From 152eb2afaf35cde5d44e50369f4cb32a72e72fdb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jul 2019 15:39:35 -0700 Subject: [PATCH 0001/1004] Use pysqlite3 if available --- sqlite_utils/cli.py | 2 +- sqlite_utils/db.py | 2 +- sqlite_utils/utils.py | 4 ++++ tests/conftest.py | 2 +- tests/test_cli.py | 2 +- tests/test_create.py | 4 ++-- 6 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 sqlite_utils/utils.py diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 7b4098d..7c09578 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -7,7 +7,7 @@ import json import sys import csv as csv_std import tabulate -import sqlite3 +from .utils import sqlite3 def output_options(fn): diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d40951e..e03ef63 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,4 +1,4 @@ -import sqlite3 +from .utils import sqlite3 from collections import namedtuple import datetime import hashlib diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py new file mode 100644 index 0000000..40b1819 --- /dev/null +++ b/sqlite_utils/utils.py @@ -0,0 +1,4 @@ +try: + import pysqlite3 as sqlite3 +except ImportError: + import sqlite3 diff --git a/tests/conftest.py b/tests/conftest.py index 322d627..c7b4077 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,5 @@ from sqlite_utils import Database -import sqlite3 +from sqlite_utils.utils import sqlite3 import pytest diff --git a/tests/test_cli.py b/tests/test_cli.py index 2fb1d04..ddb936b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,7 +4,7 @@ from click.testing import CliRunner import json import os import pytest -import sqlite3 +from sqlite_utils.utils import sqlite3 from .utils import collapse_whitespace diff --git a/tests/test_create.py b/tests/test_create.py index 91ad169..b6519c8 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -549,7 +549,7 @@ def test_create_index_if_not_exists(fresh_db): assert [] == dogs.indexes dogs.create_index(["name"]) assert 1 == len(dogs.indexes) - with pytest.raises(sqlite3.OperationalError): + with pytest.raises(Exception, match="index idx_dogs_name already exists"): dogs.create_index(["name"]) dogs.create_index(["name"], if_not_exists=True) @@ -593,7 +593,7 @@ def test_insert_thousands_ignores_extra_columns_after_first_100(fresh_db): def test_insert_ignore(fresh_db): fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") # Should raise an error if we try this again - with pytest.raises(sqlite3.IntegrityError): + with pytest.raises(Exception, match="UNIQUE constraint failed"): fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") # Using ignore=True should cause our insert to be silently ignored fresh_db["test"].insert({"id": 1, "bar": 3}, pk="id", ignore=True) From 57e43baece0936d75c74711f5f6835a5c1c1ac42 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jul 2019 16:30:54 -0700 Subject: [PATCH 0002/1004] Table options can now be passed to constructor OR to insert_all() If you want to set default options for a table, you can do this: table = db.table("dogs", pk="id", column_order=["name", "age"]) If you pass those keyword arguments to the .insert/.update/etc methods they will over-ride the defaults you set on the table. table = db["dogs"] # This still works too --- sqlite_utils/db.py | 116 +++++++++++++++++++++++++++++-------------- tests/test_create.py | 100 +++++++++++++++++++++++-------------- 2 files changed, 142 insertions(+), 74 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e03ef63..39a7416 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -18,6 +18,7 @@ ForeignKey = namedtuple( "ForeignKey", ("table", "column", "other_table", "other_column") ) Index = namedtuple("Index", ("seq", "name", "unique", "origin", "partial", "columns")) +DEFAULT = object() COLUMN_TYPE_MAPPING = { float: "FLOAT", @@ -93,11 +94,14 @@ class Database: self.conn = filename_or_conn def __getitem__(self, table_name): - return Table(self, table_name) + return self.table(table_name) def __repr__(self): return "".format(self.conn) + def table(self, table_name, **kwargs): + return Table(self, table_name, **kwargs) + def escape(self, value): # Normally we would use .execute(sql, [params]) for escaping, but # occasionally that isn't available - most notable when we need @@ -343,10 +347,36 @@ class Database: class Table: - def __init__(self, db, name): + def __init__( + self, + db, + name, + pk=None, + foreign_keys=None, + column_order=None, + not_null=None, + defaults=None, + upsert=False, + batch_size=100, + hash_id=None, + alter=False, + ignore=False, + ): self.db = db self.name = name self.exists = self.name in self.db.table_names() + self._defaults = dict( + pk=pk, + foreign_keys=foreign_keys, + column_order=column_order, + not_null=not_null, + defaults=defaults, + upsert=upsert, + batch_size=batch_size, + hash_id=hash_id, + alter=alter, + ignore=ignore, + ) def __repr__(self): return "".format( @@ -716,18 +746,21 @@ class Table: ) return self.db.conn.execute(sql, (q,)).fetchall() + def value_or_default(self, key, value): + return self._defaults[key] if value is DEFAULT else value + def insert( self, record, - pk=None, - foreign_keys=None, - column_order=None, - not_null=None, - defaults=None, - upsert=False, - hash_id=None, - alter=False, - ignore=False, + pk=DEFAULT, + foreign_keys=DEFAULT, + column_order=DEFAULT, + not_null=DEFAULT, + defaults=DEFAULT, + upsert=DEFAULT, + hash_id=DEFAULT, + alter=DEFAULT, + ignore=DEFAULT, ): return self.insert_all( [record], @@ -745,22 +778,33 @@ class Table: def insert_all( self, records, - pk=None, - foreign_keys=None, - column_order=None, - not_null=None, - defaults=None, - upsert=False, - batch_size=100, - hash_id=None, - alter=False, - ignore=False, + pk=DEFAULT, + foreign_keys=DEFAULT, + column_order=DEFAULT, + not_null=DEFAULT, + defaults=DEFAULT, + upsert=DEFAULT, + batch_size=DEFAULT, + hash_id=DEFAULT, + alter=DEFAULT, + ignore=DEFAULT, ): """ Like .insert() but takes a list of records and ensures that the table that it creates (if table does not exist) has columns for ALL of that data """ + pk = self.value_or_default("pk", pk) + foreign_keys = self.value_or_default("foreign_keys", foreign_keys) + column_order = self.value_or_default("column_order", column_order) + not_null = self.value_or_default("not_null", not_null) + defaults = self.value_or_default("defaults", defaults) + upsert = self.value_or_default("upsert", upsert) + batch_size = self.value_or_default("batch_size", batch_size) + hash_id = self.value_or_default("hash_id", hash_id) + alter = self.value_or_default("alter", alter) + ignore = self.value_or_default("ignore", ignore) + assert not (hash_id and pk), "Use either pk= or hash_id=" assert not ( ignore and upsert @@ -842,13 +886,13 @@ class Table: def upsert( self, record, - pk=None, - foreign_keys=None, - column_order=None, - not_null=None, - defaults=None, - hash_id=None, - alter=False, + pk=DEFAULT, + foreign_keys=DEFAULT, + column_order=DEFAULT, + not_null=DEFAULT, + defaults=DEFAULT, + hash_id=DEFAULT, + alter=DEFAULT, ): return self.insert( record, @@ -865,14 +909,14 @@ class Table: def upsert_all( self, records, - pk=None, - foreign_keys=None, - column_order=None, - not_null=None, - defaults=None, - batch_size=100, - hash_id=None, - alter=False, + pk=DEFAULT, + foreign_keys=DEFAULT, + column_order=DEFAULT, + not_null=DEFAULT, + defaults=DEFAULT, + batch_size=DEFAULT, + hash_id=DEFAULT, + alter=DEFAULT, ): return self.insert_all( records, diff --git a/tests/test_create.py b/tests/test_create.py index b6519c8..30131fc 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -144,19 +144,22 @@ def test_create_table_from_example_with_compound_primary_keys(fresh_db): assert record == table.get(("staff", 2)) -def test_create_table_column_order(fresh_db): - fresh_db["table"].insert( - collections.OrderedDict( - ( - ("zzz", "third"), - ("abc", "first"), - ("ccc", "second"), - ("bbb", "second-to-last"), - ("aaa", "last"), - ) - ), - column_order=("abc", "ccc", "zzz"), +@pytest.mark.parametrize("use_class_constructor", [True, False]) +def test_create_table_column_order(fresh_db, use_class_constructor): + row = collections.OrderedDict( + ( + ("zzz", "third"), + ("abc", "first"), + ("ccc", "second"), + ("bbb", "second-to-last"), + ("aaa", "last"), + ) ) + column_order = ("abc", "ccc", "zzz") + if use_class_constructor: + fresh_db.table("table", column_order=column_order).insert(row) + else: + fresh_db["table"].insert(row, column_order=column_order) assert [ {"name": "abc", "type": "TEXT"}, {"name": "ccc", "type": "TEXT"}, @@ -191,21 +194,31 @@ def test_create_table_column_order(fresh_db): (({"one_id": "one"},), AssertionError), ), ) +@pytest.mark.parametrize("use_class_constructor", [True, False]) def test_create_table_works_for_m2m_with_only_foreign_keys( - fresh_db, foreign_key_specification, expected_exception + fresh_db, foreign_key_specification, expected_exception, use_class_constructor ): - fresh_db["one"].insert({"id": 1}, pk="id") - fresh_db["two"].insert({"id": 1}, pk="id") + if use_class_constructor: + fresh_db.table("one", pk="id").insert({"id": 1}) + fresh_db.table("two", pk="id").insert({"id": 1}) + else: + fresh_db["one"].insert({"id": 1}, pk="id") + fresh_db["two"].insert({"id": 1}, pk="id") + + row = {"one_id": 1, "two_id": 1} + + def do_it(): + if use_class_constructor: + fresh_db.table("m2m", foreign_keys=foreign_key_specification).insert(row) + else: + fresh_db["m2m"].insert(row, foreign_keys=foreign_key_specification) + if expected_exception: with pytest.raises(expected_exception): - fresh_db["m2m"].insert( - {"one_id": 1, "two_id": 1}, foreign_keys=foreign_key_specification - ) + do_it() return else: - fresh_db["m2m"].insert( - {"one_id": 1, "two_id": 1}, foreign_keys=foreign_key_specification - ) + do_it() assert [ {"name": "one_id", "type": "INTEGER"}, {"name": "two_id", "type": "INTEGER"}, @@ -407,7 +420,10 @@ def test_index_foreign_keys(fresh_db): ), ], ) -def test_insert_row_alter_table(fresh_db, extra_data, expected_new_columns): +@pytest.mark.parametrize("use_class_constructor", [True, False]) +def test_insert_row_alter_table( + fresh_db, extra_data, expected_new_columns, use_class_constructor +): table = fresh_db["books"] table.insert({"title": "Hedgehogs of the world", "author_id": 1}) assert [ @@ -416,7 +432,10 @@ def test_insert_row_alter_table(fresh_db, extra_data, expected_new_columns): ] == [{"name": col.name, "type": col.type} for col in table.columns] record = {"title": "Squirrels of the world", "author_id": 2} record.update(extra_data) - fresh_db["books"].insert(record, alter=True) + if use_class_constructor: + fresh_db.table("books", alter=True).insert(record) + else: + fresh_db["books"].insert(record, alter=True) assert [ {"name": "title", "type": "TEXT"}, {"name": "author_id", "type": "INTEGER"}, @@ -425,21 +444,26 @@ def test_insert_row_alter_table(fresh_db, extra_data, expected_new_columns): ] -def test_upsert_rows_alter_table(fresh_db): - table = fresh_db["books"] - table.insert({"id": 1, "title": "Hedgehogs of the world", "author_id": 1}, pk="id") - table.upsert_all( - [ - {"id": 1, "title": "Hedgehogs of the World", "species": "hedgehogs"}, - {"id": 2, "title": "Squirrels of the World", "num_species": 200}, - { - "id": 3, - "title": "Badgers of the World", - "significant_continents": ["Europe", "North America"], - }, - ], - alter=True, - ) +@pytest.mark.parametrize("use_class_constructor", [True, False]) +def test_upsert_rows_alter_table(fresh_db, use_class_constructor): + first_row = {"id": 1, "title": "Hedgehogs of the world", "author_id": 1} + next_rows = [ + {"id": 1, "title": "Hedgehogs of the World", "species": "hedgehogs"}, + {"id": 2, "title": "Squirrels of the World", "num_species": 200}, + { + "id": 3, + "title": "Badgers of the World", + "significant_continents": ["Europe", "North America"], + }, + ] + if use_class_constructor: + table = fresh_db.table("books", pk="id", alter=True) + table.insert(first_row) + table.upsert_all(next_rows) + else: + table = fresh_db["books"] + table.insert(first_row, pk="id") + table.upsert_all(next_rows, alter=True) assert { "author_id": int, "id": int, From be655827b4a4ff88eb4813075339b0b01e018145 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jul 2019 16:33:37 -0700 Subject: [PATCH 0003/1004] use_table_factory is a better name than use_class_constructor --- tests/test_create.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test_create.py b/tests/test_create.py index 30131fc..9360395 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -144,8 +144,8 @@ def test_create_table_from_example_with_compound_primary_keys(fresh_db): assert record == table.get(("staff", 2)) -@pytest.mark.parametrize("use_class_constructor", [True, False]) -def test_create_table_column_order(fresh_db, use_class_constructor): +@pytest.mark.parametrize("use_table_factory", [True, False]) +def test_create_table_column_order(fresh_db, use_table_factory): row = collections.OrderedDict( ( ("zzz", "third"), @@ -156,7 +156,7 @@ def test_create_table_column_order(fresh_db, use_class_constructor): ) ) column_order = ("abc", "ccc", "zzz") - if use_class_constructor: + if use_table_factory: fresh_db.table("table", column_order=column_order).insert(row) else: fresh_db["table"].insert(row, column_order=column_order) @@ -194,11 +194,11 @@ def test_create_table_column_order(fresh_db, use_class_constructor): (({"one_id": "one"},), AssertionError), ), ) -@pytest.mark.parametrize("use_class_constructor", [True, False]) +@pytest.mark.parametrize("use_table_factory", [True, False]) def test_create_table_works_for_m2m_with_only_foreign_keys( - fresh_db, foreign_key_specification, expected_exception, use_class_constructor + fresh_db, foreign_key_specification, expected_exception, use_table_factory ): - if use_class_constructor: + if use_table_factory: fresh_db.table("one", pk="id").insert({"id": 1}) fresh_db.table("two", pk="id").insert({"id": 1}) else: @@ -208,7 +208,7 @@ def test_create_table_works_for_m2m_with_only_foreign_keys( row = {"one_id": 1, "two_id": 1} def do_it(): - if use_class_constructor: + if use_table_factory: fresh_db.table("m2m", foreign_keys=foreign_key_specification).insert(row) else: fresh_db["m2m"].insert(row, foreign_keys=foreign_key_specification) @@ -420,9 +420,9 @@ def test_index_foreign_keys(fresh_db): ), ], ) -@pytest.mark.parametrize("use_class_constructor", [True, False]) +@pytest.mark.parametrize("use_table_factory", [True, False]) def test_insert_row_alter_table( - fresh_db, extra_data, expected_new_columns, use_class_constructor + fresh_db, extra_data, expected_new_columns, use_table_factory ): table = fresh_db["books"] table.insert({"title": "Hedgehogs of the world", "author_id": 1}) @@ -432,7 +432,7 @@ def test_insert_row_alter_table( ] == [{"name": col.name, "type": col.type} for col in table.columns] record = {"title": "Squirrels of the world", "author_id": 2} record.update(extra_data) - if use_class_constructor: + if use_table_factory: fresh_db.table("books", alter=True).insert(record) else: fresh_db["books"].insert(record, alter=True) @@ -444,8 +444,8 @@ def test_insert_row_alter_table( ] -@pytest.mark.parametrize("use_class_constructor", [True, False]) -def test_upsert_rows_alter_table(fresh_db, use_class_constructor): +@pytest.mark.parametrize("use_table_factory", [True, False]) +def test_upsert_rows_alter_table(fresh_db, use_table_factory): first_row = {"id": 1, "title": "Hedgehogs of the world", "author_id": 1} next_rows = [ {"id": 1, "title": "Hedgehogs of the World", "species": "hedgehogs"}, @@ -456,7 +456,7 @@ def test_upsert_rows_alter_table(fresh_db, use_class_constructor): "significant_continents": ["Europe", "North America"], }, ] - if use_class_constructor: + if use_table_factory: table = fresh_db.table("books", pk="id", alter=True) table.insert(first_row) table.upsert_all(next_rows) From 127a836054fb784dc02ea40dadb55b3741dcd603 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jul 2019 16:59:17 -0700 Subject: [PATCH 0004/1004] Documented new table configuration options via .table() --- docs/python-api.rst | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 287bdcb..43a9087 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -222,6 +222,27 @@ You can leave off the third item in the tuple to have the referenced column auto ("author_id", "authors") ]) +.. _python_api_table_configuration: + +Table configuration options +=========================== + +The ``.insert()``, ``.upsert()``, ``.insert_all()`` and ``.upsert_all()`` methods each take a number of keyword arguments, some of which influence what happens should they cause a table to be created and some of which affect the behavior of those methods. + +You can set default values for these methods by accessing the table through the ``db.table(...)`` method (instead of using ``db["table_name"]``), like so: + +.. code-block:: python + + table = db.table( + "authors", + pk="id", + not_null={"name", "score"}, + column_order=("id", "name", "score", "url") + ) + # Now you can call .insert() like so: + table.insert({"id": 1, "name": "Tracy", "score": 5}) + +The configuration options that can be specified in this way are ``pk``, ``foreign_keys``, ``column_order``, ``not_null``, ``defaults``, ``upsert``, ``batch_size``, ``hash_id``, ``alter``, ``ignore``. These are all documented below. .. _python_api_defaults_not_null: @@ -395,7 +416,7 @@ You can set a ``NOT NULL DEFAULT 'x'`` constraint on the new column using ``not_ Adding columns automatically on insert/update ============================================= -You can insert or update data that includes new columns and have the table automatically altered to fit the new schema using the ``alter=True`` argument. This can be passed to all four of ``.insert()``, ``.upsert()``, ``.insert_all()`` and ``.upsert_all()``: +You can insert or update data that includes new columns and have the table automatically altered to fit the new schema using the ``alter=True`` argument. This can be passed to all four of ``.insert()``, ``.upsert()``, ``.insert_all()`` and ``.upsert_all()``, or it can be passed to ``db.table(table_name, alter=True)`` to enable it by default for all method calls against that table instance. .. code-block:: python @@ -409,6 +430,10 @@ You can insert or update data that includes new columns and have the table autom # Outputs this: # {'name': , 'age': } + # This works too: + new_table = db.table("new_table", alter=True) + new_table.insert({"name": "Gareth", "age": 32, "shoe_size": 11}) + .. _python_api_add_foreign_key: Adding foreign key constraints From 58db40d67c12cb4353a825d4aa215141a51b9e6e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jul 2019 17:05:51 -0700 Subject: [PATCH 0005/1004] Better __repr__ for tables --- docs/python-api.rst | 2 +- sqlite_utils/db.py | 5 ++++- tests/test_introspect.py | 6 ++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 43a9087..9444df9 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -571,7 +571,7 @@ Introspection If you have loaded an existing table, you can use introspection to find out more about it:: >>> db["PlantType"] - +
The ``.count`` property shows the current number of rows (``select count(*) from table``):: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 39a7416..f4829ee 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -380,7 +380,10 @@ class Table: def __repr__(self): return "
".format( - self.name, " (does not exist yet)" if not self.exists else "" + self.name, + " (does not exist yet)" + if not self.exists + else " ({})".format(", ".join(c.name for c in self.columns)), ) @property diff --git a/tests/test_introspect.py b/tests/test_introspect.py index a49028c..0dd544e 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -57,8 +57,10 @@ def test_schema(existing_db): assert "CREATE TABLE foo (text TEXT)" == existing_db["foo"].schema -def test_table_repr(existing_db): - assert "
" == repr(existing_db["foo"]) +def test_table_repr(fresh_db): + table = fresh_db["dogs"].insert({"name": "Cleo", "age": 4}) + assert "
" == repr(table) + assert "
" == repr(fresh_db["cats"]) def test_indexes(fresh_db): From 034d498b319d37b0639203fa4fbb304715b3ae03 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jul 2019 17:12:54 -0700 Subject: [PATCH 0006/1004] Support Database(memory=True) for in-memory databases --- docs/python-api.rst | 10 +++++++++- sqlite_utils/db.py | 9 +++++++-- tests/conftest.py | 5 ++--- tests/test_create.py | 7 +++++++ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 9444df9..378b5d9 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -27,7 +27,7 @@ If you want to create an in-memory database, you can do so like this: .. code-block:: python - db = Database(sqlite3.connect(":memory:")) + db = Database(sqlite3.connect(memory=True)) Tables are accessed using the indexing operator, like so: @@ -37,6 +37,14 @@ Tables are accessed using the indexing operator, like so: If the table does not yet exist, it will be created the first time you attempt to insert or upsert data into it. +You can also access tables using the ``.table()`` method like so: + +.. code-block:: python + + table = db.table("my_table") + +Using this factory function allows you to set :ref:`python_api_table_configuration`. + Listing tables ============== diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f4829ee..9f21a3b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -85,8 +85,13 @@ class NotFoundError(Exception): class Database: - def __init__(self, filename_or_conn): - if isinstance(filename_or_conn, str): + def __init__(self, filename_or_conn=None, memory=False): + assert (filename_or_conn is not None and not memory) or ( + filename_or_conn is None and memory + ), "Either specify a filename_or_conn or pass memory=True" + if memory: + self.conn = sqlite3.connect(":memory:") + elif isinstance(filename_or_conn, str): self.conn = sqlite3.connect(filename_or_conn) elif isinstance(filename_or_conn, pathlib.Path): self.conn = sqlite3.connect(str(filename_or_conn)) diff --git a/tests/conftest.py b/tests/conftest.py index c7b4077..1e1fc5e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,16 +1,15 @@ from sqlite_utils import Database -from sqlite_utils.utils import sqlite3 import pytest @pytest.fixture def fresh_db(): - return Database(sqlite3.connect(":memory:")) + return Database(memory=True) @pytest.fixture def existing_db(): - database = Database(sqlite3.connect(":memory:")) + database = Database(memory=True) database.conn.executescript( """ CREATE TABLE foo (text TEXT); diff --git a/tests/test_create.py b/tests/test_create.py index 9360395..4634191 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -727,3 +727,10 @@ def test_create_table_numpy(fresh_db): "np.uint8": 8, } ] == list(fresh_db["types"].rows) + + +def test_cannot_provide_both_filename_and_memory(): + with pytest.raises( + AssertionError, match="Either specify a filename_or_conn or pass memory=True" + ): + db = Database("/tmp/foo.db", memory=True) From 535a731b9310a07b10a8649313c8bd8fafcdadea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jul 2019 09:41:34 +0200 Subject: [PATCH 0007/1004] Fixed lint error --- tests/test_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_create.py b/tests/test_create.py index 4634191..288a8ee 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -733,4 +733,4 @@ def test_cannot_provide_both_filename_and_memory(): with pytest.raises( AssertionError, match="Either specify a filename_or_conn or pass memory=True" ): - db = Database("/tmp/foo.db", memory=True) + Database("/tmp/foo.db", memory=True) From f3a4c3d3ee6475a6caf3c9606656dbaf1df020b7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jul 2019 09:47:19 +0200 Subject: [PATCH 0008/1004] db.create_table() now remembers configs --- sqlite_utils/db.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9f21a3b..c48483d 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -263,7 +263,15 @@ class Database: table=name, columns_sql=columns_sql, extra_pk=extra_pk ) self.conn.execute(sql) - return self[name] + return self.table( + name, + pk=pk, + foreign_keys=foreign_keys, + column_order=column_order, + not_null=not_null, + defaults=defaults, + hash_id=hash_id, + ) def create_view(self, name, sql): self.conn.execute( From 580502431614d3653c93249988290265f3163d4b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jul 2019 06:06:59 -0700 Subject: [PATCH 0009/1004] Implemented table.lookup(...), closes #44 * Add pk column if missing from insert * Implemented table.lookup(...) --- docs/python-api.rst | 26 +++++++++++++++++ sqlite_utils/db.py | 25 ++++++++++++++++ tests/test_create.py | 5 ++++ tests/test_lookup.py | 68 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+) create mode 100644 tests/test_lookup.py diff --git a/docs/python-api.rst b/docs/python-api.rst index 378b5d9..0bbf19d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -354,6 +354,32 @@ Note that the ``pk`` and ``column_order`` parameters here are optional if you ar An ``upsert_all()`` method is also available, which behaves like ``insert_all()`` but performs upserts instead. +.. _python_api_lookup_tables: + +Creating lookup tables +====================== + +A useful pattern when populating large tables in to break common values out into lookup tables. Consider a table of ``Trees``, where each tree has a species. Ideally these species would be split out into a separate ``Species`` table, with each one assigned an integer primary key that can be referenced from the ``Trees`` table ``species_id`` column. + +Calling ``db["Species"].lookup({"name": "Palm"})`` creates a table called ``Species`` (if one does not already exist) with two columns: ``id`` and ``name``. It sets up a unique constraint on the ``name`` column to guarantee it will not contain duplicate rows. It then inserts a new row with the ``name`` set to ``Palm`` and returns the new integer primary key value. + +If the ``Species`` table already exists, it will insert the new row and return the primary key. If a row with that ``name`` already exists, it will return the corresponding primary key value directly. + +If you call ``.lookup()`` against an existing table without the unique constraint it will attempt to add the constraint, raising an ``IntegrityError`` if the constraint cannot be created. + +If you pass in a dictionary with multiple values, both values will be used to insert or retrieve the corresponding ID and any unique constraint that is created will cover all of those columns, for example: + +.. code-block:: python + + db["Trees"].insert({ + "latitude": 49.1265976, + "longitude": 2.5496218, + "species": db["Species"].lookup({ + "common_name": "Common Juniper", + "latin_name": "Juniperus communis" + }) + }) + .. _python_api_add_column: Adding columns diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c48483d..a86132b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -224,6 +224,8 @@ class Database: single_pk = None if isinstance(pk, str): single_pk = pk + if pk not in [c[0] for c in column_items]: + column_items.insert(0, (pk, int)) for column_name, column_type in column_items: column_extras = [] if column_name == single_pk: @@ -954,6 +956,29 @@ class Table: if col_name not in current_columns: self.add_column(col_name, col_type) + def lookup(self, column_values): + # lookups is a dictionary - all columns will be used for a unique index + if self.exists: + self.add_missing_columns([column_values]) + # TODO: Ensure we have a unique index on these + unique_column_sets = [set(i.columns) for i in self.indexes] + if set(column_values.keys()) not in unique_column_sets: + self.create_index(column_values.keys(), unique=True) + wheres = ["[{}] = ?".format(column) for column in column_values] + rows = list( + self.rows_where( + " and ".join(wheres), [value for _, value in column_values.items()] + ) + ) + try: + return rows[0]["id"] + except IndexError: + return self.insert(column_values, pk="id").last_pk + else: + pk = self.insert(column_values, pk="id").last_pk + self.create_index(column_values.keys(), unique=True) + return pk + def chunks(sequence, size): iterator = iter(sequence) diff --git a/tests/test_create.py b/tests/test_create.py index 288a8ee..222a967 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -734,3 +734,8 @@ def test_cannot_provide_both_filename_and_memory(): AssertionError, match="Either specify a filename_or_conn or pass memory=True" ): Database("/tmp/foo.db", memory=True) + + +def test_creates_id_column(fresh_db): + last_pk = fresh_db.table("cats", pk="id").insert({"name": "barry"}).last_pk + assert [{"name": "barry", "id": last_pk}] == list(fresh_db["cats"].rows) diff --git a/tests/test_lookup.py b/tests/test_lookup.py new file mode 100644 index 0000000..ac8e1c2 --- /dev/null +++ b/tests/test_lookup.py @@ -0,0 +1,68 @@ +from sqlite_utils.db import Index +import pytest + + +def test_lookup_new_table(fresh_db): + species = fresh_db["species"] + palm_id = species.lookup({"name": "Palm"}) + oak_id = species.lookup({"name": "Oak"}) + cherry_id = species.lookup({"name": "Cherry"}) + assert palm_id == species.lookup({"name": "Palm"}) + assert oak_id == species.lookup({"name": "Oak"}) + assert cherry_id == species.lookup({"name": "Cherry"}) + assert palm_id != oak_id != cherry_id + # Ensure the correct indexes were created + assert [ + Index( + seq=0, + name="idx_species_name", + unique=1, + origin="c", + partial=0, + columns=["name"], + ) + ] == species.indexes + + +def test_lookup_new_table_compound_key(fresh_db): + species = fresh_db["species"] + palm_id = species.lookup({"name": "Palm", "type": "Tree"}) + oak_id = species.lookup({"name": "Oak", "type": "Tree"}) + assert palm_id == species.lookup({"name": "Palm", "type": "Tree"}) + assert oak_id == species.lookup({"name": "Oak", "type": "Tree"}) + assert [ + Index( + seq=0, + name="idx_species_name_type", + unique=1, + origin="c", + partial=0, + columns=["name", "type"], + ) + ] == species.indexes + + +def test_lookup_adds_unique_constraint_to_existing_table(fresh_db): + species = fresh_db.table("species", pk="id") + palm_id = species.insert({"name": "Palm"}).last_pk + species.insert({"name": "Oak"}) + assert [] == species.indexes + assert palm_id == species.lookup({"name": "Palm"}) + assert [ + Index( + seq=0, + name="idx_species_name", + unique=1, + origin="c", + partial=0, + columns=["name"], + ) + ] == species.indexes + + +def test_lookup_fails_if_constraint_cannot_be_added(fresh_db): + species = fresh_db.table("species", pk="id") + species.insert_all([{"id": 1, "name": "Palm"}, {"id": 2, "name": "Palm"}]) + # This will fail because the name column is not unique + with pytest.raises(Exception, match="UNIQUE constraint failed"): + species.lookup({"name": "Palm"}) From e22cfcd953f967f6e9551b3a048d7c40726f349b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jul 2019 15:05:04 +0200 Subject: [PATCH 0010/1004] Removed a TODO which is now done --- sqlite_utils/db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a86132b..8cf31ed 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -960,7 +960,6 @@ class Table: # lookups is a dictionary - all columns will be used for a unique index if self.exists: self.add_missing_columns([column_values]) - # TODO: Ensure we have a unique index on these unique_column_sets = [set(i.columns) for i in self.indexes] if set(column_values.keys()) not in unique_column_sets: self.create_index(column_values.keys(), unique=True) From 941d281aee6eac20ad64b505511da7e47f697700 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jul 2019 10:00:42 -0700 Subject: [PATCH 0011/1004] extracts= table parameter, closes #46 --- docs/index.rst | 2 +- docs/python-api.rst | 42 ++++++++++++++++++++++++++-- sqlite_utils/db.py | 46 +++++++++++++++++++++++++++--- tests/test_extracts.py | 63 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 tests/test_extracts.py diff --git a/docs/index.rst b/docs/index.rst index f50d76e..f05532f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,7 +16,7 @@ Contents -------- .. toctree:: - :maxdepth: 2 + :maxdepth: 3 cli python-api diff --git a/docs/python-api.rst b/docs/python-api.rst index 0bbf19d..362fee0 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -356,11 +356,16 @@ An ``upsert_all()`` method is also available, which behaves like ``insert_all()` .. _python_api_lookup_tables: -Creating lookup tables -====================== +Working with lookup tables +========================== A useful pattern when populating large tables in to break common values out into lookup tables. Consider a table of ``Trees``, where each tree has a species. Ideally these species would be split out into a separate ``Species`` table, with each one assigned an integer primary key that can be referenced from the ``Trees`` table ``species_id`` column. +.. _python_api_explicit_lookup_tables: + +Creating lookup tables explicitly +--------------------------------- + Calling ``db["Species"].lookup({"name": "Palm"})`` creates a table called ``Species`` (if one does not already exist) with two columns: ``id`` and ``name``. It sets up a unique constraint on the ``name`` column to guarantee it will not contain duplicate rows. It then inserts a new row with the ``name`` set to ``Palm`` and returns the new integer primary key value. If the ``Species`` table already exists, it will insert the new row and return the primary key. If a row with that ``name`` already exists, it will return the corresponding primary key value directly. @@ -380,6 +385,39 @@ If you pass in a dictionary with multiple values, both values will be used to in }) }) +.. _python_api_extracts: + +Populating lookup tables automatically during insert/upsert +----------------------------------------------------------- + +A more efficient way to work with lookup tables is to define them using the ``extracts=`` parameter, which is accepted by ``.insert()``, ``.upsert()``, ``.insert_all()``, ``.upsert_all()`` and by the ``.table(...)`` factory function. + +``extracts=`` specifies columns which should be "extracted" out into a separate lookup table during the data insertion. + +It can be either a list of column names, in which case the extracted table names will match the column names exactly, or it can be a dictionary mapping column names to the desired name of the extracted table. + +To extract the ``species`` column out to a separate ``Species`` table, you can do this: + +.. code-block:: python + + # Using the table factory + trees = db.table("Trees", extracts={"species": "Species"}) + trees.insert({ + "latitude": 49.1265976, + "longitude": 2.5496218, + "species": "Common Juniper" + }) + + # If you want the table to be called 'species', you can do this: + trees = db.table("Trees", extracts=["species"]) + + # Using .insert() directly + db["Trees"].insert({ + "latitude": 49.1265976, + "longitude": 2.5496218, + "species": "Common Juniper" + }, extracts={"species": "Species"}) + .. _python_api_add_column: Adding columns diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 8cf31ed..ef55976 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -186,9 +186,20 @@ class Database: not_null=None, defaults=None, hash_id=None, + extracts=None, ): foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) foreign_keys_by_column = {fk.column: fk for fk in foreign_keys} + # any extracts will be treated as integer columns with a foreign key + extracts = resolve_extracts(extracts) + for extract_column, extract_table in extracts.items(): + # Ensure other table exists + if not self[extract_table].exists: + self.create_table(extract_table, {"id": int, "value": str}, pk="id") + columns[extract_column] = int + foreign_keys_by_column[extract_column] = ForeignKey( + name, extract_column, extract_table, "id" + ) # Sanity check not_null, and defaults if provided not_null = not_null or set() defaults = defaults or {} @@ -376,6 +387,7 @@ class Table: hash_id=None, alter=False, ignore=False, + extracts=None, ): self.db = db self.name = name @@ -391,6 +403,7 @@ class Table: hash_id=hash_id, alter=alter, ignore=ignore, + extracts=extracts, ) def __repr__(self): @@ -530,6 +543,7 @@ class Table: not_null=None, defaults=None, hash_id=None, + extracts=None, ): columns = {name: value for (name, value) in columns.items()} with self.db.conn: @@ -542,6 +556,7 @@ class Table: not_null=not_null, defaults=defaults, hash_id=hash_id, + extracts=extracts, ) self.exists = True return self @@ -779,6 +794,7 @@ class Table: hash_id=DEFAULT, alter=DEFAULT, ignore=DEFAULT, + extracts=DEFAULT, ): return self.insert_all( [record], @@ -791,6 +807,7 @@ class Table: hash_id=hash_id, alter=alter, ignore=ignore, + extracts=extracts, ) def insert_all( @@ -806,6 +823,7 @@ class Table: hash_id=DEFAULT, alter=DEFAULT, ignore=DEFAULT, + extracts=DEFAULT, ): """ Like .insert() but takes a list of records and ensures that the table @@ -822,6 +840,7 @@ class Table: hash_id = self.value_or_default("hash_id", hash_id) alter = self.value_or_default("alter", alter) ignore = self.value_or_default("ignore", ignore) + extracts = self.value_or_default("extracts", extracts) assert not (hash_id and pk), "Use either pk= or hash_id=" assert not ( @@ -842,6 +861,7 @@ class Table: not_null=not_null, defaults=defaults, hash_id=hash_id, + extracts=extracts, ) all_columns = set() for record in chunk: @@ -871,13 +891,18 @@ class Table: ), ) values = [] + extracts = resolve_extracts(extracts) for record in chunk: - values.extend( - jsonify_if_needed( + record_values = [] + for key in all_columns: + value = jsonify_if_needed( record.get(key, None if key != hash_id else _hash(record)) ) - for key in all_columns - ) + if key in extracts: + extract_table = extracts[key] + value = self.db[extract_table].lookup({"value": value}) + record_values.append(value) + values.extend(record_values) with self.db.conn: try: result = self.db.conn.execute(sql, values) @@ -911,6 +936,7 @@ class Table: defaults=DEFAULT, hash_id=DEFAULT, alter=DEFAULT, + extracts=DEFAULT, ): return self.insert( record, @@ -922,6 +948,7 @@ class Table: hash_id=hash_id, alter=alter, upsert=True, + extracts=extracts, ) def upsert_all( @@ -935,6 +962,7 @@ class Table: batch_size=DEFAULT, hash_id=DEFAULT, alter=DEFAULT, + extracts=DEFAULT, ): return self.insert_all( records, @@ -947,6 +975,7 @@ class Table: hash_id=hash_id, alter=alter, upsert=True, + extracts=extracts, ) def add_missing_columns(self, records): @@ -958,6 +987,7 @@ class Table: def lookup(self, column_values): # lookups is a dictionary - all columns will be used for a unique index + assert isinstance(column_values, dict) if self.exists: self.add_missing_columns([column_values]) unique_column_sets = [set(i.columns) for i in self.indexes] @@ -998,3 +1028,11 @@ def _hash(record): return hashlib.sha1( json.dumps(record, separators=(",", ":"), sort_keys=True).encode("utf8") ).hexdigest() + + +def resolve_extracts(extracts): + if extracts is None: + extracts = {} + if isinstance(extracts, (list, tuple)): + extracts = {item: item for item in extracts} + return extracts diff --git a/tests/test_extracts.py b/tests/test_extracts.py new file mode 100644 index 0000000..c46b0a5 --- /dev/null +++ b/tests/test_extracts.py @@ -0,0 +1,63 @@ +from sqlite_utils.db import Index +import pytest + + +@pytest.mark.parametrize( + "kwargs,expected_table", + [ + (dict(extracts={"species_id": "Species"}), "Species"), + (dict(extracts=["species_id"]), "species_id"), + (dict(extracts=("species_id",)), "species_id"), + ], +) +@pytest.mark.parametrize("use_table_factory", [True, False]) +def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): + table_kwargs = {} + insert_kwargs = {} + if use_table_factory: + table_kwargs = kwargs + else: + insert_kwargs = kwargs + trees = fresh_db.table("Trees", **table_kwargs) + trees.insert_all( + [ + {"id": 1, "species_id": "Oak"}, + {"id": 2, "species_id": "Oak"}, + {"id": 3, "species_id": "Palm"}, + ], + **insert_kwargs + ) + # Should now have two tables: Trees and Species + assert {expected_table, "Trees"} == set(fresh_db.table_names()) + assert ( + "CREATE TABLE [{}] (\n [id] INTEGER PRIMARY KEY,\n [value] TEXT\n)".format( + expected_table + ) + == fresh_db[expected_table].schema + ) + assert ( + "CREATE TABLE [Trees] (\n [id] INTEGER,\n [species_id] INTEGER REFERENCES [{}]([id])\n)".format( + expected_table + ) + == fresh_db["Trees"].schema + ) + # Should have unique index on Species + assert [ + Index( + seq=0, + name="idx_{}_value".format(expected_table), + unique=1, + origin="c", + partial=0, + columns=["value"], + ) + ] == fresh_db[expected_table].indexes + # Finally, check the rows + assert [{"id": 1, "value": "Oak"}, {"id": 2, "value": "Palm"}] == list( + fresh_db[expected_table].rows + ) + assert [ + {"id": 1, "species_id": 1}, + {"id": 2, "species_id": 1}, + {"id": 3, "species_id": 2}, + ] == list(fresh_db["Trees"].rows) From 9b7be79c86b4283f24a64f62257c918f12542997 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 24 Jul 2019 08:50:41 +0200 Subject: [PATCH 0012/1004] Release 1.7 - with lookup table support --- docs/changelog.rst | 13 +++++++++++++ setup.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 17176c2..3702b02 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,19 @@ Changelog =========== +.. _v1_7: + +1.7 (2019-07-24) +---------------- + +Support for lookup tables. + +- New ``table.lookup({...})`` utility method for building and querying lookup tables - see :ref:`python_api_lookup_tables` (`#44 `__) +- New ``extracts=`` table configuration option, see :ref:`python_api_extracts` (`#46 `__) +- Use `pysqlite3 `__ if it is available, otherwise use ``sqlite3`` from the standard library +- Table options can now be passed to the new ``db.table(name, **options)`` factory function in addition to being passed to ``insert_all(records, **options)`` and friends - see :ref:`python_api_table_configuration` +- In-memory databases can now be created using ``db = Database(memory=True)`` + .. _v1_6: 1.6 (2019-07-18) diff --git a/setup.py b/setup.py index a1635b5..131825b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "1.6" +VERSION = "1.7" def get_long_description(): From 62d292252804aa0a0c1d6fdc9ea1722b5ffb20a8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Jul 2019 14:10:56 +0300 Subject: [PATCH 0013/1004] Fix for too many SQL variables, closes #50 --- sqlite_utils/db.py | 9 ++++++++- tests/test_create.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ef55976..29b2286 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -848,7 +848,14 @@ class Table: ), "Use either ignore=True or upsert=True, not both" all_columns = None first = True - for chunk in chunks(records, batch_size): + # We can only handle a max of 999 variables in a SQL insert, so + # we need to adjust the batch_size down if we have too many cols + records = iter(records) + # Peek at first record to count its columns: + first_record = next(records) + num_columns = len(first_record.keys()) + batch_size = max(1, min(batch_size, 999 // num_columns)) + for chunk in chunks(itertools.chain([first_record], records), batch_size): chunk = list(chunk) if first: if not self.exists: diff --git a/tests/test_create.py b/tests/test_create.py index 222a967..8f30347 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -500,6 +500,42 @@ def test_upsert_rows_alter_table(fresh_db, use_table_factory): ] == list(table.rows) +def test_bulk_insert_more_than_999_values(fresh_db): + "Inserting 100 items with 11 columns should work" + fresh_db["big"].insert_all( + ( + { + "id": i + 1, + "c2": 2, + "c3": 3, + "c4": 4, + "c5": 5, + "c6": 6, + "c7": 7, + "c8": 8, + "c8": 9, + "c10": 10, + "c11": 11, + } + for i in range(100) + ), + pk="id", + ) + assert 100 == fresh_db["big"].count + + +@pytest.mark.parametrize( + "num_columns,should_error", ((900, False), (999, False), (1000, True)) +) +def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error): + record = dict([("c{}".format(i), i) for i in range(num_columns)]) + if should_error: + with pytest.raises(Exception): + fresh_db["big"].insert(record) + else: + fresh_db["big"].insert(record) + + @pytest.mark.parametrize( "columns,index_name,expected_index", ( From 0c1b8b7f96be874bb63801f69323960f277aa49a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Jul 2019 14:41:57 +0300 Subject: [PATCH 0014/1004] Use assertion to enforce <=999 columns --- sqlite_utils/db.py | 7 ++++++- tests/test_create.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 29b2286..586014b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -6,6 +6,8 @@ import itertools import json import pathlib +SQLITE_MAX_VARS = 999 + try: import numpy as np except ImportError: @@ -854,7 +856,10 @@ class Table: # Peek at first record to count its columns: first_record = next(records) num_columns = len(first_record.keys()) - batch_size = max(1, min(batch_size, 999 // num_columns)) + assert ( + num_columns <= SQLITE_MAX_VARS + ), "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) + batch_size = max(1, min(batch_size, SQLITE_MAX_VARS // num_columns)) for chunk in chunks(itertools.chain([first_record], records), batch_size): chunk = list(chunk) if first: diff --git a/tests/test_create.py b/tests/test_create.py index 8f30347..71b1583 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -530,7 +530,7 @@ def test_bulk_insert_more_than_999_values(fresh_db): def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error): record = dict([("c{}".format(i), i) for i in range(num_columns)]) if should_error: - with pytest.raises(Exception): + with pytest.raises(AssertionError): fresh_db["big"].insert(record) else: fresh_db["big"].insert(record) From 535a5ea476fb41738c839ac609b43d2f019cea96 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Jul 2019 14:22:12 +0300 Subject: [PATCH 0015/1004] Documentation and tests for table.drop() method --- docs/python-api.rst | 11 +++++++++++ sqlite_utils/db.py | 2 +- tests/test_create.py | 7 +++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 362fee0..752e813 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -567,6 +567,17 @@ If you want to ensure that every foreign key column in your database has a corre db.index_foreign_keys() +.. _python_api_drop: + +Dropping a table +================ + +You can drop a table by using the ``.drop()`` method: + +.. code-block:: python + + db["my_table"].drop() + .. _python_api_hash: Setting an ID based on the hash of the row contents diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 586014b..267d0e9 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -621,7 +621,7 @@ class Table: return self def drop(self): - return self.db.conn.execute("DROP TABLE {}".format(self.name)) + self.db.conn.execute("DROP TABLE {}".format(self.name)) def guess_foreign_table(self, column): column = column.lower() diff --git a/tests/test_create.py b/tests/test_create.py index 71b1583..ca73b2d 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -775,3 +775,10 @@ def test_cannot_provide_both_filename_and_memory(): def test_creates_id_column(fresh_db): last_pk = fresh_db.table("cats", pk="id").insert({"name": "barry"}).last_pk assert [{"name": "barry", "id": last_pk}] == list(fresh_db["cats"].rows) + + +def test_drop(fresh_db): + fresh_db["t"].insert({"foo": 1}) + assert ["t"] == fresh_db.table_names() + assert None is fresh_db["t"].drop() + assert [] == fresh_db.table_names() From a6749cdf43229c4f7864c946496e9ac0141627d9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Jul 2019 15:00:51 +0300 Subject: [PATCH 0016/1004] Release 1.7.1 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3702b02..e1663cf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v1_7_1: + +1.7.1 (2019-07-28) +------------------ + +- Fixed bug where inserting records with 11 columns in a batch of 100 triggered a "too many SQL variables" error (`#50 `__) +- Documentation and tests for ``table.drop()`` method: :ref:`python_api_drop` + .. _v1_7: 1.7 (2019-07-24) diff --git a/setup.py b/setup.py index 131825b..dab276c 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "1.7" +VERSION = "1.7.1" def get_long_description(): From 15368c5f59066fc9c6b8ce5d0578132b1b68b75d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Jul 2019 10:03:18 -0700 Subject: [PATCH 0017/1004] First working version of .update(), refs #35 --- sqlite_utils/db.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 267d0e9..da06238 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -784,6 +784,41 @@ class Table: def value_or_default(self, key, value): return self._defaults[key] if value is DEFAULT else value + def update(self, pk_values, updates=None): + updates = updates or {} + if not isinstance(pk_values, (list, tuple)): + pk_values = [pk_values] + pks = self.pks + pk_names = [] + if len(pks) == 0: + # rowid table + pk_names = ["rowid"] + last_pk = pk_values[0] + elif len(pks) == 1: + pk_names = [pks[0]] + last_pk = pk_values[0] + elif len(pks) > 1: + pk_names = pks + last_pk = pk_values + assert len(pk_names) == len(pk_values) + args = [] + sets = [] + wheres = [] + for key, value in updates.items(): + sets.append("[{}] = ?".format(key)) + args.append(value) + wheres = ["[{}] = ?".format(pk_name) for pk_name in pk_names] + args.extend(pk_values) + sql = "update [{table}] set {sets} where {wheres}".format( + table=self.name, sets=", ".join(sets), wheres=" and ".join(wheres) + ) + with self.db.conn: + rowcount = self.db.conn.execute(sql, args).rowcount + # TODO: Test this works (rolls back) - use better exception: + assert rowcount == 1 + self.last_pk = last_pk + return self + def insert( self, record, From 455071f3c5e76141926eb1e77656cb131a826707 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Jul 2019 15:30:28 +0300 Subject: [PATCH 0018/1004] Unit tests for .update() Also now set .last_pk to lastrowid for rowid tables. --- sqlite_utils/db.py | 2 +- tests/test_update.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 tests/test_update.py diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index da06238..e97f71d 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -961,7 +961,7 @@ class Table: else: raise self.last_rowid = result.lastrowid - self.last_pk = None + self.last_pk = self.last_rowid # self.last_rowid will be 0 if a "INSERT OR IGNORE" happened if (hash_id or pk) and self.last_rowid: row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] diff --git a/tests/test_update.py b/tests/test_update.py new file mode 100644 index 0000000..0b99333 --- /dev/null +++ b/tests/test_update.py @@ -0,0 +1,24 @@ +import pytest + + +def test_update_rowid_table(fresh_db): + table = fresh_db["table"] + rowid = table.insert({"foo": "bar"}).last_pk + table.update(rowid, {"foo": "baz"}) + assert [{"foo": "baz"}] == list(table.rows) + + +def test_update_pk_table(fresh_db): + table = fresh_db["table"] + pk = table.insert({"foo": "bar", "id": 5}, pk="id").last_pk + assert 5 == pk + table.update(pk, {"foo": "baz"}) + assert [{"id": 5, "foo": "baz"}] == list(table.rows) + + +def test_update_compound_pk_table(fresh_db): + table = fresh_db["table"] + pk = table.insert({"id1": 5, "id2": 3, "v": 1}, pk=("id1", "id2")).last_pk + assert (5, 3) == pk + table.update(pk, {"v": 2}) + assert [{"id1": 5, "id2": 3, "v": 2}] == list(table.rows) From e4a11b181580605b1711acee4828039137e2fcd9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Jul 2019 15:44:33 +0300 Subject: [PATCH 0019/1004] Refactor .update() to use .get() .pks introspection now returns [rowid] for rowid tables. --- sqlite_utils/db.py | 42 ++++++++++++---------------------------- tests/test_introspect.py | 8 ++++++++ 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e97f71d..36dec86 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -456,31 +456,24 @@ class Table: @property def pks(self): - return [column.name for column in self.columns if column.is_pk] + names = [column.name for column in self.columns if column.is_pk] + if not names: + names = ["rowid"] + return names def get(self, pk_values): if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] pks = self.pks - pk_names = [] - if len(pks) == 0: - # rowid table - pk_names = ["rowid"] - last_pk = pk_values[0] - elif len(pks) == 1: - pk_names = [pks[0]] - last_pk = pk_values[0] - elif len(pks) > 1: - pk_names = pks - last_pk = pk_values - if len(pk_names) != len(pk_values): + last_pk = pk_values[0] if len(pks) == 1 else pk_values + if len(pks) != len(pk_values): raise NotFoundError( "Need {} primary key value{}".format( - len(pk_names), "" if len(pk_names) == 1 else "s" + len(pks), "" if len(pks) == 1 else "s" ) ) - wheres = ["[{}] = ?".format(pk_name) for pk_name in pk_names] + wheres = ["[{}] = ?".format(pk_name) for pk_name in pks] rows = self.rows_where(" and ".join(wheres), pk_values) try: row = list(rows)[0] @@ -788,26 +781,15 @@ class Table: updates = updates or {} if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] - pks = self.pks - pk_names = [] - if len(pks) == 0: - # rowid table - pk_names = ["rowid"] - last_pk = pk_values[0] - elif len(pks) == 1: - pk_names = [pks[0]] - last_pk = pk_values[0] - elif len(pks) > 1: - pk_names = pks - last_pk = pk_values - assert len(pk_names) == len(pk_values) + # Sanity check that the record exists (raises error if not): + self.get(pk_values) args = [] sets = [] wheres = [] for key, value in updates.items(): sets.append("[{}] = ?".format(key)) args.append(value) - wheres = ["[{}] = ?".format(pk_name) for pk_name in pk_names] + wheres = ["[{}] = ?".format(pk_name) for pk_name in self.pks] args.extend(pk_values) sql = "update [{table}] set {sets} where {wheres}".format( table=self.name, sets=", ".join(sets), wheres=" and ".join(wheres) @@ -816,7 +798,7 @@ class Table: rowcount = self.db.conn.execute(sql, args).rowcount # TODO: Test this works (rolls back) - use better exception: assert rowcount == 1 - self.last_pk = last_pk + self.last_pk = pk_values[0] if len(self.pks) == 1 else pk_values return self def insert( diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 0dd544e..d3ab27a 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -98,3 +98,11 @@ def test_guess_foreign_table(fresh_db, column, expected_table_guess): fresh_db.create_table("authors", {"name": str}) fresh_db.create_table("genre", {"name": str}) assert expected_table_guess == fresh_db["books"].guess_foreign_table(column) + + +@pytest.mark.parametrize( + "pk,expected", ((None, ["rowid"]), ("id", ["id"]), (["id", "id2"], ["id", "id2"])) +) +def test_pks(fresh_db, pk, expected): + fresh_db["foo"].insert_all([{"id": 1, "id2": 2}], pk=pk) + assert expected == fresh_db["foo"].pks From 5225dbb89c08a73b1af536105f7fcff64aef5638 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Jul 2019 17:46:49 +0300 Subject: [PATCH 0020/1004] Unit tests for invalid .update() pks --- tests/test_update.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_update.py b/tests/test_update.py index 0b99333..9fab8a8 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -1,3 +1,4 @@ +from sqlite_utils.db import NotFoundError import pytest @@ -22,3 +23,24 @@ def test_update_compound_pk_table(fresh_db): assert (5, 3) == pk table.update(pk, {"v": 2}) assert [{"id1": 5, "id2": 3, "v": 2}] == list(table.rows) + + +@pytest.mark.parametrize( + "pk,update_pk", + ( + (None, 2), + (None, None), + ("id1", None), + ("id1", 4), + (("id1", "id2"), None), + (("id1", "id2"), 4), + (("id1", "id2"), (4, 5)), + ), +) +def test_update_invalid_pk(fresh_db, pk, update_pk): + table = fresh_db["table"] + table.insert({"id1": 5, "id2": 3, "v": 1}, pk=pk).last_pk + with pytest.raises(NotFoundError): + table.update(update_pk, {"v": 2}) + + From 4ab8d46b03a92c68e9694ea7c285d3852ef58530 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Jul 2019 17:51:49 +0300 Subject: [PATCH 0021/1004] Added table.update(pk, ..., alter=True) --- sqlite_utils/db.py | 19 ++++++++++++++----- sqlite_utils/utils.py | 5 +++++ tests/test_create.py | 2 +- tests/test_update.py | 20 ++++++++++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 36dec86..b842afe 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,4 +1,4 @@ -from .utils import sqlite3 +from .utils import sqlite3, OperationalError from collections import namedtuple import datetime import hashlib @@ -777,7 +777,7 @@ class Table: def value_or_default(self, key, value): return self._defaults[key] if value is DEFAULT else value - def update(self, pk_values, updates=None): + def update(self, pk_values, updates=None, alter=False): updates = updates or {} if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] @@ -795,7 +795,16 @@ class Table: table=self.name, sets=", ".join(sets), wheres=" and ".join(wheres) ) with self.db.conn: - rowcount = self.db.conn.execute(sql, args).rowcount + try: + rowcount = self.db.conn.execute(sql, args).rowcount + except OperationalError as e: + if alter and (" column" in e.args[0]): + # Attempt to add any missing columns, then try again + self.add_missing_columns([updates]) + rowcount = self.db.conn.execute(sql, args).rowcount + else: + raise + # TODO: Test this works (rolls back) - use better exception: assert rowcount == 1 self.last_pk = pk_values[0] if len(self.pks) == 1 else pk_values @@ -935,8 +944,8 @@ class Table: with self.db.conn: try: result = self.db.conn.execute(sql, values) - except sqlite3.OperationalError as e: - if alter and (" has no column " in e.args[0]): + except OperationalError as e: + if alter and (" column" in e.args[0]): # Attempt to add any missing columns, then try again self.add_missing_columns(chunk) result = self.db.conn.execute(sql, values) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 40b1819..738424d 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -1,4 +1,9 @@ try: import pysqlite3 as sqlite3 + import pysqlite3.dbapi2 + + OperationalError = pysqlite3.dbapi2.OperationalError except ImportError: import sqlite3 + + OperationalError = sqlite3.OperationalError diff --git a/tests/test_create.py b/tests/test_create.py index ca73b2d..b4778f3 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -6,12 +6,12 @@ from sqlite_utils.db import ( NoObviousTable, ForeignKey, ) +from sqlite_utils.utils import sqlite3 import collections import datetime import json import pathlib import pytest -import sqlite3 from .utils import collapse_whitespace diff --git a/tests/test_update.py b/tests/test_update.py index 9fab8a8..2d46ffe 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -44,3 +44,23 @@ def test_update_invalid_pk(fresh_db, pk, update_pk): table.update(update_pk, {"v": 2}) +def test_update_alter(fresh_db): + table = fresh_db["table"] + rowid = table.insert({"foo": "bar"}).last_pk + table.update(rowid, {"new_col": 1.2}, alter=True) + assert [{"foo": "bar", "new_col": 1.2}] == list(table.rows) + # Let's try adding three cols at once + table.update( + rowid, + {"str_col": "str", "bytes_col": b"\xa0 has bytes", "int_col": -10}, + alter=True, + ) + assert [ + { + "foo": "bar", + "new_col": 1.2, + "str_col": "str", + "bytes_col": b"\xa0 has bytes", + "int_col": -10, + } + ] == list(table.rows) From bc9c4db34b815f5385abbf4bb491bab0e10779db Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Jul 2019 17:59:52 +0300 Subject: [PATCH 0022/1004] .update(...) with no update argument sets last_pk --- sqlite_utils/db.py | 2 ++ tests/test_update.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b842afe..bbc6177 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -783,6 +783,8 @@ class Table: pk_values = [pk_values] # Sanity check that the record exists (raises error if not): self.get(pk_values) + if not updates: + return self args = [] sets = [] wheres = [] diff --git a/tests/test_update.py b/tests/test_update.py index 2d46ffe..060ef32 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -64,3 +64,20 @@ def test_update_alter(fresh_db): "int_col": -10, } ] == list(table.rows) + + +def test_update_with_no_values_sets_last_pk(fresh_db): + table = fresh_db.table("dogs", pk="id") + table.insert_all([{ + "id": 1, + "name": "Cleo", + }, { + "id": 2, + "name": "Pancakes" + }]) + table.update(1) + assert 1 == table.last_pk + table.update(2) + assert 2 == table.last_pk + with pytest.raises(NotFoundError): + table.update(3) From 598608374625cbfa0799f5b7a210ca6e192f9c0b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Jul 2019 18:03:44 +0300 Subject: [PATCH 0023/1004] Documentation for table.update() method --- docs/python-api.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 752e813..b1ad43d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -331,6 +331,30 @@ The function can accept an iterator or generator of rows and will commit them ac You can skip inserting any records that have a primary key that already exists using ``ignore=True``. This works with both ``.insert({...}, ignore=True)`` and ``.insert_all([...], ignore=True)``. +.. _python_api_update: + +Updating a specific record +========================== + +You can update a record by its primary key using ``table.update()``:: + + >>> db = sqlite_utils.Database("dogs.db") + >>> print(db["dogs"].get(1)) + {'id': 1, 'age': 4, 'name': 'Cleo'} + >>> db["dogs"].update(1, {"age": 5}) + >>> print(db["dogs"].get(1)) + {'id': 1, 'age': 5, 'name': 'Cleo'} + +The first argument to ``update()`` is the primary key. This can be a single value, or a tuple if that table has a compound primary key:: + + >>> db["compound_dogs"].update((5, 3), {"name": "Updated"}) + +The second argument is a dictonary of columns that should be updated, along with their new values. + +You can cause any missing columns to be added automatically using ``alter=True``:: + + >>> db["dogs"].update(1, {"breed": "Mutt"}, alter=True) + Upserting data ============== From 16d7008002b43cf47a973791da93e5cdd5913fc3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Jul 2019 18:37:27 +0300 Subject: [PATCH 0024/1004] Applied black --- tests/test_update.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_update.py b/tests/test_update.py index 060ef32..70f8e9e 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -68,13 +68,7 @@ def test_update_alter(fresh_db): def test_update_with_no_values_sets_last_pk(fresh_db): table = fresh_db.table("dogs", pk="id") - table.insert_all([{ - "id": 1, - "name": "Cleo", - }, { - "id": 2, - "name": "Pancakes" - }]) + table.insert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Pancakes"}]) table.update(1) assert 1 == table.last_pk table.update(2) From e1021030dd2d8d4705ad0e7bae389eeaea7fa17b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Jul 2019 18:41:42 +0300 Subject: [PATCH 0025/1004] Release 1.8 --- docs/changelog.rst | 11 +++++++++-- setup.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e1663cf..1c51ee5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v1_8: + +1.8 (2019-07-28) +---------------- + +- ``table.update(pk, values)`` method: :ref:`python_api_update` (`#35 `__) + .. _v1_7_1: 1.7.1 (2019-07-28) @@ -38,9 +45,9 @@ Support for lookup tables. - Support for compound primary keys (`#36 `__) - Configure these using the CLI tool by passing ``--pk`` multiple times - - In Python, pass a tuple of columns to the ``pk=(..., ...)`` argument: see :ref:`python_api_compound_primary_keys` + - In Python, pass a tuple of columns to the ``pk=(..., ...)`` argument: :ref:`python_api_compound_primary_keys` -- New ``table.get()`` method for retrieving a record by its primary key (`#39 `__) - :ref:`documentation ` +- New ``table.get()`` method for retrieving a record by its primary key: :ref:`python_api_get` (`#39 `__) .. _v1_4_1: diff --git a/setup.py b/setup.py index dab276c..559ddb2 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "1.7.1" +VERSION = "1.8" def get_long_description(): From 35eeafaaa33648a528cbcd57ceca966fea19c6ae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jul 2019 08:31:27 +0300 Subject: [PATCH 0026/1004] table.m2m(...) method, with tests --- sqlite_utils/db.py | 20 ++++++++++++++++++++ tests/test_m2m.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 tests/test_m2m.py diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index bbc6177..9f50a53 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1048,6 +1048,26 @@ class Table: self.create_index(column_values.keys(), unique=True) return pk + def m2m(self, other_table, record_or_list, pk=None): + our_id = self.last_pk + records = ( + [record_or_list] + if not isinstance(record_or_list, (list, tuple)) + else record_or_list + ) + tables = list(sorted([self.name, other_table])) + columns = ["{}_id".format(t) for t in tables] + m2m_table = self.db.table( + "{}_{}".format(*tables), pk=columns, foreign_keys=columns + ) + # Ensure each record exists in other table + for record in records: + id = self.db[other_table].upsert(record, pk=pk).last_pk + m2m_table.upsert( + {"{}_id".format(other_table): id, "{}_id".format(self.name): our_id} + ) + return self + def chunks(sequence, size): iterator = iter(sequence) diff --git a/tests/test_m2m.py b/tests/test_m2m.py new file mode 100644 index 0000000..e7d6e30 --- /dev/null +++ b/tests/test_m2m.py @@ -0,0 +1,43 @@ +from sqlite_utils.db import ForeignKey +import pytest + + +def test_insert_m2m_single(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( + "humans", {"id": 1, "name": "Natalie D"}, pk="id" + ) + assert {"dogs_humans", "humans", "dogs"} == set(fresh_db.table_names()) + humans = fresh_db["humans"] + dogs_humans = fresh_db["dogs_humans"] + assert [{"id": 1, "name": "Natalie D"}] == list(humans.rows) + assert [{"humans_id": 1, "dogs_id": 1}] == list(dogs_humans.rows) + + +def test_insert_m2m_list(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( + "humans", + [{"id": 1, "name": "Natalie D"}, {"id": 2, "name": "Simon W"}], + pk="id", + ) + assert {"dogs", "humans", "dogs_humans"} == set(fresh_db.table_names()) + humans = fresh_db["humans"] + dogs_humans = fresh_db["dogs_humans"] + assert [{"humans_id": 1, "dogs_id": 1}, {"humans_id": 2, "dogs_id": 1}] == list( + dogs_humans.rows + ) + assert [{"id": 1, "name": "Natalie D"}, {"id": 2, "name": "Simon W"}] == list( + humans.rows + ) + assert [ + ForeignKey( + table="dogs_humans", column="dogs_id", other_table="dogs", other_column="id" + ), + ForeignKey( + table="dogs_humans", + column="humans_id", + other_table="humans", + other_column="id", + ), + ] == dogs_humans.foreign_keys From ff2348e71af6705dfa3220d823ce0285e95b127f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jul 2019 09:16:46 +0300 Subject: [PATCH 0027/1004] Added failing tests --- tests/test_m2m.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_m2m.py b/tests/test_m2m.py index e7d6e30..c7e593d 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -41,3 +41,20 @@ def test_insert_m2m_list(fresh_db): other_column="id", ), ] == dogs_humans.foreign_keys + + +def test_m2m_explicit_argument(fresh_db): + # .m2m("humans", ..., m2m_table="relationships") + assert False + + +def test_uses_existing_m2m_table_if_exists(fresh_db): + # Code should look for an existing toble with fks to both tables + # and use that if it exists. + assert False + + +def test_requires_explicit_m2m_table_if_multiple_options(fresh_db): + # If the code scans for m2m tables and finds more than one candidate + # it should require that the m2m_table=x argument is used + assert False From ba1211d4456911bf0bd13f2e753a56ed988df3b4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 3 Aug 2019 17:28:03 +0300 Subject: [PATCH 0028/1004] Implemented .m2m(table, lookup=...) --- sqlite_utils/db.py | 28 +++++++++++++++++++--------- tests/test_m2m.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9f50a53..e759910 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1048,21 +1048,31 @@ class Table: self.create_index(column_values.keys(), unique=True) return pk - def m2m(self, other_table, record_or_list, pk=None): + def m2m(self, other_table, record_or_list=None, pk=None, lookup=None): our_id = self.last_pk - records = ( - [record_or_list] - if not isinstance(record_or_list, (list, tuple)) - else record_or_list - ) + if lookup is not None: + assert record_or_list is None, "Provide lookup= or record, not both" + else: + assert record_or_list is not None, "Provide lookup= or record, not both" tables = list(sorted([self.name, other_table])) columns = ["{}_id".format(t) for t in tables] m2m_table = self.db.table( "{}_{}".format(*tables), pk=columns, foreign_keys=columns ) - # Ensure each record exists in other table - for record in records: - id = self.db[other_table].upsert(record, pk=pk).last_pk + if lookup is None: + records = ( + [record_or_list] + if not isinstance(record_or_list, (list, tuple)) + else record_or_list + ) + # Ensure each record exists in other table + for record in records: + id = self.db[other_table].upsert(record, pk=pk).last_pk + m2m_table.upsert( + {"{}_id".format(other_table): id, "{}_id".format(self.name): our_id} + ) + else: + id = self.db[other_table].lookup(lookup) m2m_table.upsert( {"{}_id".format(other_table): id, "{}_id".format(self.name): our_id} ) diff --git a/tests/test_m2m.py b/tests/test_m2m.py index c7e593d..471166e 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -43,6 +43,37 @@ def test_insert_m2m_list(fresh_db): ] == dogs_humans.foreign_keys +def test_m2m_lookup(fresh_db): + people = fresh_db.table("people", pk="id") + people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) + people_tags = fresh_db["people_tags"] + tags = fresh_db["tags"] + assert people_tags.exists + assert tags.exists + assert [ + ForeignKey( + table="people_tags", + column="people_id", + other_table="people", + other_column="id", + ), + ForeignKey( + table="people_tags", column="tags_id", other_table="tags", other_column="id" + ), + ] == people_tags.foreign_keys + assert [{"people_id": 1, "tags_id": 1}] == list(people_tags.rows) + assert [{"id": 1, "name": "Wahyu"}] == list(people.rows) + assert [{"id": 1, "tag": "Coworker"}] == list(tags.rows) + + +def test_m2m_requires_either_records_or_lookup(fresh_db): + people = fresh_db.table("people", pk="id").insert({"name": "Wahyu"}) + with pytest.raises(AssertionError): + people.m2m("tags") + with pytest.raises(AssertionError): + people.m2m("tags", {"tag": "hello"}, lookup={"foo": "bar"}) + + def test_m2m_explicit_argument(fresh_db): # .m2m("humans", ..., m2m_table="relationships") assert False From b6b92980c00eda14a4d759b724139a0a2d321007 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 3 Aug 2019 20:51:22 +0300 Subject: [PATCH 0029/1004] table.m2m(..., m2m_table=x) argument --- sqlite_utils/db.py | 9 +++++---- tests/test_m2m.py | 11 ++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e759910..fdc4a99 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1048,7 +1048,9 @@ class Table: self.create_index(column_values.keys(), unique=True) return pk - def m2m(self, other_table, record_or_list=None, pk=None, lookup=None): + def m2m( + self, other_table, record_or_list=None, pk=None, lookup=None, m2m_table=None + ): our_id = self.last_pk if lookup is not None: assert record_or_list is None, "Provide lookup= or record, not both" @@ -1056,9 +1058,8 @@ class Table: assert record_or_list is not None, "Provide lookup= or record, not both" tables = list(sorted([self.name, other_table])) columns = ["{}_id".format(t) for t in tables] - m2m_table = self.db.table( - "{}_{}".format(*tables), pk=columns, foreign_keys=columns - ) + m2m_table_name = m2m_table or "{}_{}".format(*tables) + m2m_table = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) if lookup is None: records = ( [record_or_list] diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 471166e..10a3d50 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -74,9 +74,14 @@ def test_m2m_requires_either_records_or_lookup(fresh_db): people.m2m("tags", {"tag": "hello"}, lookup={"foo": "bar"}) -def test_m2m_explicit_argument(fresh_db): - # .m2m("humans", ..., m2m_table="relationships") - assert False +def test_m2m_explicit_table_name_argument(fresh_db): + people = fresh_db.table("people", pk="id") + people.insert({"name": "Wahyu"}).m2m( + "tags", lookup={"tag": "Coworker"}, m2m_table="tagged" + ) + assert fresh_db["tags"].exists + assert fresh_db["tagged"].exists + assert not fresh_db["people_tags"].exists def test_uses_existing_m2m_table_if_exists(fresh_db): From b9256413d26875c2bc3841e68b90d3842e88ccb8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 3 Aug 2019 21:07:06 +0300 Subject: [PATCH 0030/1004] db.m2m_table_candidates(table, other_table) --- sqlite_utils/db.py | 11 +++++++++++ tests/test_m2m.py | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index fdc4a99..f52b2e8 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -297,6 +297,17 @@ class Database: ) ) + def m2m_table_candidates(self, table, other_table): + "Returns potential m2m tables for arguments, based on FKs" + candidates = [] + tables = {table, other_table} + for table in self.tables: + # Does it have foreign keys to both table and other_table? + has_fks_to = {fk.other_table for fk in table.foreign_keys} + if has_fks_to.issuperset(tables): + candidates.append(table.name) + return candidates + def add_foreign_keys(self, foreign_keys): # foreign_keys is a list of explicit 4-tuples assert all( diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 10a3d50..5cfe29f 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -84,6 +84,28 @@ def test_m2m_explicit_table_name_argument(fresh_db): assert not fresh_db["people_tags"].exists +def test_m2m_table_candidates(fresh_db): + fresh_db.create_table("one", {"id": int, "name": str}, pk="id") + fresh_db.create_table("two", {"id": int, "name": str}, pk="id") + fresh_db.create_table("three", {"id": int, "name": str}, pk="id") + # No candidates at first + assert [] == fresh_db.m2m_table_candidates("one", "two") + # Create a candidate + fresh_db.create_table( + "one_m2m_two", {"one_id": int, "two_id": int}, foreign_keys=["one_id", "two_id"] + ) + assert ["one_m2m_two"] == fresh_db.m2m_table_candidates("one", "two") + # Add another table and there should be two candidates + fresh_db.create_table( + "one_m2m_two_and_three", + {"one_id": int, "two_id": int, "three_id": int}, + foreign_keys=["one_id", "two_id", "three_id"], + ) + assert {"one_m2m_two", "one_m2m_two_and_three"} == set( + fresh_db.m2m_table_candidates("one", "two") + ) + + def test_uses_existing_m2m_table_if_exists(fresh_db): # Code should look for an existing toble with fks to both tables # and use that if it exists. From d96a8f149ecb4d3fd8a8e5226774b7060c96ec95 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 3 Aug 2019 21:15:16 +0300 Subject: [PATCH 0031/1004] Use existing m2m table if one exists --- sqlite_utils/db.py | 17 ++++++++++++++++- tests/test_m2m.py | 33 +++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f52b2e8..6695580 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1069,7 +1069,22 @@ class Table: assert record_or_list is not None, "Provide lookup= or record, not both" tables = list(sorted([self.name, other_table])) columns = ["{}_id".format(t) for t in tables] - m2m_table_name = m2m_table or "{}_{}".format(*tables) + if m2m_table is not None: + m2m_table_name = m2m_table + else: + # Detect if there is a single, unambiguous option + candidates = self.db.m2m_table_candidates(self.name, other_table) + if len(candidates) == 1: + m2m_table_name = candidates[0] + elif len(candidates) > 1: + raise NoObviousTable( + "No single obvious m2m table for {}, {} - use m2m_table= parameter".format( + self.name, other_table + ) + ) + else: + # If not, create a new table + m2m_table_name = m2m_table or "{}_{}".format(*tables) m2m_table = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) if lookup is None: records = ( diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 5cfe29f..cdd7d8c 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import ForeignKey +from sqlite_utils.db import ForeignKey, NoObviousTable import pytest @@ -107,12 +107,37 @@ def test_m2m_table_candidates(fresh_db): def test_uses_existing_m2m_table_if_exists(fresh_db): - # Code should look for an existing toble with fks to both tables + # Code should look for an existing table with fks to both tables # and use that if it exists. - assert False + people = fresh_db.create_table("people", {"id": int, "name": str}, pk="id") + fresh_db["tags"].lookup({"tag": "Coworker"}) + fresh_db.create_table( + "tagged", + {"people_id": int, "tags_id": int}, + foreign_keys=["people_id", "tags_id"], + ) + people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) + assert fresh_db["tags"].exists + assert fresh_db["tagged"].exists + assert not fresh_db["people_tags"].exists + assert not fresh_db["tags_people"].exists + assert [{"people_id": 1, "tags_id": 1}] == list(fresh_db["tagged"].rows) def test_requires_explicit_m2m_table_if_multiple_options(fresh_db): # If the code scans for m2m tables and finds more than one candidate # it should require that the m2m_table=x argument is used - assert False + people = fresh_db.create_table("people", {"id": int, "name": str}, pk="id") + fresh_db["tags"].lookup({"tag": "Coworker"}) + fresh_db.create_table( + "tagged", + {"people_id": int, "tags_id": int}, + foreign_keys=["people_id", "tags_id"], + ) + fresh_db.create_table( + "tagged2", + {"people_id": int, "tags_id": int}, + foreign_keys=["people_id", "tags_id"], + ) + with pytest.raises(NoObviousTable): + people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) From 5516175ca6b9b2d48b7a929ba074b1ef69e981b0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 4 Aug 2019 05:09:17 +0300 Subject: [PATCH 0032/1004] Allow table objects to be passed to .m2m() --- sqlite_utils/db.py | 24 ++++++++++++++++-------- tests/test_m2m.py | 13 +++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6695580..3fcb133 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1060,26 +1060,28 @@ class Table: return pk def m2m( - self, other_table, record_or_list=None, pk=None, lookup=None, m2m_table=None + self, other_table, record_or_list=None, pk=DEFAULT, lookup=None, m2m_table=None ): + if isinstance(other_table, str): + other_table = self.db.table(other_table, pk=pk) our_id = self.last_pk if lookup is not None: assert record_or_list is None, "Provide lookup= or record, not both" else: assert record_or_list is not None, "Provide lookup= or record, not both" - tables = list(sorted([self.name, other_table])) + tables = list(sorted([self.name, other_table.name])) columns = ["{}_id".format(t) for t in tables] if m2m_table is not None: m2m_table_name = m2m_table else: # Detect if there is a single, unambiguous option - candidates = self.db.m2m_table_candidates(self.name, other_table) + candidates = self.db.m2m_table_candidates(self.name, other_table.name) if len(candidates) == 1: m2m_table_name = candidates[0] elif len(candidates) > 1: raise NoObviousTable( "No single obvious m2m table for {}, {} - use m2m_table= parameter".format( - self.name, other_table + self.name, other_table.name ) ) else: @@ -1094,14 +1096,20 @@ class Table: ) # Ensure each record exists in other table for record in records: - id = self.db[other_table].upsert(record, pk=pk).last_pk + id = other_table.upsert(record, pk=pk).last_pk m2m_table.upsert( - {"{}_id".format(other_table): id, "{}_id".format(self.name): our_id} + { + "{}_id".format(other_table.name): id, + "{}_id".format(self.name): our_id, + } ) else: - id = self.db[other_table].lookup(lookup) + id = other_table.lookup(lookup) m2m_table.upsert( - {"{}_id".format(other_table): id, "{}_id".format(self.name): our_id} + { + "{}_id".format(other_table.name): id, + "{}_id".format(self.name): our_id, + } ) return self diff --git a/tests/test_m2m.py b/tests/test_m2m.py index cdd7d8c..b5b897b 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -43,6 +43,19 @@ def test_insert_m2m_list(fresh_db): ] == dogs_humans.foreign_keys +def test_m2m_with_table_objects(fresh_db): + dogs = fresh_db.table("dogs", pk="id") + humans = fresh_db.table("humans", pk="id") + dogs.insert({"id": 1, "name": "Cleo"}).m2m( + humans, [{"id": 1, "name": "Natalie D"}, {"id": 2, "name": "Simon W"}] + ) + expected_tables = {"dogs", "humans", "dogs_humans"} + assert expected_tables == set(fresh_db.table_names()) + assert 1 == dogs.count + assert 2 == humans.count + assert 2 == fresh_db["dogs_humans"].count + + def test_m2m_lookup(fresh_db): people = fresh_db.table("people", pk="id") people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) From 243bcaa1acd32a173c07b24dca553991493005a0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 4 Aug 2019 05:29:19 +0300 Subject: [PATCH 0033/1004] Documentation for .m2m() table method --- docs/python-api.rst | 89 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index b1ad43d..de3c076 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -442,6 +442,95 @@ To extract the ``species`` column out to a separate ``Species`` table, you can d "species": "Common Juniper" }, extracts={"species": "Species"}) +.. _python_api_m2m: + +Working with many-to-many relationships +======================================= + +``sqlite-utils`` includes a shortcut for creating records using many-to-many relationships in the form of the ``table.m2m(...)`` method. + +Here's how to create two new records and connect them via a many-to-many table in a single line of code: + +.. code-block:: python + + db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id").m2m( + "humans", {"id": 1, "name": "Natalie"}, pk="id" + ) + +Running this example actually creates three tables: ``dogs``, ``humans`` and a many-to-many ``dogs_humans`` table. It will insert a record into each of those tables. + +The ``.m2m()`` method executes against the last record that was affected by ``.insert()`` or ``.update()`` - the record identified by the ``table.last_pk`` property. To execute ``.m2m()`` against a specific record you can first select it by passing its primary key to ``.update()``: + +.. code-block:: python + + db["dogs"].update(1).m2m( + "humans", {"id": 2, "name": "Simon"}, pk="id" + ) + +The first argument to ``.m2m()`` can be either the name of a table as a string or it can be the table object itself. + +The second argument can be a single dictionary record or a list of dictionaries. Thesee dictionaries will be passed to ``.upsert()`` against the specified table. + +Here's alternative code that creates the dog record and adds two people to it: + +.. code-block:: python + + db = Database(memory=True) + dogs = db.table("dogs", pk="id") + humans = db.table("humans", pk="id") + dogs.insert({"id": 1, "name": "Cleo"}).m2m( + humans, [ + {"id": 1, "name": "Natalie"}, + {"id": 2, "name": "Simon"} + ] + ) + +The method will attempt to find an existing many-to-many table by looking for a table that has foreign key relationships against both of the tables in the relationship. + +If it cannot find such a table, it will create a new one using the names of the two tables - ``dogs_humans`` in this example. You can customize the name of this table using the ``m2m_table=`` argument to ``.m2m()``. + +It it finds multiple candidate tables with foreign keys to both of the specified tables it will raise a ``sqlite_utils.db.NoObviousTable`` exception. You can avoid this error by specifying the correct table using ``m2m_table=``. + +.. _python_api_m2m_lookup: + +Using m2m and lookup tables together +------------------------------------ + +You can work with (or create) lookup tables as part of a call to ``.m2m()`` using the ``lookup=`` parameter. This accepts the same argument as ``table.lookup()`` does - a dictionary of values that should be used to lookup or create a row in the lookup table. + +This example creates a dogs table, populates it, creates a characteristics table, populates that and sets up a many-to-many relationship between the two. It chains ``.m2m()`` twice to create two associated characteristics: + +.. code-block:: python + + db = Database(memory=True) + dogs = db.table("dogs", pk="id") + dogs.insert({"id": 1, "name": "Cleo"}).m2m( + "characteristics", lookup={ + "name": "Playful" + } + ).m2m( + "characteristics", lookup={ + "name": "Opinionated" + } + ) + +You can inspect the database to see the results like this:: + + >>> db.table_names() + ['dogs', 'characteristics', 'characteristics_dogs'] + >>> list(db["dogs"].rows) + [{'id': 1, 'name': 'Cleo'}] + >>> list(db["characteristics"].rows) + [{'id': 1, 'name': 'Playful'}, {'id': 2, 'name': 'Opinionated'}] + >>> list(db["characteristics_dogs"].rows) + [{'characteristics_id': 1, 'dogs_id': 1}, {'characteristics_id': 2, 'dogs_id': 1}] + >>> print(db["characteristics_dogs"].schema) + CREATE TABLE [characteristics_dogs] ( + [characteristics_id] INTEGER REFERENCES [characteristics]([id]), + [dogs_id] INTEGER REFERENCES [dogs]([id]), + PRIMARY KEY ([characteristics_id], [dogs_id]) + ) + .. _python_api_add_column: Adding columns From 6ac0a5df5d714f6f3b33a92eed9ccbd8eebe66a4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 4 Aug 2019 06:35:30 +0300 Subject: [PATCH 0034/1004] Release 1.9 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1c51ee5..78df24b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v1_9: + +1.9 (2019-08-04) +---------------- + +- ``table.m2m(...)`` method for creating many-to-many relationships: :ref:`python_api_m2m` (`#23 `__) + .. _v1_8: 1.8 (2019-07-28) diff --git a/setup.py b/setup.py index 559ddb2..bb42bdf 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "1.8" +VERSION = "1.9" def get_long_description(): From 0e7b461eb3e925aef713206c15794ceae9259c57 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 4 Aug 2019 07:13:31 +0300 Subject: [PATCH 0035/1004] Fixed typo --- docs/python-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index de3c076..1dd0ceb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -469,7 +469,7 @@ The ``.m2m()`` method executes against the last record that was affected by ``.i The first argument to ``.m2m()`` can be either the name of a table as a string or it can be the table object itself. -The second argument can be a single dictionary record or a list of dictionaries. Thesee dictionaries will be passed to ``.upsert()`` against the specified table. +The second argument can be a single dictionary record or a list of dictionaries. These dictionaries will be passed to ``.upsert()`` against the specified table. Here's alternative code that creates the dog record and adds two people to it: From 9faa98222669723d31e918bb16a42c13c363817f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 23 Aug 2019 15:19:41 +0300 Subject: [PATCH 0036/1004] Ability to introspect and run queries against views (#55) * db.views_names() method and and db.views property * Separate View and Table classes, both subclassing new Queryable class * view.drop() method * Updated documentation --- docs/python-api.rst | 45 +++++++++++--- sqlite_utils/db.py | 129 ++++++++++++++++++++++++--------------- tests/test_create.py | 7 +++ tests/test_introspect.py | 17 +++++- 4 files changed, 142 insertions(+), 56 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 1dd0ceb..28ae96f 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -45,6 +45,8 @@ You can also access tables using the ``.table()`` method like so: Using this factory function allows you to set :ref:`python_api_table_configuration`. +.. _python_api_tables: + Listing tables ============== @@ -60,6 +62,31 @@ You can also iterate through the table objects themselves using the ``.tables`` >>> db.tables [
] +.. _python_api_views: + +Listing views +============= + +``.table_views()`` shows you a list of views in the database:: + + >>> db.table_views() + ['good_dogs'] + +You can iterate through view objects using the ``.views`` property:: + + >>> db.views + [] + +View objects are similar to Table objects, except that any attempts to insert or update data will throw an error. The full list of methods and properties available on a view object is as follows: + +* ``columns`` +* ``columns_dict`` +* ``count`` +* ``schema`` +* ``rows`` +* ``rows_where(where, where_args)`` +* ``drop()`` + .. _python_api_rows: Listing rows @@ -682,10 +709,10 @@ If you want to ensure that every foreign key column in your database has a corre .. _python_api_drop: -Dropping a table -================ +Dropping a table or view +======================== -You can drop a table by using the ``.drop()`` method: +You can drop a table or view using the ``.drop()`` method: .. code-block:: python @@ -764,7 +791,7 @@ For example: Introspection ============= -If you have loaded an existing table, you can use introspection to find out more about it:: +If you have loaded an existing table or view, you can use introspection to find out more about it:: >>> db["PlantType"]
@@ -776,7 +803,7 @@ The ``.count`` property shows the current number of rows (``select count(*) from >>> db["Street_Tree_List"].count 189144 -The ``.columns`` property shows the columns in the table:: +The ``.columns`` property shows the columns in the table or view:: >>> db["PlantType"].columns [Column(cid=0, name='id', type='INTEGER', notnull=0, default_value=None, is_pk=1), @@ -787,7 +814,9 @@ The ``.columns_dict`` property returns a dictionary version of this with just th >>> db["PlantType"].columns_dict {'id': , 'value': } -The ``.foreign_keys`` property shows if the table has any foreign key relationships:: +The ``.foreign_keys`` property shows if the table has any foreign key relationships. It is not available on views. + +:: >>> db["Street_Tree_List"].foreign_keys [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id'), @@ -827,7 +856,9 @@ The ``.schema`` property outputs the table's schema as a SQL string:: FOREIGN KEY ("qCareAssistant") REFERENCES [qCareAssistant](id), FOREIGN KEY ("qLegalStatus") REFERENCES [qLegalStatus](id)) -The ``.indexes`` property shows you all indexes created for a table:: +The ``.indexes`` property shows you all indexes created for a table. It is not available on views. + +:: >>> db["Street_Tree_List"].indexes [Index(seq=0, name='"Street_Tree_List_qLegalStatus"', unique=0, origin='c', partial=0, columns=['qLegalStatus']), diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3fcb133..6247428 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -63,6 +63,7 @@ if np: REVERSE_COLUMN_TYPE_MAPPING = { + "": str, # Columns in views sometimes have type = '' "TEXT": str, "BLOB": bytes, "INTEGER": int, @@ -107,7 +108,8 @@ class Database: return "".format(self.conn) def table(self, table_name, **kwargs): - return Table(self, table_name, **kwargs) + klass = View if table_name in self.view_names() else Table + return klass(self, table_name, **kwargs) def escape(self, value): # Normally we would use .execute(sql, [params]) for escaping, but @@ -128,10 +130,22 @@ class Database: sql = "select name from sqlite_master where {}".format(" AND ".join(where)) return [r[0] for r in self.conn.execute(sql).fetchall()] + def view_names(self): + return [ + r[0] + for r in self.conn.execute( + "select name from sqlite_master where type = 'view'" + ).fetchall() + ] + @property def tables(self): return [self[name] for name in self.table_names()] + @property + def views(self): + return [self[name] for name in self.view_names()] + def execute_returning_dicts(self, sql, params=None): cursor = self.conn.execute(sql, params or tuple()) keys = [d[0] for d in cursor.description] @@ -385,7 +399,59 @@ class Database: self.conn.execute("VACUUM;") -class Table: +class Queryable: + exists = False + + def __init__(self, db, name): + self.db = db + self.name = name + + @property + def count(self): + return self.db.conn.execute( + "select count(*) from [{}]".format(self.name) + ).fetchone()[0] + + @property + def rows(self): + return self.rows_where() + + def rows_where(self, where=None, where_args=None): + if not self.exists: + return [] + sql = "select * from [{}]".format(self.name) + if where is not None: + sql += " where " + where + cursor = self.db.conn.execute(sql, where_args or []) + columns = [c[0] for c in cursor.description] + for row in cursor: + yield dict(zip(columns, row)) + + @property + def columns(self): + if not self.exists: + return [] + rows = self.db.conn.execute( + "PRAGMA table_info([{}])".format(self.name) + ).fetchall() + return [Column(*row) for row in rows] + + @property + def columns_dict(self): + "Returns {column: python-type} dictionary" + return { + column.name: REVERSE_COLUMN_TYPE_MAPPING[column.type] + for column in self.columns + } + + @property + def schema(self): + return self.db.conn.execute( + "select sql from sqlite_master where name = ?", (self.name,) + ).fetchone()[0] + + +class Table(Queryable): def __init__( self, db, @@ -402,8 +468,7 @@ class Table: ignore=False, extracts=None, ): - self.db = db - self.name = name + super().__init__(db, name) self.exists = self.name in self.db.table_names() self._defaults = dict( pk=pk, @@ -427,44 +492,6 @@ class Table: else " ({})".format(", ".join(c.name for c in self.columns)), ) - @property - def count(self): - return self.db.conn.execute( - "select count(*) from [{}]".format(self.name) - ).fetchone()[0] - - @property - def columns(self): - if not self.exists: - return [] - rows = self.db.conn.execute( - "PRAGMA table_info([{}])".format(self.name) - ).fetchall() - return [Column(*row) for row in rows] - - @property - def columns_dict(self): - "Returns {column: python-type} dictionary" - return { - column.name: REVERSE_COLUMN_TYPE_MAPPING[column.type] - for column in self.columns - } - - @property - def rows(self): - return self.rows_where() - - def rows_where(self, where=None, where_args=None): - if not self.exists: - return [] - sql = "select * from [{}]".format(self.name) - if where is not None: - sql += " where " + where - cursor = self.db.conn.execute(sql, where_args or []) - columns = [c[0] for c in cursor.description] - for row in cursor: - yield dict(zip(columns, row)) - @property def pks(self): names = [column.name for column in self.columns if column.is_pk] @@ -511,12 +538,6 @@ class Table: ) return fks - @property - def schema(self): - return self.db.conn.execute( - "select sql from sqlite_master where name = ?", (self.name,) - ).fetchone()[0] - @property def indexes(self): sql = 'PRAGMA index_list("{}")'.format(self.name) @@ -1114,6 +1135,18 @@ class Table: return self +class View(Queryable): + exists = True + + def __repr__(self): + return "".format( + self.name, ", ".join(c.name for c in self.columns) + ) + + def drop(self): + self.db.conn.execute("DROP VIEW {}".format(self.name)) + + def chunks(sequence, size): iterator = iter(sequence) for item in iterator: diff --git a/tests/test_create.py b/tests/test_create.py index b4778f3..05dfe4a 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -782,3 +782,10 @@ def test_drop(fresh_db): assert ["t"] == fresh_db.table_names() assert None is fresh_db["t"].drop() assert [] == fresh_db.table_names() + + +def test_drop_view(fresh_db): + fresh_db.create_view("foo_view", "select 1") + assert ["foo_view"] == fresh_db.view_names() + assert None is fresh_db["foo_view"].drop() + assert [] == fresh_db.view_names() diff --git a/tests/test_introspect.py b/tests/test_introspect.py index d3ab27a..b04af6d 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import Index +from sqlite_utils.db import Index, View import pytest @@ -6,6 +6,11 @@ def test_table_names(existing_db): assert ["foo"] == existing_db.table_names() +def test_view_names(fresh_db): + fresh_db.create_view("foo_view", "select 1") + assert ["foo_view"] == fresh_db.view_names() + + def test_table_names_fts4(existing_db): existing_db["woo"].insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS4" @@ -36,6 +41,16 @@ def test_tables(existing_db): assert "foo" == existing_db.tables[0].name +def test_views(fresh_db): + fresh_db.create_view("foo_view", "select 1") + assert 1 == len(fresh_db.views) + view = fresh_db.views[0] + assert isinstance(view, View) + assert "foo_view" == view.name + assert "" == repr(view) + assert {"1": str} == view.columns_dict + + def test_count(existing_db): assert 3 == existing_db["foo"].count From 53124bc02fac5a89b154513f5fdc67431901fad9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 23 Aug 2019 15:24:04 +0300 Subject: [PATCH 0037/1004] Release 1.10 --- docs/changelog.rst | 13 +++++++++++++ setup.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 78df24b..ea4b13c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,19 @@ Changelog =========== +.. _v1_10: + +1.10 (2019-08-23) +----------------- + +Ability to introspect and run queries against views (`#54 `__) + +- ``db.views_names()`` method and and ``db.views`` property +- Separate ``View`` and ``Table`` classes, both subclassing new ``Queryable`` class +- ``view.drop()`` method + +See :ref:`python_api_views`. + .. _v1_9: 1.9 (2019-08-04) diff --git a/setup.py b/setup.py index bb42bdf..9d0a9a6 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "1.9" +VERSION = "1.10" def get_long_description(): From 68a5cb1b8ef2264bed68d4763f04bd47b1aa5d05 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Aug 2019 07:03:22 +0200 Subject: [PATCH 0038/1004] Corrected .table_views() -> .view_names() --- docs/python-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 28ae96f..9d272c3 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -67,9 +67,9 @@ You can also iterate through the table objects themselves using the ``.tables`` Listing views ============= -``.table_views()`` shows you a list of views in the database:: +``.view_names()`` shows you a list of views in the database:: - >>> db.table_views() + >>> db.view_names() ['good_dogs'] You can iterate through view objects using the ``.views`` property:: From cb70f7d10996b844154bf3da88779dd1f65590bc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Aug 2019 07:04:14 +0200 Subject: [PATCH 0039/1004] Corrected .views_names() -> .view_names() --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ea4b13c..02b5e41 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ Ability to introspect and run queries against views (`#54 `__) -- ``db.views_names()`` method and and ``db.views`` property +- ``db.view_names()`` method and and ``db.views`` property - Separate ``View`` and ``Table`` classes, both subclassing new ``Queryable`` class - ``view.drop()`` method From 405e092d5916e70df10f82d15e9c052aa9ee8d80 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 2 Sep 2019 16:42:28 -0700 Subject: [PATCH 0040/1004] Option to add triggers when enabling FTS (#57) --create-triggers CLI option and create_triggers=True in the Python library * Add an option to create triggers for fts table. * Add cli option for the create-update-trigger. * Add tests for the create-update-trigger option. * Change FTS table escaping to square brackets. --- sqlite_utils/cli.py | 12 ++++++++-- sqlite_utils/db.py | 51 ++++++++++++++++++++++++++++------------ tests/test_cli.py | 38 ++++++++++++++++++++++++++++++ tests/test_enable_fts.py | 44 ++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 17 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 7c09578..0b51517 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -265,7 +265,13 @@ def create_index(path, table, column, name, unique, if_not_exists): @click.argument("column", nargs=-1, required=True) @click.option("--fts4", help="Use FTS4", default=False, is_flag=True) @click.option("--fts5", help="Use FTS5", default=False, is_flag=True) -def enable_fts(path, table, column, fts4, fts5): +@click.option( + "--create-triggers", + help="Create triggers to update the FTS tables when the parent table changes.", + default=False, + is_flag=True, +) +def enable_fts(path, table, column, fts4, fts5, create_triggers): "Enable FTS for specific table and columns" fts_version = "FTS5" if fts4 and fts5: @@ -275,7 +281,9 @@ def enable_fts(path, table, column, fts4, fts5): fts_version = "FTS4" db = sqlite_utils.Database(path) - db[table].enable_fts(column, fts_version=fts_version) + db[table].enable_fts( + column, fts_version=fts_version, create_triggers=create_triggers + ) @cli.command(name="populate-fts") diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6247428..b7837c7 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -646,7 +646,7 @@ class Table(Queryable): return self def drop(self): - self.db.conn.execute("DROP TABLE {}".format(self.name)) + self.db.conn.execute("DROP TABLE [{}]".format(self.name)) def guess_foreign_table(self, column): column = column.lower() @@ -710,12 +710,12 @@ class Table(Queryable): ) self.db.add_foreign_keys([(self.name, column, other_table, other_column)]) - def enable_fts(self, columns, fts_version="FTS5"): - "Enables FTS on the specified columns" + def enable_fts(self, columns, fts_version="FTS5", create_triggers=False): + "Enables FTS on the specified columns." sql = """ - CREATE VIRTUAL TABLE "{table}_fts" USING {fts_version} ( + CREATE VIRTUAL TABLE [{table}_fts] USING {fts_version} ( {columns}, - content="{table}" + content=[{table}] ); """.format( table=self.name, @@ -724,12 +724,34 @@ class Table(Queryable): ) self.db.conn.executescript(sql) self.populate_fts(columns) + + if create_triggers: + old_cols = ", ".join("old.[{}]".format(c) for c in columns) + new_cols = ", ".join("new.[{}]".format(c) for c in columns) + triggers = """ + CREATE TRIGGER [{table}_ai] AFTER INSERT ON [{table}] BEGIN + INSERT INTO [{table}_fts] (rowid, {columns}) VALUES (new.rowid, {new_cols}); + END; + CREATE TRIGGER [{table}_ad] AFTER DELETE ON [{table}] BEGIN + INSERT INTO [{table}_fts] ([{table}_fts], rowid, {columns}) VALUES('delete', old.rowid, {old_cols}); + END; + CREATE TRIGGER [{table}_au] AFTER UPDATE ON [{table}] BEGIN + INSERT INTO [{table}_fts] ([{table}_fts], rowid, {columns}) VALUES('delete', old.rowid, {old_cols}); + INSERT INTO [{table}_fts] (rowid, {columns}) VALUES (new.rowid, {new_cols}); + END; + """.format( + table=self.name, + columns=", ".join("[{}]".format(c) for c in columns), + old_cols=old_cols, + new_cols=new_cols, + ) + self.db.conn.executescript(triggers) return self def populate_fts(self, columns): sql = """ - INSERT INTO "{table}_fts" (rowid, {columns}) - SELECT rowid, {columns} FROM {table}; + INSERT INTO [{table}_fts] (rowid, {columns}) + SELECT rowid, {columns} FROM [{table}]; """.format( table=self.name, columns=", ".join(columns) ) @@ -738,21 +760,20 @@ class Table(Queryable): def detect_fts(self): "Detect if table has a corresponding FTS virtual table and return it" - rows = self.db.conn.execute( - """ + sql = """ SELECT name FROM sqlite_master WHERE rootpage = 0 AND ( - sql LIKE '%VIRTUAL TABLE%USING FTS%content="{table}"%' + sql LIKE '%VIRTUAL TABLE%USING FTS%content=%{table}%' OR ( tbl_name = "{table}" AND sql LIKE '%VIRTUAL TABLE%USING FTS%' ) ) """.format( - table=self.name - ) - ).fetchall() + table=self.name + ) + rows = self.db.conn.execute(sql).fetchall() if len(rows) == 0: return None else: @@ -796,7 +817,7 @@ class Table(Queryable): def search(self, q): sql = """ - select * from {table} where rowid in ( + select * from "{table}" where rowid in ( select rowid from [{table}_fts] where [{table}_fts] match :search ) @@ -1144,7 +1165,7 @@ class View(Queryable): ) def drop(self): - self.db.conn.execute("DROP VIEW {}".format(self.name)) + self.db.conn.execute("DROP VIEW [{}]".format(self.name)) def chunks(sequence, size): diff --git a/tests/test_cli.py b/tests/test_cli.py index ddb936b..ca91bdb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -315,6 +315,7 @@ def test_index_foreign_keys(db_path): db = Database(db_path) assert [] == db["books"].indexes result = CliRunner().invoke(cli.cli, ["index-foreign-keys", db_path]) + assert 0 == result.exit_code assert [["author_id"], ["author_name_ref"]] == [ i.columns for i in db["books"].indexes ] @@ -328,6 +329,43 @@ def test_enable_fts(db_path): assert 0 == result.exit_code assert "Gosh_fts" == Database(db_path)["Gosh"].detect_fts() + # Table names with restricted chars are handled correctly. + # colons and dots are restricted characters for table names. + Database(db_path)["http://example.com"].create({"c1": str, "c2": str, "c3": str}) + assert None == Database(db_path)["http://example.com"].detect_fts() + result = CliRunner().invoke( + cli.cli, ["enable-fts", db_path, "http://example.com", "c1", "--fts4"] + ) + assert 0 == result.exit_code + assert ( + "http://example.com_fts" == Database(db_path)["http://example.com"].detect_fts() + ) + Database(db_path)["http://example.com"].drop() + + +def test_enable_fts_with_triggers(db_path): + Database(db_path)["Gosh"].insert_all([{"c1": "baz"}]) + exit_code = ( + CliRunner() + .invoke( + cli.cli, + ["enable-fts", db_path, "Gosh", "c1", "--fts4", "--create-triggers"], + ) + .exit_code + ) + assert 0 == exit_code + + def search(q): + return ( + Database(db_path) + .conn.execute("select c1 from Gosh_fts where c1 match ?", [q]) + .fetchall() + ) + + assert [("baz",)] == search("baz") + Database(db_path)["Gosh"].insert_all([{"c1": "martha"}]) + assert [("martha",)] == search("martha") + def test_populate_fts(db_path): Database(db_path)["Gosh"].insert_all([{"c1": "baz"}]) diff --git a/tests/test_enable_fts.py b/tests/test_enable_fts.py index 6cefe54..7f58916 100644 --- a/tests/test_enable_fts.py +++ b/tests/test_enable_fts.py @@ -22,6 +22,26 @@ def test_enable_fts(fresh_db): assert [] == table.search("bar") +def test_enable_fts_escape_table_names(fresh_db): + # Table names with restricted chars are handled correctly. + # colons and dots are restricted characters for table names. + table = fresh_db["http://example.com"] + table.insert_all(search_records) + assert ["http://example.com"] == fresh_db.table_names() + table.enable_fts(["text", "country"], fts_version="FTS4") + assert [ + "http://example.com", + "http://example.com_fts", + "http://example.com_fts_segments", + "http://example.com_fts_segdir", + "http://example.com_fts_docsize", + "http://example.com_fts_stat", + ] == fresh_db.table_names() + assert [("tanuki are tricksters", "Japan", "foo")] == table.search("tanuki") + assert [("racoons are trash pandas", "USA", "bar")] == table.search("usa") + assert [] == table.search("bar") + + def test_populate_fts(fresh_db): table = fresh_db["populatable"] table.insert(search_records[0]) @@ -34,6 +54,19 @@ def test_populate_fts(fresh_db): assert [("racoons are trash pandas", "USA", "bar")] == table.search("usa") +def test_populate_fts_escape_table_names(fresh_db): + # Restricted characters such as colon and dots should be escaped. + table = fresh_db["http://example.com"] + table.insert(search_records[0]) + table.enable_fts(["text", "country"], fts_version="FTS4") + assert [] == table.search("trash pandas") + table.insert(search_records[1]) + assert [] == table.search("trash pandas") + # Now run populate_fts to make this record available + table.populate_fts(["text", "country"]) + assert [("racoons are trash pandas", "USA", "bar")] == table.search("usa") + + def test_optimize_fts(fresh_db): for fts_version in ("4", "5"): table_name = "searchable_{}".format(fts_version) @@ -48,3 +81,14 @@ def test_optimize_fts(fresh_db): "searchable_5_fts", ): fresh_db[table_name].optimize() + + +def test_enable_fts_w_triggers(fresh_db): + table = fresh_db["searchable"] + table.insert(search_records[0]) + table.enable_fts(["text", "country"], fts_version="FTS4", create_triggers=True) + assert [("tanuki are tricksters", "Japan", "foo")] == table.search("tanuki") + table.insert(search_records[1]) + # Triggers will auto-populate FTS virtual table, not need to call populate_fts() + assert [("racoons are trash pandas", "USA", "bar")] == table.search("usa") + assert [] == table.search("bar") From 2ca63e3b2de5408a860c6c7c1852deb9a138279e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Sep 2019 17:09:41 -0700 Subject: [PATCH 0041/1004] db.triggers and table.triggers introspection (#60) Closes #59 --- docs/python-api.rst | 11 +++++++++++ sqlite_utils/db.py | 23 +++++++++++++++++++++++ tests/test_introspect.py | 27 +++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 9d272c3..997bd97 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -868,6 +868,17 @@ The ``.indexes`` property shows you all indexes created for a table. It is not a Index(seq=4, name='"Street_Tree_List_qCaretaker"', unique=0, origin='c', partial=0, columns=['qCaretaker']), Index(seq=5, name='"Street_Tree_List_PlantType"', unique=0, origin='c', partial=0, columns=['PlantType'])] +The ``.triggers`` property lists database triggers. It can be used on both database and table objects. + +:: + + >>> db["authors"].triggers + [Trigger(name='authors_ai', table='authors', sql='CREATE TRIGGER [authors_ai] AFTER INSERT...'), + Trigger(name='authors_ad', table='authors', sql="CREATE TRIGGER [authors_ad] AFTER DELETE..."), + Trigger(name='authors_au', table='authors', sql="CREATE TRIGGER [authors_au] AFTER UPDATE")] + >>> db.triggers + ... similar output to db["authors"].triggers + Enabling full-text search ========================= diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b7837c7..e36e191 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -20,6 +20,9 @@ ForeignKey = namedtuple( "ForeignKey", ("table", "column", "other_table", "other_column") ) Index = namedtuple("Index", ("seq", "name", "unique", "origin", "partial", "columns")) +Trigger = namedtuple("Trigger", ("name", "table", "sql")) + + DEFAULT = object() COLUMN_TYPE_MAPPING = { @@ -146,6 +149,15 @@ class Database: def views(self): return [self[name] for name in self.view_names()] + @property + def triggers(self): + return [ + Trigger(*r) + for r in self.conn.execute( + "select name, tbl_name, sql from sqlite_master where type = 'trigger'" + ).fetchall() + ] + def execute_returning_dicts(self, sql, params=None): cursor = self.conn.execute(sql, params or tuple()) keys = [d[0] for d in cursor.description] @@ -561,6 +573,17 @@ class Table(Queryable): indexes.append(Index(**row)) return indexes + @property + def triggers(self): + return [ + Trigger(*r) + for r in self.db.conn.execute( + "select name, tbl_name, sql from sqlite_master where type = 'trigger'" + " and tbl_name = ?", + (self.name,), + ).fetchall() + ] + def create( self, columns, diff --git a/tests/test_introspect.py b/tests/test_introspect.py index b04af6d..7c01a0b 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -121,3 +121,30 @@ def test_guess_foreign_table(fresh_db, column, expected_table_guess): def test_pks(fresh_db, pk, expected): fresh_db["foo"].insert_all([{"id": 1, "id2": 2}], pk=pk) assert expected == fresh_db["foo"].pks + + +def test_triggers(fresh_db): + assert [] == fresh_db.triggers + authors = fresh_db["authors"] + authors.insert_all( + [ + {"name": "Frank Herbert", "famous_works": "Dune"}, + {"name": "Neal Stephenson", "famous_works": "Cryptonomicon"}, + ] + ) + fresh_db["other"].insert({"foo": "bar"}) + assert [] == authors.triggers + assert [] == fresh_db["other"].triggers + authors.enable_fts( + ["name", "famous_works"], fts_version="FTS4", create_triggers=True + ) + expected_triggers = { + ("authors_ai", "authors"), + ("authors_ad", "authors"), + ("authors_au", "authors"), + } + assert expected_triggers == {(t.name, t.table) for t in fresh_db.triggers} + assert expected_triggers == { + (t.name, t.table) for t in fresh_db["authors"].triggers + } + assert [] == fresh_db["other"].triggers From 3a4dddaca23157984dbda7c16a957d3fc459024f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Sep 2019 17:10:26 -0700 Subject: [PATCH 0042/1004] Documentation for create-triggers, refs #57 --- docs/cli.rst | 4 ++++ docs/python-api.rst | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 6597fb6..b67fbf9 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -379,6 +379,10 @@ The ``enable-fts`` command will populate the new index with all existing documen $ sqlite-utils populate-fts mydb.db documents title summary +A better solution here is to use database triggers. You can set up database triggers to automatically update the full-text index using the ``--create-triggers`` option when you first run ``enable-fts``:: + + $ sqlite-utils enable-fts mydb.db documents title summary --create-triggers + Vacuum ====== diff --git a/docs/python-api.rst b/docs/python-api.rst index 997bd97..572e330 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -907,6 +907,12 @@ If you insert additional records into the table you will need to refresh the sea }, pk="id") dogs.populate_fts(["name", "twitter"]) +A better solution is to use database triggers. You can set up database triggers to automatically update the full-text index using ``create_triggers=True``: + +.. code-block:: python + + dogs.enable_fts(["name", "twitter"], create_triggers=True) + ``.enable_fts()`` defaults to using `FTS5 `__. If you wish to use `FTS4 `__ instead, use the following: .. code-block:: python From d5e1f8ac77d91b9b713358e80d9542abbf5f3633 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Sep 2019 17:58:41 -0700 Subject: [PATCH 0043/1004] Release 1.11 --- docs/changelog.rst | 11 +++++++++++ docs/python-api.rst | 4 ++++ setup.py | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 02b5e41..4d56e96 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,17 @@ Changelog =========== +.. _v1_11: + +1.11 (2019-09-02) +----------------- + +Option to create triggers to automatically keep FTS tables up-to-date with newly inserted, updated and deleted records. Thanks, Amjith Ramanujam! (`#57 `__) + +- ``sqlite-utils enable-fts ... --create-triggers`` - see :ref:`Configuring full-text search using the CLI ` +- ``db["tablename"].enable_fts(..., create_triggers=True)`` - see :ref:`Configuring full-text search using the Python library ` +- Support for introspecting triggers for a database or table - see :ref:`python_api_introspection` (`#59 `__) + .. _v1_10: 1.10 (2019-08-23) diff --git a/docs/python-api.rst b/docs/python-api.rst index 572e330..4d9ba7e 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -788,6 +788,8 @@ For example: """).fetchall() # Returns [('Felton, CA',)] +.. _python_api_introspection: + Introspection ============= @@ -879,6 +881,8 @@ The ``.triggers`` property lists database triggers. It can be used on both datab >>> db.triggers ... similar output to db["authors"].triggers +.. _python_api_fts: + Enabling full-text search ========================= diff --git a/setup.py b/setup.py index 9d0a9a6..8e216c6 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "1.10" +VERSION = "1.11" def get_long_description(): From eb39c84a8f27443abb7aaebc1724c99f68e441fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 4 Oct 2019 09:17:27 -0700 Subject: [PATCH 0044/1004] Test and docs for using :memory: as a filename --- docs/cli.rst | 5 +++++ tests/test_cli.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index b67fbf9..10c3345 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -55,6 +55,11 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] +You can run queries against a temporary in-memory database by passing ``:memory:`` as the filename:: + + $ sqlite-utils :memory: "select sqlite_version()" + [{"sqlite_version()": "3.29.0"}] + .. _cli_json_values: Nested JSON values diff --git a/tests/test_cli.py b/tests/test_cli.py index ca91bdb..d9ebe55 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -724,6 +724,20 @@ def test_query_json_with_json_cols(db_path): assert expected == result_rows.output.strip() +def test_query_memory_does_not_create_file(tmpdir): + owd = os.getcwd() + try: + os.chdir(tmpdir) + # This should create a foo.db file + CliRunner().invoke(cli.cli, ["foo.db", "select sqlite_version()"]) + # This should NOT create a file + result = CliRunner().invoke(cli.cli, [":memory:", "select sqlite_version()"]) + assert ["sqlite_version()"] == list(json.loads(result.output)[0].keys()) + finally: + os.chdir(owd) + assert ["foo.db"] == os.listdir(tmpdir) + + @pytest.mark.parametrize( "args,expected", [ From 19073d6d972fad9d68dd74c28544cd29083f1c12 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 4 Nov 2019 08:07:44 -0800 Subject: [PATCH 0045/1004] Added table.delete(pk) method, refs #62 --- docs/python-api.rst | 14 ++++++++++++++ sqlite_utils/db.py | 11 +++++++++++ tests/test_delete.py | 14 ++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 tests/test_delete.py diff --git a/docs/python-api.rst b/docs/python-api.rst index 4d9ba7e..374bdc6 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -382,6 +382,20 @@ You can cause any missing columns to be added automatically using ``alter=True`` >>> db["dogs"].update(1, {"breed": "Mutt"}, alter=True) +.. _python_api_delete: + +Deleting a specific record +========================== + +You can delete a record using ``table.delete()``:: + + >>> db = sqlite_utils.Database("dogs.db") + >>> db["dogs"].delete(1) + +The ``delete()`` method takes the primary key of the record. This can be a tuple of values if the row has a compound primary key:: + + >>> db["compound_dogs"].delete((5, 3)) + Upserting data ============== diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e36e191..68a2002 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -853,6 +853,17 @@ class Table(Queryable): def value_or_default(self, key, value): return self._defaults[key] if value is DEFAULT else value + def delete(self, pk_values): + if not isinstance(pk_values, (list, tuple)): + pk_values = [pk_values] + self.get(pk_values) + wheres = ["[{}] = ?".format(pk_name) for pk_name in self.pks] + sql = "delete from [{table}] where {wheres}".format( + table=self.name, wheres=" and ".join(wheres) + ) + with self.db.conn: + self.db.conn.execute(sql, pk_values) + def update(self, pk_values, updates=None, alter=False): updates = updates or {} if not isinstance(pk_values, (list, tuple)): diff --git a/tests/test_delete.py b/tests/test_delete.py new file mode 100644 index 0000000..6758831 --- /dev/null +++ b/tests/test_delete.py @@ -0,0 +1,14 @@ +def test_delete_rowid_table(fresh_db): + table = fresh_db["table"] + table.insert({"foo": 1}).last_pk + rowid = table.insert({"foo": 2}).last_pk + table.delete(rowid) + assert [{"foo": 1}] == list(table.rows) + + +def test_delete_pk_table(fresh_db): + table = fresh_db["table"] + table.insert({"id": 1}, pk="id") + table.insert({"id": 2}, pk="id") + table.delete(1) + assert [{"id": 2}] == list(table.rows) From 169ea455fc1f1d5e5b6e44cb339ba7ffa9d49c31 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 4 Nov 2019 08:18:06 -0800 Subject: [PATCH 0046/1004] Added table.delete_where(), closes #62 --- docs/python-api.rst | 11 +++++++++++ sqlite_utils/db.py | 8 ++++++++ tests/test_delete.py | 18 ++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 374bdc6..cadad15 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -396,6 +396,17 @@ The ``delete()`` method takes the primary key of the record. This can be a tuple >>> db["compound_dogs"].delete((5, 3)) +Deleting multiple records +========================= + +You can delete all records in a table that match a specific WHERE statement using ``table.delete_where()``:: + + >>> db = sqlite_utils.Database("dogs.db") + >>> # Delete every dog with age less than 3 + >>> db["dogs"].delete_where("age < ?", [3]): + +Calling ``table.delete_where()`` with no other arguments will delete every row in the table. + Upserting data ============== diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 68a2002..a51bba7 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -864,6 +864,14 @@ class Table(Queryable): with self.db.conn: self.db.conn.execute(sql, pk_values) + def delete_where(self, where=None, where_args=None): + if not self.exists: + return [] + sql = "delete from [{}]".format(self.name) + if where is not None: + sql += " where " + where + self.db.conn.execute(sql, where_args or []) + def update(self, pk_values, updates=None, alter=False): updates = updates or {} if not isinstance(pk_values, (list, tuple)): diff --git a/tests/test_delete.py b/tests/test_delete.py index 6758831..1198d06 100644 --- a/tests/test_delete.py +++ b/tests/test_delete.py @@ -12,3 +12,21 @@ def test_delete_pk_table(fresh_db): table.insert({"id": 2}, pk="id") table.delete(1) assert [{"id": 2}] == list(table.rows) + + +def test_delete_where(fresh_db): + table = fresh_db["table"] + for i in range(1, 11): + table.insert({"id": i}, pk="id") + assert 10 == table.count + table.delete_where("id > ?", [5]) + assert 5 == table.count + + +def test_delete_where_all(fresh_db): + table = fresh_db["table"] + for i in range(1, 11): + table.insert({"id": i}, pk="id") + assert 10 == table.count + table.delete_where() + assert 0 == table.count From a0a65f9a6405079b01aefdbf4b5f507bc758567a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 4 Nov 2019 08:28:52 -0800 Subject: [PATCH 0047/1004] Release 1.12 --- docs/changelog.rst | 10 ++++++++++ docs/python-api.rst | 2 ++ setup.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4d56e96..ee23dd9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,16 @@ Changelog =========== +.. _v1_12: + +1.12 (2019-11-04) +----------------- + +Python library utilities for deleting records (`#62 `__) + +- ``db["tablename"].delete(4)`` to delete by primary key, see :ref:`python_api_delete` +- ``db["tablename"].delete_where("id > ?", [3])`` to delete by a where clause, see :ref:`python_api_delete_where` + .. _v1_11: 1.11 (2019-09-02) diff --git a/docs/python-api.rst b/docs/python-api.rst index cadad15..40d3289 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -396,6 +396,8 @@ The ``delete()`` method takes the primary key of the record. This can be a tuple >>> db["compound_dogs"].delete((5, 3)) +.. _python_api_delete_where: + Deleting multiple records ========================= diff --git a/setup.py b/setup.py index 8e216c6..b71fd09 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "1.11" +VERSION = "1.12" def get_long_description(): From 8dab9fd1ccf571e188eec9ccf606a0c50fccf200 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 6 Nov 2019 20:32:37 -0800 Subject: [PATCH 0048/1004] insert_all() / .upsert_all() work with empty list (#64) Closes #52 --- sqlite_utils/db.py | 5 ++++- tests/test_create.py | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a51bba7..d44b238 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -977,7 +977,10 @@ class Table(Queryable): # we need to adjust the batch_size down if we have too many cols records = iter(records) # Peek at first record to count its columns: - first_record = next(records) + try: + first_record = next(records) + except StopIteration: + return self # It was an empty list num_columns = len(first_record.keys()) assert ( num_columns <= SQLITE_MAX_VARS diff --git a/tests/test_create.py b/tests/test_create.py index 05dfe4a..e2dec84 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -789,3 +789,12 @@ def test_drop_view(fresh_db): assert ["foo_view"] == fresh_db.view_names() assert None is fresh_db["foo_view"].drop() assert [] == fresh_db.view_names() + + +def test_insert_upsert_all_empty_list(fresh_db): + fresh_db["t"].insert({"foo": 1}) + assert 1 == fresh_db["t"].count + fresh_db["t"].insert_all([]) + assert 1 == fresh_db["t"].count + fresh_db["t"].upsert_all([]) + assert 1 == fresh_db["t"].count From 0a0cec3cf27861455e8cd1c4d84937825a18bb30 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 6 Nov 2019 20:58:47 -0800 Subject: [PATCH 0049/1004] Release 1.12.1 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ee23dd9..95252c2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v1_12_1: + +1.12.1 (2019-11-06) +------------------- + +- Fixed error thrown when ``.insert_all()`` and ``.upsert_all()`` were called with empty lists (`#52 `__) + .. _v1_12: 1.12 (2019-11-04) diff --git a/setup.py b/setup.py index b71fd09..191f880 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "1.12" +VERSION = "1.12.1" def get_long_description(): From 9262c3e7c0a49859bac28e268bbcaa6523e02e41 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Dec 2019 09:00:36 +0000 Subject: [PATCH 0050/1004] Corrected Database(memory=True) documentation --- docs/python-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 40d3289..de67c1d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -27,7 +27,7 @@ If you want to create an in-memory database, you can do so like this: .. code-block:: python - db = Database(sqlite3.connect(memory=True)) + db = Database(memory=True) Tables are accessed using the indexing operator, like so: From dc0a62556ec092be7b341c5220e0410354f7cd02 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 27 Dec 2019 09:46:51 +0000 Subject: [PATCH 0051/1004] Run Travis tests on Ubuntu Bionic, fixes #71 --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3af29a6..110f60d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -dist: xenial +dist: bionic # 3.6 is listed first so it gets used for the later build stages python: @@ -8,8 +8,6 @@ python: - "3.8-dev" script: - - sudo add-apt-repository ppa:jonathonf/backports -y - - sudo apt-get update && sudo apt-get install sqlite3 - pip install -U pip wheel - pip install .[test] # Only run the numpy/pandas tests on Python 3.7: From 607a2a9ff63b2bf8b14ed67b66ead5d00c77f2b7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 27 Dec 2019 09:15:31 +0000 Subject: [PATCH 0052/1004] insert --replace and insert(..., replace=True) Refs #66 --- docs/cli.rst | 8 +++---- sqlite_utils/cli.py | 15 ++++++------ sqlite_utils/db.py | 55 +++++++++++++------------------------------- tests/test_cli.py | 18 +++++++-------- tests/test_create.py | 16 ++++++------- 5 files changed, 45 insertions(+), 67 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 10c3345..43592ea 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -265,15 +265,15 @@ For tab-delimited data, use ``--tsv``:: $ sqlite-utils insert dogs.db dogs docs.tsv --tsv -Upserting data -============== +Insert-replacing data +===================== -Upserting works exactly like inserting, with the exception that if your data has a primary key that matches an already exsting record that record will be replaced with the new data. +Insert-replacing works exactly like inserting, with the exception that if your data has a primary key that matches an already exsting record that record will be replaced with the new data. After running the above ``dogs.json`` example, try running this:: $ echo '{"id": 2, "name": "Pancakes", "age": 3}' | \ - sqlite-utils upsert dogs.db dogs - --pk=id + sqlite-utils insert dogs.db dogs - --pk=id --replace This will replace the record for id=2 (Pancakes) with a new record with an updated age. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0b51517..c5c4f86 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -353,6 +353,7 @@ def insert_upsert_implementation( alter, upsert, ignore=False, + replace=False, not_null=None, default=None, ): @@ -372,17 +373,12 @@ def insert_upsert_implementation( docs = json.load(json_file) if isinstance(docs, dict): docs = [docs] - if upsert: - method = db[table].upsert_all - extra_kwargs = {} - else: - method = db[table].insert_all - extra_kwargs = {"ignore": ignore} + extra_kwargs = {"ignore": ignore, "replace": replace} if not_null: extra_kwargs["not_null"] = set(not_null) if default: extra_kwargs["defaults"] = dict(default) - method(docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs) + db[table].insert_all(docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs) @cli.command() @@ -390,6 +386,9 @@ def insert_upsert_implementation( @click.option( "--ignore", is_flag=True, default=False, help="Ignore records if pk already exists" ) +@click.option( + "--replace", is_flag=True, default=False, help="Replace records if pk already exists" +) def insert( path, table, @@ -401,6 +400,7 @@ def insert( batch_size, alter, ignore, + replace, not_null, default, ): @@ -422,6 +422,7 @@ def insert( alter=alter, upsert=False, ignore=ignore, + replace=replace, not_null=not_null, default=default, ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d44b238..874aa8b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -473,11 +473,11 @@ class Table(Queryable): column_order=None, not_null=None, defaults=None, - upsert=False, batch_size=100, hash_id=None, alter=False, ignore=False, + replace=False, extracts=None, ): super().__init__(db, name) @@ -488,11 +488,11 @@ class Table(Queryable): column_order=column_order, not_null=not_null, defaults=defaults, - upsert=upsert, batch_size=batch_size, hash_id=hash_id, alter=alter, ignore=ignore, + replace=replace, extracts=extracts, ) @@ -915,10 +915,10 @@ class Table(Queryable): column_order=DEFAULT, not_null=DEFAULT, defaults=DEFAULT, - upsert=DEFAULT, hash_id=DEFAULT, alter=DEFAULT, ignore=DEFAULT, + replace=DEFAULT, extracts=DEFAULT, ): return self.insert_all( @@ -928,10 +928,10 @@ class Table(Queryable): column_order=column_order, not_null=not_null, defaults=defaults, - upsert=upsert, hash_id=hash_id, alter=alter, ignore=ignore, + replace=replace, extracts=extracts, ) @@ -943,11 +943,11 @@ class Table(Queryable): column_order=DEFAULT, not_null=DEFAULT, defaults=DEFAULT, - upsert=DEFAULT, batch_size=DEFAULT, hash_id=DEFAULT, alter=DEFAULT, ignore=DEFAULT, + replace=DEFAULT, extracts=DEFAULT, ): """ @@ -960,17 +960,17 @@ class Table(Queryable): column_order = self.value_or_default("column_order", column_order) not_null = self.value_or_default("not_null", not_null) defaults = self.value_or_default("defaults", defaults) - upsert = self.value_or_default("upsert", upsert) batch_size = self.value_or_default("batch_size", batch_size) hash_id = self.value_or_default("hash_id", hash_id) alter = self.value_or_default("alter", alter) ignore = self.value_or_default("ignore", ignore) + replace = self.value_or_default("replace", replace) extracts = self.value_or_default("extracts", extracts) assert not (hash_id and pk), "Use either pk= or hash_id=" assert not ( - ignore and upsert - ), "Use either ignore=True or upsert=True, not both" + ignore and replace + ), "Use either ignore=True or replace=True, not both" all_columns = None first = True # We can only handle a max of 999 variables in a SQL insert, so @@ -1009,7 +1009,7 @@ class Table(Queryable): all_columns.insert(0, hash_id) first = False or_what = "" - if upsert: + if replace: or_what = "OR REPLACE " elif ignore: or_what = "OR IGNORE " @@ -1076,18 +1076,7 @@ class Table(Queryable): alter=DEFAULT, extracts=DEFAULT, ): - return self.insert( - record, - pk=pk, - foreign_keys=foreign_keys, - column_order=column_order, - not_null=not_null, - defaults=defaults, - hash_id=hash_id, - alter=alter, - upsert=True, - extracts=extracts, - ) + raise NotImplementedError def upsert_all( self, @@ -1102,19 +1091,7 @@ class Table(Queryable): alter=DEFAULT, extracts=DEFAULT, ): - return self.insert_all( - records, - pk=pk, - foreign_keys=foreign_keys, - column_order=column_order, - not_null=not_null, - defaults=defaults, - batch_size=100, - hash_id=hash_id, - alter=alter, - upsert=True, - extracts=extracts, - ) + raise NotImplementedError def add_missing_columns(self, records): needed_columns = self.detect_column_types(records) @@ -1183,20 +1160,20 @@ class Table(Queryable): ) # Ensure each record exists in other table for record in records: - id = other_table.upsert(record, pk=pk).last_pk - m2m_table.upsert( + id = other_table.insert(record, pk=pk, replace=True).last_pk + m2m_table.insert( { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, - } + }, replace=True ) else: id = other_table.lookup(lookup) - m2m_table.upsert( + m2m_table.insert( { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, - } + }, replace=True ) return self diff --git a/tests/test_cli.py b/tests/test_cli.py index d9ebe55..0c3535d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -581,23 +581,23 @@ def test_only_allow_one_of_nl_tsv_csv(options, db_path, tmpdir): assert "Error: Use just one of --nl, --csv or --tsv" == result.output.strip() -def test_upsert(db_path, tmpdir): +def test_insert_replace(db_path, tmpdir): test_insert_multiple_with_primary_key(db_path, tmpdir) - json_path = str(tmpdir / "upsert.json") + json_path = str(tmpdir / "insert-replace.json") db = Database(db_path) assert 20 == db["dogs"].count - upsert_dogs = [ - {"id": 1, "name": "Upserted 1", "age": 4}, - {"id": 2, "name": "Upserted 2", "age": 4}, + insert_replace_dogs = [ + {"id": 1, "name": "Insert replaced 1", "age": 4}, + {"id": 2, "name": "Insert replaced 2", "age": 4}, {"id": 21, "name": "Fresh insert 21", "age": 6}, ] - open(json_path, "w").write(json.dumps(upsert_dogs)) + open(json_path, "w").write(json.dumps(insert_replace_dogs)) result = CliRunner().invoke( - cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"] + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--replace"] ) - assert 0 == result.exit_code + assert 0 == result.exit_code, result.output assert 21 == db["dogs"].count - assert upsert_dogs == db.execute_returning_dicts( + assert insert_replace_dogs == db.execute_returning_dicts( "select * from dogs where id in (1, 2, 21) order by id" ) diff --git a/tests/test_create.py b/tests/test_create.py index e2dec84..a7aab07 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -445,7 +445,7 @@ def test_insert_row_alter_table( @pytest.mark.parametrize("use_table_factory", [True, False]) -def test_upsert_rows_alter_table(fresh_db, use_table_factory): +def test_insert_replace_rows_alter_table(fresh_db, use_table_factory): first_row = {"id": 1, "title": "Hedgehogs of the world", "author_id": 1} next_rows = [ {"id": 1, "title": "Hedgehogs of the World", "species": "hedgehogs"}, @@ -459,11 +459,11 @@ def test_upsert_rows_alter_table(fresh_db, use_table_factory): if use_table_factory: table = fresh_db.table("books", pk="id", alter=True) table.insert(first_row) - table.upsert_all(next_rows) + table.insert_all(next_rows, replace=True) else: table = fresh_db["books"] table.insert(first_row, pk="id") - table.upsert_all(next_rows, alter=True) + table.insert_all(next_rows, alter=True, replace=True) assert { "author_id": int, "id": int, @@ -664,11 +664,11 @@ def test_insert_ignore(fresh_db): def test_insert_hash_id(fresh_db): dogs = fresh_db["dogs"] - id = dogs.upsert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk + id = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id assert 1 == dogs.count - # Upserting a second time should not create a new row - id2 = dogs.upsert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk + # Insert replacing a second time should not create a new row + id2 = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id", replace=True).last_pk assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id2 assert 1 == dogs.count @@ -791,10 +791,10 @@ def test_drop_view(fresh_db): assert [] == fresh_db.view_names() -def test_insert_upsert_all_empty_list(fresh_db): +def test_insert_all_empty_list(fresh_db): fresh_db["t"].insert({"foo": 1}) assert 1 == fresh_db["t"].count fresh_db["t"].insert_all([]) assert 1 == fresh_db["t"].count - fresh_db["t"].upsert_all([]) + fresh_db["t"].insert_all([], replace=True) assert 1 == fresh_db["t"].count From cfbc09967e1bf69df9355a4a57e3f63882019b41 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 27 Dec 2019 09:30:29 +0000 Subject: [PATCH 0053/1004] Ran black, plus added comments for next step Refs #66 --- sqlite_utils/cli.py | 9 +++++++-- sqlite_utils/db.py | 40 +++++++++++++++++++++++++++++++++++++--- tests/test_create.py | 4 +++- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c5c4f86..798931d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -378,7 +378,9 @@ def insert_upsert_implementation( extra_kwargs["not_null"] = set(not_null) if default: extra_kwargs["defaults"] = dict(default) - db[table].insert_all(docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs) + db[table].insert_all( + docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs + ) @cli.command() @@ -387,7 +389,10 @@ def insert_upsert_implementation( "--ignore", is_flag=True, default=False, help="Ignore records if pk already exists" ) @click.option( - "--replace", is_flag=True, default=False, help="Replace records if pk already exists" + "--replace", + is_flag=True, + default=False, + help="Replace records if pk already exists", ) def insert( path, diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 874aa8b..8aa05e3 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -985,6 +985,8 @@ class Table(Queryable): assert ( num_columns <= SQLITE_MAX_VARS ), "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) + # When calculating this for upsert, the primary keys are referenced twice + # so num_columns needs to have num primary keys added to it: batch_size = max(1, min(batch_size, SQLITE_MAX_VARS // num_columns)) for chunk in chunks(itertools.chain([first_record], records), batch_size): chunk = list(chunk) @@ -1008,11 +1010,21 @@ class Table(Queryable): if hash_id: all_columns.insert(0, hash_id) first = False + + # START of bit that differs for upsert v.s. insert or_what = "" if replace: or_what = "OR REPLACE " elif ignore: or_what = "OR IGNORE " + # INSERT OR IGNORE INTO book(id) VALUES(1001); + # UPDATE book SET name = 'Programming' WHERE id = 1001; + # WAIT NO: this won't work because here we create a single + # giant INSERT, but for upsert we need two statements per + # record which means we need executescript()... but + # executescript doesn't take parameters at all. + # So maybe for upsert() we execute two separate SQL queries + # for every single record? Very different implementation. sql = """ INSERT {or_what}INTO [{table}] ({columns}) VALUES {rows}; """.format( @@ -1028,6 +1040,8 @@ class Table(Queryable): for record in chunk ), ) + # END of bit that differs + values = [] extracts = resolve_extracts(extracts) for record in chunk: @@ -1041,6 +1055,9 @@ class Table(Queryable): value = self.db[extract_table].lookup({"value": value}) record_values.append(value) values.extend(record_values) + + # Except... if upsert() needs to execute multiple queries + # then this bit needs to change too with self.db.conn: try: result = self.db.conn.execute(sql, values) @@ -1051,9 +1068,11 @@ class Table(Queryable): result = self.db.conn.execute(sql, values) else: raise + # How do we get lastrowid for upserts? self.last_rowid = result.lastrowid self.last_pk = self.last_rowid # self.last_rowid will be 0 if a "INSERT OR IGNORE" happened + # Do we need this logic for upsert too? if (hash_id or pk) and self.last_rowid: row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] if hash_id: @@ -1076,7 +1095,17 @@ class Table(Queryable): alter=DEFAULT, extracts=DEFAULT, ): - raise NotImplementedError + return self.upsert_all( + [record], + pk=pk, + foreign_keys=foreign_keys, + column_order=column_order, + not_null=not_null, + defaults=defaults, + hash_id=hash_id, + alter=alter, + extracts=extracts, + ) def upsert_all( self, @@ -1091,6 +1120,9 @@ class Table(Queryable): alter=DEFAULT, extracts=DEFAULT, ): + # Perform the following for each record: + # INSERT OR IGNORE INTO books(id) VALUES(1001); + # UPDATE books SET name = 'Programming' WHERE id = 1001; raise NotImplementedError def add_missing_columns(self, records): @@ -1165,7 +1197,8 @@ class Table(Queryable): { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, - }, replace=True + }, + replace=True, ) else: id = other_table.lookup(lookup) @@ -1173,7 +1206,8 @@ class Table(Queryable): { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, - }, replace=True + }, + replace=True, ) return self diff --git a/tests/test_create.py b/tests/test_create.py index a7aab07..bfa352f 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -668,7 +668,9 @@ def test_insert_hash_id(fresh_db): assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id assert 1 == dogs.count # Insert replacing a second time should not create a new row - id2 = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id", replace=True).last_pk + id2 = dogs.insert( + {"name": "Cleo", "twitter": "cleopaws"}, hash_id="id", replace=True + ).last_pk assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id2 assert 1 == dogs.count From 84bcabd09381a98502797a7d1adee357a1d45a67 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Dec 2019 21:03:43 -0800 Subject: [PATCH 0054/1004] New upsert implementation, refs #66 --- sqlite_utils/cli.py | 2 + sqlite_utils/db.py | 130 +++++++++++++++++++++++++++---------------- tests/test_cli.py | 56 +++++++++++++++++++ tests/test_upsert.py | 16 ++++++ 4 files changed, 156 insertions(+), 48 deletions(-) create mode 100644 tests/test_upsert.py diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 798931d..14e7365 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -378,6 +378,8 @@ def insert_upsert_implementation( extra_kwargs["not_null"] = set(not_null) if default: extra_kwargs["defaults"] = dict(default) + if upsert: + extra_kwargs["upsert"] = upsert db[table].insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 8aa05e3..7516028 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -949,6 +949,7 @@ class Table(Queryable): ignore=DEFAULT, replace=DEFAULT, extracts=DEFAULT, + upsert=False, ): """ Like .insert() but takes a list of records and ensures that the table @@ -985,8 +986,6 @@ class Table(Queryable): assert ( num_columns <= SQLITE_MAX_VARS ), "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) - # When calculating this for upsert, the primary keys are referenced twice - # so num_columns needs to have num primary keys added to it: batch_size = max(1, min(batch_size, SQLITE_MAX_VARS // num_columns)) for chunk in chunks(itertools.chain([first_record], records), batch_size): chunk = list(chunk) @@ -1011,37 +1010,9 @@ class Table(Queryable): all_columns.insert(0, hash_id) first = False - # START of bit that differs for upsert v.s. insert - or_what = "" - if replace: - or_what = "OR REPLACE " - elif ignore: - or_what = "OR IGNORE " - # INSERT OR IGNORE INTO book(id) VALUES(1001); - # UPDATE book SET name = 'Programming' WHERE id = 1001; - # WAIT NO: this won't work because here we create a single - # giant INSERT, but for upsert we need two statements per - # record which means we need executescript()... but - # executescript doesn't take parameters at all. - # So maybe for upsert() we execute two separate SQL queries - # for every single record? Very different implementation. - sql = """ - INSERT {or_what}INTO [{table}] ({columns}) VALUES {rows}; - """.format( - or_what=or_what, - table=self.name, - columns=", ".join("[{}]".format(c) for c in all_columns), - rows=", ".join( - """ - ({placeholders}) - """.format( - placeholders=", ".join(["?"] * len(all_columns)) - ) - for record in chunk - ), - ) - # END of bit that differs - + # values is the list of insert data that is passed to the + # .execute() method - but some of them may be replaced by + # new primary keys if we are extracting any columns. values = [] extracts = resolve_extracts(extracts) for record in chunk: @@ -1054,25 +1025,76 @@ class Table(Queryable): extract_table = extracts[key] value = self.db[extract_table].lookup({"value": value}) record_values.append(value) - values.extend(record_values) + values.append(record_values) + + queries_and_params = [] + if upsert: + if isinstance(pk, str): + pks = [pk] + else: + pks = pk + for record_values in values: + # TODO: make more efficient: + record = dict(zip(all_columns, record_values)) + params = [] + sql = "INSERT OR IGNORE INTO [{table}]({pks}) VALUES({pk_placeholders});".format( + table=self.name, + pks=", ".join(["[{}]".format(p) for p in pks]), + pk_placeholders=", ".join(["?" for p in pks]), + ) + queries_and_params.append((sql, [record[col] for col in pks])) + # UPDATE [book] SET [name] = 'Programming' WHERE [id] = 1001; + set_cols = [col for col in all_columns if col not in pks] + sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format( + table=self.name, + pairs=", ".join("[{}] = ?".format(col) for col in set_cols), + wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks), + ) + queries_and_params.append( + ( + sql2, + [record[col] for col in set_cols] + + [record[pk] for pk in pks], + ) + ) + else: + or_what = "" + if replace: + or_what = "OR REPLACE " + elif ignore: + or_what = "OR IGNORE " + sql = """ + INSERT {or_what}INTO [{table}] ({columns}) VALUES {rows}; + """.format( + or_what=or_what, + table=self.name, + columns=", ".join("[{}]".format(c) for c in all_columns), + rows=", ".join( + """ + ({placeholders}) + """.format( + placeholders=", ".join(["?"] * len(all_columns)) + ) + for record in chunk + ), + ) + flat_values = list(itertools.chain(*values)) + queries_and_params = [(sql, flat_values)] - # Except... if upsert() needs to execute multiple queries - # then this bit needs to change too with self.db.conn: - try: - result = self.db.conn.execute(sql, values) - except OperationalError as e: - if alter and (" column" in e.args[0]): - # Attempt to add any missing columns, then try again - self.add_missing_columns(chunk) - result = self.db.conn.execute(sql, values) - else: - raise - # How do we get lastrowid for upserts? + for query, params in queries_and_params: + try: + result = self.db.conn.execute(query, params) + except OperationalError as e: + if alter and (" column" in e.args[0]): + # Attempt to add any missing columns, then try again + self.add_missing_columns(chunk) + result = self.db.conn.execute(query, params) + else: + raise self.last_rowid = result.lastrowid self.last_pk = self.last_rowid # self.last_rowid will be 0 if a "INSERT OR IGNORE" happened - # Do we need this logic for upsert too? if (hash_id or pk) and self.last_rowid: row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] if hash_id: @@ -1123,7 +1145,19 @@ class Table(Queryable): # Perform the following for each record: # INSERT OR IGNORE INTO books(id) VALUES(1001); # UPDATE books SET name = 'Programming' WHERE id = 1001; - raise NotImplementedError + return self.insert_all( + records, + pk=pk, + foreign_keys=foreign_keys, + column_order=column_order, + not_null=not_null, + defaults=defaults, + batch_size=batch_size, + hash_id=hash_id, + alter=alter, + extracts=extracts, + upsert=True, + ) def add_missing_columns(self, records): needed_columns = self.detect_column_types(records) diff --git a/tests/test_cli.py b/tests/test_cli.py index 0c3535d..1f24090 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -765,3 +765,59 @@ def test_rows(db_path, args, expected): ) result = CliRunner().invoke(cli.cli, ["rows", db_path, "dogs"] + args) assert expected == result.output.strip() + + +def test_upsert(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + db = Database(db_path) + insert_dogs = [ + {"id": 1, "name": "Cleo", "age": 4}, + {"id": 2, "name": "Nixie", "age": 4}, + ] + open(json_path, "w").write(json.dumps(insert_dogs)) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert 0 == result.exit_code, result.output + assert 2 == db["dogs"].count + # Now run the upsert to update just their ages + upsert_dogs = [ + {"id": 1, "age": 5}, + {"id": 2, "age": 5}, + ] + open(json_path, "w").write(json.dumps(insert_dogs)) + result = CliRunner().invoke( + cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert 0 == result.exit_code, result.output + assert [ + {"id": 1, "name": "Cleo", "age": 4}, + {"id": 2, "name": "Nixie", "age": 4}, + ] == db.execute_returning_dicts("select * from dogs order by id") + + +def test_upsert_alter(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + db = Database(db_path) + insert_dogs = [{"id": 1, "name": "Cleo"}] + open(json_path, "w").write(json.dumps(insert_dogs)) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert 0 == result.exit_code, result.output + # Should fail with error code if no --alter + upsert_dogs = [{"id": 1, "age": 5}] + open(json_path, "w").write(json.dumps(upsert_dogs)) + result = CliRunner().invoke( + cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert 1 == result.exit_code + assert "no such column: age" == str(result.exception) + # Should succeed with --alter + result = CliRunner().invoke( + cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id", "--alter"] + ) + assert 0 == result.exit_code + assert [{"id": 1, "name": "Cleo", "age": 5},] == db.execute_returning_dicts( + "select * from dogs order by id" + ) diff --git a/tests/test_upsert.py b/tests/test_upsert.py new file mode 100644 index 0000000..96f9d0d --- /dev/null +++ b/tests/test_upsert.py @@ -0,0 +1,16 @@ +def test_upsert(fresh_db): + table = fresh_db["table"] + table.insert({"id": 1, "name": "Cleo"}, pk="id") + table.upsert({"id": 1, "age": 5}, pk="id", alter=True) + assert [{"id": 1, "name": "Cleo", "age": 5}] == list(table.rows) + + +def test_upsert_all(fresh_db): + table = fresh_db["table"] + table.upsert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Nixie"}], pk="id") + table.upsert_all([{"id": 1, "age": 5}, {"id": 2, "age": 5}], pk="id", alter=True) + assert [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Nixie", "age": 5}, + ] == list(table.rows) + assert 2 == table.last_pk From 9f47e8b9a4cb788b48b76aee1333c6f3baaebbd6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Dec 2019 21:23:58 -0800 Subject: [PATCH 0055/1004] Documentation for new upsert v.s insert-replace Refs #66 --- docs/cli.rst | 23 +++++++++++++++++++++++ docs/python-api.rst | 31 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 43592ea..557a860 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -265,6 +265,8 @@ For tab-delimited data, use ``--tsv``:: $ sqlite-utils insert dogs.db dogs docs.tsv --tsv +.. _cli_insert_replace: + Insert-replacing data ===================== @@ -277,6 +279,27 @@ After running the above ``dogs.json`` example, try running this:: This will replace the record for id=2 (Pancakes) with a new record with an updated age. +.. _cli_upsert: + +Upserting data +============== + +Upserting is update-or-insert. If a row exists with the specified primary key the provided columns will be updated. If no row exists that row will be created. + +Unlike ``insert --replace``, an upsert will ignore any column values that exist but are not present in the upsert document. + +For example:: + + $ echo '{"id": 2, "age": 4}' | \ + sqlite-utils upsert dogs.db dogs - --pk=id + +This will update the dog with id=2 to have an age of 4, creating a new record (with a null name) if one does not exist. If a row DOES exist the name will be left as-is. + +The command will fail if you reference columns that do not exist on the table. To automatically create missing columns, use the ``--alter`` option. + +.. note:: + ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 `__ for details of this change. + .. _cli_add_column: Adding columns diff --git a/docs/python-api.rst b/docs/python-api.rst index de67c1d..db988fe 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -358,6 +358,30 @@ The function can accept an iterator or generator of rows and will commit them ac You can skip inserting any records that have a primary key that already exists using ``ignore=True``. This works with both ``.insert({...}, ignore=True)`` and ``.insert_all([...], ignore=True)``. +.. _python_api_insert_replace: + +Insert-replacing data +===================== + +If you want to insert a record or replace an existing record with the same primary key, using the ``replace=True`` argument to ``.insert()`` or ``.insert_all()``:: + + dogs.insert_all([{ + "id": 1, + "name": "Cleo", + "twitter": "cleopaws", + "age": 3, + "is_good_dog": True, + }, { + "id": 2, + "name": "Marnie", + "twitter": "MarnieTheDog", + "age": 16, + "is_good_dog": True, + }], pk="id", replace=True) + +.. note:: + Prior to sqlite-utils 2.x the ``.upsert()`` and ``.upsert_all()`` methods did this. See :ref:`python_api_upsert` for the new behaviour of those methods in 2.x. + .. _python_api_update: Updating a specific record @@ -409,6 +433,8 @@ You can delete all records in a table that match a specific WHERE statement usin Calling ``table.delete_where()`` with no other arguments will delete every row in the table. +.. _python_api_upsert: + Upserting data ============== @@ -428,10 +454,15 @@ For example, given the dogs database you could upsert the record for Cleo like s If a record exists with id=1, it will be updated to match those fields. If it does not exist it will be created. +Any existing columns that are not referenced in the dictionary passed to ``.upsert()`` will be unchanged. If you want to replace a record entirely, use ``.insert(doc, replace=True)`` instead. + Note that the ``pk`` and ``column_order`` parameters here are optional if you are certain that the table has already been created. You should pass them if the table may not exist at the time the first upsert is performed. An ``upsert_all()`` method is also available, which behaves like ``insert_all()`` but performs upserts instead. +.. note:: + ``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 `__ for details of this change. + .. _python_api_lookup_tables: Working with lookup tables From 468d51314adac193e63b3a6ef9d67f0d43501e9b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Dec 2019 21:31:03 -0800 Subject: [PATCH 0056/1004] test_upsert_compound_primary_key --- tests/test_upsert.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 96f9d0d..9b93f88 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -14,3 +14,13 @@ def test_upsert_all(fresh_db): {"id": 2, "name": "Nixie", "age": 5}, ] == list(table.rows) assert 2 == table.last_pk + + +def test_upsert_compound_primary_key(fresh_db): + table = fresh_db["table"] + table.upsert_all([{"species": "dog", "id": 1, "name": "Cleo", "age": 4}, {"species": "cat", "id": 1, "name": "Catbag"}], pk=("species", "id")) + table.upsert_all([{"species": "dog", "id": 1, "age": 5}], pk=("species", "id")) + assert [ + {"species": "dog", "id": 1, "name": "Cleo", "age": 5}, + {"species": "cat", "id": 1, "name": "Catbag", "age": None}, + ] == list(table.rows) \ No newline at end of file From a0f0175d64d3c52529703755b89daf9c24d12d8b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Dec 2019 22:05:31 -0800 Subject: [PATCH 0057/1004] Updated help for upsert, refs #66 --- sqlite_utils/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 14e7365..ba55e61 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -443,7 +443,7 @@ def upsert( """ Upsert records based on their primary key. Works like 'insert' but if an incoming record has a primary key that matches an existing record - the existing record will be replaced. + the existing record will be updated. """ insert_upsert_implementation( path, From f0f15d3dc8dc686642f2c40894c011a2e6bac240 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Dec 2019 22:09:52 -0800 Subject: [PATCH 0058/1004] Reformatted with black --- tests/test_upsert.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 9b93f88..edf7f29 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -18,9 +18,15 @@ def test_upsert_all(fresh_db): def test_upsert_compound_primary_key(fresh_db): table = fresh_db["table"] - table.upsert_all([{"species": "dog", "id": 1, "name": "Cleo", "age": 4}, {"species": "cat", "id": 1, "name": "Catbag"}], pk=("species", "id")) + table.upsert_all( + [ + {"species": "dog", "id": 1, "name": "Cleo", "age": 4}, + {"species": "cat", "id": 1, "name": "Catbag"}, + ], + pk=("species", "id"), + ) table.upsert_all([{"species": "dog", "id": 1, "age": 5}], pk=("species", "id")) assert [ {"species": "dog", "id": 1, "name": "Cleo", "age": 5}, {"species": "cat", "id": 1, "name": "Catbag", "age": None}, - ] == list(table.rows) \ No newline at end of file + ] == list(table.rows) From 0b0a431bff94d24866fc10d82dc91ab00287de2d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Dec 2019 22:18:44 -0800 Subject: [PATCH 0059/1004] Changelog for 2.0 release --- docs/changelog.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 95252c2..3117dda 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,21 @@ Changelog =========== +.. _v2: + +2.0 (2019-12-29) +---------------- + +This release changes the behaviour of ``upsert``. It's a breaking change, hence ``2.0``. + +The ``upsert`` command-line utility and the ``.upsert()`` and ``.upsert_all()`` Python API methods have had their behaviour altered. They used to completely replace the affected records: now, they update the specified values on existing records but leave other columns unaffected. + +See :ref:`Upserting data using the Python API ` and :ref:`Upserting data using the CLI ` for full details. + +If you want the old behaviour - where records were completely replaced - you can use ``$ sqlite-utils insert ... --replace`` on the command-line and ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` in the Python API. See :ref:`Insert-replacing data using the Python API ` and :ref:`Insert-replacing data using the CLI ` for more. + +For full background on this change, see `issue #66 `__). + .. _v1_12_1: 1.12.1 (2019-11-06) From c6c2e7184bbfeaa84fd78ec0cd9d878715f64f98 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Dec 2019 22:18:58 -0800 Subject: [PATCH 0060/1004] Release 2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 191f880..270ce31 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "1.12.1" +VERSION = "2.0" def get_long_description(): From 6b79cb706a6d6252d1b66e4565283b73b3090851 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Dec 2019 22:51:07 -0800 Subject: [PATCH 0061/1004] Removed rogue parenthesis --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3117dda..c853357 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,7 +15,7 @@ See :ref:`Upserting data using the Python API ` and :ref:`Ups If you want the old behaviour - where records were completely replaced - you can use ``$ sqlite-utils insert ... --replace`` on the command-line and ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` in the Python API. See :ref:`Insert-replacing data using the Python API ` and :ref:`Insert-replacing data using the CLI ` for more. -For full background on this change, see `issue #66 `__). +For full background on this change, see `issue #66 `__. .. _v1_12_1: From 1f3f902ea4c991e6b8ad0fcfd6cffd01e3aa1c23 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Dec 2019 05:01:36 -0800 Subject: [PATCH 0062/1004] Typo --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 557a860..8590456 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -270,7 +270,7 @@ For tab-delimited data, use ``--tsv``:: Insert-replacing data ===================== -Insert-replacing works exactly like inserting, with the exception that if your data has a primary key that matches an already exsting record that record will be replaced with the new data. +Insert-replacing works exactly like inserting, with the exception that if your data has a primary key that matches an already existing record that record will be replaced with the new data. After running the above ``dogs.json`` example, try running this:: From 489eda92bc3b528c90b76ac90a3f9b78c8ea02a3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jan 2020 09:20:11 -0800 Subject: [PATCH 0063/1004] .upsert() and upsert_all() require pk=, closes #73 --- sqlite_utils/db.py | 9 ++++++--- tests/test_upsert.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 7516028..0a81a60 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -90,6 +90,10 @@ class NotFoundError(Exception): pass +class PrimaryKeyRequired(Exception): + pass + + class Database: def __init__(self, filename_or_conn=None, memory=False): assert (filename_or_conn is not None and not memory) or ( @@ -957,6 +961,8 @@ class Table(Queryable): data """ pk = self.value_or_default("pk", pk) + if upsert and not pk: + raise PrimaryKeyRequired("upsert() requires a pk") foreign_keys = self.value_or_default("foreign_keys", foreign_keys) column_order = self.value_or_default("column_order", column_order) not_null = self.value_or_default("not_null", not_null) @@ -1142,9 +1148,6 @@ class Table(Queryable): alter=DEFAULT, extracts=DEFAULT, ): - # Perform the following for each record: - # INSERT OR IGNORE INTO books(id) VALUES(1001); - # UPDATE books SET name = 'Programming' WHERE id = 1001; return self.insert_all( records, pk=pk, diff --git a/tests/test_upsert.py b/tests/test_upsert.py index edf7f29..4d53fa8 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -1,3 +1,7 @@ +from sqlite_utils.db import PrimaryKeyRequired +import pytest + + def test_upsert(fresh_db): table = fresh_db["table"] table.insert({"id": 1, "name": "Cleo"}, pk="id") @@ -16,6 +20,14 @@ def test_upsert_all(fresh_db): assert 2 == table.last_pk +def test_upsert_error_if_no_pk(fresh_db): + table = fresh_db["table"] + with pytest.raises(PrimaryKeyRequired): + table.upsert_all([{"id": 1, "name": "Cleo"}]) + with pytest.raises(PrimaryKeyRequired): + table.upsert({"id": 1, "name": "Cleo"}) + + def test_upsert_compound_primary_key(fresh_db): table = fresh_db["table"] table.upsert_all( From 59a2e8ebdcbde7e6fb091b0556713ca5a20ea4e7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jan 2020 09:23:02 -0800 Subject: [PATCH 0064/1004] Release 2.0.1 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c853357..72b19fc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_0_1: + +2.0.1 (2020-01-05) +------------------ + +The ``.upsert()`` and ``.upsert_all()`` methods now raise a ``sqlite_utils.db.PrimaryKeyRequired`` exception if you call them without specifying the primary key column using ``pk=`` (`#73 `__). + .. _v2: 2.0 (2019-12-29) diff --git a/setup.py b/setup.py index 270ce31..32619ae 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.0" +VERSION = "2.0.1" def get_long_description(): From 0988f2eccc2dfa26b1a55243582222f540a72838 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Fri, 31 Jan 2020 07:21:26 +0700 Subject: [PATCH 0065/1004] Explicitly include tests and docs in sdist (#75) Also exclude 'tests' from runtime installation - thanks, @jayvdb --- MANIFEST.in | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..e3c9212 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include LICENSE +include README.md +recursive-include docs *.rst +recursive-include tests *.py diff --git a/setup.py b/setup.py index 32619ae..a37c6b0 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( author="Simon Willison", version=VERSION, license="Apache License, Version 2.0", - packages=find_packages(exclude="tests"), + packages=find_packages(exclude=["tests", "tests.*"]), install_requires=["click", "click-default-group", "tabulate"], setup_requires=["pytest-runner"], extras_require={ From e8b2b7383bd94659d3b7a857a1414328bc48bc19 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jan 2020 16:24:30 -0800 Subject: [PATCH 0066/1004] New conversions= feature, closes #77 Pull request: #78 --- docs/python-api.rst | 62 +++++++++++++++++++++++++++++++++++++++ sqlite_utils/db.py | 24 ++++++++++++--- tests/test_conversions.py | 44 +++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 tests/test_conversions.py diff --git a/docs/python-api.rst b/docs/python-api.rst index db988fe..a1f0461 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -846,6 +846,68 @@ For example: """).fetchall() # Returns [('Felton, CA',)] +.. _python_api_conversions: + +Converting column values using SQL functions +============================================ + +Sometimes it can be useful to run values through a SQL function prior to inserting them. A simple example might be converting a value to upper case while it is being inserted. + +The ``conversions={...}`` parameter can be used to specify custom SQL to be used as part of a ``INSERT`` or ``UPDATE`` SQL statement. + +You can specify an upper case conversion for a specific column like so: + +.. code-block:: python + + db["example"].insert({ + "name": "The Bigfoot Discovery Museum" + }, conversions={"name": "upper(?)"}) + + # list(db["example"].rows) now returns: + # [{'name': 'THE BIGFOOT DISCOVERY MUSEUM'}] + +The dictionary key is the column name to be converted. The value is the SQL fragment to use, with a ``?`` placeholder for the original value. + +A more useful example: if you are working with `SpatiaLite `__ you may find yourself wanting to create geometry values from a WKT value. Code to do that could look like this: + +.. code-block:: python + + import sqlite3 + import sqlite_utils + from shapely.geometry import shape + import requests + + # Open a database and load the SpatiaLite extension: + import sqlite3 + + conn = sqlite3.connect("places.db") + conn.enable_load_extension(True) + conn.load_extension("/usr/local/lib/mod_spatialite.dylib") + + # Use sqlite-utils to create a places table: + db = sqlite_utils.Database(conn) + places = db["places"].create({"id": int, "name": str,}) + + # Add a SpatiaLite 'geometry' column: + db.conn.execute("select InitSpatialMetadata(1)") + db.conn.execute( + "SELECT AddGeometryColumn('places', 'geometry', 4326, 'MULTIPOLYGON', 2);" + ) + + # Fetch some GeoJSON from Who's On First: + geojson = requests.get( + "https://data.whosonfirst.org/404/227/475/404227475.geojson" + ).json() + + # Convert to "Well Known Text" format using shapely + wkt = shape(geojson["geometry"]).wkt + + # Insert the record, converting the WKT to a SpatiaLite geometry: + db["places"].insert( + {"name": "Wales", "geometry": wkt}, + conversions={"geometry": "GeomFromText(?, 4326)"}, + ) + .. _python_api_introspection: Introspection diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0a81a60..b061e98 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -483,6 +483,7 @@ class Table(Queryable): ignore=False, replace=False, extracts=None, + conversions=None, ): super().__init__(db, name) self.exists = self.name in self.db.table_names() @@ -498,6 +499,7 @@ class Table(Queryable): ignore=ignore, replace=replace, extracts=extracts, + conversions=conversions or {}, ) def __repr__(self): @@ -876,8 +878,9 @@ class Table(Queryable): sql += " where " + where self.db.conn.execute(sql, where_args or []) - def update(self, pk_values, updates=None, alter=False): + def update(self, pk_values, updates=None, alter=False, conversions=None): updates = updates or {} + conversions = conversions or {} if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] # Sanity check that the record exists (raises error if not): @@ -888,7 +891,7 @@ class Table(Queryable): sets = [] wheres = [] for key, value in updates.items(): - sets.append("[{}] = ?".format(key)) + sets.append("[{}] = {}".format(key, conversions.get(key, "?"))) args.append(value) wheres = ["[{}] = ?".format(pk_name) for pk_name in self.pks] args.extend(pk_values) @@ -924,6 +927,7 @@ class Table(Queryable): ignore=DEFAULT, replace=DEFAULT, extracts=DEFAULT, + conversions=DEFAULT, ): return self.insert_all( [record], @@ -937,6 +941,7 @@ class Table(Queryable): ignore=ignore, replace=replace, extracts=extracts, + conversions=conversions, ) def insert_all( @@ -953,6 +958,7 @@ class Table(Queryable): ignore=DEFAULT, replace=DEFAULT, extracts=DEFAULT, + conversions=DEFAULT, upsert=False, ): """ @@ -973,6 +979,7 @@ class Table(Queryable): ignore = self.value_or_default("ignore", ignore) replace = self.value_or_default("replace", replace) extracts = self.value_or_default("extracts", extracts) + conversions = self.value_or_default("conversions", conversions) assert not (hash_id and pk), "Use either pk= or hash_id=" assert not ( @@ -1053,7 +1060,10 @@ class Table(Queryable): set_cols = [col for col in all_columns if col not in pks] sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format( table=self.name, - pairs=", ".join("[{}] = ?".format(col) for col in set_cols), + pairs=", ".join( + "[{}] = {}".format(col, conversions.get(col, "?")) + for col in set_cols + ), wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks), ) queries_and_params.append( @@ -1079,7 +1089,9 @@ class Table(Queryable): """ ({placeholders}) """.format( - placeholders=", ".join(["?"] * len(all_columns)) + placeholders=", ".join( + [conversions.get(col, "?") for col in all_columns] + ) ) for record in chunk ), @@ -1122,6 +1134,7 @@ class Table(Queryable): hash_id=DEFAULT, alter=DEFAULT, extracts=DEFAULT, + conversions=DEFAULT, ): return self.upsert_all( [record], @@ -1133,6 +1146,7 @@ class Table(Queryable): hash_id=hash_id, alter=alter, extracts=extracts, + conversions=conversions, ) def upsert_all( @@ -1147,6 +1161,7 @@ class Table(Queryable): hash_id=DEFAULT, alter=DEFAULT, extracts=DEFAULT, + conversions=DEFAULT, ): return self.insert_all( records, @@ -1159,6 +1174,7 @@ class Table(Queryable): hash_id=hash_id, alter=alter, extracts=extracts, + conversions=conversions, upsert=True, ) diff --git a/tests/test_conversions.py b/tests/test_conversions.py new file mode 100644 index 0000000..ebe2a50 --- /dev/null +++ b/tests/test_conversions.py @@ -0,0 +1,44 @@ +import pytest + + +def test_insert_conversion(fresh_db): + table = fresh_db["table"] + table.insert({"foo": "bar"}, conversions={"foo": "upper(?)"}) + assert [{"foo": "BAR"}] == list(table.rows) + + +def test_insert_all_conversion(fresh_db): + table = fresh_db["table"] + table.insert_all([{"foo": "bar"}], conversions={"foo": "upper(?)"}) + assert [{"foo": "BAR"}] == list(table.rows) + + +def test_upsert_conversion(fresh_db): + table = fresh_db["table"] + table.upsert({"id": 1, "foo": "bar"}, pk="id", conversions={"foo": "upper(?)"}) + assert [{"id": 1, "foo": "BAR"}] == list(table.rows) + table.upsert( + {"id": 1, "bar": "baz"}, pk="id", conversions={"bar": "upper(?)"}, alter=True + ) + assert [{"id": 1, "foo": "BAR", "bar": "BAZ"}] == list(table.rows) + + +def test_upsert_all_conversion(fresh_db): + table = fresh_db["table"] + table.upsert_all( + [{"id": 1, "foo": "bar"}], pk="id", conversions={"foo": "upper(?)"} + ) + assert [{"id": 1, "foo": "BAR"}] == list(table.rows) + + +def test_update_conversion(fresh_db): + table = fresh_db["table"] + table.insert({"id": 5, "foo": "bar"}, pk="id") + table.update(5, {"foo": "baz"}, conversions={"foo": "upper(?)"}) + assert [{"id": 5, "foo": "BAZ"}] == list(table.rows) + + +def test_table_constructor_conversion(fresh_db): + table = fresh_db.table("table", conversions={"bar": "upper(?)"}) + table.insert({"bar": "baz"}) + assert [{"bar": "BAZ"}] == list(table.rows) From f7289174e66ae4d91d57de94bbd9d09fabf7aff4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Jan 2020 16:25:20 -0800 Subject: [PATCH 0067/1004] Release 2.1 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 72b19fc..fc27dca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v2_1: + +2.1 (2020-01-30) +---------------- + +New feature: ``conversions={...}`` can be passed to the ``.insert()`` family of functions to specify SQL conversions that should be applied to values that are being inserted or updated. See :ref:`python_api_conversions` . (`#77 `__). + + .. _v2_0_1: 2.0.1 (2020-01-05) diff --git a/setup.py b/setup.py index a37c6b0..66854c9 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.0.1" +VERSION = "2.1" def get_long_description(): From 5ecf3ffdeae0ab90b54044d34428b348b8473c94 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 1 Feb 2020 13:38:26 -0800 Subject: [PATCH 0068/1004] Extracted detect_column_types as suggest_column_types, refs #81 --- sqlite_utils/db.py | 30 +++--------------------------- sqlite_utils/utils.py | 25 +++++++++++++++++++++++++ tests/test_suggest_column_types.py | 19 +++++++++++++++++++ 3 files changed, 47 insertions(+), 27 deletions(-) create mode 100644 tests/test_suggest_column_types.py diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b061e98..22cf02e 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,4 +1,4 @@ -from .utils import sqlite3, OperationalError +from .utils import sqlite3, OperationalError, suggest_column_types from collections import namedtuple import datetime import hashlib @@ -820,30 +820,6 @@ class Table(Queryable): ) return self - def detect_column_types(self, records): - all_column_types = {} - for record in records: - for key, value in record.items(): - all_column_types.setdefault(key, set()).add(type(value)) - column_types = {} - for key, types in all_column_types.items(): - if len(types) == 1: - t = list(types)[0] - # But if it's list / tuple / dict, use str instead as we - # will be storing it as JSON in the table - if t in (list, tuple, dict): - t = str - elif {int, bool}.issuperset(types): - t = int - elif {int, float, bool}.issuperset(types): - t = float - elif {bytes, str}.issuperset(types): - t = bytes - else: - t = str - column_types[key] = t - return column_types - def search(self, q): sql = """ select * from "{table}" where rowid in ( @@ -1006,7 +982,7 @@ class Table(Queryable): if not self.exists: # Use the first batch to derive the table names self.create( - self.detect_column_types(chunk), + suggest_column_types(chunk), pk, foreign_keys, column_order=column_order, @@ -1179,7 +1155,7 @@ class Table(Queryable): ) def add_missing_columns(self, records): - needed_columns = self.detect_column_types(records) + needed_columns = suggest_column_types(records) current_columns = self.columns_dict for col_name, col_type in needed_columns.items(): if col_name not in current_columns: diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 738424d..0645300 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -7,3 +7,28 @@ except ImportError: import sqlite3 OperationalError = sqlite3.OperationalError + + +def suggest_column_types(records): + all_column_types = {} + for record in records: + for key, value in record.items(): + all_column_types.setdefault(key, set()).add(type(value)) + column_types = {} + for key, types in all_column_types.items(): + if len(types) == 1: + t = list(types)[0] + # But if it's list / tuple / dict, use str instead as we + # will be storing it as JSON in the table + if t in (list, tuple, dict): + t = str + elif {int, bool}.issuperset(types): + t = int + elif {int, float, bool}.issuperset(types): + t = float + elif {bytes, str}.issuperset(types): + t = bytes + else: + t = str + column_types[key] = t + return column_types diff --git a/tests/test_suggest_column_types.py b/tests/test_suggest_column_types.py new file mode 100644 index 0000000..d9fe6ff --- /dev/null +++ b/tests/test_suggest_column_types.py @@ -0,0 +1,19 @@ +import pytest +from sqlite_utils.utils import suggest_column_types + + +@pytest.mark.parametrize( + "records,types", + [ + ([{"a": 1}], {"a": int}), + ([{"a": "baz"}], {"a": str}), + ([{"a": 1.2}], {"a": float}), + ([{"a": [1]}], {"a": str}), + ([{"a": (1,)}], {"a": str}), + ([{"a": {"b": 1}}], {"a": str}), + ([{"a": 1}, {"a": 1.1}], {"a": float}), + ([{"a": b"b"}], {"a": bytes}), + ], +) +def test_suggest_column_types(records, types): + assert types == suggest_column_types(records) From de76168be5a3e18e9fda32670ff219e04e239d8f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 1 Feb 2020 13:55:13 -0800 Subject: [PATCH 0069/1004] Docs for suggest_column_types, closes #81 --- docs/python-api.rst | 70 ++++++++++++++++++++++++++++++++++++++++ sqlite_utils/__init__.py | 3 +- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index a1f0461..2b2611d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -182,6 +182,8 @@ After inserting a row like this, the ``dogs.last_rowid`` property will return th The ``dogs.last_pk`` property will return the last inserted primary key value, if you specified one. This can be very useful when writing code that creates foreign keys or many-to-many relationships. +.. _python_api_explicit_create: + Explicitly creating a table --------------------------- @@ -1090,3 +1092,71 @@ You can optimize your database by running VACUUM against it like so: .. code-block:: python Database("my_database.db").vacuum() + +.. _python_api_suggest_column_types: + +Suggesting column types +======================= + +When you create a new table for a list of inserted or upserted Python dictionaries, those methods detect the correct types for the database columns based on the data you pass in. + +In some situations you may need to intervene in this process, to customize the columns that are being created in some way - see :ref:`python_api_explicit_create`. + +That table ``.create()`` method takes a dictionary mapping column names to the Python type they should store: + +.. code-block:: python + + db["cats"].create({ + "id": int, + "name": str, + "weight": float, + }) + +You can use the ``suggest_column_types()`` helper function to derive a dictionary of column names and types from a list of records, suitable to be passed to ``table.create()``. + +For example: + +.. code-block:: python + + from sqlite_utils import Database, suggest_column_types + + cats = [{ + "id": 1, + "name": "Snowflake" + }, { + "id": 2, + "name": "Crabtree", + "age": 4 + }] + types = suggest_column_types(cats) + # types now looks like this: + # {"id": , + # "name": , + # "age": } + + # Manually add an extra field: + types["thumbnail"] = bytes + # types now looks like this: + # {"id": , + # "name": , + # "age": , + # "thumbnail": } + + # Create the table + db = Database("cats.db") + db["cats"].create(types, pk="id") + # Insert the records + db["cats"].insert_all(cats) + + # list(db["cats"].rows) now returns: + # [{"id": 1, "name": "Snowflake", "age": None, "thumbnail": None} + # {"id": 2, "name": "Crabtree", "age": 4, "thumbnail": None}] + + # The table schema looks like this: + # print(db["cats"].schema) + # CREATE TABLE [cats] ( + # [id] INTEGER PRIMARY KEY, + # [name] TEXT, + # [age] INTEGER, + # [thumbnail] BLOB + # ) diff --git a/sqlite_utils/__init__.py b/sqlite_utils/__init__.py index c0e52e9..614f984 100644 --- a/sqlite_utils/__init__.py +++ b/sqlite_utils/__init__.py @@ -1,3 +1,4 @@ from .db import Database +from .utils import suggest_column_types -__all__ = ["Database"] +__all__ = ["Database", "suggest_column_types"] From 72fa16b3d9033525ea6a798c99a870db93ece9e6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 1 Feb 2020 13:59:08 -0800 Subject: [PATCH 0070/1004] Release 2.2 --- docs/changelog.rst | 10 +++++++++- setup.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fc27dca..5268a7f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v2_2: + +2.2 (2020-02-01) +---------------- + +New feature: ``sqlite_utils.suggest_column_types([records])`` returns the suggested column types for a list of records. See :ref:`python_api_suggest_column_types`. (`#81 `__). + +This replaces the undocumented ``table.detect_column_types()`` method. + .. _v2_1: 2.1 (2020-01-30) @@ -9,7 +18,6 @@ New feature: ``conversions={...}`` can be passed to the ``.insert()`` family of functions to specify SQL conversions that should be applied to values that are being inserted or updated. See :ref:`python_api_conversions` . (`#77 `__). - .. _v2_0_1: 2.0.1 (2020-01-05) diff --git a/setup.py b/setup.py index 66854c9..9f2b1cd 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.1" +VERSION = "2.2" def get_long_description(): From 7c28a4d133b6a639fa70ba22b22cd43cb0746394 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 6 Feb 2020 23:17:06 -0800 Subject: [PATCH 0071/1004] Fix for upsert(hash_id=) bug, closes #84 --- sqlite_utils/db.py | 7 +++++-- tests/test_upsert.py | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 22cf02e..0858323 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -943,8 +943,6 @@ class Table(Queryable): data """ pk = self.value_or_default("pk", pk) - if upsert and not pk: - raise PrimaryKeyRequired("upsert() requires a pk") foreign_keys = self.value_or_default("foreign_keys", foreign_keys) column_order = self.value_or_default("column_order", column_order) not_null = self.value_or_default("not_null", not_null) @@ -957,7 +955,12 @@ class Table(Queryable): extracts = self.value_or_default("extracts", extracts) conversions = self.value_or_default("conversions", conversions) + if upsert and (not pk and not hash_id): + raise PrimaryKeyRequired("upsert() requires a pk") assert not (hash_id and pk), "Use either pk= or hash_id=" + if hash_id: + pk = hash_id + assert not ( ignore and replace ), "Use either ignore=True or replace=True, not both" diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 4d53fa8..584a496 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -28,6 +28,14 @@ def test_upsert_error_if_no_pk(fresh_db): table.upsert({"id": 1, "name": "Cleo"}) +def test_upsert_with_hash_id(fresh_db): + table = fresh_db["table"] + table.upsert({"foo": "bar"}, hash_id="pk") + assert [{"pk": "a5e744d0164540d33b1d7ea616c28f2fa97e754a", "foo": "bar"}] == list( + table.rows + ) + + def test_upsert_compound_primary_key(fresh_db): table = fresh_db["table"] table.upsert_all( From 4cbade256414844961af5006e93c1cdc5bb84868 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 6 Feb 2020 23:20:03 -0800 Subject: [PATCH 0072/1004] Release 2.2.1 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5268a7f..6e7eadb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_2_1: + +2.2.1 (2020-02-06) +------------------ + +Fixed a bug where ``.upsert(..., hash_id="pk")`` threw an error (`#84 `__). + .. _v2_2: 2.2 (2020-02-01) diff --git a/setup.py b/setup.py index 9f2b1cd..710140c 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.2" +VERSION = "2.2.1" def get_long_description(): From 0eda638d81280c7e585db071c35444e14b0b00f6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Feb 2020 15:56:03 -0800 Subject: [PATCH 0073/1004] table.exists() now a documented method, closes #83 --- docs/python-api.rst | 7 +++++++ sqlite_utils/db.py | 29 ++++++++++++++++------------- tests/test_m2m.py | 14 +++++++------- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 2b2611d..14de937 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -920,6 +920,13 @@ If you have loaded an existing table or view, you can use introspection to find >>> db["PlantType"]
+The ``.exists()`` method can be used to find out if a table exists or not:: + + >>> db["PlantType"].exists() + True + >>> db["PlantType2"].exists() + False + The ``.count`` property shows the current number of rows (``select count(*) from table``):: >>> db["PlantType"].count diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0858323..1cd19a0 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -226,7 +226,7 @@ class Database: extracts = resolve_extracts(extracts) for extract_column, extract_table in extracts.items(): # Ensure other table exists - if not self[extract_table].exists: + if not self[extract_table].exists(): self.create_table(extract_table, {"id": int, "value": str}, pk="id") columns[extract_column] = int foreign_keys_by_column[extract_column] = ForeignKey( @@ -348,11 +348,11 @@ class Database: # Verify that all tables and columns exist for table, column, other_table, other_column in foreign_keys: - if not self[table].exists: + if not self[table].exists(): raise AlterError("No such table: {}".format(table)) if column not in self[table].columns_dict: raise AlterError("No such column: {} in {}".format(column, table)) - if not self[other_table].exists: + if not self[other_table].exists(): raise AlterError("No such other_table: {}".format(other_table)) if ( other_column != "rowid" @@ -416,7 +416,8 @@ class Database: class Queryable: - exists = False + def exists(self): + return False def __init__(self, db, name): self.db = db @@ -433,7 +434,7 @@ class Queryable: return self.rows_where() def rows_where(self, where=None, where_args=None): - if not self.exists: + if not self.exists(): return [] sql = "select * from [{}]".format(self.name) if where is not None: @@ -445,7 +446,7 @@ class Queryable: @property def columns(self): - if not self.exists: + if not self.exists(): return [] rows = self.db.conn.execute( "PRAGMA table_info([{}])".format(self.name) @@ -486,7 +487,6 @@ class Table(Queryable): conversions=None, ): super().__init__(db, name) - self.exists = self.name in self.db.table_names() self._defaults = dict( pk=pk, foreign_keys=foreign_keys, @@ -506,10 +506,13 @@ class Table(Queryable): return "
".format( self.name, " (does not exist yet)" - if not self.exists + if not self.exists() else " ({})".format(", ".join(c.name for c in self.columns)), ) + def exists(self): + return self.name in self.db.table_names() + @property def pks(self): names = [column.name for column in self.columns if column.is_pk] @@ -614,7 +617,6 @@ class Table(Queryable): hash_id=hash_id, extracts=extracts, ) - self.exists = True return self def create_index(self, columns, index_name=None, unique=False, if_not_exists=False): @@ -847,7 +849,7 @@ class Table(Queryable): self.db.conn.execute(sql, pk_values) def delete_where(self, where=None, where_args=None): - if not self.exists: + if not self.exists(): return [] sql = "delete from [{}]".format(self.name) if where is not None: @@ -982,7 +984,7 @@ class Table(Queryable): for chunk in chunks(itertools.chain([first_record], records), batch_size): chunk = list(chunk) if first: - if not self.exists: + if not self.exists(): # Use the first batch to derive the table names self.create( suggest_column_types(chunk), @@ -1167,7 +1169,7 @@ class Table(Queryable): def lookup(self, column_values): # lookups is a dictionary - all columns will be used for a unique index assert isinstance(column_values, dict) - if self.exists: + if self.exists(): self.add_missing_columns([column_values]) unique_column_sets = [set(i.columns) for i in self.indexes] if set(column_values.keys()) not in unique_column_sets: @@ -1245,7 +1247,8 @@ class Table(Queryable): class View(Queryable): - exists = True + def exists(self): + return True def __repr__(self): return "".format( diff --git a/tests/test_m2m.py b/tests/test_m2m.py index b5b897b..52b3959 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -61,8 +61,8 @@ def test_m2m_lookup(fresh_db): people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) people_tags = fresh_db["people_tags"] tags = fresh_db["tags"] - assert people_tags.exists - assert tags.exists + assert people_tags.exists() + assert tags.exists() assert [ ForeignKey( table="people_tags", @@ -94,7 +94,7 @@ def test_m2m_explicit_table_name_argument(fresh_db): ) assert fresh_db["tags"].exists assert fresh_db["tagged"].exists - assert not fresh_db["people_tags"].exists + assert not fresh_db["people_tags"].exists() def test_m2m_table_candidates(fresh_db): @@ -130,10 +130,10 @@ def test_uses_existing_m2m_table_if_exists(fresh_db): foreign_keys=["people_id", "tags_id"], ) people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) - assert fresh_db["tags"].exists - assert fresh_db["tagged"].exists - assert not fresh_db["people_tags"].exists - assert not fresh_db["tags_people"].exists + assert fresh_db["tags"].exists() + assert fresh_db["tagged"].exists() + assert not fresh_db["people_tags"].exists() + assert not fresh_db["tags_people"].exists() assert [{"people_id": 1, "tags_id": 1}] == list(fresh_db["tagged"].rows) From 0c2451e0690c5f4e6463a2f339b0a280e30ed806 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Feb 2020 15:56:16 -0800 Subject: [PATCH 0074/1004] Release 2.3 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6e7eadb..47eb18c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v2_3: + +2.3 (2020-02-08) +---------------- + +``table.exists()`` is now a method, not a property. This was not a documented part of the API before so I'm considering this a non-breaking change. (`#83 `__) + + .. _v2_2_1: 2.2.1 (2020-02-06) diff --git a/setup.py b/setup.py index 710140c..76caf69 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.2.1" +VERSION = "2.3" def get_long_description(): From 6f3cb2c106ae99f0a14201e6b4c61ec2f492e766 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Feb 2020 21:13:15 -0800 Subject: [PATCH 0075/1004] create_index now works with columns with spaces, closes #85 --- sqlite_utils/db.py | 6 +++--- tests/test_create.py | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1cd19a0..0d23e0b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -625,12 +625,12 @@ class Table(Queryable): self.name.replace(" ", "_"), "_".join(columns) ) sql = """ - CREATE {unique}INDEX {if_not_exists}{index_name} - ON {table_name} ({columns}); + CREATE {unique}INDEX {if_not_exists}[{index_name}] + ON [{table_name}] ({columns}); """.format( index_name=index_name, table_name=self.name, - columns=", ".join(columns), + columns=", ".join("[{}]".format(c) for c in columns), unique="UNIQUE " if unique else "", if_not_exists="IF NOT EXISTS " if if_not_exists else "", ) diff --git a/tests/test_create.py b/tests/test_create.py index bfa352f..15e73cf 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -540,27 +540,27 @@ def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error): "columns,index_name,expected_index", ( ( - ["is_good_dog"], + ["is good dog"], None, Index( seq=0, - name="idx_dogs_is_good_dog", + name="idx_dogs_is good dog", unique=0, origin="c", partial=0, - columns=["is_good_dog"], + columns=["is good dog"], ), ), ( - ["is_good_dog", "age"], + ["is good dog", "age"], None, Index( seq=0, - name="idx_dogs_is_good_dog_age", + name="idx_dogs_is good dog_age", unique=0, origin="c", partial=0, - columns=["is_good_dog", "age"], + columns=["is good dog", "age"], ), ), ( @@ -579,7 +579,7 @@ def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error): ) def test_create_index(fresh_db, columns, index_name, expected_index): dogs = fresh_db["dogs"] - dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is_good_dog": True}) + dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True}) assert [] == dogs.indexes dogs.create_index(columns, index_name) assert expected_index == dogs.indexes[0] From de45597327c5561913efac528062c70fe14235fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Feb 2020 21:15:10 -0800 Subject: [PATCH 0076/1004] Release 2.3.1 --- docs/changelog.rst | 8 +++++++- setup.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 47eb18c..1963097 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_3_1: + +2.3.1 (2020-02-10) +------------------ + +``table.create_index()`` now works for columns that contain spaces. (`#85 `__) + .. _v2_3: 2.3 (2020-02-08) @@ -9,7 +16,6 @@ ``table.exists()`` is now a method, not a property. This was not a documented part of the API before so I'm considering this a non-breaking change. (`#83 `__) - .. _v2_2_1: 2.2.1 (2020-02-06) diff --git a/setup.py b/setup.py index 76caf69..ce39b2e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.3" +VERSION = "2.3.1" def get_long_description(): From 45df15fe23227306aca53dc99eeb66d9e272e38e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Feb 2020 21:19:54 -0800 Subject: [PATCH 0077/1004] Attempt to fix the build Suggestion from here: https://github.com/bluethon/bluethon/blob/fedbdb506a49ddba3b972f3fd35772b65b241390/languages/python/pip/pip_note.md#L13 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 110f60d..ca4a8f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ python: - "3.8-dev" script: + - python3 -m pip install -U pip - pip install -U pip wheel - pip install .[test] # Only run the numpy/pandas tests on Python 3.7: From 5e0000609f9be6efafea1b96f610988eb18d6d89 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Feb 2020 21:52:23 -0800 Subject: [PATCH 0078/1004] Try using Travis Pythons that are not -dev --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ca4a8f3..c75c1a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,10 @@ dist: bionic # 3.6 is listed first so it gets used for the later build stages python: - "3.6" - - "3.7-dev" - - "3.8-dev" + - "3.7" + - "3.8" script: - - python3 -m pip install -U pip - pip install -U pip wheel - pip install .[test] # Only run the numpy/pandas tests on Python 3.7: From 685e6a1bb3ca8c14b6b8297e438e88cabebc5c56 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 15 Feb 2020 18:20:39 -0800 Subject: [PATCH 0079/1004] Detect subclasses of dict/tuple/list, fixes #87 --- sqlite_utils/db.py | 2 +- sqlite_utils/utils.py | 9 +++++---- tests/test_create.py | 1 + tests/test_suggest_column_types.py | 2 ++ 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0d23e0b..e781e2a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,5 +1,5 @@ from .utils import sqlite3, OperationalError, suggest_column_types -from collections import namedtuple +from collections import namedtuple, OrderedDict import datetime import hashlib import itertools diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 0645300..6da6d47 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -18,10 +18,11 @@ def suggest_column_types(records): for key, types in all_column_types.items(): if len(types) == 1: t = list(types)[0] - # But if it's list / tuple / dict, use str instead as we - # will be storing it as JSON in the table - if t in (list, tuple, dict): - t = str + # But if it's a subclass of list / tuple / dict, use str + # instead as we will be storing it as JSON in the table + for superclass in (list, tuple, dict): + if issubclass(t, superclass): + t = str elif {int, bool}.issuperset(types): t = int elif {int, float, bool}.issuperset(types): diff --git a/tests/test_create.py b/tests/test_create.py index 15e73cf..7f22ded 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -621,6 +621,7 @@ def test_create_index_if_not_exists(fresh_db): ["list with", "two items"], {"dictionary": "simple"}, {"dictionary": {"nested": "complex"}}, + collections.OrderedDict([("key1", {"nested": "complex"}), ("key2", "foo"),]), [{"list": "of"}, {"two": "dicts"}], ), ) diff --git a/tests/test_suggest_column_types.py b/tests/test_suggest_column_types.py index d9fe6ff..9427444 100644 --- a/tests/test_suggest_column_types.py +++ b/tests/test_suggest_column_types.py @@ -1,4 +1,5 @@ import pytest +from collections import OrderedDict from sqlite_utils.utils import suggest_column_types @@ -11,6 +12,7 @@ from sqlite_utils.utils import suggest_column_types ([{"a": [1]}], {"a": str}), ([{"a": (1,)}], {"a": str}), ([{"a": {"b": 1}}], {"a": str}), + ([{"a": OrderedDict({"b": 1})}], {"a": str}), ([{"a": 1}, {"a": 1.1}], {"a": float}), ([{"a": b"b"}], {"a": bytes}), ], From b0ca657f49dcecb9985f92d481cfe77f7d3ad0f4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Feb 2020 20:16:02 -0800 Subject: [PATCH 0080/1004] Disallow square braces in column names, closes #86 --- sqlite_utils/db.py | 5 +++++ tests/test_create.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e781e2a..2078525 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -245,6 +245,11 @@ class Database: ), "defaults set {} includes items not in columns {}".format( repr(set(defaults)), repr(set(columns.keys())) ) + # Validate no columns contain '[' or ']' - #86 + for column in columns.keys(): + assert ( + "[" not in column and "]" not in column + ), "'[' and ']' cannot be used in column names" column_items = list(columns.items()) if column_order is not None: column_items.sort( diff --git a/tests/test_create.py b/tests/test_create.py index 7f22ded..6ad54d0 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -77,6 +77,11 @@ def test_create_table_with_bad_defaults(fresh_db): ) +def test_create_table_with_invalid_column_charactters(fresh_db): + with pytest.raises(AssertionError): + fresh_db.create_table("players", {"name[foo]": str}) + + def test_create_table_with_defaults(fresh_db): table = fresh_db.create_table( "players", From f9473ace14878212c1fa968b7bd2f51e4f064dba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Feb 2020 20:40:35 -0800 Subject: [PATCH 0081/1004] disable-fts and .disable_fts(), closes #88 --- docs/cli.rst | 4 +++ docs/python-api.rst | 6 ++++ sqlite_utils/cli.py | 13 ++++++++ sqlite_utils/db.py | 19 ++++++++++++ tests/test_cli.py | 18 ++++++++++++ tests/{test_enable_fts.py => test_fts.py} | 36 +++++++++++++++++++++++ 6 files changed, 96 insertions(+) rename tests/{test_enable_fts.py => test_fts.py} (77%) diff --git a/docs/cli.rst b/docs/cli.rst index 8590456..35e34b2 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -411,6 +411,10 @@ A better solution here is to use database triggers. You can set up database trig $ sqlite-utils enable-fts mydb.db documents title summary --create-triggers +To remove the FTS tables and triggers you created, use ``disable-fts``:: + + $ sqlite-utils disable-fts mydb.db documents + Vacuum ====== diff --git a/docs/python-api.rst b/docs/python-api.rst index 14de937..878752a 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1052,6 +1052,12 @@ A better solution is to use database triggers. You can set up database triggers dogs.enable_fts(["name", "twitter"], fts_version="FTS4") +To remove the FTS tables and triggers you created, use the ``disable_fts()`` table method: + +.. code-block:: python + + dogs.disable_fts() + Optimizing a full-text search table =================================== diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index ba55e61..54667e5 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -300,6 +300,19 @@ def populate_fts(path, table, column): db[table].populate_fts(column) +@cli.command(name="disable-fts") +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table") +def disable_fts(path, table): + "Disable FTS for specific table" + db = sqlite_utils.Database(path) + db[table].disable_fts() + + def insert_upsert_options(fn): for decorator in reversed( ( diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 2078525..40b734c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -794,6 +794,25 @@ class Table(Queryable): self.db.conn.executescript(sql) return self + def disable_fts(self): + fts_table = self.detect_fts() + if fts_table: + self.db[fts_table].drop() + # Now delete the triggers that related to that table + sql = """ + SELECT name FROM sqlite_master + WHERE type = 'trigger' + AND sql LIKE '% INSERT INTO [{}]%' + """.format( + fts_table + ) + trigger_names = [] + for row in self.db.conn.execute(sql).fetchall(): + trigger_names.append(row[0]) + with self.db.conn: + for trigger_name in trigger_names: + self.db.conn.execute("DROP TRIGGER IF EXISTS [{}]".format(trigger_name)) + def detect_fts(self): "Detect if table has a corresponding FTS virtual table and return it" sql = """ diff --git a/tests/test_cli.py b/tests/test_cli.py index 1f24090..52c50da 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -393,6 +393,24 @@ def test_populate_fts(db_path): assert [("martha",)] == search("martha") +def test_disable_fts(db_path): + db = Database(db_path) + assert {"Gosh", "Gosh2"} == set(db.table_names()) + db["Gosh"].enable_fts(["c1"], create_triggers=True) + assert { + "Gosh_fts", + "Gosh_fts_idx", + "Gosh_fts_data", + "Gosh2", + "Gosh_fts_config", + "Gosh", + "Gosh_fts_docsize", + } == set(db.table_names()) + exit_code = CliRunner().invoke(cli.cli, ["disable-fts", db_path, "Gosh"]).exit_code + assert 0 == exit_code + assert {"Gosh", "Gosh2"} == set(db.table_names()) + + def test_vacuum(db_path): result = CliRunner().invoke(cli.cli, ["vacuum", db_path]) assert 0 == result.exit_code diff --git a/tests/test_enable_fts.py b/tests/test_fts.py similarity index 77% rename from tests/test_enable_fts.py rename to tests/test_fts.py index 7f58916..134e0e7 100644 --- a/tests/test_enable_fts.py +++ b/tests/test_fts.py @@ -1,3 +1,5 @@ +import pytest + search_records = [ {"text": "tanuki are tricksters", "country": "Japan", "not_searchable": "foo"}, {"text": "racoons are trash pandas", "country": "USA", "not_searchable": "bar"}, @@ -92,3 +94,37 @@ def test_enable_fts_w_triggers(fresh_db): # Triggers will auto-populate FTS virtual table, not need to call populate_fts() assert [("racoons are trash pandas", "USA", "bar")] == table.search("usa") assert [] == table.search("bar") + + +@pytest.mark.parametrize("create_triggers", [True, False]) +def test_disable_fts(fresh_db, create_triggers): + table = fresh_db["searchable"] + table.insert(search_records[0]) + table.enable_fts(["text", "country"], create_triggers=create_triggers) + assert { + "searchable", + "searchable_fts", + "searchable_fts_data", + "searchable_fts_idx", + "searchable_fts_docsize", + "searchable_fts_config", + } == set(fresh_db.table_names()) + if create_triggers: + expected_triggers = {"searchable_ai", "searchable_ad", "searchable_au"} + else: + expected_triggers = set() + assert expected_triggers == set( + r[0] + for r in fresh_db.conn.execute( + "select name from sqlite_master where type = 'trigger'" + ).fetchall() + ) + # Now run .disable_fts() and confirm it worked + table.disable_fts() + assert ( + 0 + == fresh_db.conn.execute( + "select count(*) from sqlite_master where type = 'trigger'" + ).fetchone()[0] + ) + assert ["searchable"] == fresh_db.table_names() From 67dd3106d56adfa335fdfc7f8737cf693b1db088 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Feb 2020 20:46:13 -0800 Subject: [PATCH 0082/1004] Changelog for 2.4 --- docs/changelog.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1963097..25e3c4e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,16 @@ Changelog =========== +.. _v2_4: + +2.4 (2020-02-26) +---------------- + +- ``table.disable_fts()`` can now be used to remove FTS tables and triggers that were created using ``table.enable_fts(...)``. (`#88 `__) +- The ``sqlite-utils disable-fts`` command can be used to remove FTS tables and triggers from the command-line. (`#88 `__) +- Trying to create table columns wih square braces (``[`` or ``]``) in the name now raises an error. (`#86 `__) +- Subclasses of ``dict``, ``list`` and ``tuple`` are now detected as needing a JSON column. (`#87 `__) + .. _v2_3_1: 2.3.1 (2020-02-10) From 04ec53c039feb590c7832d499a14a9caba081f11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Feb 2020 20:55:17 -0800 Subject: [PATCH 0083/1004] Validate column names in more places, refs #86 --- sqlite_utils/db.py | 17 +++++++++++------ tests/test_create.py | 9 ++++++++- tests/test_update.py | 7 +++++++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 40b734c..5d17ad2 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -245,11 +245,7 @@ class Database: ), "defaults set {} includes items not in columns {}".format( repr(set(defaults)), repr(set(columns.keys())) ) - # Validate no columns contain '[' or ']' - #86 - for column in columns.keys(): - assert ( - "[" not in column and "]" not in column - ), "'[' and ']' cannot be used in column names" + validate_column_names(columns.keys()) column_items = list(columns.items()) if column_order is not None: column_items.sort( @@ -892,6 +888,7 @@ class Table(Queryable): args = [] sets = [] wheres = [] + validate_column_names(updates.keys()) for key, value in updates.items(): sets.append("[{}] = {}".format(key, conversions.get(key, "?"))) args.append(value) @@ -1026,8 +1023,8 @@ class Table(Queryable): all_columns = list(sorted(all_columns)) if hash_id: all_columns.insert(0, hash_id) + validate_column_names(all_columns) first = False - # values is the list of insert data that is passed to the # .execute() method - but some of them may be replaced by # new primary keys if we are extracting any columns. @@ -1310,3 +1307,11 @@ def resolve_extracts(extracts): if isinstance(extracts, (list, tuple)): extracts = {item: item for item in extracts} return extracts + + +def validate_column_names(columns): + # Validate no columns contain '[' or ']' - #86 + for column in columns: + assert ( + "[" not in column and "]" not in column + ), "'[' and ']' cannot be used in column names" diff --git a/tests/test_create.py b/tests/test_create.py index 6ad54d0..92e0f8a 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -77,7 +77,7 @@ def test_create_table_with_bad_defaults(fresh_db): ) -def test_create_table_with_invalid_column_charactters(fresh_db): +def test_create_table_with_invalid_column_characters(fresh_db): with pytest.raises(AssertionError): fresh_db.create_table("players", {"name[foo]": str}) @@ -449,6 +449,13 @@ def test_insert_row_alter_table( ] +def test_insert_row_alter_table_invalid_column_characters(fresh_db): + table = fresh_db["table"] + rowid = table.insert({"foo": "bar"}).last_pk + with pytest.raises(AssertionError): + table.insert({"foo": "baz", "new_col[abc]": 1.2}, alter=True) + + @pytest.mark.parametrize("use_table_factory", [True, False]) def test_insert_replace_rows_alter_table(fresh_db, use_table_factory): first_row = {"id": 1, "title": "Hedgehogs of the world", "author_id": 1} diff --git a/tests/test_update.py b/tests/test_update.py index 70f8e9e..3f4c4ba 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -66,6 +66,13 @@ def test_update_alter(fresh_db): ] == list(table.rows) +def test_update_alter_with_invalid_column_characters(fresh_db): + table = fresh_db["table"] + rowid = table.insert({"foo": "bar"}).last_pk + with pytest.raises(AssertionError): + table.update(rowid, {"new_col[abc]": 1.2}, alter=True) + + def test_update_with_no_values_sets_last_pk(fresh_db): table = fresh_db.table("dogs", pk="id") table.insert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Pancakes"}]) From 277d4e55c496dbe289656bf7649a47db9d1ec5d1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Feb 2020 20:55:58 -0800 Subject: [PATCH 0084/1004] Release 2.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ce39b2e..cd75fa5 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.3.1" +VERSION = "2.4" def get_long_description(): From 2ac4ea3c950d380f4bc44370db229ea9cd82527a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Feb 2020 20:59:01 -0800 Subject: [PATCH 0085/1004] Fixed typo in changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 25e3c4e..da55707 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ - ``table.disable_fts()`` can now be used to remove FTS tables and triggers that were created using ``table.enable_fts(...)``. (`#88 `__) - The ``sqlite-utils disable-fts`` command can be used to remove FTS tables and triggers from the command-line. (`#88 `__) -- Trying to create table columns wih square braces (``[`` or ``]``) in the name now raises an error. (`#86 `__) +- Trying to create table columns with square braces ([ or ]) in the name now raises an error. (`#86 `__) - Subclasses of ``dict``, ``list`` and ``tuple`` are now detected as needing a JSON column. (`#87 `__) .. _v2_3_1: From 0c36feb6ca5c3ffb9a6df4c8ea4bb732fcab74f4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 1 Mar 2020 22:10:43 -0800 Subject: [PATCH 0086/1004] .enable_fts() now works with columns with spaces in them, closes #90 --- sqlite_utils/db.py | 2 +- tests/test_fts.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 5d17ad2..746978a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -785,7 +785,7 @@ class Table(Queryable): INSERT INTO [{table}_fts] (rowid, {columns}) SELECT rowid, {columns} FROM [{table}]; """.format( - table=self.name, columns=", ".join(columns) + table=self.name, columns=", ".join("[{}]".format(c) for c in columns) ) self.db.conn.executescript(sql) return self diff --git a/tests/test_fts.py b/tests/test_fts.py index 134e0e7..7817477 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -44,6 +44,20 @@ def test_enable_fts_escape_table_names(fresh_db): assert [] == table.search("bar") +def test_enable_fts_table_names_containing_spaces(fresh_db): + table = fresh_db["test"] + table.insert({"column with spaces": "in its name"}) + table.enable_fts(["column with spaces"]) + assert [ + "test", + "test_fts", + "test_fts_data", + "test_fts_idx", + "test_fts_docsize", + "test_fts_config", + ] == fresh_db.table_names() + + def test_populate_fts(fresh_db): table = fresh_db["populatable"] table.insert(search_records[0]) From 8f19bbed029df0009cd30f357d26fdf27c7ba5c4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 1 Mar 2020 22:12:21 -0800 Subject: [PATCH 0087/1004] Release 2.4.1 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index da55707..618e62a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_4_1: + +2.4.1 (2020-03-01) +------------------ + +- ``table.enable_fts()`` now works with columns that contain spaces. (`#90 `__) + .. _v2_4: 2.4 (2020-02-26) diff --git a/setup.py b/setup.py index cd75fa5..544d1f5 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.4" +VERSION = "2.4.1" def get_long_description(): From 43f1c6ab4e3a6b76531fb6f5447adb83d26f3971 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Mar 2020 15:08:21 -0600 Subject: [PATCH 0088/1004] Documentation for NotFoundError --- docs/python-api.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 878752a..1b2a2c5 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -122,6 +122,17 @@ If the table has a compound primary key you can pass in the primary key values a >>> db["compound_dogs"].get(("mixed", 3)) +If the record does not exist a ``NotFoundError`` will be raised: + +.. code-block:: python + + from sqlite_utils.db import NotFoundError + + try: + row = db["dogs"].get(5) + except NotFoundError: + print("Dog not found") + Creating tables =============== From 1125460497e0891e730f3e5feff2bb04a78c9163 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 14 Mar 2020 13:04:06 -0700 Subject: [PATCH 0089/1004] Improved column type introspection, closes #92 --- sqlite_utils/db.py | 16 ++-------------- sqlite_utils/utils.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 746978a..15fdcfb 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,4 +1,4 @@ -from .utils import sqlite3, OperationalError, suggest_column_types +from .utils import sqlite3, OperationalError, suggest_column_types, column_affinity from collections import namedtuple, OrderedDict import datetime import hashlib @@ -65,15 +65,6 @@ if np: ) -REVERSE_COLUMN_TYPE_MAPPING = { - "": str, # Columns in views sometimes have type = '' - "TEXT": str, - "BLOB": bytes, - "INTEGER": int, - "FLOAT": float, -} - - class AlterError(Exception): pass @@ -457,10 +448,7 @@ class Queryable: @property def columns_dict(self): "Returns {column: python-type} dictionary" - return { - column.name: REVERSE_COLUMN_TYPE_MAPPING[column.type] - for column in self.columns - } + return {column.name: column_affinity(column.type) for column in self.columns} @property def schema(self): diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 6da6d47..8968a05 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -33,3 +33,22 @@ def suggest_column_types(records): t = str column_types[key] = t return column_types + + +def column_affinity(column_type): + # Implementation of SQLite affinity rules from + # https://www.sqlite.org/datatype3.html#determination_of_column_affinity + assert isinstance(column_type, str) + column_type = column_type.upper().strip() + if column_type == "": + return str # We differ from spec, which says it should be BLOB + if "INT" in column_type: + return int + if "CHAR" in column_type or "CLOB" in column_type or "TEXT" in column_type: + return str + if "BLOB" in column_type: + return bytes + if "REAL" in column_type or "FLOA" in column_type or "DOUB" in column_type: + return float + # Default is 'NUMERIC', which we currently also treat as float + return float From daf2a245aa4e0b0cf62a94c1232cfb858821803b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 14 Mar 2020 13:05:07 -0700 Subject: [PATCH 0090/1004] Unit tests covering column_affinity, refs #92 --- tests/test_column_affinity.py | 45 +++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/test_column_affinity.py diff --git a/tests/test_column_affinity.py b/tests/test_column_affinity.py new file mode 100644 index 0000000..4a34b25 --- /dev/null +++ b/tests/test_column_affinity.py @@ -0,0 +1,45 @@ +import pytest +from sqlite_utils.utils import column_affinity + +EXAMPLES = [ + # Examples from https://www.sqlite.org/datatype3.html#affinity_name_examples + ("INT", int), + ("INTEGER", int), + ("TINYINT", int), + ("SMALLINT", int), + ("MEDIUMINT", int), + ("BIGINT", int), + ("UNSIGNED BIG INT", int), + ("INT2", int), + ("INT8", int), + ("CHARACTER(20)", str), + ("VARCHAR(255)", str), + ("VARYING CHARACTER(255)", str), + ("NCHAR(55)", str), + ("NATIVE CHARACTER(70)", str), + ("NVARCHAR(100)", str), + ("TEXT", str), + ("CLOB", str), + ("BLOB", bytes), + ("REAL", float), + ("DOUBLE", float), + ("DOUBLE PRECISION", float), + ("FLOAT", float), + # Numeric, treated as float: + ("NUMERIC", float), + ("DECIMAL(10,5)", float), + ("BOOLEAN", float), + ("DATE", float), + ("DATETIME", float), +] + + +@pytest.mark.parametrize("column_def,expected_type", EXAMPLES) +def test_column_affinity(column_def, expected_type): + assert expected_type is column_affinity(column_def) + + +@pytest.mark.parametrize("column_def,expected_type", EXAMPLES) +def test_columns_dict(fresh_db, column_def, expected_type): + fresh_db.conn.execute("create table foo (col {})".format(column_def)) + assert {"col": expected_type} == fresh_db["foo"].columns_dict From 755580e2f3020d6af214a41634ab2ab6ed776e10 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 14 Mar 2020 13:09:56 -0700 Subject: [PATCH 0091/1004] Release 2.4.2 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 618e62a..f2ced11 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v2_4_2: + +2.4.2 (2020-03-14) +------------------ + +- ``table.column_dicts`` now works with all column types - previously it would throw errors on types other than ``TEXT``, ``BLOB``, ``INTEGER`` or ``FLOAT``. (`#92 `__) +- Documentation for ``NotFoundError`` thrown by ``table.get(pk)`` - see :ref:`python_api_get`. + .. _v2_4_1: 2.4.1 (2020-03-01) diff --git a/setup.py b/setup.py index 544d1f5..5ddc106 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.4.1" +VERSION = "2.4.2" def get_long_description(): From 1c745df92340ff861750643181a6a3c6685c3d55 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 23 Mar 2020 12:57:02 -0700 Subject: [PATCH 0092/1004] Suggest column types ignores nulls, closes #94 --- sqlite_utils/utils.py | 3 ++- tests/test_suggest_column_types.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 8968a05..d086ddb 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -13,7 +13,8 @@ def suggest_column_types(records): all_column_types = {} for record in records: for key, value in record.items(): - all_column_types.setdefault(key, set()).add(type(value)) + if value is not None: + all_column_types.setdefault(key, set()).add(type(value)) column_types = {} for key, types in all_column_types.items(): if len(types) == 1: diff --git a/tests/test_suggest_column_types.py b/tests/test_suggest_column_types.py index 9427444..a392c09 100644 --- a/tests/test_suggest_column_types.py +++ b/tests/test_suggest_column_types.py @@ -7,14 +7,20 @@ from sqlite_utils.utils import suggest_column_types "records,types", [ ([{"a": 1}], {"a": int}), + ([{"a": 1}, {"a": None}], {"a": int}), ([{"a": "baz"}], {"a": str}), + ([{"a": "baz"}, {"a": None}], {"a": str}), ([{"a": 1.2}], {"a": float}), + ([{"a": 1.2}, {"a": None}], {"a": float}), ([{"a": [1]}], {"a": str}), + ([{"a": [1]}, {"a": None}], {"a": str}), ([{"a": (1,)}], {"a": str}), ([{"a": {"b": 1}}], {"a": str}), + ([{"a": {"b": 1}}, {"a": None}], {"a": str}), ([{"a": OrderedDict({"b": 1})}], {"a": str}), ([{"a": 1}, {"a": 1.1}], {"a": float}), ([{"a": b"b"}], {"a": bytes}), + ([{"a": b"b"}, {"a": None}], {"a": bytes}), ], ) def test_suggest_column_types(records, types): From 3db9881970eead7e27b5411360b0c86296c99a32 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 23 Mar 2020 12:58:55 -0700 Subject: [PATCH 0093/1004] Release 2.4.3 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f2ced11..459efac 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_4_3: + +2.4.3 (2020-03-23) +------------------ + +- Column type suggestion code is no longer confused by null values. (`#94 `__) + .. _v2_4_2: 2.4.2 (2020-03-14) diff --git a/setup.py b/setup.py index 5ddc106..92ffb1f 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.4.2" +VERSION = "2.4.3" def get_long_description(): From b436bdb594fad3134ce6eba2219809faf1472c6e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 23 Mar 2020 13:31:06 -0700 Subject: [PATCH 0094/1004] Fixed bug with null columns, closes #95 --- sqlite_utils/utils.py | 11 ++++++++--- tests/test_create.py | 6 ++++++ tests/test_suggest_column_types.py | 1 + 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index d086ddb..ecb3568 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -13,11 +13,16 @@ def suggest_column_types(records): all_column_types = {} for record in records: for key, value in record.items(): - if value is not None: - all_column_types.setdefault(key, set()).add(type(value)) + all_column_types.setdefault(key, set()).add(type(value)) column_types = {} + for key, types in all_column_types.items(): - if len(types) == 1: + # Ignore null values if at least one other type present: + if len(types) > 1: + types.discard(None.__class__) + if {None.__class__} == types: + t = str + elif len(types) == 1: t = list(types)[0] # But if it's a subclass of list / tuple / dict, use str # instead as we will be storing it as JSON in the table diff --git a/tests/test_create.py b/tests/test_create.py index 92e0f8a..905f49d 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -813,3 +813,9 @@ def test_insert_all_empty_list(fresh_db): assert 1 == fresh_db["t"].count fresh_db["t"].insert_all([], replace=True) assert 1 == fresh_db["t"].count + + +def test_create_with_a_null_column(fresh_db): + record = {"name": "Name", "description": None} + fresh_db["t"].insert(record) + assert [record] == list(fresh_db["t"].rows) diff --git a/tests/test_suggest_column_types.py b/tests/test_suggest_column_types.py index a392c09..e36c58f 100644 --- a/tests/test_suggest_column_types.py +++ b/tests/test_suggest_column_types.py @@ -21,6 +21,7 @@ from sqlite_utils.utils import suggest_column_types ([{"a": 1}, {"a": 1.1}], {"a": float}), ([{"a": b"b"}], {"a": bytes}), ([{"a": b"b"}, {"a": None}], {"a": bytes}), + ([{"a": "a", "b": None}], {"a": str, "b": str}), ], ) def test_suggest_column_types(records, types): From 22250a9c735077d6f365b73bf824e6c67b122c83 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 23 Mar 2020 13:32:09 -0700 Subject: [PATCH 0095/1004] Release 2.4.4 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 459efac..4fab081 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_4_4: + +2.4.4 (2020-03-23) +------------------ + +- Fixed bug where columns with only null values were not correctly created. (`#95 `__) + .. _v2_4_3: 2.4.3 (2020-03-23) diff --git a/setup.py b/setup.py index 92ffb1f..39bce69 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.4.3" +VERSION = "2.4.4" def get_long_description(): From 8ea626e5fcdc4c9e52f615c6347e68173805f8b4 Mon Sep 17 00:00:00 2001 From: bob <32605365+b0b5h4rp13@users.noreply.github.com> Date: Tue, 31 Mar 2020 00:40:48 -0400 Subject: [PATCH 0096/1004] Add type conversion for Panda's Timestamp (#96) Thanks, @b0b5h4rp13! --- sqlite_utils/db.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 15fdcfb..8d22e0c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -8,6 +8,12 @@ import pathlib SQLITE_MAX_VARS = 999 + +try: + import pandas as pd +except ImportError: + pd = None + try: import numpy as np except ImportError: @@ -64,6 +70,10 @@ if np: } ) +# If pandas is available, add more types +if pd: + COLUMN_TYPE_MAPPING.update({pd.Timestamp: "TEXT"}) + class AlterError(Exception): pass From 6161ebf4de44411b3f33feeacaf4501e803d1116 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Apr 2020 11:44:08 -0700 Subject: [PATCH 0097/1004] Fixed incorrect usage example --- docs/cli.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 35e34b2..f09c47e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -242,11 +242,11 @@ You can also import newline-delimited JSON using the ``--nl`` option. Since `Dat This also means you pipe ``sqlite-utils`` together to easily create a new SQLite database file containing the results of a SQL query against another database:: - $ sqlite-utils json sf-trees.db \ + $ sqlite-utils sf-trees.db \ "select TreeID, qAddress, Latitude, Longitude from Street_Tree_List" --nl \ | sqlite-utils insert saved.db trees - --nl # This creates saved.db with a single table called trees: - $ sqlite-utils csv saved.db "select * from trees limit 5" + $ sqlite-utils saved.db "select * from trees limit 5" --csv TreeID,qAddress,Latitude,Longitude 141565,501X Baker St,37.7759676911831,-122.441396661871 232565,940 Elizabeth St,37.7517102172731,-122.441498017841 From 635c91475aa52e58b467797a95fec4554908f7dc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Apr 2020 20:22:32 -0700 Subject: [PATCH 0098/1004] Only set last_pk on singular .insert()/.update(), refs #98 --- sqlite_utils/db.py | 37 ++++++++++++++++++++++++++----------- tests/test_create.py | 7 ++++++- tests/test_upsert.py | 11 +++++++++-- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 8d22e0c..d51ede3 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -468,6 +468,9 @@ class Queryable: class Table(Queryable): + last_rowid = None + last_pk = None + def __init__( self, db, @@ -987,6 +990,7 @@ class Table(Queryable): ), "Use either ignore=True or replace=True, not both" all_columns = None first = True + num_records_processed = 0 # We can only handle a max of 999 variables in a SQL insert, so # we need to adjust the batch_size down if we have too many cols records = iter(records) @@ -1000,8 +1004,11 @@ class Table(Queryable): num_columns <= SQLITE_MAX_VARS ), "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) batch_size = max(1, min(batch_size, SQLITE_MAX_VARS // num_columns)) + self.last_rowid = None + self.last_pk = None for chunk in chunks(itertools.chain([first_record], records), batch_size): chunk = list(chunk) + num_records_processed += len(chunk) if first: if not self.exists(): # Use the first batch to derive the table names @@ -1046,6 +1053,7 @@ class Table(Queryable): pks = [pk] else: pks = pk + self.last_pk = None for record_values in values: # TODO: make more efficient: record = dict(zip(all_columns, record_values)) @@ -1073,6 +1081,12 @@ class Table(Queryable): + [record[pk] for pk in pks], ) ) + # We can populate .last_pk right here + if num_records_processed == 1: + self.last_pk = tuple(record[pk] for pk in pks) + if len(self.last_pk) == 1: + self.last_pk = self.last_pk[0] + else: or_what = "" if replace: @@ -1110,17 +1124,18 @@ class Table(Queryable): result = self.db.conn.execute(query, params) else: raise - self.last_rowid = result.lastrowid - self.last_pk = self.last_rowid - # self.last_rowid will be 0 if a "INSERT OR IGNORE" happened - if (hash_id or pk) and self.last_rowid: - row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] - if hash_id: - self.last_pk = row[hash_id] - elif isinstance(pk, str): - self.last_pk = row[pk] - else: - self.last_pk = tuple(row[p] for p in pk) + if num_records_processed == 1 and not upsert: + self.last_rowid = result.lastrowid + self.last_pk = self.last_rowid + # self.last_rowid will be 0 if a "INSERT OR IGNORE" happened + if (hash_id or pk) and self.last_rowid: + row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] + if hash_id: + self.last_pk = row[hash_id] + elif isinstance(pk, str): + self.last_pk = row[pk] + else: + self.last_pk = tuple(row[p] for p in pk) return self def upsert( diff --git a/tests/test_create.py b/tests/test_create.py index 905f49d..7edcd8d 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -135,7 +135,12 @@ def test_create_table_with_not_null(fresh_db): ), ) def test_create_table_from_example(fresh_db, example, expected_columns): - fresh_db["people"].insert(example) + people_table = fresh_db["people"] + assert None == people_table.last_rowid + assert None == people_table.last_pk + people_table.insert(example) + assert 1 == people_table.last_rowid + assert 1 == people_table.last_pk assert ["people"] == fresh_db.table_names() assert expected_columns == [ {"name": col.name, "type": col.type} for col in fresh_db["people"].columns diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 584a496..100c6c1 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -7,6 +7,7 @@ def test_upsert(fresh_db): table.insert({"id": 1, "name": "Cleo"}, pk="id") table.upsert({"id": 1, "age": 5}, pk="id", alter=True) assert [{"id": 1, "name": "Cleo", "age": 5}] == list(table.rows) + assert 1 == table.last_pk def test_upsert_all(fresh_db): @@ -17,7 +18,7 @@ def test_upsert_all(fresh_db): {"id": 1, "name": "Cleo", "age": 5}, {"id": 2, "name": "Nixie", "age": 5}, ] == list(table.rows) - assert 2 == table.last_pk + assert table.last_pk is None def test_upsert_error_if_no_pk(fresh_db): @@ -34,6 +35,7 @@ def test_upsert_with_hash_id(fresh_db): assert [{"pk": "a5e744d0164540d33b1d7ea616c28f2fa97e754a", "foo": "bar"}] == list( table.rows ) + assert "a5e744d0164540d33b1d7ea616c28f2fa97e754a" == table.last_pk def test_upsert_compound_primary_key(fresh_db): @@ -45,8 +47,13 @@ def test_upsert_compound_primary_key(fresh_db): ], pk=("species", "id"), ) - table.upsert_all([{"species": "dog", "id": 1, "age": 5}], pk=("species", "id")) + assert None == table.last_pk + table.upsert({"species": "dog", "id": 1, "age": 5}, pk=("species", "id")) + assert ("dog", 1) == table.last_pk assert [ {"species": "dog", "id": 1, "name": "Cleo", "age": 5}, {"species": "cat", "id": 1, "name": "Catbag", "age": None}, ] == list(table.rows) + # .upsert_all() with a single item should set .last_pk + table.upsert_all([{"species": "cat", "id": 1, "age": 5}], pk=("species", "id")) + assert ("cat", 1) == table.last_pk From 7e4b9997c2cec4c2af42bd3088847a81c970b6fc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Apr 2020 20:46:51 -0700 Subject: [PATCH 0099/1004] Database(..., recreate=True) option, refs #97 --- docs/python-api.rst | 10 +++++++++- sqlite_utils/db.py | 12 +++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 1b2a2c5..310fb01 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -15,7 +15,15 @@ Database objects are constructed by passing in either a path to a file on disk o db = Database("my_database.db") -This will create ``my_database.db`` if it does not already exist. You can also pass in an existing SQLite connection: +This will create ``my_database.db`` if it does not already exist. + +If you want to recreate a database from scratch (first removing the existing file from disk if it already exists) you can use the ``recreate=True`` argument: + +.. code-block:: python + + db = Database("my_database.db", recreate=True) + +Instead of a file path you can pass in an existing SQLite connection: .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d51ede3..3f9bb6f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -4,6 +4,7 @@ import datetime import hashlib import itertools import json +import os import pathlib SQLITE_MAX_VARS = 999 @@ -96,17 +97,18 @@ class PrimaryKeyRequired(Exception): class Database: - def __init__(self, filename_or_conn=None, memory=False): + def __init__(self, filename_or_conn=None, memory=False, recreate=False): assert (filename_or_conn is not None and not memory) or ( filename_or_conn is None and memory ), "Either specify a filename_or_conn or pass memory=True" - if memory: + if memory or filename_or_conn == ":memory:": self.conn = sqlite3.connect(":memory:") - elif isinstance(filename_or_conn, str): - self.conn = sqlite3.connect(filename_or_conn) - elif isinstance(filename_or_conn, pathlib.Path): + elif isinstance(filename_or_conn, (str, pathlib.Path)): + if recreate and os.path.exists(filename_or_conn): + os.remove(filename_or_conn) self.conn = sqlite3.connect(str(filename_or_conn)) else: + assert not recreate, "recreate cannot be used with connections, only paths" self.conn = filename_or_conn def __getitem__(self, table_name): From 729a3e7869e26bb1ec97c685d3eadc0443404adc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Apr 2020 20:47:36 -0700 Subject: [PATCH 0100/1004] Tests for Database(..., recreate=True), refs #97 --- tests/test_recreate.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/test_recreate.py diff --git a/tests/test_recreate.py b/tests/test_recreate.py new file mode 100644 index 0000000..ddef115 --- /dev/null +++ b/tests/test_recreate.py @@ -0,0 +1,32 @@ +from sqlite_utils import Database +import sqlite3 +import pathlib +import pytest + + +def test_recreate_ignored_for_in_memory(): + # None of these should raise an exception: + Database(memory=True, recreate=False) + Database(memory=True, recreate=True) + Database(":memory:", recreate=False) + Database(":memory:", recreate=True) + + +def test_recreate_not_allowed_for_connection(): + conn = sqlite3.connect(":memory:") + with pytest.raises(AssertionError): + db = Database(conn, recreate=True) + + +@pytest.mark.parametrize( + "use_path,file_exists", [(True, True), (True, False), (False, True), (False, False)] +) +def test_recreate(tmpdir, use_path, file_exists): + filepath = str(tmpdir / "data.db") + if use_path: + filepath = pathlib.Path(filepath) + if file_exists: + Database(filepath)["t1"].insert({"foo": "bar"}) + assert ["t1"] == Database(filepath).table_names() + Database(filepath, recreate=True)["t2"].insert({"foo": "bar"}) + assert ["t2"] == Database(filepath).table_names() From ad6ac19470a67867b96cb4c086450b8e4e46bf02 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Apr 2020 20:52:19 -0700 Subject: [PATCH 0101/1004] Release 2.5 Refs #96. Refs #98. Closes #97. --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4fab081..8a893a3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v2_5: + +2.5 (2020-04-12) +--------------- + +- Panda's Timestamp is now stored as a SQLite TEXT column. Thanks, b0b5h4rp13! (`#96 `__) +- ``table.last_pk`` is now only available for inserts or upserts of a single record. (`#98 `__) +- New ``Database(filepath, recreate=True)`` parameter for deleting and recreating the database. (`#97 `__) + .. _v2_4_4: 2.4.4 (2020-03-23) diff --git a/setup.py b/setup.py index 39bce69..3bec907 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.4.4" +VERSION = "2.5" def get_long_description(): From fc38868bd4c97acfc65b1aefbff80dfdea5e0d54 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 15 Apr 2020 18:04:51 -0700 Subject: [PATCH 0102/1004] Refactored tests into new test_rows.py, refs #76 --- tests/test_get.py | 21 --------------------- tests/test_introspect.py | 6 ------ tests/test_rows.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 27 deletions(-) create mode 100644 tests/test_rows.py diff --git a/tests/test_get.py b/tests/test_get.py index ae5ffdf..63c4a2e 100644 --- a/tests/test_get.py +++ b/tests/test_get.py @@ -29,24 +29,3 @@ def test_get_not_found(argument, expected_msg, fresh_db): fresh_db["dogs"].get(argument) if expected_msg is not None: assert expected_msg == excinfo.value.args[0] - - -@pytest.mark.parametrize( - "where,where_args,expected_ids", - [ - ("name = ?", ["Pancakes"], {2}), - ("age > ?", [3], {1}), - ("name is not null", [], {1, 2}), - ("is_good = ?", [True], {1, 2}), - ], -) -def test_rows_where(where, where_args, expected_ids, fresh_db): - table = fresh_db["dogs"] - table.insert_all( - [ - {"id": 1, "name": "Cleo", "age": 4, "is_good": True}, - {"id": 2, "name": "Pancakes", "age": 3, "is_good": True}, - ], - pk="id", - ) - assert expected_ids == {r["id"] for r in table.rows_where(where, where_args)} diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 7c01a0b..bebe537 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -62,12 +62,6 @@ def test_columns(existing_db): ] -def test_rows(existing_db): - assert [{"text": "one"}, {"text": "two"}, {"text": "three"}] == list( - existing_db["foo"].rows - ) - - def test_schema(existing_db): assert "CREATE TABLE foo (text TEXT)" == existing_db["foo"].schema diff --git a/tests/test_rows.py b/tests/test_rows.py new file mode 100644 index 0000000..115f26a --- /dev/null +++ b/tests/test_rows.py @@ -0,0 +1,29 @@ +from sqlite_utils.db import Index, View +import pytest + + +def test_rows(existing_db): + assert [{"text": "one"}, {"text": "two"}, {"text": "three"}] == list( + existing_db["foo"].rows + ) + + +@pytest.mark.parametrize( + "where,where_args,expected_ids", + [ + ("name = ?", ["Pancakes"], {2}), + ("age > ?", [3], {1}), + ("name is not null", [], {1, 2}), + ("is_good = ?", [True], {1, 2}), + ], +) +def test_rows_where(where, where_args, expected_ids, fresh_db): + table = fresh_db["dogs"] + table.insert_all( + [ + {"id": 1, "name": "Cleo", "age": 4, "is_good": True}, + {"id": 2, "name": "Pancakes", "age": 3, "is_good": True}, + ], + pk="id", + ) + assert expected_ids == {r["id"] for r in table.rows_where(where, where_args)} From 125c625fbc46244a4b4025732e1526fb13c55843 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 15 Apr 2020 20:12:55 -0700 Subject: [PATCH 0103/1004] .rows_where(..., order_by=) argument, closes #76 --- docs/python-api.rst | 18 +++++++++++++++++- sqlite_utils/db.py | 4 +++- tests/test_rows.py | 22 ++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 310fb01..51f05fb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -92,7 +92,7 @@ View objects are similar to Table objects, except that any attempts to insert or * ``count`` * ``schema`` * ``rows`` -* ``rows_where(where, where_args)`` +* ``rows_where(where, where_args, order_by)`` * ``drop()`` .. _python_api_rows: @@ -115,6 +115,22 @@ You can filter rows by a WHERE clause using ``.rows_where(where, where_args)``:: ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} +To specify an order, use the ``order_my=`` argument:: + + >>> for row in db["dogs"].rows_where("age > 1", order_by="age"): + ... print(row) + {'id': 2, 'age': 2, 'name': 'Pancakes'} + {'id': 1, 'age': 4, 'name': 'Cleo'} + +You can use ``order_by="age desc"`` for descending order. + +You can order all records in the table by excluding the ``where`` argument:: + + >>> for row in db["dogs"].rows_where(order_by="age desc"): + ... print(row) + {'id': 1, 'age': 4, 'name': 'Cleo'} + {'id': 2, 'age': 2, 'name': 'Pancakes'} + .. _python_api_get: Retrieving a specific record diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3f9bb6f..d92e92e 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -437,12 +437,14 @@ class Queryable: def rows(self): return self.rows_where() - def rows_where(self, where=None, where_args=None): + def rows_where(self, where=None, where_args=None, order_by=None): if not self.exists(): return [] sql = "select * from [{}]".format(self.name) if where is not None: sql += " where " + where + if order_by is not None: + sql += " order by " + order_by cursor = self.db.conn.execute(sql, where_args or []) columns = [c[0] for c in cursor.description] for row in cursor: diff --git a/tests/test_rows.py b/tests/test_rows.py index 115f26a..539d71d 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -27,3 +27,25 @@ def test_rows_where(where, where_args, expected_ids, fresh_db): pk="id", ) assert expected_ids == {r["id"] for r in table.rows_where(where, where_args)} + + +@pytest.mark.parametrize( + "where,order_by,expected_ids", + [ + (None, None, [1, 2, 3]), + (None, "id desc", [3, 2, 1]), + (None, "age", [3, 2, 1]), + ("id > 1", "age", [3, 2]), + ], +) +def test_rows_where_order_by(where, order_by, expected_ids, fresh_db): + table = fresh_db["dogs"] + table.insert_all( + [ + {"id": 1, "name": "Cleo", "age": 4}, + {"id": 2, "name": "Pancakes", "age": 3}, + {"id": 3, "name": "Bailey", "age": 2}, + ], + pk="id", + ) + assert expected_ids == [r["id"] for r in table.rows_where(where, order_by=order_by)] From 13528faa817d79bc3900d3af7473300686b145d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 15 Apr 2020 20:13:13 -0700 Subject: [PATCH 0104/1004] Release 2.6 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8a893a3..3f315b2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_6: + +2.6 (2020-04-15) +--------------- + +- New ``table.rows_where(..., order_by="age desc")`` argument, see :ref:`python_api_rows`. (`#76 `__) + .. _v2_5: 2.5 (2020-04-12) diff --git a/setup.py b/setup.py index 3bec907..af6f637 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.5" +VERSION = "2.6" def get_long_description(): From 31d3df0f798db16394fd662e42206cdf768ded12 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2020 15:21:40 -0700 Subject: [PATCH 0105/1004] Typo fix --- docs/python-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 51f05fb..8a2ad23 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -115,7 +115,7 @@ You can filter rows by a WHERE clause using ``.rows_where(where, where_args)``:: ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} -To specify an order, use the ``order_my=`` argument:: +To specify an order, use the ``order_by=`` argument:: >>> for row in db["dogs"].rows_where("age > 1", order_by="age"): ... print(row) From cd146bbbfa43c501adb9edd57f231c653aaa2397 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 17 Apr 2020 10:58:08 -0700 Subject: [PATCH 0106/1004] Fixed RST underlines --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3f315b2..19298ad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,14 +5,14 @@ .. _v2_6: 2.6 (2020-04-15) ---------------- +---------------- - New ``table.rows_where(..., order_by="age desc")`` argument, see :ref:`python_api_rows`. (`#76 `__) .. _v2_5: 2.5 (2020-04-12) ---------------- +---------------- - Panda's Timestamp is now stored as a SQLite TEXT column. Thanks, b0b5h4rp13! (`#96 `__) - ``table.last_pk`` is now only available for inserts or upserts of a single record. (`#98 `__) From 2ab62bcd54f8aa73a7a77d209133f0da6c73f3ea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 17 Apr 2020 16:53:25 -0700 Subject: [PATCH 0107/1004] New columns= parameter for over-riding column types, closes #100 --- docs/python-api.rst | 32 ++++++++++++++++++++++++++++++-- sqlite_utils/db.py | 14 +++++++++++++- tests/test_create.py | 23 +++++++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 8a2ad23..e3c40e6 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -157,6 +157,8 @@ If the record does not exist a ``NotFoundError`` will be raised: except NotFoundError: print("Dog not found") +.. _python_api_creating_tables: + Creating tables =============== @@ -197,6 +199,15 @@ You can also specify a primary key by passing the ``pk=`` parameter to the ``.in "is_good_dog": True, }, pk="id") +After inserting a row like this, the ``dogs.last_rowid`` property will return the SQLite ``rowid`` assigned to the most recently inserted record. + +The ``dogs.last_pk`` property will return the last inserted primary key value, if you specified one. This can be very useful when writing code that creates foreign keys or many-to-many relationships. + +.. _python_api_custom_columns: + +Custom column order and column types +------------------------------------ + The order of the columns in the table will be derived from the order of the keys in the dictionary, provided you are using Python 3.6 or later. If you want to explicitly set the order of the columns you can do so using the ``column_order=`` parameter: @@ -213,9 +224,26 @@ If you want to explicitly set the order of the columns you can do so using the ` You don't need to pass all of the columns to the ``column_order`` parameter. If you only pass a subset of the columns the remaining columns will be ordered based on the key order of the dictionary. -After inserting a row like this, the ``dogs.last_rowid`` property will return the SQLite ``rowid`` assigned to the most recently inserted record. +Column types are detected based on the example data provided. Sometimes you may find you need to over-ride these detected types - to create an integer column for data that was provided as a string for example, or to ensure that a table where the first example was ``None`` is created as an ``INTEGER`` rather than a ``TEXT`` column. You can do this using the ``columns=`` parameter: -The ``dogs.last_pk`` property will return the last inserted primary key value, if you specified one. This can be very useful when writing code that creates foreign keys or many-to-many relationships. +.. code-block:: python + + dogs.insert({ + "id": 1, + "name": "Cleo", + "age": "5", + }, pk="id", columns={"age": int, "weight": float}) + +This will create a table with the following schema: + +.. code-block:: sql + + CREATE TABLE [dogs] ( + [id] INTEGER PRIMARY KEY, + [name] TEXT, + [age] INTEGER, + [weight] FLOAT + ) .. _python_api_explicit_create: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d92e92e..9e3987e 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -491,6 +491,7 @@ class Table(Queryable): replace=False, extracts=None, conversions=None, + columns=None, ): super().__init__(db, name) self._defaults = dict( @@ -506,6 +507,7 @@ class Table(Queryable): replace=replace, extracts=extracts, conversions=conversions or {}, + columns=columns, ) def __repr__(self): @@ -932,6 +934,7 @@ class Table(Queryable): replace=DEFAULT, extracts=DEFAULT, conversions=DEFAULT, + columns=DEFAULT, ): return self.insert_all( [record], @@ -946,6 +949,7 @@ class Table(Queryable): replace=replace, extracts=extracts, conversions=conversions, + columns=columns, ) def insert_all( @@ -963,6 +967,7 @@ class Table(Queryable): replace=DEFAULT, extracts=DEFAULT, conversions=DEFAULT, + columns=DEFAULT, upsert=False, ): """ @@ -982,6 +987,7 @@ class Table(Queryable): replace = self.value_or_default("replace", replace) extracts = self.value_or_default("extracts", extracts) conversions = self.value_or_default("conversions", conversions) + columns = self.value_or_default("columns", columns) if upsert and (not pk and not hash_id): raise PrimaryKeyRequired("upsert() requires a pk") @@ -1016,8 +1022,10 @@ class Table(Queryable): if first: if not self.exists(): # Use the first batch to derive the table names + column_types = suggest_column_types(chunk) + column_types.update(columns or {}) self.create( - suggest_column_types(chunk), + column_types, pk, foreign_keys, column_order=column_order, @@ -1154,6 +1162,7 @@ class Table(Queryable): alter=DEFAULT, extracts=DEFAULT, conversions=DEFAULT, + columns=DEFAULT, ): return self.upsert_all( [record], @@ -1166,6 +1175,7 @@ class Table(Queryable): alter=alter, extracts=extracts, conversions=conversions, + columns=columns, ) def upsert_all( @@ -1181,6 +1191,7 @@ class Table(Queryable): alter=DEFAULT, extracts=DEFAULT, conversions=DEFAULT, + columns=DEFAULT, ): return self.insert_all( records, @@ -1194,6 +1205,7 @@ class Table(Queryable): alter=alter, extracts=extracts, conversions=conversions, + columns=columns, upsert=True, ) diff --git a/tests/test_create.py b/tests/test_create.py index 7edcd8d..95b64ab 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -154,6 +154,29 @@ def test_create_table_from_example_with_compound_primary_keys(fresh_db): assert record == table.get(("staff", 2)) +@pytest.mark.parametrize( + "method_name", ("insert", "upsert", "insert_all", "upsert_all") +) +def test_create_table_with_custom_columns(fresh_db, method_name): + table = fresh_db["dogs"] + method = getattr(table, method_name) + record = {"id": 1, "name": "Cleo", "age": "5"} + if method_name.endswith("_all"): + record = [record] + method(record, pk="id", columns={"age": int, "weight": float}) + assert ["dogs"] == fresh_db.table_names() + expected_columns = [ + {"name": "id", "type": "INTEGER"}, + {"name": "name", "type": "TEXT"}, + {"name": "age", "type": "INTEGER"}, + {"name": "weight", "type": "FLOAT"}, + ] + assert expected_columns == [ + {"name": col.name, "type": col.type} for col in table.columns + ] + assert [{"id": 1, "name": "Cleo", "age": 5, "weight": None}] == list(table.rows) + + @pytest.mark.parametrize("use_table_factory", [True, False]) def test_create_table_column_order(fresh_db, use_table_factory): row = collections.OrderedDict( From 7ce07705ed5ba4f54015cb1a5cea4b97c54bd45f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 17 Apr 2020 16:59:47 -0700 Subject: [PATCH 0108/1004] Improved README. Fixes #101 --- README.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1e41cfb..b7a100d 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,12 @@ Python CLI utility and library for manipulating SQLite databases. Read more on my blog: [ sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) -Install it like this: +## Installation pip3 install sqlite-utils +## Using as a CLI tool + Now you can do things with the CLI utility like this: $ sqlite-utils tables dogs.db --counts @@ -34,7 +36,16 @@ Now you can do things with the CLI utility like this: 1 4 Cleo 2 2 Pancakes -Or you can import it and use it as a Python library like this: +You can even import data into a new database table like this: + + $ curl https://api.github.com/repos/simonw/sqlite-utils/releases \ + | sqlite-utils insert releases.db releases - --pk + +Full CLI documentation: https://sqlite-utils.readthedocs.io/en/stable/cli.html + +## Using as a library + +You can also `import sqlite_utils` and use it as a Python library like this: ```python import sqlite_utils @@ -46,10 +57,11 @@ db["dogs"].insert_all([ ], pk="id") ``` -Full documentation: https://sqlite-utils.readthedocs.io/ +Full library documentation: https://sqlite-utils.readthedocs.io/en/stable/python-api.html -Related projects: +## Related projects * [Datasette](https://github.com/simonw/datasette): A tool for exploring and publishing data * [csvs-to-sqlite](https://github.com/simonw/csvs-to-sqlite): Convert CSV files into a SQLite database * [db-to-sqlite](https://github.com/simonw/db-to-sqlite): CLI tool for exporting a MySQL or PostgreSQL database as a SQLite file +* [dogsheep](https://dogsheep.github.io/): A family of tools for personal analytics, built on top of `sqlite-utils` From f58f7464243f75367da2ad3cab240246e6e2e618 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 17 Apr 2020 17:04:50 -0700 Subject: [PATCH 0109/1004] Release 2.7 - refs #100 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 19298ad..43cfcc5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_7: + +2.7 (2020-04-17) +---------------- + +- New ``columns=`` argument for the ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()`` methods, for over-riding the auto-detected types for columns and specifying additional columns that should be added when the table is created. See :ref:`python_api_custom_columns`. (`#100 `__) + .. _v2_6: 2.6 (2020-04-15) diff --git a/setup.py b/setup.py index af6f637..90ccde8 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.6" +VERSION = "2.7" def get_long_description(): From 2b40710e9d05ae2dd7ec2301b0054bf25eb3c085 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 30 Apr 2020 11:32:23 -0700 Subject: [PATCH 0110/1004] Changelog badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b7a100d..1757802 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # sqlite-utils [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/) +[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.readthedocs.io/en/stable/changelog.html) [![Travis CI](https://travis-ci.com/simonw/sqlite-utils.svg?branch=master)](https://travis-ci.com/simonw/sqlite-utils) [![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.readthedocs.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/master/LICENSE) From 147b52622d68473ba6ab184657258d8576100b05 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 May 2020 10:09:36 -0700 Subject: [PATCH 0111/1004] sqlite-utils tables ... --schema option, closes #104 --- docs/cli.rst | 14 +++++++++++++- sqlite_utils/cli.py | 10 +++++++++- tests/test_cli.py | 12 ++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index f09c47e..be9970f 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -162,7 +162,7 @@ You can list the names of tables in a database using the ``tables`` subcommand:: {"table": "cats"}, {"table": "chickens"}] -You can output this list in CSV using the ``-csv`` option:: +You can output this list in CSV using the ``--csv`` option:: $ sqlite-utils tables mydb.db --csv --no-headers dogs @@ -188,6 +188,18 @@ Use ``--columns`` to include a list of columns in each table:: {"table": "Gosh2", "count": 0, "columns": ["c1", "c2", "c3"]}, {"table": "dogs", "count": 2, "columns": ["id", "age", "name"]}] +Use ``--schema`` to include the schema of each table:: + + $ sqlite-utils tables dogs.db --schema --table + table schema + ------- ----------------------------------------------- + Gosh CREATE TABLE Gosh (c1 text, c2 text, c3 text) + Gosh2 CREATE TABLE Gosh2 (c1 text, c2 text, c3 text) + dogs CREATE TABLE [dogs] ( + [id] INTEGER, + [age] INTEGER, + [name] TEXT) + The ``--nl``, ``--csv`` and ``--table`` options are all available. .. _cli_inserting_data: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 54667e5..f53bcd0 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -77,6 +77,9 @@ def cli(): is_flag=True, default=False, ) +@click.option( + "--schema", help="Include schema for each table", is_flag=True, default=False, +) def tables( path, fts4, @@ -88,8 +91,9 @@ def tables( no_headers, table, fmt, - columns, json_cols, + columns, + schema, ): """List the tables in the database""" db = sqlite_utils.Database(path) @@ -98,6 +102,8 @@ def tables( headers.append("count") if columns: headers.append("columns") + if schema: + headers.append("schema") def _iter(): for name in db.table_names(fts4=fts4, fts5=fts5): @@ -110,6 +116,8 @@ def tables( row.append("\n".join(cols)) else: row.append(cols) + if schema: + row.append(db[name].schema) yield row if table: diff --git a/tests/test_cli.py b/tests/test_cli.py index 52c50da..0bcefff 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -72,6 +72,18 @@ def test_tables_counts_and_columns_csv(db_path): ) == result.output.strip() +def test_tables_schema(db_path): + db = Database(db_path) + with db.conn: + db["lots"].insert_all([{"id": i, "age": i + 1} for i in range(30)]) + result = CliRunner().invoke(cli.cli, ["tables", "--schema", db_path]) + assert ( + '[{"table": "Gosh", "schema": "CREATE TABLE Gosh (c1 text, c2 text, c3 text)"},\n' + ' {"table": "Gosh2", "schema": "CREATE TABLE Gosh2 (c1 text, c2 text, c3 text)"},\n' + ' {"table": "lots", "schema": "CREATE TABLE [lots] (\\n [id] INTEGER,\\n [age] INTEGER\\n)"}]' + ) == result.output.strip() + + @pytest.mark.parametrize( "fmt,expected", [ From 344e9573ca1cf7c59482af21a0a517bdae70f7d5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 May 2020 13:38:28 -0700 Subject: [PATCH 0112/1004] Added sqlite-utils views command, closes #105 --- docs/cli.rst | 20 ++++++++++++++++++ sqlite_utils/cli.py | 50 +++++++++++++++++++++++++++++++++++++++++++-- tests/test_cli.py | 10 +++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index be9970f..eca3df3 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -202,6 +202,26 @@ Use ``--schema`` to include the schema of each table:: The ``--nl``, ``--csv`` and ``--table`` options are all available. +Listing views +============= + +The `views` command shows any views defined in the database:: + + $ sqlite-utils views sf-trees.db --table --counts --columns --schema + view count columns schema + --------- ------- -------------------- -------------------------------------------------------------- + demo_view 189144 ['qSpecies'] CREATE VIEW demo_view AS select qSpecies from Street_Tree_List + hello 1 ['sqlite_version()'] CREATE VIEW hello as select sqlite_version() + +It takes the same options as the ``tables`` command: + +* ``--columns`` +* ``--schema`` +* ``--counts`` +* ``--nl`` +* ``--csv`` +* ``--table`` + .. _cli_inserting_data: Inserting JSON data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index f53bcd0..14aa338 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -94,10 +94,11 @@ def tables( json_cols, columns, schema, + views=False, ): """List the tables in the database""" db = sqlite_utils.Database(path) - headers = ["table"] + headers = ["view" if views else "table"] if counts: headers.append("count") if columns: @@ -106,7 +107,11 @@ def tables( headers.append("schema") def _iter(): - for name in db.table_names(fts4=fts4, fts5=fts5): + if views: + items = db.view_names() + else: + items = db.table_names(fts4=fts4, fts5=fts5) + for name in items: row = [name] if counts: row.append(db[name].count) @@ -133,6 +138,47 @@ def tables( click.echo(line) +@cli.command() +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.option( + "--counts", help="Include row counts per view", default=False, is_flag=True +) +@output_options +@click.option( + "--columns", + help="Include list of columns for each view", + is_flag=True, + default=False, +) +@click.option( + "--schema", help="Include schema for each view", is_flag=True, default=False, +) +def views( + path, counts, nl, arrays, csv, no_headers, table, fmt, json_cols, columns, schema, +): + """List the views in the database""" + tables.callback( + path=path, + fts4=False, + fts5=False, + counts=counts, + nl=nl, + arrays=arrays, + csv=csv, + no_headers=no_headers, + table=table, + fmt=fmt, + json_cols=json_cols, + columns=columns, + schema=schema, + views=True, + ) + + @cli.command() @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index 0bcefff..fe6bbb2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -28,6 +28,16 @@ def test_tables(db_path): assert '[{"table": "Gosh"},\n {"table": "Gosh2"}]' == result.output.strip() +def test_views(db_path): + Database(db_path).create_view("hello", "select sqlite_version()") + result = CliRunner().invoke(cli.cli, ["views", db_path, "--table", "--schema"]) + assert ( + "view schema\n" + "------ --------------------------------------------\n" + "hello CREATE VIEW hello AS select sqlite_version()" + ) == result.output.strip() + + def test_tables_fts4(db_path): Database(db_path)["Gosh"].enable_fts(["c2"], fts_version="FTS4") result = CliRunner().invoke(cli.cli, ["tables", "--fts4", db_path]) From d56029549acae0b0ea94c5a0f783e3b3895d9218 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 May 2020 13:45:39 -0700 Subject: [PATCH 0113/1004] Serialize JSON with non-JSON values, closes #102 --- sqlite_utils/db.py | 6 ++++-- tests/test_create.py | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9e3987e..dd49d5c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1317,7 +1317,7 @@ def chunks(sequence, size): def jsonify_if_needed(value): if isinstance(value, (dict, list, tuple)): - return json.dumps(value) + return json.dumps(value, default=repr) elif isinstance(value, (datetime.time, datetime.date, datetime.datetime)): return value.isoformat() else: @@ -1326,7 +1326,9 @@ def jsonify_if_needed(value): def _hash(record): return hashlib.sha1( - json.dumps(record, separators=(",", ":"), sort_keys=True).encode("utf8") + json.dumps(record, separators=(",", ":"), sort_keys=True, default=repr).encode( + "utf8" + ) ).hexdigest() diff --git a/tests/test_create.py b/tests/test_create.py index 95b64ab..5290cd8 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -847,3 +847,9 @@ def test_create_with_a_null_column(fresh_db): record = {"name": "Name", "description": None} fresh_db["t"].insert(record) assert [record] == list(fresh_db["t"].rows) + + +def test_create_with_nested_bytes(fresh_db): + record = {"id": 1, "data": {"foo": b"bytes"}} + fresh_db["t"].insert(record) + assert [{"id": 1, "data": '{"foo": "b\'bytes\'"}'}] == list(fresh_db["t"].rows) From b4d953d3ccef28bb81cea40ca165a647b59971fa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 May 2020 15:08:37 -0700 Subject: [PATCH 0114/1004] Release 2.7.1, refs #102 #104 #105 --- docs/changelog.rst | 9 +++++++++ docs/cli.rst | 10 ++++++---- setup.py | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 43cfcc5..3434b8f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v2_7.1: + +2.7.1 (2020-05-01) +------------------ + +- New ``sqlite-utils views my.db`` command for listing views in a database, see :ref:`cli_views`. (`#105 `__) +- ``sqlite-utils tables`` (and ``views``) has a new ``--schema`` option which outputs the table/view schema, see :ref:`cli_tables`. (`#104 `__) +- Nested structures containing invalid JSON values (e.g. Python bytestrings) are now serialized using ``repr()`` instead of throwing an error. (`#102 `__) + .. _v2_7: 2.7 (2020-04-17) diff --git a/docs/cli.rst b/docs/cli.rst index eca3df3..ef218a0 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -17,7 +17,7 @@ You can execute a SQL query against a database and get the results back as JSON [{"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}] -This is the default subcommand for ``sqlite-utils``, so you can instead use this:: +This is the default command for ``sqlite-utils``, so you can instead use this:: $ sqlite-utils dogs.db "select * from dogs" @@ -142,7 +142,7 @@ For a full list of table format options, run ``sqlite-utils query --help``. Returning all rows in a table ============================= -You can return every row in a specified table using the ``rows`` subcommand:: +You can return every row in a specified table using the ``rows`` command:: $ sqlite-utils rows dogs.db dogs [{"id": 1, "age": 4, "name": "Cleo"}, @@ -155,7 +155,7 @@ This command accepts the same output options as ``query`` - so you can pass ``-- Listing tables ============== -You can list the names of tables in a database using the ``tables`` subcommand:: +You can list the names of tables in a database using the ``tables`` command:: $ sqlite-utils tables mydb.db [{"table": "dogs"}, @@ -202,6 +202,8 @@ Use ``--schema`` to include the schema of each table:: The ``--nl``, ``--csv`` and ``--table`` options are all available. +.. _cli_views: + Listing views ============= @@ -410,7 +412,7 @@ You can use the ``--not-null`` and ``--default`` options (to both ``insert`` and Creating indexes ================ -You can add an index to an existing table using the ``create-index`` subcommand:: +You can add an index to an existing table using the ``create-index`` command:: $ sqlite-utils create-index mydb.db mytable col1 [col2...] diff --git a/setup.py b/setup.py index 90ccde8..8e94913 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.7" +VERSION = "2.7.1" def get_long_description(): From 5c1df4e3063cf47229a305fbe75757a8d412a8af Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 May 2020 09:02:04 -0700 Subject: [PATCH 0115/1004] replace=True and ignore=True parameters for create_view(), closes #106 --- docs/python-api.rst | 10 +++++++++ sqlite_utils/db.py | 25 +++++++++++++++-------- tests/test_create.py | 7 ------- tests/test_create_view.py | 43 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 15 deletions(-) create mode 100644 tests/test_create_view.py diff --git a/docs/python-api.rst b/docs/python-api.rst index e3c40e6..90c5396 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -879,6 +879,16 @@ The ``.create_view()`` method on the database class can be used to create a view select * from dogs where is_good_dog = 1 """) +This will raise a ``sqlite_utils.utils.OperationalError`` if a view with that name already exists. + +You can pass ``ignore=True`` to silently ignore an existing view and do nothing, or ``replace=True`` to replace an existing view with a new definition if your select statement differs from the current view: + +.. code-block:: python + + db.create_view("good_dogs", """ + select * from dogs where is_good_dog = 1 + """, replace=True) + Storing JSON ============ diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index dd49d5c..cb249b3 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -322,14 +322,23 @@ class Database: hash_id=hash_id, ) - def create_view(self, name, sql): - self.conn.execute( - """ - CREATE VIEW {name} AS {sql} - """.format( - name=name, sql=sql - ) - ) + def create_view(self, name, sql, ignore=False, replace=False): + assert not ( + ignore and replace + ), "Use one or the other of ignore/replace, not both" + create_sql = "CREATE VIEW {name} AS {sql}".format(name=name, sql=sql) + if ignore or replace: + # Does view exist already? + if name in self.view_names(): + if ignore: + return self + elif replace: + # If SQL is the same, do nothing + if create_sql == self[name].schema: + return self + self[name].drop() + self.conn.execute(create_sql) + return self def m2m_table_candidates(self, table, other_table): "Returns potential m2m tables for arguments, based on FKs" diff --git a/tests/test_create.py b/tests/test_create.py index 5290cd8..79123b8 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -716,13 +716,6 @@ def test_insert_hash_id(fresh_db): assert 1 == dogs.count -def test_create_view(fresh_db): - fresh_db["data"].insert({"foo": "foo", "bar": "bar"}) - fresh_db.create_view("bar", "select bar from data") - rows = fresh_db.conn.execute("select * from bar").fetchall() - assert [("bar",)] == rows - - def test_vacuum(fresh_db): fresh_db["data"].insert({"foo": "foo", "bar": "bar"}) fresh_db.vacuum() diff --git a/tests/test_create_view.py b/tests/test_create_view.py new file mode 100644 index 0000000..26cf420 --- /dev/null +++ b/tests/test_create_view.py @@ -0,0 +1,43 @@ +import pytest +from sqlite_utils.utils import OperationalError + + +def test_create_view(fresh_db): + fresh_db.create_view("bar", "select 1 + 1") + rows = fresh_db.conn.execute("select * from bar").fetchall() + assert [(2,)] == rows + + +def test_create_view_error(fresh_db): + fresh_db.create_view("bar", "select 1 + 1") + with pytest.raises(OperationalError): + fresh_db.create_view("bar", "select 1 + 2") + + +def test_create_view_only_arrow_one_param(fresh_db): + with pytest.raises(AssertionError): + fresh_db.create_view("bar", "select 1 + 2", ignore=True, replace=True) + + +def test_create_view_ignore(fresh_db): + fresh_db.create_view("bar", "select 1 + 1").create_view( + "bar", "select 1 + 2", ignore=True + ) + rows = fresh_db.conn.execute("select * from bar").fetchall() + assert [(2,)] == rows + + +def test_create_view_replace(fresh_db): + fresh_db.create_view("bar", "select 1 + 1").create_view( + "bar", "select 1 + 2", replace=True + ) + rows = fresh_db.conn.execute("select * from bar").fetchall() + assert [(3,)] == rows + + +def test_create_view_replace_with_same_does_nothing(fresh_db): + fresh_db.create_view("bar", "select 1 + 1") + initial_version = fresh_db.conn.execute("PRAGMA schema_version").fetchone()[0] + fresh_db.create_view("bar", "select 1 + 1", replace=True) + after_version = fresh_db.conn.execute("PRAGMA schema_version").fetchone()[0] + assert after_version == initial_version From 79541d3a6d71a9f888628686c3236eb0205bad35 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 May 2020 09:05:27 -0700 Subject: [PATCH 0116/1004] Release 2.7.2, refs #106 --- docs/changelog.rst | 7 +++++++ docs/python-api.rst | 2 ++ setup.py | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3434b8f..344615e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_7.2: + +2.7.2 (2020-05-02) +------------------ + +- ``db.create_view(...)`` now has additional parameters ``ignore=True`` or ``replace=True``, see :ref:`python_api_create_view`. (`#106 `__) + .. _v2_7.1: 2.7.1 (2020-05-01) diff --git a/docs/python-api.rst b/docs/python-api.rst index 90c5396..ded39ce 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -868,6 +868,8 @@ If you are going to use that ID straight away, you can access it using ``last_pk }, hash_id="id").last_pk # dog_id is now "f501265970505d9825d8d9f590bfab3519fb20b1" +.. _python_api_create_view: + Creating views ============== diff --git a/setup.py b/setup.py index 8e94913..dc18a69 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.7.1" +VERSION = "2.7.2" def get_long_description(): From 36d256b047ecd77761d24fe570fa117bc6dc917c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 May 2020 20:55:40 -0700 Subject: [PATCH 0117/1004] Initial implementation of create-table command, refs #27 --- sqlite_utils/cli.py | 31 +++++++++++++++++++++++++++++++ tests/test_cli.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 14aa338..a6ad7ca 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -9,6 +9,8 @@ import csv as csv_std import tabulate from .utils import sqlite3 +VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB") + def output_options(fn): for decorator in reversed( @@ -528,6 +530,35 @@ def upsert( ) +@cli.command(name="create-table") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table") +@click.argument("columns", nargs=-1, required=True) +@click.option("--pk", help="Column to use as primary key") +def create_table(path, table, columns, pk): + "Add an index to the specified table covering the specified columns" + db = sqlite_utils.Database(path) + if len(columns) % 2 == 1: + raise click.ClickException( + "columns must be an even number of 'name' 'type' pairs" + ) + coltypes = {} + columns = list(columns) + while columns: + name = columns.pop(0) + ctype = columns.pop(0) + if ctype.upper() not in VALID_COLUMN_TYPES: + raise click.ClickException( + "column types must be one of {}".format(VALID_COLUMN_TYPES) + ) + coltypes[name] = ctype.upper() + db[table].create(coltypes, pk=pk) + + @cli.command() @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index fe6bbb2..d002ac2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -861,3 +861,39 @@ def test_upsert_alter(db_path, tmpdir): assert [{"id": 1, "name": "Cleo", "age": 5},] == db.execute_returning_dicts( "select * from dogs order by id" ) + + +def test_create_table(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + cli.cli, + [ + "create-table", + "test.db", + "t", + "id", + "integer", + "name", + "text", + "age", + "integer", + "weight", + "float", + "thumbnail", + "blob", + "--pk", + "id", + ], + ) + assert 0 == result.exit_code + db = Database("test.db") + assert ( + "CREATE TABLE [t] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [name] TEXT,\n" + " [age] INTEGER,\n" + " [weight] FLOAT,\n" + " [thumbnail] BLOB\n" + ")" + ) == db["t"].schema From 99a7906fd93ce1c6400733b855255ed62e3e9fa1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 May 2020 21:13:49 -0700 Subject: [PATCH 0118/1004] sqlite-utils create-table docs, plus doc unit test Refs #27. Closes #108 --- docs/cli.rst | 14 ++++++++++++++ tests/test_docs.py | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 tests/test_docs.py diff --git a/docs/cli.rst b/docs/cli.rst index ef218a0..cd5b6fd 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -334,6 +334,20 @@ The command will fail if you reference columns that do not exist on the table. T .. note:: ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 `__ for details of this change. + +.. _cli_create_table: + +Creating tables +=============== + +Most of the time creating tables by inserting example data is the quickest approach. If you need to create an empty table in advance of inserting data you can do so using the ``create-table`` command:: + + $ sqlite-utils create-table mydb.db mytable id integer name text --pk=id + +This will create a table called ``mytable`` with two columns - an integer ``id`` column and a text ``name`` column. It will set the ``id`` column to be the primary key. + +You can pass as many column-name column-type pairs as you like. Valid types are ``integer``, ``text``, ``float`` and ``blob``. + .. _cli_add_column: Adding columns diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 0000000..61f0ece --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,22 @@ +from sqlite_utils import cli +from pathlib import Path +import pytest +import re + +docs_path = Path(__file__).parent.parent / "docs" +commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+) ") + + +@pytest.fixture(scope="session") +def documented_commands(): + rst = (docs_path / "cli.rst").read_text() + return { + command + for command in commands_re.findall(rst) + if "." not in command and ":" not in command + } + + +@pytest.mark.parametrize("command", cli.cli.commands.keys()) +def test_plugin_hooks_are_documented(documented_commands, command): + assert command in documented_commands From 4d5916075d42b390302b8018d84eafdc709dadc8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 May 2020 08:09:00 -0700 Subject: [PATCH 0119/1004] create-table --not-null, --default, --fk, refs #27 --- docs/cli.rst | 51 +++++++++++++++++++++++++ sqlite_utils/cli.py | 21 ++++++++++- tests/test_cli.py | 91 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 147 insertions(+), 16 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index cd5b6fd..6aa2833 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -348,6 +348,57 @@ This will create a table called ``mytable`` with two columns - an integer ``id`` You can pass as many column-name column-type pairs as you like. Valid types are ``integer``, ``text``, ``float`` and ``blob``. +You can specify columns that should be NOT NULL using ``--not-null colname``. You can specify default values for columns using ``--default colname defaultvalue``. + +:: + + $ sqlite-utils create-table mydb.db mytable \ + id integer \ + name text \ + age integer \ + is_good integer \ + --not-null name \ + --not-null age \ + --default is_good 1 \ + --pk=id + + $ sqlite-utils tables mydb.db --schema -t + table schema + ------- -------------------------------- + mytable CREATE TABLE [mytable] ( + [id] INTEGER PRIMARY KEY, + [name] TEXT NOT NULL, + [age] INTEGER NOT NULL, + [is_good] INTEGER DEFAULT '1' + ) + +You can specify foreign key relationships between the tables you are creating using ``--fk colname othertable othercolumn``:: + + $ sqlite-utils create-table books.db authors \ + id integer \ + name text \ + --pk=id + + $ sqlite-utils create-table books.db books \ + id integer \ + title text \ + author_id integer \ + --pk=id \ + --fk author_id authors id + + $ sqlite-utils tables books.db --schema -t + table schema + ------- ------------------------------------------------- + authors CREATE TABLE [authors] ( + [id] INTEGER PRIMARY KEY, + [name] TEXT + ) + books CREATE TABLE [books] ( + [id] INTEGER PRIMARY KEY, + [title] TEXT, + [author_id] INTEGER REFERENCES [authors]([id]) + ) + .. _cli_add_column: Adding columns diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index a6ad7ca..a29c3af 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -539,7 +539,22 @@ def upsert( @click.argument("table") @click.argument("columns", nargs=-1, required=True) @click.option("--pk", help="Column to use as primary key") -def create_table(path, table, columns, pk): +@click.option( + "--not-null", multiple=True, help="Columns that should be created as NOT NULL", +) +@click.option( + "--default", + multiple=True, + type=(str, str), + help="Default value that should be set for a column", +) +@click.option( + "--fk", + multiple=True, + type=(str, str, str), + help="Column, other table, other column to set as a foreign key", +) +def create_table(path, table, columns, pk, not_null, default, fk): "Add an index to the specified table covering the specified columns" db = sqlite_utils.Database(path) if len(columns) % 2 == 1: @@ -556,7 +571,9 @@ def create_table(path, table, columns, pk): "column types must be one of {}".format(VALID_COLUMN_TYPES) ) coltypes[name] = ctype.upper() - db[table].create(coltypes, pk=pk) + db[table].create( + coltypes, pk=pk, not_null=not_null, defaults=dict(default), foreign_keys=fk + ) @cli.command() diff --git a/tests/test_cli.py b/tests/test_cli.py index d002ac2..36e9b72 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -863,15 +863,17 @@ def test_upsert_alter(db_path, tmpdir): ) -def test_create_table(): - runner = CliRunner() - with runner.isolated_filesystem(): - result = runner.invoke( - cli.cli, +@pytest.mark.parametrize( + "args,schema", + [ + # No primary key + ( + ["name", "text", "age", "integer",], + ("CREATE TABLE [t] (\n [name] TEXT,\n [age] INTEGER\n)"), + ), + # All types: + ( [ - "create-table", - "test.db", - "t", "id", "integer", "name", @@ -885,15 +887,76 @@ def test_create_table(): "--pk", "id", ], + ( + "CREATE TABLE [t] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [name] TEXT,\n" + " [age] INTEGER,\n" + " [weight] FLOAT,\n" + " [thumbnail] BLOB\n" + ")" + ), + ), + # Not null: + ( + ["name", "text", "--not-null", "name"], + ("CREATE TABLE [t] (\n" " [name] TEXT NOT NULL\n" ")"), + ), + # Default: + ( + ["age", "integer", "--default", "age", "3"], + ("CREATE TABLE [t] (\n" " [age] INTEGER DEFAULT '3'\n" ")"), + ), + ], +) +def test_create_table(args, schema): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + cli.cli, ["create-table", "test.db", "t",] + args, catch_exceptions=False ) assert 0 == result.exit_code db = Database("test.db") + assert schema == db["t"].schema + + +def test_create_table_foreign_key(): + runner = CliRunner() + creates = ( + ["authors", "id", "integer", "name", "text", "--pk", "id"], + [ + "books", + "id", + "integer", + "title", + "text", + "author_id", + "integer", + "--pk", + "id", + "--fk", + "author_id", + "authors", + "id", + ], + ) + with runner.isolated_filesystem(): + for args in creates: + result = runner.invoke( + cli.cli, ["create-table", "books.db"] + args, catch_exceptions=False + ) + assert 0 == result.exit_code + db = Database("books.db") assert ( - "CREATE TABLE [t] (\n" + "CREATE TABLE [authors] (\n" " [id] INTEGER PRIMARY KEY,\n" - " [name] TEXT,\n" - " [age] INTEGER,\n" - " [weight] FLOAT,\n" - " [thumbnail] BLOB\n" + " [name] TEXT\n" ")" - ) == db["t"].schema + ) == db["authors"].schema + assert ( + "CREATE TABLE [books] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [title] TEXT,\n" + " [author_id] INTEGER REFERENCES [authors]([id])\n" + ")" + ) == db["books"].schema From 9f6085b4e4c8289b34c6a3d40ba72d77ed62b4ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 May 2020 08:24:39 -0700 Subject: [PATCH 0120/1004] create-table --ignore and --replace, refs #27 --- docs/cli.rst | 2 ++ docs/python-api.rst | 2 ++ sqlite_utils/cli.py | 20 +++++++++++++++++++- tests/test_cli.py | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 6aa2833..7c0c9c5 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -399,6 +399,8 @@ You can specify foreign key relationships between the tables you are creating us [author_id] INTEGER REFERENCES [authors]([id]) ) +If a table with the same name already exists, you will get an error. You can choose to silently ignore this error with ``--ignore``, or you can replace the existing table with a new, empty table using ``--replace``. + .. _cli_add_column: Adding columns diff --git a/docs/python-api.rst b/docs/python-api.rst index ded39ce..148f794 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -262,6 +262,8 @@ The first argument here is a dictionary specifying the columns you would like to This method takes optional arguments ``pk=``, ``column_order=``, ``foreign_keys=``, ``not_null=set()`` and ``defaults=dict()`` - explained below. +You can pass ``ignore=True`` to silently ignore an existing table and do nothing, or ``replace=True`` to replace that table with a new, empty table. + .. _python_api_compound_primary_keys: Compound primary keys diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index a29c3af..8a7fabb 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -554,7 +554,13 @@ def upsert( type=(str, str, str), help="Column, other table, other column to set as a foreign key", ) -def create_table(path, table, columns, pk, not_null, default, fk): +@click.option( + "--ignore", is_flag=True, help="If table already exists, do nothing", +) +@click.option( + "--replace", is_flag=True, help="If table already exists, replace it", +) +def create_table(path, table, columns, pk, not_null, default, fk, ignore, replace): "Add an index to the specified table covering the specified columns" db = sqlite_utils.Database(path) if len(columns) % 2 == 1: @@ -571,6 +577,18 @@ def create_table(path, table, columns, pk, not_null, default, fk): "column types must be one of {}".format(VALID_COLUMN_TYPES) ) coltypes[name] = ctype.upper() + # Does table already exist? + if table in db.table_names(): + if ignore: + return + elif replace: + db[table].drop() + else: + raise click.ClickException( + 'Table "{}" already exists. Use --replace to delete and replace it.'.format( + table + ) + ) db[table].create( coltypes, pk=pk, not_null=not_null, defaults=dict(default), foreign_keys=fk ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 36e9b72..7d653e1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -960,3 +960,42 @@ def test_create_table_foreign_key(): " [author_id] INTEGER REFERENCES [authors]([id])\n" ")" ) == db["books"].schema + + +def test_create_table_error_if_table_exists(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db["dogs"].insert({"name": "Cleo"}) + result = runner.invoke( + cli.cli, ["create-table", "test.db", "dogs", "id", "integer"] + ) + assert 1 == result.exit_code + assert ( + 'Error: Table "dogs" already exists. Use --replace to delete and replace it.' + == result.output.strip() + ) + + +def test_create_table_ignore(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db["dogs"].insert({"name": "Cleo"}) + result = runner.invoke( + cli.cli, ["create-table", "test.db", "dogs", "id", "integer", "--ignore"] + ) + assert 0 == result.exit_code + assert "CREATE TABLE [dogs] (\n [name] TEXT\n)" == db["dogs"].schema + + +def test_create_table_replace(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db["dogs"].insert({"name": "Cleo"}) + result = runner.invoke( + cli.cli, ["create-table", "test.db", "dogs", "id", "integer", "--replace"] + ) + assert 0 == result.exit_code + assert "CREATE TABLE [dogs] (\n [id] INTEGER\n)" == db["dogs"].schema From 78264b738cd72ffad6e5c32ede3f074f8aad0ca4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 May 2020 08:25:21 -0700 Subject: [PATCH 0121/1004] Removed docs for feature I decided not to implement, refs #27 --- docs/python-api.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 148f794..ded39ce 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -262,8 +262,6 @@ The first argument here is a dictionary specifying the columns you would like to This method takes optional arguments ``pk=``, ``column_order=``, ``foreign_keys=``, ``not_null=set()`` and ``defaults=dict()`` - explained below. -You can pass ``ignore=True`` to silently ignore an existing table and do nothing, or ``replace=True`` to replace that table with a new, empty table. - .. _python_api_compound_primary_keys: Compound primary keys From d16097231c5e51ea857b58c700f97a17b68dc583 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 May 2020 08:36:29 -0700 Subject: [PATCH 0122/1004] Added sqlite-utils create-view command, closes #107 --- docs/cli.rst | 15 +++++++++++ sqlite_utils/cli.py | 32 ++++++++++++++++++++++ tests/test_cli.py | 66 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 7c0c9c5..40ac963 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -401,6 +401,21 @@ You can specify foreign key relationships between the tables you are creating us If a table with the same name already exists, you will get an error. You can choose to silently ignore this error with ``--ignore``, or you can replace the existing table with a new, empty table using ``--replace``. + +.. _cli_create_view: + +Creating views +============== + +You can create a view using the ``create-view`` command:: + + $ sqlite-utils create-view mydb.db version "select sqlite_version()" + + $ sqlite-utils mydb.db "select * from version" + [{"sqlite_version()": "3.31.1"}] + +Use ``--replace`` to replace an existing view of the same name, and ``--ignore`` to do nothing if a view already exists. + .. _cli_add_column: Adding columns diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8a7fabb..f1b5ac6 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -594,6 +594,38 @@ def create_table(path, table, columns, pk, not_null, default, fk, ignore, replac ) +@cli.command(name="create-view") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("view") +@click.argument("select") +@click.option( + "--ignore", is_flag=True, help="If view already exists, do nothing", +) +@click.option( + "--replace", is_flag=True, help="If view already exists, replace it", +) +def create_view(path, view, select, ignore, replace): + "Create a view for the provided SELECT query" + db = sqlite_utils.Database(path) + # Does view already exist? + if view in db.view_names(): + if ignore: + return + elif replace: + db[view].drop() + else: + raise click.ClickException( + 'View "{}" already exists. Use --replace to delete and replace it.'.format( + view + ) + ) + db.create_view(view, select) + + @cli.command() @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index 7d653e1..32ea1db 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -999,3 +999,69 @@ def test_create_table_replace(): ) assert 0 == result.exit_code assert "CREATE TABLE [dogs] (\n [id] INTEGER\n)" == db["dogs"].schema + + +def test_create_view(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + result = runner.invoke( + cli.cli, ["create-view", "test.db", "version", "select sqlite_version()"] + ) + assert 0 == result.exit_code + assert "CREATE VIEW version AS select sqlite_version()" == db["version"].schema + + +def test_create_view_error_if_view_exists(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db.create_view("version", "select sqlite_version() + 1") + result = runner.invoke( + cli.cli, ["create-view", "test.db", "version", "select sqlite_version()"] + ) + assert 1 == result.exit_code + assert ( + 'Error: View "version" already exists. Use --replace to delete and replace it.' + == result.output.strip() + ) + + +def test_create_view_ignore(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db.create_view("version", "select sqlite_version() + 1") + result = runner.invoke( + cli.cli, + [ + "create-view", + "test.db", + "version", + "select sqlite_version()", + "--ignore", + ], + ) + assert 0 == result.exit_code + assert ( + "CREATE VIEW version AS select sqlite_version() + 1" == db["version"].schema + ) + + +def test_create_view_replace(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db.create_view("version", "select sqlite_version() + 1") + result = runner.invoke( + cli.cli, + [ + "create-view", + "test.db", + "version", + "select sqlite_version()", + "--replace", + ], + ) + assert 0 == result.exit_code + assert "CREATE VIEW version AS select sqlite_version()" == db["version"].schema From 30a390780aa1946f9430d7f473031e95ab02675d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 May 2020 08:39:50 -0700 Subject: [PATCH 0123/1004] Release 2.8, refs #27 and #107 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 344615e..02e4014 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v2_8: + +2.8 (2020-05-03) +---------------- + +- New ``sqlite-utils create-table`` command, see :ref:`cli_create_table`. (`#27 `__) +- New ``sqlite-utils create-view`` command, see :ref:`cli_create_view`. (`#107 `__) + .. _v2_7.2: 2.7.2 (2020-05-02) diff --git a/setup.py b/setup.py index dc18a69..88e385b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.7.2" +VERSION = "2.8" def get_long_description(): From 60e380e551b44028af0d73ac30c99a8bb04b458b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 May 2020 08:44:41 -0700 Subject: [PATCH 0124/1004] Add badges to documentation index --- docs/index.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index f05532f..9b0ace7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,6 +2,19 @@ sqlite-utils |version| ======================= +|PyPI| |Changelog| |Travis CI| |Documentation Status| |License| + +.. |PyPI| image:: https://img.shields.io/pypi/v/sqlite-utils.svg + :target: https://pypi.org/project/sqlite-utils/ +.. |Changelog| image:: https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog + :target: https://sqlite-utils.readthedocs.io/en/stable/changelog.html +.. |Travis CI| image:: https://travis-ci.com/simonw/sqlite-utils.svg?branch=master + :target: https://travis-ci.com/simonw/sqlite-utils +.. |Documentation Status| image:: https://readthedocs.org/projects/sqlite-utils/badge/?version=latest + :target: http://sqlite-utils.readthedocs.io/en/latest/?badge=latest +.. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg + :target: https://github.com/simonw/sqlite-utils/blob/master/LICENSE + *Python utility functions for manipulating SQLite databases* This library and command-line utility helps create SQLite databases from an existing collection of data. From 396bee92364fc3a88f6c76969366dd1c4c9c944d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 May 2020 08:47:28 -0700 Subject: [PATCH 0125/1004] Don't show documentation badge on docs index --- docs/index.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 9b0ace7..35ffcb0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ sqlite-utils |version| ======================= -|PyPI| |Changelog| |Travis CI| |Documentation Status| |License| +|PyPI| |Changelog| |Travis CI| |License| .. |PyPI| image:: https://img.shields.io/pypi/v/sqlite-utils.svg :target: https://pypi.org/project/sqlite-utils/ @@ -10,8 +10,6 @@ :target: https://sqlite-utils.readthedocs.io/en/stable/changelog.html .. |Travis CI| image:: https://travis-ci.com/simonw/sqlite-utils.svg?branch=master :target: https://travis-ci.com/simonw/sqlite-utils -.. |Documentation Status| image:: https://readthedocs.org/projects/sqlite-utils/badge/?version=latest - :target: http://sqlite-utils.readthedocs.io/en/latest/?badge=latest .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg :target: https://github.com/simonw/sqlite-utils/blob/master/LICENSE From 4e9cb739c757948b63e7bffaf4d8cfed0dd5de23 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 10 May 2020 17:44:21 -0700 Subject: [PATCH 0126/1004] drop-table and drop-view commands, closes #111 --- docs/cli.rst | 17 +++++++++++ sqlite_utils/cli.py | 33 +++++++++++++++++++++ tests/test_cli.py | 70 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 40ac963..119e49d 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -401,6 +401,14 @@ You can specify foreign key relationships between the tables you are creating us If a table with the same name already exists, you will get an error. You can choose to silently ignore this error with ``--ignore``, or you can replace the existing table with a new, empty table using ``--replace``. +.. _cli_drop_table: + +Dropping tables +=============== + +You can drop a table using the ``drop-table`` command:: + + $ sqlite-utils drop-table mytable .. _cli_create_view: @@ -416,6 +424,15 @@ You can create a view using the ``create-view`` command:: Use ``--replace`` to replace an existing view of the same name, and ``--ignore`` to do nothing if a view already exists. +.. _cli_drop_view: + +Dropping views +============== + +You can drop a view using the ``drop-view`` command:: + + $ sqlite-utils drop-view myview + .. _cli_add_column: Adding columns diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index f1b5ac6..fd74499 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -594,6 +594,22 @@ def create_table(path, table, columns, pk, not_null, default, fk, ignore, replac ) +@cli.command(name="drop-table") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table") +def drop_table(path, table): + "Drop the specified table" + db = sqlite_utils.Database(path) + if table in db.table_names(): + db[table].drop() + else: + raise click.ClickException('Table "{}" does not exist'.format(table)) + + @cli.command(name="create-view") @click.argument( "path", @@ -626,6 +642,23 @@ def create_view(path, view, select, ignore, replace): db.create_view(view, select) +@cli.command(name="drop-view") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("view") +def drop_view(path, view): + "Drop the specified view" + db = sqlite_utils.Database(path) + if view in db.view_names(): + db[view].drop() + else: + raise click.ClickException('View "{}" does not exist'.format(view)) + + + @cli.command() @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index 32ea1db..e7ded29 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1065,3 +1065,73 @@ def test_create_view_replace(): ) assert 0 == result.exit_code assert "CREATE VIEW version AS select sqlite_version()" == db["version"].schema + + +def test_drop_table(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db["t"].create({"pk": int}, pk="pk") + assert "t" in db.table_names() + result = runner.invoke( + cli.cli, + [ + "drop-table", + "test.db", + "t", + ], + ) + assert 0 == result.exit_code + assert "t" not in db.table_names() + + +def test_drop_table_error(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db["t"].create({"pk": int}, pk="pk") + result = runner.invoke( + cli.cli, + [ + "drop-table", + "test.db", + "t2", + ], + ) + assert 1 == result.exit_code + assert 'Error: Table "t2" does not exist' == result.output.strip() + + +def test_drop_view(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db.create_view("hello", "select 1") + assert "hello" in db.view_names() + result = runner.invoke( + cli.cli, + [ + "drop-view", + "test.db", + "hello", + ], + ) + assert 0 == result.exit_code + assert "hello" not in db.view_names() + + +def test_drop_view_error(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db["t"].create({"pk": int}, pk="pk") + result = runner.invoke( + cli.cli, + [ + "drop-view", + "test.db", + "t2", + ], + ) + assert 1 == result.exit_code + assert 'Error: View "t2" does not exist' == result.output.strip() From 98019e92d0f27efa87e844b89876344ff5403c8b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 10 May 2020 18:26:16 -0700 Subject: [PATCH 0127/1004] Ran black, refs #111 --- sqlite_utils/cli.py | 1 - tests/test_cli.py | 36 ++++-------------------------------- 2 files changed, 4 insertions(+), 33 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index fd74499..4cbb13e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -658,7 +658,6 @@ def drop_view(path, view): raise click.ClickException('View "{}" does not exist'.format(view)) - @cli.command() @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index e7ded29..9cd5965 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1073,14 +1073,7 @@ def test_drop_table(): db = Database("test.db") db["t"].create({"pk": int}, pk="pk") assert "t" in db.table_names() - result = runner.invoke( - cli.cli, - [ - "drop-table", - "test.db", - "t", - ], - ) + result = runner.invoke(cli.cli, ["drop-table", "test.db", "t",],) assert 0 == result.exit_code assert "t" not in db.table_names() @@ -1090,14 +1083,7 @@ def test_drop_table_error(): with runner.isolated_filesystem(): db = Database("test.db") db["t"].create({"pk": int}, pk="pk") - result = runner.invoke( - cli.cli, - [ - "drop-table", - "test.db", - "t2", - ], - ) + result = runner.invoke(cli.cli, ["drop-table", "test.db", "t2",],) assert 1 == result.exit_code assert 'Error: Table "t2" does not exist' == result.output.strip() @@ -1108,14 +1094,7 @@ def test_drop_view(): db = Database("test.db") db.create_view("hello", "select 1") assert "hello" in db.view_names() - result = runner.invoke( - cli.cli, - [ - "drop-view", - "test.db", - "hello", - ], - ) + result = runner.invoke(cli.cli, ["drop-view", "test.db", "hello",],) assert 0 == result.exit_code assert "hello" not in db.view_names() @@ -1125,13 +1104,6 @@ def test_drop_view_error(): with runner.isolated_filesystem(): db = Database("test.db") db["t"].create({"pk": int}, pk="pk") - result = runner.invoke( - cli.cli, - [ - "drop-view", - "test.db", - "t2", - ], - ) + result = runner.invoke(cli.cli, ["drop-view", "test.db", "t2",],) assert 1 == result.exit_code assert 'Error: View "t2" does not exist' == result.output.strip() From e8c57e09b60daf406761398d2712ea3ba9277542 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 10 May 2020 18:29:29 -0700 Subject: [PATCH 0128/1004] More things to .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 53605b7..634af83 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ venv .pytest_cache *.egg-info .DS_Store +.mypy_cache +.coverage +.schema +.vscode From af3f81b540923f2cf04c76cfa81b0d811c0084bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 10 May 2020 18:50:03 -0700 Subject: [PATCH 0129/1004] Store decimal.Decimal in DB as FLOAT, closes #110 --- sqlite_utils/db.py | 4 ++++ tests/test_create.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index cb249b3..edac5d9 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,6 +1,7 @@ from .utils import sqlite3, OperationalError, suggest_column_types, column_affinity from collections import namedtuple, OrderedDict import datetime +import decimal import hashlib import itertools import json @@ -42,6 +43,7 @@ COLUMN_TYPE_MAPPING = { datetime.datetime: "TEXT", datetime.date: "TEXT", datetime.time: "TEXT", + decimal.Decimal: "FLOAT", None.__class__: "TEXT", # SQLite explicit types "TEXT": "TEXT", @@ -1325,6 +1327,8 @@ def chunks(sequence, size): def jsonify_if_needed(value): + if isinstance(value, decimal.Decimal): + return float(value) if isinstance(value, (dict, list, tuple)): return json.dumps(value, default=repr) elif isinstance(value, (datetime.time, datetime.date, datetime.datetime)): diff --git a/tests/test_create.py b/tests/test_create.py index 79123b8..22e4b7b 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -9,6 +9,7 @@ from sqlite_utils.db import ( from sqlite_utils.utils import sqlite3 import collections import datetime +import decimal import json import pathlib import pytest @@ -132,6 +133,7 @@ def test_create_table_with_not_null(fresh_db): [{"name": "create", "type": "TEXT"}, {"name": "table", "type": "TEXT"}], ), ({"day": datetime.time(11, 0)}, [{"name": "day", "type": "TEXT"}]), + ({"decimal": decimal.Decimal("1.2")}, [{"name": "decimal", "type": "FLOAT"}]), ), ) def test_create_table_from_example(fresh_db, example, expected_columns): From 3936512edd34854f3290ffbac55d6ee0c673a36d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 10 May 2020 18:54:56 -0700 Subject: [PATCH 0130/1004] Release 2.9 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 02e4014..d84e536 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v2_9: + +2.9 (2020-05-10) +---------------- + +- New ``sqlite-utils drop-table`` command, see :ref:`cli_drop_table`. (`#111 `__) +- New ``sqlite-utils drop-view`` command, see :ref:`cli_drop_view`. +- Python ``decimal.Decimal`` objects are now stored as ``FLOAT``. (`#110 `__) + .. _v2_8: 2.8 (2020-05-03) diff --git a/setup.py b/setup.py index 88e385b..e619aa7 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.8" +VERSION = "2.9" def get_long_description(): From 74b30af31bf5169559c06aa6e57e1e4873076720 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 May 2020 12:16:22 -0700 Subject: [PATCH 0131/1004] Added project_urls --- setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup.py b/setup.py index e619aa7..7c540cc 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,13 @@ setup( """, tests_require=["sqlite-utils[test]"], url="https://github.com/simonw/sqlite-utils", + project_urls={ + "Documentation": "https://sqlite-utils.readthedocs.io/en/stable/", + "Changelog": "https://sqlite-utils.readthedocs.io/en/stable/changelog.html", + "Source code": "https://github.com/simonw/sqlite-utils", + "Issues": "https://github.com/simonw/sqlite-utils/issues", + "CI": "https://travis-ci.com/simonw/sqlite-utils", + }, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", From 3b5c931287646e6ef448cd6d99d410270c1e8fb1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 May 2020 12:20:29 -0700 Subject: [PATCH 0132/1004] Release 2.9.1 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d84e536..46cc1b6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_9_1: + +2.9.1 (2020-05-11) +------------------ + +- Added custom project links to the `PyPI listing `__. + .. _v2_9: 2.9 (2020-05-10) diff --git a/setup.py b/setup.py index 7c540cc..6d27b13 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.9" +VERSION = "2.9.1" def get_long_description(): From 8eaac7c5f1c8543ce4b8a6a0545862bc9404e334 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 11 May 2020 12:23:11 -0700 Subject: [PATCH 0133/1004] pip, not pip3 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1757802..29c04da 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ sqlite-utils: a Python library and CLI tool for building SQLite databases](https ## Installation - pip3 install sqlite-utils + pip install sqlite-utils ## Using as a CLI tool From 03ee97d2258254581bea72842518904fc1cbe60f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jun 2020 10:40:53 -0700 Subject: [PATCH 0134/1004] CLI now supports upsert/insert - closes #115 --- docs/cli.rst | 5 +++++ sqlite_utils/cli.py | 11 ++++++++--- tests/test_cli.py | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 119e49d..f75f370 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -55,6 +55,11 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] +If you execute an `UPDATE` or `INSERT` query the comand will return the number of affected rows:: + + $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" + [{"rows_affected": 1}] + You can run queries against a temporary in-memory database by passing ``:memory:`` as the filename:: $ sqlite-utils :memory: "select sqlite_version()" diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4cbb13e..fe5cd64 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -669,14 +669,19 @@ def drop_view(path, view): def query(path, sql, nl, arrays, csv, no_headers, table, fmt, json_cols): "Execute SQL query and return the results as JSON" db = sqlite_utils.Database(path) - cursor = iter(db.conn.execute(sql)) - headers = [c[0] for c in cursor.description] + cursor = db.conn.execute(sql) + if cursor.description is None: + # This was an update/insert + headers = ["rows_affected"] + cursor = [[cursor.rowcount]] + else: + headers = [c[0] for c in cursor.description] if table: print(tabulate.tabulate(list(cursor), headers=headers, tablefmt=fmt)) elif csv: writer = csv_std.writer(sys.stdout) if not no_headers: - writer.writerow([c[0] for c in cursor.description]) + writer.writerow(headers) for row in cursor: writer.writerow(row) else: diff --git a/tests/test_cli.py b/tests/test_cli.py index 9cd5965..9e43fb9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1107,3 +1107,22 @@ def test_drop_view_error(): result = runner.invoke(cli.cli, ["drop-view", "test.db", "t2",],) assert 1 == result.exit_code assert 'Error: View "t2" does not exist' == result.output.strip() + + +@pytest.mark.parametrize( + "args,expected", + [ + ([], '[{"rows_affected": 1}]',), + (["-t"], "rows_affected\n---------------\n 1"), + ], +) +def test_query_update(db_path, args, expected): + db = Database(db_path) + with db.conn: + db["dogs"].insert_all( + [{"id": 1, "age": 4, "name": "Cleo"},] + ) + result = CliRunner().invoke( + cli.cli, [db_path, "update dogs set age = 5 where name = 'Cleo'"] + args + ) + assert expected == result.output.strip() From 4d9a3204361d956440307a57bd18c829a15861db Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jun 2020 10:43:45 -0700 Subject: [PATCH 0135/1004] Release 2.10 Refs #115 --- docs/changelog.rst | 7 +++++++ docs/cli.rst | 2 +- setup.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 46cc1b6..31b8dc0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_10: + +2.10 (2020-06-12) +----------------- + +- The ``sqlite-utils`` command now supports UPDATE/INSERT/DELETE in addition to SELECT. (`#115 `__) + .. _v2_9_1: 2.9.1 (2020-05-11) diff --git a/docs/cli.rst b/docs/cli.rst index f75f370..af48a3e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -55,7 +55,7 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] -If you execute an `UPDATE` or `INSERT` query the comand will return the number of affected rows:: +If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the comand will return the number of affected rows:: $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" [{"rows_affected": 1}] diff --git a/setup.py b/setup.py index 6d27b13..0e1cb23 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.9.1" +VERSION = "2.10" def get_long_description(): From fbeb61e49c940ee96d1423c76300ba3ce2cadf80 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2020 14:03:13 -0700 Subject: [PATCH 0136/1004] Documentation for table.pks, closes #116 --- docs/python-api.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index ded39ce..c2c75c7 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1020,6 +1020,11 @@ The ``.columns_dict`` property returns a dictionary version of this with just th >>> db["PlantType"].columns_dict {'id': , 'value': } +The ``.pks`` property returns a list of strings naming the primary key columns for the table:: + + >>> db["PlantType"].pks + ['id'] + The ``.foreign_keys`` property shows if the table has any foreign key relationships. It is not available on views. :: From 97246f9ef7dfa38a5fd71841f397fe3222be0875 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2020 14:04:07 -0700 Subject: [PATCH 0137/1004] Release 2.10.1 Refs #116 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0e1cb23..0044bdc 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.10" +VERSION = "2.10.1" def get_long_description(): From d0cdaaaf00249230e847be3a3b393ee2689fbfe4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2020 14:13:18 -0700 Subject: [PATCH 0138/1004] Release notes for 2.10.1 --- docs/changelog.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 31b8dc0..1f0b2c5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== + +.. _v2_10_1: + +2.10.1 (2020-06-23) +------------------- + +- Added documentation for the ``table.pks`` introspection property. (`#116 `__) + .. _v2_10: 2.10 (2020-06-12) From f8277d0fb9c05a88a9ff01d996e31d55f0f0a645 Mon Sep 17 00:00:00 2001 From: Thomas Sibley Date: Tue, 7 Jul 2020 22:14:04 -0700 Subject: [PATCH 0139/1004] sqlite-utils query can now run DML (#120) * Failing test showing that DML in `sqlite-utils query` doesn't work * Run `sqlite-utils query` in a transaction so that DML is committed Thanks, @tsibley! --- sqlite_utils/cli.py | 37 +++++++++++++++++++------------------ tests/test_cli.py | 3 +++ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index fe5cd64..ab77366 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -669,24 +669,25 @@ def drop_view(path, view): def query(path, sql, nl, arrays, csv, no_headers, table, fmt, json_cols): "Execute SQL query and return the results as JSON" db = sqlite_utils.Database(path) - cursor = db.conn.execute(sql) - if cursor.description is None: - # This was an update/insert - headers = ["rows_affected"] - cursor = [[cursor.rowcount]] - else: - headers = [c[0] for c in cursor.description] - if table: - print(tabulate.tabulate(list(cursor), headers=headers, tablefmt=fmt)) - elif csv: - writer = csv_std.writer(sys.stdout) - if not no_headers: - writer.writerow(headers) - for row in cursor: - writer.writerow(row) - else: - for line in output_rows(cursor, headers, nl, arrays, json_cols): - click.echo(line) + with db.conn: + cursor = db.conn.execute(sql) + if cursor.description is None: + # This was an update/insert + headers = ["rows_affected"] + cursor = [[cursor.rowcount]] + else: + headers = [c[0] for c in cursor.description] + if table: + print(tabulate.tabulate(list(cursor), headers=headers, tablefmt=fmt)) + elif csv: + writer = csv_std.writer(sys.stdout) + if not no_headers: + writer.writerow(headers) + for row in cursor: + writer.writerow(row) + else: + for line in output_rows(cursor, headers, nl, arrays, json_cols): + click.echo(line) @cli.command() diff --git a/tests/test_cli.py b/tests/test_cli.py index 9e43fb9..c9f4658 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1126,3 +1126,6 @@ def test_query_update(db_path, args, expected): cli.cli, [db_path, "update dogs set age = 5 where name = 'Cleo'"] + args ) assert expected == result.output.strip() + assert db.execute_returning_dicts("select * from dogs") == [ + {"id": 1, "age": 5, "name": "Cleo"}, + ] From ae4593316ccf5e42ad26f27033193834a7e696c8 Mon Sep 17 00:00:00 2001 From: Thomas Sibley Date: Mon, 6 Jul 2020 14:18:23 -0700 Subject: [PATCH 0140/1004] Add insert --truncate option Deletes all rows in the table (if it exists) before inserting new rows. SQLite doesn't implement a TRUNCATE TABLE statement but does optimize an unqualified DELETE FROM. This can be handy if you want to refresh the entire contents of a table but a) don't have a PK (so can't use --replace), b) don't want the table to disappear (even briefly) for other connections, and c) have to handle records that used to exist being deleted. Ideally the replacement of rows would appear instantaneous to other connections by putting the DELETE + INSERT in a transaction, but this is very difficult without breaking other code as the current transaction handling is inconsistent and non-systematic. There exists the possibility for the DELETE to succeed but the INSERT to fail, leaving an empty table. This is not much worse, however, than the current possibility of one chunked INSERT succeeding and being committed while the next chunked INSERT fails, leaving a partially complete operation. --- docs/cli.rst | 4 ++++ docs/python-api.rst | 3 +++ sqlite_utils/cli.py | 11 ++++++++++- sqlite_utils/db.py | 3 +++ tests/test_cli.py | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index af48a3e..55be13c 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -274,6 +274,10 @@ You can skip inserting any records that have a primary key that already exists u $ sqlite-utils insert dogs.db dogs dogs.json --ignore +You can delete all the existing rows in the table before inserting the new records using ``--truncate``:: + + $ sqlite-utils insert dogs.db dogs dogs.json --truncate + You can also import newline-delimited JSON using the ``--nl`` option. Since `Datasette `__ can export newline-delimited JSON, you can combine the two tools like so:: $ curl -L "https://latest.datasette.io/fixtures/facetable.json?_shape=array&_nl=on" \ diff --git a/docs/python-api.rst b/docs/python-api.rst index c2c75c7..e6b1af6 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -423,6 +423,9 @@ The function can accept an iterator or generator of rows and will commit them ac You can skip inserting any records that have a primary key that already exists using ``ignore=True``. This works with both ``.insert({...}, ignore=True)`` and ``.insert_all([...], ignore=True)``. +You can delete all the existing rows in the table before inserting the new +records using ``truncate=True``. This is useful if you want to replace the data in the table. + .. _python_api_insert_replace: Insert-replacing data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index ab77366..7c87945 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -423,6 +423,7 @@ def insert_upsert_implementation( upsert, ignore=False, replace=False, + truncate=False, not_null=None, default=None, ): @@ -442,7 +443,7 @@ def insert_upsert_implementation( docs = json.load(json_file) if isinstance(docs, dict): docs = [docs] - extra_kwargs = {"ignore": ignore, "replace": replace} + extra_kwargs = {"ignore": ignore, "replace": replace, "truncate": truncate} if not_null: extra_kwargs["not_null"] = set(not_null) if default: @@ -465,6 +466,12 @@ def insert_upsert_implementation( default=False, help="Replace records if pk already exists", ) +@click.option( + "--truncate", + is_flag=True, + default=False, + help="Truncate table before inserting records, if table already exists", +) def insert( path, table, @@ -477,6 +484,7 @@ def insert( alter, ignore, replace, + truncate, not_null, default, ): @@ -499,6 +507,7 @@ def insert( upsert=False, ignore=ignore, replace=replace, + truncate=truncate, not_null=not_null, default=default, ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index edac5d9..d6b9ecf 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -976,6 +976,7 @@ class Table(Queryable): alter=DEFAULT, ignore=DEFAULT, replace=DEFAULT, + truncate=False, extracts=DEFAULT, conversions=DEFAULT, columns=DEFAULT, @@ -1027,6 +1028,8 @@ class Table(Queryable): batch_size = max(1, min(batch_size, SQLITE_MAX_VARS // num_columns)) self.last_rowid = None self.last_pk = None + if truncate and self.exists(): + self.db.conn.execute("DELETE FROM [{}];".format(self.name)) for chunk in chunks(itertools.chain([first_record], records), batch_size): chunk = list(chunk) num_records_processed += len(chunk) diff --git a/tests/test_cli.py b/tests/test_cli.py index c9f4658..e9a317d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -642,6 +642,39 @@ def test_insert_replace(db_path, tmpdir): ) +def test_insert_truncate(db_path): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_json_nl", "-", "--nl", "--batch-size=1"], + input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', + ) + assert 0 == result.exit_code, result.output + db = Database(db_path) + assert [ + {"foo": "bar", "n": 1}, + {"foo": "baz", "n": 2}, + ] == db.execute_returning_dicts("select foo, n from from_json_nl") + # Truncate and insert new rows + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "from_json_nl", + "-", + "--nl", + "--truncate", + "--batch-size=1", + ], + input='{"foo": "bam", "n": 3}\n{"foo": "bat", "n": 4}', + ) + assert 0 == result.exit_code, result.output + assert [ + {"foo": "bam", "n": 3}, + {"foo": "bat", "n": 4}, + ] == db.execute_returning_dicts("select foo, n from from_json_nl") + + def test_insert_alter(db_path, tmpdir): result = CliRunner().invoke( cli.cli, From 0f8b042b47ab4516829a2e56a2668fff0c5329e6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 8 Jul 2020 10:28:29 -0700 Subject: [PATCH 0141/1004] Release 2.11 Refs #118, #120 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0044bdc..22987e6 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.10.1" +VERSION = "2.11" def get_long_description(): From a236a6bc771a5a6a9d7e814f1986d461afc422d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 8 Jul 2020 10:36:07 -0700 Subject: [PATCH 0142/1004] Release notes for 2.11 Refs #118, #120 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1f0b2c5..4b308be 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_11: + +2.11 (2020-07-08) +----------------- + +- New ``--truncate`` option to ``sqlite-utils insert``, and ``truncate=True`` argument to ``.insert_all()``. Thanks, Thomas Sibley. (`#118 `__) +- The ``sqlite-utils query`` command now runs updates in a transaction. Thanks, Thomas Sibley. (`#120 `__) .. _v2_10_1: From bc8409941fb609eba646c29ae3ec40b8cdd122a4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2020 09:43:45 -0700 Subject: [PATCH 0143/1004] --raw option, refs #123 --- docs/cli.rst | 11 +++++++++++ sqlite_utils/cli.py | 11 +++++++++-- tests/test_cli.py | 15 +++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 55be13c..cbcf1ee 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -142,6 +142,17 @@ You can use the ``--fmt`` (or ``-f``) option to specify different table formats, For a full list of table format options, run ``sqlite-utils query --help``. +.. _cli_query_raw: + +Returning raw data from a query, such as binary content +======================================================= + +If your table contains binary data in a ``BLOB`` you can use the ``--raw`` option to output specific columns directly to standard out. + +For example, to retrieve a binary image from a ``BLOB`` column and store it in a file you can use the following:: + + $ sqlite-utils photos.db "select contents from photos where id=1" --raw > myphoto.jpg + .. _cli_rows: Returning all rows in a table diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 7c87945..4e70fd4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -675,7 +675,8 @@ def drop_view(path, view): ) @click.argument("sql") @output_options -def query(path, sql, nl, arrays, csv, no_headers, table, fmt, json_cols): +@click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") +def query(path, sql, nl, arrays, csv, no_headers, table, fmt, json_cols, raw): "Execute SQL query and return the results as JSON" db = sqlite_utils.Database(path) with db.conn: @@ -686,7 +687,13 @@ def query(path, sql, nl, arrays, csv, no_headers, table, fmt, json_cols): cursor = [[cursor.rowcount]] else: headers = [c[0] for c in cursor.description] - if table: + if raw: + data = cursor.fetchone()[0] + if isinstance(data, bytes): + sys.stdout.buffer.write(data) + else: + sys.stdout.write(str(data)) + elif table: print(tabulate.tabulate(list(cursor), headers=headers, tablefmt=fmt)) elif csv: writer = csv_std.writer(sys.stdout) diff --git a/tests/test_cli.py b/tests/test_cli.py index e9a317d..bf4e8ce 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -797,6 +797,21 @@ def test_query_json_with_json_cols(db_path): assert expected == result_rows.output.strip() +@pytest.mark.parametrize( + "content,is_binary", + [(b"\x00\x0Fbinary", True), ("this is text", False), (1, False), (1.5, False)], +) +def test_query_raw(db_path, content, is_binary): + Database(db_path)["files"].insert({"content": content}) + result = CliRunner().invoke( + cli.cli, [db_path, "select content from files", "--raw"] + ) + if is_binary: + assert result.stdout_bytes == content + else: + assert result.output == str(content) + + def test_query_memory_does_not_create_file(tmpdir): owd = os.getcwd() try: From 20e543e9a492f2e764caae73c38e87f18eaec444 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2020 17:48:36 -0700 Subject: [PATCH 0144/1004] Output binary columns as base64 in JSON, closes #125 --- docs/cli.rst | 13 +++++++++++++ sqlite_utils/cli.py | 10 +++++++++- tests/test_cli.py | 24 ++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index cbcf1ee..8baa194 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -55,6 +55,19 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] +Binary strings are not valid JSON, so BLOB columns containing binary data will be returned as a JSON object containing base64 encoded data, that looks like this:: + + $ sqlite-utils dogs.db "select name, content from images" | python -mjson.tool + [ + { + "name": "smile.gif", + "content": { + "$base64": true, + "encoded": "eJzt0c1x..." + } + } + ] + If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the comand will return the number of affected rows:: $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4e70fd4..f387602 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,3 +1,4 @@ +import base64 import click from click_default_group import DefaultGroup import sqlite_utils @@ -748,7 +749,7 @@ def output_rows(iterator, headers, nl, arrays, json_cols): data = dict(zip(headers, data)) line = "{firstchar}{serialized}{maybecomma}{lastchar}".format( firstchar=("[" if first else " ") if not nl else "", - serialized=json.dumps(data), + serialized=json.dumps(data, default=json_binary), maybecomma="," if (not nl and not is_last) else "", lastchar="]" if (is_last and not nl) else "", ) @@ -766,3 +767,10 @@ def maybe_json(value): return json.loads(stripped) except ValueError: return value + + +def json_binary(value): + if isinstance(value, bytes): + return {"$base64": True, "encoded": base64.b64encode(value).decode("latin-1")} + else: + raise TypeError diff --git a/tests/test_cli.py b/tests/test_cli.py index bf4e8ce..3584840 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -765,6 +765,30 @@ def test_query_json(db_path, sql, args, expected): assert expected == result.output.strip() +LOREM_IPSUM_COMPRESSED = b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8ef\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J\xb9\x97>i\xa9\x11W\xb13\xa5\xde\x96$\x13\xf3I\x9cu\xe8J\xda\xee$EcsI\x8e\x0b$\xea\xab\xf6L&u\xc4emI\xb3foFnT\xf83\xca\x93\xd8QZ\xa8\xf2\xbd1q\xd1\x87\xf3\x85>\x8c\xa4i\x8d\xdaTu\x7f\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\xfb\x8f\xef\x1b\x9b\x06\x83}" + + +def test_query_json_binary(db_path): + db = Database(db_path) + with db.conn: + db["files"].insert( + {"name": "lorem.txt", "sz": 16984, "data": LOREM_IPSUM_COMPRESSED,}, + pk="name", + ) + result = CliRunner().invoke(cli.cli, [db_path, "select name, sz, data from files"]) + assert result.exit_code == 0, str(result) + assert json.loads(result.output.strip()) == [ + { + "name": "lorem.txt", + "sz": 16984, + "data": { + "$base64": True, + "encoded": "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uIjnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3fiCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9", + }, + } + ] + + def test_query_json_with_json_cols(db_path): db = Database(db_path) with db.conn: From 814d4a7f90991be865d38aac45ff12e36df1c67d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2020 20:53:51 -0700 Subject: [PATCH 0145/1004] -p for passing named params to query, closes #124 --- docs/cli.rst | 5 +++++ sqlite_utils/cli.py | 11 +++++++++-- tests/test_cli.py | 21 +++++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 8baa194..013d323 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -21,6 +21,11 @@ This is the default command for ``sqlite-utils``, so you can instead use this:: $ sqlite-utils dogs.db "select * from dogs" +You can pass named parameters to the query using ``-p``:: + + $ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6 + [{":num * :num2": 30}] + Use ``--nl`` to get back newline-delimited JSON objects:: $ sqlite-utils dogs.db "select * from dogs" --nl diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index f387602..155ea54 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -677,11 +677,18 @@ def drop_view(path, view): @click.argument("sql") @output_options @click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") -def query(path, sql, nl, arrays, csv, no_headers, table, fmt, json_cols, raw): +@click.option( + "-p", + "--param", + multiple=True, + type=(str, str), + help="Named :parameters for SQL query", +) +def query(path, sql, nl, arrays, csv, no_headers, table, fmt, json_cols, raw, param): "Execute SQL query and return the results as JSON" db = sqlite_utils.Database(path) with db.conn: - cursor = db.conn.execute(sql) + cursor = db.conn.execute(sql, dict(param)) if cursor.description is None: # This was an update/insert headers = ["rows_affected"] diff --git a/tests/test_cli.py b/tests/test_cli.py index 3584840..4012c97 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -789,6 +789,27 @@ def test_query_json_binary(db_path): ] +@pytest.mark.parametrize( + "sql,params,expected", + [ + ("select 1 + 1 as out", {"p": "2"}, 2), + ("select 1 + :p as out", {"p": "2"}, 3), + ( + "select :hello as out", + {"hello": """This"has'many'quote"s"""}, + """This"has'many'quote"s""", + ), + ], +) +def test_query_params(db_path, sql, params, expected): + extra_args = [] + for key, value in params.items(): + extra_args.extend(["-p", key, value]) + result = CliRunner().invoke(cli.cli, [db_path, sql] + extra_args) + assert result.exit_code == 0, str(result) + assert json.loads(result.output.strip()) == [{"out": expected}] + + def test_query_json_with_json_cols(db_path): db = Database(db_path) with db.conn: From 1a61a6d3d65d14af325889fb5149794bb6c7b214 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Jul 2020 20:59:15 -0700 Subject: [PATCH 0146/1004] Ability to insert base64 binary data as JSON, closes #126 --- docs/cli.rst | 16 ++++++++++++++-- sqlite_utils/cli.py | 4 +++- sqlite_utils/utils.py | 16 ++++++++++++++++ tests/test_cli.py | 12 ++++++++++++ tests/test_utils.py | 22 ++++++++++++++++++++++ 5 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 tests/test_utils.py diff --git a/docs/cli.rst b/docs/cli.rst index 013d323..f3892ca 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -65,10 +65,10 @@ Binary strings are not valid JSON, so BLOB columns containing binary data will b $ sqlite-utils dogs.db "select name, content from images" | python -mjson.tool [ { - "name": "smile.gif", + "name": "transparent.gif", "content": { "$base64": true, - "encoded": "eJzt0c1x..." + "encoded": "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" } } ] @@ -295,6 +295,18 @@ If you feed it a JSON list it will insert multiple records. For example, if ``do } ] +You can insert binary data into a BLOB column by first encoding it using base64 and then structuring it like this:: + + [ + { + "name": "transparent.gif", + "content": { + "$base64": true, + "encoded": "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" + } + } + ] + You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so:: $ sqlite-utils insert dogs.db dogs dogs.json --pk=id diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 155ea54..219c0bf 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -8,7 +8,7 @@ import json import sys import csv as csv_std import tabulate -from .utils import sqlite3 +from .utils import sqlite3, decode_base64_values VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB") @@ -451,6 +451,8 @@ def insert_upsert_implementation( extra_kwargs["defaults"] = dict(default) if upsert: extra_kwargs["upsert"] = upsert + # Apply {"$base64": true, ...} decoding, if needed + docs = (decode_base64_values(doc) for doc in docs) db[table].insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs ) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index ecb3568..e2d3451 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -1,3 +1,5 @@ +import base64 + try: import pysqlite3 as sqlite3 import pysqlite3.dbapi2 @@ -58,3 +60,17 @@ def column_affinity(column_type): return float # Default is 'NUMERIC', which we currently also treat as float return float + + +def decode_base64_values(doc): + # Looks for '{"$base64": true..., "encoded": ...}' values and decodes them + to_fix = [ + k + for k in doc + if isinstance(doc[k], dict) + and doc[k].get("$base64") is True + and "encoded" in doc[k] + ] + if not to_fix: + return doc + return dict(doc, **{k: base64.b64decode(doc[k]["encoded"]) for k in to_fix}) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4012c97..2156def 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -554,6 +554,18 @@ def test_insert_not_null_default(db_path, tmpdir): ) == db["dogs"].schema +def test_insert_binary_base64(db_path): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "files", "-"], + input=r'{"content": {"$base64": true, "encoded": "aGVsbG8="}}', + ) + assert 0 == result.exit_code, result.output + db = Database(db_path) + actual = db.execute_returning_dicts("select content from files") + assert actual == [{"content": b"hello"}] + + def test_insert_newline_delimited(db_path): result = CliRunner().invoke( cli.cli, diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..6489f2b --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,22 @@ +from sqlite_utils import utils +import pytest + + +@pytest.mark.parametrize( + "input,expected,should_be_is", + [ + ({}, None, True), + ({"foo": "bar"}, None, True), + ( + {"content": {"$base64": True, "encoded": "aGVsbG8="}}, + {"content": b"hello"}, + False, + ), + ], +) +def test_decode_base64_values(input, expected, should_be_is): + actual = utils.decode_base64_values(input) + if should_be_is: + assert actual is input + else: + assert actual == expected From ebc802f7ff0e640b6ae11ea525290fea0115228c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jul 2020 00:08:57 -0700 Subject: [PATCH 0147/1004] sqlite-utils insert-files command, closes #122 --- docs/cli.rst | 74 ++++++++++++++++++++++++++ sqlite_utils/cli.py | 106 +++++++++++++++++++++++++++++++++++++ tests/test_docs.py | 2 +- tests/test_insert_files.py | 85 +++++++++++++++++++++++++++++ 4 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 tests/test_insert_files.py diff --git a/docs/cli.rst b/docs/cli.rst index f3892ca..4050824 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -384,6 +384,80 @@ The command will fail if you reference columns that do not exist on the table. T .. note:: ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 `__ for details of this change. +.. _cli_insert_files: + +Inserting binary data from files +================================ + +SQLite ``BLOB`` columns can be used to store binary content. It can be useful to insert the contents of files into a SQLite table. + +The ``insert-files`` command can be used to insert the content of files, along with their metadata. + +Here's an example that inserts all of the GIF files in the current directory into a ``gifs.db`` database, placing the file contents in an ``images`` table:: + + $ sqlite-utils insert-files gifs.db images *.gif + +You can also pass one or more directories, in which case every file in those directories will be added recursively:: + + $ sqlite-utils insert-files gifs.db images path/to/my-gifs + +By default this command will create a table with the following schema:: + + CREATE TABLE [images] ( + [path] TEXT PRIMARY KEY, + [content] BLOB, + [size] INTEGER + ); + +You can customize the schema using one or more ``-c`` options. For a table schema that includes just the path, MD5 hash and last modification time of the file, you would use this:: + + $ sqlite-utils insert-files gifs.db images *.gif -c path -c content -c mtime --pk=path + +This will result in the following schema:: + + CREATE TABLE [images] ( + [path] TEXT PRIMARY KEY, + [content] BLOB, + [mtime] FLOAT + ); + +You can change the name of one of these columns using a ``-c colname:coldef`` parameter. To rename the ``mtime`` column to ``last_modified`` you would use this:: + + $ sqlite-utils insert-files gifs.db images *.gif \ + -c path -c content -c last_modified:mtime --pk=path + +You can pass ``--replace`` or ``--upsert`` to indicate what should happen if you try to insert a file with an existing primary key. Pass ``--alter`` to cause any missing columns to be added to the table. + +The full list of column definitions you can use is as follows: + +``name`` + The name of the file, e.g. ``cleo.jpg`` +``path`` + The path to the file relative to the root folder, e.g. ``pictures/cleo.jpg`` +``fullpath`` + The fully resolved path to the image, e.g. ``/home/simonw/pictures/cleo.jpg`` +``sha256`` + The SHA256 hash of the file contents +``md5`` + The MD5 hash of the file contents +``mode`` + The permission bits of the file, as an integer - you may want to convert this to octal +``content`` + The binary file contents, which will be stored as a BLOB +``mtime`` + The modification time of the file, as floating point seconds since the Unix epoch +``ctime`` + The creation time of the file, as floating point seconds since the Unix epoch +``mtime_int`` + The modification time as an integer rather than a float +``ctime_int`` + The creation time as an integer rather than a float +``mtime_iso`` + The modification time as an ISO timestamp, e.g. ``2020-07-27T04:24:06.654246`` +``ctime_iso`` + The creation time is an ISO timestamp +``size`` + The integer size of the file in bytes .. _cli_create_table: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 219c0bf..3dc40ea 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,6 +1,9 @@ import base64 import click from click_default_group import DefaultGroup +from datetime import datetime +import hashlib +import pathlib import sqlite_utils from sqlite_utils.db import AlterError import itertools @@ -741,6 +744,109 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols) ) +@cli.command(name="insert-files") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table") +@click.argument( + "file_or_dir", + nargs=-1, + required=True, + type=click.Path(file_okay=True, dir_okay=True, allow_dash=True), +) +@click.option( + "-c", "--column", type=str, multiple=True, help="Column definitions for the table", +) +@click.option("--pk", type=str, help="Column to use as primary key") +@click.option("--alter", is_flag=True, help="Alter table to add missing columns") +@click.option("--replace", is_flag=True, help="Replace files with matching primary key") +@click.option("--upsert", is_flag=True, help="Upsert files with matching primary key") +def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert): + """ + Insert one or more files using BLOB columns in the specified table + + Example usage: + + \b + sqlite-utils insert-files pics.db images *.gif \\ + -c name:name \\ + -c content:content \\ + -c content_hash:sha256 \\ + -c created:ctime_iso \\ + -c modified:mtime_iso \\ + -c size:size \\ + --pk name + """ + if not column: + column = ["path:path", "content:content", "size:size"] + if not pk: + pk = "path" + + def yield_paths_and_relative_paths(): + for f_or_d in file_or_dir: + path = pathlib.Path(f_or_d) + if path.is_dir(): + for subpath in path.rglob("*"): + if subpath.is_file(): + yield subpath, subpath.relative_to(path) + elif path.is_file(): + yield path, path + + # Load all paths so we can show a progress bar + paths_and_relative_paths = list(yield_paths_and_relative_paths()) + + with click.progressbar(paths_and_relative_paths) as bar: + + def to_insert(): + for path, relative_path in bar: + row = {} + for coldef in column: + if ":" in coldef: + colname, coltype = coldef.rsplit(":", 1) + else: + colname, coltype = coldef, coldef + try: + if coltype == "path": + value = str(relative_path) + else: + value = FILE_COLUMNS[coltype](path) + row[colname] = value + except KeyError: + raise click.ClickException( + "'{}' is not a valid column definition - options are {}".format( + coltype, ", ".join(FILE_COLUMNS.keys()) + ) + ) + yield row + + db = sqlite_utils.Database(path) + with db.conn: + db[table].insert_all( + to_insert(), pk=pk, alter=alter, replace=replace, upsert=upsert + ) + + +FILE_COLUMNS = { + "name": lambda p: p.name, + "path": lambda p: str(p), + "fullpath": lambda p: str(p.resolve()), + "sha256": lambda p: hashlib.sha256(p.resolve().read_bytes()).hexdigest(), + "md5": lambda p: hashlib.md5(p.resolve().read_bytes()).hexdigest(), + "mode": lambda p: p.stat().st_mode, + "content": lambda p: p.resolve().read_bytes(), + "mtime": lambda p: p.stat().st_mtime, + "ctime": lambda p: p.stat().st_ctime, + "mtime_int": lambda p: int(p.stat().st_mtime), + "ctime_int": lambda p: int(p.stat().st_ctime), + "mtime_iso": lambda p: datetime.utcfromtimestamp(p.stat().st_mtime).isoformat(), + "ctime_iso": lambda p: datetime.utcfromtimestamp(p.stat().st_ctime).isoformat(), + "size": lambda p: p.stat().st_size, +} + + def output_rows(iterator, headers, nl, arrays, json_cols): # We have to iterate two-at-a-time so we can know if we # should output a trailing comma or if we have reached diff --git a/tests/test_docs.py b/tests/test_docs.py index 61f0ece..3b3e4a5 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -18,5 +18,5 @@ def documented_commands(): @pytest.mark.parametrize("command", cli.cli.commands.keys()) -def test_plugin_hooks_are_documented(documented_commands, command): +def test_commands_are_documented(documented_commands, command): assert command in documented_commands diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py new file mode 100644 index 0000000..7a0dd8a --- /dev/null +++ b/tests/test_insert_files.py @@ -0,0 +1,85 @@ +from sqlite_utils import cli, Database +from click.testing import CliRunner +import pathlib + + +def test_insert_files(): + runner = CliRunner() + with runner.isolated_filesystem(): + tmpdir = pathlib.Path(".") + print("tmpdir = ", tmpdir.resolve()) + db_path = str(tmpdir / "files.db") + (tmpdir / "one.txt").write_text("This is file one", "utf-8") + (tmpdir / "two.txt").write_text("Two is shorter", "utf-8") + (tmpdir / "nested").mkdir() + (tmpdir / "nested" / "three.txt").write_text("Three is nested", "utf-8") + coltypes = ( + "name", + "path", + "fullpath", + "sha256", + "md5", + "mode", + "content", + "mtime", + "ctime", + "mtime_int", + "ctime_int", + "mtime_iso", + "ctime_iso", + "size", + ) + cols = [] + for coltype in coltypes: + cols += ["-c", "{}:{}".format(coltype, coltype)] + result = runner.invoke( + cli.cli, + ["insert-files", db_path, "files", str(tmpdir)] + cols + ["--pk", "path"], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.stdout + db = Database(db_path) + rows_by_path = {r["path"]: r for r in db["files"].rows} + one, two, three = ( + rows_by_path["one.txt"], + rows_by_path["two.txt"], + rows_by_path["nested/three.txt"], + ) + assert { + "content": b"This is file one", + "md5": "556dfb57fce9ca301f914e2273adf354", + "name": "one.txt", + "path": "one.txt", + "sha256": "e34138f26b5f7368f298b4e736fea0aad87ddec69fbd04dc183b20f4d844bad5", + "size": 16, + }.items() <= one.items() + assert { + "content": b"Two is shorter", + "md5": "f86f067b083af1911043eb215e74ac70", + "name": "two.txt", + "path": "two.txt", + "sha256": "9368988ed16d4a2da0af9db9b686d385b942cb3ffd4e013f43aed2ec041183d9", + "size": 14, + }.items() <= two.items() + assert { + "content": b"Three is nested", + "md5": "12580f341781f5a5b589164d3cd39523", + "name": "three.txt", + "path": "nested/three.txt", + "sha256": "6dd45aaaaa6b9f96af19363a92c8fca5d34791d3c35c44eb19468a6a862cc8cd", + "size": 15, + }.items() <= three.items() + # Assert the other int/str/float columns exist and are of the right types + expected_types = { + "ctime": float, + "ctime_int": int, + "ctime_iso": str, + "mtime": float, + "mtime_int": int, + "mtime_iso": str, + "mode": int, + "fullpath": str, + } + for colname, expected_type in expected_types.items(): + for row in (one, two, three): + assert isinstance(row[colname], expected_type) From 3f6a10b807c8b2df6e1f971d9747cbefd858e63a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jul 2020 00:21:26 -0700 Subject: [PATCH 0148/1004] Release 2.12 Refs #122, #123, #124, #125, #126 --- docs/changelog.rst | 13 +++++++++++++ setup.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4b308be..c4c2ae7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,19 @@ Changelog =========== +.. _v2_12: + +2.12 (2020-07-27) +----------------- + +The theme of this release is better tools for working with binary data. The new ``insert-files`` command can be used to insert binary files directly into a database table, and other commands have been improved with better support for BLOB columns. + +- ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (`#122 `__) +- ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (`#123 `__) +- JSON output now encodes BLOB values as special base64 obects - see :ref:`cli_query_json`. (`#125 `__) +- The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (`#126 `__) +- The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (`#124 `__) + .. _v2_11: 2.11 (2020-07-08) diff --git a/setup.py b/setup.py index 22987e6..f737e80 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.11" +VERSION = "2.12" def get_long_description(): From 3214af4a20b5c09e2ddaebc922ac9ac12fe93344 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jul 2020 18:53:57 -0700 Subject: [PATCH 0149/1004] Fixed bug in one of the insert-files examples --- docs/cli.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 4050824..67714a5 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -411,20 +411,20 @@ By default this command will create a table with the following schema:: You can customize the schema using one or more ``-c`` options. For a table schema that includes just the path, MD5 hash and last modification time of the file, you would use this:: - $ sqlite-utils insert-files gifs.db images *.gif -c path -c content -c mtime --pk=path + $ sqlite-utils insert-files gifs.db images *.gif -c path -c md5 -c mtime --pk=path This will result in the following schema:: CREATE TABLE [images] ( [path] TEXT PRIMARY KEY, - [content] BLOB, + [md5] TEXT, [mtime] FLOAT ); You can change the name of one of these columns using a ``-c colname:coldef`` parameter. To rename the ``mtime`` column to ``last_modified`` you would use this:: $ sqlite-utils insert-files gifs.db images *.gif \ - -c path -c content -c last_modified:mtime --pk=path + -c path -c md5 -c last_modified:mtime --pk=path You can pass ``--replace`` or ``--upsert`` to indicate what should happen if you try to insert a file with an existing primary key. Pass ``--alter`` to cause any missing columns to be added to the table. From f804690274ce1bd93cc9e173a9d3b393312666cb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2020 18:10:25 -0700 Subject: [PATCH 0150/1004] Support inserting UUID and memoryview, closes #128 --- sqlite_utils/db.py | 5 +++++ tests/test_create.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d6b9ecf..ee26433 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -7,6 +7,7 @@ import itertools import json import os import pathlib +import uuid SQLITE_MAX_VARS = 999 @@ -40,11 +41,13 @@ COLUMN_TYPE_MAPPING = { str: "TEXT", bytes.__class__: "BLOB", bytes: "BLOB", + memoryview: "BLOB", datetime.datetime: "TEXT", datetime.date: "TEXT", datetime.time: "TEXT", decimal.Decimal: "FLOAT", None.__class__: "TEXT", + uuid.UUID: "TEXT", # SQLite explicit types "TEXT": "TEXT", "INTEGER": "INTEGER", @@ -1336,6 +1339,8 @@ def jsonify_if_needed(value): return json.dumps(value, default=repr) elif isinstance(value, (datetime.time, datetime.date, datetime.datetime)): return value.isoformat() + elif isinstance(value, uuid.UUID): + return str(value) else: return value diff --git a/tests/test_create.py b/tests/test_create.py index 22e4b7b..a84eb8d 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -13,6 +13,7 @@ import decimal import json import pathlib import pytest +import uuid from .utils import collapse_whitespace @@ -134,6 +135,11 @@ def test_create_table_with_not_null(fresh_db): ), ({"day": datetime.time(11, 0)}, [{"name": "day", "type": "TEXT"}]), ({"decimal": decimal.Decimal("1.2")}, [{"name": "decimal", "type": "FLOAT"}]), + ( + {"memoryview": memoryview(b"hello")}, + [{"name": "memoryview", "type": "BLOB"}], + ), + ({"uuid": uuid.uuid4()}, [{"name": "uuid", "type": "TEXT"}]), ), ) def test_create_table_from_example(fresh_db, example, expected_columns): @@ -674,6 +680,23 @@ def test_insert_dictionaries_and_lists_as_json(fresh_db, data_structure): assert data_structure == json.loads(row[1]) +def test_insert_uuid(fresh_db): + uuid4 = uuid.uuid4() + fresh_db["test"].insert({"uuid": uuid4}) + row = list(fresh_db["test"].rows)[0] + assert {"uuid"} == row.keys() + assert isinstance(row["uuid"], str) + assert row["uuid"] == str(uuid4) + + +def test_insert_memoryview(fresh_db): + fresh_db["test"].insert({"data": memoryview(b"hello")}) + row = list(fresh_db["test"].rows)[0] + assert {"data"} == row.keys() + assert isinstance(row["data"], bytes) + assert row["data"] == b"hello" + + def test_insert_thousands_using_generator(fresh_db): fresh_db["test"].insert_all( {"i": i, "word": "word_{}".format(i)} for i in range(10000) From 710454d72aed5094573e642344fd075a0ef5372c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2020 18:13:53 -0700 Subject: [PATCH 0151/1004] Release 2.13 Refs #128 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c4c2ae7..9cd608c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_13: + +2.13 (2020-07-29) +----------------- + +- ``memoryview`` and ``uuid.UUID`` objects are now supported. ``memoryview`` objects will be stored using ``BLOB`` and ``uuid.UUID`` objects will be stored using ``TEXT``. (`#128 `__) + .. _v2_12: 2.12 (2020-07-27) diff --git a/setup.py b/setup.py index f737e80..1aa233e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.12" +VERSION = "2.13" def get_long_description(): From 8fe1e6d1be021aeeb8f08b0f77f03b75a83b6f75 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 29 Jul 2020 20:08:12 -0700 Subject: [PATCH 0152/1004] insert-files can now read from stdin, closes #127 --- docs/cli.rst | 8 ++++++++ sqlite_utils/cli.py | 29 ++++++++++++++++++++++------- tests/test_insert_files.py | 18 +++++++++++++++++- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 67714a5..a9adf63 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -459,6 +459,14 @@ The full list of column definitions you can use is as follows: ``size`` The integer size of the file in bytes +You can insert data piped from standard input like this:: + + cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg + +The ``-`` argument indicates data should be read from standard input. The string passed using the ``--name`` option will be used for the file name and path values. + +When inserting data from standard input only the following column definitions are supported: ``name``, ``path``, ``content``, ``sha256``, ``md5`` and ``size``. + .. _cli_create_table: Creating tables diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 3dc40ea..c60fd01 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -764,7 +764,8 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols) @click.option("--alter", is_flag=True, help="Alter table to add missing columns") @click.option("--replace", is_flag=True, help="Replace files with matching primary key") @click.option("--upsert", is_flag=True, help="Upsert files with matching primary key") -def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert): +@click.option("--name", type=str, help="File name to use") +def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert, name): """ Insert one or more files using BLOB columns in the specified table @@ -788,7 +789,9 @@ def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert): def yield_paths_and_relative_paths(): for f_or_d in file_or_dir: path = pathlib.Path(f_or_d) - if path.is_dir(): + if f_or_d == "-": + yield "-", "-" + elif path.is_dir(): for subpath in path.rglob("*"): if subpath.is_file(): yield subpath, subpath.relative_to(path) @@ -803,23 +806,35 @@ def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert): def to_insert(): for path, relative_path in bar: row = {} + lookups = FILE_COLUMNS + if path == "-": + stdin_data = sys.stdin.buffer.read() + # We only support a subset of columns for this case + lookups = { + "name": lambda p: name or "-", + "path": lambda p: name or "-", + "content": lambda p: stdin_data, + "sha256": lambda p: hashlib.sha256(stdin_data).hexdigest(), + "md5": lambda p: hashlib.md5(stdin_data).hexdigest(), + "size": lambda p: len(stdin_data), + } for coldef in column: if ":" in coldef: colname, coltype = coldef.rsplit(":", 1) else: colname, coltype = coldef, coldef try: - if coltype == "path": - value = str(relative_path) - else: - value = FILE_COLUMNS[coltype](path) + value = lookups[coltype](path) row[colname] = value except KeyError: raise click.ClickException( "'{}' is not a valid column definition - options are {}".format( - coltype, ", ".join(FILE_COLUMNS.keys()) + coltype, ", ".join(lookups.keys()) ) ) + # Special case for --name + if coltype == "name" and name: + row[colname] = name yield row db = sqlite_utils.Database(path) diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 7a0dd8a..f17f279 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -7,7 +7,6 @@ def test_insert_files(): runner = CliRunner() with runner.isolated_filesystem(): tmpdir = pathlib.Path(".") - print("tmpdir = ", tmpdir.resolve()) db_path = str(tmpdir / "files.db") (tmpdir / "one.txt").write_text("This is file one", "utf-8") (tmpdir / "two.txt").write_text("Two is shorter", "utf-8") @@ -83,3 +82,20 @@ def test_insert_files(): for colname, expected_type in expected_types.items(): for row in (one, two, three): assert isinstance(row[colname], expected_type) + + +def test_insert_files_stdin(): + runner = CliRunner() + with runner.isolated_filesystem(): + tmpdir = pathlib.Path(".") + db_path = str(tmpdir / "files.db") + result = runner.invoke( + cli.cli, + ["insert-files", db_path, "files", "-", "--name", "stdin-name"], + catch_exceptions=False, + input="hello world", + ) + assert result.exit_code == 0, result.stdout + db = Database(db_path) + row = list(db["files"].rows)[0] + assert {"path": "stdin-name", "content": b"hello world", "size": 11} == row From 617e6f070c85be66ea04c80b78dafd08c875f8c8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 1 Aug 2020 13:40:36 -0700 Subject: [PATCH 0153/1004] enable_fts(..., tokenize=X) parameter, refs #130 --- docs/python-api.rst | 8 ++++++++ sqlite_utils/db.py | 9 ++++++-- tests/test_fts.py | 50 ++++++++++++++++++++++++++++++++++++--------- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index e6b1af6..9fc51f7 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1135,6 +1135,14 @@ A better solution is to use database triggers. You can set up database triggers dogs.enable_fts(["name", "twitter"], fts_version="FTS4") +You can customize the tokenizer configured for the table using the ``tokenize=`` parameter. For example, to enable Porter stemming, where English words like "running" will match stemmed alternatives such as "run", use ``tokenize="porter"``: + +.. code-block:: python + + db["articles"].enable_fts(["headline", "body"], tokenize="porter") + +The SQLite documentation has more on `FTS5 tokenizers `__ and `FTS4 tokenizers `__. ``porter`` is a valid option for both. + To remove the FTS tables and triggers you created, use the ``disable_fts()`` table method: .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ee26433..7d3a8fe 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -763,17 +763,22 @@ class Table(Queryable): ) self.db.add_foreign_keys([(self.name, column, other_table, other_column)]) - def enable_fts(self, columns, fts_version="FTS5", create_triggers=False): + def enable_fts( + self, columns, fts_version="FTS5", create_triggers=False, tokenize=None + ): "Enables FTS on the specified columns." sql = """ CREATE VIRTUAL TABLE [{table}_fts] USING {fts_version} ( - {columns}, + {columns},{tokenize} content=[{table}] ); """.format( table=self.name, columns=", ".join("[{}]".format(c) for c in columns), fts_version=fts_version, + tokenize="\n tokenize='{}',\n".format(tokenize) + if tokenize + else "", ) self.db.conn.executescript(sql) self.populate_fts(columns) diff --git a/tests/test_fts.py b/tests/test_fts.py index 7817477..bac4117 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -1,8 +1,16 @@ import pytest search_records = [ - {"text": "tanuki are tricksters", "country": "Japan", "not_searchable": "foo"}, - {"text": "racoons are trash pandas", "country": "USA", "not_searchable": "bar"}, + { + "text": "tanuki are running tricksters", + "country": "Japan", + "not_searchable": "foo", + }, + { + "text": "racoons are biting trash pandas", + "country": "USA", + "not_searchable": "bar", + }, ] @@ -19,8 +27,8 @@ def test_enable_fts(fresh_db): "searchable_fts_docsize", "searchable_fts_stat", ] == fresh_db.table_names() - assert [("tanuki are tricksters", "Japan", "foo")] == table.search("tanuki") - assert [("racoons are trash pandas", "USA", "bar")] == table.search("usa") + assert [("tanuki are running tricksters", "Japan", "foo")] == table.search("tanuki") + assert [("racoons are biting trash pandas", "USA", "bar")] == table.search("usa") assert [] == table.search("bar") @@ -39,8 +47,8 @@ def test_enable_fts_escape_table_names(fresh_db): "http://example.com_fts_docsize", "http://example.com_fts_stat", ] == fresh_db.table_names() - assert [("tanuki are tricksters", "Japan", "foo")] == table.search("tanuki") - assert [("racoons are trash pandas", "USA", "bar")] == table.search("usa") + assert [("tanuki are running tricksters", "Japan", "foo")] == table.search("tanuki") + assert [("racoons are biting trash pandas", "USA", "bar")] == table.search("usa") assert [] == table.search("bar") @@ -67,7 +75,7 @@ def test_populate_fts(fresh_db): assert [] == table.search("trash pandas") # Now run populate_fts to make this record available table.populate_fts(["text", "country"]) - assert [("racoons are trash pandas", "USA", "bar")] == table.search("usa") + assert [("racoons are biting trash pandas", "USA", "bar")] == table.search("usa") def test_populate_fts_escape_table_names(fresh_db): @@ -80,7 +88,29 @@ def test_populate_fts_escape_table_names(fresh_db): assert [] == table.search("trash pandas") # Now run populate_fts to make this record available table.populate_fts(["text", "country"]) - assert [("racoons are trash pandas", "USA", "bar")] == table.search("usa") + assert [("racoons are biting trash pandas", "USA", "bar")] == table.search("usa") + + +def test_fts_tokenize(fresh_db): + for fts_version in ("4", "5"): + table_name = "searchable_{}".format(fts_version) + table = fresh_db[table_name] + table.insert_all(search_records) + # Test without porter stemming + table.enable_fts( + ["text", "country"], fts_version="FTS{}".format(fts_version), + ) + assert [] == table.search("bite") + # Test WITH stemming + table.disable_fts() + table.enable_fts( + ["text", "country"], + fts_version="FTS{}".format(fts_version), + tokenize="porter", + ) + assert [("racoons are biting trash pandas", "USA", "bar")] == table.search( + "bite" + ) def test_optimize_fts(fresh_db): @@ -103,10 +133,10 @@ def test_enable_fts_w_triggers(fresh_db): table = fresh_db["searchable"] table.insert(search_records[0]) table.enable_fts(["text", "country"], fts_version="FTS4", create_triggers=True) - assert [("tanuki are tricksters", "Japan", "foo")] == table.search("tanuki") + assert [("tanuki are running tricksters", "Japan", "foo")] == table.search("tanuki") table.insert(search_records[1]) # Triggers will auto-populate FTS virtual table, not need to call populate_fts() - assert [("racoons are trash pandas", "USA", "bar")] == table.search("usa") + assert [("racoons are biting trash pandas", "USA", "bar")] == table.search("usa") assert [] == table.search("bar") From 57e4eb8e5564af5d97f892b3be8342451ee177a2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 1 Aug 2020 13:51:05 -0700 Subject: [PATCH 0154/1004] sqlite-utils populate-fts --tokenize= option, closes #130 --- docs/cli.rst | 4 ++++ sqlite_utils/cli.py | 8 ++++++-- sqlite_utils/db.py | 2 +- tests/test_cli.py | 32 ++++++++++++++++++++++++-------- 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index a9adf63..1136a9e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -676,6 +676,10 @@ A better solution here is to use database triggers. You can set up database trig $ sqlite-utils enable-fts mydb.db documents title summary --create-triggers +To set a custom FTS tokenizer, e.g. to enable Porter stemming, use ``--tokenize=``:: + + $ sqlite-utils populate-fts mydb.db documents title summary --tokenize=porter + To remove the FTS tables and triggers you created, use ``disable-fts``:: $ sqlite-utils disable-fts mydb.db documents diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c60fd01..ff3f50f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -325,13 +325,14 @@ def create_index(path, table, column, name, unique, if_not_exists): @click.argument("column", nargs=-1, required=True) @click.option("--fts4", help="Use FTS4", default=False, is_flag=True) @click.option("--fts5", help="Use FTS5", default=False, is_flag=True) +@click.option("--tokenize", help="Tokenizer to use, e.g. porter") @click.option( "--create-triggers", help="Create triggers to update the FTS tables when the parent table changes.", default=False, is_flag=True, ) -def enable_fts(path, table, column, fts4, fts5, create_triggers): +def enable_fts(path, table, column, fts4, fts5, tokenize, create_triggers): "Enable FTS for specific table and columns" fts_version = "FTS5" if fts4 and fts5: @@ -342,7 +343,10 @@ def enable_fts(path, table, column, fts4, fts5, create_triggers): db = sqlite_utils.Database(path) db[table].enable_fts( - column, fts_version=fts_version, create_triggers=create_triggers + column, + fts_version=fts_version, + tokenize=tokenize, + create_triggers=create_triggers, ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 7d3a8fe..88f5440 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -776,7 +776,7 @@ class Table(Queryable): table=self.name, columns=", ".join("[{}]".format(c) for c in columns), fts_version=fts_version, - tokenize="\n tokenize='{}',\n".format(tokenize) + tokenize="\n tokenize='{}',".format(tokenize) if tokenize else "", ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2156def..22d42e5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -344,25 +344,41 @@ def test_index_foreign_keys(db_path): def test_enable_fts(db_path): - assert None == Database(db_path)["Gosh"].detect_fts() + db = Database(db_path) + assert None == db["Gosh"].detect_fts() result = CliRunner().invoke( cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] ) assert 0 == result.exit_code - assert "Gosh_fts" == Database(db_path)["Gosh"].detect_fts() + assert "Gosh_fts" == db["Gosh"].detect_fts() # Table names with restricted chars are handled correctly. # colons and dots are restricted characters for table names. - Database(db_path)["http://example.com"].create({"c1": str, "c2": str, "c3": str}) - assert None == Database(db_path)["http://example.com"].detect_fts() + db["http://example.com"].create({"c1": str, "c2": str, "c3": str}) + assert None == db["http://example.com"].detect_fts() result = CliRunner().invoke( - cli.cli, ["enable-fts", db_path, "http://example.com", "c1", "--fts4"] + cli.cli, + [ + "enable-fts", + db_path, + "http://example.com", + "c1", + "--fts4", + "--tokenize", + "porter", + ], ) assert 0 == result.exit_code + assert "http://example.com_fts" == db["http://example.com"].detect_fts() + # Check tokenize was set to porter assert ( - "http://example.com_fts" == Database(db_path)["http://example.com"].detect_fts() - ) - Database(db_path)["http://example.com"].drop() + "CREATE VIRTUAL TABLE [http://example.com_fts] USING FTS4 (\n" + " [c1],\n" + " tokenize='porter',\n" + " content=[http://example.com]" + "\n )" + ) == db["http://example.com_fts"].schema + db["http://example.com"].drop() def test_enable_fts_with_triggers(db_path): From a8b922bcb91186c44fc163f7f6566598d962d364 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 1 Aug 2020 13:58:47 -0700 Subject: [PATCH 0155/1004] Release 2.14 Refs #127, #130 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9cd608c..1239f39 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v2_14: + +2.14 (2020-08-01) +----------------- + +- The :ref:`insert-files command ` can now read from standard input: ``cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg``. (`#127 `__) +- You can now specify a full-text search tokenizer using the new ``tokenize=`` parameter to :ref:`enable_fts() `. This means you can enable Porter stemming on a table by running ``db["articles"].enable_fts(["headline", "body"], tokenize="porter")``. (`#130 `__) +- You can also set a custom tokenizer using the :ref:`sqlite-utils enable-fts ` CLI command, via the new ``--tokenize`` option. + .. _v2_13: 2.13 (2020-07-29) diff --git a/setup.py b/setup.py index 1aa233e..60bc917 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.13" +VERSION = "2.14" def get_long_description(): From 5560d717cb6c3ba378f2b79de452fa2a737e9d6d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 1 Aug 2020 14:30:43 -0700 Subject: [PATCH 0156/1004] Updated list of supported parameters to db.table() --- docs/python-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 9fc51f7..496bff7 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -342,7 +342,7 @@ You can set default values for these methods by accessing the table through the # Now you can call .insert() like so: table.insert({"id": 1, "name": "Tracy", "score": 5}) -The configuration options that can be specified in this way are ``pk``, ``foreign_keys``, ``column_order``, ``not_null``, ``defaults``, ``upsert``, ``batch_size``, ``hash_id``, ``alter``, ``ignore``. These are all documented below. +The configuration options that can be specified in this way are ``pk``, ``foreign_keys``, ``column_order``, ``not_null``, ``defaults``, ``batch_size``, ``hash_id``, ``alter``, ``ignore``, ``replace``, ``extracts``, ``conversions``, ``columns``. These are all documented below. .. _python_api_defaults_not_null: From db1e08c2c89ac8e93bf8650a69f7ec6585c9b804 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Aug 2020 19:21:49 -0700 Subject: [PATCH 0157/1004] Documentation for table.detect_fts() method --- docs/python-api.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 496bff7..75ecca0 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1093,6 +1093,13 @@ The ``.triggers`` property lists database triggers. It can be used on both datab >>> db.triggers ... similar output to db["authors"].triggers +The ``detect_fts()`` method returns the associated SQLite FTS table name, if one exists for this table. If the table has not been configured for full-text search it returns ``None``. + +:: + + >> db["authors"].detect_fts() + "authors_fts" + .. _python_api_fts: Enabling full-text search From 5de24ee0a4652ce3d3b2e9726ccea25343a10ed1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Aug 2020 23:28:50 -0700 Subject: [PATCH 0158/1004] Release 2.14.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 60bc917..e52b054 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.14" +VERSION = "2.14.1" def get_long_description(): From 957f8c9b4ca76e32b03ca5dbf50dd61d01f25292 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Aug 2020 23:31:11 -0700 Subject: [PATCH 0159/1004] 2.14.1 release notes --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1239f39..3946b81 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_14_1: + +2.14.1 (2020-08-05) +------------------- + +- Documentation improvements. + .. _v2_14: 2.14 (2020-08-01) From 2d2d724e32824095b0bf267a38d9c6fd628cc706 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Aug 2020 11:59:21 -0700 Subject: [PATCH 0160/1004] Tools for enabling and disabling WAL, closes #132 --- docs/cli.rst | 19 +++++++++++++++++++ docs/python-api.rst | 27 +++++++++++++++++++++++++++ sqlite_utils/cli.py | 26 ++++++++++++++++++++++++++ sqlite_utils/db.py | 12 ++++++++++++ tests/test_cli.py | 31 +++++++++++++++++++++++++++++++ 5 files changed, 115 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 1136a9e..6ac2c2d 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -684,6 +684,8 @@ To remove the FTS tables and triggers you created, use ``disable-fts``:: $ sqlite-utils disable-fts mydb.db documents +.. _cli_vacuum: + Vacuum ====== @@ -691,6 +693,8 @@ You can run VACUUM to optimize your database like so:: $ sqlite-utils vacuum mydb.db +.. _cli_optimize: + Optimize ======== @@ -705,3 +709,18 @@ If you just want to run OPTIMIZE without the VACUUM, use the ``--no-vacuum`` fla # Optimize but skip the VACUUM $ sqlite-utils optimize --no-vacuum mydb.db + +.. _cli_wal: + +WAL mode +======== + +You can enable `Write-Ahead Logging `__ for a database file using the ``enable-wal`` command:: + + $ sqlite-utils enable-wal mydb.db + +You can disable WAL mode using ``disable-wal``:: + + $ sqlite-utils disable-wal mydb.db + +Both of these commands accept one or more database files as arguments. diff --git a/docs/python-api.rst b/docs/python-api.rst index 75ecca0..b9b6013 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1195,6 +1195,8 @@ You can create a unique index by passing ``unique=True``: Use ``if_not_exists=True`` to do nothing if an index with that name already exists. +.. _python_api_vacuum: + Vacuum ====== @@ -1204,6 +1206,31 @@ You can optimize your database by running VACUUM against it like so: Database("my_database.db").vacuum() +.. _python_api_wal: + +WAL mode +======== + +You can enable `Write-Ahead Logging `__ for a database with ``.enable_wal()``: + +.. code-block:: python + + Database("my_database.db").enable_wal() + +You can disable WAL mode using ``.disable_wal()``: + +.. code-block:: python + + Database("my_database.db").disable_wal() + +You can check the current journal mode for a database using the ``journal_mode`` property: + +.. code-block:: python + + journal_mode = Database("my_database.db").journal_mode + +This will usually be ``wal`` or ``delete`` (meaning WAL is disabled), but can have other values - see the `PRAGMA journal_mode `__ documentation. + .. _python_api_suggest_column_types: Suggesting column types diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index ff3f50f..79a8df0 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -377,6 +377,32 @@ def disable_fts(path, table): db[table].disable_fts() +@cli.command(name="enable-wal") +@click.argument( + "path", + nargs=-1, + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +def enable_wal(path): + "Enable WAL for database files" + for path_ in path: + sqlite_utils.Database(path_).enable_wal() + + +@cli.command(name="disable-wal") +@click.argument( + "path", + nargs=-1, + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +def disable_wal(path): + "Disable WAL for database files" + for path_ in path: + sqlite_utils.Database(path_).disable_wal() + + def insert_upsert_options(fn): for decorator in reversed( ( diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 88f5440..a8791c3 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -170,6 +170,18 @@ class Database: ).fetchall() ] + @property + def journal_mode(self): + return self.conn.execute("PRAGMA journal_mode;").fetchone()[0] + + def enable_wal(self): + if self.journal_mode != "wal": + self.conn.execute("PRAGMA journal_mode=wal;") + + def disable_wal(self): + if self.journal_mode != "delete": + self.conn.execute("PRAGMA journal_mode=delete;") + def execute_returning_dicts(self, sql, params=None): cursor = self.conn.execute(sql, params or tuple()) keys = [d[0] for d in cursor.description] diff --git a/tests/test_cli.py b/tests/test_cli.py index 22d42e5..fea89dd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1230,6 +1230,37 @@ def test_drop_view_error(): assert 'Error: View "t2" does not exist' == result.output.strip() +def test_enable_wal(): + runner = CliRunner() + dbs = ["test.db", "test2.db"] + with runner.isolated_filesystem(): + for dbname in dbs: + db = Database(dbname) + db["t"].create({"pk": int}, pk="pk") + assert db.journal_mode == "delete" + result = runner.invoke(cli.cli, ["enable-wal"] + dbs) + assert 0 == result.exit_code + for dbname in dbs: + db = Database(dbname) + assert db.journal_mode == "wal" + + +def test_disable_wal(): + runner = CliRunner() + dbs = ["test.db", "test2.db"] + with runner.isolated_filesystem(): + for dbname in dbs: + db = Database(dbname) + db["t"].create({"pk": int}, pk="pk") + db.enable_wal() + assert db.journal_mode == "wal" + result = runner.invoke(cli.cli, ["disable-wal"] + dbs) + assert 0 == result.exit_code + for dbname in dbs: + db = Database(dbname) + assert db.journal_mode == "delete" + + @pytest.mark.parametrize( "args,expected", [ From 7536a5a0f6fc49ce1a6cb961f9fbe5edb7662c68 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Aug 2020 12:06:02 -0700 Subject: [PATCH 0161/1004] Release 0.15 Refs #132 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3946b81..ce39170 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v2_15: + +2.15 (2020-08-10) +----------------- + +- New ``db.enable_wal()`` and ``db.disable_wal()`` methods for enabling and disabling `Write-Ahead Logging `__ for a database file - see :ref:`python_api_wal` in the Python API documentation. +- Also ``sqlite-utils enable-wal file.db`` and ``sqlite-utils disable-wal file.db`` commands for doing the same thing on the command-line, see :ref:`WAL mode (CLI) `. (`#132 `__) + .. _v2_14_1: 2.14.1 (2020-08-05) diff --git a/setup.py b/setup.py index e52b054..345ec0c 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.14.1" +VERSION = "2.15" def get_long_description(): From 66ed36258a64d11d99794e9ac9b3c5c9bc1727a8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Aug 2020 14:04:32 -0700 Subject: [PATCH 0162/1004] Renaming from master to main --- .travis.yml | 2 +- README.md | 4 ++-- docs/index.rst | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index c75c1a2..43fba89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,6 @@ jobs: distributions: bdist_wheel password: ${PYPI_PASSWORD} on: - branch: master + branch: main tags: true repo: simonw/sqlite-utils diff --git a/README.md b/README.md index 29c04da..e5a3c30 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/) [![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.readthedocs.io/en/stable/changelog.html) -[![Travis CI](https://travis-ci.com/simonw/sqlite-utils.svg?branch=master)](https://travis-ci.com/simonw/sqlite-utils) +[![Travis CI](https://travis-ci.com/simonw/sqlite-utils.svg?branch=main)](https://travis-ci.com/simonw/sqlite-utils) [![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.readthedocs.io/en/latest/?badge=latest) -[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/master/LICENSE) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE) Python CLI utility and library for manipulating SQLite databases. diff --git a/docs/index.rst b/docs/index.rst index 35ffcb0..f828df4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,10 +8,10 @@ :target: https://pypi.org/project/sqlite-utils/ .. |Changelog| image:: https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog :target: https://sqlite-utils.readthedocs.io/en/stable/changelog.html -.. |Travis CI| image:: https://travis-ci.com/simonw/sqlite-utils.svg?branch=master +.. |Travis CI| image:: https://travis-ci.com/simonw/sqlite-utils.svg?branch=main :target: https://travis-ci.com/simonw/sqlite-utils .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg - :target: https://github.com/simonw/sqlite-utils/blob/master/LICENSE + :target: https://github.com/simonw/sqlite-utils/blob/main/LICENSE *Python utility functions for manipulating SQLite databases* From d03fc607b355f22f7bfee4387d46f13f12463420 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2020 09:56:36 -0700 Subject: [PATCH 0163/1004] Package as sdist as well, refs #133 Also refs https://github.com/simonw/homebrew-datasette/issues/10 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 43fba89..e5c629e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ jobs: deploy: - provider: pypi user: simonw - distributions: bdist_wheel + distributions: sdist bdist_wheel password: ${PYPI_PASSWORD} on: branch: main From c8d796919281e6d97fe470d74f8580cc35fea625 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2020 09:59:48 -0700 Subject: [PATCH 0164/1004] Release 2.15.1 Refs #133 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ce39170..062683f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v2_15_1: + +2.15.1 (2020-08-12) +------------------- + +- Now available as a ``sdist`` package on PyPI in addition to a wheel. (`#133 `__) + .. _v2_15: 2.15 (2020-08-10) diff --git a/setup.py b/setup.py index 345ec0c..412a059 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.15" +VERSION = "2.15.1" def get_long_description(): From bf4c6b7c82fab6b2400e48424f8dac1ae2f0a2dc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 21 Aug 2020 13:30:02 -0700 Subject: [PATCH 0165/1004] find_spatialite() utility function, closes #135 --- docs/python-api.rst | 20 ++++++++++++++++++++ sqlite_utils/utils.py | 13 +++++++++++++ tests/test_utils.py | 5 +++++ 3 files changed, 38 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index b9b6013..9697fd3 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1298,3 +1298,23 @@ For example: # [age] INTEGER, # [thumbnail] BLOB # ) + +.. _find_spatialite: + +Finding SpatiaLite +================== + +The ``find_spatialite()`` function searches for the `SpatiaLite `__ SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found. + +You can use it in code like this: + +.. code-block:: python + + from sqlite_utils import Database + from sqlite_utils.utils import find_spatialite + + db = Database("mydb.db") + spatialite = find_spatialite() + if spatialite: + db.conn.enable_load_extension(True) + db.conn.load_extension(spatialite) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index e2d3451..89b9241 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -1,4 +1,5 @@ import base64 +import os try: import pysqlite3 as sqlite3 @@ -10,6 +11,11 @@ except ImportError: OperationalError = sqlite3.OperationalError +SPATIALITE_PATHS = ( + "/usr/lib/x86_64-linux-gnu/mod_spatialite.so", + "/usr/local/lib/mod_spatialite.dylib", +) + def suggest_column_types(records): all_column_types = {} @@ -74,3 +80,10 @@ def decode_base64_values(doc): if not to_fix: return doc return dict(doc, **{k: base64.b64decode(doc[k]["encoded"]) for k in to_fix}) + + +def find_spatialite(): + for path in SPATIALITE_PATHS: + if os.path.exists(path): + return path + return None diff --git a/tests/test_utils.py b/tests/test_utils.py index 6489f2b..a44b2e0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -20,3 +20,8 @@ def test_decode_base64_values(input, expected, should_be_is): assert actual is input else: assert actual == expected + + +def test_find_spatialite(): + spatialite = utils.find_spatialite() + assert spatialite is None or isinstance(spatialite, str) From 7e9aad7e1c09d1cf80d0b4d17d6157212a4b857d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 21 Aug 2020 13:54:11 -0700 Subject: [PATCH 0166/1004] --load-extension option for sqlite-utils query, closes #134 --- docs/cli.rst | 5 +++++ sqlite_utils/cli.py | 22 +++++++++++++++++++++- tests/test_cli.py | 25 ++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 6ac2c2d..15e2f3a 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -83,6 +83,11 @@ You can run queries against a temporary in-memory database by passing ``:memory: $ sqlite-utils :memory: "select sqlite_version()" [{"sqlite_version()": "3.29.0"}] +You can load SQLite extension modules using the `--load-extension` option:: + + $ sqlite-utils :memory: "select spatialite_version()" --load-extension=/usr/local/lib/mod_spatialite.dylib + [{"spatialite_version()": "4.3.0a"}] + .. _cli_json_values: Nested JSON values diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 79a8df0..c61d163 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -719,9 +719,29 @@ def drop_view(path, view): type=(str, str), help="Named :parameters for SQL query", ) -def query(path, sql, nl, arrays, csv, no_headers, table, fmt, json_cols, raw, param): +@click.option( + "--load-extension", multiple=True, help="SQLite extensions to load", +) +def query( + path, + sql, + nl, + arrays, + csv, + no_headers, + table, + fmt, + json_cols, + raw, + param, + load_extension, +): "Execute SQL query and return the results as JSON" db = sqlite_utils.Database(path) + if load_extension: + db.conn.enable_load_extension(True) + for ext in load_extension: + db.conn.load_extension(ext) with db.conn: cursor = db.conn.execute(sql, dict(param)) if cursor.description is None: diff --git a/tests/test_cli.py b/tests/test_cli.py index fea89dd..71a2503 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,7 +4,7 @@ from click.testing import CliRunner import json import os import pytest -from sqlite_utils.utils import sqlite3 +from sqlite_utils.utils import sqlite3, find_spatialite from .utils import collapse_whitespace @@ -885,6 +885,29 @@ def test_query_raw(db_path, content, is_binary): assert result.output == str(content) +@pytest.mark.skipif(not find_spatialite(), reason="Could not find SpatiaLite extension") +@pytest.mark.skipif( + not hasattr(sqlite3.Connection, "enable_load_extension"), + reason="sqlite3.Connection missing enable_load_extension", +) +def test_query_load_extension(): + # Without --load-extension: + result = CliRunner().invoke(cli.cli, [":memory:", "select spatialite_version()"]) + assert result.exit_code == 1 + assert "no such function: spatialite_version" in repr(result) + # With --load-extension: + result = CliRunner().invoke( + cli.cli, + [ + ":memory:", + "select spatialite_version()", + "--load-extension={}".format(find_spatialite()), + ], + ) + assert result.exit_code == 0, result.stdout + assert ["spatialite_version()"] == list(json.loads(result.output)[0].keys()) + + def test_query_memory_does_not_create_file(tmpdir): owd = os.getcwd() try: From c8b243348197c540710154fce7e7009d8e7f6699 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 21 Aug 2020 14:01:44 -0700 Subject: [PATCH 0167/1004] Install spatialite in Travis for --load-extension tests, refs #134 --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index e5c629e..0360337 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,11 @@ python: - "3.7" - "3.8" +before_install: + # SpatiaLite needed for the --load-extension test: + - sudo apt-get update + - sudo apt-get -y install libsqlite3-mod-spatialite + script: - pip install -U pip wheel - pip install .[test] From ea87c2b943fdd162c42a900ac0aea5ecc2f4b9d9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 21 Aug 2020 14:02:29 -0700 Subject: [PATCH 0168/1004] Release 2.16 Refs #134, #135 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 062683f..041c8b1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v2_16: + +2.16 (2020-08-21) +----------------- + +- ``--load-extension`` option for ``sqlite-utils query`` for loading SQLite extensions. (`#134 `__) +- New ``sqlite_utils.utils.find_spatialite()`` function for finding SpatiaLite in common locations. (`#135 `__) + .. _v2_15_1: 2.15.1 (2020-08-12) diff --git a/setup.py b/setup.py index 412a059..f3f97f1 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.15.1" +VERSION = "2.16" def get_long_description(): From 947bb7626fd1763608a470adf9cf5f156ef003e9 Mon Sep 17 00:00:00 2001 From: Simon Wiles Date: Fri, 28 Aug 2020 15:30:13 -0700 Subject: [PATCH 0169/1004] insert_all(..., alter=True) works for columns introduced after first 100 records * Insert all columns for every chunk * Update unit test to reflect new behaviour * Test that exception is raised * Update documentation Closes #139. Thanks, Simon Wiles! --- docs/python-api.rst | 2 +- sqlite_utils/db.py | 8 ++++++++ tests/test_create.py | 17 ++++++++++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 9697fd3..5860397 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -410,7 +410,7 @@ Use it like this: "is_good_dog": True, }], pk="id", column_order=("id", "twitter", "name")) -The column types used in the ``CREATE TABLE`` statement are automatically derived from the types of data in that first batch of rows. Any additional or missing columns in subsequent batches will be ignored. +The column types used in the ``CREATE TABLE`` statement are automatically derived from the types of data in that first batch of rows. Any additional columns in subsequent batches will cause a ``sqlite3.OperationalError`` exception to be raised unless the ``alter=True`` argument is supplied, in which case the new columns will be created. The function can accept an iterator or generator of rows and will commit them according to the batch size. The default batch size is 100, but you can specify a different size using the ``batch_size`` parameter: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a8791c3..75599f6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1074,6 +1074,14 @@ class Table(Queryable): all_columns = list(sorted(all_columns)) if hash_id: all_columns.insert(0, hash_id) + else: + all_columns += [ + column + for record in chunk + for column in record + if column not in all_columns + ] + validate_column_names(all_columns) first = False # values is the list of insert data that is passed to the diff --git a/tests/test_create.py b/tests/test_create.py index a84eb8d..fc8edc0 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -707,13 +707,24 @@ def test_insert_thousands_using_generator(fresh_db): assert 10000 == fresh_db["test"].count -def test_insert_thousands_ignores_extra_columns_after_first_100(fresh_db): +def test_insert_thousands_raises_exception_wtih_extra_columns_after_first_100(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/139 + with pytest.raises(Exception, match="table test has no column named extra"): + fresh_db["test"].insert_all( + [{"i": i, "word": "word_{}".format(i)} for i in range(100)] + + [{"i": 101, "extra": "This extra column should cause an exception"}], + ) + + +def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/139 fresh_db["test"].insert_all( [{"i": i, "word": "word_{}".format(i)} for i in range(100)] - + [{"i": 101, "extra": "This extra column should cause an exception"}] + + [{"i": 101, "extra": "Should trigger ALTER"}], + alter=True, ) rows = fresh_db.execute_returning_dicts("select * from test where i = 101") - assert [{"i": 101, "word": None}] == rows + assert [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] == rows def test_insert_ignore(fresh_db): From 10c6fbc3689311091a18ad4f8d1098e6761c35bc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 Aug 2020 15:30:57 -0700 Subject: [PATCH 0170/1004] Applied Black 20.8b1, refs #142 --- sqlite_utils/cli.py | 52 +++++++++++++++++++++++++------ tests/test_cli.py | 73 ++++++++++++++++++++++++++++++++++++-------- tests/test_create.py | 7 ++++- tests/test_fts.py | 3 +- 4 files changed, 111 insertions(+), 24 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c61d163..e4ac573 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -84,7 +84,10 @@ def cli(): default=False, ) @click.option( - "--schema", help="Include schema for each table", is_flag=True, default=False, + "--schema", + help="Include schema for each table", + is_flag=True, + default=False, ) def tables( path, @@ -161,10 +164,23 @@ def tables( default=False, ) @click.option( - "--schema", help="Include schema for each view", is_flag=True, default=False, + "--schema", + help="Include schema for each view", + is_flag=True, + default=False, ) def views( - path, counts, nl, arrays, csv, no_headers, table, fmt, json_cols, columns, schema, + path, + counts, + nl, + arrays, + csv, + no_headers, + table, + fmt, + json_cols, + columns, + schema, ): """List the views in the database""" tables.callback( @@ -585,7 +601,9 @@ def upsert( @click.argument("columns", nargs=-1, required=True) @click.option("--pk", help="Column to use as primary key") @click.option( - "--not-null", multiple=True, help="Columns that should be created as NOT NULL", + "--not-null", + multiple=True, + help="Columns that should be created as NOT NULL", ) @click.option( "--default", @@ -600,10 +618,14 @@ def upsert( help="Column, other table, other column to set as a foreign key", ) @click.option( - "--ignore", is_flag=True, help="If table already exists, do nothing", + "--ignore", + is_flag=True, + help="If table already exists, do nothing", ) @click.option( - "--replace", is_flag=True, help="If table already exists, replace it", + "--replace", + is_flag=True, + help="If table already exists, replace it", ) def create_table(path, table, columns, pk, not_null, default, fk, ignore, replace): "Add an index to the specified table covering the specified columns" @@ -664,10 +686,14 @@ def drop_table(path, table): @click.argument("view") @click.argument("select") @click.option( - "--ignore", is_flag=True, help="If view already exists, do nothing", + "--ignore", + is_flag=True, + help="If view already exists, do nothing", ) @click.option( - "--replace", is_flag=True, help="If view already exists, replace it", + "--replace", + is_flag=True, + help="If view already exists, replace it", ) def create_view(path, view, select, ignore, replace): "Create a view for the provided SELECT query" @@ -720,7 +746,9 @@ def drop_view(path, view): help="Named :parameters for SQL query", ) @click.option( - "--load-extension", multiple=True, help="SQLite extensions to load", + "--load-extension", + multiple=True, + help="SQLite extensions to load", ) def query( path, @@ -808,7 +836,11 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols) type=click.Path(file_okay=True, dir_okay=True, allow_dash=True), ) @click.option( - "-c", "--column", type=str, multiple=True, help="Column definitions for the table", + "-c", + "--column", + type=str, + multiple=True, + help="Column definitions for the table", ) @click.option("--pk", type=str, help="Column to use as primary key") @click.option("--alter", is_flag=True, help="Alter table to add missing columns") diff --git a/tests/test_cli.py b/tests/test_cli.py index 71a2503..e2ae298 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -800,7 +800,11 @@ def test_query_json_binary(db_path): db = Database(db_path) with db.conn: db["files"].insert( - {"name": "lorem.txt", "sz": 16984, "data": LOREM_IPSUM_COMPRESSED,}, + { + "name": "lorem.txt", + "sz": 16984, + "data": LOREM_IPSUM_COMPRESSED, + }, pk="name", ) result = CliRunner().invoke(cli.cli, [db_path, "select name, sz, data from files"]) @@ -1002,9 +1006,9 @@ def test_upsert_alter(db_path, tmpdir): cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id", "--alter"] ) assert 0 == result.exit_code - assert [{"id": 1, "name": "Cleo", "age": 5},] == db.execute_returning_dicts( - "select * from dogs order by id" - ) + assert [ + {"id": 1, "name": "Cleo", "age": 5}, + ] == db.execute_returning_dicts("select * from dogs order by id") @pytest.mark.parametrize( @@ -1012,7 +1016,12 @@ def test_upsert_alter(db_path, tmpdir): [ # No primary key ( - ["name", "text", "age", "integer",], + [ + "name", + "text", + "age", + "integer", + ], ("CREATE TABLE [t] (\n [name] TEXT,\n [age] INTEGER\n)"), ), # All types: @@ -1057,7 +1066,14 @@ def test_create_table(args, schema): runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke( - cli.cli, ["create-table", "test.db", "t",] + args, catch_exceptions=False + cli.cli, + [ + "create-table", + "test.db", + "t", + ] + + args, + catch_exceptions=False, ) assert 0 == result.exit_code db = Database("test.db") @@ -1217,7 +1233,14 @@ def test_drop_table(): db = Database("test.db") db["t"].create({"pk": int}, pk="pk") assert "t" in db.table_names() - result = runner.invoke(cli.cli, ["drop-table", "test.db", "t",],) + result = runner.invoke( + cli.cli, + [ + "drop-table", + "test.db", + "t", + ], + ) assert 0 == result.exit_code assert "t" not in db.table_names() @@ -1227,7 +1250,14 @@ def test_drop_table_error(): with runner.isolated_filesystem(): db = Database("test.db") db["t"].create({"pk": int}, pk="pk") - result = runner.invoke(cli.cli, ["drop-table", "test.db", "t2",],) + result = runner.invoke( + cli.cli, + [ + "drop-table", + "test.db", + "t2", + ], + ) assert 1 == result.exit_code assert 'Error: Table "t2" does not exist' == result.output.strip() @@ -1238,7 +1268,14 @@ def test_drop_view(): db = Database("test.db") db.create_view("hello", "select 1") assert "hello" in db.view_names() - result = runner.invoke(cli.cli, ["drop-view", "test.db", "hello",],) + result = runner.invoke( + cli.cli, + [ + "drop-view", + "test.db", + "hello", + ], + ) assert 0 == result.exit_code assert "hello" not in db.view_names() @@ -1248,7 +1285,14 @@ def test_drop_view_error(): with runner.isolated_filesystem(): db = Database("test.db") db["t"].create({"pk": int}, pk="pk") - result = runner.invoke(cli.cli, ["drop-view", "test.db", "t2",],) + result = runner.invoke( + cli.cli, + [ + "drop-view", + "test.db", + "t2", + ], + ) assert 1 == result.exit_code assert 'Error: View "t2" does not exist' == result.output.strip() @@ -1287,7 +1331,10 @@ def test_disable_wal(): @pytest.mark.parametrize( "args,expected", [ - ([], '[{"rows_affected": 1}]',), + ( + [], + '[{"rows_affected": 1}]', + ), (["-t"], "rows_affected\n---------------\n 1"), ], ) @@ -1295,7 +1342,9 @@ def test_query_update(db_path, args, expected): db = Database(db_path) with db.conn: db["dogs"].insert_all( - [{"id": 1, "age": 4, "name": "Cleo"},] + [ + {"id": 1, "age": 4, "name": "Cleo"}, + ] ) result = CliRunner().invoke( cli.cli, [db_path, "update dogs set age = 5 where name = 'Cleo'"] + args diff --git a/tests/test_create.py b/tests/test_create.py index fc8edc0..f6fad00 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -669,7 +669,12 @@ def test_create_index_if_not_exists(fresh_db): ["list with", "two items"], {"dictionary": "simple"}, {"dictionary": {"nested": "complex"}}, - collections.OrderedDict([("key1", {"nested": "complex"}), ("key2", "foo"),]), + collections.OrderedDict( + [ + ("key1", {"nested": "complex"}), + ("key2", "foo"), + ] + ), [{"list": "of"}, {"two": "dicts"}], ), ) diff --git a/tests/test_fts.py b/tests/test_fts.py index bac4117..b1a946d 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -98,7 +98,8 @@ def test_fts_tokenize(fresh_db): table.insert_all(search_records) # Test without porter stemming table.enable_fts( - ["text", "country"], fts_version="FTS{}".format(fts_version), + ["text", "country"], + fts_version="FTS{}".format(fts_version), ) assert [] == table.search("bite") # Test WITH stemming From 8c405965e9f0b55900a41ff16ae7e6077bca6ef6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 Aug 2020 15:36:55 -0700 Subject: [PATCH 0171/1004] GitHub Actions workflows, refs #143 --- .github/workflows/publish.yml | 57 +++++++++++++++++++++++++++++++++++ .github/workflows/test.yml | 31 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..0a55018 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,57 @@ +name: Publish Python Package + +on: + release: + types: [created] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install dependencies + run: | + pip install -e '.[test]' + - name: Run tests + run: | + pytest + deploy: + runs-on: ubuntu-latest + needs: [test] + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-publish-pip- + - name: Install dependencies + run: | + pip install setuptools wheel twine + - name: Publish + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..72ce06a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,31 @@ +name: Test + +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install dependencies + run: | + pip install -e '.[test]' + - name: Run tests + run: | + pytest + - name: Check formatting + run: black . --check From d7d3f962861ef32c5ead8f514c8756f5b6f7c4a0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 Aug 2020 15:37:45 -0700 Subject: [PATCH 0172/1004] Black now runs in GitHub actions, refs #143 --- tests/test_black.py | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 tests/test_black.py diff --git a/tests/test_black.py b/tests/test_black.py deleted file mode 100644 index e4b1b01..0000000 --- a/tests/test_black.py +++ /dev/null @@ -1,20 +0,0 @@ -import black -from click.testing import CliRunner -from pathlib import Path -import pytest -import sys - -code_root = Path(__file__).parent.parent - - -@pytest.mark.skipif( - sys.version_info[:2] > (3, 6), - reason="Breaks on 3.7 at the moment, but it only needs to run under one Python version", -) -def test_black(): - runner = CliRunner() - result = runner.invoke( - black.main, - [str(code_root / "tests"), str(code_root / "sqlite_utils"), "--check"], - ) - assert result.exit_code == 0, result.output From 01c7784be54d14ee5b653753c38005d823fcdd09 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 Aug 2020 15:41:29 -0700 Subject: [PATCH 0173/1004] CI is now GitHub Actions, closes #143 --- .travis.yml | 42 ------------------------------------------ README.md | 2 +- docs/index.rst | 6 +++--- setup.py | 2 +- 4 files changed, 5 insertions(+), 47 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0360337..0000000 --- a/.travis.yml +++ /dev/null @@ -1,42 +0,0 @@ -language: python -dist: bionic - -# 3.6 is listed first so it gets used for the later build stages -python: - - "3.6" - - "3.7" - - "3.8" - -before_install: - # SpatiaLite needed for the --load-extension test: - - sudo apt-get update - - sudo apt-get -y install libsqlite3-mod-spatialite - -script: - - pip install -U pip wheel - - pip install .[test] - # Only run the numpy/pandas tests on Python 3.7: - - python -c "import sys; print(sys.version_info.minor == 7)" | grep True > /dev/null && pip install pandas || true - - pytest - -cache: - directories: - - $HOME/.cache/pip - -jobs: - include: - - stage: release tagged version - if: tag IS present - language: python - python: 3.6 - script: - - pip install -U pip wheel - deploy: - - provider: pypi - user: simonw - distributions: sdist bdist_wheel - password: ${PYPI_PASSWORD} - on: - branch: main - tags: true - repo: simonw/sqlite-utils diff --git a/README.md b/README.md index e5a3c30..acd5356 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/) [![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.readthedocs.io/en/stable/changelog.html) -[![Travis CI](https://travis-ci.com/simonw/sqlite-utils.svg?branch=main)](https://travis-ci.com/simonw/sqlite-utils) +[![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest) [![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.readthedocs.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE) diff --git a/docs/index.rst b/docs/index.rst index f828df4..6fe980f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,14 +2,14 @@ sqlite-utils |version| ======================= -|PyPI| |Changelog| |Travis CI| |License| +|PyPI| |Changelog| |CI| |License| .. |PyPI| image:: https://img.shields.io/pypi/v/sqlite-utils.svg :target: https://pypi.org/project/sqlite-utils/ .. |Changelog| image:: https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog :target: https://sqlite-utils.readthedocs.io/en/stable/changelog.html -.. |Travis CI| image:: https://travis-ci.com/simonw/sqlite-utils.svg?branch=main - :target: https://travis-ci.com/simonw/sqlite-utils +.. |CI| image:: https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg + :target: https://github.com/simonw/sqlite-utils/actions .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg :target: https://github.com/simonw/sqlite-utils/blob/main/LICENSE diff --git a/setup.py b/setup.py index f3f97f1..df11f38 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ setup( "Changelog": "https://sqlite-utils.readthedocs.io/en/stable/changelog.html", "Source code": "https://github.com/simonw/sqlite-utils", "Issues": "https://github.com/simonw/sqlite-utils/issues", - "CI": "https://travis-ci.com/simonw/sqlite-utils", + "CI": "https://github.com/simonw/sqlite-utils/actions", }, classifiers=[ "Development Status :: 5 - Production/Stable", From 1a9dab86fe22b122ea44e2161887fe3c0129297f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 Aug 2020 15:45:11 -0700 Subject: [PATCH 0174/1004] Release 2.16.1 Refs #139, #142, #143 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 041c8b1..175e468 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v2_16_1: + +2.16.1 (2020-08-28) +------------------- + +- ``insert_all(..., alter=True)`` now works for columns introduced after the first 100 records. Thanks, Simon Wiles! (`#139 `__) +- Continuous Integration is now powered by GitHub Actions. (`#143 `__) + .. _v2_16: 2.16 (2020-08-21) diff --git a/setup.py b/setup.py index df11f38..8aa60cb 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.16" +VERSION = "2.16.1" def get_long_description(): From cbc22ef20cf7326b90a11661931f155f81f700fd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 Aug 2020 15:56:06 -0700 Subject: [PATCH 0175/1004] Add numpy to the matrix, refs #144 --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 72ce06a..350b07d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,6 +8,7 @@ jobs: strategy: matrix: python-version: [3.6, 3.7, 3.8] + numpy: [0, 1] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -24,6 +25,9 @@ jobs: - name: Install dependencies run: | pip install -e '.[test]' + - name: Optionally install numpy + if: matrix.numpy == 1 + run: pip install numpy - name: Run tests run: | pytest From e0cd430e8905324bb0c9143b3adc8ea5fcf60d99 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Sep 2020 13:17:01 -0700 Subject: [PATCH 0176/1004] Docs for sqlite_utils.AlterError in add_foreign_keys() --- docs/python-api.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 5860397..650e1ab 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -802,6 +802,8 @@ The ``table.add_foreign_key(column, other_table, other_column)`` method takes th - If the column is of format ``author_id``, look for tables called ``author`` or ``authors`` - If the column does not end in ``_id``, try looking for a table with the exact name of the column or that name with an added ``s`` +This method first checks that the specified foreign key references tables and columns that exist and does not clash with an existing foreign key. It will raise a ``sqlite_utils.AlterError`` exception if these checks fail. + .. _python_api_add_foreign_keys: Adding multiple foreign key constraints at once @@ -822,6 +824,8 @@ Here's an example adding two foreign keys at once: ("dogs", "home_town_id", "towns", "id") ]) +This method runs the same checks as ``.add_foreign_keys()`` and will raise ``sqlite_utils.AlterError`` if those checks fail. + .. _python_api_index_foreign_keys: Adding indexes for all foreign keys From 0e62744da9a429093e3409575c1f881376b0361f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Sep 2020 13:29:46 -0700 Subject: [PATCH 0177/1004] Correct import path for AlterError exception --- docs/python-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 650e1ab..23de88f 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -802,7 +802,7 @@ The ``table.add_foreign_key(column, other_table, other_column)`` method takes th - If the column is of format ``author_id``, look for tables called ``author`` or ``authors`` - If the column does not end in ``_id``, try looking for a table with the exact name of the column or that name with an added ``s`` -This method first checks that the specified foreign key references tables and columns that exist and does not clash with an existing foreign key. It will raise a ``sqlite_utils.AlterError`` exception if these checks fail. +This method first checks that the specified foreign key references tables and columns that exist and does not clash with an existing foreign key. It will raise a ``sqlite_utils.db.AlterError`` exception if these checks fail. .. _python_api_add_foreign_keys: @@ -824,7 +824,7 @@ Here's an example adding two foreign keys at once: ("dogs", "home_town_id", "towns", "id") ]) -This method runs the same checks as ``.add_foreign_keys()`` and will raise ``sqlite_utils.AlterError`` if those checks fail. +This method runs the same checks as ``.add_foreign_keys()`` and will raise ``sqlite_utils.db.AlterError`` if those checks fail. .. _python_api_index_foreign_keys: From 59e3d4d1715192ef7b6710ac970f5f4849ab0f0d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Sep 2020 11:12:45 -0700 Subject: [PATCH 0178/1004] Neater indentation for SQL used in schemas, closes #148 --- sqlite_utils/db.py | 15 ++++++++------- tests/test_cli.py | 8 ++++---- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 75599f6..9ca7dd7 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -7,6 +7,7 @@ import itertools import json import os import pathlib +import textwrap import uuid SQLITE_MAX_VARS = 999 @@ -658,10 +659,10 @@ class Table(Queryable): index_name = "idx_{}_{}".format( self.name.replace(" ", "_"), "_".join(columns) ) - sql = """ + sql = textwrap.dedent(""" CREATE {unique}INDEX {if_not_exists}[{index_name}] ON [{table_name}] ({columns}); - """.format( + """).strip().format( index_name=index_name, table_name=self.name, columns=", ".join("[{}]".format(c) for c in columns), @@ -779,16 +780,16 @@ class Table(Queryable): self, columns, fts_version="FTS5", create_triggers=False, tokenize=None ): "Enables FTS on the specified columns." - sql = """ + sql = textwrap.dedent(""" CREATE VIRTUAL TABLE [{table}_fts] USING {fts_version} ( {columns},{tokenize} content=[{table}] ); - """.format( + """).strip().format( table=self.name, columns=", ".join("[{}]".format(c) for c in columns), fts_version=fts_version, - tokenize="\n tokenize='{}',".format(tokenize) + tokenize="\n tokenize='{}',".format(tokenize) if tokenize else "", ) @@ -798,7 +799,7 @@ class Table(Queryable): if create_triggers: old_cols = ", ".join("old.[{}]".format(c) for c in columns) new_cols = ", ".join("new.[{}]".format(c) for c in columns) - triggers = """ + triggers = textwrap.dedent(""" CREATE TRIGGER [{table}_ai] AFTER INSERT ON [{table}] BEGIN INSERT INTO [{table}_fts] (rowid, {columns}) VALUES (new.rowid, {new_cols}); END; @@ -809,7 +810,7 @@ class Table(Queryable): INSERT INTO [{table}_fts] ([{table}_fts], rowid, {columns}) VALUES('delete', old.rowid, {old_cols}); INSERT INTO [{table}_fts] (rowid, {columns}) VALUES (new.rowid, {new_cols}); END; - """.format( + """).strip().format( table=self.name, columns=", ".join("[{}]".format(c) for c in columns), old_cols=old_cols, diff --git a/tests/test_cli.py b/tests/test_cli.py index e2ae298..49631c9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -373,10 +373,10 @@ def test_enable_fts(db_path): # Check tokenize was set to porter assert ( "CREATE VIRTUAL TABLE [http://example.com_fts] USING FTS4 (\n" - " [c1],\n" - " tokenize='porter',\n" - " content=[http://example.com]" - "\n )" + " [c1],\n" + " tokenize='porter',\n" + " content=[http://example.com]" + "\n)" ) == db["http://example.com_fts"].schema db["http://example.com"].drop() From e878f2a8fe110ed5cf68e49c9902b641022c5b1d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Sep 2020 12:45:54 -0700 Subject: [PATCH 0179/1004] Applied latest black --- sqlite_utils/db.py | 58 +++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9ca7dd7..6bd779a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -659,15 +659,21 @@ class Table(Queryable): index_name = "idx_{}_{}".format( self.name.replace(" ", "_"), "_".join(columns) ) - sql = textwrap.dedent(""" + sql = ( + textwrap.dedent( + """ CREATE {unique}INDEX {if_not_exists}[{index_name}] ON [{table_name}] ({columns}); - """).strip().format( - index_name=index_name, - table_name=self.name, - columns=", ".join("[{}]".format(c) for c in columns), - unique="UNIQUE " if unique else "", - if_not_exists="IF NOT EXISTS " if if_not_exists else "", + """ + ) + .strip() + .format( + index_name=index_name, + table_name=self.name, + columns=", ".join("[{}]".format(c) for c in columns), + unique="UNIQUE " if unique else "", + if_not_exists="IF NOT EXISTS " if if_not_exists else "", + ) ) self.db.conn.execute(sql) return self @@ -780,18 +786,22 @@ class Table(Queryable): self, columns, fts_version="FTS5", create_triggers=False, tokenize=None ): "Enables FTS on the specified columns." - sql = textwrap.dedent(""" + sql = ( + textwrap.dedent( + """ CREATE VIRTUAL TABLE [{table}_fts] USING {fts_version} ( {columns},{tokenize} content=[{table}] ); - """).strip().format( - table=self.name, - columns=", ".join("[{}]".format(c) for c in columns), - fts_version=fts_version, - tokenize="\n tokenize='{}',".format(tokenize) - if tokenize - else "", + """ + ) + .strip() + .format( + table=self.name, + columns=", ".join("[{}]".format(c) for c in columns), + fts_version=fts_version, + tokenize="\n tokenize='{}',".format(tokenize) if tokenize else "", + ) ) self.db.conn.executescript(sql) self.populate_fts(columns) @@ -799,7 +809,9 @@ class Table(Queryable): if create_triggers: old_cols = ", ".join("old.[{}]".format(c) for c in columns) new_cols = ", ".join("new.[{}]".format(c) for c in columns) - triggers = textwrap.dedent(""" + triggers = ( + textwrap.dedent( + """ CREATE TRIGGER [{table}_ai] AFTER INSERT ON [{table}] BEGIN INSERT INTO [{table}_fts] (rowid, {columns}) VALUES (new.rowid, {new_cols}); END; @@ -810,11 +822,15 @@ class Table(Queryable): INSERT INTO [{table}_fts] ([{table}_fts], rowid, {columns}) VALUES('delete', old.rowid, {old_cols}); INSERT INTO [{table}_fts] (rowid, {columns}) VALUES (new.rowid, {new_cols}); END; - """).strip().format( - table=self.name, - columns=", ".join("[{}]".format(c) for c in columns), - old_cols=old_cols, - new_cols=new_cols, + """ + ) + .strip() + .format( + table=self.name, + columns=", ".join("[{}]".format(c) for c in columns), + old_cols=old_cols, + new_cols=new_cols, + ) ) self.db.conn.executescript(triggers) return self From de1059034486166131f2b2cd59ad69b4d26d6e25 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Sep 2020 13:45:06 -0700 Subject: [PATCH 0180/1004] recursive_triggers=on by default, closes #152 Refs #149 --- docs/python-api.rst | 6 ++++++ sqlite_utils/db.py | 10 +++++++++- tests/test_constructor.py | 12 ++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/test_constructor.py diff --git a/docs/python-api.rst b/docs/python-api.rst index 23de88f..ad83b99 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -37,6 +37,12 @@ If you want to create an in-memory database, you can do so like this: db = Database(memory=True) +Connections use ``PRAGMA recursive_triggers=on`` by default. If you don't want to use `recursive triggers `__ you can turn them off using: + +.. code-block:: python + + db = Database(memory=True, recursive_triggers=False) + Tables are accessed using the indexing operator, like so: .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6bd779a..de67bce 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -103,7 +103,13 @@ class PrimaryKeyRequired(Exception): class Database: - def __init__(self, filename_or_conn=None, memory=False, recreate=False): + def __init__( + self, + filename_or_conn=None, + memory=False, + recreate=False, + recursive_triggers=True, + ): assert (filename_or_conn is not None and not memory) or ( filename_or_conn is None and memory ), "Either specify a filename_or_conn or pass memory=True" @@ -116,6 +122,8 @@ class Database: else: assert not recreate, "recreate cannot be used with connections, only paths" self.conn = filename_or_conn + if recursive_triggers: + self.conn.execute("PRAGMA recursive_triggers=on;") def __getitem__(self, table_name): return self.table(table_name) diff --git a/tests/test_constructor.py b/tests/test_constructor.py new file mode 100644 index 0000000..8a790c2 --- /dev/null +++ b/tests/test_constructor.py @@ -0,0 +1,12 @@ +from sqlite_utils import Database +import pytest + + +def test_recursive_triggers(): + db = Database(memory=True) + assert db.conn.execute("PRAGMA recursive_triggers").fetchone()[0] + + +def test_recursive_triggers_off(): + db = Database(memory=True, recursive_triggers=False) + assert not db.conn.execute("PRAGMA recursive_triggers").fetchone()[0] From c44906429735e9c23774404dc105913f3ff90b7c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Sep 2020 13:46:12 -0700 Subject: [PATCH 0181/1004] Additional tests for WAL mode This should have been included in 2d2d724e32824095b0bf267a38d9c6fd628cc706 Refs #132 --- tests/test_wal.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 tests/test_wal.py diff --git a/tests/test_wal.py b/tests/test_wal.py new file mode 100644 index 0000000..1303eed --- /dev/null +++ b/tests/test_wal.py @@ -0,0 +1,24 @@ +import pytest +from sqlite_utils import Database +import sqlite3 + + +@pytest.fixture +def db_path_tmpdir(tmpdir): + path = tmpdir / "test.db" + db = Database(str(path)) + return db, path, tmpdir + + +def test_enable_disable_wal(db_path_tmpdir): + db, path, tmpdir = db_path_tmpdir + assert len(tmpdir.listdir()) == 1 + assert "delete" == db.journal_mode + assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()] + db.enable_wal() + assert "wal" == db.journal_mode + db["test"].insert({"foo": "bar"}) + assert "test.db-wal" in [f.basename for f in tmpdir.listdir()] + db.disable_wal() + assert "delete" == db.journal_mode + assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()] From 3e87500e1561f5c4e105cd026d33e0f715cc7dea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Sep 2020 14:16:13 -0700 Subject: [PATCH 0182/1004] table.optimize() deletes junk docsize rows Closes #153. Closes #149. --- docs/dogs.db | 0 docs/python-api.rst | 4 ++++ sqlite_utils/db.py | 10 ++++++++++ tests/test_optimize.py | 20 ++++++++++++++++++++ 4 files changed, 34 insertions(+) create mode 100644 docs/dogs.db create mode 100644 tests/test_optimize.py diff --git a/docs/dogs.db b/docs/dogs.db new file mode 100644 index 0000000..e69de29 diff --git a/docs/python-api.rst b/docs/python-api.rst index ad83b99..134c824 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1178,6 +1178,10 @@ Once you have populated a FTS table you can optimize it to dramatically reduce i This runs the following SQL:: INSERT INTO dogs_fts (dogs_fts) VALUES ("optimize"); + DELETE FROM [dogs_fts_docsize] WHERE id NOT IN ( + SELECT rowid FROM [dogs_fts]); + +That ``DELETE`` statement cleans up rows that may have been created by `an obscure bug `__ in previous versions of ``sqlite-utils``. Creating indexes ================ diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index de67bce..f5b1b12 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -903,6 +903,16 @@ class Table(Queryable): table=fts_table ) ) + self.db.conn.execute( + """ + DELETE FROM [{table}_docsize] WHERE {column} NOT IN ( + SELECT rowid FROM [{table}]); + """.format( + # FTS5 uses 'id' but FTS4 uses 'docid' + column=self.db["{}_docsize".format(fts_table)].columns[0].name, + table=fts_table, + ) + ) return self def search(self, q): diff --git a/tests/test_optimize.py b/tests/test_optimize.py new file mode 100644 index 0000000..302532c --- /dev/null +++ b/tests/test_optimize.py @@ -0,0 +1,20 @@ +import pytest +from sqlite_utils import Database +import sqlite3 + + +@pytest.mark.parametrize("fts_version", ["FTS4", "FTS5"]) +def test_optimizes_removes_junk_docsize_rows(tmpdir, fts_version): + # Recreating https://github.com/simonw/sqlite-utils/issues/149 + path = tmpdir / "test.db" + db = Database(str(path), recursive_triggers=False) + licenses = [{"key": "apache2", "name": "Apache 2"}, {"key": "bsd", "name": "BSD"}] + db["licenses"].insert_all(licenses, pk="key", replace=True) + db["licenses"].enable_fts(["name"], create_triggers=True, fts_version=fts_version) + assert db["licenses_fts_docsize"].count == 2 + # Bug: insert with replace increases the number of rows in _docsize: + db["licenses"].insert_all(licenses, pk="key", replace=True) + assert db["licenses_fts_docsize"].count == 4 + # Optimize should fix this: + db["licenses_fts"].optimize() + assert db["licenses_fts_docsize"].count == 2 From cf2cb244faf992118f34aa196387a4ef8b39a20f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Sep 2020 14:56:59 -0700 Subject: [PATCH 0183/1004] Tracer mechanism for showing underlying SQL queries * Pass a tracer= function to Database constructor * New db.tracer() contextmanager * Neater SQL indentation, because tracer means it could be visible now * New db.execute() and db.executescript() methods Closes #150 --- docs/python-api.rst | 53 ++++++++++- sqlite_utils/cli.py | 2 +- sqlite_utils/db.py | 163 +++++++++++++++++++++------------- tests/conftest.py | 2 +- tests/test_cli.py | 4 +- tests/test_column_affinity.py | 2 +- tests/test_constructor.py | 4 +- tests/test_create.py | 2 +- tests/test_create_view.py | 10 +-- tests/test_fts.py | 4 +- tests/test_introspect.py | 2 +- tests/test_tracer.py | 62 +++++++++++++ 12 files changed, 231 insertions(+), 79 deletions(-) create mode 100644 tests/test_tracer.py diff --git a/docs/python-api.rst b/docs/python-api.rst index 134c824..5d3409d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -43,6 +43,53 @@ Connections use ``PRAGMA recursive_triggers=on`` by default. If you don't want t db = Database(memory=True, recursive_triggers=False) +.. _python_api_tracing: + +Tracing queries +--------------- + +You can use the ``tracer`` mechanism to see SQL queries that are being executed by SQLite. A tracer is a function that you provide which will be called with ``sql`` and ``params`` arguments every time SQL is executed, for example: + +.. code-block:: python + + def tracer(sql, params): + print("SQL: {} - params: {}".format(sql, params)) + +You can pass this function to the ``Database()`` constructor like so: + +.. code-block:: python + + db = Database(memory=True, tracer=tracer) + +You can also turn on a tracer function temporarily for a block of code using the ``with db.tracer(...)`` context manager: + +.. code-block:: python + + db = Database(memory=True) + # ... later + with db.tracer(tracer): + db["dogs"].insert({"name": "Cleo"}) + +Queries will be passed to your ``tracer()`` function only for the duration of the ``with`` block. + +.. _python_api_execute: + +Executing queries +================= + +The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around ``.execute()`` and ``.executescript()`` on the underlying SQLite connection. These wrappers log to the tracer function if one has been registered. + +.. code-block:: python + + db = Database(memory=True) + db["dogs"].insert({"name": "Cleo"}) + db.execute("update dogs set name = 'Cleopaws'") + +.. _python_api_table: + +Accessing tables +================ + Tables are accessed using the indexing operator, like so: .. code-block:: python @@ -930,7 +977,7 @@ For example: "postalCode": "95018" } }) - db.conn.execute(""" + db.execute(""" select json_extract(address, '$.addressLocality') from niche_museums """).fetchall() @@ -979,8 +1026,8 @@ A more useful example: if you are working with `SpatiaLite ".format(self.conn) + def execute(self, sql, parameters=None): + if self._tracer: + self._tracer(sql, parameters) + if parameters is not None: + return self.conn.execute(sql, parameters) + else: + return self.conn.execute(sql) + + def executescript(self, sql): + if self._tracer: + self._tracer(sql, None) + return self.conn.executescript(sql) + def table(self, table_name, **kwargs): klass = View if table_name in self.view_names() else Table return klass(self, table_name, **kwargs) @@ -139,7 +164,7 @@ class Database: # Normally we would use .execute(sql, [params]) for escaping, but # occasionally that isn't available - most notable when we need # to include a "... DEFAULT 'value'" in a column definition. - return self.conn.execute( + return self.execute( # Use SQLite itself to correctly escape this string: "SELECT quote(:value)", {"value": value}, @@ -152,12 +177,12 @@ class Database: if fts5: where.append("sql like '%FTS5%'") sql = "select name from sqlite_master where {}".format(" AND ".join(where)) - return [r[0] for r in self.conn.execute(sql).fetchall()] + return [r[0] for r in self.execute(sql).fetchall()] def view_names(self): return [ r[0] - for r in self.conn.execute( + for r in self.execute( "select name from sqlite_master where type = 'view'" ).fetchall() ] @@ -174,25 +199,25 @@ class Database: def triggers(self): return [ Trigger(*r) - for r in self.conn.execute( + for r in self.execute( "select name, tbl_name, sql from sqlite_master where type = 'trigger'" ).fetchall() ] @property def journal_mode(self): - return self.conn.execute("PRAGMA journal_mode;").fetchone()[0] + return self.execute("PRAGMA journal_mode;").fetchone()[0] def enable_wal(self): if self.journal_mode != "wal": - self.conn.execute("PRAGMA journal_mode=wal;") + self.execute("PRAGMA journal_mode=wal;") def disable_wal(self): if self.journal_mode != "delete": - self.conn.execute("PRAGMA journal_mode=delete;") + self.execute("PRAGMA journal_mode=delete;") def execute_returning_dicts(self, sql, params=None): - cursor = self.conn.execute(sql, params or tuple()) + cursor = self.execute(sql, params or tuple()) keys = [d[0] for d in cursor.description] return [dict(zip(keys, row)) for row in cursor.fetchall()] @@ -337,7 +362,7 @@ class Database: """.format( table=name, columns_sql=columns_sql, extra_pk=extra_pk ) - self.conn.execute(sql) + self.execute(sql) return self.table( name, pk=pk, @@ -363,7 +388,7 @@ class Database: if create_sql == self[name].schema: return self self[name].drop() - self.conn.execute(create_sql) + self.execute(create_sql) return self def m2m_table_candidates(self, table, other_table): @@ -451,7 +476,7 @@ class Database: table.create_index([fk.column]) def vacuum(self): - self.conn.execute("VACUUM;") + self.execute("VACUUM;") class Queryable: @@ -464,7 +489,7 @@ class Queryable: @property def count(self): - return self.db.conn.execute( + return self.db.execute( "select count(*) from [{}]".format(self.name) ).fetchone()[0] @@ -480,7 +505,7 @@ class Queryable: sql += " where " + where if order_by is not None: sql += " order by " + order_by - cursor = self.db.conn.execute(sql, where_args or []) + cursor = self.db.execute(sql, where_args or []) columns = [c[0] for c in cursor.description] for row in cursor: yield dict(zip(columns, row)) @@ -489,9 +514,7 @@ class Queryable: def columns(self): if not self.exists(): return [] - rows = self.db.conn.execute( - "PRAGMA table_info([{}])".format(self.name) - ).fetchall() + rows = self.db.execute("PRAGMA table_info([{}])".format(self.name)).fetchall() return [Column(*row) for row in rows] @property @@ -501,7 +524,7 @@ class Queryable: @property def schema(self): - return self.db.conn.execute( + return self.db.execute( "select sql from sqlite_master where name = ?", (self.name,) ).fetchone()[0] @@ -587,7 +610,7 @@ class Table(Queryable): @property def foreign_keys(self): fks = [] - for row in self.db.conn.execute( + for row in self.db.execute( "PRAGMA foreign_key_list([{}])".format(self.name) ).fetchall(): if row is not None: @@ -615,7 +638,7 @@ class Table(Queryable): ) column_sql = "PRAGMA index_info({})".format(index_name_quoted) columns = [] - for seqno, cid, name in self.db.conn.execute(column_sql).fetchall(): + for seqno, cid, name in self.db.execute(column_sql).fetchall(): columns.append(name) row["columns"] = columns # These columns may be missing on older SQLite versions: @@ -629,7 +652,7 @@ class Table(Queryable): def triggers(self): return [ Trigger(*r) - for r in self.db.conn.execute( + for r in self.db.execute( "select name, tbl_name, sql from sqlite_master where type = 'trigger'" " and tbl_name = ?", (self.name,), @@ -683,7 +706,7 @@ class Table(Queryable): if_not_exists="IF NOT EXISTS " if if_not_exists else "", ) ) - self.db.conn.execute(sql) + self.db.execute(sql) return self def add_column( @@ -720,13 +743,13 @@ class Table(Queryable): col_type=fk_col_type or COLUMN_TYPE_MAPPING[col_type], not_null_default=(" " + not_null_sql) if not_null_sql else "", ) - self.db.conn.execute(sql) + self.db.execute(sql) if fk is not None: self.add_foreign_key(col_name, fk, fk_col) return self def drop(self): - self.db.conn.execute("DROP TABLE [{}]".format(self.name)) + self.db.execute("DROP TABLE [{}]".format(self.name)) def guess_foreign_table(self, column): column = column.lower() @@ -811,7 +834,7 @@ class Table(Queryable): tokenize="\n tokenize='{}',".format(tokenize) if tokenize else "", ) ) - self.db.conn.executescript(sql) + self.db.executescript(sql) self.populate_fts(columns) if create_triggers: @@ -840,17 +863,23 @@ class Table(Queryable): new_cols=new_cols, ) ) - self.db.conn.executescript(triggers) + self.db.executescript(triggers) return self def populate_fts(self, columns): - sql = """ + sql = ( + textwrap.dedent( + """ INSERT INTO [{table}_fts] (rowid, {columns}) SELECT rowid, {columns} FROM [{table}]; - """.format( - table=self.name, columns=", ".join("[{}]".format(c) for c in columns) + """ + ) + .strip() + .format( + table=self.name, columns=", ".join("[{}]".format(c) for c in columns) + ) ) - self.db.conn.executescript(sql) + self.db.executescript(sql) return self def disable_fts(self): @@ -858,23 +887,29 @@ class Table(Queryable): if fts_table: self.db[fts_table].drop() # Now delete the triggers that related to that table - sql = """ + sql = ( + textwrap.dedent( + """ SELECT name FROM sqlite_master WHERE type = 'trigger' AND sql LIKE '% INSERT INTO [{}]%' - """.format( - fts_table + """ + ) + .strip() + .format(fts_table) ) trigger_names = [] - for row in self.db.conn.execute(sql).fetchall(): + for row in self.db.execute(sql).fetchall(): trigger_names.append(row[0]) with self.db.conn: for trigger_name in trigger_names: - self.db.conn.execute("DROP TRIGGER IF EXISTS [{}]".format(trigger_name)) + self.db.execute("DROP TRIGGER IF EXISTS [{}]".format(trigger_name)) def detect_fts(self): "Detect if table has a corresponding FTS virtual table and return it" - sql = """ + sql = ( + textwrap.dedent( + """ SELECT name FROM sqlite_master WHERE rootpage = 0 AND ( @@ -884,10 +919,12 @@ class Table(Queryable): AND sql LIKE '%VIRTUAL TABLE%USING FTS%' ) ) - """.format( - table=self.name + """ + ) + .strip() + .format(table=self.name) ) - rows = self.db.conn.execute(sql).fetchall() + rows = self.db.execute(sql).fetchall() if len(rows) == 0: return None else: @@ -896,18 +933,22 @@ class Table(Queryable): def optimize(self): fts_table = self.detect_fts() if fts_table is not None: - self.db.conn.execute( + self.db.execute( """ INSERT INTO [{table}] ([{table}]) VALUES ("optimize"); - """.format( + """.strip().format( table=fts_table ) ) - self.db.conn.execute( - """ + self.db.execute( + textwrap.dedent( + """ DELETE FROM [{table}_docsize] WHERE {column} NOT IN ( SELECT rowid FROM [{table}]); - """.format( + """ + ) + .strip() + .format( # FTS5 uses 'id' but FTS4 uses 'docid' column=self.db["{}_docsize".format(fts_table)].columns[0].name, table=fts_table, @@ -916,16 +957,20 @@ class Table(Queryable): return self def search(self, q): - sql = """ + sql = ( + textwrap.dedent( + """ select * from "{table}" where rowid in ( select rowid from [{table}_fts] where [{table}_fts] match :search ) order by rowid - """.format( - table=self.name + """ + ) + .strip() + .format(table=self.name) ) - return self.db.conn.execute(sql, (q,)).fetchall() + return self.db.execute(sql, (q,)).fetchall() def value_or_default(self, key, value): return self._defaults[key] if value is DEFAULT else value @@ -939,7 +984,7 @@ class Table(Queryable): table=self.name, wheres=" and ".join(wheres) ) with self.db.conn: - self.db.conn.execute(sql, pk_values) + self.db.execute(sql, pk_values) def delete_where(self, where=None, where_args=None): if not self.exists(): @@ -947,7 +992,7 @@ class Table(Queryable): sql = "delete from [{}]".format(self.name) if where is not None: sql += " where " + where - self.db.conn.execute(sql, where_args or []) + self.db.execute(sql, where_args or []) def update(self, pk_values, updates=None, alter=False, conversions=None): updates = updates or {} @@ -972,12 +1017,12 @@ class Table(Queryable): ) with self.db.conn: try: - rowcount = self.db.conn.execute(sql, args).rowcount + rowcount = self.db.execute(sql, args).rowcount except OperationalError as e: if alter and (" column" in e.args[0]): # Attempt to add any missing columns, then try again self.add_missing_columns([updates]) - rowcount = self.db.conn.execute(sql, args).rowcount + rowcount = self.db.execute(sql, args).rowcount else: raise @@ -1084,7 +1129,7 @@ class Table(Queryable): self.last_rowid = None self.last_pk = None if truncate and self.exists(): - self.db.conn.execute("DELETE FROM [{}];".format(self.name)) + self.db.execute("DELETE FROM [{}];".format(self.name)) for chunk in chunks(itertools.chain([first_record], records), batch_size): chunk = list(chunk) num_records_processed += len(chunk) @@ -1184,14 +1229,12 @@ class Table(Queryable): or_what = "OR IGNORE " sql = """ INSERT {or_what}INTO [{table}] ({columns}) VALUES {rows}; - """.format( + """.strip().format( or_what=or_what, table=self.name, columns=", ".join("[{}]".format(c) for c in all_columns), rows=", ".join( - """ - ({placeholders}) - """.format( + "({placeholders})".format( placeholders=", ".join( [conversions.get(col, "?") for col in all_columns] ) @@ -1205,12 +1248,12 @@ class Table(Queryable): with self.db.conn: for query, params in queries_and_params: try: - result = self.db.conn.execute(query, params) + result = self.db.execute(query, params) except OperationalError as e: if alter and (" column" in e.args[0]): # Attempt to add any missing columns, then try again self.add_missing_columns(chunk) - result = self.db.conn.execute(query, params) + result = self.db.execute(query, params) else: raise if num_records_processed == 1 and not upsert: @@ -1383,7 +1426,7 @@ class View(Queryable): ) def drop(self): - self.db.conn.execute("DROP VIEW [{}]".format(self.name)) + self.db.execute("DROP VIEW [{}]".format(self.name)) def chunks(sequence, size): diff --git a/tests/conftest.py b/tests/conftest.py index 1e1fc5e..4541689 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,7 @@ def fresh_db(): @pytest.fixture def existing_db(): database = Database(memory=True) - database.conn.executescript( + database.executescript( """ CREATE TABLE foo (text TEXT); INSERT INTO foo (text) values ("one"); diff --git a/tests/test_cli.py b/tests/test_cli.py index 49631c9..964b6a2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -396,7 +396,7 @@ def test_enable_fts_with_triggers(db_path): def search(q): return ( Database(db_path) - .conn.execute("select c1 from Gosh_fts where c1 match ?", [q]) + .execute("select c1 from Gosh_fts where c1 match ?", [q]) .fetchall() ) @@ -417,7 +417,7 @@ def test_populate_fts(db_path): def search(q): return ( Database(db_path) - .conn.execute("select c1 from Gosh_fts where c1 match ?", [q]) + .execute("select c1 from Gosh_fts where c1 match ?", [q]) .fetchall() ) diff --git a/tests/test_column_affinity.py b/tests/test_column_affinity.py index 4a34b25..fb8f340 100644 --- a/tests/test_column_affinity.py +++ b/tests/test_column_affinity.py @@ -41,5 +41,5 @@ def test_column_affinity(column_def, expected_type): @pytest.mark.parametrize("column_def,expected_type", EXAMPLES) def test_columns_dict(fresh_db, column_def, expected_type): - fresh_db.conn.execute("create table foo (col {})".format(column_def)) + fresh_db.execute("create table foo (col {})".format(column_def)) assert {"col": expected_type} == fresh_db["foo"].columns_dict diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 8a790c2..b3cd963 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -4,9 +4,9 @@ import pytest def test_recursive_triggers(): db = Database(memory=True) - assert db.conn.execute("PRAGMA recursive_triggers").fetchone()[0] + assert db.execute("PRAGMA recursive_triggers").fetchone()[0] def test_recursive_triggers_off(): db = Database(memory=True, recursive_triggers=False) - assert not db.conn.execute("PRAGMA recursive_triggers").fetchone()[0] + assert not db.execute("PRAGMA recursive_triggers").fetchone()[0] diff --git a/tests/test_create.py b/tests/test_create.py index f6fad00..30a3d0e 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -680,7 +680,7 @@ def test_create_index_if_not_exists(fresh_db): ) def test_insert_dictionaries_and_lists_as_json(fresh_db, data_structure): fresh_db["test"].insert({"id": 1, "data": data_structure}, pk="id") - row = fresh_db.conn.execute("select id, data from test").fetchone() + row = fresh_db.execute("select id, data from test").fetchone() assert row[0] == 1 assert data_structure == json.loads(row[1]) diff --git a/tests/test_create_view.py b/tests/test_create_view.py index 26cf420..f0e2855 100644 --- a/tests/test_create_view.py +++ b/tests/test_create_view.py @@ -4,7 +4,7 @@ from sqlite_utils.utils import OperationalError def test_create_view(fresh_db): fresh_db.create_view("bar", "select 1 + 1") - rows = fresh_db.conn.execute("select * from bar").fetchall() + rows = fresh_db.execute("select * from bar").fetchall() assert [(2,)] == rows @@ -23,7 +23,7 @@ def test_create_view_ignore(fresh_db): fresh_db.create_view("bar", "select 1 + 1").create_view( "bar", "select 1 + 2", ignore=True ) - rows = fresh_db.conn.execute("select * from bar").fetchall() + rows = fresh_db.execute("select * from bar").fetchall() assert [(2,)] == rows @@ -31,13 +31,13 @@ def test_create_view_replace(fresh_db): fresh_db.create_view("bar", "select 1 + 1").create_view( "bar", "select 1 + 2", replace=True ) - rows = fresh_db.conn.execute("select * from bar").fetchall() + rows = fresh_db.execute("select * from bar").fetchall() assert [(3,)] == rows def test_create_view_replace_with_same_does_nothing(fresh_db): fresh_db.create_view("bar", "select 1 + 1") - initial_version = fresh_db.conn.execute("PRAGMA schema_version").fetchone()[0] + initial_version = fresh_db.execute("PRAGMA schema_version").fetchone()[0] fresh_db.create_view("bar", "select 1 + 1", replace=True) - after_version = fresh_db.conn.execute("PRAGMA schema_version").fetchone()[0] + after_version = fresh_db.execute("PRAGMA schema_version").fetchone()[0] assert after_version == initial_version diff --git a/tests/test_fts.py b/tests/test_fts.py index b1a946d..defb886 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -160,7 +160,7 @@ def test_disable_fts(fresh_db, create_triggers): expected_triggers = set() assert expected_triggers == set( r[0] - for r in fresh_db.conn.execute( + for r in fresh_db.execute( "select name from sqlite_master where type = 'trigger'" ).fetchall() ) @@ -168,7 +168,7 @@ def test_disable_fts(fresh_db, create_triggers): table.disable_fts() assert ( 0 - == fresh_db.conn.execute( + == fresh_db.execute( "select count(*) from sqlite_master where type = 'trigger'" ).fetchone()[0] ) diff --git a/tests/test_introspect.py b/tests/test_introspect.py index bebe537..cbec0a1 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -73,7 +73,7 @@ def test_table_repr(fresh_db): def test_indexes(fresh_db): - fresh_db.conn.executescript( + fresh_db.executescript( """ create table Gosh (c1 text, c2 text, c3 text); create index Gosh_c1 on Gosh(c1); diff --git a/tests/test_tracer.py b/tests/test_tracer.py new file mode 100644 index 0000000..e23ae86 --- /dev/null +++ b/tests/test_tracer.py @@ -0,0 +1,62 @@ +import pytest +from sqlite_utils import Database + + +def test_tracer(): + collected = [] + db = Database( + memory=True, tracer=lambda sql, params: collected.append((sql, params)) + ) + db["dogs"].insert({"name": "Cleopaws"}) + db["dogs"].enable_fts(["name"]) + db["dogs"].search("Cleopaws") + assert collected == [ + ("PRAGMA recursive_triggers=on;", None), + ("select name from sqlite_master where type = 'view'", None), + ("select name from sqlite_master where type = 'table'", None), + ("CREATE TABLE [dogs] (\n [name] TEXT\n);\n ", None), + ("select name from sqlite_master where type = 'view'", None), + ("INSERT INTO [dogs] ([name]) VALUES (?);", ["Cleopaws"]), + ("select name from sqlite_master where type = 'view'", None), + ( + "CREATE VIRTUAL TABLE [dogs_fts] USING FTS5 (\n [name],\n content=[dogs]\n);", + None, + ), + ( + "INSERT INTO [dogs_fts] (rowid, [name])\n SELECT rowid, [name] FROM [dogs];", + None, + ), + ("select name from sqlite_master where type = 'view'", None), + ( + 'select * from "dogs" where rowid in (\n select rowid from [dogs_fts]\n where [dogs_fts] match :search\n)\norder by rowid', + ("Cleopaws",), + ), + ] + + +def test_with_tracer(): + collected = [] + tracer = lambda sql, params: collected.append((sql, params)) + + db = Database(memory=True) + + db["dogs"].insert({"name": "Cleopaws"}) + db["dogs"].enable_fts(["name"]) + + assert len(collected) == 0 + + with db.tracer(tracer): + db["dogs"].search("Cleopaws") + + assert len(collected) == 2 + assert collected == [ + ("select name from sqlite_master where type = 'view'", None), + ( + 'select * from "dogs" where rowid in (\n select rowid from [dogs_fts]\n where [dogs_fts] match :search\n)\norder by rowid', + ("Cleopaws",), + ), + ] + + # Outside the with block collected should not be appended to + db["dogs"].insert({"name": "Cleopaws"}) + assert len(collected) == 2 From f28cd4de6d0c87e892999adb7d23699d6c00af05 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Sep 2020 14:58:49 -0700 Subject: [PATCH 0184/1004] Release 2.17 Refs #144, #148, #149, #150, #151, #152, #153 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8aa60cb..c4a3afe 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.16.1" +VERSION = "2.17" def get_long_description(): From deb2eb013ff85bbc828ebc244a9654f0d9c3139e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Sep 2020 15:07:21 -0700 Subject: [PATCH 0185/1004] Release notes for 2.17 --- docs/changelog.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 175e468..26322c3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,19 @@ Changelog =========== +.. _v2_17: + +2.17 (2020-09-07) +----------------- + +This release handles a bug where replacing rows in FTS tables could result in growing numbers of unneccessary rows in the associated ``*_fts_docsize`` table. (`#149 `__) + +- ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (`#152 `__) +- ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (`#153 `__) +- New tracer method for tracking underlying SQL queries, see :ref:`python_api_tracing`. (`#150 `__) +- Neater indentation for schema SQL. (`#148 `__) +- Documentation for ``sqlite_utils.AlterError`` exception thrown by in ``add_foreign_keys()``. + .. _v2_16_1: 2.16.1 (2020-08-28) From 4c0f79398fa8a08515781d12243af21af8d9004e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Sep 2020 15:09:25 -0700 Subject: [PATCH 0186/1004] table.rebuild_fts() method, refs #155 --- docs/python-api.rst | 15 +++++++++++++++ sqlite_utils/db.py | 11 +++++++++++ tests/test_fts.py | 26 ++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 5d3409d..d29133e 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1213,6 +1213,21 @@ To remove the FTS tables and triggers you created, use the ``disable_fts()`` tab dogs.disable_fts() +Rebuilding a full-text search table +=================================== + +You can rebuild a table using the ``table.rebuild_fts()`` method. This is useful for if the table configuration changes or the indexed data has become corrupted in some way. + +.. code-block:: python + + dogs.rebuild_fts() + +This method can be called on a table that has been configured for full-text search - ``dogs`` in this instance - or directly on a ``_fts`` table: + +.. code-block:: python + + db["dogs_fts"].rebuild_fts() + Optimizing a full-text search table =================================== diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0a22a3a..e077231 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -905,6 +905,17 @@ class Table(Queryable): for trigger_name in trigger_names: self.db.execute("DROP TRIGGER IF EXISTS [{}]".format(trigger_name)) + def rebuild_fts(self): + fts_table = self.detect_fts() + if fts_table is None: + # Assume this is itself an FTS table + fts_table = self.name + self.db.execute( + "INSERT INTO [{table}]([{table}]) VALUES('rebuild');".format( + table=fts_table + ) + ) + def detect_fts(self): "Detect if table has a corresponding FTS virtual table and return it" sql = ( diff --git a/tests/test_fts.py b/tests/test_fts.py index defb886..e0b5475 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -1,4 +1,5 @@ import pytest +from sqlite_utils.utils import sqlite3 search_records = [ { @@ -173,3 +174,28 @@ def test_disable_fts(fresh_db, create_triggers): ).fetchone()[0] ) assert ["searchable"] == fresh_db.table_names() + + +@pytest.mark.parametrize("table_to_fix", ["searchable", "searchable_fts"]) +def test_rebuild_fts(fresh_db, table_to_fix): + table = fresh_db["searchable"] + table.insert(search_records[0]) + table.enable_fts(["text", "country"]) + # Run a search + assert [("tanuki are running tricksters", "Japan", "foo")] == table.search("tanuki") + # Delete from searchable_fts_data + fresh_db["searchable_fts_data"].delete_where() + # This should have broken the index + with pytest.raises(sqlite3.DatabaseError): + table.search("tanuki") + # Running rebuild_fts() should fix it + fresh_db[table_to_fix].rebuild_fts() + assert [("tanuki are running tricksters", "Japan", "foo")] == table.search("tanuki") + + +@pytest.mark.parametrize("invalid_table", ["does_not_exist", "not_searchable"]) +def test_rebuild_fts_invalid(fresh_db, invalid_table): + fresh_db["not_searchable"].insert({"foo": "bar"}) + # Raise OperationalError on invalid table + with pytest.raises(sqlite3.OperationalError): + fresh_db[invalid_table].rebuild_fts() From 64799df78b14a12084d1def91c561abdcbcd8773 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Sep 2020 15:18:12 -0700 Subject: [PATCH 0187/1004] .optimize() no longer cleans up _docsize This isn't necessary any more since the new .rebuild_fts() method can achieve the same thing. Refs #155, #153 --- docs/python-api.rst | 8 ++++---- sqlite_utils/db.py | 14 -------------- tests/test_fts.py | 18 ++++++++++++++++++ tests/test_optimize.py | 20 -------------------- 4 files changed, 22 insertions(+), 38 deletions(-) delete mode 100644 tests/test_optimize.py diff --git a/docs/python-api.rst b/docs/python-api.rst index d29133e..2564376 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1228,6 +1228,10 @@ This method can be called on a table that has been configured for full-text sear db["dogs_fts"].rebuild_fts() +This runs the following SQL:: + + INSERT INTO dogs_fts (dogs_fts) VALUES ("rebuild"); + Optimizing a full-text search table =================================== @@ -1240,10 +1244,6 @@ Once you have populated a FTS table you can optimize it to dramatically reduce i This runs the following SQL:: INSERT INTO dogs_fts (dogs_fts) VALUES ("optimize"); - DELETE FROM [dogs_fts_docsize] WHERE id NOT IN ( - SELECT rowid FROM [dogs_fts]); - -That ``DELETE`` statement cleans up rows that may have been created by `an obscure bug `__ in previous versions of ``sqlite-utils``. Creating indexes ================ diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e077231..c03530f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -951,20 +951,6 @@ class Table(Queryable): table=fts_table ) ) - self.db.execute( - textwrap.dedent( - """ - DELETE FROM [{table}_docsize] WHERE {column} NOT IN ( - SELECT rowid FROM [{table}]); - """ - ) - .strip() - .format( - # FTS5 uses 'id' but FTS4 uses 'docid' - column=self.db["{}_docsize".format(fts_table)].columns[0].name, - table=fts_table, - ) - ) return self def search(self, q): diff --git a/tests/test_fts.py b/tests/test_fts.py index e0b5475..fb94a2e 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -1,4 +1,5 @@ import pytest +from sqlite_utils import Database from sqlite_utils.utils import sqlite3 search_records = [ @@ -199,3 +200,20 @@ def test_rebuild_fts_invalid(fresh_db, invalid_table): # Raise OperationalError on invalid table with pytest.raises(sqlite3.OperationalError): fresh_db[invalid_table].rebuild_fts() + + +@pytest.mark.parametrize("fts_version", ["FTS4", "FTS5"]) +def test_rebuild_removes_junk_docsize_rows(tmpdir, fts_version): + # Recreating https://github.com/simonw/sqlite-utils/issues/149 + path = tmpdir / "test.db" + db = Database(str(path), recursive_triggers=False) + licenses = [{"key": "apache2", "name": "Apache 2"}, {"key": "bsd", "name": "BSD"}] + db["licenses"].insert_all(licenses, pk="key", replace=True) + db["licenses"].enable_fts(["name"], create_triggers=True, fts_version=fts_version) + assert db["licenses_fts_docsize"].count == 2 + # Bug: insert with replace increases the number of rows in _docsize: + db["licenses"].insert_all(licenses, pk="key", replace=True) + assert db["licenses_fts_docsize"].count == 4 + # rebuild should fix this: + db["licenses_fts"].rebuild_fts() + assert db["licenses_fts_docsize"].count == 2 diff --git a/tests/test_optimize.py b/tests/test_optimize.py deleted file mode 100644 index 302532c..0000000 --- a/tests/test_optimize.py +++ /dev/null @@ -1,20 +0,0 @@ -import pytest -from sqlite_utils import Database -import sqlite3 - - -@pytest.mark.parametrize("fts_version", ["FTS4", "FTS5"]) -def test_optimizes_removes_junk_docsize_rows(tmpdir, fts_version): - # Recreating https://github.com/simonw/sqlite-utils/issues/149 - path = tmpdir / "test.db" - db = Database(str(path), recursive_triggers=False) - licenses = [{"key": "apache2", "name": "Apache 2"}, {"key": "bsd", "name": "BSD"}] - db["licenses"].insert_all(licenses, pk="key", replace=True) - db["licenses"].enable_fts(["name"], create_triggers=True, fts_version=fts_version) - assert db["licenses_fts_docsize"].count == 2 - # Bug: insert with replace increases the number of rows in _docsize: - db["licenses"].insert_all(licenses, pk="key", replace=True) - assert db["licenses_fts_docsize"].count == 4 - # Optimize should fix this: - db["licenses_fts"].optimize() - assert db["licenses_fts_docsize"].count == 2 From 9680a0291c7f5692076c468985c71f7fc6f5f199 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Sep 2020 15:24:39 -0700 Subject: [PATCH 0188/1004] 'Soundness check' is better --- sqlite_utils/db.py | 8 ++++---- tests/test_cli.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c03530f..f535f6b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -286,7 +286,7 @@ class Database: foreign_keys_by_column[extract_column] = ForeignKey( name, extract_column, extract_table, "id" ) - # Sanity check not_null, and defaults if provided + # Soundness check not_null, and defaults if provided not_null = not_null or set() defaults = defaults or {} assert all( @@ -308,7 +308,7 @@ class Database: if hash_id: column_items.insert(0, (hash_id, str)) pk = hash_id - # Sanity check foreign_keys point to existing tables + # Soundness check foreign_keys point to existing tables for fk in foreign_keys: if not any( c for c in self[fk.other_table].columns if c.name == fk.other_column @@ -792,7 +792,7 @@ class Table(Queryable): if other_column is None: other_column = self.guess_foreign_column(other_table) - # Sanity check that the other column exists + # Soundness check that the other column exists if ( not [c for c in self.db[other_table].columns if c.name == other_column] and other_column != "rowid" @@ -996,7 +996,7 @@ class Table(Queryable): conversions = conversions or {} if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] - # Sanity check that the record exists (raises error if not): + # Soundness check that the record exists (raises error if not): self.get(pk_values) if not updates: return self diff --git a/tests/test_cli.py b/tests/test_cli.py index 964b6a2..77efc56 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -475,7 +475,7 @@ def test_optimize(db_path): assert 0 == result.exit_code size_after_optimize = os.stat(db_path).st_size assert size_after_optimize < size_before_optimize - # Sanity check that --no-vacuum doesn't throw errors: + # Soundness check that --no-vacuum doesn't throw errors: result = CliRunner().invoke(cli.cli, ["optimize", "--no-vacuum", db_path]) assert 0 == result.exit_code @@ -724,7 +724,7 @@ def test_insert_alter(db_path, tmpdir): input='{"foo": "bar", "baz": 5}', ) assert 0 == result.exit_code, result.output - # Sanity check the database itself + # Soundness check the database itself db = Database(db_path) assert {"foo": str, "n": int, "baz": int} == db["from_json_nl"].columns_dict assert [ From 76548596a6397336042fffeb0fcab24e6ef59cfe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Sep 2020 15:34:55 -0700 Subject: [PATCH 0189/1004] optimize command now accepts optional tables, refs #155 --- docs/cli.rst | 7 ++++++- sqlite_utils/cli.py | 6 ++++-- tests/test_cli.py | 5 +++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 15e2f3a..100f3e0 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -703,7 +703,7 @@ You can run VACUUM to optimize your database like so:: Optimize ======== -The optimize command can dramatically reduce the size of your database if you are using SQLite full-text search. It runs OPTIMIZE against all of our FTS4 and FTS5 tables, then runs VACUUM. +The optimize command can dramatically reduce the size of your database if you are using SQLite full-text search. It runs OPTIMIZE against all of your FTS4 and FTS5 tables, then runs VACUUM. If you just want to run OPTIMIZE without the VACUUM, use the ``--no-vacuum`` flag. @@ -715,6 +715,11 @@ If you just want to run OPTIMIZE without the VACUUM, use the ``--no-vacuum`` fla # Optimize but skip the VACUUM $ sqlite-utils optimize --no-vacuum mydb.db +To optimize specific tables rather than every FTS table, pass those tables as extra arguments: + +:: + $ sqlite-utils optimize mydb.db table_1 table_2 + .. _cli_wal: WAL mode diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 7990d7d..beb9943 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -218,11 +218,13 @@ def vacuum(path): type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), required=True, ) +@click.argument("tables", nargs=-1) @click.option("--no-vacuum", help="Don't run VACUUM", default=False, is_flag=True) -def optimize(path, no_vacuum): +def optimize(path, tables, no_vacuum): """Optimize all FTS tables and then run VACUUM - should shrink the database file""" db = sqlite_utils.Database(path) - tables = db.table_names(fts4=True) + db.table_names(fts5=True) + if not tables: + tables = db.table_names(fts4=True) + db.table_names(fts5=True) with db.conn: for table in tables: db[table].optimize() diff --git a/tests/test_cli.py b/tests/test_cli.py index 77efc56..5c4404a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -454,7 +454,8 @@ def test_vacuum(db_path): assert 0 == result.exit_code -def test_optimize(db_path): +@pytest.mark.parametrize("tables", ([], ["Gosh"], ["Gosh2"])) +def test_optimize(db_path, tables): db = Database(db_path) with db.conn: for table in ("Gosh", "Gosh2"): @@ -471,7 +472,7 @@ def test_optimize(db_path): db["Gosh"].enable_fts(["c1", "c2", "c3"], fts_version="FTS4") db["Gosh2"].enable_fts(["c1", "c2", "c3"], fts_version="FTS5") size_before_optimize = os.stat(db_path).st_size - result = CliRunner().invoke(cli.cli, ["optimize", db_path]) + result = CliRunner().invoke(cli.cli, ["optimize", db_path] + tables) assert 0 == result.exit_code size_after_optimize = os.stat(db_path).st_size assert size_after_optimize < size_before_optimize From 176f4e0ef4a4825ae3b61a5f7169a8943fccb073 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Sep 2020 16:16:03 -0700 Subject: [PATCH 0190/1004] sqlite-utils rebuild-fts command, closes #155 --- docs/cli.rst | 8 +++++++ docs/python-api.rst | 4 ++++ sqlite_utils/cli.py | 28 +++++++++++++++++++++++++ sqlite_utils/db.py | 4 ++-- tests/test_cli.py | 51 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 100f3e0..5eeb507 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -689,6 +689,14 @@ To remove the FTS tables and triggers you created, use ``disable-fts``:: $ sqlite-utils disable-fts mydb.db documents +To rebuild one or more FTS tables (see :ref:`python_api_fts_rebuild`), use ``rebuild-fts``:: + + $ sqlite-utils rebuild-fts mydb.db documents + +You can rebuild every FTS table by running ``rebuild-fts`` without passing any table names:: + + $ sqlite-utils rebuild-fts mydb.db + .. _cli_vacuum: Vacuum diff --git a/docs/python-api.rst b/docs/python-api.rst index 2564376..9404c78 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1213,6 +1213,8 @@ To remove the FTS tables and triggers you created, use the ``disable_fts()`` tab dogs.disable_fts() +.. _python_api_fts_rebuild: + Rebuilding a full-text search table =================================== @@ -1232,6 +1234,8 @@ This runs the following SQL:: INSERT INTO dogs_fts (dogs_fts) VALUES ("rebuild"); +.. _python_api_fts_optimize: + Optimizing a full-text search table =================================== diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index beb9943..4a01cc8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -232,6 +232,34 @@ def optimize(path, tables, no_vacuum): db.vacuum() +@cli.command(name="rebuild-fts") +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("tables", nargs=-1) +def rebuild_fts(path, tables): + """Rebuild specific FTS tables, or all FTS tables if none are specified""" + db = sqlite_utils.Database(path) + if not tables: + tables = db.table_names(fts4=True) + db.table_names(fts5=True) + with db.conn: + for table in tables: + db[table].rebuild_fts() + + +@cli.command() +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +def vacuum(path): + """Run VACUUM against the database""" + sqlite_utils.Database(path).vacuum() + + @cli.command(name="add-column") @click.argument( "path", diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f535f6b..f938ef4 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -173,9 +173,9 @@ class Database: def table_names(self, fts4=False, fts5=False): where = ["type = 'table'"] if fts4: - where.append("sql like '%FTS4%'") + where.append("sql like '%USING FTS4%'") if fts5: - where.append("sql like '%FTS5%'") + where.append("sql like '%USING FTS5%'") sql = "select name from sqlite_master where {}".format(" AND ".join(where)) return [r[0] for r in self.execute(sql).fetchall()] diff --git a/tests/test_cli.py b/tests/test_cli.py index 5c4404a..0a8c0ba 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -481,6 +481,57 @@ def test_optimize(db_path, tables): assert 0 == result.exit_code +@pytest.mark.parametrize("tables", ([], ["fts4_table"], ["fts5_table"])) +def test_rebuild_fts(db_path, tables): + db = Database(db_path, recursive_triggers=False) + records = [ + { + "c1": "verb{}".format(i), + "c2": "noun{}".format(i), + "c3": "adjective{}".format(i), + } + for i in range(10000) + ] + with db.conn: + db["fts4_table"].insert_all(records, pk="c1") + db["fts5_table"].insert_all(records, pk="c1") + db["fts4_table"].enable_fts( + ["c1", "c2", "c3"], fts_version="FTS4", create_triggers=True + ) + db["fts5_table"].enable_fts( + ["c1", "c2", "c3"], fts_version="FTS5", create_triggers=True + ) + # Search should work + assert db["fts4_table"].search("verb1") + assert db["fts5_table"].search("verb1") + # Deleting _fts_segments to break FTS4 + with db.conn: + db["fts4_table_fts_segments"].delete_where() + # Now this should error: + with pytest.raises(sqlite3.DatabaseError): + db["fts4_table"].search("verb1") + # Replicate docsize error from this issue for FTS5 + # https://github.com/simonw/sqlite-utils/issues/149 + assert db["fts5_table_fts_docsize"].count == 10000 + db["fts5_table"].insert_all(records, replace=True) + assert db["fts5_table"].count == 10000 + assert db["fts5_table_fts_docsize"].count == 20000 + # Running rebuild-fts should fix both issues + print(["rebuild-fts", db_path] + tables) + result = CliRunner().invoke(cli.cli, ["rebuild-fts", db_path] + tables) + assert 0 == result.exit_code + fixed_tables = tables or ["fts4_table", "fts5_table"] + if "fts4_table" in fixed_tables: + assert db["fts4_table"].search("verb1") + else: + with pytest.raises(sqlite3.DatabaseError): + db["fts4_table"].search("verb1") + if "fts5_table" in fixed_tables: + assert db["fts5_table_fts_docsize"].count == 10000 + else: + assert db["fts5_table_fts_docsize"].count == 20000 + + def test_insert_simple(tmpdir): json_path = str(tmpdir / "dog.json") db_path = str(tmpdir / "dogs.db") From e6d202b742a7b531fffa593703d34f8337632d68 Mon Sep 17 00:00:00 2001 From: Simon Wiles Date: Tue, 8 Sep 2020 16:20:36 -0700 Subject: [PATCH 0191/1004] Handle case where subsequent records (after first batch) include extra columns Refs #145. * Extract build_insert_queries_and_params * Extract insert_chunk so it can be called recursively Thanks, @simonwiles --- sqlite_utils/db.py | 295 ++++++++++++++++++++++++++++--------------- tests/test_create.py | 24 ++++ 2 files changed, 217 insertions(+), 102 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f938ef4..86f417b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1028,6 +1028,186 @@ class Table(Queryable): self.last_pk = pk_values[0] if len(self.pks) == 1 else pk_values return self + def build_insert_queries_and_params( + self, + extracts, + chunk, + all_columns, + hash_id, + upsert, + pk, + conversions, + num_records_processed, + replace, + ignore, + ): + # values is the list of insert data that is passed to the + # .execute() method - but some of them may be replaced by + # new primary keys if we are extracting any columns. + values = [] + extracts = resolve_extracts(extracts) + for record in chunk: + record_values = [] + for key in all_columns: + value = jsonify_if_needed( + record.get(key, None if key != hash_id else _hash(record)) + ) + if key in extracts: + extract_table = extracts[key] + value = self.db[extract_table].lookup({"value": value}) + record_values.append(value) + values.append(record_values) + + queries_and_params = [] + if upsert: + if isinstance(pk, str): + pks = [pk] + else: + pks = pk + self.last_pk = None + for record_values in values: + # TODO: make more efficient: + record = dict(zip(all_columns, record_values)) + sql = "INSERT OR IGNORE INTO [{table}]({pks}) VALUES({pk_placeholders});".format( + table=self.name, + pks=", ".join(["[{}]".format(p) for p in pks]), + pk_placeholders=", ".join(["?" for p in pks]), + ) + queries_and_params.append((sql, [record[col] for col in pks])) + # UPDATE [book] SET [name] = 'Programming' WHERE [id] = 1001; + set_cols = [col for col in all_columns if col not in pks] + sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format( + table=self.name, + pairs=", ".join( + "[{}] = {}".format(col, conversions.get(col, "?")) + for col in set_cols + ), + wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks), + ) + queries_and_params.append( + ( + sql2, + [record[col] for col in set_cols] + [record[pk] for pk in pks], + ) + ) + # We can populate .last_pk right here + if num_records_processed == 1: + self.last_pk = tuple(record[pk] for pk in pks) + if len(self.last_pk) == 1: + self.last_pk = self.last_pk[0] + + else: + or_what = "" + if replace: + or_what = "OR REPLACE " + elif ignore: + or_what = "OR IGNORE " + sql = """ + INSERT {or_what}INTO [{table}] ({columns}) VALUES {rows}; + """.strip().format( + or_what=or_what, + table=self.name, + columns=", ".join("[{}]".format(c) for c in all_columns), + rows=", ".join( + "({placeholders})".format( + placeholders=", ".join( + [conversions.get(col, "?") for col in all_columns] + ) + ) + for record in chunk + ), + ) + flat_values = list(itertools.chain(*values)) + queries_and_params = [(sql, flat_values)] + + return queries_and_params + + def insert_chunk( + self, + alter, + extracts, + chunk, + all_columns, + hash_id, + upsert, + pk, + conversions, + num_records_processed, + replace, + ignore, + ): + queries_and_params = self.build_insert_queries_and_params( + extracts, + chunk, + all_columns, + hash_id, + upsert, + pk, + conversions, + num_records_processed, + replace, + ignore, + ) + + with self.db.conn: + for query, params in queries_and_params: + try: + result = self.db.execute(query, params) + except OperationalError as e: + if alter and (" column" in e.args[0]): + # Attempt to add any missing columns, then try again + self.add_missing_columns(chunk) + result = self.db.execute(query, params) + elif e.args[0] == "too many SQL variables": + + first_half = chunk[: len(chunk) // 2] + second_half = chunk[len(chunk) // 2 :] + + self.insert_chunk( + alter, + extracts, + first_half, + all_columns, + hash_id, + upsert, + pk, + conversions, + num_records_processed, + replace, + ignore, + ) + + self.insert_chunk( + alter, + extracts, + second_half, + all_columns, + hash_id, + upsert, + pk, + conversions, + num_records_processed, + replace, + ignore, + ) + + else: + raise + if num_records_processed == 1 and not upsert: + self.last_rowid = result.lastrowid + self.last_pk = self.last_rowid + # self.last_rowid will be 0 if a "INSERT OR IGNORE" happened + if (hash_id or pk) and self.last_rowid: + row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] + if hash_id: + self.last_pk = row[hash_id] + elif isinstance(pk, str): + self.last_pk = row[pk] + else: + self.last_pk = tuple(row[p] for p in pk) + + return + def insert( self, record, @@ -1161,110 +1341,21 @@ class Table(Queryable): validate_column_names(all_columns) first = False - # values is the list of insert data that is passed to the - # .execute() method - but some of them may be replaced by - # new primary keys if we are extracting any columns. - values = [] - extracts = resolve_extracts(extracts) - for record in chunk: - record_values = [] - for key in all_columns: - value = jsonify_if_needed( - record.get(key, None if key != hash_id else _hash(record)) - ) - if key in extracts: - extract_table = extracts[key] - value = self.db[extract_table].lookup({"value": value}) - record_values.append(value) - values.append(record_values) - queries_and_params = [] - if upsert: - if isinstance(pk, str): - pks = [pk] - else: - pks = pk - self.last_pk = None - for record_values in values: - # TODO: make more efficient: - record = dict(zip(all_columns, record_values)) - params = [] - sql = "INSERT OR IGNORE INTO [{table}]({pks}) VALUES({pk_placeholders});".format( - table=self.name, - pks=", ".join(["[{}]".format(p) for p in pks]), - pk_placeholders=", ".join(["?" for p in pks]), - ) - queries_and_params.append((sql, [record[col] for col in pks])) - # UPDATE [book] SET [name] = 'Programming' WHERE [id] = 1001; - set_cols = [col for col in all_columns if col not in pks] - sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format( - table=self.name, - pairs=", ".join( - "[{}] = {}".format(col, conversions.get(col, "?")) - for col in set_cols - ), - wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks), - ) - queries_and_params.append( - ( - sql2, - [record[col] for col in set_cols] - + [record[pk] for pk in pks], - ) - ) - # We can populate .last_pk right here - if num_records_processed == 1: - self.last_pk = tuple(record[pk] for pk in pks) - if len(self.last_pk) == 1: - self.last_pk = self.last_pk[0] + self.insert_chunk( + alter, + extracts, + chunk, + all_columns, + hash_id, + upsert, + pk, + conversions, + num_records_processed, + replace, + ignore, + ) - else: - or_what = "" - if replace: - or_what = "OR REPLACE " - elif ignore: - or_what = "OR IGNORE " - sql = """ - INSERT {or_what}INTO [{table}] ({columns}) VALUES {rows}; - """.strip().format( - or_what=or_what, - table=self.name, - columns=", ".join("[{}]".format(c) for c in all_columns), - rows=", ".join( - "({placeholders})".format( - placeholders=", ".join( - [conversions.get(col, "?") for col in all_columns] - ) - ) - for record in chunk - ), - ) - flat_values = list(itertools.chain(*values)) - queries_and_params = [(sql, flat_values)] - - with self.db.conn: - for query, params in queries_and_params: - try: - result = self.db.execute(query, params) - except OperationalError as e: - if alter and (" column" in e.args[0]): - # Attempt to add any missing columns, then try again - self.add_missing_columns(chunk) - result = self.db.execute(query, params) - else: - raise - if num_records_processed == 1 and not upsert: - self.last_rowid = result.lastrowid - self.last_pk = self.last_rowid - # self.last_rowid will be 0 if a "INSERT OR IGNORE" happened - if (hash_id or pk) and self.last_rowid: - row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] - if hash_id: - self.last_pk = row[hash_id] - elif isinstance(pk, str): - self.last_pk = row[pk] - else: - self.last_pk = tuple(row[p] for p in pk) return self def upsert( diff --git a/tests/test_create.py b/tests/test_create.py index 30a3d0e..e139557 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -584,6 +584,30 @@ def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error): fresh_db["big"].insert(record) +def test_columns_not_in_first_record_should_not_cause_batch_to_be_too_large(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/145 + # sqlite on homebrew and Debian/Ubuntu etc. is typically compiled with + # SQLITE_MAX_VARIABLE_NUMBER set to 250,000, so we need to exceed this value to + # trigger the error on these systems. + THRESHOLD = 250000 + batch_size = 999 + extra_columns = 1 + (THRESHOLD - 1) // (batch_size - 1) + records = [ + {"c0": "first record"}, # one column in first record -> batch size = 999 + # fill out the batch with 99 records with enough columns to exceed THRESHOLD + *[ + dict([("c{}".format(i), j) for i in range(extra_columns)]) + for j in range(batch_size - 1) + ], + ] + try: + fresh_db["too_many_columns"].insert_all( + records, alter=True, batch_size=batch_size + ) + except sqlite3.OperationalError: + raise + + @pytest.mark.parametrize( "columns,index_name,expected_index", ( From 32f1badfec7302dd4b1fd2a60be8af40a990c30d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Sep 2020 16:27:55 -0700 Subject: [PATCH 0192/1004] Tracer example using print, refs #150 --- docs/python-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 9404c78..d070b86 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -67,10 +67,10 @@ You can also turn on a tracer function temporarily for a block of code using the db = Database(memory=True) # ... later - with db.tracer(tracer): + with db.tracer(print): db["dogs"].insert({"name": "Cleo"}) -Queries will be passed to your ``tracer()`` function only for the duration of the ``with`` block. +This example will print queries only for the duration of the ``with`` block. .. _python_api_execute: From 6be61263642d8e46ec54cf5f51af74e0df2f2393 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Sep 2020 16:37:28 -0700 Subject: [PATCH 0193/1004] Release 2.18 Refs #145. #155 --- docs/changelog.rst | 10 ++++++++++ setup.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 26322c3..48afc0b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,16 @@ Changelog =========== +.. _v2_18: + +2.18 (2020-09-08) +----------------- + +- ``table.rebuild_fts()`` method for rebuilding a FTS index, see :ref:`python_api_fts_rebuild`. (`#155 `__) +- ``sqlite-utils rebuild-fts data.db`` command for rebuilding FTS indexes across all tables, or just specific tables. (`#155 `__) +- ``table.optimize()`` method no longer deletes junk rows from the ``*_fts_docsize`` table. This was added in 2.17 but it turns out running ``table.rebuild_fts()`` is a better solution to this problem. +- Fixed a bug where rows with additional columns that are inserted after the first batch of records could cause an error due to breaking SQLite's maximum number of parameters. Thanks, Simon Wiles. (`#145 `__) + .. _v2_17: 2.17 (2020-09-07) diff --git a/setup.py b/setup.py index c4a3afe..6f50822 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.17" +VERSION = "2.18" def get_long_description(): From 367082e787101fb90901ef3214804ab23a92ce46 Mon Sep 17 00:00:00 2001 From: Simon Wiles Date: Wed, 9 Sep 2020 11:21:22 -0700 Subject: [PATCH 0194/1004] Typos in tests (#156) Thanks @simonwiles --- tests/test_create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_create.py b/tests/test_create.py index e139557..1dfc541 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -561,7 +561,7 @@ def test_bulk_insert_more_than_999_values(fresh_db): "c6": 6, "c7": 7, "c8": 8, - "c8": 9, + "c9": 9, "c10": 10, "c11": 11, } @@ -736,7 +736,7 @@ def test_insert_thousands_using_generator(fresh_db): assert 10000 == fresh_db["test"].count -def test_insert_thousands_raises_exception_wtih_extra_columns_after_first_100(fresh_db): +def test_insert_thousands_raises_exception_with_extra_columns_after_first_100(fresh_db): # https://github.com/simonw/sqlite-utils/issues/139 with pytest.raises(Exception, match="table test has no column named extra"): fresh_db["test"].insert_all( From 7805d53bcf11199bd1f2b07e05ae90151f9d0eb0 Mon Sep 17 00:00:00 2001 From: Tom V Date: Wed, 16 Sep 2020 07:21:42 +0100 Subject: [PATCH 0195/1004] Fix accidental mega long line in docs (#158) Thanks @tomviner --- docs/python-api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index d070b86..b1c2e0d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -431,7 +431,8 @@ Here's an example that uses these features: # Outputs: # [{'id': 1, 'name': 'Sally', 'score': 2}, # {'id': 3, 'name': 'Dharma', 'score': 1}] - print(db["authors"].schema) # Outputs: + print(db["authors"].schema) + # Outputs: # CREATE TABLE [authors] ( # [id] INTEGER PRIMARY KEY, # [name] TEXT NOT NULL, From 3cc1944e53b75749644f558cbe1717397cae72ea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Sep 2020 13:14:25 -0700 Subject: [PATCH 0196/1004] sqlite-utils add-foreign-keys command, closes #157 --- docs/cli.rst | 13 +++++++++++++ sqlite_utils/cli.py | 30 ++++++++++++++++++++++++++++++ tests/test_cli.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 5eeb507..9c5344c 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -621,6 +621,19 @@ If you omit the other table and other column references ``sqlite-utils`` will at See :ref:`python_api_add_foreign_key` in the Python API documentation for further details, including how the automatic table guessing mechanism works. +.. _cli_add_foreign_keys: + +Adding multiple foreign keys at once +------------------------------------ + +Adding a foreign key requires a ``VACUUM``. On large databases this can be an expensive operation, so if you are adding multiple foreign keys you can combine them into one operation (and hence one ``VACUUM``) using ``add-foreign-keys``:: + + $ sqlite-utils add-foreign-keys books.db \ + books author_id authors id \ + authors country_id countries id + +When you are using this command each foreign key needs to be defined in full, as four arguments - the table, column, other table and other column. + .. _cli_index_foreign_keys: Adding indexes for all foreign keys diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4a01cc8..eb44a8e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -323,6 +323,36 @@ def add_foreign_key(path, table, column, other_table, other_column): raise click.ClickException(e) +@cli.command(name="add-foreign-keys") +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("foreign_key", nargs=-1) +def add_foreign_keys(path, foreign_key): + """ + Add multiple new foreign key constraints to a database. Example usage: + + \b + sqlite-utils add-foreign-keys my.db \\ + books author_id authors id \\ + authors country_id countries id + """ + db = sqlite_utils.Database(path) + if len(foreign_key) % 4 != 0: + raise click.ClickException( + "Each foreign key requires four values: table, column, other_table, other_column" + ) + tuples = [] + for i in range(len(foreign_key) // 4): + tuples.append(tuple(foreign_key[i * 4 : (i * 4) + 4])) + try: + db.add_foreign_keys(tuples) + except AlterError as e: + raise click.ClickException(e) + + @cli.command(name="index-foreign-keys") @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index 0a8c0ba..dba182b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1405,3 +1405,41 @@ def test_query_update(db_path, args, expected): assert db.execute_returning_dicts("select * from dogs") == [ {"id": 1, "age": 5, "name": "Cleo"}, ] + + +def test_add_foreign_keys(db_path): + db = Database(db_path) + db["countries"].insert({"id": 7, "name": "Panama"}, pk="id") + db["authors"].insert({"id": 3, "name": "Matilda", "country_id": 7}, pk="id") + db["books"].insert({"id": 2, "title": "Wolf anatomy", "author_id": 3}, pk="id") + assert db["authors"].foreign_keys == [] + assert db["books"].foreign_keys == [] + result = CliRunner().invoke( + cli.cli, + [ + "add-foreign-keys", + db_path, + "authors", + "country_id", + "countries", + "id", + "books", + "author_id", + "authors", + "id", + ], + ) + assert result.exit_code == 0 + assert db["authors"].foreign_keys == [ + ForeignKey( + table="authors", + column="country_id", + other_table="countries", + other_column="id", + ) + ] + assert db["books"].foreign_keys == [ + ForeignKey( + table="books", column="author_id", other_table="authors", other_column="id" + ) + ] From ecb50c8f76535754f76bffdf77bf99e8f829b832 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Sep 2020 15:05:46 -0700 Subject: [PATCH 0197/1004] .enable_fts(..., replace=True) argument, closes #160 --- docs/python-api.rst | 10 ++++++++ sqlite_utils/db.py | 31 ++++++++++++++++++++--- tests/test_fts.py | 59 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_tracer.py | 2 +- 4 files changed, 97 insertions(+), 5 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index b1c2e0d..e6a0dd1 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1208,6 +1208,16 @@ You can customize the tokenizer configured for the table using the ``tokenize=`` The SQLite documentation has more on `FTS5 tokenizers `__ and `FTS4 tokenizers `__. ``porter`` is a valid option for both. +If you attempt to configure a FTS table where one already exists, a ``sqlite3.OperationalError`` exception will be raised. + +You can replace the existing table with a new configuration using ``replace=True``: + +.. code-block:: python + + db["articles"].enable_fts(["headline"], tokenize="porter", replace=True) + +This will have no effect if the FTS table already exists, otherwise it will drop and recreate the table with the new settings. This takes into consideration the columns, the tokenizer, the FTS version used and whether or not the table has triggers. + To remove the FTS tables and triggers you created, use the ``disable_fts()`` table method: .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 86f417b..7a3e4d6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -814,16 +814,21 @@ class Table(Queryable): self.db.add_foreign_keys([(self.name, column, other_table, other_column)]) def enable_fts( - self, columns, fts_version="FTS5", create_triggers=False, tokenize=None + self, + columns, + fts_version="FTS5", + create_triggers=False, + tokenize=None, + replace=False, ): "Enables FTS on the specified columns." - sql = ( + create_fts_sql = ( textwrap.dedent( """ CREATE VIRTUAL TABLE [{table}_fts] USING {fts_version} ( {columns},{tokenize} content=[{table}] - ); + ) """ ) .strip() @@ -834,7 +839,25 @@ class Table(Queryable): tokenize="\n tokenize='{}',".format(tokenize) if tokenize else "", ) ) - self.db.executescript(sql) + should_recreate = False + if replace and self.db["{}_fts".format(self.name)].exists(): + # Does the table need to be recreated? + fts_schema = self.db["{}_fts".format(self.name)].schema + if fts_schema != create_fts_sql: + should_recreate = True + expected_triggers = {self.name + suffix for suffix in ("_ai", "_ad", "_au")} + existing_triggers = {t.name for t in self.triggers} + has_triggers = existing_triggers.issuperset(expected_triggers) + if has_triggers != create_triggers: + should_recreate = True + if not should_recreate: + # Table with correct configuration already exists + return self + + if should_recreate: + self.disable_fts() + + self.db.executescript(create_fts_sql) self.populate_fts(columns) if create_triggers: diff --git a/tests/test_fts.py b/tests/test_fts.py index fb94a2e..359d911 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -217,3 +217,62 @@ def test_rebuild_removes_junk_docsize_rows(tmpdir, fts_version): # rebuild should fix this: db["licenses_fts"].rebuild_fts() assert db["licenses_fts_docsize"].count == 2 + + +@pytest.mark.parametrize( + "kwargs", + [ + {"columns": ["title"]}, + {"fts_version": "FTS4"}, + {"create_triggers": True}, + {"tokenize": "porter"}, + ], +) +def test_enable_fts_replace(kwargs): + db = Database(memory=True) + db["books"].insert( + { + "id": 1, + "title": "Habits of Australian Marsupials", + "author": "Marlee Hawkins", + }, + pk="id", + ) + db["books"].enable_fts(["title", "author"]) + assert not db["books"].triggers + assert db["books_fts"].columns_dict.keys() == {"title", "author"} + assert "FTS5" in db["books_fts"].schema + assert "porter" not in db["books_fts"].schema + # Now modify the FTS configuration + should_have_changed_columns = "columns" in kwargs + if "columns" not in kwargs: + kwargs["columns"] = ["title", "author"] + db["books"].enable_fts(**kwargs, replace=True) + # Check that the new configuration is correct + if should_have_changed_columns: + assert db["books_fts"].columns_dict.keys() == set(["title"]) + if "create_triggers" in kwargs: + assert db["books"].triggers + if "fts_version" in kwargs: + assert "FTS4" in db["books_fts"].schema + if "tokenize" in kwargs: + assert "porter" in db["books_fts"].schema + + +def test_enable_fts_replace_does_nothing_if_args_the_same(): + queries = [] + db = Database(memory=True, tracer=lambda sql, params: queries.append((sql, params))) + db["books"].insert( + { + "id": 1, + "title": "Habits of Australian Marsupials", + "author": "Marlee Hawkins", + }, + pk="id", + ) + db["books"].enable_fts(["title", "author"], create_triggers=True) + queries.clear() + # Running that again shouldn't run much SQL: + db["books"].enable_fts(["title", "author"], create_triggers=True, replace=True) + # The only SQL that executed should be select statements + assert all(q[0].startswith("select ") for q in queries) diff --git a/tests/test_tracer.py b/tests/test_tracer.py index e23ae86..6a58ec0 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -19,7 +19,7 @@ def test_tracer(): ("INSERT INTO [dogs] ([name]) VALUES (?);", ["Cleopaws"]), ("select name from sqlite_master where type = 'view'", None), ( - "CREATE VIRTUAL TABLE [dogs_fts] USING FTS5 (\n [name],\n content=[dogs]\n);", + "CREATE VIRTUAL TABLE [dogs_fts] USING FTS5 (\n [name],\n content=[dogs]\n)", None, ), ( From e23eedb4ce4efbf24fd01b80c0209de4b9aba2bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Sep 2020 15:17:25 -0700 Subject: [PATCH 0198/1004] ignore=True argument for add_foreign_key, closes #112 Also --ignore for add-foreign-key command Plus table.add_foreign_key(...) now returns self, allowing more chaining --- docs/cli.rst | 4 ++++ docs/python-api.rst | 6 ++++++ sqlite_utils/cli.py | 9 +++++++-- sqlite_utils/db.py | 16 +++++++++++----- tests/test_cli.py | 10 +++++++++- tests/test_create.py | 12 +++++++++++- 6 files changed, 48 insertions(+), 9 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 9c5344c..bd73eff 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -619,6 +619,10 @@ If you omit the other table and other column references ``sqlite-utils`` will at $ sqlite-utils add-foreign-key books.db books author_id +Add ``--ignore`` to ignore an existing foreign key (as opposed to returning an error):: + + $ sqlite-utils add-foreign-key books.db books author_id --ignore + See :ref:`python_api_add_foreign_key` in the Python API documentation for further details, including how the automatic table guessing mechanism works. .. _cli_add_foreign_keys: diff --git a/docs/python-api.rst b/docs/python-api.rst index e6a0dd1..fae8cee 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -858,6 +858,12 @@ The ``table.add_foreign_key(column, other_table, other_column)`` method takes th This method first checks that the specified foreign key references tables and columns that exist and does not clash with an existing foreign key. It will raise a ``sqlite_utils.db.AlterError`` exception if these checks fail. +To ignore the case where the key already exists, use ``ignore=True``: + +.. code-block:: python + + db["books"].add_foreign_key("author_id", "authors", "id", ignore=True) + .. _python_api_add_foreign_keys: Adding multiple foreign key constraints at once diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index eb44a8e..cfdffed 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -308,7 +308,12 @@ def add_column(path, table, col_name, col_type, fk, fk_col, not_null_default): @click.argument("column") @click.argument("other_table", required=False) @click.argument("other_column", required=False) -def add_foreign_key(path, table, column, other_table, other_column): +@click.option( + "--ignore", + is_flag=True, + help="If foreign key already exists, do nothing", +) +def add_foreign_key(path, table, column, other_table, other_column, ignore): """ Add a new foreign key constraint to an existing table. Example usage: @@ -318,7 +323,7 @@ def add_foreign_key(path, table, column, other_table, other_column): """ db = sqlite_utils.Database(path) try: - db[table].add_foreign_key(column, other_table, other_column) + db[table].add_foreign_key(column, other_table, other_column, ignore=ignore) except AlterError as e: raise click.ClickException(e) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 7a3e4d6..ebffabf 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -781,7 +781,9 @@ class Table(Queryable): else: return pks[0].name - def add_foreign_key(self, column, other_table=None, other_column=None): + def add_foreign_key( + self, column, other_table=None, other_column=None, ignore=False + ): # Ensure column exists if column not in self.columns_dict: raise AlterError("No such column: {}".format(column)) @@ -806,12 +808,16 @@ class Table(Queryable): and fk.other_table == other_table and fk.other_column == other_column ): - raise AlterError( - "Foreign key already exists for {} => {}.{}".format( - column, other_table, other_column + if ignore: + return self + else: + raise AlterError( + "Foreign key already exists for {} => {}.{}".format( + column, other_table, other_column + ) ) - ) self.db.add_foreign_keys([(self.name, column, other_table, other_column)]) + return self def enable_fts( self, diff --git a/tests/test_cli.py b/tests/test_cli.py index dba182b..7d8ccf4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -272,16 +272,24 @@ def test_add_foreign_key(db_path, args, assert_message): table="books", column="author_id", other_table="authors", other_column="id" ) ] == db["books"].foreign_keys + # Error if we try to add it twice: result = CliRunner().invoke( cli.cli, ["add-foreign-key", db_path, "books", "author_id", "authors", "id"] ) - assert 0 != result.exit_code assert ( "Error: Foreign key already exists for author_id => authors.id" == result.output.strip() ) + + # No error if we add it twice with --ignore + result = CliRunner().invoke( + cli.cli, + ["add-foreign-key", db_path, "books", "author_id", "authors", "id", "--ignore"], + ) + assert 0 == result.exit_code + # Error if we try against an invalid column result = CliRunner().invoke( cli.cli, ["add-foreign-key", db_path, "books", "author_id", "authors", "bad"] diff --git a/tests/test_create.py b/tests/test_create.py index 1dfc541..9e30d2b 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -5,6 +5,7 @@ from sqlite_utils.db import ( AlterError, NoObviousTable, ForeignKey, + Table, ) from sqlite_utils.utils import sqlite3 import collections @@ -348,7 +349,9 @@ def test_add_foreign_key(fresh_db): ] ) assert [] == fresh_db["books"].foreign_keys - fresh_db["books"].add_foreign_key("author_id", "authors", "id") + t = fresh_db["books"].add_foreign_key("author_id", "authors", "id") + # Ensure it returned self: + assert isinstance(t, Table) and t.name == "books" assert [ ForeignKey( table="books", column="author_id", other_table="authors", other_column="id" @@ -379,6 +382,13 @@ def test_add_foreign_key_error_if_already_exists(fresh_db): assert "Foreign key already exists for author_id => authors.id" == ex.value.args[0] +def test_add_foreign_key_no_error_if_exists_and_ignore_true(fresh_db): + fresh_db["books"].insert({"title": "Hedgehogs of the world", "author_id": 1}) + fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fresh_db["books"].add_foreign_key("author_id", "authors", "id", ignore=True) + + def test_add_foreign_keys(fresh_db): fresh_db["authors"].insert_all( [{"id": 1, "name": "Sally"}, {"id": 2, "name": "Asheesh"}], pk="id" From ef882986d07f157b6bcc6be3d7b64270fda3e523 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Sep 2020 15:22:56 -0700 Subject: [PATCH 0199/1004] Release 2.19 Refs #112, #157, #160 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 48afc0b..9a2ab44 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v2_19: + +2.19 (2020-09-20) +----------------- + +- New ``sqlite-utils add-foreign-keys`` command for :ref:`cli_add_foreign_keys`. (`#157 `__) +- New ``table.enable_fts(..., replace=True)`` argument for replacing an existing FTS table with a new configuration. (`#160 `__) +- New ``table.add_foreign_key(..., ignore=True)`` argument for ignoring a foreign key if it already exists. (`#112 `__) + .. _v2_18: 2.18 (2020-09-08) diff --git a/setup.py b/setup.py index 6f50822..ccaf564 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.18" +VERSION = "2.19" def get_long_description(): From 482477585a0f3aec1ef3210dee941742d2a02e5e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Sep 2020 17:31:43 -0700 Subject: [PATCH 0200/1004] @db.register_function decorator, closes #162 --- docs/python-api.rst | 22 ++++++++++++++++++++++ sqlite_utils/db.py | 6 ++++++ tests/test_register_function.py | 16 ++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 tests/test_register_function.py diff --git a/docs/python-api.rst b/docs/python-api.rst index fae8cee..2ded7b0 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1415,3 +1415,25 @@ You can use it in code like this: if spatialite: db.conn.enable_load_extension(True) db.conn.load_extension(spatialite) + +.. _python_api_register_function: + +Registering custom SQL functions +================================ + +SQLite supports registering custom SQL functions written in Python. The ``@db.register_function`` decorator provides syntactic sugar for registering these functions. + +.. code-block:: python + + from sqlite_utils import Database + + db = Database(memory=True) + + @db.register_function + def reverse_string(s): + return "".join(reversed(list(s))) + + print(db.execute('select reverse_string("hello")').fetchone())[0] + # This prints "olleh" + +If your Python function accepts multiple arguments the SQL version will accept the same number of arguments. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ebffabf..daf3e25 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -4,6 +4,7 @@ import contextlib import datetime import decimal import hashlib +import inspect import itertools import json import os @@ -143,6 +144,11 @@ class Database: def __repr__(self): return "".format(self.conn) + def register_function(self, fn): + name = fn.__name__ + arity = len(inspect.signature(fn).parameters) + self.conn.create_function(name, arity, fn) + def execute(self, sql, parameters=None): if self._tracer: self._tracer(sql, parameters) diff --git a/tests/test_register_function.py b/tests/test_register_function.py new file mode 100644 index 0000000..4aa5a3b --- /dev/null +++ b/tests/test_register_function.py @@ -0,0 +1,16 @@ +def test_register_function(fresh_db): + @fresh_db.register_function + def reverse_string(s): + return "".join(reversed(list(s))) + + result = fresh_db.execute('select reverse_string("hello")').fetchone()[0] + assert result == "olleh" + + +def test_register_function_multiple_arguments(fresh_db): + @fresh_db.register_function + def a_times_b_plus_c(a, b, c): + return a * b + c + + result = fresh_db.execute("select a_times_b_plus_c(2, 3, 4)").fetchone()[0] + assert result == 10 From 987dd123f2ac43c5ab66d69e59d454fe09660606 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Sep 2020 21:20:01 -0700 Subject: [PATCH 0201/1004] table.transform() method - closes #114 --- docs/python-api.rst | 93 +++++++++++++ sqlite_utils/db.py | 179 +++++++++++++++++++++++- tests/test_transform.py | 298 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 568 insertions(+), 2 deletions(-) create mode 100644 tests/test_transform.py diff --git a/docs/python-api.rst b/docs/python-api.rst index 2ded7b0..026e2e0 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -908,6 +908,99 @@ You can drop a table or view using the ``.drop()`` method: db["my_table"].drop() +.. _python_api_transform: + +Transforming a table +==================== + +The SQLite ``ALTER TABLE`` statement is limited. It can add columns and rename tables, but it cannot rename columns, drop columns, change column types, change ``NOT NULL`` status or change the primary key for a table. + +The ``table.transform()`` method can do all of these things, by implementing a multi-step pattern `described in the SQLite documentation `__: + +1. Start a transaction +2. ``CREATE TABLE tablename_new_x123`` with the required changes +3. Copy the old data into the new table using ``INSERT INTO tablename_new_x123 SELECT * FROM tablename;`` +4. ``DROP TABLE tablename;`` +5. ``ALTER TABLE tablename_new_x123 RENAME TO tablename;`` +6. Commit the transaction + +The ``.transform()`` method takes a number of parameters, all of which are optional. + +To alter the type of a column, use the first argument: + +.. code-block:: python + + # Convert the 'age' column to an integer, and 'weight' to a float + table.transform({"age": int, "weight": float}) + +The ``rename=`` parameter can rename columns: + +.. code-block:: python + + # Rename 'age' to 'initial_age': + table.transform(rename={"age": "initial_age"}) + +To drop columns, pass them in the ``drop=`` set: + +.. code-block:: python + + # Drop the 'age' column: + table.transform(drop={"age"}) + +To change the primary key for a table, use ``pk=``. This can be passed a single column for a regular primary key, or a tuple of columns to create a compound primary key. Passing ``pk=None`` will remove the primary key and convert the table into a ``rowid`` table. + +.. code-block:: python + + # Make `user_id` the new primary key + table.transform(pk="user_id") + +You can change the ``NOT NULL`` status of columns by using ``not_null=``. You can pass this a set of columns to make those columns ``NOT NULL``: + +.. code-block:: python + + # Make the 'age' and 'weight' columns NOT NULL + table.transform(not_null={"age", "weight"}) + +If you want to take existing ``NOT NULL`` columns and change them to allow null values, you can do so by passing a dictionary of true/false values instead: + +.. code-block:: python + + # 'age' is NOT NULL but we want to allow NULL: + table.transform(not_null={"age": False}) + + # Make age allow NULL and switch weight to being NOT NULL: + table.transform(not_null={"age": False, "weight": True}) + +The ``defaults=`` parameter can be used to set or change the defaults for different columns: + +.. code-block:: python + + # Set default age to 1: + table.transform(defaults={"age": 1}) + + # Now remove the default from that column: + table.transform(defaults={"age": None}) + +You can use ``.transform()`` to remove foreign key constraints from a table. You will need to know the name of the column, the name of the table it points to and the name of the column it references on that other table. + +This example drops two foreign keys - the one from ``places.country`` to ``country.id`` and the one from ``places.continent`` to ``continent.id``: + +.. code-block:: python + + db["places"].transform( + drop_foreign_keys=( + ("country", "country", "id"), + ("continent", "continent", "id"), + ) + ) + +.. _python_api_transform_sql: + +Custom transformations with .transform_sql() +-------------------------------------------- + +If you want to do something more advanced, you can call the ``table.transform_sql(...)`` method with the same arguments that you would have passed to ``table.transform(...)``. This method will return a list of SQL statements that should be executed to implement the change. You can then make modifications to that SQL before executing it yourself. + .. _python_api_hash: Setting an ID based on the hash of the row contents diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index daf3e25..d8ad05b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -268,7 +268,7 @@ class Database: ) return fks - def create_table( + def create_table_sql( self, name, columns, @@ -336,7 +336,7 @@ class Database: column_extras.append("PRIMARY KEY") if column_name in not_null: column_extras.append("NOT NULL") - if column_name in defaults: + if column_name in defaults and defaults[column_name] is not None: column_extras.append( "DEFAULT {}".format(self.escape(defaults[column_name])) ) @@ -368,6 +368,31 @@ class Database: """.format( table=name, columns_sql=columns_sql, extra_pk=extra_pk ) + return sql + + def create_table( + self, + name, + columns, + pk=None, + foreign_keys=None, + column_order=None, + not_null=None, + defaults=None, + hash_id=None, + extracts=None, + ): + sql = self.create_table_sql( + name=name, + columns=columns, + pk=pk, + foreign_keys=foreign_keys, + column_order=column_order, + not_null=not_null, + defaults=defaults, + hash_id=hash_id, + extracts=extracts, + ) self.execute(sql) return self.table( name, @@ -691,6 +716,156 @@ class Table(Queryable): ) return self + def transform( + self, + columns=None, + rename=None, + drop=None, + pk=DEFAULT, + not_null=None, + defaults=None, + drop_foreign_keys=None, + ): + assert self.exists(), "Cannot transform a table that doesn't exist yet" + sqls = self.transform_sql( + columns=columns, + rename=rename, + drop=None, + pk=pk, + not_null=not_null, + defaults=defaults, + drop_foreign_keys=drop_foreign_keys, + ) + initial_pragma_foreign_keys = self.db.execute("PRAGMA foreign_keys").fetchone()[ + 0 + ] + try: + with self.db.conn: + for sql in sqls: + self.db.execute(sql) + finally: + # Make sure we reset PRAGMA foreign_keys correctly + if ( + initial_pragma_foreign_keys + and not self.db.execute("PRAGMA foreign_keys").fetchone()[0] + ): + self.db.execute("PRAGMA foreign_keys=1") + return self + + def transform_sql( + self, + columns=None, + rename=None, + drop=None, + pk=DEFAULT, + not_null=None, + defaults=None, + drop_foreign_keys=None, + tmp_suffix=None, + ): + columns = columns or {} + rename = rename or {} + drop = drop or set() + new_table_name = "{}_new_{}".format( + self.name, tmp_suffix or os.urandom(6).hex() + ) + current_column_pairs = list(self.columns_dict.items()) + new_column_pairs = [] + copy_from_to = {column: column for column, _ in current_column_pairs} + for name, type_ in current_column_pairs: + type_ = columns.get(name) or type_ + if name in drop: + del [copy_from_to[name]] + continue + new_name = rename.get(name) or name + new_column_pairs.append((new_name, type_)) + copy_from_to[name] = new_name + + should_flip_foreign_keys_pragma = self.db.execute( + "PRAGMA foreign_keys" + ).fetchone()[0] + + sqls = [] + + if should_flip_foreign_keys_pragma: + sqls.append("PRAGMA foreign_keys=OFF") + + if pk is DEFAULT: + pks_renamed = tuple(rename.get(p) or p for p in self.pks) + if len(pks_renamed) == 1: + pk = pks_renamed[0] + else: + pk = pks_renamed + + # not_null may be a set or dict, need to convert to a set + create_table_not_null = {c.name for c in self.columns if c.notnull} + if isinstance(not_null, dict): + # Remove any columns with a value of False + for key, value in not_null.items(): + # Column may have been renamed + key = rename.get(key) or key + if value is False and key in create_table_not_null: + create_table_not_null.remove(key) + else: + create_table_not_null.add(key) + elif isinstance(not_null, set): + create_table_not_null.update(rename.get(k) or k for k in not_null) + + # defaults= + create_table_defaults = { + (rename.get(c.name) or c.name): c.default_value + for c in self.columns + if c.default_value is not None + } + if defaults is not None: + create_table_defaults.update( + {rename.get(c) or c: v for c, v in defaults.items()} + ) + + # foreign_keys + create_table_foreign_keys = [] + for table, column, other_table, other_column in self.foreign_keys: + if (drop_foreign_keys is None) or ( + (column, other_table, other_column) not in drop_foreign_keys + ): + create_table_foreign_keys.append( + (rename.get(column) or column, other_table, other_column) + ) + + sqls.append( + self.db.create_table_sql( + new_table_name, + dict(new_column_pairs), + pk=pk, + not_null=create_table_not_null, + defaults=create_table_defaults, + foreign_keys=create_table_foreign_keys, + ).strip() + ) + # Copy across data, respecting any renamed columns + new_cols = [] + old_cols = [] + for from_, to_ in copy_from_to.items(): + old_cols.append(from_) + new_cols.append(to_) + copy_sql = "INSERT INTO [{new_table}] ({new_cols}) SELECT {old_cols} FROM [{old_table}]".format( + new_table=new_table_name, + old_table=self.name, + old_cols=", ".join("[{}]".format(col) for col in old_cols), + new_cols=", ".join("[{}]".format(col) for col in new_cols), + ) + sqls.append(copy_sql) + # Drop the old table + sqls.append("DROP TABLE [{}]".format(self.name)) + # Rename the new one + sqls.append("ALTER TABLE [{}] RENAME TO [{}]".format(new_table_name, self.name)) + + if should_flip_foreign_keys_pragma: + sqls.append("PRAGMA foreign_key_check") + sqls.append("PRAGMA foreign_keys=ON") + + return sqls + def create_index(self, columns, index_name=None, unique=False, if_not_exists=False): if index_name is None: index_name = "idx_{}_{}".format( diff --git a/tests/test_transform.py b/tests/test_transform.py new file mode 100644 index 0000000..c49efe1 --- /dev/null +++ b/tests/test_transform.py @@ -0,0 +1,298 @@ +from sqlite_utils.db import ForeignKey +from sqlite_utils.utils import OperationalError +import pytest + + +@pytest.mark.parametrize( + "params,expected_sql", + [ + # Identity transform - nothing changes + ( + {}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age]) SELECT [id], [name], [age] FROM [dogs]", + "DROP TABLE [dogs]", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + ], + ), + # Change column type + ( + {"columns": {"age": int}}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] INTEGER\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age]) SELECT [id], [name], [age] FROM [dogs]", + "DROP TABLE [dogs]", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + ], + ), + # Rename a column + ( + {"rename": {"age": "dog_age"}}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [dog_age] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age]) SELECT [id], [name], [age] FROM [dogs]", + "DROP TABLE [dogs]", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + ], + ), + # Drop a column + ( + {"drop": ["age"]}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name]) SELECT [id], [name] FROM [dogs]", + "DROP TABLE [dogs]", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + ], + ), + # Convert type AND rename column + ( + {"columns": {"age": int}, "rename": {"age": "dog_age"}}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [dog_age] INTEGER\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age]) SELECT [id], [name], [age] FROM [dogs]", + "DROP TABLE [dogs]", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + ], + ), + # Change primary key + ( + {"pk": "age"}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT PRIMARY KEY\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age]) SELECT [id], [name], [age] FROM [dogs]", + "DROP TABLE [dogs]", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + ], + ), + # Change primary key to a compound pk + ( + {"pk": ("age", "name")}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT,\n PRIMARY KEY ([age], [name])\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age]) SELECT [id], [name], [age] FROM [dogs]", + "DROP TABLE [dogs]", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + ], + ), + # Remove primary key, creating a rowid table + ( + {"pk": None}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age]) SELECT [id], [name], [age] FROM [dogs]", + "DROP TABLE [dogs]", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + ], + ), + ], +) +@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) +def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys): + dogs = fresh_db["dogs"] + if use_pragma_foreign_keys: + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + expected_sql.insert(0, "PRAGMA foreign_keys=OFF") + expected_sql.append("PRAGMA foreign_key_check") + expected_sql.append("PRAGMA foreign_keys=ON") + dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") + sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}}) + assert sql == expected_sql + # Check that .transform() runs without exceptions: + dogs.transform(**params) + + +def test_transform_sql_rowid_to_id(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) + assert ( + dogs.schema + == "CREATE TABLE [dogs] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT\n)" + ) + dogs.transform(pk="id") + # Slight oddity: [dogs] becomes "dogs" during the rename: + assert ( + dogs.schema + == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] TEXT\n)' + ) + + +def test_transform_rename_pk(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") + dogs.transform(rename={"id": "pk"}) + assert ( + dogs.schema + == 'CREATE TABLE "dogs" (\n [pk] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] TEXT\n)' + ) + + +def test_transform_not_null(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") + dogs.transform(not_null={"name"}) + assert ( + dogs.schema + == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT NOT NULL,\n [age] TEXT\n)' + ) + + +def test_transform_remove_a_not_null(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, not_null={"age"}, pk="id") + dogs.transform(not_null={"name": True, "age": False}) + assert ( + dogs.schema + == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT NOT NULL,\n [age] TEXT\n)' + ) + + +@pytest.mark.parametrize("not_null", [{"age"}, {"age": True}]) +def test_transform_add_not_null_with_rename(fresh_db, not_null): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") + dogs.transform(not_null=not_null, rename={"age": "dog_age"}) + assert ( + dogs.schema + == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [dog_age] TEXT NOT NULL\n)' + ) + + +def test_transform_defaults(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id") + dogs.transform(defaults={"age": 1}) + assert ( + dogs.schema + == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] INTEGER DEFAULT 1\n)' + ) + + +def test_transform_defaults_and_rename_column(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id") + dogs.transform(rename={"age": "dog_age"}, defaults={"age": 1}) + assert ( + dogs.schema + == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [dog_age] INTEGER DEFAULT 1\n)' + ) + + +def test_remove_defaults(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "age": 5}, defaults={"age": 1}, pk="id") + dogs.transform(defaults={"age": None}) + assert ( + dogs.schema + == 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] INTEGER\n)' + ) + + +@pytest.fixture +def authors_db(fresh_db): + books = fresh_db["books"] + authors = fresh_db["authors"] + authors.insert({"id": 5, "name": "Jane McGonical"}, pk="id") + books.insert( + {"id": 2, "title": "Reality is Broken", "author_id": 5}, + foreign_keys=("author_id",), + pk="id", + ) + return fresh_db + + +def test_transform_foreign_keys_persist(authors_db): + assert authors_db["books"].foreign_keys == [ + ForeignKey( + table="books", column="author_id", other_table="authors", other_column="id" + ) + ] + authors_db["books"].transform(rename={"title": "book_title"}) + assert authors_db["books"].foreign_keys == [ + ForeignKey( + table="books", column="author_id", other_table="authors", other_column="id" + ) + ] + + +@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) +def test_transform_foreign_keys_survive_renamed_column( + authors_db, use_pragma_foreign_keys +): + if use_pragma_foreign_keys: + authors_db.conn.execute("PRAGMA foreign_keys=ON") + authors_db["books"].transform(rename={"author_id": "author_id_2"}) + assert authors_db["books"].foreign_keys == [ + ForeignKey( + table="books", + column="author_id_2", + other_table="authors", + other_column="id", + ) + ] + + +@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) +def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys): + if use_pragma_foreign_keys: + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + # Create table with three foreign keys so we can drop two of them + fresh_db["country"].insert({"id": 1, "name": "France"}, pk="id") + fresh_db["continent"].insert({"id": 2, "name": "Europe"}, pk="id") + fresh_db["city"].insert({"id": 24, "name": "Paris"}, pk="id") + fresh_db["places"].insert( + { + "id": 32, + "name": "Caveau de la Huchette", + "country": 1, + "continent": 2, + "city": 24, + }, + foreign_keys=("country", "continent", "city"), + ) + assert fresh_db["places"].foreign_keys == [ + ForeignKey( + table="places", column="city", other_table="city", other_column="id" + ), + ForeignKey( + table="places", + column="continent", + other_table="continent", + other_column="id", + ), + ForeignKey( + table="places", column="country", other_table="country", other_column="id" + ), + ] + # Drop two of those foreign keys + fresh_db["places"].transform( + drop_foreign_keys=( + ("country", "country", "id"), + ("continent", "continent", "id"), + ) + ) + # Should be only one foreign key now + assert fresh_db["places"].foreign_keys == [ + ForeignKey(table="places", column="city", other_table="city", other_column="id") + ] + if use_pragma_foreign_keys: + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + +def test_transform_verify_foreign_keys(fresh_db): + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db["authors"].insert({"id": 3, "name": "Tina"}, pk="id") + fresh_db["books"].insert( + {"id": 1, "title": "Book", "author_id": 3}, pk="id", foreign_keys={"author_id"} + ) + # Renaming the id column on authors should break everything + with pytest.raises(OperationalError) as e: + fresh_db["authors"].transform(rename={"id": "id2"}) + assert e.value.args[0] == 'foreign key mismatch - "books" referencing "authors"' + # This should have rolled us back + assert ( + fresh_db["authors"].schema + == "CREATE TABLE [authors] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)" + ) + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] From f8e10df00eae209fb0a1ea03384d9153f673a3ec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Sep 2020 23:39:10 -0700 Subject: [PATCH 0202/1004] Keyword only arguments for transform() Also renamed columns= to types= Closes #165 --- docs/python-api.rst | 4 ++-- sqlite_utils/db.py | 12 +++++++----- tests/test_transform.py | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 026e2e0..ba903a3 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -926,12 +926,12 @@ The ``table.transform()`` method can do all of these things, by implementing a m The ``.transform()`` method takes a number of parameters, all of which are optional. -To alter the type of a column, use the first argument: +To alter the type of a column, use the ``types=`` argument: .. code-block:: python # Convert the 'age' column to an integer, and 'weight' to a float - table.transform({"age": int, "weight": float}) + table.transform(types={"age": int, "weight": float}) The ``rename=`` parameter can rename columns: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d8ad05b..df95c3e 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -718,7 +718,8 @@ class Table(Queryable): def transform( self, - columns=None, + *, + types=None, rename=None, drop=None, pk=DEFAULT, @@ -728,7 +729,7 @@ class Table(Queryable): ): assert self.exists(), "Cannot transform a table that doesn't exist yet" sqls = self.transform_sql( - columns=columns, + types=types, rename=rename, drop=None, pk=pk, @@ -754,7 +755,8 @@ class Table(Queryable): def transform_sql( self, - columns=None, + *, + types=None, rename=None, drop=None, pk=DEFAULT, @@ -763,7 +765,7 @@ class Table(Queryable): drop_foreign_keys=None, tmp_suffix=None, ): - columns = columns or {} + types = types or {} rename = rename or {} drop = drop or set() new_table_name = "{}_new_{}".format( @@ -773,7 +775,7 @@ class Table(Queryable): new_column_pairs = [] copy_from_to = {column: column for column, _ in current_column_pairs} for name, type_ in current_column_pairs: - type_ = columns.get(name) or type_ + type_ = types.get(name) or type_ if name in drop: del [copy_from_to[name]] continue diff --git a/tests/test_transform.py b/tests/test_transform.py index c49efe1..f1bb894 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -18,7 +18,7 @@ import pytest ), # Change column type ( - {"columns": {"age": int}}, + {"types": {"age": int}}, [ "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] INTEGER\n);", "INSERT INTO [dogs_new_suffix] ([id], [name], [age]) SELECT [id], [name], [age] FROM [dogs]", @@ -48,7 +48,7 @@ import pytest ), # Convert type AND rename column ( - {"columns": {"age": int}, "rename": {"age": "dog_age"}}, + {"types": {"age": int}, "rename": {"age": "dog_age"}}, [ "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [dog_age] INTEGER\n);", "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age]) SELECT [id], [name], [age] FROM [dogs]", From 752d2612296a553cdbeadecad769eb199099e88c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 00:46:32 -0700 Subject: [PATCH 0203/1004] Implemented sqlite-utils transform command, closes #164 --- docs/cli.rst | 65 +++++++++++++++++++++++++- sqlite_utils/cli.py | 100 ++++++++++++++++++++++++++++++++++++++++ sqlite_utils/db.py | 31 ++++++++----- tests/test_cli.py | 98 +++++++++++++++++++++++++++++++++++++++ tests/test_transform.py | 54 +++++++++++----------- 5 files changed, 309 insertions(+), 39 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index bd73eff..198f19e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -545,7 +545,70 @@ Dropping tables You can drop a table using the ``drop-table`` command:: - $ sqlite-utils drop-table mytable + $ sqlite-utils drop-table mydb.db mytable + +.. _cli_transform_table: + +Transforming tables +=================== + +The ``transform`` command allows you to apply complex transformations to a table that cannot be implemented using a regular SQLite ``ALTER TABLE`` command. See :ref:`python_api_transform` for details of how this works. + +:: + + $ sqlite-utils transform mydb.db mytable \ + --drop column1 \ + --rename column2 column_renamed + +Every option for this table (with the exception of ``--pk-none``) can be specified multiple times. The options are as follows: + +``--type column-name new-type`` + Change the type of the specified column. Valid types are ``integer``, ``text``, ``float``, ``blob``. + +``--drop column-name`` + Drop the specified column. + +``--rename column-name new-name`` + Rename this column to a new name. + +``--not-null column-name`` + Set this column as ``NOT NULL``. + +``--not-null-false column-name`` + For a column that is currently set as ``NOT NULL``, remove the ``NOT NULL``. + +``--pk column-name`` + Change the primary key column for this table. Pass ``--pk`` multiple times if you want to create a compound primary key. + +``--pk-none`` + Remove the primary key from this table, turning it into a ``rowid`` table. + +``--default column-name value`` + Set the default value of this column. + +``--default-none column`` + Remove the default value for this column. + +``--drop-foreign-key col other_table other_column`` + Drop the specified foreign key. + +If you want to see the SQL that will be executed to make the change without actually executing it, add the ``--sql`` flag. For example:: + + % sqlite-utils transform fixtures.db roadside_attractions \ + --rename pk id \ + --default name Untitled \ + --drop address \ + --sql + CREATE TABLE [roadside_attractions_new_4033a60276b9] ( + [id] INTEGER PRIMARY KEY, + [name] TEXT DEFAULT 'Untitled', + [latitude] FLOAT, + [longitude] FLOAT + ); + INSERT INTO [roadside_attractions_new_4033a60276b9] ([id], [name], [latitude], [longitude]) + SELECT [pk], [name], [latitude], [longitude] FROM [roadside_attractions]; + DROP TABLE [roadside_attractions]; + ALTER TABLE [roadside_attractions_new_4033a60276b9] RENAME TO [roadside_attractions]; .. _cli_create_view: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cfdffed..058bf04 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -887,6 +887,106 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols) ) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table") +@click.option( + "--type", + type=(str, str), + multiple=True, + help="Change column type to X", +) +@click.option("--drop", type=str, multiple=True, help="Drop this column") +@click.option( + "--rename", type=(str, str), multiple=True, help="Rename this column to X" +) +@click.option("--not-null", type=str, multiple=True, help="Set this column to NOT NULL") +@click.option( + "--not-null-false", type=str, multiple=True, help="Remove NOT NULL from this column" +) +@click.option("--pk", type=str, multiple=True, help="Make this column the primary key") +@click.option( + "--pk-none", is_flag=True, help="Remove primary key (convert to rowid table)" +) +@click.option( + "--default", + type=(str, str), + multiple=True, + help="Set default value for this column", +) +@click.option( + "--default-none", type=str, multiple=True, help="Remove default from this column" +) +@click.option( + "--drop-foreign-key", + type=(str, str, str), + multiple=True, + help="Drop this foreign key constraint", +) +@click.option("--sql", is_flag=True, help="Output SQL without executing it") +def transform( + path, + table, + type, + drop, + rename, + not_null, + not_null_false, + pk, + pk_none, + default, + default_none, + drop_foreign_key, + sql, +): + db = sqlite_utils.Database(path) + types = {} + kwargs = {} + for column, ctype in type: + if ctype.upper() not in VALID_COLUMN_TYPES: + raise click.ClickException( + "column types must be one of {}".format(VALID_COLUMN_TYPES) + ) + types[column] = ctype.upper() + + not_null_dict = {} + for column in not_null: + not_null_dict[column] = True + for column in not_null_false: + not_null_dict[column] = False + + default_dict = {} + for column, value in default: + default_dict[column] = value + for column in default_none: + default_dict[column] = None + + kwargs["types"] = types + kwargs["drop"] = set(drop) + kwargs["rename"] = dict(rename) + kwargs["not_null"] = not_null_dict + if pk: + if len(pk) == 1: + kwargs["pk"] = pk[0] + else: + kwargs["pk"] = pk + elif pk_none: + kwargs["pk"] = None + kwargs["defaults"] = default_dict + if drop_foreign_key: + kwargs["drop_foreign_keys"] = drop_foreign_key + + if sql: + for line in db[table].transform_sql(**kwargs): + click.echo(line) + else: + db[table].transform(**kwargs) + + @cli.command(name="insert-files") @click.argument( "path", diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index df95c3e..bcab8a5 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -731,7 +731,7 @@ class Table(Queryable): sqls = self.transform_sql( types=types, rename=rename, - drop=None, + drop=drop, pk=pk, not_null=not_null, defaults=defaults, @@ -790,7 +790,7 @@ class Table(Queryable): sqls = [] if should_flip_foreign_keys_pragma: - sqls.append("PRAGMA foreign_keys=OFF") + sqls.append("PRAGMA foreign_keys=OFF;") if pk is DEFAULT: pks_renamed = tuple(rename.get(p) or p for p in self.pks) @@ -800,7 +800,12 @@ class Table(Queryable): pk = pks_renamed # not_null may be a set or dict, need to convert to a set - create_table_not_null = {c.name for c in self.columns if c.notnull} + create_table_not_null = { + rename.get(c.name) or c.name + for c in self.columns + if c.notnull + if c.name not in drop + } if isinstance(not_null, dict): # Remove any columns with a value of False for key, value in not_null.items(): @@ -811,13 +816,16 @@ class Table(Queryable): else: create_table_not_null.add(key) elif isinstance(not_null, set): - create_table_not_null.update(rename.get(k) or k for k in not_null) - + create_table_not_null.update((rename.get(k) or k) for k in not_null) + elif not_null is None: + pass + else: + assert False, "not_null must be a dict or a set or None" # defaults= create_table_defaults = { (rename.get(c.name) or c.name): c.default_value for c in self.columns - if c.default_value is not None + if c.default_value is not None and c.name not in drop } if defaults is not None: create_table_defaults.update( @@ -844,13 +852,14 @@ class Table(Queryable): foreign_keys=create_table_foreign_keys, ).strip() ) + # Copy across data, respecting any renamed columns new_cols = [] old_cols = [] for from_, to_ in copy_from_to.items(): old_cols.append(from_) new_cols.append(to_) - copy_sql = "INSERT INTO [{new_table}] ({new_cols}) SELECT {old_cols} FROM [{old_table}]".format( + copy_sql = "INSERT INTO [{new_table}] ({new_cols})\n SELECT {old_cols} FROM [{old_table}];".format( new_table=new_table_name, old_table=self.name, old_cols=", ".join("[{}]".format(col) for col in old_cols), @@ -858,13 +867,13 @@ class Table(Queryable): ) sqls.append(copy_sql) # Drop the old table - sqls.append("DROP TABLE [{}]".format(self.name)) + sqls.append("DROP TABLE [{}];".format(self.name)) # Rename the new one - sqls.append("ALTER TABLE [{}] RENAME TO [{}]".format(new_table_name, self.name)) + sqls.append("ALTER TABLE [{}] RENAME TO [{}];".format(new_table_name, self.name)) if should_flip_foreign_keys_pragma: - sqls.append("PRAGMA foreign_key_check") - sqls.append("PRAGMA foreign_keys=ON") + sqls.append("PRAGMA foreign_key_check;") + sqls.append("PRAGMA foreign_keys=ON;") return sqls diff --git a/tests/test_cli.py b/tests/test_cli.py index 7d8ccf4..43ab4b0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1451,3 +1451,101 @@ def test_add_foreign_keys(db_path): table="books", column="author_id", other_table="authors", other_column="id" ) ] + + +@pytest.mark.parametrize( + "args,expected_schema", + [ + ( + [], + "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ), + ( + ["--type", "age", "text"], + "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] TEXT NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ), + ( + ["--drop", "age"], + 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)', + ), + ( + ["--rename", "age", "age2", "--rename", "id", "pk"], + "CREATE TABLE \"dogs\" (\n [pk] INTEGER PRIMARY KEY,\n [age2] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ), + ( + ["--not-null", "name"], + "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT NOT NULL\n)", + ), + ( + ["--not-null-false", "age"], + "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER DEFAULT '1',\n [name] TEXT\n)", + ), + ( + ["--pk", "name"], + "CREATE TABLE \"dogs\" (\n [id] INTEGER,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT PRIMARY KEY\n)", + ), + ( + ["--pk-none"], + "CREATE TABLE \"dogs\" (\n [id] INTEGER,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ), + ( + ["--default", "name", "Turnip"], + "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT DEFAULT 'Turnip'\n)", + ), + ( + ["--default-none", "age"], + 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL,\n [name] TEXT\n)', + ), + ], +) +def test_transform(db_path, args, expected_schema): + db = Database(db_path) + with db.conn: + db["dogs"].insert( + {"id": 1, "age": 4, "name": "Cleo"}, + not_null={"age"}, + defaults={"age": 1}, + pk="id", + ) + result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs"] + args) + print(result.output) + assert result.exit_code == 0 + schema = db["dogs"].schema + assert schema == expected_schema + + +def test_transform_drop_foreign_key(db_path): + db = Database(db_path) + with db.conn: + # Create table with three foreign keys so we can drop two of them + db["country"].insert({"id": 1, "name": "France"}, pk="id") + db["city"].insert({"id": 24, "name": "Paris"}, pk="id") + db["places"].insert( + { + "id": 32, + "name": "Caveau de la Huchette", + "country": 1, + "city": 24, + }, + foreign_keys=("country", "city"), + pk="id", + ) + result = CliRunner().invoke( + cli.cli, + [ + "transform", + db_path, + "places", + "--drop-foreign-key", + "country", + "country", + "id", + ], + ) + print(result.output) + assert result.exit_code == 0 + schema = db["places"].schema + assert ( + schema + == 'CREATE TABLE "places" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [country] INTEGER,\n [city] INTEGER REFERENCES [city]([id])\n)' + ) diff --git a/tests/test_transform.py b/tests/test_transform.py index f1bb894..352efe4 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -11,9 +11,9 @@ import pytest {}, [ "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] TEXT\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age]) SELECT [id], [name], [age] FROM [dogs]", - "DROP TABLE [dogs]", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", ], ), # Change column type @@ -21,9 +21,9 @@ import pytest {"types": {"age": int}}, [ "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] INTEGER\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age]) SELECT [id], [name], [age] FROM [dogs]", - "DROP TABLE [dogs]", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", ], ), # Rename a column @@ -31,9 +31,9 @@ import pytest {"rename": {"age": "dog_age"}}, [ "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [dog_age] TEXT\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age]) SELECT [id], [name], [age] FROM [dogs]", - "DROP TABLE [dogs]", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", ], ), # Drop a column @@ -41,9 +41,9 @@ import pytest {"drop": ["age"]}, [ "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name]) SELECT [id], [name] FROM [dogs]", - "DROP TABLE [dogs]", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + "INSERT INTO [dogs_new_suffix] ([id], [name])\n SELECT [id], [name] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", ], ), # Convert type AND rename column @@ -51,9 +51,9 @@ import pytest {"types": {"age": int}, "rename": {"age": "dog_age"}}, [ "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [dog_age] INTEGER\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age]) SELECT [id], [name], [age] FROM [dogs]", - "DROP TABLE [dogs]", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", ], ), # Change primary key @@ -61,9 +61,9 @@ import pytest {"pk": "age"}, [ "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT PRIMARY KEY\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age]) SELECT [id], [name], [age] FROM [dogs]", - "DROP TABLE [dogs]", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", ], ), # Change primary key to a compound pk @@ -71,9 +71,9 @@ import pytest {"pk": ("age", "name")}, [ "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT,\n PRIMARY KEY ([age], [name])\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age]) SELECT [id], [name], [age] FROM [dogs]", - "DROP TABLE [dogs]", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", ], ), # Remove primary key, creating a rowid table @@ -81,9 +81,9 @@ import pytest {"pk": None}, [ "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT\n);", - "INSERT INTO [dogs_new_suffix] ([id], [name], [age]) SELECT [id], [name], [age] FROM [dogs]", - "DROP TABLE [dogs]", - "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs]", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", ], ), ], @@ -93,9 +93,9 @@ def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys): dogs = fresh_db["dogs"] if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") - expected_sql.insert(0, "PRAGMA foreign_keys=OFF") - expected_sql.append("PRAGMA foreign_key_check") - expected_sql.append("PRAGMA foreign_keys=ON") + expected_sql.insert(0, "PRAGMA foreign_keys=OFF;") + expected_sql.append("PRAGMA foreign_key_check;") + expected_sql.append("PRAGMA foreign_keys=ON;") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}}) assert sql == expected_sql From f29f6821f2d08e91c5c6d65d885a1bbc0c743bdd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 00:49:27 -0700 Subject: [PATCH 0204/1004] Applied Black --- sqlite_utils/db.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index bcab8a5..452c74a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -869,7 +869,9 @@ class Table(Queryable): # Drop the old table sqls.append("DROP TABLE [{}];".format(self.name)) # Rename the new one - sqls.append("ALTER TABLE [{}] RENAME TO [{}];".format(new_table_name, self.name)) + sqls.append( + "ALTER TABLE [{}] RENAME TO [{}];".format(new_table_name, self.name) + ) if should_flip_foreign_keys_pragma: sqls.append("PRAGMA foreign_key_check;") From f8553799d38deece370f890f6c90af32f52a609b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 15:20:18 -0700 Subject: [PATCH 0205/1004] table.extract() method, refs #42 --- docs/python-api.rst | 135 ++++++++++++++++++++++++++++++++++++++++++ sqlite_utils/db.py | 32 ++++++++++ tests/test_extract.py | 105 ++++++++++++++++++++++++++++++++ 3 files changed, 272 insertions(+) create mode 100644 tests/test_extract.py diff --git a/docs/python-api.rst b/docs/python-api.rst index ba903a3..ebfc203 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1001,6 +1001,141 @@ Custom transformations with .transform_sql() If you want to do something more advanced, you can call the ``table.transform_sql(...)`` method with the same arguments that you would have passed to ``table.transform(...)``. This method will return a list of SQL statements that should be executed to implement the change. You can then make modifications to that SQL before executing it yourself. +.. _python_api_extract: + +Extracting columns into a separate table +======================================== + +The ``table.extract()`` method can be used to extract specified columns into a separate table. + +Imagine a ``Trees`` table that looks like this: + +=== ============ ======= + id TreeAddress Species +=== ============ ======= + 1 52 Vine St Palm + 2 12 Draft St Oak + 3 51 Dark Ave Palm + 4 1252 Left St Palm +=== ============ ======= + +The ``Species`` column contains duplicate values. This database could be improved by extracting that column out into a separate ``Species`` table and pointing to it using a foreign key column. + +The schema of the above table is: + +.. code-block:: sql + + CREATE TABLE [Trees] ( + [id] INTEGER PRIMARY KEY, + [TreeAddress] TEXT, + [Species] TEXT + ) + +Here's how to extract the ``Species`` column using ``.extract()``: + +.. code-block:: python + + db["Trees"].extract("Species") + +After running this code the table schema now looks like this: + +.. code-block:: sql + + CREATE TABLE "Trees" ( + [id] INTEGER PRIMARY KEY, + [TreeAddress] TEXT, + [Species_id] INTEGER, + FOREIGN KEY(Species_id) REFERENCES Species(id) + ) + +A new ``Species`` table will have been created with the following schema: + +.. code-block:: sql + + CREATE TABLE [Species] ( + [id] INTEGER PRIMARY KEY, + [Species] TEXT + ) + +The ``.extract()`` method defaults to creating a table with the same name as the column that was extracted, and adding a foreign key column called ``tablename_id``. + +You can specify a custom table name using ``table=``, and a custom foreign key name using ``fk_column=``. This example creates a table called ``tree_species`` and a foreign key column called ``tree_species_id``: + +.. code-block:: python + + db["Trees"].extract("Species", table="tree_species", fk_column="tree_species_id") + +The resulting schema looks like this: + +.. code-block:: sql + + CREATE TABLE "Trees" ( + [id] INTEGER PRIMARY KEY, + [TreeAddress] TEXT, + [tree_species_id] INTEGER, + FOREIGN KEY(tree_species_id) REFERENCES tree_species(id) + ) + + CREATE TABLE [tree_species] ( + [id] INTEGER PRIMARY KEY, + [Species] TEXT + ) + +You can also extract multiple columns into the same external table. Say for example you have a table like this: + +=== ============ ========== ========= + id TreeAddress CommonName LatinName +=== ============ ========== ========= + 1 52 Vine St Palm Arecaceae + 2 12 Draft St Oak Quercus + 3 51 Dark Ave Palm Arecaceae + 4 1252 Left St Palm Arecaceae +=== ============ ========== ========= + +You can pass ``["CommonName", "LatinName"]`` to ``.extract()`` to extract both of those columns: + +.. code-block:: python + + db["Trees"].extract(["CommonName", "LatinName"]) + +This produces the following schema: + +.. code-block:: sql + + CREATE TABLE "Trees" ( + [id] INTEGER PRIMARY KEY, + [TreeAddress] TEXT, + [CommonName_LatinName_id] INTEGER, + FOREIGN KEY(CommonName_LatinName_id) REFERENCES CommonName_LatinName(id) + ) + CREATE TABLE [CommonName_LatinName] ( + [id] INTEGER PRIMARY KEY, + [CommonName] TEXT, + [LatinName] TEXT + ) + +The table name ``CommonName_LatinName`` is derived from the extract columns. You can use ``table=`` and ``fk_column=`` to specify custom names like this: + +.. code-block:: python + + db["Trees"].extract(["CommonName", "LatinName"], table="Species", fk_column="species_id") + +This produces the following schema: + +.. code-block:: sql + + CREATE TABLE "Trees" ( + [id] INTEGER PRIMARY KEY, + [TreeAddress] TEXT, + [species_id] INTEGER, + FOREIGN KEY(species_id) REFERENCES Species(id) + ) + CREATE TABLE [Species] ( + [id] INTEGER PRIMARY KEY, + [CommonName] TEXT, + [LatinName] TEXT + ) + .. _python_api_hash: Setting an ID based on the hash of the row contents diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 452c74a..846f97d 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -104,6 +104,10 @@ class PrimaryKeyRequired(Exception): pass +class InvalidColumns(Exception): + pass + + class Database: def __init__( self, @@ -879,6 +883,34 @@ class Table(Queryable): return sqls + def extract(self, columns, table=None, fk_column=None): + if isinstance(columns, str): + columns = [columns] + if not set(columns).issubset(self.columns_dict.keys()): + raise InvalidColumns( + "Invalid columns {} for table with columns {}".format( + columns, list(self.columns_dict.keys()) + ) + ) + table = table or "_".join(columns) + first_column = columns[0] + pks = self.pks + lookup_table = self.db[table] + for row in self.rows: + row_pks = tuple(row[pk] for pk in pks) + lookups = {column: row[column] for column in columns} + self.update(row_pks, {first_column: lookup_table.lookup(lookups)}) + fk_column = fk_column or "{}_id".format(table) + # Now rename first_column and change its type to integer, and drop + # any other extracted columns: + self.transform( + types={first_column: int}, + drop=set(columns[1:]), + rename={first_column: fk_column}, + ) + # And add the foreign key constraint + self.add_foreign_key(fk_column, table, "id") + def create_index(self, columns, index_name=None, unique=False, if_not_exists=False): if index_name is None: index_name = "idx_{}_{}".format( diff --git a/tests/test_extract.py b/tests/test_extract.py new file mode 100644 index 0000000..31b6d3e --- /dev/null +++ b/tests/test_extract.py @@ -0,0 +1,105 @@ +from sqlite_utils.db import Index, InvalidColumns +import itertools +import pytest + + +@pytest.mark.parametrize("table", [None, "Species"]) +@pytest.mark.parametrize("fk_column", [None, "species"]) +def test_extract_single_column(fresh_db, table, fk_column): + expected_table = table or "species" + expected_fk = fk_column or "{}_id".format(expected_table) + iter_species = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) + fresh_db["tree"].insert_all( + ( + {"id": i, "name": "Tree {}".format(i), "species": next(iter_species)} + for i in range(1, 1001) + ), + pk="id", + ) + fresh_db["tree"].extract("species", table=table, fk_column=fk_column) + assert fresh_db["tree"].schema == ( + 'CREATE TABLE "tree" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [name] TEXT,\n" + " [{}] INTEGER,\n".format(expected_fk) + + " FOREIGN KEY({}) REFERENCES {}(id)\n".format(expected_fk, expected_table) + + ")" + ) + assert fresh_db[expected_table].schema == ( + "CREATE TABLE [{}] (\n".format(expected_table) + + " [id] INTEGER PRIMARY KEY,\n" + " [species] TEXT\n" + ")" + ) + assert list(fresh_db[expected_table].rows) == [ + {"id": 1, "species": "Palm"}, + {"id": 2, "species": "Spruce"}, + {"id": 3, "species": "Mangrove"}, + {"id": 4, "species": "Oak"}, + ] + assert list(itertools.islice(fresh_db["tree"].rows, 0, 4)) == [ + {"id": 1, "name": "Tree 1", expected_fk: 1}, + {"id": 2, "name": "Tree 2", expected_fk: 2}, + {"id": 3, "name": "Tree 3", expected_fk: 3}, + {"id": 4, "name": "Tree 4", expected_fk: 4}, + ] + + +def test_extract_multiple_columns(fresh_db): + iter_common = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) + iter_latin = itertools.cycle(["Arecaceae", "Picea", "Rhizophora", "Quercus"]) + fresh_db["tree"].insert_all( + ( + { + "id": i, + "name": "Tree {}".format(i), + "common_name": next(iter_common), + "latin_name": next(iter_latin), + } + for i in range(1, 1001) + ), + pk="id", + ) + + fresh_db["tree"].extract(["common_name", "latin_name"]) + assert fresh_db["tree"].schema == ( + 'CREATE TABLE "tree" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [name] TEXT,\n" + " [common_name_latin_name_id] INTEGER,\n" + " FOREIGN KEY(common_name_latin_name_id) REFERENCES common_name_latin_name(id)\n" + ")" + ) + assert fresh_db["common_name_latin_name"].schema == ( + "CREATE TABLE [common_name_latin_name] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [common_name] TEXT,\n" + " [latin_name] TEXT\n" + ")" + ) + assert list(fresh_db["common_name_latin_name"].rows) == [ + {"common_name": "Palm", "id": 1, "latin_name": "Arecaceae"}, + {"common_name": "Spruce", "id": 2, "latin_name": "Picea"}, + {"common_name": "Mangrove", "id": 3, "latin_name": "Rhizophora"}, + {"common_name": "Oak", "id": 4, "latin_name": "Quercus"}, + ] + assert list(itertools.islice(fresh_db["tree"].rows, 0, 4)) == [ + {"id": 1, "name": "Tree 1", "common_name_latin_name_id": 1}, + {"id": 2, "name": "Tree 2", "common_name_latin_name_id": 2}, + {"id": 3, "name": "Tree 3", "common_name_latin_name_id": 3}, + {"id": 4, "name": "Tree 4", "common_name_latin_name_id": 4}, + ] + + +def test_extract_invalid_columns(fresh_db): + fresh_db["tree"].insert( + { + "id": 1, + "name": "Tree 1", + "common_name": "Palm", + "latin_name": "Arecaceae", + }, + pk="id", + ) + with pytest.raises(InvalidColumns): + fresh_db["tree"].extract(["bad_column"]) From c755f2852d8ef0a2142ba9e41197b2a6dd801d1d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 15:47:11 -0700 Subject: [PATCH 0206/1004] Docstring for sqlite-utils transform --- sqlite_utils/cli.py | 1 + tests/test_docs.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 058bf04..4fb8ebc 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -943,6 +943,7 @@ def transform( drop_foreign_key, sql, ): + "Transform a table beyond the capabilities of ALTER TABLE" db = sqlite_utils.Database(path) types = {} kwargs = {} diff --git a/tests/test_docs.py b/tests/test_docs.py index 3b3e4a5..1d659d1 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -20,3 +20,8 @@ def documented_commands(): @pytest.mark.parametrize("command", cli.cli.commands.keys()) def test_commands_are_documented(documented_commands, command): assert command in documented_commands + + +@pytest.mark.parametrize("command", cli.cli.commands.values()) +def test_commands_have_docstrings(command): + assert command.__doc__, '{} is missing a docstring'.format(command) From c3210f2ffb291ecbf23d4a80d17793f9f9bebfc9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 15:57:02 -0700 Subject: [PATCH 0207/1004] Added table.extract(rename=) option, refs #42 --- docs/python-api.rst | 21 +++++++++++++++++++++ sqlite_utils/db.py | 5 +++-- tests/test_extract.py | 16 +++++++++------- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index ebfc203..35023d5 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1136,6 +1136,27 @@ This produces the following schema: [LatinName] TEXT ) +You can use the ``rename=`` argument to rename columns in the lookup table. To create a ``Species`` table with columns called ``name`` and ``latin`` you can do this: + +.. code-block:: python + + db["Trees"].extract( + ["CommonName", "LatinName"], + table="Species", + fk_column="species_id", + rename={"CommonName": "name", "LatinName": "latin"} + ) + +This produces a lookup table like so: + +.. code-block:: sql + + CREATE TABLE [Species] ( + [id] INTEGER PRIMARY KEY, + [name] TEXT, + [latin] TEXT + ) + .. _python_api_hash: Setting an ID based on the hash of the row contents diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 846f97d..33af372 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -883,7 +883,8 @@ class Table(Queryable): return sqls - def extract(self, columns, table=None, fk_column=None): + def extract(self, columns, table=None, fk_column=None, rename=None): + rename = rename or {} if isinstance(columns, str): columns = [columns] if not set(columns).issubset(self.columns_dict.keys()): @@ -898,7 +899,7 @@ class Table(Queryable): lookup_table = self.db[table] for row in self.rows: row_pks = tuple(row[pk] for pk in pks) - lookups = {column: row[column] for column in columns} + lookups = {rename.get(column) or column: row[column] for column in columns} self.update(row_pks, {first_column: lookup_table.lookup(lookups)}) fk_column = fk_column or "{}_id".format(table) # Now rename first_column and change its type to integer, and drop diff --git a/tests/test_extract.py b/tests/test_extract.py index 31b6d3e..4855441 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -45,7 +45,7 @@ def test_extract_single_column(fresh_db, table, fk_column): ] -def test_extract_multiple_columns(fresh_db): +def test_extract_multiple_columns_with_rename(fresh_db): iter_common = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) iter_latin = itertools.cycle(["Arecaceae", "Picea", "Rhizophora", "Quercus"]) fresh_db["tree"].insert_all( @@ -61,7 +61,9 @@ def test_extract_multiple_columns(fresh_db): pk="id", ) - fresh_db["tree"].extract(["common_name", "latin_name"]) + fresh_db["tree"].extract(["common_name", "latin_name"], rename={ + "common_name": "name" + }) assert fresh_db["tree"].schema == ( 'CREATE TABLE "tree" (\n' " [id] INTEGER PRIMARY KEY,\n" @@ -73,15 +75,15 @@ def test_extract_multiple_columns(fresh_db): assert fresh_db["common_name_latin_name"].schema == ( "CREATE TABLE [common_name_latin_name] (\n" " [id] INTEGER PRIMARY KEY,\n" - " [common_name] TEXT,\n" + " [name] TEXT,\n" " [latin_name] TEXT\n" ")" ) assert list(fresh_db["common_name_latin_name"].rows) == [ - {"common_name": "Palm", "id": 1, "latin_name": "Arecaceae"}, - {"common_name": "Spruce", "id": 2, "latin_name": "Picea"}, - {"common_name": "Mangrove", "id": 3, "latin_name": "Rhizophora"}, - {"common_name": "Oak", "id": 4, "latin_name": "Quercus"}, + {"name": "Palm", "id": 1, "latin_name": "Arecaceae"}, + {"name": "Spruce", "id": 2, "latin_name": "Picea"}, + {"name": "Mangrove", "id": 3, "latin_name": "Rhizophora"}, + {"name": "Oak", "id": 4, "latin_name": "Quercus"}, ] assert list(itertools.islice(fresh_db["tree"].rows, 0, 4)) == [ {"id": 1, "name": "Tree 1", "common_name_latin_name_id": 1}, From 317071a552003384c939a7551684f7299792ad18 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 16:09:28 -0700 Subject: [PATCH 0208/1004] Applied Black --- tests/test_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_docs.py b/tests/test_docs.py index 1d659d1..87b685e 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -24,4 +24,4 @@ def test_commands_are_documented(documented_commands, command): @pytest.mark.parametrize("command", cli.cli.commands.values()) def test_commands_have_docstrings(command): - assert command.__doc__, '{} is missing a docstring'.format(command) + assert command.__doc__, "{} is missing a docstring".format(command) From 71782311ce5a4535a0820c7a55fc813e6a12ae16 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 16:10:14 -0700 Subject: [PATCH 0209/1004] New .rows_where(select=) argument --- docs/python-api.rst | 9 ++++++++- sqlite_utils/db.py | 4 ++-- tests/test_rows.py | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 35023d5..4041d4d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -145,7 +145,7 @@ View objects are similar to Table objects, except that any attempts to insert or * ``count`` * ``schema`` * ``rows`` -* ``rows_where(where, where_args, order_by)`` +* ``rows_where(where, where_args, order_by, select)`` * ``drop()`` .. _python_api_rows: @@ -168,6 +168,13 @@ You can filter rows by a WHERE clause using ``.rows_where(where, where_args)``:: ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} +To return custom columns (instead of using ``select *``) pass ``select=``:: + + >>> db = sqlite_utils.Database("dogs.db") + >>> for row in db["dogs"].rows_where(select='name, age'): + ... print(row) + {'name': 'Cleo', 'age': 4} + To specify an order, use the ``order_by=`` argument:: >>> for row in db["dogs"].rows_where("age > 1", order_by="age"): diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 33af372..6ba718a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -532,10 +532,10 @@ class Queryable: def rows(self): return self.rows_where() - def rows_where(self, where=None, where_args=None, order_by=None): + def rows_where(self, where=None, where_args=None, order_by=None, select="*"): if not self.exists(): return [] - sql = "select * from [{}]".format(self.name) + sql = "select {} from [{}]".format(select, self.name) if where is not None: sql += " where " + where if order_by is not None: diff --git a/tests/test_rows.py b/tests/test_rows.py index 539d71d..8885802 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -26,7 +26,7 @@ def test_rows_where(where, where_args, expected_ids, fresh_db): ], pk="id", ) - assert expected_ids == {r["id"] for r in table.rows_where(where, where_args)} + assert expected_ids == {r["id"] for r in table.rows_where(where, where_args, select="id")} @pytest.mark.parametrize( From 2db6c5b2d57f8f35124c8da70d60331b3fbc658b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 16:11:07 -0700 Subject: [PATCH 0210/1004] table.extract() now works with rowid tables, refs #42 --- sqlite_utils/db.py | 6 +++++- tests/test_extract.py | 25 ++++++++++++++++++++++--- tests/test_rows.py | 4 +++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6ba718a..2430f0d 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -897,7 +897,11 @@ class Table(Queryable): first_column = columns[0] pks = self.pks lookup_table = self.db[table] - for row in self.rows: + if pks == ["rowid"]: + rows_iter = self.rows_where(select="rowid, *") + else: + rows_iter = self.rows + for row in rows_iter: row_pks = tuple(row[pk] for pk in pks) lookups = {rename.get(column) or column: row[column] for column in columns} self.update(row_pks, {first_column: lookup_table.lookup(lookups)}) diff --git a/tests/test_extract.py b/tests/test_extract.py index 4855441..4108a5a 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -61,9 +61,9 @@ def test_extract_multiple_columns_with_rename(fresh_db): pk="id", ) - fresh_db["tree"].extract(["common_name", "latin_name"], rename={ - "common_name": "name" - }) + fresh_db["tree"].extract( + ["common_name", "latin_name"], rename={"common_name": "name"} + ) assert fresh_db["tree"].schema == ( 'CREATE TABLE "tree" (\n' " [id] INTEGER PRIMARY KEY,\n" @@ -105,3 +105,22 @@ def test_extract_invalid_columns(fresh_db): ) with pytest.raises(InvalidColumns): fresh_db["tree"].extract(["bad_column"]) + + +def test_extract_rowid_table(fresh_db): + fresh_db["tree"].insert( + { + "name": "Tree 1", + "common_name": "Palm", + "latin_name": "Arecaceae", + } + ) + fresh_db["tree"].extract(["common_name", "latin_name"]) + assert fresh_db["tree"].schema == ( + 'CREATE TABLE "tree" (\n' + " [rowid] INTEGER PRIMARY KEY,\n" + " [name] TEXT,\n" + " [common_name_latin_name_id] INTEGER,\n" + " FOREIGN KEY(common_name_latin_name_id) REFERENCES common_name_latin_name(id)\n" + ")" + ) diff --git a/tests/test_rows.py b/tests/test_rows.py index 8885802..603f254 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -26,7 +26,9 @@ def test_rows_where(where, where_args, expected_ids, fresh_db): ], pk="id", ) - assert expected_ids == {r["id"] for r in table.rows_where(where, where_args, select="id")} + assert expected_ids == { + r["id"] for r in table.rows_where(where, where_args, select="id") + } @pytest.mark.parametrize( From 55cf928f73254273370d8489b4143875de4cabf2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 16:37:39 -0700 Subject: [PATCH 0211/1004] sqlite-utils extract, closes #42 --- docs/cli.rst | 91 +++++++++++++++++++++++++++++++++++++++++++++ sqlite_utils/cli.py | 33 ++++++++++++++++ tests/test_cli.py | 50 +++++++++++++++++++++++++ 3 files changed, 174 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 198f19e..8971fe9 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -610,6 +610,97 @@ If you want to see the SQL that will be executed to make the change without actu DROP TABLE [roadside_attractions]; ALTER TABLE [roadside_attractions_new_4033a60276b9] RENAME TO [roadside_attractions]; +.. _cli_extract: + +Extracting columns into a separate table +======================================== + +The ``sqlite-utils extract`` command can be used to extract specified columns into a separate table. + +Take a look at the Python API documentation for :ref:`python_api_extract` for a detailed description of how this works, including examples of table schemas before and after running an extraction operation. + +The command takes a database, table and one or more columns that should be extracted. To extract the ``species`` column from the ``trees`` table you would run:: + + $ sqlite-utils extract my.db trees species + +This would produce the following schema: + +.. code-block:: sql + + CREATE TABLE "trees" ( + [id] INTEGER PRIMARY KEY, + [TreeAddress] TEXT, + [species_id] INTEGER, + FOREIGN KEY(species_id) REFERENCES species(id) + ) + + CREATE TABLE [species] ( + [id] INTEGER PRIMARY KEY, + [species] TEXT + ) + +The command takes the following options: + +``--table TEXT`` + The name of the lookup to extract columns to. This defaults to using the name of the columns that are being extracted. + +``--fk-column TEXT`` + The name of the foreign key column to add to the table. Defaults to ``columnname_id``. + +``--rename `` + Use this option to rename the columns created in the new lookup table. + +Here's a more complex example that makes use of these options. It converts `this CSV file `__ full of global power plants into SQLite, then extracts the ``country`` and ``country_long`` columns into a separate ``countries`` table:: + + wget 'https://github.com/wri/global-power-plant-database/blob/232a6666/output_database/global_power_plant_database.csv?raw=true' + sqlite-utils insert global.db power_plants \ + 'global_power_plant_database.csv?raw=true' --csv + # Extract those columns: + sqlite-utils extract global.db power_plants country country_long \ + --table countries \ + --fk-column country_id \ + --rename country_long name + +After running the above, the command ``sqlite3 global.db .schema`` reveals the following schema: + +.. code-block:: sql + + CREATE TABLE [countries] ( + [id] INTEGER PRIMARY KEY, + [country] TEXT, + [name] TEXT + ); + CREATE UNIQUE INDEX [idx_countries_country_name] + ON [countries] ([country], [name]); + CREATE TABLE IF NOT EXISTS "power_plants" ( + [rowid] INTEGER PRIMARY KEY, + [country_id] INTEGER, + [name] TEXT, + [gppd_idnr] TEXT, + [capacity_mw] TEXT, + [latitude] TEXT, + [longitude] TEXT, + [primary_fuel] TEXT, + [other_fuel1] TEXT, + [other_fuel2] TEXT, + [other_fuel3] TEXT, + [commissioning_year] TEXT, + [owner] TEXT, + [source] TEXT, + [url] TEXT, + [geolocation_source] TEXT, + [wepp_id] TEXT, + [year_of_capacity_data] TEXT, + [generation_gwh_2013] TEXT, + [generation_gwh_2014] TEXT, + [generation_gwh_2015] TEXT, + [generation_gwh_2016] TEXT, + [generation_gwh_2017] TEXT, + [generation_data_source] TEXT, + [estimated_generation_gwh] TEXT, + FOREIGN KEY(country_id) REFERENCES countries(id) + ); + .. _cli_create_view: Creating views diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4fb8ebc..adf33ef 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -988,6 +988,39 @@ def transform( db[table].transform(**kwargs) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table") +@click.argument("columns", nargs=-1, required=True) +@click.option( + "--table", "other_table", help="Name of the other table to extract columns to" +) +@click.option("--fk-column", help="Name of the foreign key column to add to the table") +@click.option( + "--rename", + type=(str, str), + multiple=True, + help="Rename this column in extracted table", +) +def extract( + path, + table, + columns, + other_table, + fk_column, + rename, +): + "Extract one or more columns into a separate table" + db = sqlite_utils.Database(path) + db[table].extract( + columns, table=other_table, fk_column=fk_column, rename=dict(rename) + ) + + @cli.command(name="insert-files") @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index 43ab4b0..0dcf331 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1549,3 +1549,53 @@ def test_transform_drop_foreign_key(db_path): schema == 'CREATE TABLE "places" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [country] INTEGER,\n [city] INTEGER REFERENCES [city]([id])\n)' ) + + +_common_other_schema = ( + "CREATE TABLE [species] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)" +) + + +@pytest.mark.parametrize( + "args,expected_table_schema,expected_other_schema", + [ + ( + [], + 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY(species_id) REFERENCES species(id)\n)', + _common_other_schema, + ), + ( + ["--table", "custom_table"], + 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_table_id] INTEGER,\n FOREIGN KEY(custom_table_id) REFERENCES custom_table(id)\n)', + "CREATE TABLE [custom_table] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)", + ), + ( + ["--fk-column", "custom_fk"], + 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_fk] INTEGER,\n FOREIGN KEY(custom_fk) REFERENCES species(id)\n)', + _common_other_schema, + ), + ( + ["--rename", "name", "name2"], + 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY(species_id) REFERENCES species(id)\n)', + "CREATE TABLE [species] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)", + ), + ], +) +def test_extract(db_path, args, expected_table_schema, expected_other_schema): + db = Database(db_path) + with db.conn: + db["trees"].insert( + {"id": 1, "address": "4 Park Ave", "species": "Palm"}, + pk="id", + ) + result = CliRunner().invoke( + cli.cli, ["extract", db_path, "trees", "species"] + args + ) + print(result.output) + assert result.exit_code == 0 + schema = db["trees"].schema + assert schema == expected_table_schema + other_schema = [t for t in db.tables if t.name not in ("trees", "Gosh", "Gosh2")][ + 0 + ].schema + assert other_schema == expected_other_schema From 5c4d58d1528367c15ec6490024bf2658f251acd3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 17:02:29 -0700 Subject: [PATCH 0212/1004] Progress bar for "sqlite-utils extract", closes #169 --- docs/cli.rst | 3 +++ sqlite_utils/cli.py | 22 ++++++++++++++++++++-- sqlite_utils/db.py | 4 +++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 8971fe9..85fba97 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -650,6 +650,9 @@ The command takes the following options: ``--rename `` Use this option to rename the columns created in the new lookup table. +``--silent`` + Don't display the progress bar. + Here's a more complex example that makes use of these options. It converts `this CSV file `__ full of global power plants into SQLite, then extracts the ``country`` and ``country_long`` columns into a separate ``countries`` table:: wget 'https://github.com/wri/global-power-plant-database/blob/232a6666/output_database/global_power_plant_database.csv?raw=true' diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index adf33ef..9ff56f5 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1006,6 +1006,12 @@ def transform( multiple=True, help="Rename this column in extracted table", ) +@click.option( + "-s", + "--silent", + is_flag=True, + help="Don't show progress bar", +) def extract( path, table, @@ -1013,12 +1019,24 @@ def extract( other_table, fk_column, rename, + silent, ): "Extract one or more columns into a separate table" db = sqlite_utils.Database(path) - db[table].extract( - columns, table=other_table, fk_column=fk_column, rename=dict(rename) + kwargs = dict( + columns=columns, + table=other_table, + fk_column=fk_column, + rename=dict(rename), ) + if silent: + db[table].extract(**kwargs) + else: + with click.progressbar( + length=db[table].count, label="Extracting columns" + ) as bar: + kwargs["progress"] = bar.update + db[table].extract(**kwargs) @cli.command(name="insert-files") diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 2430f0d..e9d5298 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -883,7 +883,7 @@ class Table(Queryable): return sqls - def extract(self, columns, table=None, fk_column=None, rename=None): + def extract(self, columns, table=None, fk_column=None, rename=None, progress=None): rename = rename or {} if isinstance(columns, str): columns = [columns] @@ -905,6 +905,8 @@ class Table(Queryable): row_pks = tuple(row[pk] for pk in pks) lookups = {rename.get(column) or column: row[column] for column in columns} self.update(row_pks, {first_column: lookup_table.lookup(lookups)}) + if progress: + progress(1) fk_column = fk_column or "{}_id".format(table) # Now rename first_column and change its type to integer, and drop # any other extracted columns: From b8e0048485a76cdf056f06c3bf1b641f02b9ea40 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 17:12:56 -0700 Subject: [PATCH 0213/1004] Fixed PRAGMA foreign_keys handling for .transform, closes #167 --- sqlite_utils/db.py | 32 ++++++++++---------------------- tests/test_transform.py | 16 ++++++++++++---- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e9d5298..cbd657d 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -741,20 +741,21 @@ class Table(Queryable): defaults=defaults, drop_foreign_keys=drop_foreign_keys, ) - initial_pragma_foreign_keys = self.db.execute("PRAGMA foreign_keys").fetchone()[ - 0 - ] + pragma_foreign_keys_was_on = self.db.execute( + "PRAGMA foreign_keys" + ).fetchone()[0] try: + if pragma_foreign_keys_was_on: + self.db.execute("PRAGMA foreign_keys=0;") with self.db.conn: for sql in sqls: self.db.execute(sql) + # Run the foreign_key_check before we commit + if pragma_foreign_keys_was_on: + self.db.execute("PRAGMA foreign_key_check;") finally: - # Make sure we reset PRAGMA foreign_keys correctly - if ( - initial_pragma_foreign_keys - and not self.db.execute("PRAGMA foreign_keys").fetchone()[0] - ): - self.db.execute("PRAGMA foreign_keys=1") + if pragma_foreign_keys_was_on: + self.db.execute("PRAGMA foreign_keys=1;") return self def transform_sql( @@ -787,15 +788,7 @@ class Table(Queryable): new_column_pairs.append((new_name, type_)) copy_from_to[name] = new_name - should_flip_foreign_keys_pragma = self.db.execute( - "PRAGMA foreign_keys" - ).fetchone()[0] - sqls = [] - - if should_flip_foreign_keys_pragma: - sqls.append("PRAGMA foreign_keys=OFF;") - if pk is DEFAULT: pks_renamed = tuple(rename.get(p) or p for p in self.pks) if len(pks_renamed) == 1: @@ -876,11 +869,6 @@ class Table(Queryable): sqls.append( "ALTER TABLE [{}] RENAME TO [{}];".format(new_table_name, self.name) ) - - if should_flip_foreign_keys_pragma: - sqls.append("PRAGMA foreign_key_check;") - sqls.append("PRAGMA foreign_keys=ON;") - return sqls def extract(self, columns, table=None, fk_column=None, rename=None, progress=None): diff --git a/tests/test_transform.py b/tests/test_transform.py index 352efe4..7075fcf 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -90,17 +90,25 @@ import pytest ) @pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys): + captured = [] + tracer = lambda sql, params: captured.append((sql, params)) dogs = fresh_db["dogs"] if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") - expected_sql.insert(0, "PRAGMA foreign_keys=OFF;") - expected_sql.append("PRAGMA foreign_key_check;") - expected_sql.append("PRAGMA foreign_keys=ON;") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}}) assert sql == expected_sql # Check that .transform() runs without exceptions: - dogs.transform(**params) + with fresh_db.tracer(tracer): + dogs.transform(**params) + # If use_pragma_foreign_keys, check that we did the right thing + if use_pragma_foreign_keys: + assert ('PRAGMA foreign_keys=0;', None) in captured + assert captured[-2] == ('PRAGMA foreign_key_check;', None) + assert captured[-1] == ('PRAGMA foreign_keys=1;', None) + else: + assert ('PRAGMA foreign_keys=0;', None) not in captured + assert ('PRAGMA foreign_keys=1;', None) not in captured def test_transform_sql_rowid_to_id(fresh_db): From dcdef136dbe05ecbd156e99688d2a938f307a581 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 17:31:15 -0700 Subject: [PATCH 0214/1004] Release 2.20 Refs #114, #42, #162, #164, #165, #167, #169. Closes #170 --- docs/changelog.rst | 29 +++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9a2ab44..0051480 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,35 @@ Changelog =========== +.. _v2_20: + +2.20 (2020-09-22) +----------------- + +This release introduces two key new capabilities: **transform** (`#114 `__) and **extract** (`#42 `__). + +Transform +~~~~~~~~~ + +SQLite's ALTER TABLE has `several documented limitations `__. The ``table.transform()`` Python method and ``sqlite-utils transform`` CLI command work around these limitations using a pattern where a new table with the desired structure is created, data is copied over to it and the old table is then dropped and replaced by the new one. + +You can use these tools to drop columns, change column types, rename columns, add and remove ``NOT NULL`` and defaults, remove foreign key constraints and more. See the :ref:`transforming tables (CLI) ` and :ref:`transforming tables (Python library) ` documentation for full details of how to use them. + +Extract +~~~~~~~ + +Sometimes a database table - especially one imported from a CSV file - will contain duplicate data. A ``Trees`` table may include a ``Species`` column with only a few dozen unique values, when the table itself contains thousands of rows. + +The ``table.extract()`` method and ``sqlite-utils extract`` commands can extract a column - or multiple columns - out into a separate lookup table, and set up a foreign key relationship from the original table. + +The Python library :ref:`extract() documentation ` describes how extraction works in detail, and :ref:`cli_extract` in the CLI documentation includes a detailed example. + +Other changes +~~~~~~~~~~~~~ + +- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref;`python_api_register_function`. (`#162 `__) +- The ``table.rows_where()`` method now accepts an optional ``select=`` argument for specifying which columns should be selected, see :ref:`python_api_rows`. + .. _v2_19: 2.19 (2020-09-20) diff --git a/setup.py b/setup.py index ccaf564..7e3d792 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.19" +VERSION = "2.20" def get_long_description(): From 5534c320e4dfdf0ee854704a40ced275f70edb05 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 17:32:40 -0700 Subject: [PATCH 0215/1004] Applied Black --- sqlite_utils/db.py | 6 +++--- tests/test_transform.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index cbd657d..953455a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -741,9 +741,9 @@ class Table(Queryable): defaults=defaults, drop_foreign_keys=drop_foreign_keys, ) - pragma_foreign_keys_was_on = self.db.execute( - "PRAGMA foreign_keys" - ).fetchone()[0] + pragma_foreign_keys_was_on = self.db.execute("PRAGMA foreign_keys").fetchone()[ + 0 + ] try: if pragma_foreign_keys_was_on: self.db.execute("PRAGMA foreign_keys=0;") diff --git a/tests/test_transform.py b/tests/test_transform.py index 7075fcf..a96f9ba 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -103,12 +103,12 @@ def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys): dogs.transform(**params) # If use_pragma_foreign_keys, check that we did the right thing if use_pragma_foreign_keys: - assert ('PRAGMA foreign_keys=0;', None) in captured - assert captured[-2] == ('PRAGMA foreign_key_check;', None) - assert captured[-1] == ('PRAGMA foreign_keys=1;', None) + assert ("PRAGMA foreign_keys=0;", None) in captured + assert captured[-2] == ("PRAGMA foreign_key_check;", None) + assert captured[-1] == ("PRAGMA foreign_keys=1;", None) else: - assert ('PRAGMA foreign_keys=0;', None) not in captured - assert ('PRAGMA foreign_keys=1;', None) not in captured + assert ("PRAGMA foreign_keys=0;", None) not in captured + assert ("PRAGMA foreign_keys=1;", None) not in captured def test_transform_sql_rowid_to_id(fresh_db): From 9f59a7a325851b0026ffd3c385985e473fdff92c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 17:35:44 -0700 Subject: [PATCH 0216/1004] Fixed typo in release notes, refs #170 --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0051480..d4fb6db 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,7 +28,7 @@ The Python library :ref:`extract() documentation ` describes Other changes ~~~~~~~~~~~~~ -- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref;`python_api_register_function`. (`#162 `__) +- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref:`python_api_register_function`. (`#162 `__) - The ``table.rows_where()`` method now accepts an optional ``select=`` argument for specifying which columns should be selected, see :ref:`python_api_rows`. .. _v2_19: From 1ebffe1dbeaed7311e5b61ed988f4cd701e84808 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 20:09:42 -0700 Subject: [PATCH 0217/1004] Correction: SQLite ALTER TABLE can rename columns --- docs/changelog.rst | 2 +- docs/python-api.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d4fb6db..85378ee 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,7 +14,7 @@ Transform SQLite's ALTER TABLE has `several documented limitations `__. The ``table.transform()`` Python method and ``sqlite-utils transform`` CLI command work around these limitations using a pattern where a new table with the desired structure is created, data is copied over to it and the old table is then dropped and replaced by the new one. -You can use these tools to drop columns, change column types, rename columns, add and remove ``NOT NULL`` and defaults, remove foreign key constraints and more. See the :ref:`transforming tables (CLI) ` and :ref:`transforming tables (Python library) ` documentation for full details of how to use them. +You can use these tools to change column types, rename columns, drop columns, add and remove ``NOT NULL`` and defaults, remove foreign key constraints and more. See the :ref:`transforming tables (CLI) ` and :ref:`transforming tables (Python library) ` documentation for full details of how to use them. Extract ~~~~~~~ diff --git a/docs/python-api.rst b/docs/python-api.rst index 4041d4d..dcc4bf1 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -920,7 +920,7 @@ You can drop a table or view using the ``.drop()`` method: Transforming a table ==================== -The SQLite ``ALTER TABLE`` statement is limited. It can add columns and rename tables, but it cannot rename columns, drop columns, change column types, change ``NOT NULL`` status or change the primary key for a table. +The SQLite ``ALTER TABLE`` statement is limited. It can add columns and rename tables, but it cannot drop columns, change column types, change ``NOT NULL`` status or change the primary key for a table. The ``table.transform()`` method can do all of these things, by implementing a multi-step pattern `described in the SQLite documentation `__: From 66d506587eba9f0715267d6560b97c1fa44cc781 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 23 Sep 2020 13:12:09 -0700 Subject: [PATCH 0218/1004] Some optimizations for extract() Refs #172 - seems to give me about 20% speedup. --- sqlite_utils/db.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 953455a..e9e6ffa 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3,6 +3,7 @@ from collections import namedtuple, OrderedDict import contextlib import datetime import decimal +import functools import hashlib import inspect import itertools @@ -885,14 +886,26 @@ class Table(Queryable): first_column = columns[0] pks = self.pks lookup_table = self.db[table] + + @functools.lru_cache(maxsize=128) + def cached_lookup(lookups): + return lookup_table.lookup(dict(lookups)) + if pks == ["rowid"]: rows_iter = self.rows_where(select="rowid, *") else: rows_iter = self.rows for row in rows_iter: row_pks = tuple(row[pk] for pk in pks) - lookups = {rename.get(column) or column: row[column] for column in columns} - self.update(row_pks, {first_column: lookup_table.lookup(lookups)}) + lookups = tuple( + (rename.get(column) or column, row[column]) for column in columns + ) + self.update( + row_pks, + {first_column: cached_lookup(lookups)}, + assume_exists=True, + pks=self.pks, + ) if progress: progress(1) fk_column = fk_column or "{}_id".format(table) @@ -1241,13 +1254,23 @@ class Table(Queryable): sql += " where " + where self.db.execute(sql, where_args or []) - def update(self, pk_values, updates=None, alter=False, conversions=None): + def update( + self, + pk_values, + updates=None, + alter=False, + conversions=None, + assume_exists=False, + pks=None, + ): updates = updates or {} + pks = pks or self.pks conversions = conversions or {} if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] # Soundness check that the record exists (raises error if not): - self.get(pk_values) + if not assume_exists: + self.get(pk_values) if not updates: return self args = [] @@ -1257,7 +1280,7 @@ class Table(Queryable): for key, value in updates.items(): sets.append("[{}] = {}".format(key, conversions.get(key, "?"))) args.append(value) - wheres = ["[{}] = ?".format(pk_name) for pk_name in self.pks] + wheres = ["[{}] = ?".format(pk_name) for pk_name in pks] args.extend(pk_values) sql = "update [{table}] set {sets} where {wheres}".format( table=self.name, sets=", ".join(sets), wheres=" and ".join(wheres) @@ -1275,7 +1298,7 @@ class Table(Queryable): # TODO: Test this works (rolls back) - use better exception: assert rowcount == 1 - self.last_pk = pk_values[0] if len(self.pks) == 1 else pk_values + self.last_pk = pk_values[0] if len(pks) == 1 else pk_values return self def build_insert_queries_and_params( From 0ca5585fcb834122193e8e7186f926217b23cb8a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 23 Sep 2020 13:16:01 -0700 Subject: [PATCH 0219/1004] Clarify why you would want transform_sql() --- docs/python-api.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index dcc4bf1..3d826f1 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1006,7 +1006,11 @@ This example drops two foreign keys - the one from ``places.country`` to ``count Custom transformations with .transform_sql() -------------------------------------------- -If you want to do something more advanced, you can call the ``table.transform_sql(...)`` method with the same arguments that you would have passed to ``table.transform(...)``. This method will return a list of SQL statements that should be executed to implement the change. You can then make modifications to that SQL before executing it yourself. +The ``.transform()`` method can handle most cases, but it does not automatically upgrade indexes, views or triggers associated with the table that is being transformed. + +If you want to do something more advanced, you can call the ``table.transform_sql(...)`` method with the same arguments that you would have passed to ``table.transform(...)``. + +This method will return a list of SQL statements that should be executed to implement the change. You can then make modifications to that SQL - or add additional SQL statements - before executing it yourself. .. _python_api_extract: From 5eb14d1c1f2e76d67fb70128a61ad22a7b38cd6a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Sep 2020 07:51:36 -0700 Subject: [PATCH 0220/1004] Added several missing 'return self' to support chaining --- sqlite_utils/db.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e9e6ffa..a00e7ba 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1167,6 +1167,7 @@ class Table(Queryable): with self.db.conn: for trigger_name in trigger_names: self.db.execute("DROP TRIGGER IF EXISTS [{}]".format(trigger_name)) + return self def rebuild_fts(self): fts_table = self.detect_fts() @@ -1178,6 +1179,7 @@ class Table(Queryable): table=fts_table ) ) + return self def detect_fts(self): "Detect if table has a corresponding FTS virtual table and return it" @@ -1245,6 +1247,7 @@ class Table(Queryable): ) with self.db.conn: self.db.execute(sql, pk_values) + return self def delete_where(self, where=None, where_args=None): if not self.exists(): @@ -1253,6 +1256,7 @@ class Table(Queryable): if where is not None: sql += " where " + where self.db.execute(sql, where_args or []) + return self def update( self, @@ -1696,6 +1700,7 @@ class Table(Queryable): for col_name, col_type in needed_columns.items(): if col_name not in current_columns: self.add_column(col_name, col_type) + return self def lookup(self, column_values): # lookups is a dictionary - all columns will be used for a unique index From 022cdd97a9ddab1a152e23e1e1c42e78c1ed0fa9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Sep 2020 08:43:55 -0700 Subject: [PATCH 0221/1004] Much, much faster extract() implementation Takes my test down from ten minutes to four seconds! * Removed unnecessary update() optimization * Added column_order= to .transform() and .transform_sql() * Tests for reusing lookup table in extract() Closes #172 --- sqlite_utils/cli.py | 16 +----- sqlite_utils/db.py | 128 ++++++++++++++++++++++++++++-------------- tests/test_extract.py | 57 +++++++++++++++++-- 3 files changed, 139 insertions(+), 62 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 9ff56f5..7948d27 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1006,12 +1006,6 @@ def transform( multiple=True, help="Rename this column in extracted table", ) -@click.option( - "-s", - "--silent", - is_flag=True, - help="Don't show progress bar", -) def extract( path, table, @@ -1019,7 +1013,6 @@ def extract( other_table, fk_column, rename, - silent, ): "Extract one or more columns into a separate table" db = sqlite_utils.Database(path) @@ -1029,14 +1022,7 @@ def extract( fk_column=fk_column, rename=dict(rename), ) - if silent: - db[table].extract(**kwargs) - else: - with click.progressbar( - length=db[table].count, label="Extracting columns" - ) as bar: - kwargs["progress"] = bar.update - db[table].extract(**kwargs) + db[table].extract(**kwargs) @cli.command(name="insert-files") diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a00e7ba..f5868ed 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3,7 +3,6 @@ from collections import namedtuple, OrderedDict import contextlib import datetime import decimal -import functools import hashlib import inspect import itertools @@ -731,6 +730,7 @@ class Table(Queryable): not_null=None, defaults=None, drop_foreign_keys=None, + column_order=None, ): assert self.exists(), "Cannot transform a table that doesn't exist yet" sqls = self.transform_sql( @@ -741,6 +741,7 @@ class Table(Queryable): not_null=not_null, defaults=defaults, drop_foreign_keys=drop_foreign_keys, + column_order=column_order, ) pragma_foreign_keys_was_on = self.db.execute("PRAGMA foreign_keys").fetchone()[ 0 @@ -769,6 +770,7 @@ class Table(Queryable): not_null=None, defaults=None, drop_foreign_keys=None, + column_order=None, tmp_suffix=None, ): types = types or {} @@ -840,6 +842,9 @@ class Table(Queryable): (rename.get(column) or column, other_table, other_column) ) + if column_order is not None: + column_order = [rename.get(col) or col for col in column_order] + sqls.append( self.db.create_table_sql( new_table_name, @@ -848,6 +853,7 @@ class Table(Queryable): not_null=create_table_not_null, defaults=create_table_defaults, foreign_keys=create_table_foreign_keys, + column_order=column_order, ).strip() ) @@ -872,7 +878,7 @@ class Table(Queryable): ) return sqls - def extract(self, columns, table=None, fk_column=None, rename=None, progress=None): + def extract(self, columns, table=None, fk_column=None, rename=None): rename = rename or {} if isinstance(columns, str): columns = [columns] @@ -886,38 +892,85 @@ class Table(Queryable): first_column = columns[0] pks = self.pks lookup_table = self.db[table] - - @functools.lru_cache(maxsize=128) - def cached_lookup(lookups): - return lookup_table.lookup(dict(lookups)) - - if pks == ["rowid"]: - rows_iter = self.rows_where(select="rowid, *") - else: - rows_iter = self.rows - for row in rows_iter: - row_pks = tuple(row[pk] for pk in pks) - lookups = tuple( - (rename.get(column) or column, row[column]) for column in columns - ) - self.update( - row_pks, - {first_column: cached_lookup(lookups)}, - assume_exists=True, - pks=self.pks, - ) - if progress: - progress(1) fk_column = fk_column or "{}_id".format(table) - # Now rename first_column and change its type to integer, and drop - # any other extracted columns: - self.transform( - types={first_column: int}, - drop=set(columns[1:]), - rename={first_column: fk_column}, + magic_lookup_column = "{}_{}".format(fk_column, os.urandom(6).hex()) + + # Populate the lookup table with all of the extracted unique values + lookup_columns_definition = { + (rename.get(col) or col): typ + for col, typ in self.columns_dict.items() + if col in columns + } + if lookup_table.exists(): + if not set(lookup_columns_definition.items()).issubset( + lookup_table.columns_dict.items() + ): + raise InvalidColumns( + "Lookup table {} already exists but does not have columns {}".format( + table, lookup_columns_definition + ) + ) + else: + lookup_table.create( + { + **{ + "id": int, + }, + **lookup_columns_definition, + }, + pk="id", + ) + lookup_columns = [(rename.get(col) or col) for col in columns] + lookup_table.create_index(lookup_columns, unique=True, if_not_exists=True) + self.db.execute( + "INSERT OR IGNORE INTO [{lookup_table}] ({lookup_columns}) SELECT DISTINCT {table_cols} FROM [{table}]".format( + lookup_table=table, + lookup_columns=", ".join("[{}]".format(c) for c in lookup_columns), + table_cols=", ".join("[{}]".format(c) for c in columns), + table=self.name, + ) ) + + # Now add the new fk_column + self.add_column(magic_lookup_column, int) + + # And populate it + self.db.execute( + "UPDATE [{table}] SET [{magic_lookup_column}] = (SELECT id FROM [{lookup_table}] WHERE {where})".format( + table=self.name, + magic_lookup_column=magic_lookup_column, + lookup_table=table, + where=" AND ".join( + "[{table}].[{column}] = [{lookup_table}].[{lookup_column}]".format( + table=self.name, + lookup_table=table, + column=column, + lookup_column=rename.get(column) or column, + ) + for column in columns + ), + ) + ) + # Figure out the right column order + column_order = [] + for c in self.columns: + if c.name in columns and magic_lookup_column not in column_order: + column_order.append(magic_lookup_column) + elif c.name == magic_lookup_column: + continue + else: + column_order.append(c.name) + + # Drop the unnecessary columns and rename lookup column + self.transform( + drop=set(columns), + rename={magic_lookup_column: fk_column}, + column_order=column_order, + ) + # And add the foreign key constraint self.add_foreign_key(fk_column, table, "id") + return self def create_index(self, columns, index_name=None, unique=False, if_not_exists=False): if index_name is None: @@ -1258,28 +1311,19 @@ class Table(Queryable): self.db.execute(sql, where_args or []) return self - def update( - self, - pk_values, - updates=None, - alter=False, - conversions=None, - assume_exists=False, - pks=None, - ): + def update(self, pk_values, updates=None, alter=False, conversions=None): updates = updates or {} - pks = pks or self.pks conversions = conversions or {} if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] # Soundness check that the record exists (raises error if not): - if not assume_exists: - self.get(pk_values) + self.get(pk_values) if not updates: return self args = [] sets = [] wheres = [] + pks = self.pks validate_column_names(updates.keys()) for key, value in updates.items(): sets.append("[{}] = {}".format(key, conversions.get(key, "?"))) diff --git a/tests/test_extract.py b/tests/test_extract.py index 4108a5a..25dd6e2 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -11,7 +11,12 @@ def test_extract_single_column(fresh_db, table, fk_column): iter_species = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) fresh_db["tree"].insert_all( ( - {"id": i, "name": "Tree {}".format(i), "species": next(iter_species)} + { + "id": i, + "name": "Tree {}".format(i), + "species": next(iter_species), + "end": 1, + } for i in range(1, 1001) ), pk="id", @@ -22,6 +27,7 @@ def test_extract_single_column(fresh_db, table, fk_column): " [id] INTEGER PRIMARY KEY,\n" " [name] TEXT,\n" " [{}] INTEGER,\n".format(expected_fk) + + " [end] INTEGER,\n" + " FOREIGN KEY({}) REFERENCES {}(id)\n".format(expected_fk, expected_table) + ")" ) @@ -38,10 +44,10 @@ def test_extract_single_column(fresh_db, table, fk_column): {"id": 4, "species": "Oak"}, ] assert list(itertools.islice(fresh_db["tree"].rows, 0, 4)) == [ - {"id": 1, "name": "Tree 1", expected_fk: 1}, - {"id": 2, "name": "Tree 2", expected_fk: 2}, - {"id": 3, "name": "Tree 3", expected_fk: 3}, - {"id": 4, "name": "Tree 4", expected_fk: 4}, + {"id": 1, "name": "Tree 1", expected_fk: 1, "end": 1}, + {"id": 2, "name": "Tree 2", expected_fk: 2, "end": 1}, + {"id": 3, "name": "Tree 3", expected_fk: 3, "end": 1}, + {"id": 4, "name": "Tree 4", expected_fk: 4, "end": 1}, ] @@ -124,3 +130,44 @@ def test_extract_rowid_table(fresh_db): " FOREIGN KEY(common_name_latin_name_id) REFERENCES common_name_latin_name(id)\n" ")" ) + + +def test_reuse_lookup_table(fresh_db): + fresh_db["species"].insert({"id": 1, "name": "Wolf"}, pk="id") + fresh_db["sightings"].insert({"id": 10, "species": "Wolf"}, pk="id") + fresh_db["individuals"].insert( + {"id": 10, "name": "Terriana", "species": "Fox"}, pk="id" + ) + fresh_db["sightings"].extract("species", rename={"species": "name"}) + fresh_db["individuals"].extract("species", rename={"species": "name"}) + assert fresh_db["sightings"].schema == ( + 'CREATE TABLE "sightings" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [species_id] INTEGER,\n" + " FOREIGN KEY(species_id) REFERENCES species(id)\n" + ")" + ) + assert fresh_db["individuals"].schema == ( + 'CREATE TABLE "individuals" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [name] TEXT,\n" + " [species_id] INTEGER,\n" + " FOREIGN KEY(species_id) REFERENCES species(id)\n" + ")" + ) + assert list(fresh_db["species"].rows) == [ + {"id": 1, "name": "Wolf"}, + {"id": 2, "name": "Fox"}, + ] + + +def test_extract_error_on_incompatible_existing_lookup_table(fresh_db): + fresh_db["species"].insert({"id": 1}) + fresh_db["tree"].insert({"name": "Tree 1", "common_name": "Palm"}) + with pytest.raises(InvalidColumns): + fresh_db["tree"].extract("common_name", table="species") + + # Try again with incompatible existing column type + fresh_db["species2"].insert({"id": 1, "common_name": 3.5}) + with pytest.raises(InvalidColumns): + fresh_db["tree"].extract("common_name", table="species2") From 725f206949441e4679ef4d1c5995c1cf7015a83e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Sep 2020 09:00:50 -0700 Subject: [PATCH 0222/1004] Documentation for .transform(column_order=), closes #175 --- docs/python-api.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 3d826f1..b2452d7 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -988,6 +988,13 @@ The ``defaults=`` parameter can be used to set or change the defaults for differ # Now remove the default from that column: table.transform(defaults={"age": None}) +The ``column_order=`` parameter can be used to change the order of the columns. If you pass the names of a subset of the columns those will go first and columns you omitted will appear in their existing order after them. + +.. code-block:: python + + # Change column order + table.transform(column_order=("name", "age", "id") + You can use ``.transform()`` to remove foreign key constraints from a table. You will need to know the name of the column, the name of the table it points to and the name of the column it references on that other table. This example drops two foreign keys - the one from ``places.country`` to ``country.id`` and the one from ``places.continent`` to ``continent.id``: From d13c123100bddbe53b56cad6f9f0e7a0e50c4e0d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Sep 2020 09:11:53 -0700 Subject: [PATCH 0223/1004] sqlite-utils transform --column-order option, closes #176 --- docs/cli.rst | 3 +++ sqlite_utils/cli.py | 3 +++ tests/test_cli.py | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 85fba97..c05f05d 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -571,6 +571,9 @@ Every option for this table (with the exception of ``--pk-none``) can be specifi ``--rename column-name new-name`` Rename this column to a new name. +``--column-order column`` + Use this multiple times to specify a new order for your columns. ``-o`` shortcut is also available. + ``--not-null column-name`` Set this column as ``NOT NULL``. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 7948d27..535d3d0 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -904,6 +904,7 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols) @click.option( "--rename", type=(str, str), multiple=True, help="Rename this column to X" ) +@click.option("-o", "--column-order", type=str, multiple=True, help="Reorder columns") @click.option("--not-null", type=str, multiple=True, help="Set this column to NOT NULL") @click.option( "--not-null-false", type=str, multiple=True, help="Remove NOT NULL from this column" @@ -934,6 +935,7 @@ def transform( type, drop, rename, + column_order, not_null, not_null_false, pk, @@ -969,6 +971,7 @@ def transform( kwargs["types"] = types kwargs["drop"] = set(drop) kwargs["rename"] = dict(rename) + kwargs["column_order"] = column_order or None kwargs["not_null"] = not_null_dict if pk: if len(pk) == 1: diff --git a/tests/test_cli.py b/tests/test_cli.py index 0dcf331..5aad909 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1496,6 +1496,10 @@ def test_add_foreign_keys(db_path): ["--default-none", "age"], 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL,\n [name] TEXT\n)', ), + ( + ["-o", "name", "--column-order", "age", "-o", "id"], + "CREATE TABLE \"dogs\" (\n [name] TEXT,\n [age] INTEGER NOT NULL DEFAULT '1',\n [id] INTEGER PRIMARY KEY\n)", + ), ], ) def test_transform(db_path, args, expected_schema): From 5a63b9e88c5887432eb1d7df39f304ea55038437 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Sep 2020 09:19:07 -0700 Subject: [PATCH 0224/1004] Simplify drop-foreign-key, and drop_foreign_keys, closes #177 --- docs/cli.rst | 2 +- docs/python-api.rst | 7 ++----- sqlite_utils/cli.py | 2 +- sqlite_utils/db.py | 4 +--- tests/test_cli.py | 2 -- tests/test_transform.py | 7 +------ 6 files changed, 6 insertions(+), 18 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index c05f05d..bdd10a3 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -592,7 +592,7 @@ Every option for this table (with the exception of ``--pk-none``) can be specifi ``--default-none column`` Remove the default value for this column. -``--drop-foreign-key col other_table other_column`` +``--drop-foreign-key column`` Drop the specified foreign key. If you want to see the SQL that will be executed to make the change without actually executing it, add the ``--sql`` flag. For example:: diff --git a/docs/python-api.rst b/docs/python-api.rst index b2452d7..b0aabf9 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -995,17 +995,14 @@ The ``column_order=`` parameter can be used to change the order of the columns. # Change column order table.transform(column_order=("name", "age", "id") -You can use ``.transform()`` to remove foreign key constraints from a table. You will need to know the name of the column, the name of the table it points to and the name of the column it references on that other table. +You can use ``.transform()`` to remove foreign key constraints from a table. This example drops two foreign keys - the one from ``places.country`` to ``country.id`` and the one from ``places.continent`` to ``continent.id``: .. code-block:: python db["places"].transform( - drop_foreign_keys=( - ("country", "country", "id"), - ("continent", "continent", "id"), - ) + drop_foreign_keys=("country", "continent") ) .. _python_api_transform_sql: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 535d3d0..56dc0c8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -924,7 +924,7 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols) ) @click.option( "--drop-foreign-key", - type=(str, str, str), + type=str, multiple=True, help="Drop this foreign key constraint", ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f5868ed..178d48b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -835,9 +835,7 @@ class Table(Queryable): # foreign_keys create_table_foreign_keys = [] for table, column, other_table, other_column in self.foreign_keys: - if (drop_foreign_keys is None) or ( - (column, other_table, other_column) not in drop_foreign_keys - ): + if (drop_foreign_keys is None) or (column not in drop_foreign_keys): create_table_foreign_keys.append( (rename.get(column) or column, other_table, other_column) ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 5aad909..c50f87f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1542,8 +1542,6 @@ def test_transform_drop_foreign_key(db_path): "places", "--drop-foreign-key", "country", - "country", - "id", ], ) print(result.output) diff --git a/tests/test_transform.py b/tests/test_transform.py index a96f9ba..b3ef009 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -274,12 +274,7 @@ def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys): ), ] # Drop two of those foreign keys - fresh_db["places"].transform( - drop_foreign_keys=( - ("country", "country", "id"), - ("continent", "continent", "id"), - ) - ) + fresh_db["places"].transform(drop_foreign_keys=("country", "continent")) # Should be only one foreign key now assert fresh_db["places"].foreign_keys == [ ForeignKey(table="places", column="city", other_table="city", other_column="id") From a57acf84f77aac01fd6b8aaa2ce089145ff3c3e7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Sep 2020 09:44:30 -0700 Subject: [PATCH 0225/1004] Release 2.21 Refs #172, #175, #176, #177 --- docs/changelog.rst | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 85378ee..bac5431 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,18 @@ Changelog =========== +.. _v2_21: + +2.21 (2020-09-24) +----------------- + +- ``table.extract()`` and ``sqlite-utils extract`` now apply much, much faster - one example operation reduced from twelve minutes to just four seconds! (`#172 `__) +- ``sqlite-utils extract`` no longer shows a progress bar, because it's fast enough not to need one. +- New ``column_order=`` option for ``table.transform()`` which can be used to alter the order of columns in a table. (`#175 `__) +- ``sqlite-utils transform --column-order=`` option (with a ``-o`` shortcut) for changing column order. (`#176 `__) +- The ``table.transform(drop_foreign_keys=)`` parameter and the ``sqlite-utils transform --drop-foreign-key`` option have changed. They now accept just the name of the column rather than requiring all three of the column, other table and other column. This is technically a backwards-incompatible change but I chose not to bump the major version number because the transform feature is so new. (`#177 `__) +- The table ``.disable_fts()``, ``.rebuild_fts()``, ``.delete()``, ``.delete_where()`` and ``.add_missing_columns()`` methods all now ``return self``, which means they can be chained together with other table operations. + .. _v2_20: 2.20 (2020-09-22) diff --git a/setup.py b/setup.py index 7e3d792..9ed6491 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.20" +VERSION = "2.21" def get_long_description(): From cda559f8353ea65d7db031fa57ea25b515b5fa24 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Sep 2020 13:33:19 -0700 Subject: [PATCH 0226/1004] Include --column-order in combined example, refs #176 --- docs/cli.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index bdd10a3..f3854ba 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -600,16 +600,19 @@ If you want to see the SQL that will be executed to make the change without actu % sqlite-utils transform fixtures.db roadside_attractions \ --rename pk id \ --default name Untitled \ + --column-order id \ + --column-order longitude \ + --column-order latitude \ --drop address \ --sql CREATE TABLE [roadside_attractions_new_4033a60276b9] ( [id] INTEGER PRIMARY KEY, - [name] TEXT DEFAULT 'Untitled', + [longitude] FLOAT, [latitude] FLOAT, - [longitude] FLOAT + [name] TEXT DEFAULT 'Untitled' ); - INSERT INTO [roadside_attractions_new_4033a60276b9] ([id], [name], [latitude], [longitude]) - SELECT [pk], [name], [latitude], [longitude] FROM [roadside_attractions]; + INSERT INTO [roadside_attractions_new_4033a60276b9] ([longitude], [latitude], [id], [name]) + SELECT [longitude], [latitude], [pk], [name] FROM [roadside_attractions]; DROP TABLE [roadside_attractions]; ALTER TABLE [roadside_attractions_new_4033a60276b9] RENAME TO [roadside_attractions]; From 94fc62857ee2655a21d85f6dae84b67bbfa5956d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Sep 2020 15:46:46 -0700 Subject: [PATCH 0227/1004] Demonstrate extract= creates correct foreign keys Closes #138 --- tests/test_extracts.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_extracts.py b/tests/test_extracts.py index c46b0a5..0edd002 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import Index +from sqlite_utils.db import Index, ForeignKey import pytest @@ -41,6 +41,12 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): ) == fresh_db["Trees"].schema ) + # Should have a foreign key reference + assert len(fresh_db["Trees"].foreign_keys) == 1 + fk = fresh_db["Trees"].foreign_keys[0] + assert fk.table == "Trees" + assert fk.column == "species_id" + # Should have unique index on Species assert [ Index( From cada1017edcfa691c2314d7ad1b7c7576495317f Mon Sep 17 00:00:00 2001 From: Shakeel Mahate Date: Wed, 30 Sep 2020 16:29:27 -0400 Subject: [PATCH 0228/1004] Fixed incorrect example in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index acd5356..5c1b23b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Now you can do things with the CLI utility like this: You can even import data into a new database table like this: $ curl https://api.github.com/repos/simonw/sqlite-utils/releases \ - | sqlite-utils insert releases.db releases - --pk + | sqlite-utils insert releases.db releases - --pk id Full CLI documentation: https://sqlite-utils.readthedocs.io/en/stable/cli.html From 7f4fe9190c1df7f0e72f9d7040327a43cf252c48 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Sep 2020 15:17:23 -0700 Subject: [PATCH 0229/1004] Configure code scanning, refs #183 --- .github/workflows/codeql-analysis.yml | 66 +++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..d86c2e3 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,66 @@ +name: "CodeQL" + +on: + push: + branches: [main] + pull_request: + # The branches below must be a subset of the branches above + branches: [main] + schedule: + - cron: '0 4 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['python'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 8e91de8e4edf6c8abeeccdf084b8870fff40a51a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Oct 2020 18:44:05 -0700 Subject: [PATCH 0230/1004] Python 3.9 (#184) * Test against Python 3.9 * Programming Language :: Python :: 3.9 classifier * Python versions badge --- .github/workflows/test.yml | 2 +- README.md | 1 + setup.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 350b07d..4f2b030 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] numpy: [0, 1] steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index 5c1b23b..2e4946b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/) [![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.readthedocs.io/en/stable/changelog.html) +[![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/) [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest) [![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.readthedocs.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE) diff --git a/setup.py b/setup.py index 9ed6491..371895a 100644 --- a/setup.py +++ b/setup.py @@ -51,5 +51,6 @@ setup( "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], ) From 4e8e157b5df4ccddbcaaec847807d730437cffb5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Oct 2020 18:45:07 -0700 Subject: [PATCH 0231/1004] Test against Python 3.9 on publish --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0a55018..3755c3a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -37,7 +37,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.9' - uses: actions/cache@v2 name: Configure pip caching with: From 7eda0532e800bb54e2a304632ce510a024a4ee60 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 11 Oct 2020 17:13:24 -0700 Subject: [PATCH 0232/1004] Consistent usage of db["dogs"], closes #185 --- docs/python-api.rst | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index b0aabf9..259336d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -274,7 +274,7 @@ If you want to explicitly set the order of the columns you can do so using the ` .. code-block:: python - dogs.insert({ + db["dogs"].insert({ "id": 1, "name": "Cleo", "twitter": "cleopaws", @@ -288,7 +288,7 @@ Column types are detected based on the example data provided. Sometimes you may .. code-block:: python - dogs.insert({ + db["dogs"].insert({ "id": 1, "name": "Cleo", "age": "5", @@ -457,7 +457,7 @@ Use it like this: .. code-block:: python - dogs.insert_all([{ + db["dogs"].insert_all([{ "id": 1, "name": "Cleo", "twitter": "cleopaws", @@ -494,7 +494,7 @@ Insert-replacing data If you want to insert a record or replace an existing record with the same primary key, using the ``replace=True`` argument to ``.insert()`` or ``.insert_all()``:: - dogs.insert_all([{ + db["dogs"].insert_all([{ "id": 1, "name": "Cleo", "twitter": "cleopaws", @@ -573,7 +573,7 @@ For example, given the dogs database you could upsert the record for Cleo like s .. code-block:: python - dogs.upsert([{ + db["dogs"].upsert([{ "id": 1, "name": "Cleo", "twitter": "cleopaws", @@ -1437,38 +1437,38 @@ You can enable full-text search on a table using ``.enable_fts(columns)``: .. code-block:: python - dogs.enable_fts(["name", "twitter"]) + db["dogs"].enable_fts(["name", "twitter"]) You can then run searches using the ``.search()`` method: .. code-block:: python - rows = dogs.search("cleo") + rows = db["dogs"].search("cleo") If you insert additional records into the table you will need to refresh the search index using ``populate_fts()``: .. code-block:: python - dogs.insert({ + db["dogs"].insert({ "id": 2, "name": "Marnie", "twitter": "MarnieTheDog", "age": 16, "is_good_dog": True, }, pk="id") - dogs.populate_fts(["name", "twitter"]) + db["dogs"].populate_fts(["name", "twitter"]) A better solution is to use database triggers. You can set up database triggers to automatically update the full-text index using ``create_triggers=True``: .. code-block:: python - dogs.enable_fts(["name", "twitter"], create_triggers=True) + db["dogs"].enable_fts(["name", "twitter"], create_triggers=True) ``.enable_fts()`` defaults to using `FTS5 `__. If you wish to use `FTS4 `__ instead, use the following: .. code-block:: python - dogs.enable_fts(["name", "twitter"], fts_version="FTS4") + db["dogs"].enable_fts(["name", "twitter"], fts_version="FTS4") You can customize the tokenizer configured for the table using the ``tokenize=`` parameter. For example, to enable Porter stemming, where English words like "running" will match stemmed alternatives such as "run", use ``tokenize="porter"``: @@ -1492,7 +1492,7 @@ To remove the FTS tables and triggers you created, use the ``disable_fts()`` tab .. code-block:: python - dogs.disable_fts() + db["dogs"].disable_fts() .. _python_api_fts_rebuild: @@ -1503,7 +1503,7 @@ You can rebuild a table using the ``table.rebuild_fts()`` method. This is useful .. code-block:: python - dogs.rebuild_fts() + db["dogs"].rebuild_fts() This method can be called on a table that has been configured for full-text search - ``dogs`` in this instance - or directly on a ``_fts`` table: @@ -1524,7 +1524,7 @@ Once you have populated a FTS table you can optimize it to dramatically reduce i .. code-block:: python - dogs.optimize() + db["dogs"].optimize() This runs the following SQL:: @@ -1537,13 +1537,13 @@ You can create an index on a table using the ``.create_index(columns)`` method. .. code-block:: python - dogs.create_index(["is_good_dog"]) + db["dogs"].create_index(["is_good_dog"]) By default the index will be named ``idx_{table-name}_{columns}`` - if you want to customize the name of the created index you can pass the ``index_name`` parameter: .. code-block:: python - dogs.create_index( + db["dogs"].create_index( ["is_good_dog", "age"], index_name="good_dogs_by_age" ) @@ -1552,7 +1552,7 @@ You can create a unique index by passing ``unique=True``: .. code-block:: python - dogs.create_index(["name"], unique=True) + db["dogs"].create_index(["name"], unique=True) Use ``if_not_exists=True`` to do nothing if an index with that name already exists. From 7c0ef116edd78f8970be32523d230340675db5bd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 14 Oct 2020 14:59:38 -0700 Subject: [PATCH 0233/1004] pk=['id'] now equivalent to pk='id', closes #181 --- sqlite_utils/db.py | 2 ++ tests/test_create.py | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 178d48b..d517bdc 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -330,6 +330,8 @@ class Database: column_defs = [] # ensure pk is a tuple single_pk = None + if isinstance(pk, list) and len(pk) == 1 and isinstance(pk[0], str): + pk = pk[0] if isinstance(pk, str): single_pk = pk if pk not in [c[0] for c in column_items]: diff --git a/tests/test_create.py b/tests/test_create.py index 9e30d2b..83936e4 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -73,11 +73,12 @@ def test_create_table_compound_primary_key(fresh_db): assert ["id1", "id2"] == table.pks -def test_create_table_with_bad_defaults(fresh_db): - with pytest.raises(AssertionError): - fresh_db.create_table( - "players", {"name": str, "score": int}, defaults={"mouse": 1} - ) +@pytest.mark.parametrize("pk", ("id", ["id"])) +def test_create_table_with_single_primary_key(fresh_db, pk): + fresh_db["foo"].insert({"id": 1}, pk=pk) + assert ( + fresh_db["foo"].schema == "CREATE TABLE [foo] (\n [id] INTEGER PRIMARY KEY\n)" + ) def test_create_table_with_invalid_column_characters(fresh_db): From 2c541fac352632e23e40b0d21e3f233f7a744a57 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Oct 2020 10:18:46 -0700 Subject: [PATCH 0234/1004] --encoding option for non-utf8 CSV/TSV, closes #182 --- docs/cli.rst | 4 ++ sqlite_utils/cli.py | 108 ++++++++++++++++++++++++++++++-------------- tests/test_cli.py | 61 +++++++++++++++++++++++-- 3 files changed, 137 insertions(+), 36 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index f3854ba..2d439f6 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -354,6 +354,10 @@ For tab-delimited data, use ``--tsv``:: $ sqlite-utils insert dogs.db dogs docs.tsv --tsv +Data is expected to be encoded as Unicode UTF-8. If your data is an another character encoding you can specify it using the ``--encoding`` option:: + + $ sqlite-utils insert dogs.db dogs docs.tsv --tsv --encoding=latin-1 + .. _cli_insert_replace: Insert-replacing data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 56dc0c8..cfa7303 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,5 +1,6 @@ import base64 import click +import codecs from click_default_group import DefaultGroup from datetime import datetime import hashlib @@ -15,6 +16,18 @@ from .utils import sqlite3, decode_base64_values VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB") +UNICODE_ERROR = """ +{} + +The input you provided uses a character encoding other than utf-8. + +You can fix this by passing the --encoding= option with the encoding of the file. + +If you do not know the encoding, running 'file filename.csv' may tell you. + +It's often worth trying: --encoding=latin-1 +""".strip() + def output_options(fn): for decorator in reversed( @@ -493,7 +506,7 @@ def insert_upsert_options(fn): required=True, ), click.argument("table"), - click.argument("json_file", type=click.File(), required=True), + click.argument("json_file", type=click.File("rb"), required=True), click.option( "--pk", help="Columns to use as the primary key, e.g. id", multiple=True ), @@ -519,6 +532,10 @@ def insert_upsert_options(fn): type=(str, str), help="Default value that should be set for a column", ), + click.option( + "--encoding", + help="Character encoding for input, defaults to utf-8", + ), ) ): fn = decorator(fn) @@ -541,10 +558,15 @@ def insert_upsert_implementation( truncate=False, not_null=None, default=None, + encoding=None, ): db = sqlite_utils.Database(path) if (nl + csv + tsv) >= 2: raise click.ClickException("Use just one of --nl, --csv or --tsv") + if encoding and not (csv or tsv): + raise click.ClickException("--encoding must be used with --csv or --tsv") + encoding = encoding or "utf-8" + json_file = codecs.getreader(encoding)(json_file) if pk and len(pk) == 1: pk = pk[0] if csv or tsv: @@ -599,6 +621,7 @@ def insert( tsv, batch_size, alter, + encoding, ignore, replace, truncate, @@ -611,49 +634,68 @@ def insert( Input should be a JSON array of objects, unless --nl or --csv is used. """ - insert_upsert_implementation( - path, - table, - json_file, - pk, - nl, - csv, - tsv, - batch_size, - alter=alter, - upsert=False, - ignore=ignore, - replace=replace, - truncate=truncate, - not_null=not_null, - default=default, - ) + try: + insert_upsert_implementation( + path, + table, + json_file, + pk, + nl, + csv, + tsv, + batch_size, + alter=alter, + upsert=False, + ignore=ignore, + replace=replace, + truncate=truncate, + encoding=encoding, + not_null=not_null, + default=default, + ) + except UnicodeDecodeError as ex: + raise click.ClickException(UNICODE_ERROR.format(ex)) @cli.command() @insert_upsert_options def upsert( - path, table, json_file, pk, nl, csv, tsv, batch_size, alter, not_null, default + path, + table, + json_file, + pk, + nl, + csv, + tsv, + batch_size, + alter, + not_null, + default, + encoding, ): """ Upsert records based on their primary key. Works like 'insert' but if an incoming record has a primary key that matches an existing record the existing record will be updated. """ - insert_upsert_implementation( - path, - table, - json_file, - pk, - nl, - csv, - tsv, - batch_size, - alter=alter, - upsert=True, - not_null=not_null, - default=default, - ) + try: + insert_upsert_implementation( + path, + table, + json_file, + pk, + nl, + csv, + tsv, + batch_size, + alter=alter, + upsert=True, + not_null=not_null, + default=default, + encoding=encoding, + ) + except UnicodeDecodeError as ex: + raise click.ClickException(UNICODE_ERROR.format(ex)) @cli.command(name="create-table") diff --git a/tests/test_cli.py b/tests/test_cli.py index c50f87f..d70ff6f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -685,7 +685,9 @@ def test_insert_csv_tsv(content, option, db_path, tmpdir): db = Database(db_path) file_path = str(tmpdir / "insert.csv-tsv") open(file_path, "w").write(content) - result = CliRunner().invoke(cli.cli, ["insert", db_path, "data", file_path, option]) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "data", file_path, option], catch_exceptions=False + ) assert 0 == result.exit_code assert [{"foo": "1", "bar": "2", "baz": "3"}] == list(db["data"].rows) @@ -1024,7 +1026,9 @@ def test_upsert(db_path, tmpdir): ] open(json_path, "w").write(json.dumps(insert_dogs)) result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] + cli.cli, + ["insert", db_path, "dogs", json_path, "--pk", "id"], + catch_exceptions=False, ) assert 0 == result.exit_code, result.output assert 2 == db["dogs"].count @@ -1035,7 +1039,9 @@ def test_upsert(db_path, tmpdir): ] open(json_path, "w").write(json.dumps(insert_dogs)) result = CliRunner().invoke( - cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"] + cli.cli, + ["upsert", db_path, "dogs", json_path, "--pk", "id"], + catch_exceptions=False, ) assert 0 == result.exit_code, result.output assert [ @@ -1601,3 +1607,52 @@ def test_extract(db_path, args, expected_table_schema, expected_other_schema): 0 ].schema assert other_schema == expected_other_schema + + +def test_insert_encoding(tmpdir): + db_path = str(tmpdir / "test.db") + latin1_csv = ( + b"date,name,latitude,longitude\n" + b"2020-01-01,Barra da Lagoa,-27.574,-48.422\n" + b"2020-03-04,S\xe3o Paulo,-23.561,-46.645\n" + b"2020-04-05,Salta,-24.793:-65.408" + ) + assert latin1_csv.decode("latin-1").split("\n")[2].split(",")[1] == "São Paulo" + csv_path = str(tmpdir / "test.csv") + open(csv_path, "wb").write(latin1_csv) + # First attempt should error: + bad_result = CliRunner().invoke( + cli.cli, ["insert", db_path, "places", csv_path, "--csv"] + ) + assert bad_result.exit_code == 1 + assert ( + "The input you provided uses a character encoding other than utf-8" + in bad_result.output + ) + # Using --encoding=latin-1 should work + good_result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "places", csv_path, "--encoding", "latin-1", "--csv"], + ) + assert good_result.exit_code == 0 + db = Database(db_path) + assert list(db["places"].rows) == [ + { + "date": "2020-01-01", + "name": "Barra da Lagoa", + "latitude": "-27.574", + "longitude": "-48.422", + }, + { + "date": "2020-03-04", + "name": "São Paulo", + "latitude": "-23.561", + "longitude": "-46.645", + }, + { + "date": "2020-04-05", + "name": "Salta", + "latitude": "-24.793:-65.408", + "longitude": None, + }, + ] From 21ff60e3b119af48bb9b8c9635bc701576dd1b1d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Oct 2020 12:14:22 -0700 Subject: [PATCH 0235/1004] --load-extension= for many more commands, closes #137 Also added --load-extension=spatialite shortcut, closes #136 --- docs/cli.rst | 17 +++++ docs/python-api.rst | 2 + sqlite_utils/cli.py | 150 +++++++++++++++++++++++++++++++++++--------- sqlite_utils/db.py | 2 + tests/test_cli.py | 9 ++- 5 files changed, 148 insertions(+), 32 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 2d439f6..40d0f77 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -6,6 +6,8 @@ The ``sqlite-utils`` command-line tool can be used to manipulate SQLite databases in a number of different ways. +.. contents:: :local: + .. _cli_query_json: Running queries and returning JSON @@ -910,6 +912,7 @@ If you just want to run OPTIMIZE without the VACUUM, use the ``--no-vacuum`` fla To optimize specific tables rather than every FTS table, pass those tables as extra arguments: :: + $ sqlite-utils optimize mydb.db table_1 table_2 .. _cli_wal: @@ -926,3 +929,17 @@ You can disable WAL mode using ``disable-wal``:: $ sqlite-utils disable-wal mydb.db Both of these commands accept one or more database files as arguments. + +.. _cli_load_extension: + +Loading SQLite extensions +========================= + +Many of these commands have the ablity to load additional SQLite extensions using the ``--load-extension=/path/to/extension`` option - use ``--help`` to check for support, e.g. ``sqlite-utils rows --help``. + +This option can be applied multiple times to load multiple extensions. + +Since `SpatiaLite `__ is commonly used with SQLite, the value ``spatialite`` is special: it will search for SpatiaLite in the most common installation locations, saving you from needing to remember exactly where that module is located:: + + $ sqlite-utils :memory: "select spatialite_version()" --load-extension=spatialite + [{"spatialite_version()": "4.3.0a"}] diff --git a/docs/python-api.rst b/docs/python-api.rst index 259336d..cb7461b 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -4,6 +4,8 @@ Python API ============ +.. contents:: :local: + Connecting to or creating a database ==================================== diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cfa7303..0e56c9f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -9,10 +9,11 @@ import sqlite_utils from sqlite_utils.db import AlterError import itertools import json +import os import sys import csv as csv_std import tabulate -from .utils import sqlite3, decode_base64_values +from .utils import find_spatialite, sqlite3, decode_base64_values VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB") @@ -67,6 +68,14 @@ def output_options(fn): return fn +def load_extension_option(fn): + return click.option( + "--load-extension", + multiple=True, + help="SQLite extensions to load", + )(fn) + + @click.group(cls=DefaultGroup, default="query", default_if_no_args=True) @click.version_option() def cli(): @@ -102,6 +111,7 @@ def cli(): is_flag=True, default=False, ) +@load_extension_option def tables( path, fts4, @@ -116,10 +126,12 @@ def tables( json_cols, columns, schema, + load_extension, views=False, ): """List the tables in the database""" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) headers = ["view" if views else "table"] if counts: headers.append("count") @@ -182,6 +194,7 @@ def tables( is_flag=True, default=False, ) +@load_extension_option def views( path, counts, @@ -194,6 +207,7 @@ def views( json_cols, columns, schema, + load_extension, ): """List the views in the database""" tables.callback( @@ -210,6 +224,7 @@ def views( json_cols=json_cols, columns=columns, schema=schema, + load_extension=load_extension, views=True, ) @@ -233,9 +248,11 @@ def vacuum(path): ) @click.argument("tables", nargs=-1) @click.option("--no-vacuum", help="Don't run VACUUM", default=False, is_flag=True) -def optimize(path, tables, no_vacuum): +@load_extension_option +def optimize(path, tables, no_vacuum, load_extension): """Optimize all FTS tables and then run VACUUM - should shrink the database file""" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) if not tables: tables = db.table_names(fts4=True) + db.table_names(fts5=True) with db.conn: @@ -252,9 +269,11 @@ def optimize(path, tables, no_vacuum): required=True, ) @click.argument("tables", nargs=-1) -def rebuild_fts(path, tables): +@load_extension_option +def rebuild_fts(path, tables, load_extension): """Rebuild specific FTS tables, or all FTS tables if none are specified""" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) if not tables: tables = db.table_names(fts4=True) + db.table_names(fts5=True) with db.conn: @@ -303,9 +322,13 @@ def vacuum(path): required=False, help="Add NOT NULL DEFAULT 'TEXT' constraint", ) -def add_column(path, table, col_name, col_type, fk, fk_col, not_null_default): +@load_extension_option +def add_column( + path, table, col_name, col_type, fk, fk_col, not_null_default, load_extension +): "Add a column to the specified table" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) db[table].add_column( col_name, col_type, fk=fk, fk_col=fk_col, not_null_default=not_null_default ) @@ -326,7 +349,10 @@ def add_column(path, table, col_name, col_type, fk, fk_col, not_null_default): is_flag=True, help="If foreign key already exists, do nothing", ) -def add_foreign_key(path, table, column, other_table, other_column, ignore): +@load_extension_option +def add_foreign_key( + path, table, column, other_table, other_column, ignore, load_extension +): """ Add a new foreign key constraint to an existing table. Example usage: @@ -335,6 +361,7 @@ def add_foreign_key(path, table, column, other_table, other_column, ignore): WARNING: Could corrupt your database! Back up your database file first. """ db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) try: db[table].add_foreign_key(column, other_table, other_column, ignore=ignore) except AlterError as e: @@ -348,7 +375,8 @@ def add_foreign_key(path, table, column, other_table, other_column, ignore): required=True, ) @click.argument("foreign_key", nargs=-1) -def add_foreign_keys(path, foreign_key): +@load_extension_option +def add_foreign_keys(path, foreign_key, load_extension): """ Add multiple new foreign key constraints to a database. Example usage: @@ -358,6 +386,7 @@ def add_foreign_keys(path, foreign_key): authors country_id countries id """ db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) if len(foreign_key) % 4 != 0: raise click.ClickException( "Each foreign key requires four values: table, column, other_table, other_column" @@ -377,11 +406,13 @@ def add_foreign_keys(path, foreign_key): type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), required=True, ) -def index_foreign_keys(path): +@load_extension_option +def index_foreign_keys(path, load_extension): """ Ensure every foreign key column has an index on it. """ db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) db.index_foreign_keys() @@ -401,9 +432,11 @@ def index_foreign_keys(path): default=False, is_flag=True, ) -def create_index(path, table, column, name, unique, if_not_exists): +@load_extension_option +def create_index(path, table, column, name, unique, if_not_exists, load_extension): "Add an index to the specified table covering the specified columns" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) db[table].create_index( column, index_name=name, unique=unique, if_not_exists=if_not_exists ) @@ -426,7 +459,10 @@ def create_index(path, table, column, name, unique, if_not_exists): default=False, is_flag=True, ) -def enable_fts(path, table, column, fts4, fts5, tokenize, create_triggers): +@load_extension_option +def enable_fts( + path, table, column, fts4, fts5, tokenize, create_triggers, load_extension +): "Enable FTS for specific table and columns" fts_version = "FTS5" if fts4 and fts5: @@ -436,6 +472,7 @@ def enable_fts(path, table, column, fts4, fts5, tokenize, create_triggers): fts_version = "FTS4" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) db[table].enable_fts( column, fts_version=fts_version, @@ -452,9 +489,11 @@ def enable_fts(path, table, column, fts4, fts5, tokenize, create_triggers): ) @click.argument("table") @click.argument("column", nargs=-1, required=True) -def populate_fts(path, table, column): +@load_extension_option +def populate_fts(path, table, column, load_extension): "Re-populate FTS for specific table and columns" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) db[table].populate_fts(column) @@ -465,9 +504,11 @@ def populate_fts(path, table, column): required=True, ) @click.argument("table") -def disable_fts(path, table): +@load_extension_option +def disable_fts(path, table, load_extension): "Disable FTS for specific table" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) db[table].disable_fts() @@ -478,10 +519,13 @@ def disable_fts(path, table): type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), required=True, ) -def enable_wal(path): +@load_extension_option +def enable_wal(path, load_extension): "Enable WAL for database files" for path_ in path: - sqlite_utils.Database(path_).enable_wal() + db = sqlite_utils.Database(path_) + _load_extensions(db, load_extension) + db.enable_wal() @cli.command(name="disable-wal") @@ -491,10 +535,13 @@ def enable_wal(path): type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), required=True, ) -def disable_wal(path): +@load_extension_option +def disable_wal(path, load_extension): "Disable WAL for database files" for path_ in path: - sqlite_utils.Database(path_).disable_wal() + db = sqlite_utils.Database(path_) + _load_extensions(db, load_extension) + db.disable_wal() def insert_upsert_options(fn): @@ -536,6 +583,7 @@ def insert_upsert_options(fn): "--encoding", help="Character encoding for input, defaults to utf-8", ), + load_extension_option, ) ): fn = decorator(fn) @@ -559,8 +607,10 @@ def insert_upsert_implementation( not_null=None, default=None, encoding=None, + load_extension=None, ): db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) if (nl + csv + tsv) >= 2: raise click.ClickException("Use just one of --nl, --csv or --tsv") if encoding and not (csv or tsv): @@ -622,6 +672,7 @@ def insert( batch_size, alter, encoding, + load_extension, ignore, replace, truncate, @@ -650,6 +701,7 @@ def insert( replace=replace, truncate=truncate, encoding=encoding, + load_extension=load_extension, not_null=not_null, default=default, ) @@ -672,6 +724,7 @@ def upsert( not_null, default, encoding, + load_extension, ): """ Upsert records based on their primary key. Works like 'insert' but if @@ -693,6 +746,7 @@ def upsert( not_null=not_null, default=default, encoding=encoding, + load_extension=load_extension, ) except UnicodeDecodeError as ex: raise click.ClickException(UNICODE_ERROR.format(ex)) @@ -734,9 +788,13 @@ def upsert( is_flag=True, help="If table already exists, replace it", ) -def create_table(path, table, columns, pk, not_null, default, fk, ignore, replace): +@load_extension_option +def create_table( + path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension +): "Add an index to the specified table covering the specified columns" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) if len(columns) % 2 == 1: raise click.ClickException( "columns must be an even number of 'name' 'type' pairs" @@ -775,9 +833,11 @@ def create_table(path, table, columns, pk, not_null, default, fk, ignore, replac required=True, ) @click.argument("table") -def drop_table(path, table): +@load_extension_option +def drop_table(path, table, load_extension): "Drop the specified table" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) if table in db.table_names(): db[table].drop() else: @@ -802,9 +862,11 @@ def drop_table(path, table): is_flag=True, help="If view already exists, replace it", ) -def create_view(path, view, select, ignore, replace): +@load_extension_option +def create_view(path, view, select, ignore, replace, load_extension): "Create a view for the provided SELECT query" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) # Does view already exist? if view in db.view_names(): if ignore: @@ -827,9 +889,11 @@ def create_view(path, view, select, ignore, replace): required=True, ) @click.argument("view") -def drop_view(path, view): +@load_extension_option +def drop_view(path, view, load_extension): "Drop the specified view" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) if view in db.view_names(): db[view].drop() else: @@ -852,11 +916,7 @@ def drop_view(path, view): type=(str, str), help="Named :parameters for SQL query", ) -@click.option( - "--load-extension", - multiple=True, - help="SQLite extensions to load", -) +@load_extension_option def query( path, sql, @@ -873,10 +933,7 @@ def query( ): "Execute SQL query and return the results as JSON" db = sqlite_utils.Database(path) - if load_extension: - db.conn.enable_load_extension(True) - for ext in load_extension: - db.conn.load_extension(ext) + _load_extensions(db, load_extension) with db.conn: cursor = db.execute(sql, dict(param)) if cursor.description is None: @@ -912,8 +969,21 @@ def query( ) @click.argument("dbtable") @output_options +@load_extension_option @click.pass_context -def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols): +def rows( + ctx, + path, + dbtable, + nl, + arrays, + csv, + no_headers, + table, + fmt, + json_cols, + load_extension, +): "Output all rows in the specified table" ctx.invoke( query, @@ -926,6 +996,7 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols) table=table, fmt=fmt, json_cols=json_cols, + load_extension=load_extension, ) @@ -971,6 +1042,7 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols) help="Drop this foreign key constraint", ) @click.option("--sql", is_flag=True, help="Output SQL without executing it") +@load_extension_option def transform( path, table, @@ -986,9 +1058,11 @@ def transform( default_none, drop_foreign_key, sql, + load_extension, ): "Transform a table beyond the capabilities of ALTER TABLE" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) types = {} kwargs = {} for column, ctype in type: @@ -1051,6 +1125,7 @@ def transform( multiple=True, help="Rename this column in extracted table", ) +@load_extension_option def extract( path, table, @@ -1058,9 +1133,11 @@ def extract( other_table, fk_column, rename, + load_extension, ): "Extract one or more columns into a separate table" db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) kwargs = dict( columns=columns, table=other_table, @@ -1095,7 +1172,10 @@ def extract( @click.option("--replace", is_flag=True, help="Replace files with matching primary key") @click.option("--upsert", is_flag=True, help="Upsert files with matching primary key") @click.option("--name", type=str, help="File name to use") -def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert, name): +@load_extension_option +def insert_files( + path, table, file_or_dir, column, pk, alter, replace, upsert, name, load_extension +): """ Insert one or more files using BLOB columns in the specified table @@ -1168,6 +1248,7 @@ def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert, n yield row db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) with db.conn: db[table].insert_all( to_insert(), pk=pk, alter=alter, replace=replace, upsert=upsert @@ -1234,3 +1315,12 @@ def json_binary(value): return {"$base64": True, "encoded": base64.b64encode(value).decode("latin-1")} else: raise TypeError + + +def _load_extensions(db, load_extension): + if load_extension: + db.conn.enable_load_extension(True) + for ext in load_extension: + if ext == "spatialite" and not os.path.exists(ext): + ext = find_spatialite() + db.conn.load_extension(ext) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d517bdc..af5bb97 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -289,6 +289,8 @@ class Database: # any extracts will be treated as integer columns with a foreign key extracts = resolve_extracts(extracts) for extract_column, extract_table in extracts.items(): + if isinstance(extract_column, tuple): + assert False # Ensure other table exists if not self[extract_table].exists(): self.create_table(extract_table, {"id": int, "value": str}, pk="id") diff --git a/tests/test_cli.py b/tests/test_cli.py index d70ff6f..9bc112b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -956,18 +956,23 @@ def test_query_raw(db_path, content, is_binary): not hasattr(sqlite3.Connection, "enable_load_extension"), reason="sqlite3.Connection missing enable_load_extension", ) -def test_query_load_extension(): +@pytest.mark.parametrize("use_spatialite_shortcut", [True, False]) +def test_query_load_extension(use_spatialite_shortcut): # Without --load-extension: result = CliRunner().invoke(cli.cli, [":memory:", "select spatialite_version()"]) assert result.exit_code == 1 assert "no such function: spatialite_version" in repr(result) # With --load-extension: + if use_spatialite_shortcut: + load_extension = "spatialite" + else: + load_extension = find_spatialite() result = CliRunner().invoke( cli.cli, [ ":memory:", "select spatialite_version()", - "--load-extension={}".format(find_spatialite()), + "--load-extension={}".format(load_extension), ], ) assert result.exit_code == 0, result.stdout From 47af71f6038327a0aba5ab9bbb7de21ee21924f9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Oct 2020 12:30:25 -0700 Subject: [PATCH 0236/1004] Release 2.22 Refs #182 #137 #136 #184 #181 --- docs/changelog.rst | 11 +++++++++++ setup.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index bac5431..8f4abb4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,17 @@ Changelog =========== +.. _v2_22: + +2.22 (2020-10-16) +----------------- + +- New ``--encoding`` option for processing CSV and TSV files that use a non-utf-8 encoding, for both the ``insert`` and ``update`` commands. (`#182 `__) +- The ``--load-extension`` option is now available to many more commands. (`#137 `__) +- ``--load-extension=spatialite`` can be used to load SpatiaLite from common installation locations, if it is available. (`#136 `__) +- Tests now also run against Python 3.9. (`#184 `__) +- Passing ``pk=["id"]`` now has the same effect as passing ``pk="id"``. (`#181 `__) + .. _v2_21: 2.21 (2020-09-24) diff --git a/setup.py b/setup.py index 371895a..45fc517 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.21" +VERSION = "2.22" def get_long_description(): From 0b5edd646926d6e01e3bf9f2897d072f4302ce2d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 18 Oct 2020 21:51:50 -0700 Subject: [PATCH 0237/1004] Added basic tests using hypothesis, closes #180 --- setup.py | 2 +- tests/test_hypothesis.py | 43 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 tests/test_hypothesis.py diff --git a/setup.py b/setup.py index 45fc517..41b6f54 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setup( install_requires=["click", "click-default-group", "tabulate"], setup_requires=["pytest-runner"], extras_require={ - "test": ["pytest", "black"], + "test": ["pytest", "black", "hypothesis"], "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], }, entry_points=""" diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py new file mode 100644 index 0000000..5759c5e --- /dev/null +++ b/tests/test_hypothesis.py @@ -0,0 +1,43 @@ +from hypothesis import given +import hypothesis.strategies as st +import sqlite_utils + +# SQLite integers are -(2^63) to 2^63 - 1 +@given(st.integers(-9223372036854775808, 9223372036854775807)) +def test_roundtrip_integers(integer): + db = sqlite_utils.Database(memory=True) + row = { + "integer": integer, + } + db["test"].insert(row) + assert list(db["test"].rows) == [row] + + +@given(st.text()) +def test_roundtrip_text(text): + db = sqlite_utils.Database(memory=True) + row = { + "text": text, + } + db["test"].insert(row) + assert list(db["test"].rows) == [row] + + +@given(st.binary(max_size=1024 * 1024)) +def test_roundtrip_binary(binary): + db = sqlite_utils.Database(memory=True) + row = { + "binary": binary, + } + db["test"].insert(row) + assert list(db["test"].rows) == [row] + + +@given(st.floats(allow_nan=False)) +def test_roundtrip_floats(floats): + db = sqlite_utils.Database(memory=True) + row = { + "floats": floats, + } + db["test"].insert(row) + assert list(db["test"].rows) == [row] From 55133b596620392244530a09eb879bfe1b6e34b0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 21 Oct 2020 11:08:28 -0700 Subject: [PATCH 0238/1004] Link to sqliteutils tag on my blog --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2e4946b..f3a6e14 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Python CLI utility and library for manipulating SQLite databases. Read more on my blog: [ -sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) +sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) and other [entries tagged sqliteutils](https://simonwillison.net/tags/sqliteutils/). ## Installation From e4f1c7b936981de29823730c5dbef4f4ba7a4286 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 23 Oct 2020 14:19:30 -0700 Subject: [PATCH 0239/1004] python_requires=">=3.6" Inspired by https://github.com/simonw/datasette/pull/1044 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 41b6f54..405e6f3 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ setup( "Issues": "https://github.com/simonw/sqlite-utils/issues", "CI": "https://github.com/simonw/sqlite-utils/actions", }, + python_requires=">=3.6", classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", From 2771ab96e750ab946a74bda81a514c755c5b8a06 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 25 Oct 2020 20:05:56 -0700 Subject: [PATCH 0240/1004] Test showing stdin inserts work --- tests/test_cli.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9bc112b..062f319 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -554,6 +554,19 @@ def test_insert_simple(tmpdir): assert [] == db["dogs"].indexes +def test_insert_from_stdin(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", "-"], + input=json.dumps({"name": "Cleo", "age": 4}), + ) + assert 0 == result.exit_code + assert [{"age": 4, "name": "Cleo"}] == Database(db_path).execute_returning_dicts( + "select * from dogs" + ) + + def test_insert_with_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dog.json") open(json_path, "w").write(json.dumps({"id": 1, "name": "Cleo", "age": 4})) From f045d8559a6d2cb922a2de30fbcc896a4486b82f Mon Sep 17 00:00:00 2001 From: Adam Wolf Date: Tue, 27 Oct 2020 11:24:21 -0500 Subject: [PATCH 0241/1004] Allow iterables other than lists in m2m records (#189) * Allow iterables other than Lists in m2m records * Add test for iterable m2m records Thanks, @adamwolf! --- sqlite_utils/db.py | 14 +++++---- tests/test_m2m.py | 76 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 64 insertions(+), 26 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index af5bb97..0a16380 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,5 +1,6 @@ from .utils import sqlite3, OperationalError, suggest_column_types, column_affinity from collections import namedtuple, OrderedDict +from collections.abc import Mapping import contextlib import datetime import decimal @@ -1772,15 +1773,15 @@ class Table(Queryable): return pk def m2m( - self, other_table, record_or_list=None, pk=DEFAULT, lookup=None, m2m_table=None + self, other_table, record_or_iterable=None, pk=DEFAULT, lookup=None, m2m_table=None ): if isinstance(other_table, str): other_table = self.db.table(other_table, pk=pk) our_id = self.last_pk if lookup is not None: - assert record_or_list is None, "Provide lookup= or record, not both" + assert record_or_iterable is None, "Provide lookup= or record, not both" else: - assert record_or_list is not None, "Provide lookup= or record, not both" + assert record_or_iterable is not None, "Provide lookup= or record, not both" tables = list(sorted([self.name, other_table.name])) columns = ["{}_id".format(t) for t in tables] if m2m_table is not None: @@ -1801,10 +1802,11 @@ class Table(Queryable): m2m_table_name = m2m_table or "{}_{}".format(*tables) m2m_table = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) if lookup is None: + # if records is only one record, put the record in a list records = ( - [record_or_list] - if not isinstance(record_or_list, (list, tuple)) - else record_or_list + [record_or_iterable] + if isinstance(record_or_iterable, Mapping) + else record_or_iterable ) # Ensure each record exists in other table for record in records: diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 52b3959..33cd0a8 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -31,16 +31,52 @@ def test_insert_m2m_list(fresh_db): humans.rows ) assert [ - ForeignKey( - table="dogs_humans", column="dogs_id", other_table="dogs", other_column="id" - ), - ForeignKey( - table="dogs_humans", - column="humans_id", - other_table="humans", - other_column="id", - ), - ] == dogs_humans.foreign_keys + ForeignKey( + table="dogs_humans", column="dogs_id", other_table="dogs", other_column="id" + ), + ForeignKey( + table="dogs_humans", + column="humans_id", + other_table="humans", + other_column="id", + ), + ] == dogs_humans.foreign_keys + + +def test_insert_m2m_iterable(fresh_db): + iterable_records = ({"id": 1, "name": "Phineas"}, {"id": 2, "name": "Ferb"}) + + def iterable(): + for record in iterable_records: + yield record + + platypuses = fresh_db["platypuses"] + platypuses.insert({"id": 1, "name": "Perry"}, pk="id").m2m( + "humans", + iterable(), + pk="id", + ) + + assert {"platypuses", "humans", "humans_platypuses"} == set(fresh_db.table_names()) + humans = fresh_db["humans"] + humans_platypuses = fresh_db["humans_platypuses"] + assert [{"humans_id": 1, "platypuses_id": 1}, {"humans_id": 2, "platypuses_id": 1}] == list( + humans_platypuses.rows + ) + assert [{"id": 1, "name": "Phineas"}, {"id": 2, "name": "Ferb"}] == list( + humans.rows + ) + assert [ + ForeignKey( + table="humans_platypuses", column="platypuses_id", other_table="platypuses", other_column="id" + ), + ForeignKey( + table="humans_platypuses", + column="humans_id", + other_table="humans", + other_column="id", + ), + ] == humans_platypuses.foreign_keys def test_m2m_with_table_objects(fresh_db): @@ -64,16 +100,16 @@ def test_m2m_lookup(fresh_db): assert people_tags.exists() assert tags.exists() assert [ - ForeignKey( - table="people_tags", - column="people_id", - other_table="people", - other_column="id", - ), - ForeignKey( - table="people_tags", column="tags_id", other_table="tags", other_column="id" - ), - ] == people_tags.foreign_keys + ForeignKey( + table="people_tags", + column="people_id", + other_table="people", + other_column="id", + ), + ForeignKey( + table="people_tags", column="tags_id", other_table="tags", other_column="id" + ), + ] == people_tags.foreign_keys assert [{"people_id": 1, "tags_id": 1}] == list(people_tags.rows) assert [{"id": 1, "name": "Wahyu"}] == list(people.rows) assert [{"id": 1, "tag": "Coworker"}] == list(tags.rows) From c7e5dd64513c0ec2b2df4c51c8df924c282417f9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Oct 2020 09:26:01 -0700 Subject: [PATCH 0242/1004] Applied latest Black --- sqlite_utils/db.py | 7 ++++- tests/test_m2m.py | 70 ++++++++++++++++++++++++---------------------- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0a16380..877c17f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1773,7 +1773,12 @@ class Table(Queryable): return pk def m2m( - self, other_table, record_or_iterable=None, pk=DEFAULT, lookup=None, m2m_table=None + self, + other_table, + record_or_iterable=None, + pk=DEFAULT, + lookup=None, + m2m_table=None, ): if isinstance(other_table, str): other_table = self.db.table(other_table, pk=pk) diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 33cd0a8..cbc9015 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -31,16 +31,16 @@ def test_insert_m2m_list(fresh_db): humans.rows ) assert [ - ForeignKey( - table="dogs_humans", column="dogs_id", other_table="dogs", other_column="id" - ), - ForeignKey( - table="dogs_humans", - column="humans_id", - other_table="humans", - other_column="id", - ), - ] == dogs_humans.foreign_keys + ForeignKey( + table="dogs_humans", column="dogs_id", other_table="dogs", other_column="id" + ), + ForeignKey( + table="dogs_humans", + column="humans_id", + other_table="humans", + other_column="id", + ), + ] == dogs_humans.foreign_keys def test_insert_m2m_iterable(fresh_db): @@ -60,23 +60,27 @@ def test_insert_m2m_iterable(fresh_db): assert {"platypuses", "humans", "humans_platypuses"} == set(fresh_db.table_names()) humans = fresh_db["humans"] humans_platypuses = fresh_db["humans_platypuses"] - assert [{"humans_id": 1, "platypuses_id": 1}, {"humans_id": 2, "platypuses_id": 1}] == list( - humans_platypuses.rows - ) + assert [ + {"humans_id": 1, "platypuses_id": 1}, + {"humans_id": 2, "platypuses_id": 1}, + ] == list(humans_platypuses.rows) assert [{"id": 1, "name": "Phineas"}, {"id": 2, "name": "Ferb"}] == list( humans.rows ) assert [ - ForeignKey( - table="humans_platypuses", column="platypuses_id", other_table="platypuses", other_column="id" - ), - ForeignKey( - table="humans_platypuses", - column="humans_id", - other_table="humans", - other_column="id", - ), - ] == humans_platypuses.foreign_keys + ForeignKey( + table="humans_platypuses", + column="platypuses_id", + other_table="platypuses", + other_column="id", + ), + ForeignKey( + table="humans_platypuses", + column="humans_id", + other_table="humans", + other_column="id", + ), + ] == humans_platypuses.foreign_keys def test_m2m_with_table_objects(fresh_db): @@ -100,16 +104,16 @@ def test_m2m_lookup(fresh_db): assert people_tags.exists() assert tags.exists() assert [ - ForeignKey( - table="people_tags", - column="people_id", - other_table="people", - other_column="id", - ), - ForeignKey( - table="people_tags", column="tags_id", other_table="tags", other_column="id" - ), - ] == people_tags.foreign_keys + ForeignKey( + table="people_tags", + column="people_id", + other_table="people", + other_column="id", + ), + ForeignKey( + table="people_tags", column="tags_id", other_table="tags", other_column="id" + ), + ] == people_tags.foreign_keys assert [{"people_id": 1, "tags_id": 1}] == list(people_tags.rows) assert [{"id": 1, "name": "Wahyu"}] == list(people.rows) assert [{"id": 1, "tag": "Coworker"}] == list(tags.rows) From f99a23652910b03ac4669bbbb35a9b484451aabb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Oct 2020 11:16:02 -0700 Subject: [PATCH 0243/1004] Progress bar for sqlite-utils insert command, closes #173 --- docs/cli.rst | 2 ++ sqlite_utils/cli.py | 15 +++++++++++---- sqlite_utils/utils.py | 24 ++++++++++++++++++++++++ tests/test_cli.py | 5 ++++- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 40d0f77..54591cc 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -360,6 +360,8 @@ Data is expected to be encoded as Unicode UTF-8. If your data is an another char $ sqlite-utils insert dogs.db dogs docs.tsv --tsv --encoding=latin-1 +A progress bar is displayed when inserting data from a file. You can hide the progress bar using the ``--silent`` option. + .. _cli_insert_replace: Insert-replacing data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0e56c9f..fe42ec2 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -13,7 +13,7 @@ import os import sys import csv as csv_std import tabulate -from .utils import find_spatialite, sqlite3, decode_base64_values +from .utils import file_progress, find_spatialite, sqlite3, decode_base64_values VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB") @@ -584,6 +584,7 @@ def insert_upsert_options(fn): help="Character encoding for input, defaults to utf-8", ), load_extension_option, + click.option("--silent", is_flag=True, help="Do not show progress bar"), ) ): fn = decorator(fn) @@ -608,6 +609,7 @@ def insert_upsert_implementation( default=None, encoding=None, load_extension=None, + silent=False, ): db = sqlite_utils.Database(path) _load_extensions(db, load_extension) @@ -621,9 +623,10 @@ def insert_upsert_implementation( pk = pk[0] if csv or tsv: dialect = "excel-tab" if tsv else "excel" - reader = csv_std.reader(json_file, dialect=dialect) - headers = next(reader) - docs = (dict(zip(headers, row)) for row in reader) + with file_progress(json_file, silent=silent) as json_file: + reader = csv_std.reader(json_file, dialect=dialect) + headers = next(reader) + docs = (dict(zip(headers, row)) for row in reader) elif nl: docs = (json.loads(line) for line in json_file) else: @@ -673,6 +676,7 @@ def insert( alter, encoding, load_extension, + silent, ignore, replace, truncate, @@ -702,6 +706,7 @@ def insert( truncate=truncate, encoding=encoding, load_extension=load_extension, + silent=silent, not_null=not_null, default=default, ) @@ -725,6 +730,7 @@ def upsert( default, encoding, load_extension, + silent, ): """ Upsert records based on their primary key. Works like 'insert' but if @@ -747,6 +753,7 @@ def upsert( default=default, encoding=encoding, load_extension=load_extension, + silent=silent, ) except UnicodeDecodeError as ex: raise click.ClickException(UNICODE_ERROR.format(ex)) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 89b9241..a158b2b 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -1,4 +1,7 @@ import base64 +import click +import contextlib +import io import os try: @@ -87,3 +90,24 @@ def find_spatialite(): if os.path.exists(path): return path return None + + +class UpdateWrapper: + def __init__(self, wrapped, update): + self._wrapped = wrapped + self._update = update + + def __iter__(self): + for line in self._wrapped: + self._update(len(line)) + yield line + + +@contextlib.contextmanager +def file_progress(file, silent=False, **kwargs): + if silent or file.raw.fileno() == 0: # 0 = stdin + yield file + else: + file_length = os.path.getsize(file.raw.name) + with click.progressbar(length=file_length, **kwargs) as bar: + yield UpdateWrapper(file, bar.update) diff --git a/tests/test_cli.py b/tests/test_cli.py index 062f319..7e1365d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1640,7 +1640,9 @@ def test_insert_encoding(tmpdir): open(csv_path, "wb").write(latin1_csv) # First attempt should error: bad_result = CliRunner().invoke( - cli.cli, ["insert", db_path, "places", csv_path, "--csv"] + cli.cli, + ["insert", db_path, "places", csv_path, "--csv"], + catch_exceptions=False, ) assert bad_result.exit_code == 1 assert ( @@ -1651,6 +1653,7 @@ def test_insert_encoding(tmpdir): good_result = CliRunner().invoke( cli.cli, ["insert", db_path, "places", csv_path, "--encoding", "latin-1", "--csv"], + catch_exceptions=False, ) assert good_result.exit_code == 0 db = Database(db_path) From 0789bad8f7581fd96dec5bde51a75e937dffb1e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 28 Oct 2020 14:24:03 -0700 Subject: [PATCH 0244/1004] @db.register_function(deterministic=True), closes #191 --- docs/python-api.rst | 10 ++++++++++ sqlite_utils/db.py | 19 +++++++++++++++---- tests/test_register_function.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index cb7461b..79ad510 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1703,3 +1703,13 @@ SQLite supports registering custom SQL functions written in Python. The ``@db.re # This prints "olleh" If your Python function accepts multiple arguments the SQL version will accept the same number of arguments. + +Python 3.8 added the ability to register `deterministic SQLite functions `__, allowing you to indicate that a function will return the exact same result for any given inputs and hence allowing SQLite to apply some performance optimizations. You can mark a function as deterministic using ``deterministic=True``, like this: + +.. code-block:: python + + @db.register_function(deterministic=True) + def reverse_string(s): + return "".join(reversed(list(s))) + +If you run this on a version of Python prior to 3.8 your code will still work, but the ``deterministic=True`` parameter will be ignored. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 877c17f..2e70f9d 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -10,6 +10,7 @@ import itertools import json import os import pathlib +import sys import textwrap import uuid @@ -149,10 +150,20 @@ class Database: def __repr__(self): return "".format(self.conn) - def register_function(self, fn): - name = fn.__name__ - arity = len(inspect.signature(fn).parameters) - self.conn.create_function(name, arity, fn) + def register_function(self, fn=None, deterministic=None): + def register(fn): + name = fn.__name__ + arity = len(inspect.signature(fn).parameters) + kwargs = {} + if deterministic and sys.version_info >= (3, 8): + kwargs["deterministic"] = True + self.conn.create_function(name, arity, fn, **kwargs) + return fn + + if fn is None: + return register + else: + register(fn) def execute(self, sql, parameters=None): if self._tracer: diff --git a/tests/test_register_function.py b/tests/test_register_function.py index 4aa5a3b..dab0a8d 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -1,3 +1,8 @@ +import pytest +import sys +from unittest.mock import MagicMock + + def test_register_function(fresh_db): @fresh_db.register_function def reverse_string(s): @@ -14,3 +19,28 @@ def test_register_function_multiple_arguments(fresh_db): result = fresh_db.execute("select a_times_b_plus_c(2, 3, 4)").fetchone()[0] assert result == 10 + + +def test_register_function_deterministic(fresh_db): + @fresh_db.register_function(deterministic=True) + def to_lower(s): + return s.lower() + + result = fresh_db.execute("select to_lower('BOB')").fetchone()[0] + assert result == "bob" + + +@pytest.mark.skipif( + sys.version_info < (3, 8), reason="deterministic=True was added in Python 3.8" +) +def test_register_function_deterministic_registered(fresh_db): + fresh_db.conn = MagicMock() + fresh_db.conn.create_function = MagicMock() + + @fresh_db.register_function(deterministic=True) + def to_lower_2(s): + return s.lower() + + fresh_db.conn.create_function.assert_called_with( + "to_lower_2", 1, to_lower_2, deterministic=True + ) From 43eae8b193d362f2b292df73e087ed6f10838144 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 28 Oct 2020 14:38:10 -0700 Subject: [PATCH 0245/1004] Release 2.23 Refs #189, #173, #191 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8f4abb4..018d59c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v2_23: + +2.23 (2020-10-28) +----------------- + +- ``table.m2m(other_table, records)`` method now takes any iterable, not just a list or tuple. Thanks, Adam Wolf. (`#189 `__) +- ``sqlite-utils insert`` now displays a progress bar for CSV or TSV imports. (`#173 `__) +- New ``@db.register_function(deterministic=True)`` option for registering deterministic SQLite functions in Python 3.8 or higher. (`#191 `__) + .. _v2_22: 2.22 (2020-10-16) diff --git a/setup.py b/setup.py index 405e6f3..304f69c 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.22" +VERSION = "2.23" def get_long_description(): From 59d8689ed0e6e042d99fd650896def680ca3c657 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 4 Nov 2020 19:53:32 -0800 Subject: [PATCH 0246/1004] table.virtual_table_using property, closes #196 --- docs/python-api.rst | 6 +++++ sqlite_utils/db.py | 28 +++++++++++++++++++++++ tests/test_introspect.py | 48 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 79ad510..9f69b34 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1430,6 +1430,12 @@ The ``detect_fts()`` method returns the associated SQLite FTS table name, if one >> db["authors"].detect_fts() "authors_fts" +The ``.virtual_table_using`` property reveals if a table is a virtual table. It returns ``None`` for regular tables and the upper case version of the type of virtual table otherwise. For example:: + + >> db["authors"].enable_fts(["name"]) + >> db["authors_fts"].virtual_table_using + "FTS5" + .. _python_api_fts: Enabling full-text search diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 2e70f9d..3600e88 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -10,12 +10,32 @@ import itertools import json import os import pathlib +import re import sys import textwrap import uuid SQLITE_MAX_VARS = 999 +_virtual_table_using_re = re.compile( + r""" +^ # Start of string +\s*CREATE\s+VIRTUAL\s+TABLE\s+ # CREATE VIRTUAL TABLE +( + '(?P[^']*(?:''[^']*)*)' | # single quoted name + "(?P[^"]*(?:""[^"]*)*)" | # double quoted name + `(?P[^`]+)` | # `backtick` quoted name + \[(?P[^\]]+)\] | # [...] quoted name + (?P # SQLite non-quoted identifier + [A-Za-z_\u0080-\uffff] # \u0080-\uffff = "any character larger than u007f" + [A-Za-z_\u0080-\uffff0-9\$]* # zero-or-more alphanemuric or $ + ) +) +\s+(IF\s+NOT\s+EXISTS\s+)? # IF NOT EXISTS (optional) +USING\s+(?P\w+) # e.g. USING FTS5 +""", + re.VERBOSE | re.IGNORECASE, +) try: import pandas as pd @@ -676,6 +696,14 @@ class Table(Queryable): ) return fks + @property + def virtual_table_using(self): + "Returns type of virtual table or None if this is not a virtual table" + match = _virtual_table_using_re.match(self.schema) + if match is None: + return None + return match.groupdict()["using"].upper() + @property def indexes(self): sql = 'PRAGMA index_list("{}")'.format(self.name) diff --git a/tests/test_introspect.py b/tests/test_introspect.py index cbec0a1..80cfee6 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import Index, View +from sqlite_utils.db import Index, View, Database import pytest @@ -142,3 +142,49 @@ def test_triggers(fresh_db): (t.name, t.table) for t in fresh_db["authors"].triggers } assert [] == fresh_db["other"].triggers + + +@pytest.mark.parametrize( + "sql,expected_name,expected_using", + [ + ( + """ + CREATE VIRTUAL TABLE foo USING FTS5(name) + """, + "foo", + "FTS5", + ), + ( + """ + CREATE VIRTUAL TABLE "foo" USING FTS4(name) + """, + "foo", + "FTS4", + ), + ( + """ + CREATE VIRTUAL TABLE IF NOT EXISTS `foo` USING FTS4(name) + """, + "foo", + "FTS4", + ), + ( + """ + CREATE VIRTUAL TABLE IF NOT EXISTS `foo` USING fts5(name) + """, + "foo", + "FTS5", + ), + ( + """ + CREATE TABLE IF NOT EXISTS `foo` (id integer primary key) + """, + "foo", + None, + ), + ], +) +def test_virtual_table_using(sql, expected_name, expected_using): + db = Database(memory=True) + db.execute(sql) + assert db[expected_name].virtual_table_using == expected_using From 2c00567aac6d9c79087cfff0d054f64922b1473d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Nov 2020 14:01:14 -0800 Subject: [PATCH 0247/1004] sqlite-utils search WIP, refs #192 --- docs/cli.rst | 44 ++++++++++++++++++ sqlite_utils/cli.py | 111 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 154 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 54591cc..5d20116 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -885,6 +885,50 @@ You can rebuild every FTS table by running ``rebuild-fts`` without passing any t $ sqlite-utils rebuild-fts mydb.db +.. _cli_search: + +Executing searches +================== + +Once you have configured full-text search for a table, you can search it using ``sqlite-utils search``:: + + $ sqlite-utils search mydb.db documents searchterm + +This command accepts the same output options as ``sqlite-utils query``: ``--table``, ``--csv``, ``--nl`` etc. + +By default it shows the most relevant matches first. You can specify a different sort order using the ``-o`` option, which can take a column or a column followed by ``desc``:: + + # Sort by rowid + $ sqlite-utils search mydb.db documents searchterm -o rowid + # Sort by created in descending order + $ sqlite-utils search mydb.db documents searchterm -o 'created desc' + +You can specify a subset of columns to be returned using the ``-c`` option one or more times:: + + $ sqlite-utils search mydb.db documents searchterm -c title -c created + +Use the ``--sql`` option to output the SQL that would be executed, rather than running the query:: + + $ sqlite-utils search mydb.db documents searchterm --sql + with original as ( + select + rowid, + * + from [documents] + ) + select + original.*, + [documents_fts].rank as rank + from + [original] + join [documents_fts] on [original].rowid = [documents_fts].rowid + where + [documents_fts] match :search + order by + rank desc + limit + 20 + .. _cli_vacuum: Vacuum diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index fe42ec2..15fc4b2 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -7,6 +7,7 @@ import hashlib import pathlib import sqlite_utils from sqlite_utils.db import AlterError +import textwrap import itertools import json import os @@ -45,7 +46,7 @@ def output_options(fn): is_flag=True, default=False, ), - click.option("-c", "--csv", is_flag=True, help="Output CSV"), + click.option("--csv", is_flag=True, help="Output CSV"), click.option("--no-headers", is_flag=True, help="Omit CSV headers"), click.option("-t", "--table", is_flag=True, help="Output as a table"), click.option( @@ -968,6 +969,114 @@ def query( click.echo(line) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("dbtable") +@click.argument("q") +@click.option("-o", "--order", type=str, help="Order by ('column' or 'column desc')") +@click.option("-c", "--column", type=str, multiple=True, help="Columns to return") +@click.option( + "--sql", "show_sql", is_flag=True, help="Show SQL query that would be run" +) +@output_options +@load_extension_option +@click.pass_context +def search( + ctx, + path, + dbtable, + q, + order, + show_sql, + column, + nl, + arrays, + csv, + no_headers, + table, + fmt, + json_cols, + load_extension, +): + "Execute a full-text search against this table" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + # Check table exists + table_obj = db[dbtable] + if not table_obj.exists(): + raise click.ClickException("Table '{}' does not exist".format(dbtable)) + fts_table = table_obj.detect_fts() + if not fts_table: + raise click.ClickException( + "Table '{}' is not configured for full-text search".format(dbtable) + ) + # Pick names for table and rank column that don't clash + original = "original_" if dbtable == "original" else "original" + rank = "rank" + while rank in table_obj.columns_dict: + rank = rank + "_" + columns = "*" + if column: + # Check they all exist + for c in column: + if c not in table_obj.columns_dict: + raise click.ClickException( + "Table '{}' has no column '{}".format(dbtable, c) + ) + columns = ", ".join("[{}]".format(c) for c in column) + sql = textwrap.dedent( + """ + with {original} as ( + select + rowid, + {columns} + from [{dbtable}] + ) + select + {original}.*, + [{fts}].rank as {rank} + from + [{original}] + join [{fts}] on [{original}].rowid = [{fts}].rowid + where + [{fts}] match :search + order by + {order} + limit + {limit} + """.format( + dbtable=dbtable, + original=original, + columns=columns, + rank=rank, + fts=fts_table, + order=order if order else "{} desc".format(rank), + limit=20, + ) + ).strip() + if show_sql: + click.echo(sql) + return + ctx.invoke( + query, + path=path, + sql=sql, + nl=nl, + arrays=arrays, + csv=csv, + no_headers=no_headers, + table=table, + fmt=fmt, + json_cols=json_cols, + param=[("search", q)], + load_extension=load_extension, + ) + + @cli.command() @click.argument( "path", From de39e8db1ee8b18755b9d83c69371a65664106fd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Nov 2020 14:46:18 -0800 Subject: [PATCH 0248/1004] Refactored to table.search_sql() method, added --limit --- docs/cli.rst | 6 ++--- sqlite_utils/cli.py | 54 +++++++++------------------------------ sqlite_utils/db.py | 43 ++++++++++++++++++++++++++++++++ tests/test_fts.py | 61 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 119 insertions(+), 45 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 5d20116..2d7c638 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -907,6 +907,8 @@ You can specify a subset of columns to be returned using the ``-c`` option one o $ sqlite-utils search mydb.db documents searchterm -c title -c created +By default all search results will be returned. You can use ``--limit 20`` to return just the first 20 results. + Use the ``--sql`` option to output the SQL that would be executed, rather than running the query:: $ sqlite-utils search mydb.db documents searchterm --sql @@ -923,11 +925,9 @@ Use the ``--sql`` option to output the SQL that would be executed, rather than r [original] join [documents_fts] on [original].rowid = [documents_fts].rowid where - [documents_fts] match :search + [documents_fts] match :query order by rank desc - limit - 20 .. _cli_vacuum: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 15fc4b2..15c78f6 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -7,7 +7,6 @@ import hashlib import pathlib import sqlite_utils from sqlite_utils.db import AlterError -import textwrap import itertools import json import os @@ -979,6 +978,12 @@ def query( @click.argument("q") @click.option("-o", "--order", type=str, help="Order by ('column' or 'column desc')") @click.option("-c", "--column", type=str, multiple=True, help="Columns to return") +@click.option( + "--limit", + type=int, + default=20, + help="Number of rows to return, default 20, set to 0 for unlimited", +) @click.option( "--sql", "show_sql", is_flag=True, help="Show SQL query that would be run" ) @@ -993,6 +998,7 @@ def search( order, show_sql, column, + limit, nl, arrays, csv, @@ -1009,55 +1015,19 @@ def search( table_obj = db[dbtable] if not table_obj.exists(): raise click.ClickException("Table '{}' does not exist".format(dbtable)) - fts_table = table_obj.detect_fts() - if not fts_table: + if not table_obj.detect_fts(): raise click.ClickException( "Table '{}' is not configured for full-text search".format(dbtable) ) - # Pick names for table and rank column that don't clash - original = "original_" if dbtable == "original" else "original" - rank = "rank" - while rank in table_obj.columns_dict: - rank = rank + "_" - columns = "*" if column: # Check they all exist + table_columns = table_obj.columns_dict for c in column: - if c not in table_obj.columns_dict: + if c not in table_columns: raise click.ClickException( "Table '{}' has no column '{}".format(dbtable, c) ) - columns = ", ".join("[{}]".format(c) for c in column) - sql = textwrap.dedent( - """ - with {original} as ( - select - rowid, - {columns} - from [{dbtable}] - ) - select - {original}.*, - [{fts}].rank as {rank} - from - [{original}] - join [{fts}] on [{original}].rowid = [{fts}].rowid - where - [{fts}] match :search - order by - {order} - limit - {limit} - """.format( - dbtable=dbtable, - original=original, - columns=columns, - rank=rank, - fts=fts_table, - order=order if order else "{} desc".format(rank), - limit=20, - ) - ).strip() + sql = table_obj.search_sql(columns=column, order=order, limit=limit) if show_sql: click.echo(sql) return @@ -1072,7 +1042,7 @@ def search( table=table, fmt=fmt, json_cols=json_cols, - param=[("search", q)], + param=[("query", q)], load_extension=load_extension, ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3600e88..cd6a5a6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1313,6 +1313,49 @@ class Table(Queryable): ) return self + def search_sql(self, columns=None, order=None, limit=None): + # Pick names for table and rank column that don't clash + original = "original_" if self.name == "original" else "original" + rank = "rank" + while rank in self.columns_dict: + rank = rank + "_" + columns_sql = "*" + if columns: + columns_sql = ", ".join("[{}]".format(c) for c in columns) + fts_table = self.detect_fts() + assert fts_table, "Full-text search is not configured for table '{}'".format( + self.name + ) + return textwrap.dedent( + """ + with {original} as ( + select + rowid, + {columns} + from [{dbtable}] + ) + select + {original}.*, + [{fts}].rank as {rank} + from + [{original}] + join [{fts}] on [{original}].rowid = [{fts}].rowid + where + [{fts}] match :query + order by + {order} + {limit} + """.format( + dbtable=self.name, + original=original, + columns=columns_sql, + rank=rank, + fts=fts_table, + order=order or "{} desc".format(rank), + limit="limit {}".format(limit) if limit else "", + ) + ).strip() + def search(self, q): sql = ( textwrap.dedent( diff --git a/tests/test_fts.py b/tests/test_fts.py index 359d911..ddcf1bd 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -276,3 +276,64 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): db["books"].enable_fts(["title", "author"], create_triggers=True, replace=True) # The only SQL that executed should be select statements assert all(q[0].startswith("select ") for q in queries) + + +@pytest.mark.parametrize( + "kwargs,expected", + [ + ( + {}, + ( + "with original as (\n" + " select\n" + " rowid,\n" + " *\n" + " from [books]\n" + ")\n" + "select\n" + " original.*,\n" + " [books_fts].rank as rank\n" + "from\n" + " [original]\n" + " join [books_fts] on [original].rowid = [books_fts].rowid\n" + "where\n" + " [books_fts] match :query\n" + "order by\n" + " rank desc" + ), + ), + ( + {"columns": ["title"], "order": "rowid", "limit": 10}, + ( + "with original as (\n" + " select\n" + " rowid,\n" + " [title]\n" + " from [books]\n" + ")\n" + "select\n" + " original.*,\n" + " [books_fts].rank as rank\n" + "from\n" + " [original]\n" + " join [books_fts] on [original].rowid = [books_fts].rowid\n" + "where\n" + " [books_fts] match :query\n" + "order by\n" + " rowid\n" + "limit 10" + ), + ), + ], +) +def test_search_sql(kwargs, expected): + db = Database(memory=True) + db["books"].insert( + { + "title": "Habits of Australian Marsupials", + "author": "Marlee Hawkins", + } + ) + db["books"].enable_fts(["title", "author"]) + sql = db["books"].search_sql(**kwargs) + assert sql == expected From 7c22a64fb60fdf50c8a5f521ecd2c320143341d5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 5 Nov 2020 10:01:58 -0800 Subject: [PATCH 0249/1004] .search() now works differently for FTS4 v.s. FTS5 --- sqlite_utils/db.py | 85 +++++++++++++++++++++++----------------------- tests/test_fts.py | 2 +- 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index cd6a5a6..bfbd17c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1326,51 +1326,52 @@ class Table(Queryable): assert fts_table, "Full-text search is not configured for table '{}'".format( self.name ) - return textwrap.dedent( - """ - with {original} as ( - select - rowid, - {columns} - from [{dbtable}] - ) - select - {original}.*, - [{fts}].rank as {rank} - from - [{original}] - join [{fts}] on [{original}].rowid = [{fts}].rowid - where - [{fts}] match :query - order by - {order} - {limit} - """.format( - dbtable=self.name, - original=original, - columns=columns_sql, - rank=rank, - fts=fts_table, - order=order or "{} desc".format(rank), - limit="limit {}".format(limit) if limit else "", + if self.db[fts_table].virtual_table_using == "FTS5": + sql = textwrap.dedent( + """ + with {original} as ( + select + rowid, + {columns} + from [{dbtable}] ) + select + {original}.*, + [{fts}].rank as {rank} + from + [{original}] + join [{fts}] on [{original}].rowid = [{fts}].rowid + where + [{fts}] match :query + order by + {order} + {limit} + """ + ).strip() + else: + if order == rank or order is None: + order = "rowid" + sql = textwrap.dedent( + """ + select * from "{dbtable}" where rowid in ( + select rowid from [{fts}] + where [{fts}] match :query + ) + order by {order} + """ + ).strip() + return sql.format( + dbtable=self.name, + original=original, + columns=columns_sql, + rank=rank, + fts=fts_table, + order=order or "{} desc".format(rank), + limit="limit {}".format(limit) if limit else "", ).strip() - def search(self, q): - sql = ( - textwrap.dedent( - """ - select * from "{table}" where rowid in ( - select rowid from [{table}_fts] - where [{table}_fts] match :search - ) - order by rowid - """ - ) - .strip() - .format(table=self.name) - ) - return self.db.execute(sql, (q,)).fetchall() + def search(self, q, order=None): + return self.db.execute(self.search_sql(order=order), {"query": q}).fetchall() def value_or_default(self, key, value): return self._defaults[key] if value is DEFAULT else value diff --git a/tests/test_fts.py b/tests/test_fts.py index ddcf1bd..9ae692f 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -112,7 +112,7 @@ def test_fts_tokenize(fresh_db): tokenize="porter", ) assert [("racoons are biting trash pandas", "USA", "bar")] == table.search( - "bite" + "bite", order="rowid" ) From 27b67f1cae3f4edf206cf9886aa6a31b2a0ffe63 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 07:53:22 -0800 Subject: [PATCH 0250/1004] @db.register_function(..., replace=True), closes #199 --- docs/python-api.rst | 10 ++++++++++ setup.py | 2 +- sqlite_utils/db.py | 6 +++++- tests/test_register_function.py | 22 ++++++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 9f69b34..b008cb5 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1719,3 +1719,13 @@ Python 3.8 added the ability to register `deterministic SQLite functions ".format(self.conn) - def register_function(self, fn=None, deterministic=None): + def register_function(self, fn=None, deterministic=None, replace=False): def register(fn): name = fn.__name__ arity = len(inspect.signature(fn).parameters) + if not replace and (name, arity) in self._registered_functions: + return fn kwargs = {} if deterministic and sys.version_info >= (3, 8): kwargs["deterministic"] = True self.conn.create_function(name, arity, fn, **kwargs) + self._registered_functions.add((name, arity)) return fn if fn is None: diff --git a/tests/test_register_function.py b/tests/test_register_function.py index dab0a8d..e6d977a 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -44,3 +44,25 @@ def test_register_function_deterministic_registered(fresh_db): fresh_db.conn.create_function.assert_called_with( "to_lower_2", 1, to_lower_2, deterministic=True ) + + +def test_register_function_replace(fresh_db): + @fresh_db.register_function() + def one(): + return "one" + + assert "one" == fresh_db.execute("select one()").fetchone()[0] + + # This will fail to replace the function: + @fresh_db.register_function() + def one(): + return "two" + + assert "one" == fresh_db.execute("select one()").fetchone()[0] + + # This will replace it + @fresh_db.register_function(replace=True) + def one(): + return "two" + + assert "two" == fresh_db.execute("select one()").fetchone()[0] From 476825a224b382febbbd8569e89c78be3e8d426b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 08:03:25 -0800 Subject: [PATCH 0251/1004] How to use register_function as a method, not a decorator --- docs/python-api.rst | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index b008cb5..2740366 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1693,7 +1693,9 @@ You can use it in code like this: Registering custom SQL functions ================================ -SQLite supports registering custom SQL functions written in Python. The ``@db.register_function`` decorator provides syntactic sugar for registering these functions. +SQLite supports registering custom SQL functions written in Python. The ``db.register_function()`` method lets you register these functions, and keeps track of functions that have already been registered. + +If you use it as a method it will automatically detect the name and number of arguments needed by the function: .. code-block:: python @@ -1701,14 +1703,22 @@ SQLite supports registering custom SQL functions written in Python. The ``@db.re db = Database(memory=True) + def reverse_string(s): + return "".join(reversed(list(s))) + + db.register_function(reverse_string) + print(db.execute('select reverse_string("hello")').fetchone()[0]) + # This prints "olleh" + +You can also use the method as a function decorator like so: + +.. code-block:: python + @db.register_function def reverse_string(s): return "".join(reversed(list(s))) - print(db.execute('select reverse_string("hello")').fetchone())[0] - # This prints "olleh" - -If your Python function accepts multiple arguments the SQL version will accept the same number of arguments. + print(db.execute('select reverse_string("hello")').fetchone()[0]) Python 3.8 added the ability to register `deterministic SQLite functions `__, allowing you to indicate that a function will return the exact same result for any given inputs and hence allowing SQLite to apply some performance optimizations. You can mark a function as deterministic using ``deterministic=True``, like this: From d411fba1f4124047429ac9d1175a04b36bf5bee9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 10:23:16 -0800 Subject: [PATCH 0252/1004] .search() works for FTS4, yields dicts Closes #198, refs #197 --- sqlite_utils/db.py | 69 +++++++++++----------- tests/test_cli.py | 10 ++-- tests/test_fts.py | 133 +++++++++++++++++++++++++++++++++++-------- tests/test_tracer.py | 22 ++++--- 4 files changed, 164 insertions(+), 70 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6a41932..f1a1e74 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -11,6 +11,7 @@ import json import os import pathlib import re +from sqlite_fts4 import rank_bm25 import sys import textwrap import uuid @@ -189,6 +190,9 @@ class Database: else: register(fn) + def register_fts4_bm25(self): + self.register_function(rank_bm25, deterministic=True) + def execute(self, sql, parameters=None): if self._tracer: self._tracer(sql, parameters) @@ -1330,52 +1334,51 @@ class Table(Queryable): assert fts_table, "Full-text search is not configured for table '{}'".format( self.name ) - if self.db[fts_table].virtual_table_using == "FTS5": - sql = textwrap.dedent( - """ - with {original} as ( - select - rowid, - {columns} - from [{dbtable}] - ) + virtual_table_using = self.db[fts_table].virtual_table_using + sql = textwrap.dedent( + """ + with {original} as ( select - {original}.*, - [{fts}].rank as {rank} - from - [{original}] - join [{fts}] on [{original}].rowid = [{fts}].rowid - where - [{fts}] match :query - order by - {order} - {limit} - """ - ).strip() + rowid, + {columns} + from [{dbtable}] + ) + select + {original}.*, + {rank_implementation} as {rank} + from + [{original}] + join [{fts_table}] on [{original}].rowid = [{fts_table}].rowid + where + [{fts_table}] match :query + order by + {order} + {limit} + """ + ).strip() + if virtual_table_using == "FTS5": + rank_implementation = "[{}].rank".format(fts_table) else: - if order == rank or order is None: - order = "rowid" - sql = textwrap.dedent( - """ - select * from "{dbtable}" where rowid in ( - select rowid from [{fts}] - where [{fts}] match :query + self.db.register_fts4_bm25() + rank_implementation = "-rank_bm25(matchinfo([{}], 'pcnalx'))".format( + fts_table ) - order by {order} - """ - ).strip() return sql.format( dbtable=self.name, original=original, columns=columns_sql, rank=rank, - fts=fts_table, + rank_implementation=rank_implementation, + fts_table=fts_table, order=order or "{} desc".format(rank), limit="limit {}".format(limit) if limit else "", ).strip() def search(self, q, order=None): - return self.db.execute(self.search_sql(order=order), {"query": q}).fetchall() + cursor = self.db.execute(self.search_sql(order=order), {"query": q}) + columns = [c[0] for c in cursor.description] + for row in cursor: + yield dict(zip(columns, row)) def value_or_default(self, key, value): return self._defaults[key] if value is DEFAULT else value diff --git a/tests/test_cli.py b/tests/test_cli.py index 7e1365d..f9b2509 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -510,14 +510,14 @@ def test_rebuild_fts(db_path, tables): ["c1", "c2", "c3"], fts_version="FTS5", create_triggers=True ) # Search should work - assert db["fts4_table"].search("verb1") - assert db["fts5_table"].search("verb1") + assert list(db["fts4_table"].search("verb1")) + assert list(db["fts5_table"].search("verb1")) # Deleting _fts_segments to break FTS4 with db.conn: db["fts4_table_fts_segments"].delete_where() # Now this should error: with pytest.raises(sqlite3.DatabaseError): - db["fts4_table"].search("verb1") + list(db["fts4_table"].search("verb1")) # Replicate docsize error from this issue for FTS5 # https://github.com/simonw/sqlite-utils/issues/149 assert db["fts5_table_fts_docsize"].count == 10000 @@ -530,10 +530,10 @@ def test_rebuild_fts(db_path, tables): assert 0 == result.exit_code fixed_tables = tables or ["fts4_table", "fts5_table"] if "fts4_table" in fixed_tables: - assert db["fts4_table"].search("verb1") + assert list(db["fts4_table"].search("verb1")) else: with pytest.raises(sqlite3.DatabaseError): - db["fts4_table"].search("verb1") + list(db["fts4_table"].search("verb1")) if "fts5_table" in fixed_tables: assert db["fts5_table_fts_docsize"].count == 10000 else: diff --git a/tests/test_fts.py b/tests/test_fts.py index 9ae692f..0b0c106 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -29,9 +29,25 @@ def test_enable_fts(fresh_db): "searchable_fts_docsize", "searchable_fts_stat", ] == fresh_db.table_names() - assert [("tanuki are running tricksters", "Japan", "foo")] == table.search("tanuki") - assert [("racoons are biting trash pandas", "USA", "bar")] == table.search("usa") - assert [] == table.search("bar") + assert [ + { + "rowid": 1, + "text": "tanuki are running tricksters", + "country": "Japan", + "not_searchable": "foo", + "rank": 0.0, + } + ] == list(table.search("tanuki")) + assert [ + { + "rowid": 2, + "text": "racoons are biting trash pandas", + "country": "USA", + "not_searchable": "bar", + "rank": 0.0, + } + ] == list(table.search("usa")) + assert [] == list(table.search("bar")) def test_enable_fts_escape_table_names(fresh_db): @@ -49,9 +65,25 @@ def test_enable_fts_escape_table_names(fresh_db): "http://example.com_fts_docsize", "http://example.com_fts_stat", ] == fresh_db.table_names() - assert [("tanuki are running tricksters", "Japan", "foo")] == table.search("tanuki") - assert [("racoons are biting trash pandas", "USA", "bar")] == table.search("usa") - assert [] == table.search("bar") + assert [ + { + "rowid": 1, + "text": "tanuki are running tricksters", + "country": "Japan", + "not_searchable": "foo", + "rank": 0.0, + } + ] == list(table.search("tanuki")) + assert [ + { + "rowid": 2, + "text": "racoons are biting trash pandas", + "country": "USA", + "not_searchable": "bar", + "rank": 0.0, + } + ] == list(table.search("usa")) + assert [] == list(table.search("bar")) def test_enable_fts_table_names_containing_spaces(fresh_db): @@ -72,12 +104,21 @@ def test_populate_fts(fresh_db): table = fresh_db["populatable"] table.insert(search_records[0]) table.enable_fts(["text", "country"], fts_version="FTS4") - assert [] == table.search("trash pandas") + assert [] == list(table.search("trash pandas")) table.insert(search_records[1]) - assert [] == table.search("trash pandas") + assert [] == list(table.search("trash pandas")) # Now run populate_fts to make this record available table.populate_fts(["text", "country"]) - assert [("racoons are biting trash pandas", "USA", "bar")] == table.search("usa") + rows = list(table.search("usa")) + assert [ + { + "rowid": 2, + "text": "racoons are biting trash pandas", + "country": "USA", + "not_searchable": "bar", + "rank": 0.5108256237659907, + } + ] == rows def test_populate_fts_escape_table_names(fresh_db): @@ -85,12 +126,20 @@ def test_populate_fts_escape_table_names(fresh_db): table = fresh_db["http://example.com"] table.insert(search_records[0]) table.enable_fts(["text", "country"], fts_version="FTS4") - assert [] == table.search("trash pandas") + assert [] == list(table.search("trash pandas")) table.insert(search_records[1]) - assert [] == table.search("trash pandas") + assert [] == list(table.search("trash pandas")) # Now run populate_fts to make this record available table.populate_fts(["text", "country"]) - assert [("racoons are biting trash pandas", "USA", "bar")] == table.search("usa") + assert [ + { + "rowid": 2, + "text": "racoons are biting trash pandas", + "country": "USA", + "not_searchable": "bar", + "rank": 0.5108256237659907, + } + ] == list(table.search("usa")) def test_fts_tokenize(fresh_db): @@ -103,7 +152,7 @@ def test_fts_tokenize(fresh_db): ["text", "country"], fts_version="FTS{}".format(fts_version), ) - assert [] == table.search("bite") + assert [] == list(table.search("bite")) # Test WITH stemming table.disable_fts() table.enable_fts( @@ -111,9 +160,14 @@ def test_fts_tokenize(fresh_db): fts_version="FTS{}".format(fts_version), tokenize="porter", ) - assert [("racoons are biting trash pandas", "USA", "bar")] == table.search( - "bite", order="rowid" - ) + rows = list(table.search("bite", order="rowid")) + assert len(rows) == 1 + assert { + "rowid": 2, + "text": "racoons are biting trash pandas", + "country": "USA", + "not_searchable": "bar", + }.items() <= rows[0].items() def test_optimize_fts(fresh_db): @@ -132,15 +186,34 @@ def test_optimize_fts(fresh_db): fresh_db[table_name].optimize() -def test_enable_fts_w_triggers(fresh_db): +def test_enable_fts_with_triggers(fresh_db): table = fresh_db["searchable"] table.insert(search_records[0]) table.enable_fts(["text", "country"], fts_version="FTS4", create_triggers=True) - assert [("tanuki are running tricksters", "Japan", "foo")] == table.search("tanuki") + rows1 = list(table.search("tanuki")) + assert len(rows1) == 1 + assert rows1 == [ + { + "rowid": 1, + "text": "tanuki are running tricksters", + "country": "Japan", + "not_searchable": "foo", + "rank": 0.0, + } + ] table.insert(search_records[1]) # Triggers will auto-populate FTS virtual table, not need to call populate_fts() - assert [("racoons are biting trash pandas", "USA", "bar")] == table.search("usa") - assert [] == table.search("bar") + rows2 = list(table.search("usa")) + assert rows2 == [ + { + "rowid": 2, + "text": "racoons are biting trash pandas", + "country": "USA", + "not_searchable": "bar", + "rank": 0.0, + } + ] + assert [] == list(table.search("bar")) @pytest.mark.parametrize("create_triggers", [True, False]) @@ -183,15 +256,29 @@ def test_rebuild_fts(fresh_db, table_to_fix): table.insert(search_records[0]) table.enable_fts(["text", "country"]) # Run a search - assert [("tanuki are running tricksters", "Japan", "foo")] == table.search("tanuki") + rows = list(table.search("tanuki")) + assert len(rows) == 1 + assert { + "rowid": 1, + "text": "tanuki are running tricksters", + "country": "Japan", + "not_searchable": "foo", + }.items() <= rows[0].items() # Delete from searchable_fts_data fresh_db["searchable_fts_data"].delete_where() # This should have broken the index with pytest.raises(sqlite3.DatabaseError): - table.search("tanuki") + list(table.search("tanuki")) # Running rebuild_fts() should fix it fresh_db[table_to_fix].rebuild_fts() - assert [("tanuki are running tricksters", "Japan", "foo")] == table.search("tanuki") + rows2 = list(table.search("tanuki")) + assert len(rows2) == 1 + assert { + "rowid": 1, + "text": "tanuki are running tricksters", + "country": "Japan", + "not_searchable": "foo", + }.items() <= rows2[0].items() @pytest.mark.parametrize("invalid_table", ["does_not_exist", "not_searchable"]) diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 6a58ec0..dab9afb 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -27,10 +27,6 @@ def test_tracer(): None, ), ("select name from sqlite_master where type = 'view'", None), - ( - 'select * from "dogs" where rowid in (\n select rowid from [dogs_fts]\n where [dogs_fts] match :search\n)\norder by rowid', - ("Cleopaws",), - ), ] @@ -46,17 +42,25 @@ def test_with_tracer(): assert len(collected) == 0 with db.tracer(tracer): - db["dogs"].search("Cleopaws") + list(db["dogs"].search("Cleopaws")) - assert len(collected) == 2 + assert len(collected) == 7 assert collected == [ ("select name from sqlite_master where type = 'view'", None), + ("select name from sqlite_master where type = 'table'", None), + ("PRAGMA table_info([dogs])", None), ( - 'select * from "dogs" where rowid in (\n select rowid from [dogs_fts]\n where [dogs_fts] match :search\n)\norder by rowid', - ("Cleopaws",), + "SELECT name FROM sqlite_master\n WHERE rootpage = 0\n AND (\n sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n OR (\n tbl_name = \"dogs\"\n AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n )\n )", + None, + ), + ("select name from sqlite_master where type = 'view'", None), + ("select sql from sqlite_master where name = ?", ("dogs_fts",)), + ( + "with original as (\n select\n rowid,\n *\n from [dogs]\n)\nselect\n original.*,\n [dogs_fts].rank as rank\nfrom\n [original]\n join [dogs_fts] on [original].rowid = [dogs_fts].rowid\nwhere\n [dogs_fts] match :query\norder by\n rank desc", + {"query": "Cleopaws"}, ), ] # Outside the with block collected should not be appended to db["dogs"].insert({"name": "Cleopaws"}) - assert len(collected) == 2 + assert len(collected) == 7 From 63e2bdf18d9db092ebafc2f054eebf5be791db26 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 15:40:42 -0800 Subject: [PATCH 0253/1004] Added test for sqlite-utils search, refs #192 --- sqlite_utils/cli.py | 1 + sqlite_utils/db.py | 2 +- tests/test_cli.py | 32 ++++++++++++++++++++++++++++++++ tests/test_fts.py | 4 ++-- 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 15c78f6..37c44f7 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -941,6 +941,7 @@ def query( "Execute SQL query and return the results as JSON" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) + db.register_fts4_bm25() with db.conn: cursor = db.execute(sql, dict(param)) if cursor.description is None: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f1a1e74..9d067d3 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1360,7 +1360,7 @@ class Table(Queryable): rank_implementation = "[{}].rank".format(fts_table) else: self.db.register_fts4_bm25() - rank_implementation = "-rank_bm25(matchinfo([{}], 'pcnalx'))".format( + rank_implementation = "rank_bm25(matchinfo([{}], 'pcnalx'))".format( fts_table ) return sql.format( diff --git a/tests/test_cli.py b/tests/test_cli.py index f9b2509..69aef64 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1677,3 +1677,35 @@ def test_insert_encoding(tmpdir): "longitude": None, }, ] + + +@pytest.mark.parametrize("fts", ["FTS4", "FTS5"]) +@pytest.mark.parametrize( + "extra_arg,expected", + [ + ( + None, + '[{"rowid": 2, "id": 2, "title": "Title the second", "rank": -0.5108256237659907}]\n', + ), + ("--csv", "rowid,id,title,rank\n2,2,Title the second,-0.5108256237659907\n"), + ], +) +def test_search(tmpdir, fts, extra_arg, expected): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["articles"].insert_all( + [ + {"id": 1, "title": "Title the first"}, + {"id": 2, "title": "Title the second"}, + {"id": 3, "title": "Title the third"}, + ], + pk="id", + ) + db["articles"].enable_fts(["title"], fts_version=fts) + result = CliRunner().invoke( + cli.cli, + ["search", db_path, "articles", "second"] + ([extra_arg] if extra_arg else []), + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert result.output == expected diff --git a/tests/test_fts.py b/tests/test_fts.py index 0b0c106..6f469a2 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -116,7 +116,7 @@ def test_populate_fts(fresh_db): "text": "racoons are biting trash pandas", "country": "USA", "not_searchable": "bar", - "rank": 0.5108256237659907, + "rank": -0.5108256237659907, } ] == rows @@ -137,7 +137,7 @@ def test_populate_fts_escape_table_names(fresh_db): "text": "racoons are biting trash pandas", "country": "USA", "not_searchable": "bar", - "rank": 0.5108256237659907, + "rank": -0.5108256237659907, } ] == list(table.search("usa")) From 771bd81b62b73b69a61cec5e71be97c0a7e2c5f9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 15:50:02 -0800 Subject: [PATCH 0254/1004] search_sql() returns most relevant first, not least Refs #192 --- docs/cli.rst | 2 +- sqlite_utils/db.py | 2 +- tests/test_fts.py | 2 +- tests/test_tracer.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 2d7c638..03676d4 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -927,7 +927,7 @@ Use the ``--sql`` option to output the SQL that would be executed, rather than r where [documents_fts] match :query order by - rank desc + rank .. _cli_vacuum: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9d067d3..8e586d8 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1370,7 +1370,7 @@ class Table(Queryable): rank=rank, rank_implementation=rank_implementation, fts_table=fts_table, - order=order or "{} desc".format(rank), + order=order or rank, limit="limit {}".format(limit) if limit else "", ).strip() diff --git a/tests/test_fts.py b/tests/test_fts.py index 6f469a2..3c2180e 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -386,7 +386,7 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): "where\n" " [books_fts] match :query\n" "order by\n" - " rank desc" + " rank" ), ), ( diff --git a/tests/test_tracer.py b/tests/test_tracer.py index dab9afb..318d45e 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -56,7 +56,7 @@ def test_with_tracer(): ("select name from sqlite_master where type = 'view'", None), ("select sql from sqlite_master where name = ?", ("dogs_fts",)), ( - "with original as (\n select\n rowid,\n *\n from [dogs]\n)\nselect\n original.*,\n [dogs_fts].rank as rank\nfrom\n [original]\n join [dogs_fts] on [original].rowid = [dogs_fts].rowid\nwhere\n [dogs_fts] match :query\norder by\n rank desc", + "with original as (\n select\n rowid,\n *\n from [dogs]\n)\nselect\n original.*,\n [dogs_fts].rank as rank\nfrom\n [original]\n join [dogs_fts] on [original].rowid = [dogs_fts].rowid\nwhere\n [dogs_fts] match :query\norder by\n rank", {"query": "Cleopaws"}, ), ] From afee15f04b060a557897fa876f6e5d37f6d898cb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 16:09:42 -0800 Subject: [PATCH 0255/1004] --tsv output option, closes #193 --- docs/cli.rst | 20 +++++++++++----- sqlite_utils/cli.py | 17 ++++++++++---- tests/test_cli.py | 56 ++++++++++++++++++++++++++++++--------------- 3 files changed, 65 insertions(+), 28 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 03676d4..166eea1 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -122,14 +122,14 @@ You can use the ``--json-cols`` option to automatically detect these JSON column } ] } - ] + ]- .. _cli_query_csv: Running queries and returning CSV ================================= -You can use the ``--csv`` option (or ``-c`` shortcut) to return results as CSV:: +You can use the ``--csv`` option to return results as CSV:: $ sqlite-utils dogs.db "select * from dogs" --csv id,age,name @@ -142,6 +142,13 @@ This will default to including the column names as a header row. To exclude the 1,4,Cleo 2,2,Pancakes +Use ``--tsv`` instead of ``--csv`` to get back tab-separated values:: + + $ sqlite-utils dogs.db "select * from dogs" --tsv + id age name + 1 4 Cleo + 2 2 Pancakes + .. _cli_query_table: Running queries and outputting a table @@ -189,7 +196,7 @@ You can return every row in a specified table using the ``rows`` command:: [{"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}] -This command accepts the same output options as ``query`` - so you can pass ``--nl``, ``--csv``, ``--no-headers``, ``--table`` and ``--fmt``. +This command accepts the same output options as ``query`` - so you can pass ``--nl``, ``--csv``, ``--tsv``, ``--no-headers``, ``--table`` and ``--fmt``. .. _cli_tables: @@ -203,7 +210,7 @@ You can list the names of tables in a database using the ``tables`` command:: {"table": "cats"}, {"table": "chickens"}] -You can output this list in CSV using the ``--csv`` option:: +You can output this list in CSV using the ``--csv`` or ``--tsv`` options:: $ sqlite-utils tables mydb.db --csv --no-headers dogs @@ -241,7 +248,7 @@ Use ``--schema`` to include the schema of each table:: [age] INTEGER, [name] TEXT) -The ``--nl``, ``--csv`` and ``--table`` options are all available. +The ``--nl``, ``--csv``, ``--tsv`` and ``--table`` options are all available. .. _cli_views: @@ -263,6 +270,7 @@ It takes the same options as the ``tables`` command: * ``--counts`` * ``--nl`` * ``--csv`` +* ``--tsv`` * ``--table`` .. _cli_inserting_data: @@ -894,7 +902,7 @@ Once you have configured full-text search for a table, you can search it using ` $ sqlite-utils search mydb.db documents searchterm -This command accepts the same output options as ``sqlite-utils query``: ``--table``, ``--csv``, ``--nl`` etc. +This command accepts the same output options as ``sqlite-utils query``: ``--table``, ``--csv``, ``--tsv``, ``--nl`` etc. By default it shows the most relevant matches first. You can specify a different sort order using the ``-o`` option, which can take a column or a column followed by ``desc``:: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 37c44f7..18247eb 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -46,6 +46,7 @@ def output_options(fn): default=False, ), click.option("--csv", is_flag=True, help="Output CSV"), + click.option("--tsv", is_flag=True, help="Output TSV"), click.option("--no-headers", is_flag=True, help="Omit CSV headers"), click.option("-t", "--table", is_flag=True, help="Output as a table"), click.option( @@ -120,6 +121,7 @@ def tables( nl, arrays, csv, + tsv, no_headers, table, fmt, @@ -161,8 +163,8 @@ def tables( if table: print(tabulate.tabulate(_iter(), headers=headers, tablefmt=fmt)) - elif csv: - writer = csv_std.writer(sys.stdout) + elif csv or tsv: + writer = csv_std.writer(sys.stdout, dialect="excel-tab" if tsv else "excel") if not no_headers: writer.writerow(headers) for row in _iter(): @@ -201,6 +203,7 @@ def views( nl, arrays, csv, + tsv, no_headers, table, fmt, @@ -218,6 +221,7 @@ def views( nl=nl, arrays=arrays, csv=csv, + tsv=tsv, no_headers=no_headers, table=table, fmt=fmt, @@ -930,6 +934,7 @@ def query( nl, arrays, csv, + tsv, no_headers, table, fmt, @@ -958,8 +963,8 @@ def query( sys.stdout.write(str(data)) elif table: print(tabulate.tabulate(list(cursor), headers=headers, tablefmt=fmt)) - elif csv: - writer = csv_std.writer(sys.stdout) + elif csv or tsv: + writer = csv_std.writer(sys.stdout, dialect="excel-tab" if tsv else "excel") if not no_headers: writer.writerow(headers) for row in cursor: @@ -1003,6 +1008,7 @@ def search( nl, arrays, csv, + tsv, no_headers, table, fmt, @@ -1039,6 +1045,7 @@ def search( nl=nl, arrays=arrays, csv=csv, + tsv=tsv, no_headers=no_headers, table=table, fmt=fmt, @@ -1065,6 +1072,7 @@ def rows( nl, arrays, csv, + tsv, no_headers, table, fmt, @@ -1079,6 +1087,7 @@ def rows( nl=nl, arrays=arrays, csv=csv, + tsv=tsv, no_headers=no_headers, table=table, fmt=fmt, diff --git a/tests/test_cli.py b/tests/test_cli.py index 69aef64..db31121 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -62,24 +62,37 @@ def test_tables_counts_and_columns(db_path): ) == result.output.strip() -def test_tables_counts_and_columns_csv(db_path): +@pytest.mark.parametrize( + "format,expected", + [ + ( + "--csv", + ( + "table,count,columns\n" + 'Gosh,0,"c1\n' + "c2\n" + 'c3"\n' + 'Gosh2,0,"c1\n' + "c2\n" + 'c3"\n' + 'lots,30,"id\n' + 'age"' + ), + ), + ( + "--tsv", + "table\tcount\tcolumns\nGosh\t0\t['c1', 'c2', 'c3']\nGosh2\t0\t['c1', 'c2', 'c3']\nlots\t30\t['id', 'age']", + ), + ], +) +def test_tables_counts_and_columns_csv(db_path, format, expected): db = Database(db_path) with db.conn: db["lots"].insert_all([{"id": i, "age": i + 1} for i in range(30)]) result = CliRunner().invoke( - cli.cli, ["tables", "--counts", "--columns", "--csv", db_path] + cli.cli, ["tables", "--counts", "--columns", format, db_path] ) - assert ( - "table,count,columns\n" - 'Gosh,0,"c1\n' - "c2\n" - 'c3"\n' - 'Gosh2,0,"c1\n' - "c2\n" - 'c3"\n' - 'lots,30,"id\n' - 'age"' - ) == result.output.strip() + assert result.output.strip() == expected def test_tables_schema(db_path): @@ -809,7 +822,14 @@ def test_insert_alter(db_path, tmpdir): ] == db.execute_returning_dicts("select foo, n, baz from from_json_nl") -def test_query_csv(db_path): +@pytest.mark.parametrize( + "format,expected", + [ + ("--csv", "id,name,age\n1,Cleo,4\n2,Pancakes,2\n"), + ("--tsv", "id\tname\tage\n1\tCleo\t4\n2\tPancakes\t2\n"), + ], +) +def test_query_csv(db_path, format, expected): db = Database(db_path) with db.conn: db["dogs"].insert_all( @@ -819,15 +839,15 @@ def test_query_csv(db_path): ] ) result = CliRunner().invoke( - cli.cli, [db_path, "select id, name, age from dogs", "--csv"] + cli.cli, [db_path, "select id, name, age from dogs", format] ) assert 0 == result.exit_code - assert "id,name,age\n1,Cleo,4\n2,Pancakes,2\n" == result.output + assert result.output == expected # Test the no-headers option: result = CliRunner().invoke( - cli.cli, [db_path, "select id, name, age from dogs", "--no-headers", "--csv"] + cli.cli, [db_path, "select id, name, age from dogs", "--no-headers", format] ) - assert "1,Cleo,4\n2,Pancakes,2\n" == result.output + assert result.output.strip() == "\n".join(expected.split("\n")[1:]).strip() _all_query = "select id, name, age from dogs" From 309ae84336fd1d0262cad49adef7ce3dc72c531c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 16:17:15 -0800 Subject: [PATCH 0256/1004] '-f' must now always be specified as '--fmt', refs #194 --- docs/cli.rst | 2 +- sqlite_utils/cli.py | 1 - tests/test_cli.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 166eea1..9044039 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -162,7 +162,7 @@ You can use the ``--table`` option (or ``-t`` shortcut) to output query results 1 4 Cleo 2 2 Pancakes -You can use the ``--fmt`` (or ``-f``) option to specify different table formats, for example ``rst`` for reStructuredText:: +You can use the ``--fmt`` option to specify different table formats, for example ``rst`` for reStructuredText:: $ sqlite-utils dogs.db "select * from dogs" --table --fmt rst ==== ===== ======== diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 18247eb..e75d514 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -50,7 +50,6 @@ def output_options(fn): click.option("--no-headers", is_flag=True, help="Omit CSV headers"), click.option("-t", "--table", is_flag=True, help="Output as a table"), click.option( - "-f", "--fmt", help="Table format - one of {}".format( ", ".join(tabulate.tabulate_formats) diff --git a/tests/test_cli.py b/tests/test_cli.py index db31121..00efdf1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -149,7 +149,7 @@ def test_output_table(db_path, fmt, expected): for i in range(4) ] ) - result = CliRunner().invoke(cli.cli, ["rows", db_path, "rows", "-t", "-f", fmt]) + result = CliRunner().invoke(cli.cli, ["rows", db_path, "rows", "-t", "--fmt", fmt]) assert 0 == result.exit_code assert expected == result.output.strip() From 6863dc267745bc0a3392912f00c6aeb628e5ee3f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 16:28:41 -0800 Subject: [PATCH 0257/1004] sqlite-utils rows -c, closes #200 --- docs/cli.rst | 6 ++++++ sqlite_utils/cli.py | 7 ++++++- tests/test_cli.py | 4 ++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 9044039..a122234 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -198,6 +198,12 @@ You can return every row in a specified table using the ``rows`` command:: This command accepts the same output options as ``query`` - so you can pass ``--nl``, ``--csv``, ``--tsv``, ``--no-headers``, ``--table`` and ``--fmt``. +You can use the ``-c`` option to specify a subset of columns to return:: + + $ sqlite-utils rows dogs.db dogs -c age -c name + [{"age": 4, "name": "Cleo"}, + {"age": 2, "name": "Pancakes"}] + .. _cli_tables: Listing tables diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index e75d514..b3df70a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1061,6 +1061,7 @@ def search( required=True, ) @click.argument("dbtable") +@click.option("-c", "--column", type=str, multiple=True, help="Columns to return") @output_options @load_extension_option @click.pass_context @@ -1068,6 +1069,7 @@ def rows( ctx, path, dbtable, + column, nl, arrays, csv, @@ -1079,10 +1081,13 @@ def rows( load_extension, ): "Output all rows in the specified table" + columns = "*" + if column: + columns = ", ".join("[{}]".format(c) for c in column) ctx.invoke( query, path=path, - sql="select * from [{}]".format(dbtable), + sql="select {} from [{}]".format(columns, dbtable), nl=nl, arrays=arrays, csv=csv, diff --git a/tests/test_cli.py b/tests/test_cli.py index 00efdf1..c82a342 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1039,6 +1039,10 @@ def test_query_memory_does_not_create_file(tmpdir): ), (["--arrays"], '[[1, "Cleo", 4],\n [2, "Pancakes", 2]]'), (["--arrays", "--nl"], '[1, "Cleo", 4]\n[2, "Pancakes", 2]'), + ( + ["--nl", "-c", "age", "-c", "name"], + '{"age": 4, "name": "Cleo"}\n{"age": 2, "name": "Pancakes"}', + ), ], ) def test_rows(db_path, args, expected): From 2bc1e9c5b42f5ecb076e5e966d3907853b9b9055 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 16:32:40 -0800 Subject: [PATCH 0258/1004] Added test for .search_sql() and FTS4, refs #197 --- tests/test_fts.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/test_fts.py b/tests/test_fts.py index 3c2180e..9592bad 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -366,10 +366,11 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): @pytest.mark.parametrize( - "kwargs,expected", + "kwargs,fts,expected", [ ( {}, + "FTS5", ( "with original as (\n" " select\n" @@ -391,6 +392,7 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): ), ( {"columns": ["title"], "order": "rowid", "limit": 10}, + "FTS5", ( "with original as (\n" " select\n" @@ -411,9 +413,31 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): "limit 10" ), ), + ( + {"columns": ["title"]}, + "FTS4", + ( + "with original as (\n" + " select\n" + " rowid,\n" + " [title]\n" + " from [books]\n" + ")\n" + "select\n" + " original.*,\n" + " rank_bm25(matchinfo([books_fts], 'pcnalx')) as rank\n" + "from\n" + " [original]\n" + " join [books_fts] on [original].rowid = [books_fts].rowid\n" + "where\n" + " [books_fts] match :query\n" + "order by\n" + " rank" + ), + ), ], ) -def test_search_sql(kwargs, expected): +def test_search_sql(kwargs, fts, expected): db = Database(memory=True) db["books"].insert( { @@ -421,6 +445,6 @@ def test_search_sql(kwargs, expected): "author": "Marlee Hawkins", } ) - db["books"].enable_fts(["title", "author"]) + db["books"].enable_fts(["title", "author"], fts_version=fts) sql = db["books"].search_sql(**kwargs) assert sql == expected From bce18721093489c0047792444b6f6eda5e8cc20b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 16:43:33 -0800 Subject: [PATCH 0259/1004] order= is now order_by=, refs #197 --- sqlite_utils/cli.py | 2 +- sqlite_utils/db.py | 17 ++++++++++++----- tests/test_fts.py | 4 ++-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index b3df70a..468e6b4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1033,7 +1033,7 @@ def search( raise click.ClickException( "Table '{}' has no column '{}".format(dbtable, c) ) - sql = table_obj.search_sql(columns=column, order=order, limit=limit) + sql = table_obj.search_sql(columns=column, order_by=order, limit=limit) if show_sql: click.echo(sql) return diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 8e586d8..40bcb5b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1321,7 +1321,7 @@ class Table(Queryable): ) return self - def search_sql(self, columns=None, order=None, limit=None): + def search_sql(self, columns=None, order_by=None, limit=None): # Pick names for table and rank column that don't clash original = "original_" if self.name == "original" else "original" rank = "rank" @@ -1352,7 +1352,7 @@ class Table(Queryable): where [{fts_table}] match :query order by - {order} + {order_by} {limit} """ ).strip() @@ -1370,12 +1370,19 @@ class Table(Queryable): rank=rank, rank_implementation=rank_implementation, fts_table=fts_table, - order=order or rank, + order_by=order_by or rank, limit="limit {}".format(limit) if limit else "", ).strip() - def search(self, q, order=None): - cursor = self.db.execute(self.search_sql(order=order), {"query": q}) + def search(self, q, order_by=None, columns=None, limit=None): + cursor = self.db.execute( + self.search_sql( + order_by=order_by, + columns=columns, + limit=limit, + ), + {"query": q}, + ) columns = [c[0] for c in cursor.description] for row in cursor: yield dict(zip(columns, row)) diff --git a/tests/test_fts.py b/tests/test_fts.py index 9592bad..8cb05e9 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -160,7 +160,7 @@ def test_fts_tokenize(fresh_db): fts_version="FTS{}".format(fts_version), tokenize="porter", ) - rows = list(table.search("bite", order="rowid")) + rows = list(table.search("bite", order_by="rowid")) assert len(rows) == 1 assert { "rowid": 2, @@ -391,7 +391,7 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): ), ), ( - {"columns": ["title"], "order": "rowid", "limit": 10}, + {"columns": ["title"], "order_by": "rowid", "limit": 10}, "FTS5", ( "with original as (\n" From cbc9841646761718095b6efba4fd1b4cd963090a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 16:58:57 -0800 Subject: [PATCH 0260/1004] Docs for .search() and .search_sql(), refs #197 --- docs/python-api.rst | 100 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 2740366..34dd226 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1451,7 +1451,9 @@ You can then run searches using the ``.search()`` method: .. code-block:: python - rows = db["dogs"].search("cleo") + rows = list(db["dogs"].search("cleo")) + +This method returns a generator that can be looped over to get dictionaries for each row, similar to :ref:`python_api_rows`. If you insert additional records into the table you will need to refresh the search index using ``populate_fts()``: @@ -1502,6 +1504,102 @@ To remove the FTS tables and triggers you created, use the ``disable_fts()`` tab db["dogs"].disable_fts() +.. _python_api_fts_search: + +Searching with table.search() +----------------------------- + +The ``table.search(q)`` method returns a generator over Python dictionaries representing rows that match the search phrase ``q``, ordered by relevance with the most relevant results first. + +.. code-block:: python + + for article in db["articles"].search("jquery"): + print(article) + +The ``.search()`` method also accepts the following optional parameters: + +``order_by`` string + The column to sort by. Defaults to relevance score. Can optionally include a ``desc``, e.g. ``rowid desc``. + +``columns`` array of strings + Columns to return. Defaults to all columns. + +``limit`` integer + Number of results to return. Defaults to all results. + +To return just the title and published columns for three matches for ``"dog"`` ordered by ``published`` with the most recent first, use the following: + +.. code-block:: python + + for article in db["articles"].search( + "dog", + order_by="published desc", + limit=3, + columns=["title", "published"] + ): + print(article) + +.. _python_api_fts_search_sql: + +Building SQL queries with table.search_sql() +-------------------------------------------- + +You can generate the SQL query that would be used for a search using the ``table.search_sql()`` method. It takes the same arguments as ``table.search()`` with the exception of the search query itself, since the returned SQL includes a parameter that can be used for the search. + +.. code-block:: python + + print(db["articles"].search_sql(columns=["title", "author"])) + +Outputs: + +.. code-block:: sql + + with original as ( + select + rowid, + [title], [author] + from [articles] + ) + select + original.*, + [articles_fts].rank as rank + from + [original] + join [articles_fts] on [original].rowid = [articles_fts].rowid + where + [articles_fts] match :query + order by + rank + +This method detects if a SQLite table uses FTS4 or FTS5, and outputs the correct SQL for ordering by relevance depending on the search type. + +The FTS4 output looks something like this: + +.. code-block:: sql + + with original as ( + select + rowid, + [title], [author] + from [articles] + ) + select + original.*, + rank_bm25(matchinfo([articles_fts], 'pcnalx')) as rank + from + [original] + join [articles_fts] on [original].rowid = [articles_fts].rowid + where + [articles_fts] match :query + order by + rank + +This uses the ``rank_bm25()`` custom SQL function from `sqlite-fts4 `__. You can register that custom function against a ``Database`` connection using this method: + +.. code-block:: python + + db.register_fts4_bm25() + .. _python_api_fts_rebuild: Rebuilding a full-text search table From 6785e89cc04e58382ea2bff34a4ee66ebe4c4434 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 17:22:24 -0800 Subject: [PATCH 0261/1004] Release 3.0a0 Refs #192 #193 #194 #196 #199 #198 #197 #200 --- docs/changelog.rst | 23 +++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 018d59c..5835f1c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,29 @@ Changelog =========== +.. _v3_0_a0: + +3.0a0 (2020-11-06) +------------------ + +This release introduces a new ``sqlite-utils search`` command for searching tables, see :ref:`cli_search`. (`#192 `__) + +The ``table.search()`` method has been redesigned, see :ref:`python_api_fts_search`. (`#197 `__) + +The release includes minor backwards-incompatible changes, hence the version bump to 3.0. Those changes, which should not affect most users, are: + +- The ``-c`` shortcut` option for outputting CSV is no longer available. The full ``--csv`` option is required instead. +- The ``-f`` shortcut for ``--fmt`` has also been removed - use ``--fmt``. +- The ``table.search()`` method now defaults to sorting by relevance, not sorting by ``rowid``. (`#198 `__) +- The ``table.search()`` method now returns a generator over a list of Python dictionaries. It previously returned a list of tuples. + +Also in this release: + +- The ``query``, ``tables``, ``rows`` and ``search`` CLI commands now accept a new ``--tsv`` option which outputs the results in TSV. (`#193 `__) +- A new ``table.virtual_table_using`` property reveals if a table is a virtual table, and returns the upper case type of virtual table (e.g. ``FTS4`` or ``FTS5``) if it is. It returns ``None`` if the table is not a virtual table. (`#196 `__) +- The new ``table.search_sql()`` method returns the SQL for searching a table, see :ref:`python_api_fts_search_sql`. +- ``sqlite-utils rows`` now accepts multiple optional ``-c`` parameters specifying the columns to return. (`#200 `__) + .. _v2_23: 2.23 (2020-10-28) diff --git a/setup.py b/setup.py index 9e1aea9..83c34d6 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "2.23" +VERSION = "3.0a0" def get_long_description(): From 60f4aff9b002009b76dd9386c84d4b80e73f6b0e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 18:23:56 -0800 Subject: [PATCH 0262/1004] Link changelog badge to /en/latest/changelog.html That way you can see the changelog for alpha releases. Refs #194. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f3a6e14..b706050 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # sqlite-utils [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/) -[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.readthedocs.io/en/stable/changelog.html) +[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.readthedocs.io/en/latest/changelog.html) [![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/) [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest) [![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.readthedocs.io/en/latest/?badge=latest) From a8acafbfe06640ff8aebe2af6338c9d01b76b85c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Nov 2020 18:49:24 -0800 Subject: [PATCH 0263/1004] Fixed RST typo --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5835f1c..a1ce981 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,7 +13,7 @@ The ``table.search()`` method has been redesigned, see :ref:`python_api_fts_sear The release includes minor backwards-incompatible changes, hence the version bump to 3.0. Those changes, which should not affect most users, are: -- The ``-c`` shortcut` option for outputting CSV is no longer available. The full ``--csv`` option is required instead. +- The ``-c`` shortcut option for outputting CSV is no longer available. The full ``--csv`` option is required instead. - The ``-f`` shortcut for ``--fmt`` has also been removed - use ``--fmt``. - The ``table.search()`` method now defaults to sorting by relevance, not sorting by ``rowid``. (`#198 `__) - The ``table.search()`` method now returns a generator over a list of Python dictionaries. It previously returned a list of tuples. From c5a798c15fa6ee694890c5f8e87e2c7a6001a4f4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Nov 2020 08:53:53 -0800 Subject: [PATCH 0264/1004] .search_sql() fully respects columns=, closes #201 Refs #192 and #197 --- sqlite_utils/db.py | 15 +++++++-------- tests/test_cli.py | 4 ++-- tests/test_fts.py | 21 +++++---------------- tests/test_tracer.py | 8 +++----- 4 files changed, 17 insertions(+), 31 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 40bcb5b..c94216e 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1324,12 +1324,13 @@ class Table(Queryable): def search_sql(self, columns=None, order_by=None, limit=None): # Pick names for table and rank column that don't clash original = "original_" if self.name == "original" else "original" - rank = "rank" - while rank in self.columns_dict: - rank = rank + "_" columns_sql = "*" + columns_with_prefix_sql = "[{}].*".format(original) if columns: columns_sql = ", ".join("[{}]".format(c) for c in columns) + columns_with_prefix_sql = ", ".join( + "[{}].[{}]".format(original, c) for c in columns + ) fts_table = self.detect_fts() assert fts_table, "Full-text search is not configured for table '{}'".format( self.name @@ -1344,8 +1345,7 @@ class Table(Queryable): from [{dbtable}] ) select - {original}.*, - {rank_implementation} as {rank} + {columns_with_prefix} from [{original}] join [{fts_table}] on [{original}].rowid = [{fts_table}].rowid @@ -1367,10 +1367,9 @@ class Table(Queryable): dbtable=self.name, original=original, columns=columns_sql, - rank=rank, - rank_implementation=rank_implementation, + columns_with_prefix=columns_with_prefix_sql, fts_table=fts_table, - order_by=order_by or rank, + order_by=order_by or rank_implementation, limit="limit {}".format(limit) if limit else "", ).strip() diff --git a/tests/test_cli.py b/tests/test_cli.py index c82a342..0179b50 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1709,9 +1709,9 @@ def test_insert_encoding(tmpdir): [ ( None, - '[{"rowid": 2, "id": 2, "title": "Title the second", "rank": -0.5108256237659907}]\n', + '[{"rowid": 2, "id": 2, "title": "Title the second"}]\n', ), - ("--csv", "rowid,id,title,rank\n2,2,Title the second,-0.5108256237659907\n"), + ("--csv", "rowid,id,title\n2,2,Title the second\n"), ], ) def test_search(tmpdir, fts, extra_arg, expected): diff --git a/tests/test_fts.py b/tests/test_fts.py index 8cb05e9..06fac20 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -35,7 +35,6 @@ def test_enable_fts(fresh_db): "text": "tanuki are running tricksters", "country": "Japan", "not_searchable": "foo", - "rank": 0.0, } ] == list(table.search("tanuki")) assert [ @@ -44,7 +43,6 @@ def test_enable_fts(fresh_db): "text": "racoons are biting trash pandas", "country": "USA", "not_searchable": "bar", - "rank": 0.0, } ] == list(table.search("usa")) assert [] == list(table.search("bar")) @@ -71,7 +69,6 @@ def test_enable_fts_escape_table_names(fresh_db): "text": "tanuki are running tricksters", "country": "Japan", "not_searchable": "foo", - "rank": 0.0, } ] == list(table.search("tanuki")) assert [ @@ -80,7 +77,6 @@ def test_enable_fts_escape_table_names(fresh_db): "text": "racoons are biting trash pandas", "country": "USA", "not_searchable": "bar", - "rank": 0.0, } ] == list(table.search("usa")) assert [] == list(table.search("bar")) @@ -116,7 +112,6 @@ def test_populate_fts(fresh_db): "text": "racoons are biting trash pandas", "country": "USA", "not_searchable": "bar", - "rank": -0.5108256237659907, } ] == rows @@ -137,7 +132,6 @@ def test_populate_fts_escape_table_names(fresh_db): "text": "racoons are biting trash pandas", "country": "USA", "not_searchable": "bar", - "rank": -0.5108256237659907, } ] == list(table.search("usa")) @@ -198,7 +192,6 @@ def test_enable_fts_with_triggers(fresh_db): "text": "tanuki are running tricksters", "country": "Japan", "not_searchable": "foo", - "rank": 0.0, } ] table.insert(search_records[1]) @@ -210,7 +203,6 @@ def test_enable_fts_with_triggers(fresh_db): "text": "racoons are biting trash pandas", "country": "USA", "not_searchable": "bar", - "rank": 0.0, } ] assert [] == list(table.search("bar")) @@ -379,15 +371,14 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): " from [books]\n" ")\n" "select\n" - " original.*,\n" - " [books_fts].rank as rank\n" + " [original].*\n" "from\n" " [original]\n" " join [books_fts] on [original].rowid = [books_fts].rowid\n" "where\n" " [books_fts] match :query\n" "order by\n" - " rank" + " [books_fts].rank" ), ), ( @@ -401,8 +392,7 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): " from [books]\n" ")\n" "select\n" - " original.*,\n" - " [books_fts].rank as rank\n" + " [original].[title]\n" "from\n" " [original]\n" " join [books_fts] on [original].rowid = [books_fts].rowid\n" @@ -424,15 +414,14 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): " from [books]\n" ")\n" "select\n" - " original.*,\n" - " rank_bm25(matchinfo([books_fts], 'pcnalx')) as rank\n" + " [original].[title]\n" "from\n" " [original]\n" " join [books_fts] on [original].rowid = [books_fts].rowid\n" "where\n" " [books_fts] match :query\n" "order by\n" - " rank" + " rank_bm25(matchinfo([books_fts], 'pcnalx'))" ), ), ], diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 318d45e..094551a 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -44,11 +44,9 @@ def test_with_tracer(): with db.tracer(tracer): list(db["dogs"].search("Cleopaws")) - assert len(collected) == 7 + assert len(collected) == 5 assert collected == [ ("select name from sqlite_master where type = 'view'", None), - ("select name from sqlite_master where type = 'table'", None), - ("PRAGMA table_info([dogs])", None), ( "SELECT name FROM sqlite_master\n WHERE rootpage = 0\n AND (\n sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n OR (\n tbl_name = \"dogs\"\n AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n )\n )", None, @@ -56,11 +54,11 @@ def test_with_tracer(): ("select name from sqlite_master where type = 'view'", None), ("select sql from sqlite_master where name = ?", ("dogs_fts",)), ( - "with original as (\n select\n rowid,\n *\n from [dogs]\n)\nselect\n original.*,\n [dogs_fts].rank as rank\nfrom\n [original]\n join [dogs_fts] on [original].rowid = [dogs_fts].rowid\nwhere\n [dogs_fts] match :query\norder by\n rank", + "with original as (\n select\n rowid,\n *\n from [dogs]\n)\nselect\n [original].*\nfrom\n [original]\n join [dogs_fts] on [original].rowid = [dogs_fts].rowid\nwhere\n [dogs_fts] match :query\norder by\n [dogs_fts].rank", {"query": "Cleopaws"}, ), ] # Outside the with block collected should not be appended to db["dogs"].insert({"name": "Cleopaws"}) - assert len(collected) == 7 + assert len(collected) == 5 From 79109939c39c16fd206010ef179040d59704682b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Nov 2020 09:00:43 -0800 Subject: [PATCH 0265/1004] search --limit now defaults to everything, refs #192 --- docs/cli.rst | 5 ++--- sqlite_utils/cli.py | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index a122234..4e9710d 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -933,15 +933,14 @@ Use the ``--sql`` option to output the SQL that would be executed, rather than r from [documents] ) select - original.*, - [documents_fts].rank as rank + [original].* from [original] join [documents_fts] on [original].rowid = [documents_fts].rowid where [documents_fts] match :query order by - rank + [documents_fts].rank .. _cli_vacuum: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 468e6b4..cfc57c6 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -986,8 +986,7 @@ def query( @click.option( "--limit", type=int, - default=20, - help="Number of rows to return, default 20, set to 0 for unlimited", + help="Number of rows to return - defaults to everything", ) @click.option( "--sql", "show_sql", is_flag=True, help="Show SQL query that would be run" From ce2b07c358cd68e6de4c8942d7067591b7f1be96 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Nov 2020 09:04:33 -0800 Subject: [PATCH 0266/1004] Updated docs for .search_sql() method Also improved indentation of generated SQL queries. Refs #197 --- docs/python-api.rst | 18 ++++++++++-------- sqlite_utils/db.py | 4 ++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 34dd226..8dead7d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1557,19 +1557,20 @@ Outputs: with original as ( select rowid, - [title], [author] + [title], + [author] from [articles] ) select - original.*, - [articles_fts].rank as rank + [original].[title], + [original].[author] from [original] join [articles_fts] on [original].rowid = [articles_fts].rowid where [articles_fts] match :query order by - rank + [articles_fts].rank This method detects if a SQLite table uses FTS4 or FTS5, and outputs the correct SQL for ordering by relevance depending on the search type. @@ -1580,19 +1581,20 @@ The FTS4 output looks something like this: with original as ( select rowid, - [title], [author] + [title], + [author] from [articles] ) select - original.*, - rank_bm25(matchinfo([articles_fts], 'pcnalx')) as rank + [original].[title], + [original].[author] from [original] join [articles_fts] on [original].rowid = [articles_fts].rowid where [articles_fts] match :query order by - rank + rank_bm25(matchinfo([articles_fts], 'pcnalx')) This uses the ``rank_bm25()`` custom SQL function from `sqlite-fts4 `__. You can register that custom function against a ``Database`` connection using this method: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c94216e..c91ef82 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1327,8 +1327,8 @@ class Table(Queryable): columns_sql = "*" columns_with_prefix_sql = "[{}].*".format(original) if columns: - columns_sql = ", ".join("[{}]".format(c) for c in columns) - columns_with_prefix_sql = ", ".join( + columns_sql = ",\n ".join("[{}]".format(c) for c in columns) + columns_with_prefix_sql = ",\n ".join( "[{}].[{}]".format(original, c) for c in columns ) fts_table = self.detect_fts() From 47abca07643da36d0bcc589332826eeb092808be Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Nov 2020 09:16:25 -0800 Subject: [PATCH 0267/1004] Update README for 3.0 release, refs #194 --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b706050..618edf6 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,13 @@ Python CLI utility and library for manipulating SQLite databases. +## Some feature highlights + +- [Pipe JSON](https://sqlite-utils.readthedocs.io/en/stable/cli.html#inserting-json-data) (or [CSV or TSV](https://sqlite-utils.readthedocs.io/en/stable/cli.html#inserting-csv-or-tsv-data)) directly into a new SQLite database file, automatically creating a table with the appropriate schema +- [Configure SQLite full-text search](https://sqlite-utils.readthedocs.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance +- Run [transformations against your tables](https://sqlite-utils.readthedocs.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as dropping columns +- [Extract columns](https://sqlite-utils.readthedocs.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data + Read more on my blog: [ sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) and other [entries tagged sqliteutils](https://simonwillison.net/tags/sqliteutils/). @@ -43,7 +50,7 @@ You can even import data into a new database table like this: $ curl https://api.github.com/repos/simonw/sqlite-utils/releases \ | sqlite-utils insert releases.db releases - --pk id -Full CLI documentation: https://sqlite-utils.readthedocs.io/en/stable/cli.html +See the [full CLI documentation](https://sqlite-utils.readthedocs.io/en/stable/cli.html) for comprehensive coverage of many more commands. ## Using as a library @@ -59,7 +66,7 @@ db["dogs"].insert_all([ ], pk="id") ``` -Full library documentation: https://sqlite-utils.readthedocs.io/en/stable/python-api.html +Check out the [full library documentation](https://sqlite-utils.readthedocs.io/en/stable/python-api.html) for everything else you can do with the Python library. ## Related projects From 68637732e011afb699a0724854efea524b9b239c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Nov 2020 09:19:20 -0800 Subject: [PATCH 0268/1004] Release 3.0 Refs #192 #193 #196 #199 #198 #197 #200 #201 Closes #194 --- docs/changelog.rst | 6 +++--- setup.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a1ce981..8923ed6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,10 +2,10 @@ Changelog =========== -.. _v3_0_a0: +.. _v3_0: -3.0a0 (2020-11-06) ------------------- +3.0 (2020-11-08) +---------------- This release introduces a new ``sqlite-utils search`` command for searching tables, see :ref:`cli_search`. (`#192 `__) diff --git a/setup.py b/setup.py index 83c34d6..9d80bbd 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.0a0" +VERSION = "3.0" def get_long_description(): From 60d3c4821be4cf25c41097c1e8b79b2e60c5ead5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Nov 2020 09:23:38 -0800 Subject: [PATCH 0269/1004] Changes since the 3.0a0 alpha release --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8923ed6..76b6b5d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,11 @@ Also in this release: - The new ``table.search_sql()`` method returns the SQL for searching a table, see :ref:`python_api_fts_search_sql`. - ``sqlite-utils rows`` now accepts multiple optional ``-c`` parameters specifying the columns to return. (`#200 `__) +Changes since the 3.0a0 alpha release: + +- The ``sqlite-utils search`` command now defaults to returning every result, unless you add a ``--limit 20`` option. +- The ``sqlite-utils search -c`` and ``table.search(columns=[])`` options are now fully respected. (`#201 `__) + .. _v2_23: 2.23 (2020-10-28) From c5f4f0f70ce394dfec6054c3c5aaedf330887093 Mon Sep 17 00:00:00 2001 From: Andreas Madsack Date: Tue, 8 Dec 2020 18:49:42 +0100 Subject: [PATCH 0270/1004] Use jsonify_if_need for sql updates (#204) * add failing tests for update with json values * use jsonify_if_needed in for sql updates Thanks, @mfa --- sqlite_utils/db.py | 2 +- tests/test_update.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c91ef82..8849680 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1426,7 +1426,7 @@ class Table(Queryable): validate_column_names(updates.keys()) for key, value in updates.items(): sets.append("[{}] = {}".format(key, conversions.get(key, "?"))) - args.append(value) + args.append(jsonify_if_needed(value)) wheres = ["[{}] = ?".format(pk_name) for pk_name in pks] args.extend(pk_values) sql = "update [{table}] set {sets} where {wheres}".format( diff --git a/tests/test_update.py b/tests/test_update.py index 3f4c4ba..7e3e0ab 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -1,6 +1,10 @@ -from sqlite_utils.db import NotFoundError +import collections +import json + import pytest +from sqlite_utils.db import NotFoundError + def test_update_rowid_table(fresh_db): table = fresh_db["table"] @@ -82,3 +86,27 @@ def test_update_with_no_values_sets_last_pk(fresh_db): assert 2 == table.last_pk with pytest.raises(NotFoundError): table.update(3) + + +@pytest.mark.parametrize( + "data_structure", + ( + ["list with one item"], + ["list with", "two items"], + {"dictionary": "simple"}, + {"dictionary": {"nested": "complex"}}, + collections.OrderedDict( + [ + ("key1", {"nested": "complex"}), + ("key2", "foo"), + ] + ), + [{"list": "of"}, {"two": "dicts"}], + ), +) +def test_update_dictionaries_and_lists_as_json(fresh_db, data_structure): + fresh_db["test"].insert({"id": 1, "data": ""}, pk="id") + fresh_db["test"].update(1, {"data": data_structure}) + row = fresh_db.execute("select id, data from test").fetchone() + assert row[0] == 1 + assert data_structure == json.loads(row[1]) From 69a121e08847acbf95abf0c2df1759fc73dc81b8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 23:20:11 -0800 Subject: [PATCH 0271/1004] sqlite-utils analyze-tables command and table.analyze_column() method Closes #207 --- docs/cli.rst | 87 ++++++++++++++++++ docs/python-api.rst | 35 ++++++++ sqlite_utils/cli.py | 88 ++++++++++++++++++ sqlite_utils/db.py | 79 +++++++++++++++++ tests/test_analyze_tables.py | 167 +++++++++++++++++++++++++++++++++++ 5 files changed, 456 insertions(+) create mode 100644 tests/test_analyze_tables.py diff --git a/docs/cli.rst b/docs/cli.rst index 4e9710d..68ca429 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -279,6 +279,93 @@ It takes the same options as the ``tables`` command: * ``--tsv`` * ``--table`` +.. _cli_analyze_tables: + +Analyzing tables +================ + +When working with a new database it can be useful to get an idea of the shape of the data. The ``sqlite-utils analyze-tables`` command inspects specified tables (or all tables) and calculates some useful details about each of the columns in those tables. + +To inspect the ``tags`` table in the ``github.db`` database, run the following:: + + $ sqlite-utils analyze-tables github.db tags + tags.repo: (1/3) + + Total rows: 261 + Null rows: 0 + Blank rows: 0 + + Distinct values: 14 + + Most common: + 88: 107914493 + 75: 140912432 + 27: 206156866 + + Least common: + 1: 209590345 + 2: 206649770 + 2: 303218369 + + tags.name: (2/3) + + Total rows: 261 + Null rows: 0 + Blank rows: 0 + + Distinct values: 175 + + Most common: + 10: 0.2 + 9: 0.1 + 7: 0.3 + + Least common: + 1: 0.1.1 + 1: 0.11.1 + 1: 0.1a2 + + tags.sha: (3/3) + + Total rows: 261 + Null rows: 0 + Blank rows: 0 + + Distinct values: 261 + +For each column this tool dispalys the number of null rows, the number of blank rows (rows that contain an empty string), the number of distinct values and, for columns that are not entirely distinct, the most common and least common values. + +If you do not specify any tables every table in the database will be analyzed:: + + $ sqlite-utils analyze-tables github.db + +If you wish to analyze one or more specific columns, use the ``-c`` option:: + + $ sqlite-utils analyze-tables github.db tags -c sha + +.. _cli_analyze_tables_save: + +Saving the analyzed table details +--------------------------------- + +``analyze-tables`` can take quite a while to run for large database files. You can save the results of the analysis to a database table called ``_analyze_tables_`` using the ``--save`` option:: + + $ sqlite-utils analyze-tables github.db --save + +The ``_analyze_tables_`` table has the following schema:: + + CREATE TABLE [_analyze_tables_] ( + [table] TEXT, + [column] TEXT, + [total_rows] INTEGER, + [num_null] INTEGER, + [num_blank] INTEGER, + [num_distinct] INTEGER, + [most_common] TEXT, + [least_common] TEXT, + PRIMARY KEY ([table], [column]) + ); + .. _cli_inserting_data: Inserting JSON data diff --git a/docs/python-api.rst b/docs/python-api.rst index 8dead7d..130c4cb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -747,6 +747,41 @@ You can inspect the database to see the results like this:: PRIMARY KEY ([characteristics_id], [dogs_id]) ) +.. _python_api_analyze_column: + +Analyzing a column +================== + +The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` method is used by the :ref:`analyze-tables ` CLI command. It returns a ``ColumnDetails`` named tuple with the following fields: + +``table`` + The name of the table + +``column`` + The name of the column + +``total_rows`` + The total number of rows in the table` + +``num_null`` + The number of rows for which this column is null + +``num_blank`` + The number of rows for which this column is blank (the empty string) + +``num_distinct`` + The number of distinct values in this column + +``most_common`` + The ``N`` most common values as a list of ``(value, count)`` tuples`, or ``None`` if the table consists entirely of distinct values + +``least_common`` + The ``N`` least common values as a list of ``(value, count)`` tuples`, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) + +``N`` defaults to 10, or you can pass a custom ``N`` using the ``common_limit`` parameter. + +You can use the ``value_truncate`` parameter to truncate values in the ``most_common`` and ``least_common`` lists to a specified number of characters. + .. _python_api_add_column: Adding columns diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cfc57c6..6312327 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -7,6 +7,7 @@ import hashlib import pathlib import sqlite_utils from sqlite_utils.db import AlterError +import textwrap import itertools import json import os @@ -1354,6 +1355,93 @@ def insert_files( ) +@cli.command(name="analyze-tables") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False, exists=True), + required=True, +) +@click.argument("tables", nargs=-1) +@click.option( + "-c", + "--column", + "columns", + type=str, + multiple=True, + help="Specific columns to analyze", +) +@click.option("--save", is_flag=True, help="Save results to _analyze_tables table") +@load_extension_option +def analyze_tables( + path, + tables, + columns, + save, + load_extension, +): + "Analyze the columns in one or more tables" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + if not tables: + tables = db.table_names() + todo = [] + table_counts = {} + for table in tables: + table_counts[table] = db[table].count + for column in db[table].columns: + if not columns or column.name in columns: + todo.append((table, column.name)) + # Now we now how many we need to do + for i, (table, column) in enumerate(todo): + column_details = db[table].analyze_column( + column, total_rows=table_counts[table], value_truncate=80 + ) + if save: + db["_analyze_tables_"].insert( + column_details._asdict(), pk=("table", "column"), replace=True + ) + most_common_rendered = _render_common( + "\n\n Most common:", column_details.most_common + ) + least_common_rendered = _render_common( + "\n\n Least common:", column_details.least_common + ) + details = ( + ( + textwrap.dedent( + """ + {table}.{column}: ({i}/{total}) + + Total rows: {total_rows} + Null rows: {num_null} + Blank rows: {num_blank} + + Distinct values: {num_distinct}{most_common_rendered}{least_common_rendered} + """ + ) + .strip() + .format( + i=i + 1, + total=len(todo), + most_common_rendered=most_common_rendered, + least_common_rendered=least_common_rendered, + **column_details._asdict() + ) + ) + + "\n" + ) + click.echo(details) + + +def _render_common(title, values): + if values is None: + return "" + lines = [title] + for value, count in values: + lines.append(" {}: {}".format(count, value)) + return "\n".join(lines) + + FILE_COLUMNS = { "name": lambda p: p.name, "path": lambda p: str(p), diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 8849680..90427bf 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -51,6 +51,19 @@ except ImportError: Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") ) +ColumnDetails = namedtuple( + "ColumnDetails", + ( + "table", + "column", + "total_rows", + "num_null", + "num_blank", + "num_distinct", + "most_common", + "least_common", + ), +) ForeignKey = namedtuple( "ForeignKey", ("table", "column", "other_table", "other_column") ) @@ -1930,6 +1943,72 @@ class Table(Queryable): ) return self + def analyze_column( + self, column, common_limit=10, value_truncate=None, total_rows=None + ): + db = self.db + table = self.name + if total_rows is None: + total_rows = db[table].count + + def truncate(value): + if value_truncate is None or isinstance(value, (float, int)): + return value + value = str(value) + if len(value) > value_truncate: + value = value[:value_truncate] + "..." + return value + + num_null = db.execute( + "select count(*) from [{}] where [{}] is null".format(table, column) + ).fetchone()[0] + num_blank = db.execute( + "select count(*) from [{}] where [{}] = ''".format(table, column) + ).fetchone()[0] + num_distinct = db.execute( + "select count(distinct [{}]) from [{}]".format(column, table) + ).fetchone()[0] + most_common = None + least_common = None + if num_distinct == 1: + value = db.execute( + "select [{}] from [{}] limit 1".format(column, table) + ).fetchone()[0] + most_common = [(truncate(value), total_rows)] + elif num_distinct != total_rows: + most_common = [ + (truncate(r[0]), r[1]) + for r in db.execute( + "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format( + column, table, column, column, common_limit + ) + ).fetchall() + ] + most_common.sort(key=lambda p: (p[1], p[0]), reverse=True) + if num_distinct <= common_limit: + # No need to run the query if it will just return the results in revers order + least_common = None + else: + least_common = [ + (truncate(r[0]), r[1]) + for r in db.execute( + "select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format( + column, table, column, column, common_limit + ) + ).fetchall() + ] + least_common.sort(key=lambda p: (p[1], p[0])) + return ColumnDetails( + self.name, + column, + total_rows, + num_null, + num_blank, + num_distinct, + most_common, + least_common, + ) + class View(Queryable): def exists(self): diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py new file mode 100644 index 0000000..f0af2ca --- /dev/null +++ b/tests/test_analyze_tables.py @@ -0,0 +1,167 @@ +from sqlite_utils.db import Database, ForeignKey, ColumnDetails +from sqlite_utils import cli +from sqlite_utils.utils import OperationalError +from click.testing import CliRunner +import pytest +import sqlite3 +import textwrap + + +@pytest.fixture +def db_to_analyze(fresh_db): + stuff = fresh_db["stuff"] + stuff.insert_all( + [ + {"id": 1, "owner": "Terryterryterry", "size": 5}, + {"id": 2, "owner": "Joan", "size": 4}, + {"id": 3, "owner": "Kumar", "size": 5}, + {"id": 4, "owner": "Anne", "size": 5}, + {"id": 5, "owner": "Terryterryterry", "size": 5}, + {"id": 6, "owner": "Joan", "size": 4}, + {"id": 7, "owner": "Kumar", "size": 5}, + {"id": 8, "owner": "Joan", "size": 4}, + ], + pk="id", + ) + return fresh_db + + +@pytest.mark.parametrize( + "column,expected", + [ + ( + "id", + ColumnDetails( + table="stuff", + column="id", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=8, + most_common=None, + least_common=None, + ), + ), + ( + "owner", + ColumnDetails( + table="stuff", + column="owner", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=4, + most_common=[("Joan", 3), ("Kumar", 2)], + least_common=[("Anne", 1), ("Terry...", 2)], + ), + ), + ( + "size", + ColumnDetails( + table="stuff", + column="size", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=2, + most_common=[(5, 5), (4, 3)], + least_common=None, + ), + ), + ], +) +def test_analyze_column(db_to_analyze, column, expected): + assert ( + db_to_analyze["stuff"].analyze_column(column, common_limit=2, value_truncate=5) + == expected + ) + + +@pytest.fixture +def db_to_analyze_path(db_to_analyze, tmpdir): + path = str(tmpdir / "test.db") + db = sqlite3.connect(path) + db.executescript("\n".join(db_to_analyze.conn.iterdump())) + return path + + +def test_analyze_table(db_to_analyze_path): + result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path]) + assert ( + result.output.strip() + == ( + """ +stuff.id: (1/3) + + Total rows: 8 + Null rows: 0 + Blank rows: 0 + + Distinct values: 8 + +stuff.owner: (2/3) + + Total rows: 8 + Null rows: 0 + Blank rows: 0 + + Distinct values: 4 + + Most common: + 3: Joan + 2: Terryterryterry + 2: Kumar + 1: Anne + +stuff.size: (3/3) + + Total rows: 8 + Null rows: 0 + Blank rows: 0 + + Distinct values: 2 + + Most common: + 5: 5 + 3: 4""" + ).strip() + ) + + +def test_analyze_table_save(db_to_analyze_path): + result = CliRunner().invoke( + cli.cli, ["analyze-tables", db_to_analyze_path, "--save"] + ) + rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows) + assert rows == [ + { + "table": "stuff", + "column": "id", + "total_rows": 8, + "num_null": 0, + "num_blank": 0, + "num_distinct": 8, + "most_common": None, + "least_common": None, + }, + { + "table": "stuff", + "column": "owner", + "total_rows": 8, + "num_null": 0, + "num_blank": 0, + "num_distinct": 4, + "most_common": '[["Joan", 3], ["Terryterryterry", 2], ["Kumar", 2], ["Anne", 1]]', + "least_common": None, + }, + { + "table": "stuff", + "column": "size", + "total_rows": 8, + "num_null": 0, + "num_blank": 0, + "num_distinct": 2, + "most_common": "[[5, 5], [4, 3]]", + "least_common": None, + }, + ] From 1ce96d8c0854476e84216af5e5af71bcebbddb78 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 23:28:20 -0800 Subject: [PATCH 0272/1004] Release 3.1 Refs #204, #207, #208 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 76b6b5d..b0c3da1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v3_1: + +3.1 (2020-12-12) +---------------- + +- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (`#207 `__) +- New ``table.analyze_column(column)`` Python method used by the ``analyze-tables`` command - see :ref:`python_api_analyze_column`. +- The ``table.update()`` method now correctly handles values that should be stored as JSON. Thanks, Andreas Madsack. (`#204 `__) + .. _v3_0: 3.0 (2020-11-08) diff --git a/setup.py b/setup.py index 9d80bbd..a507a1b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.0" +VERSION = "3.1" def get_long_description(): From 5e06026e76cdda4ffdf89b7369b0e50be398d8fe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 13 Dec 2020 16:19:51 -0800 Subject: [PATCH 0273/1004] Typo fix --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 68ca429..bf67a11 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -333,7 +333,7 @@ To inspect the ``tags`` table in the ``github.db`` database, run the following:: Distinct values: 261 -For each column this tool dispalys the number of null rows, the number of blank rows (rows that contain an empty string), the number of distinct values and, for columns that are not entirely distinct, the most common and least common values. +For each column this tool displays the number of null rows, the number of blank rows (rows that contain an empty string), the number of distinct values and, for columns that are not entirely distinct, the most common and least common values. If you do not specify any tables every table in the database will be analyzed:: From f1277f638f3a54a821db6e03cb980adad2f2fa35 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 13 Dec 2020 20:52:24 -0800 Subject: [PATCH 0274/1004] Added Homebrew installation instructions --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 618edf6..85d9b4a 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ sqlite-utils: a Python library and CLI tool for building SQLite databases](https pip install sqlite-utils +Or if you use [Homebrew](https://brew.sh/) for macOS: + + brew install sqlite-utils + ## Using as a CLI tool Now you can do things with the CLI utility like this: From 8ca35dfb645e4c0ca5d528b496835b209f1c8414 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Dec 2020 13:33:25 -0800 Subject: [PATCH 0275/1004] Link to new datasette.io website --- README.md | 2 +- docs/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 85d9b4a..4e961fc 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Check out the [full library documentation](https://sqlite-utils.readthedocs.io/e ## Related projects -* [Datasette](https://github.com/simonw/datasette): A tool for exploring and publishing data +* [Datasette](https://datasette.io/): A tool for exploring and publishing data * [csvs-to-sqlite](https://github.com/simonw/csvs-to-sqlite): Convert CSV files into a SQLite database * [db-to-sqlite](https://github.com/simonw/db-to-sqlite): CLI tool for exporting a MySQL or PostgreSQL database as a SQLite file * [dogsheep](https://dogsheep.github.io/): A family of tools for personal analytics, built on top of `sqlite-utils` diff --git a/docs/index.rst b/docs/index.rst index 6fe980f..528dbae 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,7 +21,7 @@ Most of the functionality is available as either a Python API or through the ``s sqlite-utils is not intended to be a full ORM: the focus is utility helpers to make creating the initial database and populating it with data as productive as possible. -It is designed as a useful complement to `Datasette `_. +It is designed as a useful complement to `Datasette `_. Contents -------- From 6a34426b12e85a34f1f36bd1113e21077d1f2877 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Dec 2020 13:34:55 -0800 Subject: [PATCH 0276/1004] Docs now live at sqlite-utils.datasette.io --- README.md | 16 ++++++++-------- docs/cli.rst | 2 +- docs/index.rst | 2 +- docs/python-api.rst | 2 +- setup.py | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 4e961fc..fa78d9d 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,20 @@ # sqlite-utils [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/) -[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.readthedocs.io/en/latest/changelog.html) +[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/latest/changelog.html) [![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/) [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest) -[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.readthedocs.io/en/latest/?badge=latest) +[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.datasette.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE) Python CLI utility and library for manipulating SQLite databases. ## Some feature highlights -- [Pipe JSON](https://sqlite-utils.readthedocs.io/en/stable/cli.html#inserting-json-data) (or [CSV or TSV](https://sqlite-utils.readthedocs.io/en/stable/cli.html#inserting-csv-or-tsv-data)) directly into a new SQLite database file, automatically creating a table with the appropriate schema -- [Configure SQLite full-text search](https://sqlite-utils.readthedocs.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance -- Run [transformations against your tables](https://sqlite-utils.readthedocs.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as dropping columns -- [Extract columns](https://sqlite-utils.readthedocs.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data +- [Pipe JSON](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-json-data) (or [CSV or TSV](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-csv-or-tsv-data)) directly into a new SQLite database file, automatically creating a table with the appropriate schema +- [Configure SQLite full-text search](https://sqlite-utils.datasette.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance +- Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as dropping columns +- [Extract columns](https://sqlite-utils.datasette.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data Read more on my blog: [ sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) and other [entries tagged sqliteutils](https://simonwillison.net/tags/sqliteutils/). @@ -54,7 +54,7 @@ You can even import data into a new database table like this: $ curl https://api.github.com/repos/simonw/sqlite-utils/releases \ | sqlite-utils insert releases.db releases - --pk id -See the [full CLI documentation](https://sqlite-utils.readthedocs.io/en/stable/cli.html) for comprehensive coverage of many more commands. +See the [full CLI documentation](https://sqlite-utils.datasette.io/en/stable/cli.html) for comprehensive coverage of many more commands. ## Using as a library @@ -70,7 +70,7 @@ db["dogs"].insert_all([ ], pk="id") ``` -Check out the [full library documentation](https://sqlite-utils.readthedocs.io/en/stable/python-api.html) for everything else you can do with the Python library. +Check out the [full library documentation](https://sqlite-utils.datasette.io/en/stable/python-api.html) for everything else you can do with the Python library. ## Related projects diff --git a/docs/cli.rst b/docs/cli.rst index bf67a11..445cef2 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -427,7 +427,7 @@ You can delete all the existing rows in the table before inserting the new recor $ sqlite-utils insert dogs.db dogs dogs.json --truncate -You can also import newline-delimited JSON using the ``--nl`` option. Since `Datasette `__ can export newline-delimited JSON, you can combine the two tools like so:: +You can also import newline-delimited JSON using the ``--nl`` option. Since `Datasette `__ can export newline-delimited JSON, you can combine the two tools like so:: $ curl -L "https://latest.datasette.io/fixtures/facetable.json?_shape=array&_nl=on" \ | sqlite-utils insert nl-demo.db facetable - --pk=id --nl diff --git a/docs/index.rst b/docs/index.rst index 528dbae..571020f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ .. |PyPI| image:: https://img.shields.io/pypi/v/sqlite-utils.svg :target: https://pypi.org/project/sqlite-utils/ .. |Changelog| image:: https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog - :target: https://sqlite-utils.readthedocs.io/en/stable/changelog.html + :target: https://sqlite-utils.datasette.io/en/stable/changelog.html .. |CI| image:: https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg :target: https://github.com/simonw/sqlite-utils/actions .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg diff --git a/docs/python-api.rst b/docs/python-api.rst index 130c4cb..ac7a830 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -349,7 +349,7 @@ Specifying foreign keys Any operation that can create a table (``.create()``, ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()``) accepts an optional ``foreign_keys=`` argument which can be used to set up foreign key constraints for the table that is being created. -If you are using your database with `Datasette `__, Datasette will detect these constraints and use them to generate hyperlinks to associated records. +If you are using your database with `Datasette `__, Datasette will detect these constraints and use them to generate hyperlinks to associated records. The ``foreign_keys`` argument takes a list that indicates which foreign keys should be created. The list can take several forms. The simplest is a list of columns: diff --git a/setup.py b/setup.py index a507a1b..2a17384 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,8 @@ setup( tests_require=["sqlite-utils[test]"], url="https://github.com/simonw/sqlite-utils", project_urls={ - "Documentation": "https://sqlite-utils.readthedocs.io/en/stable/", - "Changelog": "https://sqlite-utils.readthedocs.io/en/stable/changelog.html", + "Documentation": "https://sqlite-utils.datasette.io/en/stable/", + "Changelog": "https://sqlite-utils.datasette.io/en/stable/changelog.html", "Source code": "https://github.com/simonw/sqlite-utils", "Issues": "https://github.com/simonw/sqlite-utils/issues", "CI": "https://github.com/simonw/sqlite-utils/actions", From 33c759a74c3d95b5356eade1f7f592f48c6416a5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Jan 2021 15:52:36 -0800 Subject: [PATCH 0277/1004] Test now tolerates optimize producing larger DB, closes #209 --- tests/test_cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 0179b50..c335c46 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -496,7 +496,10 @@ def test_optimize(db_path, tables): result = CliRunner().invoke(cli.cli, ["optimize", db_path] + tables) assert 0 == result.exit_code size_after_optimize = os.stat(db_path).st_size - assert size_after_optimize < size_before_optimize + # Weirdest thing: tests started failing because size after + # ended up larger than size before in some cases. I think + # it's OK to tolerate that happening, though it's very strange. + assert size_after_optimize <= (size_before_optimize + 10000) # Soundness check that --no-vacuum doesn't throw errors: result = CliRunner().invoke(cli.cli, ["optimize", "--no-vacuum", db_path]) assert 0 == result.exit_code From 0dca784dbe6b75de6e4c0da4869c0b2b9574dde4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Jan 2021 15:56:20 -0800 Subject: [PATCH 0278/1004] Release 3.1.1 Refs #209. Also updated copyright footer in documentation. --- docs/changelog.rst | 9 +++++++++ docs/conf.py | 2 +- setup.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b0c3da1..8388066 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v3_1_1: + +3.1.1 (2021-01-01) +------------------ + +- Fixed failing test caused by ``optimize`` sometimes creating larger database files. (`#209 `__) +- Documentation now lives on https://sqlite-utils.datasette.io/ +- README now includes ``brew install sqlite-utils`` installation method. + .. _v3_1: 3.1 (2020-12-12) diff --git a/docs/conf.py b/docs/conf.py index 8429616..929a41e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -46,7 +46,7 @@ master_doc = "index" # General information about the project. project = "sqlite-utils" -copyright = "2019, Simon Willison" +copyright = "2018-2021, Simon Willison" author = "Simon Willison" # The version info for the project you're documenting, acts as replacement for diff --git a/setup.py b/setup.py index 2a17384..af257d8 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.1" +VERSION = "3.1.1" def get_long_description(): From b067f1ff57372be7f520d536510adc808764243a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 1 Jan 2021 18:10:04 -0800 Subject: [PATCH 0279/1004] table.triggers_dict property, closes #211 --- docs/python-api.rst | 9 +++++++++ sqlite_utils/db.py | 5 +++++ tests/test_introspect.py | 9 ++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index ac7a830..62fa535 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1458,6 +1458,15 @@ The ``.triggers`` property lists database triggers. It can be used on both datab >>> db.triggers ... similar output to db["authors"].triggers +The ``table.triggers_dict`` property returns the triggers for that table as a dictionary mapping their names to their SQL definitions. + +:: + + >>> db["authors"].triggers_dict + {'authors_ai': 'CREATE TRIGGER [authors_ai] AFTER INSERT...', + 'authors_ad': 'CREATE TRIGGER [authors_ad] AFTER DELETE...', + 'authors_au': 'CREATE TRIGGER [authors_au] AFTER UPDATE'} + The ``detect_fts()`` method returns the associated SQLite FTS table name, if one exists for this table. If the table has not been configured for full-text search it returns ``None``. :: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 90427bf..18ac664 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -759,6 +759,11 @@ class Table(Queryable): ).fetchall() ] + @property + def triggers_dict(self): + "Returns {trigger_name: sql} dictionary" + return {trigger.name: trigger.sql for trigger in self.triggers} + def create( self, columns, diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 80cfee6..ce68155 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -117,7 +117,7 @@ def test_pks(fresh_db, pk, expected): assert expected == fresh_db["foo"].pks -def test_triggers(fresh_db): +def test_triggers_and_triggers_dict(fresh_db): assert [] == fresh_db.triggers authors = fresh_db["authors"] authors.insert_all( @@ -128,6 +128,7 @@ def test_triggers(fresh_db): ) fresh_db["other"].insert({"foo": "bar"}) assert [] == authors.triggers + assert {} == authors.triggers_dict assert [] == fresh_db["other"].triggers authors.enable_fts( ["name", "famous_works"], fts_version="FTS4", create_triggers=True @@ -141,7 +142,13 @@ def test_triggers(fresh_db): assert expected_triggers == { (t.name, t.table) for t in fresh_db["authors"].triggers } + assert authors.triggers_dict == { + "authors_ai": "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND", + "authors_ad": "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\nEND", + "authors_au": "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND", + } assert [] == fresh_db["other"].triggers + assert {} == fresh_db["other"].triggers_dict @pytest.mark.parametrize( From 1cad7fad3e7a5b734088f5cc545b69a055e636da Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Jan 2021 13:40:10 -0800 Subject: [PATCH 0280/1004] table.enable_counts() method, closes #212 --- docs/python-api.rst | 24 ++++++++++++++++++++++ sqlite_utils/db.py | 40 +++++++++++++++++++++++++++++++++++++ tests/test_enable_counts.py | 27 +++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 tests/test_enable_counts.py diff --git a/docs/python-api.rst b/docs/python-api.rst index 62fa535..400fa58 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1682,6 +1682,30 @@ This runs the following SQL:: INSERT INTO dogs_fts (dogs_fts) VALUES ("optimize"); +.. _python_api_enable_counts: + +Enabling cached counts for a table +================================== + +The ``select count(*)`` query in SQLite requires a full scan of the primary key index, and can take an increasingly long time as the table grows larger. + +The ``table.enable_counts()`` method can be used to configure triggers to continuously update a record in a ``_counts`` table. This value can then be used to quickly retrieve the count of rows in the associated table. + +.. code-block:: python + + db["dogs"].enable_counts() + +This will create the ``_counts`` table if it does not already exist, with the following schema: + +.. code-block:: sql + + CREATE TABLE [_counts] ( + [table] TEXT PRIMARY KEY, + [count] INTEGER DEFAULT 0 + ) + +Once enabled, table counts can be accessed by querying the ``counts`` table. The count records will be automatically kept up-to-date by the triggers when rows are added or deleted to the table. + Creating indexes ================ diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 18ac664..e2f55bf 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1174,6 +1174,46 @@ class Table(Queryable): self.db.add_foreign_keys([(self.name, column, other_table, other_column)]) return self + def enable_counts(self): + sql = ( + textwrap.dedent( + """ + CREATE TABLE IF NOT EXISTS [{counts_table}]([table] TEXT PRIMARY KEY, count INTEGER DEFAULT 0); + CREATE TRIGGER IF NOT EXISTS [{table}{counts_table}_insert] AFTER INSERT ON [{table}] + BEGIN + INSERT OR REPLACE INTO [{counts_table}] + VALUES ( + {table_quoted}, + COALESCE( + (SELECT count FROM [{counts_table}] WHERE [table] = {table_quoted}), + 0 + ) + 1 + ); + END; + CREATE TRIGGER IF NOT EXISTS [{table}{counts_table}_delete] AFTER DELETE ON [{table}] + BEGIN + INSERT OR REPLACE INTO [{counts_table}] + VALUES ( + {table_quoted}, + COALESCE( + (SELECT count FROM [{counts_table}] WHERE [table] = {table_quoted}), + 0 + ) - 1 + ); + END; + INSERT OR REPLACE INTO _counts VALUES ({table_quoted}, (select count(*) from [{table}])); + """ + ) + .strip() + .format( + counts_table="_counts", + table=self.name, + table_quoted=self.db.escape(self.name), + ) + ) + with self.db.conn: + self.db.conn.executescript(sql) + def enable_fts( self, columns, diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py new file mode 100644 index 0000000..9dc2d54 --- /dev/null +++ b/tests/test_enable_counts.py @@ -0,0 +1,27 @@ +def test_enable_counts(fresh_db): + foo = fresh_db["foo"] + assert fresh_db.table_names() == [] + for i in range(10): + foo.insert({"name": "item {}".format(i)}) + assert fresh_db.table_names() == ["foo"] + assert foo.count == 10 + # Now enable counts + foo.enable_counts() + assert foo.triggers_dict == { + "foo_counts_insert": "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\nBEGIN\n INSERT OR REPLACE INTO [_counts]\n VALUES (\n 'foo',\n COALESCE(\n (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n 0\n ) + 1\n );\nEND", + "foo_counts_delete": "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\nBEGIN\n INSERT OR REPLACE INTO [_counts]\n VALUES (\n 'foo',\n COALESCE(\n (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n 0\n ) - 1\n );\nEND", + } + assert fresh_db.table_names() == ["foo", "_counts"] + assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}] + # Add some items to test the triggers + for i in range(5): + foo.insert({"name": "item {}".format(10 + i)}) + assert foo.count == 15 + assert list(fresh_db["_counts"].rows) == [{"count": 15, "table": "foo"}] + # Delete some items + foo.delete_where("rowid < 7") + assert foo.count == 9 + assert list(fresh_db["_counts"].rows) == [{"count": 9, "table": "foo"}] + foo.delete_where() + assert foo.count == 0 + assert list(fresh_db["_counts"].rows) == [{"count": 0, "table": "foo"}] From 9a5c92b63e7917c93cc502478493c51c781b2ecc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Jan 2021 14:03:52 -0800 Subject: [PATCH 0281/1004] db.enable_counts() method, closes #213 --- docs/python-api.rst | 6 ++++++ sqlite_utils/db.py | 23 +++++++++++++++++++++-- tests/test_enable_counts.py | 29 ++++++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 400fa58..1fc5ba8 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1706,6 +1706,12 @@ This will create the ``_counts`` table if it does not already exist, with the fo Once enabled, table counts can be accessed by querying the ``counts`` table. The count records will be automatically kept up-to-date by the triggers when rows are added or deleted to the table. +You can enable cached counts for every table in a database (except for virtual tables and the ``_counts`` table itself) using the database ``enable_counts()`` method: + +.. code-block:: python + + db.enable_counts() + Creating indexes ================ diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e2f55bf..3118138 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -145,6 +145,11 @@ class InvalidColumns(Exception): class Database: + _counts_table_name = "_counts" + _counts_table_create = "CREATE TABLE IF NOT EXISTS [{}]([table] TEXT PRIMARY KEY, count INTEGER DEFAULT 0);".format( + _counts_table_name + ) + def __init__( self, filename_or_conn=None, @@ -279,6 +284,19 @@ class Database: if self.journal_mode != "delete": self.execute("PRAGMA journal_mode=delete;") + def _ensure_counts_table(self): + with self.conn: + self.execute(self._counts_table_create) + + def enable_counts(self): + self._ensure_counts_table() + for table in self.tables: + if ( + table.virtual_table_using is None + and table.name != self._counts_table_name + ): + table.enable_counts() + def execute_returning_dicts(self, sql, params=None): cursor = self.execute(sql, params or tuple()) keys = [d[0] for d in cursor.description] @@ -1178,7 +1196,7 @@ class Table(Queryable): sql = ( textwrap.dedent( """ - CREATE TABLE IF NOT EXISTS [{counts_table}]([table] TEXT PRIMARY KEY, count INTEGER DEFAULT 0); + {create_counts_table} CREATE TRIGGER IF NOT EXISTS [{table}{counts_table}_insert] AFTER INSERT ON [{table}] BEGIN INSERT OR REPLACE INTO [{counts_table}] @@ -1206,7 +1224,8 @@ class Table(Queryable): ) .strip() .format( - counts_table="_counts", + create_counts_table=self.db._counts_table_create, + counts_table=self.db._counts_table_name, table=self.name, table_quoted=self.db.escape(self.name), ) diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index 9dc2d54..d876a35 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -1,4 +1,4 @@ -def test_enable_counts(fresh_db): +def test_enable_counts_specific_table(fresh_db): foo = fresh_db["foo"] assert fresh_db.table_names() == [] for i in range(10): @@ -25,3 +25,30 @@ def test_enable_counts(fresh_db): foo.delete_where() assert foo.count == 0 assert list(fresh_db["_counts"].rows) == [{"count": 0, "table": "foo"}] + + +def test_enable_counts_all_tables(fresh_db): + foo = fresh_db["foo"] + bar = fresh_db["bar"] + foo.insert({"name": "Cleo"}) + bar.insert({"name": "Cleo"}) + foo.enable_fts(["name"]) + fresh_db.enable_counts() + assert set(fresh_db.table_names()) == { + "foo", + "bar", + "foo_fts", + "foo_fts_data", + "foo_fts_idx", + "foo_fts_docsize", + "foo_fts_config", + "_counts", + } + assert list(fresh_db["_counts"].rows) == [ + {"count": 1, "table": "foo"}, + {"count": 1, "table": "bar"}, + {"count": 3, "table": "foo_fts_data"}, + {"count": 1, "table": "foo_fts_idx"}, + {"count": 1, "table": "foo_fts_docsize"}, + {"count": 1, "table": "foo_fts_config"}, + ] From 5b246d17a0b1c8b5e122da2f1d9974f53b50978e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Jan 2021 19:03:15 -0800 Subject: [PATCH 0282/1004] 'sqlite-utils triggers' command, closes #218 --- docs/cli.rst | 30 ++++++++++++++++++++++++++++- sqlite_utils/cli.py | 47 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_cli.py | 38 ++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 445cef2..a90f99c 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -261,7 +261,7 @@ The ``--nl``, ``--csv``, ``--tsv`` and ``--table`` options are all available. Listing views ============= -The `views` command shows any views defined in the database:: +The ``views`` command shows any views defined in the database:: $ sqlite-utils views sf-trees.db --table --counts --columns --schema view count columns schema @@ -279,6 +279,34 @@ It takes the same options as the ``tables`` command: * ``--tsv`` * ``--table`` +.. _cli_triggers: + +Listing triggers +================ + +The ``triggers`` command shows any triggers configured for the database:: + + % sqlite-utils triggers global-power-plants.db --table + name table sql + --------------- --------- ----------------------------------------------------------------- + plants_insert plants CREATE TRIGGER [plants_insert] AFTER INSERT ON [plants] + BEGIN + INSERT OR REPLACE INTO [_counts] + VALUES ( + 'plants', + COALESCE( + (SELECT count FROM [_counts] WHERE [table] = 'plants'), + 0 + ) + 1 + ); + END + +It defaults to showing triggers for all tables. To see triggers for one or more specific tables pass their names as arguments:: + + % sqlite-utils triggers global-power-plants.db plants + +The command takes the same format options as the ``tables`` and ``views`` commands. + .. _cli_analyze_tables: Analyzing tables diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 6312327..b56e557 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1100,6 +1100,53 @@ def rows( ) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("tables", nargs=-1) +@output_options +@load_extension_option +@click.pass_context +def triggers( + ctx, + path, + tables, + nl, + arrays, + csv, + tsv, + no_headers, + table, + fmt, + json_cols, + load_extension, +): + "Show triggers configured in this database" + sql = "select name, tbl_name as [table], sql from sqlite_master where type = 'trigger'" + if tables: + quote = sqlite_utils.Database(memory=True).escape + sql += " and [table] in ({})".format( + ", ".join(quote(table) for table in tables) + ) + ctx.invoke( + query, + path=path, + sql=sql, + nl=nl, + arrays=arrays, + csv=csv, + tsv=tsv, + no_headers=no_headers, + table=table, + fmt=fmt, + json_cols=json_cols, + load_extension=load_extension, + ) + + @cli.command() @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index c335c46..59770c3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5,6 +5,7 @@ import json import os import pytest from sqlite_utils.utils import sqlite3, find_spatialite +import textwrap from .utils import collapse_whitespace @@ -1736,3 +1737,40 @@ def test_search(tmpdir, fts, extra_arg, expected): ) assert result.exit_code == 0 assert result.output == expected + + +_TRIGGERS_EXPECTED = '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah AFTER INSERT ON articles\\nBEGIN\\n UPDATE counter SET count = count + 1;\\nEND"}]\n' + + +@pytest.mark.parametrize( + "extra_args,expected", + [([], _TRIGGERS_EXPECTED), (["articles"], _TRIGGERS_EXPECTED), (["counter"], "")], +) +def test_triggers(tmpdir, extra_args, expected): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["articles"].insert( + {"id": 1, "title": "Title the first"}, + pk="id", + ) + db["counter"].insert({"count": 1}) + db.conn.execute( + textwrap.dedent( + """ + CREATE TRIGGER blah AFTER INSERT ON articles + BEGIN + UPDATE counter SET count = count + 1; + END + """ + ) + ) + args = ["triggers", db_path] + if extra_args: + args.extend(extra_args) + result = CliRunner().invoke( + cli.cli, + args, + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert result.output == expected From ce918195a4d72152569999c907937feb9d866ce3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Jan 2021 19:52:15 -0800 Subject: [PATCH 0283/1004] Use $ instead of % in CLI documentation Refs #218 --- docs/cli.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index a90f99c..e6f96bb 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -286,7 +286,7 @@ Listing triggers The ``triggers`` command shows any triggers configured for the database:: - % sqlite-utils triggers global-power-plants.db --table + $ sqlite-utils triggers global-power-plants.db --table name table sql --------------- --------- ----------------------------------------------------------------- plants_insert plants CREATE TRIGGER [plants_insert] AFTER INSERT ON [plants] @@ -303,7 +303,7 @@ The ``triggers`` command shows any triggers configured for the database:: It defaults to showing triggers for all tables. To see triggers for one or more specific tables pass their names as arguments:: - % sqlite-utils triggers global-power-plants.db plants + $ sqlite-utils triggers global-power-plants.db plants The command takes the same format options as the ``tables`` and ``views`` commands. @@ -734,7 +734,7 @@ Every option for this table (with the exception of ``--pk-none``) can be specifi If you want to see the SQL that will be executed to make the change without actually executing it, add the ``--sql`` flag. For example:: - % sqlite-utils transform fixtures.db roadside_attractions \ + $ sqlite-utils transform fixtures.db roadside_attractions \ --rename pk id \ --default name Untitled \ --column-order id \ From 3d041d34d5ee8234e0b955d2d1697f0756d1ffa5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Jan 2021 20:15:04 -0800 Subject: [PATCH 0284/1004] Renamed db.escape() to db.quote() and documented it Closes #217 --- docs/python-api.rst | 26 ++++++++++++++++++++++++++ sqlite_utils/cli.py | 2 +- sqlite_utils/db.py | 10 ++++------ tests/test_create.py | 7 +++++++ 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 1fc5ba8..a8fb705 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -87,6 +87,15 @@ The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around db["dogs"].insert({"name": "Cleo"}) db.execute("update dogs set name = 'Cleopaws'") +You can pass parameters as an optional second argument, using either a list or a dictionary. These will be correctly quoted and escaped. + +.. code-block:: python + + # Using ? and a list: + db.execute("update dogs set name = ?", ["Cleopaws"]) + # Or using :name and a dictionary: + db.execute("update dogs set name = :name", {"name": "Cleopaws"}) + .. _python_api_table: Accessing tables @@ -1913,3 +1922,20 @@ If you want to deliberately replace the registered function with a new implement @db.register_function(deterministic=True, replace=True) def reverse_string(s): return s[::-1] + +.. _python_api_quote: + +Quoting strings for use in SQL +============================== + +In almost all cases you should pass values to your SQL queries using the optional ``parameters`` argument to ``db.execute()``, as described in :ref:`python_api_execute`. + +If that option isn't relevant to your use-case you can to quote a string for use with SQLite using the ``db.quote()`` method, like so: + +:: + + >>> db = Database(memory=True) + >>> db.quote("hello") + "'hello'" + >>> db.quote("hello'this'has'quotes") + "'hello''this''has''quotes'" diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index b56e557..4a3035b 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1127,7 +1127,7 @@ def triggers( "Show triggers configured in this database" sql = "select name, tbl_name as [table], sql from sqlite_master where type = 'trigger'" if tables: - quote = sqlite_utils.Database(memory=True).escape + quote = sqlite_utils.Database(memory=True).quote sql += " and [table] in ({})".format( ", ".join(quote(table) for table in tables) ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3118138..2fedc4f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -228,7 +228,7 @@ class Database: klass = View if table_name in self.view_names() else Table return klass(self, table_name, **kwargs) - def escape(self, value): + def quote(self, value): # Normally we would use .execute(sql, [params]) for escaping, but # occasionally that isn't available - most notable when we need # to include a "... DEFAULT 'value'" in a column definition. @@ -417,7 +417,7 @@ class Database: column_extras.append("NOT NULL") if column_name in defaults and defaults[column_name] is not None: column_extras.append( - "DEFAULT {}".format(self.escape(defaults[column_name])) + "DEFAULT {}".format(self.quote(defaults[column_name])) ) if column_name in foreign_keys_by_column: column_extras.append( @@ -1107,9 +1107,7 @@ class Table(Queryable): col_type = str not_null_sql = None if not_null_default is not None: - not_null_sql = "NOT NULL DEFAULT {}".format( - self.db.escape(not_null_default) - ) + not_null_sql = "NOT NULL DEFAULT {}".format(self.db.quote(not_null_default)) sql = "ALTER TABLE [{table}] ADD COLUMN [{col_name}] {col_type}{not_null_default};".format( table=self.name, col_name=col_name, @@ -1227,7 +1225,7 @@ class Table(Queryable): create_counts_table=self.db._counts_table_create, counts_table=self.db._counts_table_name, table=self.name, - table_quoted=self.db.escape(self.name), + table_quoted=self.db.quote(self.name), ) ) with self.db.conn: diff --git a/tests/test_create.py b/tests/test_create.py index 83936e4..a658757 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -922,3 +922,10 @@ def test_create_with_nested_bytes(fresh_db): record = {"id": 1, "data": {"foo": b"bytes"}} fresh_db["t"].insert(record) assert [{"id": 1, "data": '{"foo": "b\'bytes\'"}'}] == list(fresh_db["t"].rows) + + +@pytest.mark.parametrize( + "input,expected", [("hello", "'hello'"), ("hello'there'", "'hello''there'''")] +) +def test_quote(fresh_db, input, expected): + assert fresh_db.quote(input) == expected From de08096989de1e025f0457e53404477f71a994e4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Jan 2021 20:19:55 -0800 Subject: [PATCH 0285/1004] database.triggers_dict, closes #216 --- docs/python-api.rst | 9 +++++++++ sqlite_utils/db.py | 5 +++++ tests/test_introspect.py | 15 +++++++++------ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index a8fb705..04703de 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1476,6 +1476,15 @@ The ``table.triggers_dict`` property returns the triggers for that table as a di 'authors_ad': 'CREATE TRIGGER [authors_ad] AFTER DELETE...', 'authors_au': 'CREATE TRIGGER [authors_au] AFTER UPDATE'} +The same property exists on the database, and will return all triggers across all tables: + +:: + + >>> db.triggers_dict + {'authors_ai': 'CREATE TRIGGER [authors_ai] AFTER INSERT...', + 'authors_ad': 'CREATE TRIGGER [authors_ad] AFTER DELETE...', + 'authors_au': 'CREATE TRIGGER [authors_au] AFTER UPDATE'} + The ``detect_fts()`` method returns the associated SQLite FTS table name, if one exists for this table. If the table has not been configured for full-text search it returns ``None``. :: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 2fedc4f..719d03c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -272,6 +272,11 @@ class Database: ).fetchall() ] + @property + def triggers_dict(self): + "Returns {trigger_name: sql} dictionary" + return {trigger.name: trigger.sql for trigger in self.triggers} + @property def journal_mode(self): return self.execute("PRAGMA journal_mode;").fetchone()[0] diff --git a/tests/test_introspect.py b/tests/test_introspect.py index ce68155..9a37001 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -127,9 +127,10 @@ def test_triggers_and_triggers_dict(fresh_db): ] ) fresh_db["other"].insert({"foo": "bar"}) - assert [] == authors.triggers - assert {} == authors.triggers_dict - assert [] == fresh_db["other"].triggers + assert authors.triggers == [] + assert authors.triggers_dict == {} + assert fresh_db["other"].triggers == [] + assert fresh_db.triggers_dict == {} authors.enable_fts( ["name", "famous_works"], fts_version="FTS4", create_triggers=True ) @@ -142,13 +143,15 @@ def test_triggers_and_triggers_dict(fresh_db): assert expected_triggers == { (t.name, t.table) for t in fresh_db["authors"].triggers } - assert authors.triggers_dict == { + expected_triggers = { "authors_ai": "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND", "authors_ad": "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\nEND", "authors_au": "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND", } - assert [] == fresh_db["other"].triggers - assert {} == fresh_db["other"].triggers_dict + assert authors.triggers_dict == expected_triggers + assert fresh_db["other"].triggers == [] + assert fresh_db["other"].triggers_dict == {} + assert fresh_db.triggers_dict == expected_triggers @pytest.mark.parametrize( From ce042ff1f0a398d9ad46532636e438dbe4efc0f1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Jan 2021 20:26:39 -0800 Subject: [PATCH 0286/1004] 'sqlite-utils enable-counts' command, closes #214 --- docs/cli.rst | 17 ++++++++++++++ sqlite_utils/cli.py | 23 +++++++++++++++++++ tests/test_enable_counts.py | 44 +++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index e6f96bb..3a55629 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1057,6 +1057,23 @@ Use the ``--sql`` option to output the SQL that would be executed, rather than r order by [documents_fts].rank +.. _cli_enable_counts: + +Enabling cached counts +====================== + +``select count(*)`` queries can take a long time against large tables. ``sqlite-utils`` can speed these up by adding triggers to maintain a ``_counts`` table, see :ref:`python_api_enable_counts`. + +The ``sqlite-utils enable-counts`` command can be used to configure these triggers, either for every table in the database or for specific tables. + +:: + + # Configure triggers for every table in the database + $ sqlite-utils enable-counts mydb.db + + # Configure triggers just for specific tables + $ sqlite-utils enable-counts mydb.db table1 table2 + .. _cli_vacuum: Vacuum diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4a3035b..5f81a1e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -548,6 +548,29 @@ def disable_wal(path, load_extension): db.disable_wal() +@cli.command(name="enable-counts") +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("tables", nargs=-1) +@load_extension_option +def enable_counts(path, tables, load_extension): + "Configure triggers to update a _counts table with row counts" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + if not tables: + db.enable_counts() + else: + # Check all tables exist + bad_tables = [table for table in tables if not db[table].exists()] + if bad_tables: + raise click.ClickException("Invalid tables: {}".format(bad_tables)) + for table in tables: + db[table].enable_counts() + + def insert_upsert_options(fn): for decorator in reversed( ( diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index d876a35..a6f35a0 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -1,3 +1,9 @@ +from sqlite_utils import Database +from sqlite_utils import cli +from click.testing import CliRunner +import pytest + + def test_enable_counts_specific_table(fresh_db): foo = fresh_db["foo"] assert fresh_db.table_names() == [] @@ -52,3 +58,41 @@ def test_enable_counts_all_tables(fresh_db): {"count": 1, "table": "foo_fts_docsize"}, {"count": 1, "table": "foo_fts_config"}, ] + + +@pytest.fixture +def counts_db_path(tmpdir): + path = str(tmpdir / "test.db") + db = Database(path) + db["foo"].insert({"name": "Cleo"}) + db["bar"].insert({"name": "Cleo"}) + return path + + +@pytest.mark.parametrize( + "extra_args,expected_triggers", + [ + ( + [], + [ + "foo_counts_insert", + "foo_counts_delete", + "bar_counts_insert", + "bar_counts_delete", + ], + ), + ( + ["bar"], + [ + "bar_counts_insert", + "bar_counts_delete", + ], + ), + ], +) +def test_cli_enable_counts(counts_db_path, extra_args, expected_triggers): + db = Database(counts_db_path) + assert list(db.triggers_dict.keys()) == [] + result = CliRunner().invoke(cli.cli, ["enable-counts", counts_db_path] + extra_args) + assert result.exit_code == 0 + assert list(db.triggers_dict.keys()) == expected_triggers From 6d1828e40b17a7fa3403de7a3d3e0da9f657aa30 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Jan 2021 20:30:10 -0800 Subject: [PATCH 0287/1004] Don't run CodeQL against pull requests It failed with strange errors. https://github.com/simonw/sqlite-utils/pull/203/checks?check_run_id=1549287178 --- .github/workflows/codeql-analysis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d86c2e3..713f81e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -3,9 +3,6 @@ name: "CodeQL" on: push: branches: [main] - pull_request: - # The branches below must be a subset of the branches above - branches: [main] schedule: - cron: '0 4 * * 5' From 1e38a16ea8a58ec3eee8e54eee6c024d87f99d86 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 Jan 2021 10:42:17 -0800 Subject: [PATCH 0288/1004] Nicer error message for invalid JSON insert, closes #206 --- sqlite_utils/cli.py | 17 ++++++++++++----- tests/test_cli.py | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 5f81a1e..4d888ef 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -654,12 +654,19 @@ def insert_upsert_implementation( reader = csv_std.reader(json_file, dialect=dialect) headers = next(reader) docs = (dict(zip(headers, row)) for row in reader) - elif nl: - docs = (json.loads(line) for line in json_file) else: - docs = json.load(json_file) - if isinstance(docs, dict): - docs = [docs] + try: + if nl: + docs = (json.loads(line) for line in json_file) + else: + docs = json.load(json_file) + if isinstance(docs, dict): + docs = [docs] + except json.decoder.JSONDecodeError: + raise click.ClickException( + "Invalid JSON - use --csv for CSV or --tsv for TSV files" + ) + extra_kwargs = {"ignore": ignore, "replace": replace, "truncate": truncate} if not_null: extra_kwargs["not_null"] = set(not_null) diff --git a/tests/test_cli.py b/tests/test_cli.py index 59770c3..381997b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -584,6 +584,20 @@ def test_insert_from_stdin(tmpdir): ) +def test_insert_invalid_json_error(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", "-"], + input="name,age\nCleo,4", + ) + assert result.exit_code == 1 + assert ( + result.output + == "Error: Invalid JSON - use --csv for CSV or --tsv for TSV files\n" + ) + + def test_insert_with_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dog.json") open(json_path, "w").write(json.dumps({"id": 1, "name": "Cleo", "age": 4})) From 036ec6d32313487527c66dea613a3e7118b97459 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 Jan 2021 10:43:21 -0800 Subject: [PATCH 0289/1004] Ignore test .db files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 634af83..75fa28e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .venv +*.db __pycache__/ *.py[cod] *$py.class From 94b50230666cc6657a7b447e0ef1ddcb74a44473 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 Jan 2021 12:19:34 -0800 Subject: [PATCH 0290/1004] table.count now uses cached counts if db.use_counts_table Closes #215 --- docs/cli.rst | 2 +- docs/python-api.rst | 40 ++++++++++++++++++++++++++----- sqlite_utils/db.py | 47 +++++++++++++++++++++++++++++++------ tests/test_enable_counts.py | 33 ++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 14 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 3a55629..46ad820 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1062,7 +1062,7 @@ Use the ``--sql`` option to output the SQL that would be executed, rather than r Enabling cached counts ====================== -``select count(*)`` queries can take a long time against large tables. ``sqlite-utils`` can speed these up by adding triggers to maintain a ``_counts`` table, see :ref:`python_api_enable_counts`. +``select count(*)`` queries can take a long time against large tables. ``sqlite-utils`` can speed these up by adding triggers to maintain a ``_counts`` table, see :ref:`python_api_cached_table_counts` for details. The ``sqlite-utils enable-counts`` command can be used to configure these triggers, either for every table in the database or for specific tables. diff --git a/docs/python-api.rst b/docs/python-api.rst index 04703de..b02fafa 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1386,6 +1386,8 @@ The ``.count`` property shows the current number of rows (``select count(*) from >>> db["Street_Tree_List"].count 189144 +This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.execute_count()`` instead of accessing the property. + The ``.columns`` property shows the columns in the table or view:: >>> db["PlantType"].columns @@ -1700,9 +1702,9 @@ This runs the following SQL:: INSERT INTO dogs_fts (dogs_fts) VALUES ("optimize"); -.. _python_api_enable_counts: +.. _python_api_cached_table_counts: -Enabling cached counts for a table +Cached table counts using triggers ================================== The ``select count(*)`` query in SQLite requires a full scan of the primary key index, and can take an increasingly long time as the table grows larger. @@ -1718,18 +1720,44 @@ This will create the ``_counts`` table if it does not already exist, with the fo .. code-block:: sql CREATE TABLE [_counts] ( - [table] TEXT PRIMARY KEY, - [count] INTEGER DEFAULT 0 + [table] TEXT PRIMARY KEY, + [count] INTEGER DEFAULT 0 ) -Once enabled, table counts can be accessed by querying the ``counts`` table. The count records will be automatically kept up-to-date by the triggers when rows are added or deleted to the table. - You can enable cached counts for every table in a database (except for virtual tables and the ``_counts`` table itself) using the database ``enable_counts()`` method: .. code-block:: python db.enable_counts() +Once enabled, table counts will be stored in the ``_counts`` table. The count records will be automatically kept up-to-date by the triggers when rows are added or deleted to the table. + +To access these counts you can query the ``_counts`` table directly or you can use the ``db.cached_counts()`` method. This method returns a dictionary mapping tables to their counts:: + + >>> db.cached_counts() + {'global-power-plants': 33643, + 'global-power-plants_fts_data': 136, + 'global-power-plants_fts_idx': 199, + 'global-power-plants_fts_docsize': 33643, + 'global-power-plants_fts_config': 1} + +You can pass a list of table names to this method to retrieve just those counts:: + + >>> db.cached_counts(["global-power-plants"]) + {'global-power-plants': 33643} + +The ``table.count`` property executes a ``select count(*)`` query by default, unless the ``db.use_counts_table`` property is set to ``True``. + +You can set ``use_counts_table`` to ``True`` when you instantiate the database object: + +.. code-block:: python + + db = Database("global-power-plants.db", use_counts_table=True) + +If the property is ``True`` any calls to the ``table.count`` property will first attempt to find the cached count in the ``_counts`` table, and fall back on a ``count(*)`` query if the value is not available or the table is missing. + +Calling the ``.enable_counts()`` method on a database or table object will set ``use_counts_table`` to ``True`` for the lifetime of that database object. + Creating indexes ================ diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 719d03c..f1f3de6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -144,11 +144,17 @@ class InvalidColumns(Exception): pass +_COUNTS_TABLE_CREATE_SQL = """ +CREATE TABLE IF NOT EXISTS [{}]( + [table] TEXT PRIMARY KEY, + count INTEGER DEFAULT 0 +); +""".strip() + + class Database: _counts_table_name = "_counts" - _counts_table_create = "CREATE TABLE IF NOT EXISTS [{}]([table] TEXT PRIMARY KEY, count INTEGER DEFAULT 0);".format( - _counts_table_name - ) + use_counts_table = False def __init__( self, @@ -157,6 +163,7 @@ class Database: recreate=False, recursive_triggers=True, tracer=None, + use_counts_table=False, ): assert (filename_or_conn is not None and not memory) or ( filename_or_conn is None and memory @@ -174,6 +181,7 @@ class Database: if recursive_triggers: self.execute("PRAGMA recursive_triggers=on;") self._registered_functions = set() + self.use_counts_table = use_counts_table @contextlib.contextmanager def tracer(self, tracer=None): @@ -291,7 +299,7 @@ class Database: def _ensure_counts_table(self): with self.conn: - self.execute(self._counts_table_create) + self.execute(_COUNTS_TABLE_CREATE_SQL.format(self._counts_table_name)) def enable_counts(self): self._ensure_counts_table() @@ -301,6 +309,16 @@ class Database: and table.name != self._counts_table_name ): table.enable_counts() + self.use_counts_table = True + + def cached_counts(self, tables=None): + sql = "select [table], count from {}".format(self._counts_table_name) + if tables: + sql += " where [table] in ({})".format(", ".join("?" for table in tables)) + try: + return {r[0]: r[1] for r in self.execute(sql, tables).fetchall()} + except OperationalError: + return {} def execute_returning_dicts(self, sql, params=None): cursor = self.execute(sql, params or tuple()) @@ -602,12 +620,15 @@ class Queryable: self.db = db self.name = name - @property - def count(self): + def execute_count(self): return self.db.execute( "select count(*) from [{}]".format(self.name) ).fetchone()[0] + @property + def count(self): + return self.execute_count() + @property def rows(self): return self.rows_where() @@ -691,6 +712,14 @@ class Table(Queryable): else " ({})".format(", ".join(c.name for c in self.columns)), ) + @property + def count(self): + if self.db.use_counts_table: + counts = self.db.cached_counts([self.name]) + if counts: + return next(iter(counts.values())) + return self.execute_count() + def exists(self): return self.name in self.db.table_names() @@ -1227,7 +1256,9 @@ class Table(Queryable): ) .strip() .format( - create_counts_table=self.db._counts_table_create, + create_counts_table=_COUNTS_TABLE_CREATE_SQL.format( + self.db._counts_table_name + ), counts_table=self.db._counts_table_name, table=self.name, table_quoted=self.db.quote(self.name), @@ -1235,6 +1266,7 @@ class Table(Queryable): ) with self.db.conn: self.db.conn.executescript(sql) + self.db.use_counts_table = True def enable_fts( self, @@ -1650,6 +1682,7 @@ class Table(Queryable): ) with self.db.conn: + result = None for query, params in queries_and_params: try: result = self.db.execute(query, params) diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index a6f35a0..6fabd78 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -96,3 +96,36 @@ def test_cli_enable_counts(counts_db_path, extra_args, expected_triggers): result = CliRunner().invoke(cli.cli, ["enable-counts", counts_db_path] + extra_args) assert result.exit_code == 0 assert list(db.triggers_dict.keys()) == expected_triggers + + +def test_uses_counts_after_enable_counts(counts_db_path): + db = Database(counts_db_path) + logged = [] + with db.tracer(lambda sql, parameters: logged.append((sql, parameters))): + assert db["foo"].count == 1 + assert logged == [ + ("select name from sqlite_master where type = 'view'", None), + ("select count(*) from [foo]", None), + ] + logged.clear() + assert not db.use_counts_table + db.enable_counts() + assert db.use_counts_table + assert db["foo"].count == 1 + assert logged == [ + ( + "CREATE TABLE IF NOT EXISTS [_counts](\n [table] TEXT PRIMARY KEY,\n count INTEGER DEFAULT 0\n);", + None, + ), + ("select name from sqlite_master where type = 'table'", None), + ("select name from sqlite_master where type = 'view'", None), + ("select name from sqlite_master where type = 'view'", None), + ("select name from sqlite_master where type = 'view'", None), + ("select sql from sqlite_master where name = ?", ("foo",)), + ("SELECT quote(:value)", {"value": "foo"}), + ("select sql from sqlite_master where name = ?", ("bar",)), + ("SELECT quote(:value)", {"value": "bar"}), + ("select sql from sqlite_master where name = ?", ("_counts",)), + ("select name from sqlite_master where type = 'view'", None), + ("select [table], count from _counts where [table] in (?)", ["foo"]), + ] From c265541384c1e794b167da762c49a078b2195bf8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 Jan 2021 12:22:07 -0800 Subject: [PATCH 0291/1004] Shorter help summary for rebuild_fts --- sqlite_utils/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4d888ef..0388961 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -275,7 +275,7 @@ def optimize(path, tables, no_vacuum, load_extension): @click.argument("tables", nargs=-1) @load_extension_option def rebuild_fts(path, tables, load_extension): - """Rebuild specific FTS tables, or all FTS tables if none are specified""" + """Rebuild all or specific FTS tables""" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) if not tables: From b4f09146d342ba72190f9a8543a44ccd6ea06b02 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 Jan 2021 12:41:24 -0800 Subject: [PATCH 0292/1004] table.has_counts_triggers property, refs #219 --- docs/python-api.rst | 18 ++++++++++++++---- sqlite_utils/cli.py | 14 ++++++++++++++ sqlite_utils/db.py | 10 ++++++++++ tests/test_introspect.py | 8 ++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index b02fafa..8f943f6 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1469,7 +1469,7 @@ The ``.triggers`` property lists database triggers. It can be used on both datab >>> db.triggers ... similar output to db["authors"].triggers -The ``table.triggers_dict`` property returns the triggers for that table as a dictionary mapping their names to their SQL definitions. +The ``.triggers_dict`` property returns the triggers for that table as a dictionary mapping their names to their SQL definitions. :: @@ -1491,15 +1491,25 @@ The ``detect_fts()`` method returns the associated SQLite FTS table name, if one :: - >> db["authors"].detect_fts() + >>> db["authors"].detect_fts() "authors_fts" The ``.virtual_table_using`` property reveals if a table is a virtual table. It returns ``None`` for regular tables and the upper case version of the type of virtual table otherwise. For example:: - >> db["authors"].enable_fts(["name"]) - >> db["authors_fts"].virtual_table_using + >>> db["authors"].enable_fts(["name"]) + >>> db["authors_fts"].virtual_table_using "FTS5" +The ``.has_counts_triggers`` property shows if a table has been configured with triggers for updating a ``_counts`` table, as described in :ref:`python_api_cached_table_counts`. + +:: + + >>> db["authors"].has_counts_triggers + False + >>> db["authors"].enable_counts() + >>> db["authors"].has_counts_triggers + True + .. _python_api_fts: Enabling full-text search diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0388961..0bc03fe 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -571,6 +571,20 @@ def enable_counts(path, tables, load_extension): db[table].enable_counts() +@cli.command(name="reset-counts") +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@load_extension_option +def reset_counts(path, load_extension): + "Reset calculated counts in the _counts table" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + db.reset_counts() + + def insert_upsert_options(fn): for decorator in reversed( ( diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f1f3de6..bc954e5 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1268,6 +1268,16 @@ class Table(Queryable): self.db.conn.executescript(sql) self.db.use_counts_table = True + @property + def has_counts_triggers(self): + trigger_names = { + "{table}{counts_table}_{suffix}".format( + counts_table=self.db._counts_table_name, table=self.name, suffix=suffix + ) + for suffix in ["insert", "delete"] + } + return trigger_names.issubset(self.triggers_dict.keys()) + def enable_fts( self, columns, diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 9a37001..73102f8 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -154,6 +154,14 @@ def test_triggers_and_triggers_dict(fresh_db): assert fresh_db.triggers_dict == expected_triggers +def test_has_counts_triggers(fresh_db): + authors = fresh_db["authors"] + authors.insert({"name": "Frank Herbert"}) + assert not authors.has_counts_triggers + authors.enable_counts() + assert authors.has_counts_triggers + + @pytest.mark.parametrize( "sql,expected_name,expected_using", [ From 0d2a47eab99f625be7ef4f77c6e9803177d13a2c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 Jan 2021 12:59:31 -0800 Subject: [PATCH 0293/1004] .reset_counts() method and reset-counts command, closes #219 --- docs/cli.rst | 4 ++++ docs/python-api.rst | 6 ++++++ sqlite_utils/db.py | 11 +++++++++++ tests/test_enable_counts.py | 37 +++++++++++++++++++++++++++++++++++-- 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 46ad820..974fe68 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1074,6 +1074,10 @@ The ``sqlite-utils enable-counts`` command can be used to configure these trigge # Configure triggers just for specific tables $ sqlite-utils enable-counts mydb.db table1 table2 +If the ``_counts`` table ever becomes out-of-sync with the actual table counts you can repair it using the ``reset-counts`` command:: + + $ sqlite-utils reset-counts mydb.db + .. _cli_vacuum: Vacuum diff --git a/docs/python-api.rst b/docs/python-api.rst index 8f943f6..28adfba 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1768,6 +1768,12 @@ If the property is ``True`` any calls to the ``table.count`` property will first Calling the ``.enable_counts()`` method on a database or table object will set ``use_counts_table`` to ``True`` for the lifetime of that database object. +If the ``_counts`` table ever becomes out-of-sync with the actual table counts you can repair it using the ``.reset_counts()`` method: + +.. code-block:: python + + db.reset_counts() + Creating indexes ================ diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index bc954e5..dc2a2e8 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -320,6 +320,17 @@ class Database: except OperationalError: return {} + def reset_counts(self): + tables = [table for table in self.tables if table.has_counts_triggers] + with self.conn: + self._ensure_counts_table() + counts_table = self[self._counts_table_name] + counts_table.delete_where() + counts_table.insert_all( + {"table": table.name, "count": table.execute_count()} + for table in tables + ) + def execute_returning_dicts(self, sql, params=None): cursor = self.execute(sql, params or tuple()) keys = [d[0] for d in cursor.description] diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index 6fabd78..b70378e 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -64,8 +64,10 @@ def test_enable_counts_all_tables(fresh_db): def counts_db_path(tmpdir): path = str(tmpdir / "test.db") db = Database(path) - db["foo"].insert({"name": "Cleo"}) - db["bar"].insert({"name": "Cleo"}) + db["foo"].insert({"name": "bar"}) + db["bar"].insert({"name": "bar"}) + db["bar"].insert({"name": "bar"}) + db["baz"].insert({"name": "bar"}) return path @@ -79,6 +81,8 @@ def counts_db_path(tmpdir): "foo_counts_delete", "bar_counts_insert", "bar_counts_delete", + "baz_counts_insert", + "baz_counts_delete", ], ), ( @@ -121,11 +125,40 @@ def test_uses_counts_after_enable_counts(counts_db_path): ("select name from sqlite_master where type = 'view'", None), ("select name from sqlite_master where type = 'view'", None), ("select name from sqlite_master where type = 'view'", None), + ("select name from sqlite_master where type = 'view'", None), ("select sql from sqlite_master where name = ?", ("foo",)), ("SELECT quote(:value)", {"value": "foo"}), ("select sql from sqlite_master where name = ?", ("bar",)), ("SELECT quote(:value)", {"value": "bar"}), + ("select sql from sqlite_master where name = ?", ("baz",)), + ("SELECT quote(:value)", {"value": "baz"}), ("select sql from sqlite_master where name = ?", ("_counts",)), ("select name from sqlite_master where type = 'view'", None), ("select [table], count from _counts where [table] in (?)", ["foo"]), ] + + +def test_reset_counts(counts_db_path): + db = Database(counts_db_path) + db["foo"].enable_counts() + db["bar"].enable_counts() + assert db.cached_counts() == {"foo": 1, "bar": 2} + # Corrupt the value + db["_counts"].update("foo", {"count": 3}) + assert db.cached_counts() == {"foo": 3, "bar": 2} + assert db["foo"].count == 3 + # Reset them + db.reset_counts() + assert db.cached_counts() == {"foo": 1, "bar": 2} + assert db["foo"].count == 1 + + +def test_reset_counts_cli(counts_db_path): + db = Database(counts_db_path) + db["foo"].enable_counts() + db["bar"].enable_counts() + assert db.cached_counts() == {"foo": 1, "bar": 2} + db["_counts"].update("foo", {"count": 3}) + result = CliRunner().invoke(cli.cli, ["reset-counts", counts_db_path]) + assert result.exit_code == 0 + assert db.cached_counts() == {"foo": 1, "bar": 2} From 4cc82fd0bccc9d2eeb3510beb4e691d7da099f84 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 3 Jan 2021 13:15:26 -0800 Subject: [PATCH 0294/1004] Release 3.2 Refs #206, #211, #212, #213, #214, #215, #216, #217, #218, #219 --- docs/changelog.rst | 18 ++++++++++++++++++ setup.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8388066..cf0ff85 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,24 @@ Changelog =========== +.. _v3_2: + +3.2 (2021-01-03) +---------------- + +This release introduces a new mechanism for speeding up ``count(*)`` queries using cached table counts, stored in a ``_counts`` table and updated by triggers. This mechanism is described in :ref:`python_api_cached_table_counts`, and can be enabled using Python API methods or the new ``enable-counts`` CLI command. (`#212 `__) + +- ``table.enable_counts()`` method for enabling these triggers on a specific table. +- ``db.enable_counts()`` method for enabling triggers on every table in the database. (`#213 `__) +- New ``sqlite-utils enable-counts my.db`` command for enabling counts on all or specific tables, see :ref:`cli_enable_counts`. (`#214 `__) +- New ``sqlite-utils triggers`` command for listing the triggers defined for a database or specific tables, see :ref:`cli_triggers`. (`#218 `__) +- New ``db.use_counts_table`` property which, if ``True``, causes ``table.count`` to read from the ``_counts`` table. (`#215 `__) +- ``table.has_counts_triggers`` property revealing if a table has been configured with the new ``_counts`` database triggers. +- ``db.reset_counts()`` method and ``sqlite-utils reset-counts`` command for resetting the values in the ``_counts`` table. (`#219 `__) +- The previously undocumented ``db.escape()`` method has been renamed to ``db.quote()`` and is now covered by the documentation: :ref:`python_api_quote`. (`#217 `__) +- New ``table.triggers_dict`` and ``db.triggers_dict`` introspection properties. (`#211 `__, `#216 `__) +- ``sqlite-utils insert`` now shows a more useful error message for invalid JSON. (`#206 `__) + .. _v3_1_1: 3.1.1 (2021-01-01) diff --git a/setup.py b/setup.py index af257d8..d3f141c 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.1.1" +VERSION = "3.2" def get_long_description(): From b6840646baf97e4d324d2c53c036ffeeedab9822 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Jan 2021 15:17:27 -0800 Subject: [PATCH 0295/1004] .add_missing_columns() is now case insensitive, closes #221 --- sqlite_utils/db.py | 4 ++-- tests/test_create.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index dc2a2e8..c6bc606 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1973,9 +1973,9 @@ class Table(Queryable): def add_missing_columns(self, records): needed_columns = suggest_column_types(records) - current_columns = self.columns_dict + current_columns = {c.lower() for c in self.columns_dict} for col_name, col_type in needed_columns.items(): - if col_name not in current_columns: + if col_name.lower() not in current_columns: self.add_column(col_name, col_type) return self diff --git a/tests/test_create.py b/tests/test_create.py index a658757..bee47a7 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -503,6 +503,16 @@ def test_insert_row_alter_table_invalid_column_characters(fresh_db): table.insert({"foo": "baz", "new_col[abc]": 1.2}, alter=True) +def test_add_missing_columns_case_insensitive(fresh_db): + table = fresh_db["foo"] + table.insert({"id": 1, "name": "Cleo"}, pk="id") + table.add_missing_columns([{"Name": ".", "age": 4}]) + assert ( + table.schema + == "CREATE TABLE [foo] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n, [age] INTEGER)" + ) + + @pytest.mark.parametrize("use_table_factory", [True, False]) def test_insert_replace_rows_alter_table(fresh_db, use_table_factory): first_row = {"id": 1, "title": "Hedgehogs of the world", "author_id": 1} From 0b244d207a460d872cdac029d01deec784058858 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Jan 2021 15:22:53 -0800 Subject: [PATCH 0296/1004] Release 3.2.1 Refs #221 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cf0ff85..8ba41ae 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v3_2_1: + +3.2.1 (2021-01-12) +------------------ + +- Fixed a bug where ``.add_missing_columns()`` failed to take case insensitive column names into account. (`#221 `__) + .. _v3_2: 3.2 (2021-01-03) diff --git a/setup.py b/setup.py index d3f141c..b1a7324 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.2" +VERSION = "3.2.1" def get_long_description(): From d4e00f8d0121cf5e1f1ad822dda68919d47cc5e1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 Jan 2021 20:26:02 -0800 Subject: [PATCH 0297/1004] table.m2m(..., alter=True) option, closes #222 --- docs/python-api.rst | 2 ++ sqlite_utils/db.py | 5 ++++- tests/test_m2m.py | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 28adfba..c656045 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -716,6 +716,8 @@ If it cannot find such a table, it will create a new one using the names of the It it finds multiple candidate tables with foreign keys to both of the specified tables it will raise a ``sqlite_utils.db.NoObviousTable`` exception. You can avoid this error by specifying the correct table using ``m2m_table=``. +The ``.m2m()`` method also takes an optional ``pk=`` argument to specify the primary key that should be used if the table is created, and an optional ``alter=True`` argument to specify that any missing columns of an existing table should be added if they are needed. + .. _python_api_m2m_lookup: Using m2m and lookup tables together diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c6bc606..14a9efc 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2009,6 +2009,7 @@ class Table(Queryable): pk=DEFAULT, lookup=None, m2m_table=None, + alter=False, ): if isinstance(other_table, str): other_table = self.db.table(other_table, pk=pk) @@ -2045,7 +2046,9 @@ class Table(Queryable): ) # Ensure each record exists in other table for record in records: - id = other_table.insert(record, pk=pk, replace=True).last_pk + id = other_table.insert( + record, pk=pk, replace=True, alter=alter + ).last_pk m2m_table.insert( { "{}_id".format(other_table.name): id, diff --git a/tests/test_m2m.py b/tests/test_m2m.py index cbc9015..5423d2b 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -14,6 +14,24 @@ def test_insert_m2m_single(fresh_db): assert [{"humans_id": 1, "dogs_id": 1}] == list(dogs_humans.rows) +def test_insert_m2m_alter(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( + "humans", {"id": 1, "name": "Natalie D"}, pk="id" + ) + dogs.update(1).m2m( + "humans", {"id": 2, "name": "Simon W", "nerd": True}, pk="id", alter=True + ) + assert list(fresh_db["humans"].rows) == [ + {"id": 1, "name": "Natalie D", "nerd": None}, + {"id": 2, "name": "Simon W", "nerd": 1}, + ] + assert list(fresh_db["dogs_humans"].rows) == [ + {"humans_id": 1, "dogs_id": 1}, + {"humans_id": 2, "dogs_id": 1}, + ] + + def test_insert_m2m_list(fresh_db): dogs = fresh_db["dogs"] dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( From 36dc7e3909a44878681c266b90f9be76ac749f2d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 Jan 2021 20:28:24 -0800 Subject: [PATCH 0298/1004] Release 3.3 Refs #222 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8ba41ae..c790396 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v3_3: + +3.3 (2021-01-17) +---------------- + +- The ``table.m2m()`` method now accepts an optional ``alter=True`` argument to specify that any missing columns should be added to the referenced table. See :ref:`python_api_m2m`. (`#222 `__) + .. _v3_2_1: 3.2.1 (2021-01-12) diff --git a/setup.py b/setup.py index b1a7324..eded07a 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.2.1" +VERSION = "3.3" def get_long_description(): From 1b666f9315d4ea6bb332b2e75e48480c26100199 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Feb 2021 17:34:47 -0800 Subject: [PATCH 0299/1004] --delimiter and --quotechar, closes #223 --- docs/cli.rst | 25 +++++++++++++++++++++++-- sqlite_utils/cli.py | 22 +++++++++++++++++++++- tests/test_cli.py | 20 +++++++++++++++----- 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 974fe68..3ac5c4b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -474,6 +474,8 @@ This also means you pipe ``sqlite-utils`` together to easily create a new SQLite 207368,920 Kirkham St,37.760210314285,-122.47073935813 188702,1501 Evans Ave,37.7422086702947,-122.387293152263 +.. _cli_insert_csv_tsv: + Inserting CSV or TSV data ========================= @@ -483,14 +485,33 @@ If your data is in CSV format, you can insert it using the ``--csv`` option:: For tab-delimited data, use ``--tsv``:: - $ sqlite-utils insert dogs.db dogs docs.tsv --tsv + $ sqlite-utils insert dogs.db dogs dogs.tsv --tsv Data is expected to be encoded as Unicode UTF-8. If your data is an another character encoding you can specify it using the ``--encoding`` option:: - $ sqlite-utils insert dogs.db dogs docs.tsv --tsv --encoding=latin-1 + $ sqlite-utils insert dogs.db dogs dogs.tsv --tsv --encoding=latin-1 A progress bar is displayed when inserting data from a file. You can hide the progress bar using the ``--silent`` option. +.. _cli_insert_csv_tsv_delimiter: + +Alternative delimiters and quote characters +------------------------------------------- + +If your file uses a delimiter other than ``,`` or a quote character other than ``"`` you can specify them using the ``--delimiter`` and ``--quotechar`` options. + +Here's a CSV file that uses ``;`` for delimiters and the ``|`` symbol for quote characters:: + + name;description + Cleo;|Very fine; a friendly dog| + Pancakes;A local corgi + +You can import that using:: + + $ sqlite-utils insert dogs.db dogs dogs.csv --delimiter=";" --quotechar="|" + +Passing either ``--delimiter`` and ``--quotechar`` implies ``--csv``, so you can omit that option. + .. _cli_insert_replace: Insert-replacing data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0bc03fe..48bc737 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,4 +1,5 @@ import base64 +from tests.test_cli import test_query_memory_does_not_create_file import click import codecs from click_default_group import DefaultGroup @@ -601,6 +602,8 @@ def insert_upsert_options(fn): click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), click.option("-c", "--csv", is_flag=True, help="Expect CSV"), click.option("--tsv", is_flag=True, help="Expect TSV"), + click.option("--delimiter", help="Delimiter to use for CSV files"), + click.option("--quotechar", help="Quote character to use for CSV/TSV"), click.option( "--batch-size", type=int, default=100, help="Commit every X records" ), @@ -640,6 +643,8 @@ def insert_upsert_implementation( nl, csv, tsv, + delimiter, + quotechar, batch_size, alter, upsert, @@ -654,6 +659,8 @@ def insert_upsert_implementation( ): db = sqlite_utils.Database(path) _load_extensions(db, load_extension) + if delimiter or quotechar: + csv = True if (nl + csv + tsv) >= 2: raise click.ClickException("Use just one of --nl, --csv or --tsv") if encoding and not (csv or tsv): @@ -665,7 +672,12 @@ def insert_upsert_implementation( if csv or tsv: dialect = "excel-tab" if tsv else "excel" with file_progress(json_file, silent=silent) as json_file: - reader = csv_std.reader(json_file, dialect=dialect) + csv_reader_args = {"dialect": dialect} + if delimiter: + csv_reader_args["delimiter"] = delimiter + if quotechar: + csv_reader_args["quotechar"] = quotechar + reader = csv_std.reader(json_file, **csv_reader_args) headers = next(reader) docs = (dict(zip(headers, row)) for row in reader) else: @@ -720,6 +732,8 @@ def insert( nl, csv, tsv, + delimiter, + quotechar, batch_size, alter, encoding, @@ -746,6 +760,8 @@ def insert( nl, csv, tsv, + delimiter, + quotechar, batch_size, alter=alter, upsert=False, @@ -773,6 +789,8 @@ def upsert( csv, tsv, batch_size, + delimiter, + quotechar, alter, not_null, default, @@ -794,6 +812,8 @@ def upsert( nl, csv, tsv, + delimiter, + quotechar, batch_size, alter=alter, upsert=True, diff --git a/tests/test_cli.py b/tests/test_cli.py index 381997b..52a80e9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -722,18 +722,28 @@ def test_insert_ignore(db_path, tmpdir): @pytest.mark.parametrize( - "content,option", - (("foo\tbar\tbaz\n1\t2\t3", "--tsv"), ("foo,bar,baz\n1,2,3", "--csv")), + "content,options", + [ + ("foo\tbar\tbaz\n1\t2\tcat,dog", ["--tsv"]), + ('foo,bar,baz\n1,2,"cat,dog"', ["--csv"]), + ('foo;bar;baz\n1;2;"cat,dog"', ["--csv", "--delimiter", ";"]), + # --delimiter implies --csv: + ('foo;bar;baz\n1;2;"cat,dog"', ["--delimiter", ";"]), + ("foo,bar,baz\n1,2,|cat,dog|", ["--csv", "--quotechar", "|"]), + ("foo,bar,baz\n1,2,|cat,dog|", ["--quotechar", "|"]), + ], ) -def test_insert_csv_tsv(content, option, db_path, tmpdir): +def test_insert_csv_tsv(content, options, db_path, tmpdir): db = Database(db_path) file_path = str(tmpdir / "insert.csv-tsv") open(file_path, "w").write(content) result = CliRunner().invoke( - cli.cli, ["insert", db_path, "data", file_path, option], catch_exceptions=False + cli.cli, + ["insert", db_path, "data", file_path] + options, + catch_exceptions=False, ) assert 0 == result.exit_code - assert [{"foo": "1", "bar": "2", "baz": "3"}] == list(db["data"].rows) + assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows) @pytest.mark.parametrize( From f8010ca78fed8c5fca6cde19658ec09fdd468420 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Feb 2021 17:37:27 -0800 Subject: [PATCH 0300/1004] Release 3.4 Refs #223 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c790396..ad140d6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v3_4: + +3.4 (2021-02-05) +---------------- + +- ``sqlite-utils insert --csv`` now accepts optional ``--delimiter`` and ``--quotechar`` options. See :ref:`cli_insert_csv_tsv_delimiter`. (`#223 `__) + .. _v3_3: 3.3 (2021-01-17) diff --git a/setup.py b/setup.py index eded07a..dcae521 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.3" +VERSION = "3.4" def get_long_description(): From 7d04565010c644a5f709f6076eca9d2acf3361b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Feb 2021 18:09:48 -0800 Subject: [PATCH 0301/1004] Remove rogue import, refs #226 --- sqlite_utils/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 48bc737..03ac3f5 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,5 +1,4 @@ import base64 -from tests.test_cli import test_query_memory_does_not_create_file import click import codecs from click_default_group import DefaultGroup From 726219c3503e77440975cd15b74d006639feb0f8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Feb 2021 18:10:04 -0800 Subject: [PATCH 0302/1004] Release 3.4.1 Closes #226 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ad140d6..42638bf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v3_4_1: + +3.4.1 (2021-02-05) +------------------ + +- Fixed a code import bug that slipped in to 3.4. (`#226 `__) + .. _v3_4: 3.4 (2021-02-05) diff --git a/setup.py b/setup.py index dcae521..e847dc0 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.4" +VERSION = "3.4.1" def get_long_description(): From 1e9eb875a64dfc65d786f4c6a52f6ba08b25b86b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Feb 2021 10:33:26 -0800 Subject: [PATCH 0303/1004] Switch from codecs.getreader to io.TextIOWrapper, refs #230 --- sqlite_utils/cli.py | 4 ++-- sqlite_utils/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 03ac3f5..3cb61af 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,6 +1,5 @@ import base64 import click -import codecs from click_default_group import DefaultGroup from datetime import datetime import hashlib @@ -8,6 +7,7 @@ import pathlib import sqlite_utils from sqlite_utils.db import AlterError import textwrap +import io import itertools import json import os @@ -665,7 +665,7 @@ def insert_upsert_implementation( if encoding and not (csv or tsv): raise click.ClickException("--encoding must be used with --csv or --tsv") encoding = encoding or "utf-8" - json_file = codecs.getreader(encoding)(json_file) + json_file = io.TextIOWrapper(json_file, encoding=encoding) if pk and len(pk) == 1: pk = pk[0] if csv or tsv: diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index a158b2b..9d5dac6 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -105,9 +105,9 @@ class UpdateWrapper: @contextlib.contextmanager def file_progress(file, silent=False, **kwargs): - if silent or file.raw.fileno() == 0: # 0 = stdin + if silent or file.fileno() == 0: # 0 = stdin yield file else: - file_length = os.path.getsize(file.raw.name) + file_length = os.path.getsize(file.name) with click.progressbar(length=file_length, **kwargs) as bar: yield UpdateWrapper(file, bar.update) From 99ff0a288c08ec2071139c6031eb880fa9c95310 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Feb 2021 11:23:12 -0800 Subject: [PATCH 0304/1004] sqlite-utils insert --sniff option, closes #230 --- docs/cli.rst | 10 ++++++++-- sqlite_utils/cli.py | 28 +++++++++++++++++++++------- tests/sniff/example1.csv | 5 +++++ tests/sniff/example2.csv | 5 +++++ tests/sniff/example3.csv | 5 +++++ tests/sniff/example4.csv | 5 +++++ tests/test_sniff.py | 25 +++++++++++++++++++++++++ 7 files changed, 74 insertions(+), 9 deletions(-) create mode 100644 tests/sniff/example1.csv create mode 100644 tests/sniff/example2.csv create mode 100644 tests/sniff/example3.csv create mode 100644 tests/sniff/example4.csv create mode 100644 tests/test_sniff.py diff --git a/docs/cli.rst b/docs/cli.rst index 3ac5c4b..23b1690 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -498,7 +498,13 @@ A progress bar is displayed when inserting data from a file. You can hide the pr Alternative delimiters and quote characters ------------------------------------------- -If your file uses a delimiter other than ``,`` or a quote character other than ``"`` you can specify them using the ``--delimiter`` and ``--quotechar`` options. +If your file uses a delimiter other than ``,`` or a quote character other than ``"`` you can attempt to detect delimiters or you can specify them explicitly. + +The ``--sniff`` option can be used to attempt to detect the delimiters: + + sqlite-utils insert dogs.db dogs dogs.csv --sniff + +Alternatively, you can specify them using the ``--delimiter`` and ``--quotechar`` options. Here's a CSV file that uses ``;`` for delimiters and the ``|`` symbol for quote characters:: @@ -510,7 +516,7 @@ You can import that using:: $ sqlite-utils insert dogs.db dogs dogs.csv --delimiter=";" --quotechar="|" -Passing either ``--delimiter`` and ``--quotechar`` implies ``--csv``, so you can omit that option. +Passing ``--delimiter``, ``--quotechar`` or ``--sniff`` implies ``--csv``, so you can omit the ``--csv`` option. .. _cli_insert_replace: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 3cb61af..c871232 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -603,6 +603,9 @@ def insert_upsert_options(fn): click.option("--tsv", is_flag=True, help="Expect TSV"), click.option("--delimiter", help="Delimiter to use for CSV files"), click.option("--quotechar", help="Quote character to use for CSV/TSV"), + click.option( + "--sniff", is_flag=True, help="Detect delimiter and quote character" + ), click.option( "--batch-size", type=int, default=100, help="Commit every X records" ), @@ -644,6 +647,7 @@ def insert_upsert_implementation( tsv, delimiter, quotechar, + sniff, batch_size, alter, upsert, @@ -658,33 +662,39 @@ def insert_upsert_implementation( ): db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - if delimiter or quotechar: + if delimiter or quotechar or sniff: csv = True if (nl + csv + tsv) >= 2: raise click.ClickException("Use just one of --nl, --csv or --tsv") if encoding and not (csv or tsv): raise click.ClickException("--encoding must be used with --csv or --tsv") encoding = encoding or "utf-8" - json_file = io.TextIOWrapper(json_file, encoding=encoding) + buffered = io.BufferedReader(json_file, buffer_size=4096) + decoded = io.TextIOWrapper(buffered, encoding=encoding, line_buffering=True) if pk and len(pk) == 1: pk = pk[0] if csv or tsv: - dialect = "excel-tab" if tsv else "excel" - with file_progress(json_file, silent=silent) as json_file: + if sniff: + # Read first 2048 bytes and use that to detect + first_bytes = buffered.peek(2048) + dialect = csv_std.Sniffer().sniff(first_bytes.decode(encoding, "ignore")) + else: + dialect = "excel-tab" if tsv else "excel" + with file_progress(decoded, silent=silent) as decoded: csv_reader_args = {"dialect": dialect} if delimiter: csv_reader_args["delimiter"] = delimiter if quotechar: csv_reader_args["quotechar"] = quotechar - reader = csv_std.reader(json_file, **csv_reader_args) + reader = csv_std.reader(decoded, **csv_reader_args) headers = next(reader) docs = (dict(zip(headers, row)) for row in reader) else: try: if nl: - docs = (json.loads(line) for line in json_file) + docs = (json.loads(line) for line in decoded) else: - docs = json.load(json_file) + docs = json.load(decoded) if isinstance(docs, dict): docs = [docs] except json.decoder.JSONDecodeError: @@ -733,6 +743,7 @@ def insert( tsv, delimiter, quotechar, + sniff, batch_size, alter, encoding, @@ -761,6 +772,7 @@ def insert( tsv, delimiter, quotechar, + sniff, batch_size, alter=alter, upsert=False, @@ -790,6 +802,7 @@ def upsert( batch_size, delimiter, quotechar, + sniff, alter, not_null, default, @@ -813,6 +826,7 @@ def upsert( tsv, delimiter, quotechar, + sniff, batch_size, alter=alter, upsert=True, diff --git a/tests/sniff/example1.csv b/tests/sniff/example1.csv new file mode 100644 index 0000000..3daaadd --- /dev/null +++ b/tests/sniff/example1.csv @@ -0,0 +1,5 @@ +id,species,name,age +1,dog,Cleo,5 +2,dog,Pancakes,4 +3,cat,Mozie,8 +4,spider,"Daisy, the tarantula",6 diff --git a/tests/sniff/example2.csv b/tests/sniff/example2.csv new file mode 100644 index 0000000..0452e7f --- /dev/null +++ b/tests/sniff/example2.csv @@ -0,0 +1,5 @@ +id;species;name;age +1;dog;Cleo;5 +2;dog;Pancakes;4 +3;cat;Mozie;8 +4;spider;"Daisy, the tarantula";6 diff --git a/tests/sniff/example3.csv b/tests/sniff/example3.csv new file mode 100644 index 0000000..172c3d3 --- /dev/null +++ b/tests/sniff/example3.csv @@ -0,0 +1,5 @@ +id,species,name,age +1,dog,Cleo,5 +2,dog,Pancakes,4 +3,cat,Mozie,8 +4,spider,'Daisy, the tarantula',6 diff --git a/tests/sniff/example4.csv b/tests/sniff/example4.csv new file mode 100644 index 0000000..71b671e --- /dev/null +++ b/tests/sniff/example4.csv @@ -0,0 +1,5 @@ +id species name age +1 dog Cleo 5 +2 dog Pancakes 4 +3 cat Mozie 8 +4 spider 'Daisy, the tarantula' 6 diff --git a/tests/test_sniff.py b/tests/test_sniff.py new file mode 100644 index 0000000..36cc471 --- /dev/null +++ b/tests/test_sniff.py @@ -0,0 +1,25 @@ +from sqlite_utils import cli, Database +from click.testing import CliRunner +import pathlib +import pytest + +sniff_dir = pathlib.Path(__file__).parent / "sniff" + + +@pytest.mark.parametrize("filepath", sniff_dir.glob("example*")) +def test_sniff(tmpdir, filepath): + db_path = str(tmpdir / "test.db") + runner = CliRunner() + result = runner.invoke( + cli.cli, + ["insert", db_path, "creatures", str(filepath), "--sniff"], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.stdout + db = Database(db_path) + assert list(db["creatures"].rows) == [ + {"id": "1", "species": "dog", "name": "Cleo", "age": "5"}, + {"id": "2", "species": "dog", "name": "Pancakes", "age": "4"}, + {"id": "3", "species": "cat", "name": "Mozie", "age": "8"}, + {"id": "4", "species": "spider", "name": "Daisy, the tarantula", "age": "6"}, + ] From cf811e35e1cbb78cd0347e73c2b747d1f4b8497d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Feb 2021 11:25:58 -0800 Subject: [PATCH 0305/1004] Formatting fix --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 23b1690..e32b21e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -500,7 +500,7 @@ Alternative delimiters and quote characters If your file uses a delimiter other than ``,`` or a quote character other than ``"`` you can attempt to detect delimiters or you can specify them explicitly. -The ``--sniff`` option can be used to attempt to detect the delimiters: +The ``--sniff`` option can be used to attempt to detect the delimiters:: sqlite-utils insert dogs.db dogs dogs.csv --sniff From 320f3ac33a83b32f89559ef0c162b7eca428a278 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Feb 2021 12:02:41 -0800 Subject: [PATCH 0306/1004] offset= and limit= parameters, closes #231 --- docs/python-api.rst | 9 ++++++++ sqlite_utils/db.py | 28 +++++++++++++++++++---- tests/test_fts.py | 56 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_rows.py | 17 ++++++++++++++ 4 files changed, 105 insertions(+), 5 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index c656045..246a301 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -202,6 +202,12 @@ You can order all records in the table by excluding the ``where`` argument:: {'id': 1, 'age': 4, 'name': 'Cleo'} {'id': 2, 'age': 2, 'name': 'Pancakes'} +This method also accepts ``offset=`` and ``limit=`` arguments, for specifying an OFFSET and a LIMIT for the SQL query:: + + >>> for row in db["dogs"].rows_where(order_by="age desc", limit=1): + ... print(row) + {'id': 1, 'age': 4, 'name': 'Cleo'} + .. _python_api_get: Retrieving a specific record @@ -1603,6 +1609,9 @@ The ``.search()`` method also accepts the following optional parameters: ``limit`` integer Number of results to return. Defaults to all results. +``offset`` integer + Offset to use along side the limit parameter. + To return just the title and published columns for three matches for ``"dog"`` ordered by ``published`` with the most recent first, use the following: .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 14a9efc..17d79de 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -644,7 +644,15 @@ class Queryable: def rows(self): return self.rows_where() - def rows_where(self, where=None, where_args=None, order_by=None, select="*"): + def rows_where( + self, + where=None, + where_args=None, + order_by=None, + select="*", + limit=None, + offset=None, + ): if not self.exists(): return [] sql = "select {} from [{}]".format(select, self.name) @@ -652,6 +660,10 @@ class Queryable: sql += " where " + where if order_by is not None: sql += " order by " + order_by + if limit is not None: + sql += " limit {}".format(limit) + if offset is not None: + sql += " offset {}".format(offset) cursor = self.db.execute(sql, where_args or []) columns = [c[0] for c in cursor.description] for row in cursor: @@ -1454,7 +1466,7 @@ class Table(Queryable): ) return self - def search_sql(self, columns=None, order_by=None, limit=None): + def search_sql(self, columns=None, order_by=None, limit=None, offset=None): # Pick names for table and rank column that don't clash original = "original_" if self.name == "original" else "original" columns_sql = "*" @@ -1486,7 +1498,7 @@ class Table(Queryable): [{fts_table}] match :query order by {order_by} - {limit} + {limit_offset} """ ).strip() if virtual_table_using == "FTS5": @@ -1496,6 +1508,11 @@ class Table(Queryable): rank_implementation = "rank_bm25(matchinfo([{}], 'pcnalx'))".format( fts_table ) + limit_offset = "" + if limit is not None: + limit_offset += " limit {}".format(limit) + if offset is not None: + limit_offset += " offset {}".format(offset) return sql.format( dbtable=self.name, original=original, @@ -1503,15 +1520,16 @@ class Table(Queryable): columns_with_prefix=columns_with_prefix_sql, fts_table=fts_table, order_by=order_by or rank_implementation, - limit="limit {}".format(limit) if limit else "", + limit_offset=limit_offset.strip(), ).strip() - def search(self, q, order_by=None, columns=None, limit=None): + def search(self, q, order_by=None, columns=None, limit=None, offset=None): cursor = self.db.execute( self.search_sql( order_by=order_by, columns=columns, limit=limit, + offset=offset, ), {"query": q}, ) diff --git a/tests/test_fts.py b/tests/test_fts.py index 06fac20..94650bf 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -82,6 +82,18 @@ def test_enable_fts_escape_table_names(fresh_db): assert [] == list(table.search("bar")) +def test_search_limit_offset(fresh_db): + table = fresh_db["t"] + table.insert_all(search_records) + table.enable_fts(["text", "country"], fts_version="FTS4") + assert len(list(table.search("are"))) == 2 + assert len(list(table.search("are", limit=1))) == 1 + assert list(table.search("are", limit=1, order_by="rowid"))[0]["rowid"] == 1 + assert ( + list(table.search("are", limit=1, offset=1, order_by="rowid"))[0]["rowid"] == 2 + ) + + def test_enable_fts_table_names_containing_spaces(fresh_db): table = fresh_db["test"] table.insert({"column with spaces": "in its name"}) @@ -424,6 +436,50 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): " rank_bm25(matchinfo([books_fts], 'pcnalx'))" ), ), + ( + {"offset": 1, "limit": 1}, + "FTS4", + ( + "with original as (\n" + " select\n" + " rowid,\n" + " *\n" + " from [books]\n" + ")\n" + "select\n" + " [original].*\n" + "from\n" + " [original]\n" + " join [books_fts] on [original].rowid = [books_fts].rowid\n" + "where\n" + " [books_fts] match :query\n" + "order by\n" + " rank_bm25(matchinfo([books_fts], 'pcnalx'))\n" + "limit 1 offset 1" + ), + ), + ( + {"limit": 2}, + "FTS4", + ( + "with original as (\n" + " select\n" + " rowid,\n" + " *\n" + " from [books]\n" + ")\n" + "select\n" + " [original].*\n" + "from\n" + " [original]\n" + " join [books_fts] on [original].rowid = [books_fts].rowid\n" + "where\n" + " [books_fts] match :query\n" + "order by\n" + " rank_bm25(matchinfo([books_fts], 'pcnalx'))\n" + "limit 2" + ), + ), ], ) def test_search_sql(kwargs, fts, expected): diff --git a/tests/test_rows.py b/tests/test_rows.py index 603f254..6d995e2 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -51,3 +51,20 @@ def test_rows_where_order_by(where, order_by, expected_ids, fresh_db): pk="id", ) assert expected_ids == [r["id"] for r in table.rows_where(where, order_by=order_by)] + + +@pytest.mark.parametrize( + "offset,limit,expected", + [ + (None, 3, [1, 2, 3]), + (0, 3, [1, 2, 3]), + (3, 3, [4, 5, 6]), + ], +) +def test_rows_where_offset_limit(fresh_db, offset, limit, expected): + table = fresh_db["rows"] + table.insert_all([{"id": id} for id in range(1, 101)], pk="id") + assert table.count == 100 + assert expected == [ + r["id"] for r in table.rows_where(offset=offset, limit=limit, order_by="id") + ] From f51a1f6c3cb2929bcf79cb4efe3b2a9886d9c25c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Feb 2021 12:39:54 -0800 Subject: [PATCH 0307/1004] Run tests against Ubuntu, macOS and Windows With tests fixes for Windows, thanks to @nieuwenhoven in #225. Closes #232 --- .github/workflows/test.yml | 3 ++- tests/test_cli.py | 9 +++++---- tests/test_insert_files.py | 5 +++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4f2b030..f8a79ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,11 +4,12 @@ on: [push] jobs: test: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] numpy: [0, 1] + os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/tests/test_cli.py b/tests/test_cli.py index 52a80e9..e324bdd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -93,7 +93,7 @@ def test_tables_counts_and_columns_csv(db_path, format, expected): result = CliRunner().invoke( cli.cli, ["tables", "--counts", "--columns", format, db_path] ) - assert result.output.strip() == expected + assert result.output.strip().replace("\r", "") == expected def test_tables_schema(db_path): @@ -870,12 +870,13 @@ def test_query_csv(db_path, format, expected): cli.cli, [db_path, "select id, name, age from dogs", format] ) assert 0 == result.exit_code - assert result.output == expected + assert result.output.replace("\r", "") == expected # Test the no-headers option: result = CliRunner().invoke( cli.cli, [db_path, "select id, name, age from dogs", "--no-headers", format] ) - assert result.output.strip() == "\n".join(expected.split("\n")[1:]).strip() + expected_rest = "\n".join(expected.split("\n")[1:]).strip() + assert result.output.strip().replace("\r", "") == expected_rest _all_query = "select id, name, age from dogs" @@ -1760,7 +1761,7 @@ def test_search(tmpdir, fts, extra_arg, expected): catch_exceptions=False, ) assert result.exit_code == 0 - assert result.output == expected + assert result.output.replace("\r", "") == expected _TRIGGERS_EXPECTED = '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah AFTER INSERT ON articles\\nBEGIN\\n UPDATE counter SET count = count + 1;\\nEND"}]\n' diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index f17f279..1a11d2c 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -1,5 +1,6 @@ from sqlite_utils import cli, Database from click.testing import CliRunner +import os import pathlib @@ -42,7 +43,7 @@ def test_insert_files(): one, two, three = ( rows_by_path["one.txt"], rows_by_path["two.txt"], - rows_by_path["nested/three.txt"], + rows_by_path[os.path.join("nested", "three.txt")], ) assert { "content": b"This is file one", @@ -64,7 +65,7 @@ def test_insert_files(): "content": b"Three is nested", "md5": "12580f341781f5a5b589164d3cd39523", "name": "three.txt", - "path": "nested/three.txt", + "path": os.path.join("nested", "three.txt"), "sha256": "6dd45aaaaa6b9f96af19363a92c8fca5d34791d3c35c44eb19468a6a862cc8cd", "size": 15, }.items() <= three.items() From 8f042ae1fd323995d966a94e8e6df85cc843b938 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Feb 2021 13:03:17 -0800 Subject: [PATCH 0308/1004] Fix for bug with extra columns in later chunks, closes #234 Thanks @nieuwenhoven for the fix, proposed in #225 --- sqlite_utils/db.py | 10 ++++------ tests/test_create.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 17d79de..b6315dd 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1904,12 +1904,10 @@ class Table(Queryable): if hash_id: all_columns.insert(0, hash_id) else: - all_columns += [ - column - for record in chunk - for column in record - if column not in all_columns - ] + for record in chunk: + all_columns += [ + column for column in record if column not in all_columns + ] validate_column_names(all_columns) first = False diff --git a/tests/test_create.py b/tests/test_create.py index bee47a7..2975dea 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -569,6 +569,22 @@ def test_insert_replace_rows_alter_table(fresh_db, use_table_factory): ] == list(table.rows) +def test_insert_all_with_extra_columns_in_later_chunks(fresh_db): + chunk = [ + {"record": "Record 1"}, + {"record": "Record 2"}, + {"record": "Record 3"}, + {"record": "Record 4", "extra": 1}, + ] + fresh_db["t"].insert_all(chunk, batch_size=2, alter=True) + assert list(fresh_db["t"].rows) == [ + {"record": "Record 1", "extra": None}, + {"record": "Record 2", "extra": None}, + {"record": "Record 3", "extra": None}, + {"record": "Record 4", "extra": 1}, + ] + + def test_bulk_insert_more_than_999_values(fresh_db): "Inserting 100 items with 11 columns should work" fresh_db["big"].insert_all( From 1a93b72ba710ea2271eaabc204685a27d2469374 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Feb 2021 13:33:21 -0800 Subject: [PATCH 0309/1004] Increase csv field_size_limit to maximum, closes #229 Refs #227 --- sqlite_utils/cli.py | 12 ++++++++++++ tests/test_cli.py | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c871232..5fc181a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -31,6 +31,18 @@ It's often worth trying: --encoding=latin-1 """.strip() +# Increase CSV field size limit to maximim possible +# https://stackoverflow.com/a/15063941 +field_size_limit = sys.maxsize + +while True: + try: + csv_std.field_size_limit(field_size_limit) + break + except OverflowError: + field_size_limit = int(field_size_limit / 10) + + def output_options(fn): for decorator in reversed( ( diff --git a/tests/test_cli.py b/tests/test_cli.py index e324bdd..a3669ab 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1799,3 +1799,23 @@ def test_triggers(tmpdir, extra_args, expected): ) assert result.exit_code == 0 assert result.output == expected + + +def test_long_csv_column_value(tmpdir): + db_path = str(tmpdir / "test.db") + csv_path = str(tmpdir / "test.csv") + csv_file = open(csv_path, "w") + long_string = "a" * 131073 + csv_file.write("id,text\n") + csv_file.write("1,{}\n".format(long_string)) + csv_file.close() + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "bigtable", csv_path, "--csv"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + db = Database(db_path) + rows = list(db["bigtable"].rows) + assert len(rows) == 1 + assert rows[0]["text"] == long_string From 67cce7c86139426bd8a5c60c8b48c01bd53bebe4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Feb 2021 13:35:12 -0800 Subject: [PATCH 0310/1004] Run publish tests on macOS and Windows too, refs #232 --- .github/workflows/publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3755c3a..9f05567 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,10 +6,11 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] + os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 427dace184c7da57f4a04df07b1e84cdae3261e8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Feb 2021 13:36:43 -0800 Subject: [PATCH 0311/1004] Added --csv example to README --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fa78d9d..e8b8245 100644 --- a/README.md +++ b/README.md @@ -49,11 +49,15 @@ Now you can do things with the CLI utility like this: 1 4 Cleo 2 2 Pancakes -You can even import data into a new database table like this: +You can import JSON data into a new database table like this: $ curl https://api.github.com/repos/simonw/sqlite-utils/releases \ | sqlite-utils insert releases.db releases - --pk id +Or for data in a CSV file: + + $ sqlite-utils insert dogs.db dogs docs.csv --csv + See the [full CLI documentation](https://sqlite-utils.datasette.io/en/stable/cli.html) for comprehensive coverage of many more commands. ## Using as a library From 50d2096f5ed718df5a6704c2ea265f44d6e9907f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Feb 2021 14:25:03 -0800 Subject: [PATCH 0312/1004] --no-headers option for sqlite-utils insert --csv, closes #228 --- docs/cli.rst | 11 +++++++++++ sqlite_utils/cli.py | 17 +++++++++++++++-- tests/test_cli.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index e32b21e..c07a74f 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -518,6 +518,17 @@ You can import that using:: Passing ``--delimiter``, ``--quotechar`` or ``--sniff`` implies ``--csv``, so you can omit the ``--csv`` option. +.. _cli_insert_csv_tsv_no_header: + +CSV files without a header row +------------------------------ + +The first row of any CSV or TSV file is expected to contain the names of the columns in that file. + +If your file does not include this row, you can use the ``--no-headers`` option to specify that the tool should not use that fist row as headers. + +If you do this, the table will be created with column names called ``untitled_1`` and ``untitled_2`` and so on. You can then rename them using the ``sqlite-utils transform ... --rename`` command, see :ref:`cli_transform_table`. + .. _cli_insert_replace: Insert-replacing data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 5fc181a..e2862d5 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -618,6 +618,9 @@ def insert_upsert_options(fn): click.option( "--sniff", is_flag=True, help="Detect delimiter and quote character" ), + click.option( + "--no-headers", is_flag=True, help="CSV file has no header row" + ), click.option( "--batch-size", type=int, default=100, help="Commit every X records" ), @@ -660,6 +663,7 @@ def insert_upsert_implementation( delimiter, quotechar, sniff, + no_headers, batch_size, alter, upsert, @@ -674,7 +678,7 @@ def insert_upsert_implementation( ): db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - if delimiter or quotechar or sniff: + if delimiter or quotechar or sniff or no_headers: csv = True if (nl + csv + tsv) >= 2: raise click.ClickException("Use just one of --nl, --csv or --tsv") @@ -699,7 +703,12 @@ def insert_upsert_implementation( if quotechar: csv_reader_args["quotechar"] = quotechar reader = csv_std.reader(decoded, **csv_reader_args) - headers = next(reader) + first_row = next(reader) + if no_headers: + headers = ["untitled_{}".format(i + 1) for i in range(len(first_row))] + reader = itertools.chain([first_row], reader) + else: + headers = first_row docs = (dict(zip(headers, row)) for row in reader) else: try: @@ -756,6 +765,7 @@ def insert( delimiter, quotechar, sniff, + no_headers, batch_size, alter, encoding, @@ -785,6 +795,7 @@ def insert( delimiter, quotechar, sniff, + no_headers, batch_size, alter=alter, upsert=False, @@ -815,6 +826,7 @@ def upsert( delimiter, quotechar, sniff, + no_headers, alter, not_null, default, @@ -839,6 +851,7 @@ def upsert( delimiter, quotechar, sniff, + no_headers, batch_size, alter=alter, upsert=True, diff --git a/tests/test_cli.py b/tests/test_cli.py index a3669ab..ce6e7d5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1819,3 +1819,39 @@ def test_long_csv_column_value(tmpdir): rows = list(db["bigtable"].rows) assert len(rows) == 1 assert rows[0]["text"] == long_string + + +@pytest.mark.parametrize( + "args", + ( + ["--csv", "--no-headers"], + ["--no-headers"], + ), +) +def test_csv_import_no_headers(tmpdir, args): + db_path = str(tmpdir / "test.db") + csv_path = str(tmpdir / "test.csv") + csv_file = open(csv_path, "w") + csv_file.write("Cleo,Dog,5\n") + csv_file.write("Tracy,Spider,7\n") + csv_file.close() + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", csv_path] + args, + catch_exceptions=False, + ) + assert result.exit_code == 0 + db = Database(db_path) + schema = db["creatures"].schema + assert schema == ( + "CREATE TABLE [creatures] (\n" + " [untitled_1] TEXT,\n" + " [untitled_2] TEXT,\n" + " [untitled_3] TEXT\n" + ")" + ) + rows = list(db["creatures"].rows) + assert rows == [ + {"untitled_1": "Cleo", "untitled_2": "Dog", "untitled_3": "5"}, + {"untitled_1": "Tracy", "untitled_2": "Spider", "untitled_3": "7"}, + ] From ef13bb046f525f33cda7cd56a12093a5071a3cb6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Feb 2021 14:34:22 -0800 Subject: [PATCH 0313/1004] Useful error message for enable_fts() on views, closes #220 --- sqlite_utils/db.py | 5 +++++ tests/test_fts.py | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b6315dd..0bf93bd 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2162,6 +2162,11 @@ class View(Queryable): def drop(self): self.db.execute("DROP VIEW [{}]".format(self.name)) + def enable_fts(self, *args, **kwargs): + raise NotImplementedError( + "enable_fts() is supported on tables but not on views" + ) + def chunks(sequence, size): iterator = iter(sequence) diff --git a/tests/test_fts.py b/tests/test_fts.py index 94650bf..a3efa54 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -369,6 +369,14 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): assert all(q[0].startswith("select ") for q in queries) +def test_enable_fts_error_message_on_views(): + db = Database(memory=True) + db.create_view("hello", "select 1 + 1") + with pytest.raises(NotImplementedError) as e: + db["hello"].enable_fts() + assert e.value.args[0] == "enable_fts() is supported on tables but not on views" + + @pytest.mark.parametrize( "kwargs,fts,expected", [ From 8fcaee03b718754cc429e03d74ac3ac5d49da92f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Feb 2021 14:43:06 -0800 Subject: [PATCH 0314/1004] Release 3.5 Refs #228, #229, #230, #231, #232, #234 --- docs/changelog.rst | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 42638bf..e197c6c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,18 @@ Changelog =========== +.. _v3_5: + +3.5 (2021-02-14) +---------------- + +- ``sqlite-utils insert --sniff`` option for detecting the delimiter and quote character used by a CSV file, see :ref:`cli_insert_csv_tsv_delimiter`. (`#230 `__) +- The ``table.rows_where()``, ``table.search()`` and ``table.search_sql()`` methods all now take optional ``offset=`` and ``limit=`` arguments. (`#231 `__) +- New ``--no-headers`` option for ``sqlite-utils insert --csv`` to handle CSV files that are missing the header row, see :ref:`cli_insert_csv_tsv_no_header`. (`#228 `__) +- Fixed bug where inserting data with extra columns in subsequent chunks would throw an error. Thanks `@nieuwenhoven `__ for the fix. (`#234 `__) +- Fixed bug importing CSV files with columns containing more than 128KB of data. (`#229 `__) +- Test suite now runs in CI against Ubuntu, macOS and Windows. Thanks `@nieuwenhoven `__ for the Windows test fixes. (`#232 `__) + .. _v3_4_1: 3.4.1 (2021-02-05) diff --git a/setup.py b/setup.py index e847dc0..bdcb829 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.4.1" +VERSION = "3.5" def get_long_description(): From 1f49f32814a942fa076cfe5f504d1621188097ed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 15 Feb 2021 11:18:28 -0800 Subject: [PATCH 0315/1004] Don't need line_buffering=True here, refs #230 --- sqlite_utils/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index e2862d5..4cb080f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -686,7 +686,7 @@ def insert_upsert_implementation( raise click.ClickException("--encoding must be used with --csv or --tsv") encoding = encoding or "utf-8" buffered = io.BufferedReader(json_file, buffer_size=4096) - decoded = io.TextIOWrapper(buffered, encoding=encoding, line_buffering=True) + decoded = io.TextIOWrapper(buffered, encoding=encoding) if pk and len(pk) == 1: pk = pk[0] if csv or tsv: From 2c1b9f2445d0ca4ca9f30a1433b7cde8cc0f42a2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Feb 2021 10:22:43 -0800 Subject: [PATCH 0316/1004] Create FUNDING.yml --- .github/FUNDING.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..f0bcdbe --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: [simonw] From 999f099cbe267554f679963a3964042f09c1c159 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Feb 2021 20:56:32 -0800 Subject: [PATCH 0317/1004] db.attach(alias, filepath) method, closes #113 Will also be useful for #236 --- docs/python-api.rst | 21 +++++++++++++++++++++ sqlite_utils/db.py | 8 ++++++++ tests/test_attach.py | 16 ++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 tests/test_attach.py diff --git a/docs/python-api.rst b/docs/python-api.rst index 246a301..d55f1f0 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -45,6 +45,27 @@ Connections use ``PRAGMA recursive_triggers=on`` by default. If you don't want t db = Database(memory=True, recursive_triggers=False) +.. _python_api_attach: + +Attaching additional databases +------------------------------ + +SQLite supports cross-database SQL queries, which can join data from tables in more than one database file. You can attach an additional database using the ``.attach()`` method, providing an alias to use for that database and the path to the SQLite file on disk. + +.. code-block:: python + + db = Database("first.db") + db.attach("second", "second.db") + # Now you can run queries like this one: + cursor = db.execute(""" + select * from table_in_first + union all + select * from second.table_in_second + """) + print(cursor.fetchall()) + +You can reference tables in the attached database using the alias value you passed to ``db.attach(alias, filepath)`` as a prefix, for example the ``second.table_in_second`` reference in the SQL query above. + .. _python_api_tracing: Tracing queries diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0bf93bd..971bf6f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -219,6 +219,14 @@ class Database: def register_fts4_bm25(self): self.register_function(rank_bm25, deterministic=True) + def attach(self, alias, filepath): + attach_sql = """ + ATTACH DATABASE '{}' AS [{}]; + """.format( + str(pathlib.Path(filepath).resolve()), alias + ).strip() + self.execute(attach_sql) + def execute(self, sql, parameters=None): if self._tracer: self._tracer(sql, parameters) diff --git a/tests/test_attach.py b/tests/test_attach.py new file mode 100644 index 0000000..b594b3b --- /dev/null +++ b/tests/test_attach.py @@ -0,0 +1,16 @@ +from sqlite_utils import Database + + +def test_attach(tmpdir): + foo_path = str(tmpdir / "foo.db") + bar_path = str(tmpdir / "bar.db") + db = Database(foo_path) + with db.conn: + db["foo"].insert({"id": 1, "text": "foo"}) + db2 = Database(bar_path) + with db2.conn: + db2["bar"].insert({"id": 1, "text": "bar"}) + db.attach("bar", bar_path) + assert db.execute( + "select * from foo union all select * from bar.bar" + ).fetchall() == [(1, "foo"), (1, "bar")] From 2ba558888131b58ed13bccea29e0db20c9c01087 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Feb 2021 21:08:39 -0800 Subject: [PATCH 0318/1004] sqlite-utils --attach option, closes #236 --- docs/cli.rst | 16 +++++++++++++++- docs/python-api.rst | 4 +++- sqlite_utils/cli.py | 9 +++++++++ tests/test_cli.py | 22 ++++++++++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index c07a74f..b3fbc1a 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -122,7 +122,21 @@ You can use the ``--json-cols`` option to automatically detect these JSON column } ] } - ]- + ] + +.. cli_attach: + +Attaching additional databases +------------------------------ + +SQLite supports cross-database SQL queries, which can join data from tables in more than one database file. + +You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk. + +This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database:: + + sqlite-utils dogs.db --attach books books.db \ + 'select * from sqlite_master union all select * from books.sqlite_master' .. _cli_query_csv: diff --git a/docs/python-api.rst b/docs/python-api.rst index d55f1f0..def0b14 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -50,7 +50,9 @@ Connections use ``PRAGMA recursive_triggers=on`` by default. If you don't want t Attaching additional databases ------------------------------ -SQLite supports cross-database SQL queries, which can join data from tables in more than one database file. You can attach an additional database using the ``.attach()`` method, providing an alias to use for that database and the path to the SQLite file on disk. +SQLite supports cross-database SQL queries, which can join data from tables in more than one database file. + +You can attach an additional database using the ``.attach()`` method, providing an alias to use for that database and the path to the SQLite file on disk. .. code-block:: python diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4cb080f..4b51565 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1020,6 +1020,12 @@ def drop_view(path, view, load_extension): required=True, ) @click.argument("sql") +@click.option( + "--attach", + type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), + multiple=True, + help="Additional databases to attach - specify alias and filepath", +) @output_options @click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") @click.option( @@ -1033,6 +1039,7 @@ def drop_view(path, view, load_extension): def query( path, sql, + attach, nl, arrays, csv, @@ -1047,6 +1054,8 @@ def query( ): "Execute SQL query and return the results as JSON" db = sqlite_utils.Database(path) + for alias, attach_path in attach: + db.attach(alias, attach_path) _load_extensions(db, load_extension) db.register_fts4_bm25() with db.conn: diff --git a/tests/test_cli.py b/tests/test_cli.py index ce6e7d5..988b41f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1855,3 +1855,25 @@ def test_csv_import_no_headers(tmpdir, args): {"untitled_1": "Cleo", "untitled_2": "Dog", "untitled_3": "5"}, {"untitled_1": "Tracy", "untitled_2": "Spider", "untitled_3": "7"}, ] + + +def test_attach(tmpdir): + foo_path = str(tmpdir / "foo.db") + bar_path = str(tmpdir / "bar.db") + db = Database(foo_path) + with db.conn: + db["foo"].insert({"id": 1, "text": "foo"}) + db2 = Database(bar_path) + with db2.conn: + db2["bar"].insert({"id": 1, "text": "bar"}) + db.attach("bar", bar_path) + sql = "select * from foo union all select * from bar.bar" + result = CliRunner().invoke( + cli.cli, + [foo_path, "--attach", "bar", bar_path, sql], + catch_exceptions=False, + ) + assert json.loads(result.output) == [ + {"id": 1, "text": "foo"}, + {"id": 1, "text": "bar"}, + ] From 806c21044ac8d31da35f4c90600e98115aade7c6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Feb 2021 21:18:02 -0800 Subject: [PATCH 0319/1004] Release 3.6 Refs #113, #236 --- docs/changelog.rst | 10 ++++++++++ docs/cli.rst | 2 +- setup.py | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e197c6c..9d4770f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,16 @@ Changelog =========== +.. _v3_6: + +3.6 (2021-02-18) +---------------- + +This release adds the ability to execute queries joining data from more than one database file - similar to the cross database querying feature introduced in `Datasette 0.55 `__. + +- The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation `. (`#113 `__) +- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (`#236 `__) + .. _v3_5: 3.5 (2021-02-14) diff --git a/docs/cli.rst b/docs/cli.rst index b3fbc1a..c015d78 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -124,7 +124,7 @@ You can use the ``--json-cols`` option to automatically detect these JSON column } ] -.. cli_attach: +.. _cli_attach: Attaching additional databases ------------------------------ diff --git a/setup.py b/setup.py index bdcb829..6e7f978 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.5" +VERSION = "3.6" def get_long_description(): From 38e688fb8bcb58ae888b676fe3f7dd0529b4eecc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 Feb 2021 08:28:17 -0800 Subject: [PATCH 0320/1004] table.pks_and_rows_where() method, closes #240 --- docs/python-api.rst | 34 ++++++++++++++++++++++++++++++++++ sqlite_utils/db.py | 28 ++++++++++++++++++++++++++++ tests/test_rows.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index def0b14..c0eb593 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -231,6 +231,40 @@ This method also accepts ``offset=`` and ``limit=`` arguments, for specifying an ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} +.. _python_api_pks_and_rows_where: + +Listing rows with their primary keys +==================================== + +Sometimes it can be useful to retrieve the primary key along with each row, in order to pass that key (or primary key tuple) to the ``.get()`` or ``.update()`` methods. + +The ``.pks_and_rows_where()`` method takes the same signature as ``.rows_where()`` (with the exception of the ``select=`` parameter) but returns a generator that yields pairs of ``(primary key, row dictionary)``. + +The primary key value will usually be a single value but can also be a tuple if the table has a compound primary key. + +If the table is a ``rowid`` table (with no explicit primary key column) then that ID will be returned. + +:: + + >>> db = sqlite_utils.Database(memory=True) + >>> db["dogs"].insert({"name": "Cleo"}) + >>> for pk, row in db["dogs"].pks_and_rows_where(): + ... print(pk, row) + 1 {'rowid': 1, 'name': 'Cleo'} + + >>> db["dogs_with_pk"].insert({"id": 5, "name": "Cleo"}, pk="id") + >>> for pk, row in db["dogs_with_pk"].pks_and_rows_where(): + ... print(pk, row) + 5 {'id': 5, 'name': 'Cleo'} + + >>> db["dogs_with_compound_pk"].insert( + ... {"species": "dog", "id": 3, "name": "Cleo"}, + ... pk=("species", "id") + ... ) + >>> for pk, row in db["dogs_with_compound_pk"].pks_and_rows_where(): + ... print(pk, row) + ('dog', 3) {'species': 'dog', 'id': 3, 'name': 'Cleo'} + .. _python_api_get: Retrieving a specific record diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 971bf6f..97c55b7 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -677,6 +677,34 @@ class Queryable: for row in cursor: yield dict(zip(columns, row)) + def pks_and_rows_where( + self, + where=None, + where_args=None, + order_by=None, + limit=None, + offset=None, + ): + "Like .rows_where() but returns (pk, row) pairs - pk can be a single value or tuple" + column_names = [column.name for column in self.columns] + pks = [column.name for column in self.columns if column.is_pk] + if not pks: + column_names.insert(0, "rowid") + pks = ["rowid"] + select = ",".join("[{}]".format(column_name) for column_name in column_names) + for row in self.rows_where( + select=select, + where=where, + where_args=where_args, + order_by=order_by, + limit=limit, + offset=offset, + ): + row_pk = tuple(row[pk] for pk in pks) + if len(row_pk) == 1: + row_pk = row_pk[0] + yield row_pk, row + @property def columns(self): if not self.exists(): diff --git a/tests/test_rows.py b/tests/test_rows.py index 6d995e2..3d33ecb 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -68,3 +68,40 @@ def test_rows_where_offset_limit(fresh_db, offset, limit, expected): assert expected == [ r["id"] for r in table.rows_where(offset=offset, limit=limit, order_by="id") ] + + +def test_pks_and_rows_where_rowid(fresh_db): + table = fresh_db["rowid_table"] + table.insert_all({"number": i + 10} for i in range(3)) + pks_and_rows = list(table.pks_and_rows_where()) + assert pks_and_rows == [ + (1, {"rowid": 1, "number": 10}), + (2, {"rowid": 2, "number": 11}), + (3, {"rowid": 3, "number": 12}), + ] + + +def test_pks_and_rows_where_simple_pk(fresh_db): + table = fresh_db["simple_pk_table"] + table.insert_all(({"id": i + 10} for i in range(3)), pk="id") + pks_and_rows = list(table.pks_and_rows_where()) + assert pks_and_rows == [ + (10, {"id": 10}), + (11, {"id": 11}), + (12, {"id": 12}), + ] + + +def test_pks_and_rows_where_compound_pk(fresh_db): + table = fresh_db["compound_pk_table"] + table.insert_all( + ({"type": "number", "number": i, "plusone": i + 1} for i in range(3)), + pk=("type", "number"), + ) + pks_and_rows = list(table.pks_and_rows_where()) + assert pks_and_rows == [ + (("number", 0), {"type": "number", "number": 0, "plusone": 1}), + (("number", 1), {"type": "number", "number": 1, "plusone": 2}), + (("number", 2), {"type": "number", "number": 2, "plusone": 3}), + ] + assert False From 16987bd56ef04ed1f1629b58272d8592c3a13249 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 Feb 2021 08:35:24 -0800 Subject: [PATCH 0321/1004] Removed rogue assert False, refs #240 --- tests/test_rows.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_rows.py b/tests/test_rows.py index 3d33ecb..73bd94f 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -104,4 +104,3 @@ def test_pks_and_rows_where_compound_pk(fresh_db): (("number", 1), {"type": "number", "number": 1, "plusone": 2}), (("number", 2), {"type": "number", "number": 2, "plusone": 3}), ] - assert False From a76e3b33ac983355e78169b11f2b27cacb54b551 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 Feb 2021 08:53:53 -0800 Subject: [PATCH 0322/1004] Fixed bug with add_foreign_key against columns with spaces, closes #238 --- sqlite_utils/db.py | 2 +- tests/test_cli.py | 14 +++++++------- tests/test_create.py | 23 +++++++++++++++++++---- tests/test_extract.py | 10 +++++----- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 97c55b7..246064c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -593,7 +593,7 @@ class Database: table_sql = {} for table, column, other_table, other_column in foreign_keys_to_create: old_sql = table_sql.get(table, self[table].schema) - extra_sql = ",\n FOREIGN KEY({column}) REFERENCES {other_table}({other_column})\n".format( + extra_sql = ",\n FOREIGN KEY([{column}]) REFERENCES [{other_table}]([{other_column}])\n".format( column=column, other_table=other_table, other_column=other_column ) # Stick that bit in at the very end just before the closing ')' diff --git a/tests/test_cli.py b/tests/test_cli.py index 988b41f..b9d5246 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -322,7 +322,7 @@ def test_add_column_foreign_key(db_path): ) assert 0 == result.exit_code, result.output assert ( - "CREATE TABLE [books] ( [title] TEXT , [author_id] INTEGER, FOREIGN KEY(author_id) REFERENCES authors(id) )" + "CREATE TABLE [books] ( [title] TEXT , [author_id] INTEGER, FOREIGN KEY([author_id]) REFERENCES [authors]([id]) )" == collapse_whitespace(db["books"].schema) ) # Try it again with a custom --fk-col @@ -342,8 +342,8 @@ def test_add_column_foreign_key(db_path): assert 0 == result.exit_code, result.output assert ( "CREATE TABLE [books] ( [title] TEXT , [author_id] INTEGER, [author_name_ref] TEXT, " - "FOREIGN KEY(author_id) REFERENCES authors(id), " - "FOREIGN KEY(author_name_ref) REFERENCES authors(name) )" + "FOREIGN KEY([author_id]) REFERENCES [authors]([id]), " + "FOREIGN KEY([author_name_ref]) REFERENCES [authors]([name]) )" == collapse_whitespace(db["books"].schema) ) # Throw an error if the --fk table does not exist @@ -1640,22 +1640,22 @@ _common_other_schema = ( [ ( [], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY(species_id) REFERENCES species(id)\n)', + 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)', _common_other_schema, ), ( ["--table", "custom_table"], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_table_id] INTEGER,\n FOREIGN KEY(custom_table_id) REFERENCES custom_table(id)\n)', + 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_table_id] INTEGER,\n FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n)', "CREATE TABLE [custom_table] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)", ), ( ["--fk-column", "custom_fk"], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_fk] INTEGER,\n FOREIGN KEY(custom_fk) REFERENCES species(id)\n)', + 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_fk] INTEGER,\n FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n)', _common_other_schema, ), ( ["--rename", "name", "name2"], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY(species_id) REFERENCES species(id)\n)', + 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)', "CREATE TABLE [species] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)", ), ], diff --git a/tests/test_create.py b/tests/test_create.py index 2975dea..09a695e 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -360,6 +360,21 @@ def test_add_foreign_key(fresh_db): ] == fresh_db["books"].foreign_keys +def test_add_foreign_key_if_column_contains_space(fresh_db): + fresh_db["authors"].insert_all([{"id": 1, "name": "Sally"}], pk="id") + fresh_db["books"].insert_all( + [ + {"title": "Hedgehogs of the world", "author id": 1}, + ] + ) + fresh_db["books"].add_foreign_key("author id", "authors", "id") + assert fresh_db["books"].foreign_keys == [ + ForeignKey( + table="books", column="author id", other_table="authors", other_column="id" + ) + ] + + def test_add_foreign_key_error_if_column_does_not_exist(fresh_db): fresh_db["books"].insert( {"id": 1, "title": "Hedgehogs of the world", "author_id": 1} @@ -423,7 +438,7 @@ def test_add_column_foreign_key(fresh_db): fresh_db.create_table("breeds", {"name": str}) fresh_db["dogs"].add_column("breed_id", fk="breeds") assert ( - "CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, FOREIGN KEY(breed_id) REFERENCES breeds(rowid) )" + "CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, FOREIGN KEY([breed_id]) REFERENCES [breeds]([rowid]) )" == collapse_whitespace(fresh_db["dogs"].schema) ) # And again with an explicit primary key column @@ -431,8 +446,8 @@ def test_add_column_foreign_key(fresh_db): fresh_db["dogs"].add_column("subbreed_id", fk="subbreeds") assert ( "CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, [subbreed_id] TEXT, " - "FOREIGN KEY(breed_id) REFERENCES breeds(rowid), " - "FOREIGN KEY(subbreed_id) REFERENCES subbreeds(primkey) )" + "FOREIGN KEY([breed_id]) REFERENCES [breeds]([rowid]), " + "FOREIGN KEY([subbreed_id]) REFERENCES [subbreeds]([primkey]) )" == collapse_whitespace(fresh_db["dogs"].schema) ) @@ -443,7 +458,7 @@ def test_add_foreign_key_guess_table(fresh_db): fresh_db["dogs"].add_column("breed_id", int) fresh_db["dogs"].add_foreign_key("breed_id") assert ( - "CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, FOREIGN KEY(breed_id) REFERENCES breeds(id) )" + "CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, FOREIGN KEY([breed_id]) REFERENCES [breeds]([id]) )" == collapse_whitespace(fresh_db["dogs"].schema) ) diff --git a/tests/test_extract.py b/tests/test_extract.py index 25dd6e2..7a663b5 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -28,7 +28,7 @@ def test_extract_single_column(fresh_db, table, fk_column): " [name] TEXT,\n" " [{}] INTEGER,\n".format(expected_fk) + " [end] INTEGER,\n" - + " FOREIGN KEY({}) REFERENCES {}(id)\n".format(expected_fk, expected_table) + + " FOREIGN KEY([{}]) REFERENCES [{}]([id])\n".format(expected_fk, expected_table) + ")" ) assert fresh_db[expected_table].schema == ( @@ -75,7 +75,7 @@ def test_extract_multiple_columns_with_rename(fresh_db): " [id] INTEGER PRIMARY KEY,\n" " [name] TEXT,\n" " [common_name_latin_name_id] INTEGER,\n" - " FOREIGN KEY(common_name_latin_name_id) REFERENCES common_name_latin_name(id)\n" + " FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n" ")" ) assert fresh_db["common_name_latin_name"].schema == ( @@ -127,7 +127,7 @@ def test_extract_rowid_table(fresh_db): " [rowid] INTEGER PRIMARY KEY,\n" " [name] TEXT,\n" " [common_name_latin_name_id] INTEGER,\n" - " FOREIGN KEY(common_name_latin_name_id) REFERENCES common_name_latin_name(id)\n" + " FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n" ")" ) @@ -144,7 +144,7 @@ def test_reuse_lookup_table(fresh_db): 'CREATE TABLE "sightings" (\n' " [id] INTEGER PRIMARY KEY,\n" " [species_id] INTEGER,\n" - " FOREIGN KEY(species_id) REFERENCES species(id)\n" + " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n" ")" ) assert fresh_db["individuals"].schema == ( @@ -152,7 +152,7 @@ def test_reuse_lookup_table(fresh_db): " [id] INTEGER PRIMARY KEY,\n" " [name] TEXT,\n" " [species_id] INTEGER,\n" - " FOREIGN KEY(species_id) REFERENCES species(id)\n" + " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n" ")" ) assert list(fresh_db["species"].rows) == [ From c236894caa976d4ea5c7503e9331a3e9d2fbb1c4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 Feb 2021 09:05:08 -0800 Subject: [PATCH 0323/1004] table.drop(ignore=True) option, refs #237 --- docs/python-api.rst | 6 ++++++ sqlite_utils/db.py | 16 ++++++++++++---- tests/test_create.py | 16 ++++++++++++++++ tests/test_extract.py | 4 +++- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index c0eb593..e5ffd44 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1026,6 +1026,12 @@ You can drop a table or view using the ``.drop()`` method: db["my_table"].drop() +Pass ``ignore=True`` if you want to ignore the error caused by the table or view not existing. + +.. code-block:: python + + db["my_table"].drop(ignore=True) + .. _python_api_transform: Transforming a table diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 246064c..d9a6b26 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1212,8 +1212,12 @@ class Table(Queryable): self.add_foreign_key(col_name, fk, fk_col) return self - def drop(self): - self.db.execute("DROP TABLE [{}]".format(self.name)) + def drop(self, ignore=False): + try: + self.db.execute("DROP TABLE [{}]".format(self.name)) + except sqlite3.OperationalError: + if not ignore: + raise def guess_foreign_table(self, column): column = column.lower() @@ -2195,8 +2199,12 @@ class View(Queryable): self.name, ", ".join(c.name for c in self.columns) ) - def drop(self): - self.db.execute("DROP VIEW [{}]".format(self.name)) + def drop(self, ignore=False): + try: + self.db.execute("DROP VIEW [{}]".format(self.name)) + except sqlite3.OperationalError: + if not ignore: + raise def enable_fts(self, *args, **kwargs): raise NotImplementedError( diff --git a/tests/test_create.py b/tests/test_create.py index 09a695e..c1daf6b 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -6,6 +6,7 @@ from sqlite_utils.db import ( NoObviousTable, ForeignKey, Table, + View, ) from sqlite_utils.utils import sqlite3 import collections @@ -944,6 +945,21 @@ def test_drop_view(fresh_db): assert [] == fresh_db.view_names() +def test_drop_ignore(fresh_db): + with pytest.raises(sqlite3.OperationalError): + fresh_db["does_not_exist"].drop() + fresh_db["does_not_exist"].drop(ignore=True) + # Testing view is harder, we need to create it in order + # to get a View object, then drop it twice + fresh_db.create_view("foo_view", "select 1") + view = fresh_db["foo_view"] + assert isinstance(view, View) + view.drop() + with pytest.raises(sqlite3.OperationalError): + view.drop() + view.drop(ignore=True) + + def test_insert_all_empty_list(fresh_db): fresh_db["t"].insert({"foo": 1}) assert 1 == fresh_db["t"].count diff --git a/tests/test_extract.py b/tests/test_extract.py index 7a663b5..9eae704 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -28,7 +28,9 @@ def test_extract_single_column(fresh_db, table, fk_column): " [name] TEXT,\n" " [{}] INTEGER,\n".format(expected_fk) + " [end] INTEGER,\n" - + " FOREIGN KEY([{}]) REFERENCES [{}]([id])\n".format(expected_fk, expected_table) + + " FOREIGN KEY([{}]) REFERENCES [{}]([id])\n".format( + expected_fk, expected_table + ) + ")" ) assert fresh_db[expected_table].schema == ( From bba6e241be0e4aa596da0d5d4ae787d9d4cde92e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 Feb 2021 09:11:37 -0800 Subject: [PATCH 0324/1004] --ignore for sqlite-utils drop-table and drop-view, closes #237 --- docs/cli.rst | 4 ++++ sqlite_utils/cli.py | 18 ++++++++++-------- tests/test_cli.py | 23 ++++++++++++++++++++++- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index c015d78..8a049a6 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -736,6 +736,8 @@ You can drop a table using the ``drop-table`` command:: $ sqlite-utils drop-table mydb.db mytable +Use ``--ignore`` to ignore the error if the table does not exist. + .. _cli_transform_table: Transforming tables @@ -922,6 +924,8 @@ You can drop a view using the ``drop-view`` command:: $ sqlite-utils drop-view myview +Use ``--ignore`` to ignore the error if the view does not exist. + .. _cli_add_column: Adding columns diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4b51565..c5cbadf 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -946,14 +946,15 @@ def create_table( required=True, ) @click.argument("table") +@click.option("--ignore", is_flag=True) @load_extension_option -def drop_table(path, table, load_extension): +def drop_table(path, table, ignore, load_extension): "Drop the specified table" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - if table in db.table_names(): - db[table].drop() - else: + try: + db[table].drop(ignore=ignore) + except sqlite3.OperationalError: raise click.ClickException('Table "{}" does not exist'.format(table)) @@ -1002,14 +1003,15 @@ def create_view(path, view, select, ignore, replace, load_extension): required=True, ) @click.argument("view") +@click.option("--ignore", is_flag=True) @load_extension_option -def drop_view(path, view, load_extension): +def drop_view(path, view, ignore, load_extension): "Drop the specified view" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - if view in db.view_names(): - db[view].drop() - else: + try: + db[view].drop(ignore=ignore) + except sqlite3.OperationalError: raise click.ClickException('View "{}" does not exist'.format(view)) diff --git a/tests/test_cli.py b/tests/test_cli.py index b9d5246..0be2594 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1397,7 +1397,17 @@ def test_drop_table_error(): ) assert 1 == result.exit_code assert 'Error: Table "t2" does not exist' == result.output.strip() - + # Using --ignore supresses that error + result = runner.invoke( + cli.cli, + [ + "drop-table", + "test.db", + "t2", + "--ignore" + ], + ) + assert 0 == result.exit_code def test_drop_view(): runner = CliRunner() @@ -1432,6 +1442,17 @@ def test_drop_view_error(): ) assert 1 == result.exit_code assert 'Error: View "t2" does not exist' == result.output.strip() + # Using --ignore supresses that error + result = runner.invoke( + cli.cli, + [ + "drop-view", + "test.db", + "t2", + "--ignore" + ], + ) + assert 0 == result.exit_code def test_enable_wal(): From 09c3386f55f766b135b6a1c00295646c4ae29bec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 Feb 2021 09:13:00 -0800 Subject: [PATCH 0325/1004] Applied Black, refs #237 --- tests/test_cli.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 0be2594..8775396 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1400,15 +1400,11 @@ def test_drop_table_error(): # Using --ignore supresses that error result = runner.invoke( cli.cli, - [ - "drop-table", - "test.db", - "t2", - "--ignore" - ], + ["drop-table", "test.db", "t2", "--ignore"], ) assert 0 == result.exit_code + def test_drop_view(): runner = CliRunner() with runner.isolated_filesystem(): @@ -1445,12 +1441,7 @@ def test_drop_view_error(): # Using --ignore supresses that error result = runner.invoke( cli.cli, - [ - "drop-view", - "test.db", - "t2", - "--ignore" - ], + ["drop-view", "test.db", "t2", "--ignore"], ) assert 0 == result.exit_code From 22f1d9e1999f70af4c5b0f880a820cd9eead3942 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 Mar 2021 08:41:49 -0800 Subject: [PATCH 0326/1004] Expand FTS acronym in --help output --- sqlite_utils/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c5cbadf..e42be83 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -266,7 +266,7 @@ def vacuum(path): @click.option("--no-vacuum", help="Don't run VACUUM", default=False, is_flag=True) @load_extension_option def optimize(path, tables, no_vacuum, load_extension): - """Optimize all FTS tables and then run VACUUM - should shrink the database file""" + """Optimize all full-text search tables and then run VACUUM - should shrink the database file""" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) if not tables: @@ -287,7 +287,7 @@ def optimize(path, tables, no_vacuum, load_extension): @click.argument("tables", nargs=-1) @load_extension_option def rebuild_fts(path, tables, load_extension): - """Rebuild all or specific FTS tables""" + """Rebuild all or specific full-text search tables""" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) if not tables: @@ -479,7 +479,7 @@ def create_index(path, table, column, name, unique, if_not_exists, load_extensio def enable_fts( path, table, column, fts4, fts5, tokenize, create_triggers, load_extension ): - "Enable FTS for specific table and columns" + "Enable full-text search for specific table and columns" fts_version = "FTS5" if fts4 and fts5: click.echo("Can only use one of --fts4 or --fts5", err=True) @@ -507,7 +507,7 @@ def enable_fts( @click.argument("column", nargs=-1, required=True) @load_extension_option def populate_fts(path, table, column, load_extension): - "Re-populate FTS for specific table and columns" + "Re-populate full-text search for specific table and columns" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) db[table].populate_fts(column) @@ -522,7 +522,7 @@ def populate_fts(path, table, column, load_extension): @click.argument("table") @load_extension_option def disable_fts(path, table, load_extension): - "Disable FTS for specific table" + "Disable full-text search for specific table" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) db[table].disable_fts() From 6f4f9a3effeb16de0348d3cf136664f7531f498d Mon Sep 17 00:00:00 2001 From: Dylan Wu <6586811+dylan-wu@users.noreply.github.com> Date: Tue, 18 May 2021 21:47:44 -0500 Subject: [PATCH 0327/1004] Fixing insert from JSON containing strings with non-ascii characters are escaped aps unicode for lists, tuples, dicts (#258) --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d9a6b26..0e37b12 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2222,7 +2222,7 @@ def jsonify_if_needed(value): if isinstance(value, decimal.Decimal): return float(value) if isinstance(value, (dict, list, tuple)): - return json.dumps(value, default=repr) + return json.dumps(value, default=repr, ensure_ascii=False) elif isinstance(value, (datetime.time, datetime.date, datetime.datetime)): return value.isoformat() elif isinstance(value, uuid.UUID): From a95954c481012cc46fff2df5aaa4ee24e43dddf3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 18 May 2021 19:56:53 -0700 Subject: [PATCH 0328/1004] Tests for unicode characters in nested JSON, refs #257 --- tests/test_create.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_create.py b/tests/test_create.py index c1daf6b..e08eb25 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -748,7 +748,7 @@ def test_create_index_if_not_exists(fresh_db): {"dictionary": {"nested": "complex"}}, collections.OrderedDict( [ - ("key1", {"nested": "complex"}), + ("key1", {"nested": ["cømplex"]}), ("key2", "foo"), ] ), @@ -762,6 +762,14 @@ def test_insert_dictionaries_and_lists_as_json(fresh_db, data_structure): assert data_structure == json.loads(row[1]) +def test_insert_list_nested_unicode(fresh_db): + fresh_db["test"].insert( + {"id": 1, "data": {"key1": {"nested": ["cømplex"]}}}, pk="id" + ) + row = fresh_db.execute("select id, data from test").fetchone() + assert row[1] == '{"key1": {"nested": ["cømplex"]}}' + + def test_insert_uuid(fresh_db): uuid4 = uuid.uuid4() fresh_db["test"].insert({"uuid": uuid4}) From e7b2626291040b78b9a2dbc2982ba72691fb1a0f Mon Sep 17 00:00:00 2001 From: Rob Wells Date: Wed, 19 May 2021 03:57:26 +0100 Subject: [PATCH 0329/1004] Fix incorrect create-table cli description (#254) The description for `create-table` was duplicated from `create-index`. --- sqlite_utils/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index e42be83..9d899d5 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -905,7 +905,7 @@ def upsert( def create_table( path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension ): - "Add an index to the specified table covering the specified columns" + "Add a table with the specified columns" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) if len(columns) % 2 == 1: From 3e62ab62a88992d4bbb0fe83debec3bacd93ebf3 Mon Sep 17 00:00:00 2001 From: Damien Ready Date: Tue, 18 May 2021 21:58:04 -0500 Subject: [PATCH 0330/1004] Correct some typos (#245) --- docs/changelog.rst | 8 ++++---- docs/cli.rst | 6 +++--- docs/python-api.rst | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9d4770f..7e1e6a9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -201,7 +201,7 @@ Other changes 2.17 (2020-09-07) ----------------- -This release handles a bug where replacing rows in FTS tables could result in growing numbers of unneccessary rows in the associated ``*_fts_docsize`` table. (`#149 `__) +This release handles a bug where replacing rows in FTS tables could result in growing numbers of unnecessary rows in the associated ``*_fts_docsize`` table. (`#149 `__) - ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (`#152 `__) - ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (`#153 `__) @@ -272,7 +272,7 @@ The theme of this release is better tools for working with binary data. The new - ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (`#122 `__) - ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (`#123 `__) -- JSON output now encodes BLOB values as special base64 obects - see :ref:`cli_query_json`. (`#125 `__) +- JSON output now encodes BLOB values as special base64 objects - see :ref:`cli_query_json`. (`#125 `__) - The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (`#126 `__) - The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (`#124 `__) @@ -594,7 +594,7 @@ Support for lookup tables. 1.2 (2019-06-12) ---------------- -- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by instrospecting the database. See :ref:`python_api_add_foreign_key` for details. (`#25 `__) +- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by introspecting the database. See :ref:`python_api_add_foreign_key` for details. (`#25 `__) - Ability to set ``NOT NULL`` constraints and ``DEFAULT`` values when creating tables (`#24 `__). Documentation: :ref:`Setting defaults and not null constraints (Python API) `, :ref:`Setting defaults and not null constraints (CLI) ` - Support for ``not_null_default=X`` / ``--not-null-default`` for setting a ``NOT NULL DEFAULT 'x'`` when adding a new column. Documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` @@ -603,7 +603,7 @@ Support for lookup tables. 1.1 (2019-05-28) ---------------- -- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key alread exists (`#21 `__) - documentation: :ref:`Inserting data (Python API) `, :ref:`Inserting data (CLI) ` +- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key already exists (`#21 `__) - documentation: :ref:`Inserting data (Python API) `, :ref:`Inserting data (CLI) ` - Ability to add a column that is a foreign key reference using ``fk=...`` / ``--fk`` (`#16 `__) - documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` .. _v1_0_1: diff --git a/docs/cli.rst b/docs/cli.rst index 8a049a6..b314b2d 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -34,7 +34,7 @@ Use ``--nl`` to get back newline-delimited JSON objects:: {"id": 1, "age": 4, "name": "Cleo"} {"id": 2, "age": 2, "name": "Pancakes"} -You can use ``--arrays`` to request ararys instead of objects:: +You can use ``--arrays`` to request arrays instead of objects:: $ sqlite-utils dogs.db "select * from dogs" --arrays [[1, 4, "Cleo"], @@ -75,7 +75,7 @@ Binary strings are not valid JSON, so BLOB columns containing binary data will b } ] -If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the comand will return the number of affected rows:: +If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows:: $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" [{"rows_affected": 1}] @@ -1186,7 +1186,7 @@ Both of these commands accept one or more database files as arguments. Loading SQLite extensions ========================= -Many of these commands have the ablity to load additional SQLite extensions using the ``--load-extension=/path/to/extension`` option - use ``--help`` to check for support, e.g. ``sqlite-utils rows --help``. +Many of these commands have the ability to load additional SQLite extensions using the ``--load-extension=/path/to/extension`` option - use ``--help`` to check for support, e.g. ``sqlite-utils rows --help``. This option can be applied multiple times to load multiple extensions. diff --git a/docs/python-api.rst b/docs/python-api.rst index e5ffd44..11b1f0c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -603,7 +603,7 @@ The first argument to ``update()`` is the primary key. This can be a single valu >>> db["compound_dogs"].update((5, 3), {"name": "Updated"}) -The second argument is a dictonary of columns that should be updated, along with their new values. +The second argument is a dictionary of columns that should be updated, along with their new values. You can cause any missing columns to be added automatically using ``alter=True``:: From 328211eaca1247cd6b33a2c0a54642f87866d85b Mon Sep 17 00:00:00 2001 From: "Juan E. D" Date: Tue, 18 May 2021 23:58:21 -0300 Subject: [PATCH 0331/1004] Typo in upsert example (#244) Remove extra `[` --- docs/python-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 11b1f0c..ad7b394 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -647,7 +647,7 @@ For example, given the dogs database you could upsert the record for Cleo like s .. code-block:: python - db["dogs"].upsert([{ + db["dogs"].upsert({ "id": 1, "name": "Cleo", "twitter": "cleopaws", From 2f3371ecb1ad075672d3f815993193732ed00be5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 18 May 2021 20:26:13 -0700 Subject: [PATCH 0332/1004] Suggest --alter if column is missing, closes #259, refs #256 --- sqlite_utils/cli.py | 13 ++++++++++--- tests/test_cli.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 9d899d5..011935a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -732,9 +732,16 @@ def insert_upsert_implementation( extra_kwargs["upsert"] = upsert # Apply {"$base64": true, ...} decoding, if needed docs = (decode_base64_values(doc) for doc in docs) - db[table].insert_all( - docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs - ) + try: + db[table].insert_all( + docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs + ) + except sqlite3.OperationalError as e: + if e.args and "has no column named" in e.args[0]: + raise click.ClickException( + "{}\n\nTry using --alter to add additional columns".format(e.args[0]) + ) + raise @cli.command() diff --git a/tests/test_cli.py b/tests/test_cli.py index 8775396..2e982dd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -354,6 +354,21 @@ def test_add_column_foreign_key(db_path): assert "table 'bobcats' does not exist" in str(result.exception) +def test_suggest_alter_if_column_missing(db_path): + db = Database(db_path) + db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "authors", "-"], + input='{"id": 2, "name": "Barry", "age": 43}', + ) + assert result.exit_code != 0 + assert result.output.strip() == ( + "Error: table authors has no column named age\n\n" + "Try using --alter to add additional columns" + ) + + def test_index_foreign_keys(db_path): test_add_column_foreign_key(db_path) db = Database(db_path) From b2302875c97f723e02cc39136d0b20fd706369aa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 May 2021 20:55:46 -0700 Subject: [PATCH 0333/1004] Document --type option better, closes #255 --- sqlite_utils/cli.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 011935a..960d34c 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1276,9 +1276,12 @@ def triggers( @click.argument("table") @click.option( "--type", - type=(str, str), + type=( + str, + click.Choice(["INTEGER", "TEXT", "FLOAT", "BLOB"], case_sensitive=False), + ), multiple=True, - help="Change column type to X", + help="Change column type to INTEGER, TEXT, FLOAT or BLOB", ) @click.option("--drop", type=str, multiple=True, help="Drop this column") @click.option( From 51d01da30d45c1fbc1e587e6046a933529cf915e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 May 2021 22:01:38 -0700 Subject: [PATCH 0334/1004] Ability to add descending order indexes (#262) * DescIndex(column) for descending index columns, refs #260 * Ability to add desc indexes using CLI, closes #260 --- docs/cli.rst | 8 ++++++++ docs/python-api.rst | 11 +++++++++++ sqlite_utils/cli.py | 16 +++++++++++++--- sqlite_utils/db.py | 13 ++++++++++++- tests/test_cli.py | 11 +++++++++++ tests/test_create.py | 14 ++++++++++++++ 6 files changed, 69 insertions(+), 4 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index b314b2d..e5ffcfc 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1033,6 +1033,14 @@ Use the ``--unique`` option to create a unique index. Use ``--if-not-exists`` to avoid attempting to create the index if one with that name already exists. +To add an index on a column in descending order, prefix the column with a hyphen. Since this can be confused for a command-line option you need to construct that like this:: + + $ sqlite-utils create-index mydb.db mytable -- col1 -col2 col3 + +This will create an index on that table on ``(col1, col2 desc, col3)``. + +If your column names are already prefixed with a hyphen you'll need to manually execute a ``CREATE INDEX`` SQL statement to add indexes to them rather than using this tool. + .. _cli_fts: Configuring full-text search diff --git a/docs/python-api.rst b/docs/python-api.rst index ad7b394..418c58d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1866,6 +1866,17 @@ By default the index will be named ``idx_{table-name}_{columns}`` - if you want index_name="good_dogs_by_age" ) +To create an index in descending order for a column, wrap the column name in ``db.DescIndex()`` like this: + +.. code-block:: python + + from sqlite_utils.db import DescIndex + + db["dogs"].create_index( + ["is_good_dog", DescIndex("age")], + index_name="good_dogs_by_age" + ) + You can create a unique index by passing ``unique=True``: .. code-block:: python diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 960d34c..52de64a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -5,7 +5,7 @@ from datetime import datetime import hashlib import pathlib import sqlite_utils -from sqlite_utils.db import AlterError +from sqlite_utils.db import AlterError, DescIndex import textwrap import io import itertools @@ -450,11 +450,21 @@ def index_foreign_keys(path, load_extension): ) @load_extension_option def create_index(path, table, column, name, unique, if_not_exists, load_extension): - "Add an index to the specified table covering the specified columns" + """ + Add an index to the specified table covering the specified columns. + Use "sqlite-utils create-index mydb -- -column" to specify descending + order for a column. + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) + # Treat -prefix as descending for columns + columns = [] + for col in column: + if col.startswith("-"): + col = DescIndex(col[1:]) + columns.append(col) db[table].create_index( - column, index_name=name, unique=unique, if_not_exists=if_not_exists + columns, index_name=name, unique=unique, if_not_exists=if_not_exists ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0e37b12..fdfe360 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -144,6 +144,10 @@ class InvalidColumns(Exception): pass +class DescIndex(str): + pass + + _COUNTS_TABLE_CREATE_SQL = """ CREATE TABLE IF NOT EXISTS [{}]( [table] TEXT PRIMARY KEY, @@ -1156,6 +1160,13 @@ class Table(Queryable): index_name = "idx_{}_{}".format( self.name.replace(" ", "_"), "_".join(columns) ) + columns_sql = [] + for column in columns: + if isinstance(column, DescIndex): + fmt = "[{}] desc" + else: + fmt = "[{}]" + columns_sql.append(fmt.format(column)) sql = ( textwrap.dedent( """ @@ -1167,7 +1178,7 @@ class Table(Queryable): .format( index_name=index_name, table_name=self.name, - columns=", ".join("[{}]".format(c) for c in columns), + columns=", ".join(columns_sql), unique="UNIQUE " if unique else "", if_not_exists="IF NOT EXISTS " if if_not_exists else "", ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2e982dd..17ce27d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -208,6 +208,17 @@ def test_create_index(db_path): ) +def test_create_index_desc(db_path): + db = Database(db_path) + assert [] == db["Gosh"].indexes + result = CliRunner().invoke(cli.cli, ["create-index", db_path, "Gosh", "--", "-c1"]) + assert result.exit_code == 0 + assert ( + db.execute("select sql from sqlite_master where type='index'").fetchone()[0] + == "CREATE INDEX [idx_Gosh_c1]\n ON [Gosh] ([c1] desc)" + ) + + @pytest.mark.parametrize( "col_name,col_type,expected_schema", ( diff --git a/tests/test_create.py b/tests/test_create.py index e08eb25..6e925dc 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1,6 +1,7 @@ from sqlite_utils.db import ( Index, Database, + DescIndex, ForeignKey, AlterError, NoObviousTable, @@ -739,6 +740,19 @@ def test_create_index_if_not_exists(fresh_db): dogs.create_index(["name"], if_not_exists=True) +def test_create_index_desc(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True}) + assert [] == dogs.indexes + dogs.create_index([DescIndex("age"), "name"]) + sql = fresh_db.execute( + "select sql from sqlite_master where name='idx_dogs_age_name'" + ).fetchone()[0] + assert sql == ( + "CREATE INDEX [idx_dogs_age_name]\n" " ON [dogs] ([age] desc, [name])" + ) + + @pytest.mark.parametrize( "data_structure", ( From 668e8c9fd1b0eab4000902e4226aeaae3860e802 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 May 2021 22:00:11 -0700 Subject: [PATCH 0335/1004] Better help for sqlite-utils create-table --- docs/dogs.db | 0 sqlite_utils/cli.py | 12 +++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) delete mode 100644 docs/dogs.db diff --git a/docs/dogs.db b/docs/dogs.db deleted file mode 100644 index e69de29..0000000 diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 52de64a..68bd986 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -922,7 +922,17 @@ def upsert( def create_table( path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension ): - "Add a table with the specified columns" + """ + Add a table with the specified columns. Columns should be specified using + name, type pairs, for example: + + \b + sqlite-utils create-table my.db people \\ + id integer \\ + name text \\ + height float \\ + photo blob --pk id + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) if len(columns) % 2 == 1: From 8de5595c21b9be40f120eab20192baa465bd7628 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 May 2021 22:34:17 -0700 Subject: [PATCH 0336/1004] Handle BOM in CSV files, closes #250 --- sqlite_utils/cli.py | 2 +- tests/test_cli.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 68bd986..cbc0555 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -694,7 +694,7 @@ def insert_upsert_implementation( raise click.ClickException("Use just one of --nl, --csv or --tsv") if encoding and not (csv or tsv): raise click.ClickException("--encoding must be used with --csv or --tsv") - encoding = encoding or "utf-8" + encoding = encoding or "utf-8-sig" buffered = io.BufferedReader(json_file, buffer_size=4096) decoded = io.TextIOWrapper(buffered, encoding=encoding) if pk and len(pk) == 1: diff --git a/tests/test_cli.py b/tests/test_cli.py index 17ce27d..3807eca 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1915,3 +1915,28 @@ def test_attach(tmpdir): {"id": 1, "text": "foo"}, {"id": 1, "text": "bar"}, ] + + +def test_csv_insert_bom(tmpdir): + db_path = str(tmpdir / "test.db") + bom_csv_path = str(tmpdir / "bom.csv") + with open(bom_csv_path, "wb") as fp: + fp.write(b"\xef\xbb\xbfname,age\nCleo,5") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "broken", bom_csv_path, "--encoding", "utf-8", "--csv"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + result2 = CliRunner().invoke( + cli.cli, + ["insert", db_path, "fixed", bom_csv_path, "--csv"], + catch_exceptions=False, + ) + assert result2.exit_code == 0 + db = Database(db_path) + tables = db.execute("select name, sql from sqlite_master").fetchall() + assert tables == [ + ("broken", "CREATE TABLE [broken] (\n [\ufeffname] TEXT,\n [age] TEXT\n)"), + ("fixed", "CREATE TABLE [fixed] (\n [name] TEXT,\n [age] TEXT\n)"), + ] From 8c542d20ca95069476c8c84d4078a99d07561c6e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 May 2021 22:47:59 -0700 Subject: [PATCH 0337/1004] Release 3.7 Refs #237, #238, #240, #250, #257, #259, #260 --- docs/changelog.rst | 14 ++++++++++++++ docs/python-api.rst | 2 ++ setup.py | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7e1e6a9..81e7729 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,20 @@ Changelog =========== +.. _v3_7: + +3.7 (2021-05-28) +---------------- + +- New ``table.pks_and_rows_where()`` method returning ``(primary_key, row_dictionary)`` tuples - see :ref:`python_api_pks_and_rows_where`. (`#240 `__) +- Fixed bug with `table.add_foreign_key()` against columns containing spaces. (`#238 `__) +- ``table_or_view.drop(ignore=True)`` option for avoiding errors if the table or view does not exist. (`#237 `__) +- ``sqlite-utils drop-view --ignore`` and ``sqlite-utils drop-table --ignore`` options. (`#237 `__) +- Fixed a bug with inserts of nested JSON containing non-ascii strings - thanks, Dylan Wu. (`#257 `__) +- Suggest ``--alter`` if an error occurs caused by a missing column. (`#259 `__) +- Support creating indexes with columns in descending order, see :ref:`API documentation ` and :ref:`CLI documentation `. (`#260 `__) +- Correctly handle CSV files that start with a UTF-8 BOM. (`#250 `__) + .. _v3_6: 3.6 (2021-02-18) diff --git a/docs/python-api.rst b/docs/python-api.rst index 418c58d..78b069c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1848,6 +1848,8 @@ If the ``_counts`` table ever becomes out-of-sync with the actual table counts y db.reset_counts() +.. _python_api_create_index: + Creating indexes ================ diff --git a/setup.py b/setup.py index 6e7f978..53d71ef 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.6" +VERSION = "3.7" def get_long_description(): From 670f92285fc931f706b155ca20ac2e6fb3ca45b4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 May 2021 23:31:04 -0700 Subject: [PATCH 0338/1004] Fixed RST --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 81e7729..19dd349 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,7 +8,7 @@ ---------------- - New ``table.pks_and_rows_where()`` method returning ``(primary_key, row_dictionary)`` tuples - see :ref:`python_api_pks_and_rows_where`. (`#240 `__) -- Fixed bug with `table.add_foreign_key()` against columns containing spaces. (`#238 `__) +- Fixed bug with ``table.add_foreign_key()`` against columns containing spaces. (`#238 `__) - ``table_or_view.drop(ignore=True)`` option for avoiding errors if the table or view does not exist. (`#237 `__) - ``sqlite-utils drop-view --ignore`` and ``sqlite-utils drop-table --ignore`` options. (`#237 `__) - Fixed a bug with inserts of nested JSON containing non-ascii strings - thanks, Dylan Wu. (`#257 `__) From 2dad4f583cf3f9be40a4388093ed74c9043a6989 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Jun 2021 11:57:05 -0700 Subject: [PATCH 0339/1004] Improved .rows_where() documentation, added test for :named parameters --- docs/python-api.rst | 8 +++++++- tests/test_rows.py | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 78b069c..3745ffa 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -202,7 +202,13 @@ You can filter rows by a WHERE clause using ``.rows_where(where, where_args)``:: ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} -To return custom columns (instead of using ``select *``) pass ``select=``:: +The first argument is a fragment of SQL. The second, optional argument is values to be passed to that fragment - you can use ``?`` placeholders and pass an array, or you can use ``:named`` parameters and pass a dictionary, like this:: + + >>> for row in db["dogs"].rows_where("age > :age", {"age": 3}): + ... print(row) + {'id': 1, 'age': 4, 'name': 'Cleo'} + +To return custom columns (instead of the default that uses ``select *``) pass ``select="column1, column2"``:: >>> db = sqlite_utils.Database("dogs.db") >>> for row in db["dogs"].rows_where(select='name, age'): diff --git a/tests/test_rows.py b/tests/test_rows.py index 73bd94f..3ac52f9 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -13,6 +13,7 @@ def test_rows(existing_db): [ ("name = ?", ["Pancakes"], {2}), ("age > ?", [3], {1}), + ("age > :age", {"age": 3}, {1}), ("name is not null", [], {1, 2}), ("is_good = ?", [True], {1, 2}), ], From 9c67cb925253cd5ef54a1fe0496e0ff9caeacfd6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Jun 2021 20:51:27 -0700 Subject: [PATCH 0340/1004] table.xindexes property plus improved introspection documentation, closes #261 --- docs/python-api.rst | 103 +++++++++++++++++++++++++++++++++++++-- sqlite_utils/db.py | 22 +++++++++ tests/test_introspect.py | 29 ++++++++++- 3 files changed, 148 insertions(+), 6 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 3745ffa..de26d0a 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1449,6 +1449,11 @@ If you have loaded an existing table or view, you can use introspection to find >>> db["PlantType"]
+.. _python_api_introspection_exists: + +.exists() +--------- + The ``.exists()`` method can be used to find out if a table exists or not:: >>> db["PlantType"].exists() @@ -1456,6 +1461,11 @@ The ``.exists()`` method can be used to find out if a table exists or not:: >>> db["PlantType2"].exists() False +.. _python_api_introspection_count: + +.count +------ + The ``.count`` property shows the current number of rows (``select count(*) from table``):: >>> db["PlantType"].count @@ -1465,23 +1475,45 @@ The ``.count`` property shows the current number of rows (``select count(*) from This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.execute_count()`` instead of accessing the property. -The ``.columns`` property shows the columns in the table or view:: +.. _python_api_introspection_columns: + +.columns +-------- + +The ``.columns`` property shows the columns in the table or view. It returns a list of ``Column(cid, name, type, notnull, default_value, is_pk)`` named tuples. + +:: >>> db["PlantType"].columns [Column(cid=0, name='id', type='INTEGER', notnull=0, default_value=None, is_pk=1), Column(cid=1, name='value', type='TEXT', notnull=0, default_value=None, is_pk=0)] -The ``.columns_dict`` property returns a dictionary version of this with just the names and types:: +.. _python_api_introspection_columns_dict: + +.columns_dict +------------- + +The ``.columns_dict`` property returns a dictionary version of the columns with just the names and Python types:: >>> db["PlantType"].columns_dict {'id': , 'value': } +.. _python_api_introspection_pks: + +.pks +---- + The ``.pks`` property returns a list of strings naming the primary key columns for the table:: >>> db["PlantType"].pks ['id'] -The ``.foreign_keys`` property shows if the table has any foreign key relationships. It is not available on views. +.. _python_api_introspection_foreign_keys: + +.foreign_keys +------------- + +The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey(table, column, other_table, other_column)`` named tuples. It is not available on views. :: @@ -1493,6 +1525,11 @@ The ``.foreign_keys`` property shows if the table has any foreign key relationsh ForeignKey(table='Street_Tree_List', column='qCaretaker', other_table='qCaretaker', other_column='id'), ForeignKey(table='Street_Tree_List', column='PlantType', other_table='PlantType', other_column='id')] +.. _python_api_introspection_schema: + +.schema +------- + The ``.schema`` property outputs the table's schema as a SQL string:: >>> print(db["Street_Tree_List"].schema) @@ -1523,7 +1560,12 @@ The ``.schema`` property outputs the table's schema as a SQL string:: FOREIGN KEY ("qCareAssistant") REFERENCES [qCareAssistant](id), FOREIGN KEY ("qLegalStatus") REFERENCES [qLegalStatus](id)) -The ``.indexes`` property shows you all indexes created for a table. It is not available on views. +.. _python_api_introspection_indexes: + +.indexes +-------- + +The ``.indexes`` property returns all indexes created for a table, as a list of ``Index(seq, name, unique, origin, partial, columns)`` named tuples. It is not available on views. :: @@ -1535,7 +1577,38 @@ The ``.indexes`` property shows you all indexes created for a table. It is not a Index(seq=4, name='"Street_Tree_List_qCaretaker"', unique=0, origin='c', partial=0, columns=['qCaretaker']), Index(seq=5, name='"Street_Tree_List_PlantType"', unique=0, origin='c', partial=0, columns=['PlantType'])] -The ``.triggers`` property lists database triggers. It can be used on both database and table objects. +.. _python_api_introspection_xindexes: + +.xindexes +--------- + +The ``.xindexes`` property returns more detailed information about the indexes on the table, using the SQLite `PRAGMA index_xinfo() `__ mechanism. It returns a list of ``XIndex(name, columns)`` named tuples, where ``columns`` is a list of ``XIndexColumn(seqno, cid, name, desc, coll, key)`` named tuples. + +:: + >>> db["ny_times_us_counties"].xindexes + [ + XIndex( + name='idx_ny_times_us_counties_date', + columns=[ + XIndexColumn(seqno=0, cid=0, name='date', desc=1, coll='BINARY', key=1), + XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll='BINARY', key=0) + ] + ), + XIndex( + name='idx_ny_times_us_counties_fips', + columns=[ + XIndexColumn(seqno=0, cid=3, name='fips', desc=0, coll='BINARY', key=1), + XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll='BINARY', key=0) + ] + ) + ] + +.. _python_api_introspection_triggers: + +.triggers +--------- + +The ``.triggers`` property lists database triggers. It can be used on both database and table objects. It returns a list of ``Trigger(name, table, sql)`` named tuples. :: @@ -1546,6 +1619,11 @@ The ``.triggers`` property lists database triggers. It can be used on both datab >>> db.triggers ... similar output to db["authors"].triggers +.. _python_api_introspection_triggers_dict: + +.triggers_dict +-------------- + The ``.triggers_dict`` property returns the triggers for that table as a dictionary mapping their names to their SQL definitions. :: @@ -1564,6 +1642,11 @@ The same property exists on the database, and will return all triggers across al 'authors_ad': 'CREATE TRIGGER [authors_ad] AFTER DELETE...', 'authors_au': 'CREATE TRIGGER [authors_au] AFTER UPDATE'} +.. _python_api_introspection_detect_fts: + +.detect_fts() +------------- + The ``detect_fts()`` method returns the associated SQLite FTS table name, if one exists for this table. If the table has not been configured for full-text search it returns ``None``. :: @@ -1571,12 +1654,22 @@ The ``detect_fts()`` method returns the associated SQLite FTS table name, if one >>> db["authors"].detect_fts() "authors_fts" +.. _python_api_introspection_virtual_table_using: + +.virtual_table_using +-------------------- + The ``.virtual_table_using`` property reveals if a table is a virtual table. It returns ``None`` for regular tables and the upper case version of the type of virtual table otherwise. For example:: >>> db["authors"].enable_fts(["name"]) >>> db["authors_fts"].virtual_table_using "FTS5" +.. _python_api_introspection_has_counts_triggers: + +.has_counts_triggers +-------------------- + The ``.has_counts_triggers`` property shows if a table has been configured with triggers for updating a ``_counts`` table, as described in :ref:`python_api_cached_table_counts`. :: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index fdfe360..6781735 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -68,6 +68,10 @@ ForeignKey = namedtuple( "ForeignKey", ("table", "column", "other_table", "other_column") ) Index = namedtuple("Index", ("seq", "name", "unique", "origin", "partial", "columns")) +XIndex = namedtuple("XIndex", ("name", "columns")) +XIndexColumn = namedtuple( + "XIndexColumn", ("seqno", "cid", "name", "desc", "coll", "key") +) Trigger = namedtuple("Trigger", ("name", "table", "sql")) @@ -863,6 +867,24 @@ class Table(Queryable): indexes.append(Index(**row)) return indexes + @property + def xindexes(self): + sql = 'PRAGMA index_list("{}")'.format(self.name) + indexes = [] + for row in self.db.execute_returning_dicts(sql): + index_name = row["name"] + index_name_quoted = ( + '"{}"'.format(index_name) + if not index_name.startswith('"') + else index_name + ) + column_sql = "PRAGMA index_xinfo({})".format(index_name_quoted) + index_columns = [] + for info in self.db.execute(column_sql).fetchall(): + index_columns.append(XIndexColumn(*info)) + indexes.append(XIndex(index_name, index_columns)) + return indexes + @property def triggers(self): return [ diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 73102f8..66dbcff 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import Index, View, Database +from sqlite_utils.db import Index, View, Database, XIndex, XIndexColumn import pytest @@ -93,6 +93,33 @@ def test_indexes(fresh_db): ] == fresh_db["Gosh"].indexes +def test_xindexes(fresh_db): + fresh_db.executescript( + """ + create table Gosh (c1 text, c2 text, c3 text); + create index Gosh_c1 on Gosh(c1); + create index Gosh_c2c3 on Gosh(c2, c3 desc); + """ + ) + assert fresh_db["Gosh"].xindexes == [ + XIndex( + name="Gosh_c2c3", + columns=[ + XIndexColumn(seqno=0, cid=1, name="c2", desc=0, coll="BINARY", key=1), + XIndexColumn(seqno=1, cid=2, name="c3", desc=1, coll="BINARY", key=1), + XIndexColumn(seqno=2, cid=-1, name=None, desc=0, coll="BINARY", key=0), + ], + ), + XIndex( + name="Gosh_c1", + columns=[ + XIndexColumn(seqno=0, cid=0, name="c1", desc=0, coll="BINARY", key=1), + XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll="BINARY", key=0), + ], + ), + ] + + @pytest.mark.parametrize( "column,expected_table_guess", ( From 28dc5aac347ffdecb2dff154d23a73883a2ffabf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Jun 2021 21:26:46 -0700 Subject: [PATCH 0341/1004] sqlite-utils indexes command, refs #263 --- docs/cli.rst | 27 ++++++++++++++++ sqlite_utils/cli.py | 61 +++++++++++++++++++++++++++++++++++ tests/test_cli.py | 77 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index e5ffcfc..41d9a0c 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -293,6 +293,33 @@ It takes the same options as the ``tables`` command: * ``--tsv`` * ``--table`` +.. _cli_indexes: + +Listing indexes +=============== + +The ``indexes`` command lists any indexes configured for the database:: + + $ sqlite-utils indexes covid.db --table + table index_name seqno cid name desc coll key + -------------------------------- ------------------------------------------------------ ------- ----- ----------------- ------ ------ ----- + johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_combined_key 0 12 combined_key 0 BINARY 1 + johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_country_or_region 0 1 country_or_region 0 BINARY 1 + johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_province_or_state 0 2 province_or_state 0 BINARY 1 + johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_day 0 0 day 0 BINARY 1 + ny_times_us_counties idx_ny_times_us_counties_date 0 0 date 1 BINARY 1 + ny_times_us_counties idx_ny_times_us_counties_fips 0 3 fips 0 BINARY 1 + ny_times_us_counties idx_ny_times_us_counties_county 0 1 county 0 BINARY 1 + ny_times_us_counties idx_ny_times_us_counties_state 0 2 state 0 BINARY 1 + +It shows indexes across all tables. To see indexes for specific tables, list those after the database:: + + $ sqlite-utils indexes covid.db johns_hopkins_csse_daily_reports --table + +The command defaults to only showing the columns that are explicitly part of the index. To also include auxiliary columns use the ``--aux`` option - these columns will be listed with a ``key`` of ``0``. + +The command takes the same format options as the ``tables`` and ``views`` commands. + .. _cli_triggers: Listing triggers diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cbc0555..2e3d08e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1287,6 +1287,67 @@ def triggers( ) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("tables", nargs=-1) +@click.option("--aux", is_flag=True, help="Include auxiliary columns") +@output_options +@load_extension_option +@click.pass_context +def indexes( + ctx, + path, + tables, + aux, + nl, + arrays, + csv, + tsv, + no_headers, + table, + fmt, + json_cols, + load_extension, +): + "Show indexes for this database" + sql = """ + select + sqlite_master.name as "table", + indexes.name as index_name, + xinfo.* + from sqlite_master + join pragma_index_list(sqlite_master.name) indexes + join pragma_index_xinfo(index_name) xinfo + where + sqlite_master.type = 'table' + """ + if tables: + quote = sqlite_utils.Database(memory=True).quote + sql += " and sqlite_master.name in ({})".format( + ", ".join(quote(table) for table in tables) + ) + if not aux: + sql += " and xinfo.key = 1" + ctx.invoke( + query, + path=path, + sql=sql, + nl=nl, + arrays=arrays, + csv=csv, + tsv=tsv, + no_headers=no_headers, + table=table, + fmt=fmt, + json_cols=json_cols, + load_extension=load_extension, + ) + + @cli.command() @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index 3807eca..56f1953 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1802,6 +1802,83 @@ def test_search(tmpdir, fts, extra_arg, expected): assert result.output.replace("\r", "") == expected +def test_indexes(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db.conn.executescript( + """ + create table Gosh (c1 text, c2 text, c3 text); + create index Gosh_idx on Gosh(c2, c3 desc); + """ + ) + result = CliRunner().invoke( + cli.cli, + ["indexes", str(db_path)], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert json.loads(result.output) == [ + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 0, + "cid": 1, + "name": "c2", + "desc": 0, + "coll": "BINARY", + "key": 1, + }, + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 1, + "cid": 2, + "name": "c3", + "desc": 1, + "coll": "BINARY", + "key": 1, + }, + ] + result2 = CliRunner().invoke( + cli.cli, + ["indexes", str(db_path), "--aux"], + catch_exceptions=False, + ) + assert result2.exit_code == 0 + assert json.loads(result2.output) == [ + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 0, + "cid": 1, + "name": "c2", + "desc": 0, + "coll": "BINARY", + "key": 1, + }, + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 1, + "cid": 2, + "name": "c3", + "desc": 1, + "coll": "BINARY", + "key": 1, + }, + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 2, + "cid": -1, + "name": None, + "desc": 0, + "coll": "BINARY", + "key": 0, + }, + ] + + _TRIGGERS_EXPECTED = '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah AFTER INSERT ON articles\\nBEGIN\\n UPDATE counter SET count = count + 1;\\nEND"}]\n' From d1a372b3006e6cf7d2017b3ddc484bf5c033e45d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Jun 2021 22:16:33 -0700 Subject: [PATCH 0342/1004] Release 3.8 Refs #261, #263 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 19dd349..f1d2261 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v3_8: + +3.8 (2021-06-02) +---------------- + +- New ``sqlite-utils indexes`` command to list indexes in a database, see :ref:`cli_indexes`. (`#263 `__) +- ``table.indexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (`#261 `__) + .. _v3_7: 3.7 (2021-05-28) diff --git a/setup.py b/setup.py index 53d71ef..194baba 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.7" +VERSION = "3.8" def get_long_description(): From 9dff7a38831d471b1dff16d40d89eb5c3b4e84d6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 6 Jun 2021 23:02:18 -0700 Subject: [PATCH 0343/1004] Fixed markup --- docs/python-api.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index de26d0a..f629c1d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1585,6 +1585,7 @@ The ``.indexes`` property returns all indexes created for a table, as a list of The ``.xindexes`` property returns more detailed information about the indexes on the table, using the SQLite `PRAGMA index_xinfo() `__ mechanism. It returns a list of ``XIndex(name, columns)`` named tuples, where ``columns`` is a list of ``XIndexColumn(seqno, cid, name, desc, coll, key)`` named tuples. :: + >>> db["ny_times_us_counties"].xindexes [ XIndex( From 9696abfabf883d1c877ee71425b382587c288981 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Jun 2021 13:03:12 -0700 Subject: [PATCH 0344/1004] Rearranged "Inserting JSON data" section --- docs/cli.rst | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 41d9a0c..3bb2a83 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -472,6 +472,23 @@ If you feed it a JSON list it will insert multiple records. For example, if ``do } ] +You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so:: + + $ sqlite-utils insert dogs.db dogs dogs.json --pk=id + +You can skip inserting any records that have a primary key that already exists using ``--ignore``:: + + $ sqlite-utils insert dogs.db dogs dogs.json --ignore + +You can delete all the existing rows in the table before inserting the new records using ``--truncate``:: + + $ sqlite-utils insert dogs.db dogs dogs.json --truncate + +.. _cli_inserting_data_binary: + +Inserting binary data +--------------------- + You can insert binary data into a BLOB column by first encoding it using base64 and then structuring it like this:: [ @@ -484,17 +501,10 @@ You can insert binary data into a BLOB column by first encoding it using base64 } ] -You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so:: +.. _cli_inserting_data_nl_json: - $ sqlite-utils insert dogs.db dogs dogs.json --pk=id - -You can skip inserting any records that have a primary key that already exists using ``--ignore``:: - - $ sqlite-utils insert dogs.db dogs dogs.json --ignore - -You can delete all the existing rows in the table before inserting the new records using ``--truncate``:: - - $ sqlite-utils insert dogs.db dogs dogs.json --truncate +Inserting newline-delimited JSON +-------------------------------- You can also import newline-delimited JSON using the ``--nl`` option. Since `Datasette `__ can export newline-delimited JSON, you can combine the two tools like so:: From 0d2e4f49f324de01bcd8257d0adeea3ebf391791 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Jun 2021 13:51:49 -0700 Subject: [PATCH 0345/1004] db.schema and 'sqlite-utils schema' command, closes #268 --- docs/cli.rst | 13 +++++++++++++ docs/python-api.rst | 15 +++++++++++++++ sqlite_utils/cli.py | 17 +++++++++++++++++ sqlite_utils/db.py | 12 ++++++++++++ tests/test_cli.py | 27 +++++++++++++++++++++++++++ tests/test_introspect.py | 8 ++++++-- 6 files changed, 90 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 3bb2a83..d39d367 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -348,6 +348,19 @@ It defaults to showing triggers for all tables. To see triggers for one or more The command takes the same format options as the ``tables`` and ``views`` commands. +.. _cli_schema: + +Showing the schema +================== + +The ``sqlite-utils schema`` command shows the full SQL schema for the database:: + + $ sqlite-utils schema dogs.db + CREATE TABLE "dogs" ( + [id] INTEGER PRIMARY KEY, + [name] TEXT + ); + .. _cli_analyze_tables: Analyzing tables diff --git a/docs/python-api.rst b/docs/python-api.rst index f629c1d..725445c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -297,6 +297,21 @@ If the record does not exist a ``NotFoundError`` will be raised: except NotFoundError: print("Dog not found") +.. _python_api_schema: + +Showing the schema +================== + +The ``db.schema`` property returns the full SQL schema for the database as a string:: + + >>> db = sqlite_utils.Database("dogs.db") + >>> print(db.schema) + >>> print(db.schema) + CREATE TABLE "dogs" ( + [id] INTEGER PRIMARY KEY, + [name] TEXT + ); + .. _python_api_creating_tables: Creating tables diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 2e3d08e..6c40454 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1348,6 +1348,23 @@ def indexes( ) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@load_extension_option +def schema( + path, + load_extension, +): + "Show full schema for this database" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + click.echo(db.schema) + + @cli.command() @click.argument( "path", diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6781735..53fae10 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -301,6 +301,18 @@ class Database: "Returns {trigger_name: sql} dictionary" return {trigger.name: trigger.sql for trigger in self.triggers} + @property + def schema(self): + sqls = [] + for row in self.execute( + "select sql from sqlite_master where sql is not null" + ).fetchall(): + sql = row[0] + if not sql.strip().endswith(";"): + sql += ";" + sqls.append(sql) + return "\n".join(sqls) + @property def journal_mode(self): return self.execute("PRAGMA journal_mode;").fetchone()[0] diff --git a/tests/test_cli.py b/tests/test_cli.py index 56f1953..a73cdff 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1916,6 +1916,33 @@ def test_triggers(tmpdir, extra_args, expected): assert result.output == expected +def test_schema(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["dogs"].create({"id": int, "name": str}) + db["chickens"].create({"id": int, "name": str, "breed": str}) + db["chickens"].create_index(["breed"]) + result = CliRunner().invoke( + cli.cli, + ["schema", db_path], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert result.output == ( + "CREATE TABLE [dogs] (\n" + " [id] INTEGER,\n" + " [name] TEXT\n" + ");\n" + "CREATE TABLE [chickens] (\n" + " [id] INTEGER,\n" + " [name] TEXT,\n" + " [breed] TEXT\n" + ");\n" + "CREATE INDEX [idx_chickens_breed]\n" + " ON [chickens] ([breed]);\n" + ) + + def test_long_csv_column_value(tmpdir): db_path = str(tmpdir / "test.db") csv_path = str(tmpdir / "test.csv") diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 66dbcff..d54ca88 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -62,8 +62,12 @@ def test_columns(existing_db): ] -def test_schema(existing_db): - assert "CREATE TABLE foo (text TEXT)" == existing_db["foo"].schema +def test_table_schema(existing_db): + assert existing_db["foo"].schema == "CREATE TABLE foo (text TEXT)" + + +def test_database_schema(existing_db): + assert existing_db.schema == "CREATE TABLE foo (text TEXT);" def test_table_repr(fresh_db): From baafcec4a5e653d0c242f79fa5437591604d5292 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Jun 2021 19:05:12 -0700 Subject: [PATCH 0346/1004] Release 3.9 Refs #268 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f1d2261..c52449c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v3_9: + +3.9 (2021-06-11) +---------------- + +- New ``sqlite-utils schema`` command showing the full SQL schema for a database, see :ref:`Showing the schema (CLI)`. (`#268 `__) +- ``db.schema`` introspection property exposing the same feature to the Python library, see :ref:`Showing the schema (Python library) `. + .. _v3_8: 3.8 (2021-06-02) diff --git a/setup.py b/setup.py index 194baba..fd2e98a 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.8" +VERSION = "3.9" def get_long_description(): From b0f9d1e494c9891ce407e27b0f5c6deeea361d30 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Jun 2021 19:07:18 -0700 Subject: [PATCH 0347/1004] Fixed typo in 3.8 release notes --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c52449c..7be1f9d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,7 +16,7 @@ ---------------- - New ``sqlite-utils indexes`` command to list indexes in a database, see :ref:`cli_indexes`. (`#263 `__) -- ``table.indexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (`#261 `__) +- ``table.xindexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (`#261 `__) .. _v3_7: From b9629099ab21554a00eb11506201e6972600b93c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Jun 2021 19:57:21 -0700 Subject: [PATCH 0348/1004] Fix bug with upsert_all() and single column tables, closes #271 --- sqlite_utils/db.py | 28 +++++++++++++++------------- tests/test_create.py | 7 +++++++ tests/test_upsert.py | 7 +++++++ 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 53fae10..31d4cee 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1732,20 +1732,22 @@ class Table(Queryable): queries_and_params.append((sql, [record[col] for col in pks])) # UPDATE [book] SET [name] = 'Programming' WHERE [id] = 1001; set_cols = [col for col in all_columns if col not in pks] - sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format( - table=self.name, - pairs=", ".join( - "[{}] = {}".format(col, conversions.get(col, "?")) - for col in set_cols - ), - wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks), - ) - queries_and_params.append( - ( - sql2, - [record[col] for col in set_cols] + [record[pk] for pk in pks], + if set_cols: + sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format( + table=self.name, + pairs=", ".join( + "[{}] = {}".format(col, conversions.get(col, "?")) + for col in set_cols + ), + wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks), + ) + queries_and_params.append( + ( + sql2, + [record[col] for col in set_cols] + + [record[pk] for pk in pks], + ) ) - ) # We can populate .last_pk right here if num_records_processed == 1: self.last_pk = tuple(record[pk] for pk in pks) diff --git a/tests/test_create.py b/tests/test_create.py index 6e925dc..926a7d1 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -991,6 +991,13 @@ def test_insert_all_empty_list(fresh_db): assert 1 == fresh_db["t"].count +def test_insert_all_single_column(fresh_db): + table = fresh_db["table"] + table.insert_all([{"name": "Cleo"}], pk="name") + assert [{"name": "Cleo"}] == list(table.rows) + assert table.pks == ["name"] + + def test_create_with_a_null_column(fresh_db): record = {"name": "Name", "description": None} fresh_db["t"].insert(record) diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 100c6c1..9b1990e 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -21,6 +21,13 @@ def test_upsert_all(fresh_db): assert table.last_pk is None +def test_upsert_all_single_column(fresh_db): + table = fresh_db["table"] + table.upsert_all([{"name": "Cleo"}], pk="name") + assert [{"name": "Cleo"}] == list(table.rows) + assert table.pks == ["name"] + + def test_upsert_error_if_no_pk(fresh_db): table = fresh_db["table"] with pytest.raises(PrimaryKeyRequired): From 10f4913c144c4680c0feec576c2d4080f5005b33 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Jun 2021 19:59:08 -0700 Subject: [PATCH 0349/1004] Release 3.9.1 Refs #271 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7be1f9d..3339f0d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _3.9.1: + +3.9.1 (2021-06-12) +------------------ + +- Fixed bug when using ``table.upsert_all()`` to create a table with only a single column that is treated as the primary key. (`#271 `__) + .. _v3_9: 3.9 (2021-06-11) diff --git a/setup.py b/setup.py index fd2e98a..9c73ff4 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.9" +VERSION = "3.9.1" def get_long_description(): From a81c05d2350de2fb6931ee40fc540580db366bf9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Jun 2021 20:47:34 -0700 Subject: [PATCH 0350/1004] Clarify types that can be passed to .transform() --- docs/python-api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 725445c..45dac23 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1078,6 +1078,8 @@ To alter the type of a column, use the ``types=`` argument: # Convert the 'age' column to an integer, and 'weight' to a float table.transform(types={"age": int, "weight": float}) +See :ref:`python_api_add_column` for a list of available types. + The ``rename=`` parameter can rename columns: .. code-block:: python From a54b6788b0aa915c3e85a00220c6a841f094a1a3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Jun 2021 15:34:29 -0700 Subject: [PATCH 0351/1004] Sub-headings for .transform() --- docs/python-api.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 45dac23..b25eb21 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1071,6 +1071,9 @@ The ``table.transform()`` method can do all of these things, by implementing a m The ``.transform()`` method takes a number of parameters, all of which are optional. +Altering column types +--------------------- + To alter the type of a column, use the ``types=`` argument: .. code-block:: python @@ -1080,6 +1083,9 @@ To alter the type of a column, use the ``types=`` argument: See :ref:`python_api_add_column` for a list of available types. +Renaming columns +---------------- + The ``rename=`` parameter can rename columns: .. code-block:: python @@ -1087,6 +1093,9 @@ The ``rename=`` parameter can rename columns: # Rename 'age' to 'initial_age': table.transform(rename={"age": "initial_age"}) +Dropping columns +---------------- + To drop columns, pass them in the ``drop=`` set: .. code-block:: python @@ -1094,6 +1103,9 @@ To drop columns, pass them in the ``drop=`` set: # Drop the 'age' column: table.transform(drop={"age"}) +Changing primary keys +--------------------- + To change the primary key for a table, use ``pk=``. This can be passed a single column for a regular primary key, or a tuple of columns to create a compound primary key. Passing ``pk=None`` will remove the primary key and convert the table into a ``rowid`` table. .. code-block:: python @@ -1101,6 +1113,9 @@ To change the primary key for a table, use ``pk=``. This can be passed a single # Make `user_id` the new primary key table.transform(pk="user_id") +Changing not null status +------------------------ + You can change the ``NOT NULL`` status of columns by using ``not_null=``. You can pass this a set of columns to make those columns ``NOT NULL``: .. code-block:: python @@ -1118,6 +1133,9 @@ If you want to take existing ``NOT NULL`` columns and change them to allow null # Make age allow NULL and switch weight to being NOT NULL: table.transform(not_null={"age": False, "weight": True}) +Altering column defaults +------------------------ + The ``defaults=`` parameter can be used to set or change the defaults for different columns: .. code-block:: python @@ -1128,6 +1146,9 @@ The ``defaults=`` parameter can be used to set or change the defaults for differ # Now remove the default from that column: table.transform(defaults={"age": None}) +Changing column order +--------------------- + The ``column_order=`` parameter can be used to change the order of the columns. If you pass the names of a subset of the columns those will go first and columns you omitted will appear in their existing order after them. .. code-block:: python @@ -1135,6 +1156,9 @@ The ``column_order=`` parameter can be used to change the order of the columns. # Change column order table.transform(column_order=("name", "age", "id") +Dropping foreign key constraints +-------------------------------- + You can use ``.transform()`` to remove foreign key constraints from a table. This example drops two foreign keys - the one from ``places.country`` to ``country.id`` and the one from ``places.continent`` to ``continent.id``: From 78aebb6479420217454747870737bc593a259abc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Jun 2021 19:36:16 -0700 Subject: [PATCH 0352/1004] Link to --load-extension docs --- docs/cli.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index d39d367..c4dfc0d 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -85,9 +85,11 @@ You can run queries against a temporary in-memory database by passing ``:memory: $ sqlite-utils :memory: "select sqlite_version()" [{"sqlite_version()": "3.29.0"}] -You can load SQLite extension modules using the `--load-extension` option:: +You can load SQLite extension modules using the ``--load-extension`` option, see :ref:`cli_load_extension`. - $ sqlite-utils :memory: "select spatialite_version()" --load-extension=/usr/local/lib/mod_spatialite.dylib +:: + + $ sqlite-utils :memory: "select spatialite_version()" --load-extension=spatialite [{"spatialite_version()": "4.3.0a"}] .. _cli_json_values: From 287cdcae8908916687f2ecccc87c38549d004ac6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Jun 2021 21:40:28 -0700 Subject: [PATCH 0353/1004] Turn SQL errors into click errors --- sqlite_utils/cli.py | 5 ++++- tests/test_cli.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 6c40454..4208830 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1088,7 +1088,10 @@ def query( _load_extensions(db, load_extension) db.register_fts4_bm25() with db.conn: - cursor = db.execute(sql, dict(param)) + try: + cursor = db.execute(sql, dict(param)) + except sqlite3.OperationalError as e: + raise click.ClickException(str(e)) if cursor.description is None: # This was an update/insert headers = ["rows_affected"] diff --git a/tests/test_cli.py b/tests/test_cli.py index a73cdff..5cb4af0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1049,7 +1049,7 @@ def test_query_load_extension(use_spatialite_shortcut): # Without --load-extension: result = CliRunner().invoke(cli.cli, [":memory:", "select spatialite_version()"]) assert result.exit_code == 1 - assert "no such function: spatialite_version" in repr(result) + assert "no such function: spatialite_version" in result.output # With --load-extension: if use_spatialite_shortcut: load_extension = "spatialite" From fe1562e8a69872b27c1043c4b117d07623f16274 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 16 Jun 2021 09:36:32 -0700 Subject: [PATCH 0354/1004] Structure of most_common and least_common columns --- docs/cli.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index c4dfc0d..c953a0f 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -450,6 +450,21 @@ The ``_analyze_tables_`` table has the following schema:: PRIMARY KEY ([table], [column]) ); +The ``most_common`` and ``least_common`` columns will contain nested JSON arrays of the most commond and least common values that look like this:: + + [ + ["Del Libertador, Av", 5068], + ["Alberdi Juan Bautista Av.", 4612], + ["Directorio Av.", 4552], + ["Rivadavia, Av", 4532], + ["Yerbal", 4512], + ["Cosquín", 4472], + ["Estado Plurinacional de Bolivia", 4440], + ["Gordillo Timoteo", 4424], + ["Montiel", 4360], + ["Condarco", 4288] + ] + .. _cli_inserting_data: Inserting JSON data From ca2b26130f6c5fd030973ce593b02f08d19c9d84 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 16 Jun 2021 16:51:48 -0700 Subject: [PATCH 0355/1004] sqlite-utils dump command, closes #274 --- docs/cli.rst | 13 +++++++++++++ sqlite_utils/cli.py | 15 +++++++++++++++ tests/test_cli.py | 7 +++++++ 3 files changed, 35 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index c953a0f..a45f3b8 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1256,6 +1256,19 @@ You can disable WAL mode using ``disable-wal``:: Both of these commands accept one or more database files as arguments. +.. _cli_dump: + +Dumping the database to SQL +=========================== + +The ``dump`` command outputs a SQL dump of the schema and full contents of the specified database file:: + + $ sqlite-utils dump mydb.db + BEGIN TRANSACTION; + CREATE TABLE ... + ... + COMMIT; + .. _cli_load_extension: Loading SQLite extensions diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4208830..1e43a8b 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -308,6 +308,21 @@ def vacuum(path): sqlite_utils.Database(path).vacuum() +@cli.command() +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@load_extension_option +def dump(path, load_extension): + """Output a SQL dump of the schema and full contents of the database""" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + for line in db.conn.iterdump(): + click.echo(line) + + @cli.command(name="add-column") @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index 5cb4af0..c68c7db 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -502,6 +502,13 @@ def test_vacuum(db_path): assert 0 == result.exit_code +def test_dump(db_path): + result = CliRunner().invoke(cli.cli, ["dump", db_path]) + assert result.exit_code == 0 + assert result.output.startswith("BEGIN TRANSACTION;") + assert result.output.strip().endswith("COMMIT;") + + @pytest.mark.parametrize("tables", ([], ["Gosh"], ["Gosh2"])) def test_optimize(db_path, tables): db = Database(db_path) From 276fc4297caf69da2deb4c1334dd49375a248dbb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 16 Jun 2021 16:54:16 -0700 Subject: [PATCH 0356/1004] Enable codecov.io, refs #275 --- .github/workflows/test-coverage.yml | 42 +++++++++++++++++++++++++++++ codecov.yml | 8 ++++++ 2 files changed, 50 insertions(+) create mode 100644 .github/workflows/test-coverage.yml create mode 100644 codecov.yml diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml new file mode 100644 index 0000000..31a37fa --- /dev/null +++ b/.github/workflows/test-coverage.yml @@ -0,0 +1,42 @@ +name: Calculate test coverage + +on: + push: + branches: + - main + pull_request: + branches: + - main +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e .[test] + python -m pip install pytest-cov + - name: Run tests + run: |- + ls -lah + cat .coveragerc + pytest --cov=sqlite_utils --cov-report xml:coverage.xml --cov-report term + ls -lah + - name: Upload coverage report + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.xml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..bfdc987 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true From f003d051e7e1738143a07312bc96e6c6cbc0db4c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 16 Jun 2021 16:55:53 -0700 Subject: [PATCH 0357/1004] Not using .coveragerc, refs #275 --- .github/workflows/test-coverage.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 31a37fa..4b99cc5 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -32,7 +32,6 @@ jobs: - name: Run tests run: |- ls -lah - cat .coveragerc pytest --cov=sqlite_utils --cov-report xml:coverage.xml --cov-report term ls -lah - name: Upload coverage report From a19ce1a4d0048d389411cfe11a5dbe4c503720e1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 16 Jun 2021 17:12:11 -0700 Subject: [PATCH 0358/1004] codecov badge, closes #275 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e8b8245..8c1b4ab 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/) [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest) [![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.datasette.io/en/latest/?badge=latest) +[![codecov](https://codecov.io/gh/simonw/sqlite-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/simonw/sqlite-utils) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE) Python CLI utility and library for manipulating SQLite databases. From aa652b6afe43d2b40fabc7a513c3e68866e030a5 Mon Sep 17 00:00:00 2001 From: Loren McIntyre Date: Fri, 18 Jun 2021 07:56:59 -0700 Subject: [PATCH 0359/1004] add -h support Closes #276 --- sqlite_utils/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 1e43a8b..390f43e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -16,6 +16,8 @@ import csv as csv_std import tabulate from .utils import file_progress, find_spatialite, sqlite3, decode_base64_values +CONTEXT_SETTINGS = dict(help_option_names=['-h','--help']) + VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB") UNICODE_ERROR = """ @@ -89,7 +91,7 @@ def load_extension_option(fn): )(fn) -@click.group(cls=DefaultGroup, default="query", default_if_no_args=True) +@click.group(cls=DefaultGroup, default="query", default_if_no_args=True, context_settings=CONTEXT_SETTINGS) @click.version_option() def cli(): "Commands for interacting with a SQLite database" From eea3851d40ea7e49cf27905cca19d200cf4cdbe4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 07:55:26 -0700 Subject: [PATCH 0360/1004] Added test, formatted with Black - refs #276, #277 --- sqlite_utils/cli.py | 9 +++++++-- tests/test_cli.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 390f43e..69e04e3 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -16,7 +16,7 @@ import csv as csv_std import tabulate from .utils import file_progress, find_spatialite, sqlite3, decode_base64_values -CONTEXT_SETTINGS = dict(help_option_names=['-h','--help']) +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB") @@ -91,7 +91,12 @@ def load_extension_option(fn): )(fn) -@click.group(cls=DefaultGroup, default="query", default_if_no_args=True, context_settings=CONTEXT_SETTINGS) +@click.group( + cls=DefaultGroup, + default="query", + default_if_no_args=True, + context_settings=CONTEXT_SETTINGS, +) @click.version_option() def cli(): "Commands for interacting with a SQLite database" diff --git a/tests/test_cli.py b/tests/test_cli.py index c68c7db..b085ea4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -24,6 +24,22 @@ def db_path(tmpdir): return path +@pytest.mark.parametrize( + "options", + ( + ["-h"], + ["--help"], + ["insert", "-h"], + ["insert", "--help"], + ), +) +def test_help(options): + result = CliRunner().invoke(cli.cli, options) + assert result.exit_code == 0 + assert result.output.startswith("Usage: ") + assert "-h, --help" in result.output + + def test_tables(db_path): result = CliRunner().invoke(cli.cli, ["tables", db_path]) assert '[{"table": "Gosh"},\n {"table": "Gosh2"}]' == result.output.strip() From 42ec59d8ee3fcfb8ac1affb772aed43b6e2a7381 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 08:00:52 -0700 Subject: [PATCH 0361/1004] sqlite-utils memory command for directly querying CSV/JSON data * Turn SQL errors into click errors * Initial CSV-only prototype of sqlite-utils memory, refs #272 * Implement --save plus tests for --save and --dump, refs #272 * Re-arranged CLI query documentation, refs #272 * Re-organized CLI query docs, refs #272 * Docs for --save and --dump plus made SQL optional for those, refs #273 * Replaced one last :memory: example * Documented --attach option for memory command, refs #272 * Improved arrangement of CLI query documentation --- docs/changelog.rst | 2 +- docs/cli.rst | 225 ++++++++++++++++++++++++++++++--------- sqlite_utils/cli.py | 105 +++++++++++++++++- tests/test_cli_memory.py | 72 +++++++++++++ 4 files changed, 351 insertions(+), 53 deletions(-) create mode 100644 tests/test_cli_memory.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 3339f0d..879d19b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -47,7 +47,7 @@ This release adds the ability to execute queries joining data from more than one database file - similar to the cross database querying feature introduced in `Datasette 0.55 `__. - The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation `. (`#113 `__) -- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (`#236 `__) +- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (`#236 `__) .. _v3_5: diff --git a/docs/cli.rst b/docs/cli.rst index a45f3b8..b7f0313 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -8,25 +8,31 @@ The ``sqlite-utils`` command-line tool can be used to manipulate SQLite database .. contents:: :local: -.. _cli_query_json: +.. _cli_query: -Running queries and returning JSON -================================== +Running SQL queries +=================== -You can execute a SQL query against a database and get the results back as JSON like this:: +The ``sqlite-utils query`` command lets you run queries directly against a SQLite database file. This is the default subcommand, so the following two examples work the same way:: $ sqlite-utils query dogs.db "select * from dogs" + $ sqlite-utils dogs.db "select * from dogs" + +.. _cli_query_json: + +Returning JSON +-------------- + +The default format returned for queries is JSON:: + + $ sqlite-utils dogs.db "select * from dogs" [{"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}] -This is the default command for ``sqlite-utils``, so you can instead use this:: +.. _cli_query_nl: - $ sqlite-utils dogs.db "select * from dogs" - -You can pass named parameters to the query using ``-p``:: - - $ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6 - [{":num * :num2": 30}] +Newline-delimited JSON +~~~~~~~~~~~~~~~~~~~~~~ Use ``--nl`` to get back newline-delimited JSON objects:: @@ -34,6 +40,11 @@ Use ``--nl`` to get back newline-delimited JSON objects:: {"id": 1, "age": 4, "name": "Cleo"} {"id": 2, "age": 2, "name": "Pancakes"} +.. _cli_query_arrays: + +JSON arrays +~~~~~~~~~~~ + You can use ``--arrays`` to request arrays instead of objects:: $ sqlite-utils dogs.db "select * from dogs" --arrays @@ -62,6 +73,11 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] +.. _cli_query_binary_json: + +Binary data in JSON +~~~~~~~~~~~~~~~~~~~ + Binary strings are not valid JSON, so BLOB columns containing binary data will be returned as a JSON object containing base64 encoded data, that looks like this:: $ sqlite-utils dogs.db "select name, content from images" | python -mjson.tool @@ -75,27 +91,11 @@ Binary strings are not valid JSON, so BLOB columns containing binary data will b } ] -If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows:: - - $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" - [{"rows_affected": 1}] - -You can run queries against a temporary in-memory database by passing ``:memory:`` as the filename:: - - $ sqlite-utils :memory: "select sqlite_version()" - [{"sqlite_version()": "3.29.0"}] - -You can load SQLite extension modules using the ``--load-extension`` option, see :ref:`cli_load_extension`. - -:: - - $ sqlite-utils :memory: "select spatialite_version()" --load-extension=spatialite - [{"spatialite_version()": "4.3.0a"}] .. _cli_json_values: Nested JSON values ------------------- +~~~~~~~~~~~~~~~~~~ If one of your columns contains JSON, by default it will be returned as an escaped string:: @@ -126,24 +126,10 @@ You can use the ``--json-cols`` option to automatically detect these JSON column } ] -.. _cli_attach: - -Attaching additional databases ------------------------------- - -SQLite supports cross-database SQL queries, which can join data from tables in more than one database file. - -You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk. - -This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database:: - - sqlite-utils dogs.db --attach books books.db \ - 'select * from sqlite_master union all select * from books.sqlite_master' - .. _cli_query_csv: -Running queries and returning CSV -================================= +Returning CSV or TSV +-------------------- You can use the ``--csv`` option to return results as CSV:: @@ -167,8 +153,8 @@ Use ``--tsv`` instead of ``--csv`` to get back tab-separated values:: .. _cli_query_table: -Running queries and outputting a table -====================================== +Table-formatted output +---------------------- You can use the ``--table`` option (or ``-t`` shortcut) to output query results as a table:: @@ -192,8 +178,8 @@ For a full list of table format options, run ``sqlite-utils query --help``. .. _cli_query_raw: -Returning raw data from a query, such as binary content -======================================================= +Returning raw data, such as binary content +------------------------------------------ If your table contains binary data in a ``BLOB`` you can use the ``--raw`` option to output specific columns directly to standard out. @@ -201,6 +187,145 @@ For example, to retrieve a binary image from a ``BLOB`` column and store it in a $ sqlite-utils photos.db "select contents from photos where id=1" --raw > myphoto.jpg + +.. _cli_query_parameters: + +Using named parameters +---------------------- + +You can pass named parameters to the query using ``-p``:: + + $ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6 + [{":num * :num2": 30}] + +These will be correctly quoted and escaped in the SQL query, providing a safe way to combine other values with SQL. + +.. _cli_query_update_insert_delete: + +UPDATE, INSERT and DELETE +------------------------- + +If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows:: + + $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" + [{"rows_affected": 1}] + +SQLite extensions +----------------- + +You can load SQLite extension modules using the ``--load-extension`` option, see :ref:`cli_load_extension`. + +:: + + $ sqlite-utils dogs.db "select spatialite_version()" --load-extension=spatialite + [{"spatialite_version()": "4.3.0a"}] + +.. _cli_query_attach: + +Attaching additional databases +------------------------------ + +SQLite supports cross-database SQL queries, which can join data from tables in more than one database file. + +You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk. + +This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database:: + + sqlite-utils dogs.db --attach books books.db \ + 'select * from sqlite_master union all select * from books.sqlite_master' + +.. _cli_query_memory: + +Querying CSV data directly using an in-memory database +====================================================== + +The ``sqlite-utils memory`` command works similar to ``sqlite-utils query``, but allows you to execute queries against an in-memory database. + +You can also pass this command CSV files which will be loaded into a temporary in-memory table, allowing you to execute SQL against that data without a separate step to first convert it to SQLite. + +Without any extra arguments, this command executes SQL against the in-memory database directly:: + + $ sqlite-utils memory 'select sqlite_version()' + [{"sqlite_version()": "3.35.5"}] + +It takes all of the same formatting options as :ref:`sqlite-utils query `: ``--csv`` and ``--csv`` and ``--table`` and ``--nl``:: + + $ sqlite-utils memory 'select sqlite_version()' --csv + sqlite_version() + 3.35.5 + $ sqlite-utils memory 'select sqlite_version()' --table --fmt grid + +--------------------+ + | sqlite_version() | + +====================+ + | 3.35.5 | + +--------------------+ + +.. _cli_query_memory_csv: + +Running queries directly against CSV +------------------------------------ + +If you have data in CSV format you can load it into an in-memory SQLite database and run queries against it directly in a single command using ``sqlite-utils memory`` like this:: + + $ sqlite-utils memory data.csv "select * from data" + +You can pass multiple files to the command if you want to run joins between different CSV files:: + + $ sqlite-utils memory one.csv two.csv "select * from one join two on one.id = two.other_id" + +The in-memory tables will be named after the CSV files without their ``.csv`` extension. The tool also sets up aliases for those tables (using SQL views) as ``t1``, ``t2`` and so on, or you can use the alias ``t`` to refer to the first table:: + + $ sqlite-utils memory example.csv "select * from t" + +To read from standard input, use ``-`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name:: + + $ cat example.csv | sqlite-utils memory - "select * from stdin" + +.. _cli_query_memory_attach: + +Joining in-memory data against existing databases using \-\-attach +------------------------------------------------------------------ + +The :ref:`attach option ` can be used to attach database files to the in-memory connection, enabling joins between in-memory data loaded from a file and tables in existing SQLite database files. An example:: + + $ echo "id\n1\n3\n5" | sqlite-utils memory - --attach trees trees.db \ + "select * from trees.trees where rowid in (select id from stdin)" + +Here the ``--attach trees trees.db`` option makes the ``trees.db`` database available with an alias of ``trees``. + +``select * from trees.trees where ...`` can then query the ``trees`` table in that database. + +The CSV data that was piped into the script is available in the ``stdin`` table, so ``... where rowid in (select id from stdin)`` can be used to return rows from the ``trees`` table that match IDs that were piped in as CSV content. + +.. _cli_query_memory_dump_save: + +\-\-dump and \-\-save +--------------------- + +You can dump out the SQL used for the temporary in-memory database, complete with all imported data, using the ``--dump`` option:: + + % sqlite-utils memory dogs.csv --dump + BEGIN TRANSACTION; + CREATE TABLE [dogs] ( + [rowid] TEXT, + [id] TEXT, + [dog_age] TEXT, + [name] TEXT + ); + INSERT INTO "dogs" VALUES('1','1','4','Cleo'); + INSERT INTO "dogs" VALUES('2','2','2','Pancakes'); + INSERT INTO "dogs" VALUES('3','2','3','Pancakes'); + CREATE VIEW t1 AS select * from [dogs]; + CREATE VIEW t AS select * from [dogs]; + COMMIT; + +Passing ``--save other.db`` will instead use that SQL to populate a new database file:: + + % sqlite-utils memory dogs.csv --save dogs.db + +These features are mainly intented as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`. + + .. _cli_rows: Returning all rows in a table @@ -270,7 +395,7 @@ Use ``--schema`` to include the schema of each table:: [age] INTEGER, [name] TEXT) -The ``--nl``, ``--csv``, ``--tsv`` and ``--table`` options are all available. +The ``--nl``, ``--csv``, ``--tsv``, ``--table`` and ``--fmt`` options are also available. .. _cli_views: @@ -1280,5 +1405,5 @@ This option can be applied multiple times to load multiple extensions. Since `SpatiaLite `__ is commonly used with SQLite, the value ``spatialite`` is special: it will search for SpatiaLite in the most common installation locations, saving you from needing to remember exactly where that module is located:: - $ sqlite-utils :memory: "select spatialite_version()" --load-extension=spatialite + $ sqlite-utils memory "select spatialite_version()" --load-extension=spatialite [{"spatialite_version()": "4.3.0a"}] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 69e04e3..8fe37c8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -716,11 +716,11 @@ def insert_upsert_implementation( raise click.ClickException("Use just one of --nl, --csv or --tsv") if encoding and not (csv or tsv): raise click.ClickException("--encoding must be used with --csv or --tsv") + if pk and len(pk) == 1: + pk = pk[0] encoding = encoding or "utf-8-sig" buffered = io.BufferedReader(json_file, buffer_size=4096) decoded = io.TextIOWrapper(buffered, encoding=encoding) - if pk and len(pk) == 1: - pk = pk[0] if csv or tsv: if sniff: # Read first 2048 bytes and use that to detect @@ -1109,6 +1109,107 @@ def query( db.attach(alias, attach_path) _load_extensions(db, load_extension) db.register_fts4_bm25() + + _execute_query( + db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + ) + + +@cli.command() +@click.argument( + "paths", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=True), + required=False, + nargs=-1, +) +@click.argument("sql") +@click.option( + "--attach", + type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), + multiple=True, + help="Additional databases to attach - specify alias and filepath", +) +@output_options +@click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") +@click.option( + "-p", + "--param", + multiple=True, + type=(str, str), + help="Named :parameters for SQL query", +) +@click.option("--dump", is_flag=True, help="Dump SQL for in-memory database") +@click.option( + "--save", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + help="Save in-memory database to this file", +) +@load_extension_option +def memory( + paths, + sql, + attach, + nl, + arrays, + csv, + tsv, + no_headers, + table, + fmt, + json_cols, + raw, + param, + dump, + save, + load_extension, +): + "Execute SQL query against an in-memory database, optionally populated by imported data" + db = sqlite_utils.Database(memory=True) + # If --dump or --save used but no paths detected, assume SQL query is a path: + if (dump or save) and not paths: + paths = [sql] + sql = None + for i, path in enumerate(paths): + if path == "-": + csv_fp = sys.stdin + csv_table = "stdin" + else: + csv_path = pathlib.Path(path) + csv_table = csv_path.stem + csv_fp = csv_path.open() + db[csv_table].insert_all(csv_std.DictReader(csv_fp)) + # Add convenient t / t1 / t2 views + view_names = ["t{}".format(i + 1)] + if i == 0: + view_names.append("t") + for view_name in view_names: + if not db[view_name].exists(): + db.create_view(view_name, "select * from [{}]".format(csv_table)) + + if dump: + for line in db.conn.iterdump(): + click.echo(line) + return + + if save: + db2 = sqlite_utils.Database(save) + for line in db.conn.iterdump(): + db2.execute(line) + return + + for alias, attach_path in attach: + db.attach(alias, attach_path) + _load_extensions(db, load_extension) + db.register_fts4_bm25() + + _execute_query( + db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + ) + + +def _execute_query( + db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols +): with db.conn: try: cursor = db.execute(sql, dict(param)) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py new file mode 100644 index 0000000..f018837 --- /dev/null +++ b/tests/test_cli_memory.py @@ -0,0 +1,72 @@ +from sqlite_utils import cli, Database +from click.testing import CliRunner +import pytest + + +def test_memory_basic(): + result = CliRunner().invoke(cli.cli, ["memory", "select 1 + 1"]) + assert result.exit_code == 0 + assert result.output.strip() == '[{"1 + 1": 2}]' + + +@pytest.mark.parametrize("sql_from", ("test", "t", "t1")) +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_csv(tmpdir, sql_from, use_stdin): + content = "id,name\n1,Cleo\n2,Bants" + input = None + if use_stdin: + input = content + csv_path = "-" + if sql_from == "test": + sql_from = "stdin" + else: + csv_path = str(tmpdir / "test.csv") + open(csv_path, "w").write(content) + result = CliRunner().invoke( + cli.cli, + ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], + input=input, + ) + assert result.exit_code == 0 + assert ( + result.output.strip() + == '{"id": "1", "name": "Cleo"}\n{"id": "2", "name": "Bants"}' + ) + + +@pytest.mark.parametrize("extra_args", ([], ["select 1"])) +def test_memory_dump(extra_args): + result = CliRunner().invoke( + cli.cli, + ["memory", "-"] + extra_args + ["--dump"], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + assert result.output.strip() == ( + "BEGIN TRANSACTION;\n" + "CREATE TABLE [stdin] (\n" + " [id] TEXT,\n" + " [name] TEXT\n" + ");\n" + "INSERT INTO \"stdin\" VALUES('1','Cleo');\n" + "INSERT INTO \"stdin\" VALUES('2','Bants');\n" + "CREATE VIEW t1 AS select * from [stdin];\n" + "CREATE VIEW t AS select * from [stdin];\n" + "COMMIT;" + ) + + +@pytest.mark.parametrize("extra_args", ([], ["select 1"])) +def test_memory_save(tmpdir, extra_args): + save_to = str(tmpdir / "save.db") + result = CliRunner().invoke( + cli.cli, + ["memory", "-"] + extra_args + ["--save", save_to], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + db = Database(save_to) + assert list(db["stdin"].rows) == [ + {"id": "1", "name": "Cleo"}, + {"id": "2", "name": "Bants"}, + ] From 7684bbf0976431371541bc91136779b4948dbabf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 08:29:41 -0700 Subject: [PATCH 0362/1004] --encoding option for sqlite-utils memory, closes #280 Refs #272 --- docs/cli.rst | 6 ++++++ sqlite_utils/cli.py | 13 +++++++++++-- tests/test_cli_memory.py | 42 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index b7f0313..69d00a0 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -281,6 +281,12 @@ To read from standard input, use ``-`` as the filename - then use ``stdin`` or ` $ cat example.csv | sqlite-utils memory - "select * from stdin" +Incoming CSV data will be assumed to use ``utf-8``. If your data uses a different character encoding you can specify that with ``--encoding``:: + + $ cat example.csv | sqlite-utils memory - "select * from stdin" --encoding=latin-1 + +If you are joining across multiple CSV files they must all use the same encoding. + .. _cli_query_memory_attach: Joining in-memory data against existing databases using \-\-attach diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8fe37c8..eac76d4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1138,6 +1138,10 @@ def query( type=(str, str), help="Named :parameters for SQL query", ) +@click.option( + "--encoding", + help="Character encoding for CSV input, defaults to utf-8", +) @click.option("--dump", is_flag=True, help="Dump SQL for in-memory database") @click.option( "--save", @@ -1159,6 +1163,7 @@ def memory( json_cols, raw, param, + encoding, dump, save, load_extension, @@ -1171,13 +1176,17 @@ def memory( sql = None for i, path in enumerate(paths): if path == "-": - csv_fp = sys.stdin + csv_fp = sys.stdin.buffer csv_table = "stdin" else: csv_path = pathlib.Path(path) csv_table = csv_path.stem csv_fp = csv_path.open() - db[csv_table].insert_all(csv_std.DictReader(csv_fp)) + + encoding = encoding or "utf-8-sig" + decoded_fp = io.TextIOWrapper(csv_fp, encoding=encoding) + + db[csv_table].insert_all(csv_std.DictReader(decoded_fp)) # Add convenient t / t1 / t2 views view_names = ["t{}".format(i + 1)] if i == 0: diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index f018837..f45abdf 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -1,6 +1,7 @@ from sqlite_utils import cli, Database from click.testing import CliRunner import pytest +import json def test_memory_basic(): @@ -34,6 +35,47 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): ) +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_csv_encoding(tmpdir, use_stdin): + latin1_csv = ( + b"date,name,latitude,longitude\n" b"2020-03-04,S\xe3o Paulo,-23.561,-46.645\n" + ) + input = None + if use_stdin: + input = latin1_csv + csv_path = "-" + sql_from = "stdin" + else: + csv_path = str(tmpdir / "test.csv") + with open(csv_path, "wb") as fp: + fp.write(latin1_csv) + sql_from = "test" + # Without --encoding should error: + assert ( + CliRunner() + .invoke( + cli.cli, + ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], + input=input, + ) + .exit_code + == 1 + ) + # With --encoding should work: + result = CliRunner().invoke( + cli.cli, + ["memory", "-", "select * from stdin", "--encoding", "latin-1", "--nl"], + input=latin1_csv, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == { + "date": "2020-03-04", + "name": "S\u00e3o Paulo", + "latitude": "-23.561", + "longitude": "-46.645", + } + + @pytest.mark.parametrize("extra_args", ([], ["select 1"])) def test_memory_dump(extra_args): result = CliRunner().invoke( From 93594ce15b01f5ceba3bde65abe57ed28dfde9b4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 08:36:09 -0700 Subject: [PATCH 0363/1004] Open CSV in binary mode, refs #280 --- sqlite_utils/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index eac76d4..c5176f8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1181,7 +1181,7 @@ def memory( else: csv_path = pathlib.Path(path) csv_table = csv_path.stem - csv_fp = csv_path.open() + csv_fp = csv_path.open("rb") encoding = encoding or "utf-8-sig" decoded_fp = io.TextIOWrapper(csv_fp, encoding=encoding) From 00e4bd5ff18ef4c3db6c1d67e2b974131c80d65c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 20:11:54 -0700 Subject: [PATCH 0364/1004] TSV and JSON support for sqlite-utils memory Closes #281, closes #279, refs #272 --- docs/cli.rst | 51 +++++++++++++++++-------- sqlite_utils/cli.py | 24 ++++++++---- sqlite_utils/utils.py | 62 ++++++++++++++++++++++++++++++- tests/test_cli_memory.py | 80 ++++++++++++++++++++++++++++++++++++++-- 4 files changed, 191 insertions(+), 26 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 69d00a0..c6067ac 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -234,21 +234,21 @@ This example attaches the ``books.db`` database under the alias ``books`` and th sqlite-utils dogs.db --attach books books.db \ 'select * from sqlite_master union all select * from books.sqlite_master' -.. _cli_query_memory: +.. _cli_memory: -Querying CSV data directly using an in-memory database -====================================================== +Querying data directly using an in-memory database +================================================== The ``sqlite-utils memory`` command works similar to ``sqlite-utils query``, but allows you to execute queries against an in-memory database. -You can also pass this command CSV files which will be loaded into a temporary in-memory table, allowing you to execute SQL against that data without a separate step to first convert it to SQLite. +You can also pass this command CSV or JSON files which will be loaded into a temporary in-memory table, allowing you to execute SQL against that data without a separate step to first convert it to SQLite. Without any extra arguments, this command executes SQL against the in-memory database directly:: $ sqlite-utils memory 'select sqlite_version()' [{"sqlite_version()": "3.35.5"}] -It takes all of the same formatting options as :ref:`sqlite-utils query `: ``--csv`` and ``--csv`` and ``--table`` and ``--nl``:: +It takes all of the same output formatting options as :ref:`sqlite-utils query `: ``--csv`` and ``--csv`` and ``--table`` and ``--nl``:: $ sqlite-utils memory 'select sqlite_version()' --csv sqlite_version() @@ -260,24 +260,28 @@ It takes all of the same formatting options as :ref:`sqlite-utils query ` - so either a single JSON object (treated as a single row) or a list of JSON objects. + +CSV data can be comma- or tab- delimited. + +The in-memory tables will be named after the files without their extensions. The tool also sets up aliases for those tables (using SQL views) as ``t1``, ``t2`` and so on, or you can use the alias ``t`` to refer to the first table:: $ sqlite-utils memory example.csv "select * from t" -To read from standard input, use ``-`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name:: +To read from standard input, use either ``-`` or ``stdin`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name:: $ cat example.csv | sqlite-utils memory - "select * from stdin" @@ -287,7 +291,24 @@ Incoming CSV data will be assumed to use ``utf-8``. If your data uses a differen If you are joining across multiple CSV files they must all use the same encoding. -.. _cli_query_memory_attach: +.. _cli_memory_explicit: + +Explicitly specifying the format +-------------------------------- + +By default, ``sqlite-utils memory`` will attempt to detect the incoming data format (JSON, TSV or CSV) automatically. + +You can instead specify an explicit format by adding a ``:csv``, ``:tsv``, ``:json`` or ``:nl`` (for newline-delimited JSON) suffix to the filename. For example:: + + $ sqlite-utils memory one.dat:csv two.dat:nl "select * from one union select * from two" + +Here the contents of ``one.dat`` will be treated as CSV and the contents of ``two.dat`` will be treated as newline-delimited JSON. + +To explicitly specify the format for data piped into the tool on standard input, use ``stdin:format`` - for example:: + + $ cat one.dat | sqlite-utils memory stdin:csv "select * from stdin" + +.. _cli_memory_attach: Joining in-memory data against existing databases using \-\-attach ------------------------------------------------------------------ @@ -303,7 +324,7 @@ Here the ``--attach trees trees.db`` option makes the ``trees.db`` database avai The CSV data that was piped into the script is available in the ``stdin`` table, so ``... where rowid in (select id from stdin)`` can be used to return rows from the ``trees`` table that match IDs that were piped in as CSV content. -.. _cli_query_memory_dump_save: +.. _cli_memory_dump_save: \-\-dump and \-\-save --------------------- diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c5176f8..a4c1789 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -14,7 +14,14 @@ import os import sys import csv as csv_std import tabulate -from .utils import file_progress, find_spatialite, sqlite3, decode_base64_values +from .utils import ( + file_progress, + find_spatialite, + sqlite3, + decode_base64_values, + rows_from_file, + Format, +) CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) @@ -1175,18 +1182,21 @@ def memory( paths = [sql] sql = None for i, path in enumerate(paths): - if path == "-": + # Path may have a :format suffix + if ":" in path and path.rsplit(":", 1)[-1].upper() in Format.__members__: + path, suffix = path.rsplit(":", 1) + format = Format[suffix.upper()] + else: + format = None + if path in ("-", "stdin"): csv_fp = sys.stdin.buffer csv_table = "stdin" else: csv_path = pathlib.Path(path) csv_table = csv_path.stem csv_fp = csv_path.open("rb") - - encoding = encoding or "utf-8-sig" - decoded_fp = io.TextIOWrapper(csv_fp, encoding=encoding) - - db[csv_table].insert_all(csv_std.DictReader(decoded_fp)) + rows = rows_from_file(csv_fp, format=format, encoding=encoding) + db[csv_table].insert_all(rows, alter=True) # Add convenient t / t1 / t2 views view_names = ["t{}".format(i + 1)] if i == 0: diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 9d5dac6..8a0b9ab 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -1,8 +1,13 @@ import base64 -import click import contextlib +import csv +import enum import io +import json import os +from typing import Generator + +import click try: import pysqlite3 as sqlite3 @@ -111,3 +116,58 @@ def file_progress(file, silent=False, **kwargs): file_length = os.path.getsize(file.name) with click.progressbar(length=file_length, **kwargs) as bar: yield UpdateWrapper(file, bar.update) + + +class Format(enum.Enum): + CSV = 1 + TSV = 2 + JSON = 3 + NL = 4 + + +class RowsFromFileError(Exception): + pass + + +class RowsFromFileBadJSON(RowsFromFileError): + pass + + +def rows_from_file( + fp, + format=None, + dialect=None, + encoding=None, +) -> Generator[dict, None, None]: + if format == Format.JSON: + decoded = json.load(fp) + if isinstance(decoded, dict): + decoded = [decoded] + if not isinstance(decoded, list): + raise RowsFromFileBadJSON("JSON must be a list or a dictionary") + yield from decoded + elif format == Format.NL: + yield from (json.loads(line) for line in fp if line.strip()) + elif format == Format.CSV: + decoded_fp = io.TextIOWrapper(fp, encoding=encoding or "utf-8-sig") + yield from csv.DictReader(decoded_fp, dialect=dialect) + elif format == Format.TSV: + yield from rows_from_file( + fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding + ) + elif format is None: + # Detect the format, then call this recursively + buffered = io.BufferedReader(fp, buffer_size=4096) + first_bytes = buffered.peek(2048).strip() + if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"): + # TODO: Detect newline-JSON + yield from rows_from_file(buffered, format=Format.JSON) + else: + dialect = csv.Sniffer().sniff( + first_bytes.decode(encoding or "utf-8-sig", "ignore") + ) + yield from rows_from_file( + buffered, format=Format.CSV, dialect=dialect, encoding=encoding + ) + else: + raise RowsFromFileError("Bad format") diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index f45abdf..c91beaf 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -1,8 +1,10 @@ -from sqlite_utils import cli, Database -from click.testing import CliRunner -import pytest import json +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli + def test_memory_basic(): result = CliRunner().invoke(cli.cli, ["memory", "select 1 + 1"]) @@ -35,6 +37,78 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): ) +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_tsv(tmpdir, use_stdin): + data = "id\tname\n1\tCleo\n2\tBants" + if use_stdin: + input = data + path = "stdin:tsv" + sql_from = "stdin" + else: + input = None + path = str(tmpdir / "chickens.tsv") + open(path, "w").write(data) + path = path + ":tsv" + sql_from = "chickens" + result = CliRunner().invoke( + cli.cli, + ["memory", path, "select * from {}".format(sql_from)], + input=data, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + {"id": "1", "name": "Cleo"}, + {"id": "2", "name": "Bants"}, + ] + + +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_json(tmpdir, use_stdin): + data = '[{"name": "Bants"}, {"name": "Dori", "age": 1}]' + if use_stdin: + input = data + path = "stdin:json" + sql_from = "stdin" + else: + input = None + path = str(tmpdir / "chickens.json") + open(path, "w").write(data) + path = path + ":json" + sql_from = "chickens" + result = CliRunner().invoke( + cli.cli, + ["memory", path, "select * from {}".format(sql_from)], + input=input, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + {"name": "Bants", "age": None}, + {"name": "Dori", "age": 1}, + ] + + +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_json_nl(tmpdir, use_stdin): + data = '{"name": "Bants"}\n\n{"name": "Dori"}' + if use_stdin: + input = data + path = "stdin:nl" + sql_from = "stdin" + else: + input = None + path = str(tmpdir / "chickens.json") + open(path, "w").write(data) + path = path + ":nl" + sql_from = "chickens" + result = CliRunner().invoke( + cli.cli, + ["memory", path, "select * from {}".format(sql_from)], + input=data, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [{"name": "Bants"}, {"name": "Dori"}] + + @pytest.mark.parametrize("use_stdin", (True, False)) def test_memory_csv_encoding(tmpdir, use_stdin): latin1_csv = ( From 1091a9cbd804504efa8e1126226759e736e3ccdf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 20:14:12 -0700 Subject: [PATCH 0365/1004] Add sqlite-utils memory to the README, refs #272 --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8c1b4ab..4c738ab 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,11 @@ You can import JSON data into a new database table like this: Or for data in a CSV file: - $ sqlite-utils insert dogs.db dogs docs.csv --csv + $ sqlite-utils insert dogs.db dogs dogs.csv --csv + +`sqlite-utils memory` lets you import CSV or JSON data into an in-memory database and run SQL queries against it in a single command: + + $ cat dogs.csv | sqlite-utils memory - "select name, age from dogs" See the [full CLI documentation](https://sqlite-utils.datasette.io/en/stable/cli.html) for comprehensive coverage of many more commands. From 59992d2feedf964d9f7c72110755f36de49e1c8b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 20:20:56 -0700 Subject: [PATCH 0366/1004] Better help text for 'sqlite-utils memory', refs #272 --- sqlite_utils/cli.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index a4c1789..c5f50a1 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1175,7 +1175,28 @@ def memory( save, load_extension, ): - "Execute SQL query against an in-memory database, optionally populated by imported data" + """Execute SQL query against an in-memory database, optionally populated by imported data + + To import data from CSV, TSV or JSON files pass them on the command-line: + + \b + sqlite-utils memory one.csv two.json \\ + "select * from one join two on one.two_id = two.id" + + For data piped into the tool from standard input, use "-" or "stdin": + + \b + cat animals.csv | sqlite-utils memory - \\ + "select * from stdin where species = 'dog'" + + The format of the data will be automatically detected. You can specify the format + explicitly using :json, :csv, :tsv or :nl (for newline-delimited JSON) - for example: + + \b + cat animals.csv | sqlite-utils memory stdin:csv places.dat:nl \\ + "select * from stdin where place_id in (select id from places)" + + """ db = sqlite_utils.Database(memory=True) # If --dump or --save used but no paths detected, assume SQL query is a path: if (dump or save) and not paths: From fd9867d145c11a6be6c4049e0383832d0e856f4b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 21:18:58 -0700 Subject: [PATCH 0367/1004] sqlite-utils insert --detect-types option, refs #282 --- docs/cli.rst | 26 +++++++++++++++ sqlite_utils/cli.py | 18 ++++++++++ sqlite_utils/utils.py | 76 ++++++++++++++++++++++++++++++++++++++++++- tests/test_cli.py | 31 ++++++++++++++++++ 4 files changed, 150 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index c6067ac..bad2d51 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -726,6 +726,32 @@ Data is expected to be encoded as Unicode UTF-8. If your data is an another char A progress bar is displayed when inserting data from a file. You can hide the progress bar using the ``--silent`` option. +By default every column inserted from a CSV or TSV file will be of type ``TEXT``. To automatically detect column types - resulting in a mix of ``TEXT``, ``INTEGER`` and ``FLOAT`` columns, use the ``--detect-types`` option (or its shortcut ``-d``). + +For example, given a ``creatures.csv`` file containing this:: + + name,age,weight + Cleo,6,45.5 + Dori,1,3.5 + +The following command:: + + $ sqlite-utils insert creatures.db creatures creatures.tsv --csv --detect-types + +Will produce this schema:: + + $ sqlite-utils schema creatures.db + CREATE TABLE "creatures" ( + [rowid] INTEGER PRIMARY KEY, + [name] TEXT, + [age] INTEGER, + [weight] FLOAT + ); + +You can set the ``SQLITE_UTILS_DETECT_TYPES`` environment variable if you want ``--detect-types`` to be the default behavior:: + + $ export SQLITE_UTILS_DETECT_TYPES=1 + .. _cli_insert_csv_tsv_delimiter: Alternative delimiters and quote characters diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c5f50a1..96adaee 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -21,6 +21,7 @@ from .utils import ( decode_base64_values, rows_from_file, Format, + TypeTracker, ) CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) @@ -683,6 +684,13 @@ def insert_upsert_options(fn): "--encoding", help="Character encoding for input, defaults to utf-8", ), + click.option( + "-d", + "--detect-types", + is_flag=True, + envvar="SQLITE_UTILS_DETECT_TYPES", + help="Detect types for columns in CSV/TSV data", + ), load_extension_option, click.option("--silent", is_flag=True, help="Do not show progress bar"), ) @@ -712,6 +720,7 @@ def insert_upsert_implementation( not_null=None, default=None, encoding=None, + detect_types=None, load_extension=None, silent=False, ): @@ -728,6 +737,7 @@ def insert_upsert_implementation( encoding = encoding or "utf-8-sig" buffered = io.BufferedReader(json_file, buffer_size=4096) decoded = io.TextIOWrapper(buffered, encoding=encoding) + tracker = None if csv or tsv: if sniff: # Read first 2048 bytes and use that to detect @@ -749,6 +759,9 @@ def insert_upsert_implementation( else: headers = first_row docs = (dict(zip(headers, row)) for row in reader) + if detect_types: + tracker = TypeTracker() + docs = tracker.wrap(docs) else: try: if nl: @@ -781,6 +794,8 @@ def insert_upsert_implementation( "{}\n\nTry using --alter to add additional columns".format(e.args[0]) ) raise + if tracker is not None: + db[table].transform(types=tracker.types) @cli.command() @@ -815,6 +830,7 @@ def insert( batch_size, alter, encoding, + detect_types, load_extension, silent, ignore, @@ -849,6 +865,7 @@ def insert( replace=replace, truncate=truncate, encoding=encoding, + detect_types=detect_types, load_extension=load_extension, silent=silent, not_null=not_null, @@ -877,6 +894,7 @@ def upsert( not_null, default, encoding, + detect_types, load_extension, silent, ): diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 8a0b9ab..ffc727e 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -110,7 +110,16 @@ class UpdateWrapper: @contextlib.contextmanager def file_progress(file, silent=False, **kwargs): - if silent or file.fileno() == 0: # 0 = stdin + if silent: + yield file + return + # file.fileno() throws an exception in our test suite + try: + fileno = file.fileno() + except io.UnsupportedOperation: + yield file + return + if fileno == 0: # 0 means stdin yield file else: file_length = os.path.getsize(file.name) @@ -171,3 +180,68 @@ def rows_from_file( ) else: raise RowsFromFileError("Bad format") + + +class TypeTracker: + def __init__(self): + self.trackers = {} + + def wrap(self, iterator): + for row in iterator: + for key, value in row.items(): + tracker = self.trackers.setdefault(key, ValueTracker()) + tracker.evaluate(value) + yield row + + @property + def types(self): + return {key: tracker.guessed_type for key, tracker in self.trackers.items()} + + +class ValueTracker: + def __init__(self): + self.couldbe = {key: getattr(self, "test_" + key) for key in self.get_tests()} + + @classmethod + def get_tests(cls): + return [ + key.split("test_")[-1] + for key in cls.__dict__.keys() + if key.startswith("test_") + ] + + def test_integer(self, value): + try: + int(value) + return True + except ValueError: + return False + + def test_float(self, value): + try: + float(value) + return True + except ValueError: + return False + + def __repr__(self): + return self.guessed_type + ": possibilities = " + repr(self.couldbe) + + @property + def guessed_type(self): + options = set(self.couldbe.keys()) + # Return based on precedence + for key in self.get_tests(): + if key in options: + return key + return "text" + + def evaluate(self, value): + if not value or not self.couldbe: + return + not_these = [] + for name, test in self.couldbe.items(): + if not test(value): + not_these.append(name) + for key in not_these: + del self.couldbe[key] diff --git a/tests/test_cli.py b/tests/test_cli.py index b085ea4..8ddf71b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ from sqlite_utils import cli, Database from sqlite_utils.db import Index, ForeignKey from click.testing import CliRunner +from unittest import mock import json import os import pytest @@ -2067,3 +2068,33 @@ def test_csv_insert_bom(tmpdir): ("broken", "CREATE TABLE [broken] (\n [\ufeffname] TEXT,\n [age] TEXT\n)"), ("fixed", "CREATE TABLE [fixed] (\n [name] TEXT,\n [age] TEXT\n)"), ] + + +@pytest.mark.parametrize("option_or_env_var", (None, "-d", "--detect-types")) +def test_insert_detect_types(tmpdir, option_or_env_var): + db_path = str(tmpdir / "test.db") + data = "name,age,weight\nCleo,6,45.5\nDori,1,3.5" + extra = [] + if option_or_env_var: + extra = [option_or_env_var] + + def _test(): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "-", "--csv"] + extra, + catch_exceptions=False, + input=data, + ) + assert result.exit_code == 0 + db = Database(db_path) + assert list(db["creatures"].rows) == [ + {"rowid": 1, "name": "Cleo", "age": 6, "weight": 45.5}, + {"rowid": 2, "name": "Dori", "age": 1, "weight": 3.5}, + ] + + if option_or_env_var is None: + # Use environemnt variable instead of option + with mock.patch.dict(os.environ, {"SQLITE_UTILS_DETECT_TYPES": "1"}): + _test() + else: + _test() From ec5174ed40fa283cb06f25ee0c0136297ec313ae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 21:37:56 -0700 Subject: [PATCH 0368/1004] Detect types for sqlite-utils memory CSV, opt out with --no-detect-types - closes #282 --- docs/cli.rst | 2 ++ sqlite_utils/cli.py | 13 +++++++++++ sqlite_utils/utils.py | 1 + tests/test_cli_memory.py | 49 ++++++++++++++++++++++++++++------------ 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index bad2d51..8a061c3 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -291,6 +291,8 @@ Incoming CSV data will be assumed to use ``utf-8``. If your data uses a differen If you are joining across multiple CSV files they must all use the same encoding. +Column types will be automatically detected in CSV or TSV data, using the same mechanism as ``--detect-types`` described in :ref:`cli_insert_csv_tsv`. You can pass the ``--no-detect-types`` option to disable this automatic type detection and treat all CSV and TSV columns as ``TEXT``. + .. _cli_memory_explicit: Explicitly specifying the format diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 96adaee..4ae1b21 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1167,6 +1167,12 @@ def query( "--encoding", help="Character encoding for CSV input, defaults to utf-8", ) +@click.option( + "-n", + "--no-detect-types", + is_flag=True, + help="Treat all CSV/TSV columns as TEXT", +) @click.option("--dump", is_flag=True, help="Dump SQL for in-memory database") @click.option( "--save", @@ -1189,6 +1195,7 @@ def memory( raw, param, encoding, + no_detect_types, dump, save, load_extension, @@ -1235,7 +1242,13 @@ def memory( csv_table = csv_path.stem csv_fp = csv_path.open("rb") rows = rows_from_file(csv_fp, format=format, encoding=encoding) + tracker = None + if not no_detect_types: + tracker = TypeTracker() + rows = tracker.wrap(rows) db[csv_table].insert_all(rows, alter=True) + if tracker is not None: + db[csv_table].transform(types=tracker.types) # Add convenient t / t1 / t2 views view_names = ["t{}".format(i + 1)] if i == 0: diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index ffc727e..473ad37 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -147,6 +147,7 @@ def rows_from_file( format=None, dialect=None, encoding=None, + detect_types=False, ) -> Generator[dict, None, None]: if format == Format.JSON: decoded = json.load(fp) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index c91beaf..fb0f153 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -33,7 +33,7 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): assert result.exit_code == 0 assert ( result.output.strip() - == '{"id": "1", "name": "Cleo"}\n{"id": "2", "name": "Bants"}' + == '{"rowid": 1, "id": 1, "name": "Cleo"}\n{"rowid": 2, "id": 2, "name": "Bants"}' ) @@ -57,8 +57,8 @@ def test_memory_tsv(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ - {"id": "1", "name": "Cleo"}, - {"id": "2", "name": "Bants"}, + {"rowid": 1, "id": 1, "name": "Cleo"}, + {"rowid": 2, "id": 2, "name": "Bants"}, ] @@ -82,8 +82,8 @@ def test_memory_json(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ - {"name": "Bants", "age": None}, - {"name": "Dori", "age": 1}, + {"rowid": 1, "name": "Bants", "age": None}, + {"rowid": 2, "name": "Dori", "age": 1}, ] @@ -106,7 +106,10 @@ def test_memory_json_nl(tmpdir, use_stdin): input=data, ) assert result.exit_code == 0, result.output - assert json.loads(result.output.strip()) == [{"name": "Bants"}, {"name": "Dori"}] + assert json.loads(result.output.strip()) == [ + {"rowid": 1, "name": "Bants"}, + {"rowid": 2, "name": "Dori"}, + ] @pytest.mark.parametrize("use_stdin", (True, False)) @@ -143,10 +146,11 @@ def test_memory_csv_encoding(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == { + "rowid": 1, "date": "2020-03-04", - "name": "S\u00e3o Paulo", - "latitude": "-23.561", - "longitude": "-46.645", + "name": "São Paulo", + "latitude": -23.561, + "longitude": -46.645, } @@ -160,12 +164,13 @@ def test_memory_dump(extra_args): assert result.exit_code == 0 assert result.output.strip() == ( "BEGIN TRANSACTION;\n" - "CREATE TABLE [stdin] (\n" - " [id] TEXT,\n" + 'CREATE TABLE "stdin" (\n' + " [rowid] INTEGER PRIMARY KEY,\n" + " [id] INTEGER,\n" " [name] TEXT\n" ");\n" - "INSERT INTO \"stdin\" VALUES('1','Cleo');\n" - "INSERT INTO \"stdin\" VALUES('2','Bants');\n" + "INSERT INTO \"stdin\" VALUES(1,1,'Cleo');\n" + "INSERT INTO \"stdin\" VALUES(2,2,'Bants');\n" "CREATE VIEW t1 AS select * from [stdin];\n" "CREATE VIEW t AS select * from [stdin];\n" "COMMIT;" @@ -183,6 +188,20 @@ def test_memory_save(tmpdir, extra_args): assert result.exit_code == 0 db = Database(save_to) assert list(db["stdin"].rows) == [ - {"id": "1", "name": "Cleo"}, - {"id": "2", "name": "Bants"}, + {"rowid": 1, "id": 1, "name": "Cleo"}, + {"rowid": 2, "id": 2, "name": "Bants"}, + ] + + +@pytest.mark.parametrize("option", ("-n", "--no-detect-types")) +def test_memory_no_detect_types(option): + result = CliRunner().invoke( + cli.cli, + ["memory", "-", "select * from stdin"] + [option], + input="id,name,weight\n1,Cleo,45.5\n2,Bants,3.5", + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + {"id": "1", "name": "Cleo", "weight": "45.5"}, + {"id": "2", "name": "Bants", "weight": "3.5"}, ] From dc94f4bb8cfe922bb2f9c89f8f0f29092ea63133 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 07:52:44 -0700 Subject: [PATCH 0369/1004] Don't detect types on JSON input to memory, closes #283 --- sqlite_utils/cli.py | 4 ++-- sqlite_utils/utils.py | 21 ++++++++++++--------- tests/test_cli_memory.py | 10 +++++----- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4ae1b21..32bd693 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1241,9 +1241,9 @@ def memory( csv_path = pathlib.Path(path) csv_table = csv_path.stem csv_fp = csv_path.open("rb") - rows = rows_from_file(csv_fp, format=format, encoding=encoding) + rows, format_used = rows_from_file(csv_fp, format=format, encoding=encoding) tracker = None - if not no_detect_types: + if format_used in (Format.CSV, Format.TSV) and not no_detect_types: tracker = TypeTracker() rows = tracker.wrap(rows) db[csv_table].insert_all(rows, alter=True) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 473ad37..4dd5f21 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -155,15 +155,18 @@ def rows_from_file( decoded = [decoded] if not isinstance(decoded, list): raise RowsFromFileBadJSON("JSON must be a list or a dictionary") - yield from decoded + return decoded, Format.JSON elif format == Format.NL: - yield from (json.loads(line) for line in fp if line.strip()) + return (json.loads(line) for line in fp if line.strip()), Format.NL elif format == Format.CSV: decoded_fp = io.TextIOWrapper(fp, encoding=encoding or "utf-8-sig") - yield from csv.DictReader(decoded_fp, dialect=dialect) + return csv.DictReader(decoded_fp, dialect=dialect), Format.CSV elif format == Format.TSV: - yield from rows_from_file( - fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding + return ( + rows_from_file( + fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding + )[0], + Format.TSV, ) elif format is None: # Detect the format, then call this recursively @@ -171,12 +174,12 @@ def rows_from_file( first_bytes = buffered.peek(2048).strip() if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"): # TODO: Detect newline-JSON - yield from rows_from_file(buffered, format=Format.JSON) + return rows_from_file(buffered, format=Format.JSON) else: dialect = csv.Sniffer().sniff( first_bytes.decode(encoding or "utf-8-sig", "ignore") ) - yield from rows_from_file( + return rows_from_file( buffered, format=Format.CSV, dialect=dialect, encoding=encoding ) else: @@ -215,14 +218,14 @@ class ValueTracker: try: int(value) return True - except ValueError: + except (ValueError, TypeError): return False def test_float(self, value): try: float(value) return True - except ValueError: + except (ValueError, TypeError): return False def __repr__(self): diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index fb0f153..2a1fb85 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -64,7 +64,7 @@ def test_memory_tsv(tmpdir, use_stdin): @pytest.mark.parametrize("use_stdin", (True, False)) def test_memory_json(tmpdir, use_stdin): - data = '[{"name": "Bants"}, {"name": "Dori", "age": 1}]' + data = '[{"name": "Bants"}, {"name": "Dori", "age": 1, "nested": {"nest": 1}}]' if use_stdin: input = data path = "stdin:json" @@ -82,8 +82,8 @@ def test_memory_json(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ - {"rowid": 1, "name": "Bants", "age": None}, - {"rowid": 2, "name": "Dori", "age": 1}, + {"name": "Bants", "age": None, "nested": None}, + {"name": "Dori", "age": 1, "nested": '{"nest": 1}'}, ] @@ -107,8 +107,8 @@ def test_memory_json_nl(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ - {"rowid": 1, "name": "Bants"}, - {"rowid": 2, "name": "Dori"}, + {"name": "Bants"}, + {"name": "Dori"}, ] From 5b257949d996fe43dc5d218d4308b88796a90740 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 08:12:29 -0700 Subject: [PATCH 0370/1004] table.use_rowid introspection property, closes #285 --- docs/python-api.rst | 13 +++++++++++++ sqlite_utils/db.py | 4 ++++ tests/test_introspect.py | 8 ++++++++ 3 files changed, 25 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index b25eb21..85f0296 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1549,6 +1549,19 @@ The ``.pks`` property returns a list of strings naming the primary key columns f >>> db["PlantType"].pks ['id'] +If a table has no primary keys but is a `rowid table `__, this property will return ``['rowid']``. + +.. _python_api_introspection_use_rowid: + +.use_rowid +---------- + +Almost all SQLite tables have a ``rowid`` column, but a table with no explicitly defined primary keys must use that ``rowid`` as the primary key for identifying individual rows. The ``.use_rowid`` property checks to see if a table needs to use the ``rowid`` in this way - it returns ``True`` if the table has no explicitly defined primary keys and ``False`` otherwise. + + >>> db["PlantType"].use_rowid + False + + .. _python_api_introspection_foreign_keys: .foreign_keys diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 31d4cee..2d8fb24 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -809,6 +809,10 @@ class Table(Queryable): names = ["rowid"] return names + @property + def use_rowid(self): + return not any(column for column in self.columns if column.is_pk) + def get(self, pk_values): if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] diff --git a/tests/test_introspect.py b/tests/test_introspect.py index d54ca88..6ca578c 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -237,3 +237,11 @@ def test_virtual_table_using(sql, expected_name, expected_using): db = Database(memory=True) db.execute(sql) assert db[expected_name].virtual_table_using == expected_using + + +def test_use_rowid(): + db = Database(memory=True) + db["rowid_table"].insert({"name": "Cleo"}) + db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id") + assert db["rowid_table"].use_rowid + assert not db["regular_table"].use_rowid From 0e797033f96a1c61b173a3d8af2ff36905687a2e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 08:28:26 -0700 Subject: [PATCH 0371/1004] .transform() on rowid (non-pk) tables bug fix, closes #284 --- sqlite_utils/db.py | 4 ++- tests/test_cli.py | 4 +-- tests/test_cli_memory.py | 17 ++++----- tests/test_extract.py | 15 +++++++- tests/test_transform.py | 76 ++++++++++++++++++++++++++++++++++++++-- 5 files changed, 100 insertions(+), 16 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 2d8fb24..9da1b97 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1016,7 +1016,9 @@ class Table(Queryable): sqls = [] if pk is DEFAULT: - pks_renamed = tuple(rename.get(p) or p for p in self.pks) + pks_renamed = tuple( + rename.get(p.name) or p.name for p in self.columns if p.is_pk + ) if len(pks_renamed) == 1: pk = pks_renamed[0] else: diff --git a/tests/test_cli.py b/tests/test_cli.py index 8ddf71b..8d59c0c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2088,8 +2088,8 @@ def test_insert_detect_types(tmpdir, option_or_env_var): assert result.exit_code == 0 db = Database(db_path) assert list(db["creatures"].rows) == [ - {"rowid": 1, "name": "Cleo", "age": 6, "weight": 45.5}, - {"rowid": 2, "name": "Dori", "age": 1, "weight": 3.5}, + {"name": "Cleo", "age": 6, "weight": 45.5}, + {"name": "Dori", "age": 1, "weight": 3.5}, ] if option_or_env_var is None: diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index 2a1fb85..6966d05 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -32,8 +32,7 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): ) assert result.exit_code == 0 assert ( - result.output.strip() - == '{"rowid": 1, "id": 1, "name": "Cleo"}\n{"rowid": 2, "id": 2, "name": "Bants"}' + result.output.strip() == '{"id": 1, "name": "Cleo"}\n{"id": 2, "name": "Bants"}' ) @@ -57,8 +56,8 @@ def test_memory_tsv(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ - {"rowid": 1, "id": 1, "name": "Cleo"}, - {"rowid": 2, "id": 2, "name": "Bants"}, + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Bants"}, ] @@ -146,7 +145,6 @@ def test_memory_csv_encoding(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == { - "rowid": 1, "date": "2020-03-04", "name": "São Paulo", "latitude": -23.561, @@ -165,12 +163,11 @@ def test_memory_dump(extra_args): assert result.output.strip() == ( "BEGIN TRANSACTION;\n" 'CREATE TABLE "stdin" (\n' - " [rowid] INTEGER PRIMARY KEY,\n" " [id] INTEGER,\n" " [name] TEXT\n" ");\n" - "INSERT INTO \"stdin\" VALUES(1,1,'Cleo');\n" - "INSERT INTO \"stdin\" VALUES(2,2,'Bants');\n" + "INSERT INTO \"stdin\" VALUES(1,'Cleo');\n" + "INSERT INTO \"stdin\" VALUES(2,'Bants');\n" "CREATE VIEW t1 AS select * from [stdin];\n" "CREATE VIEW t AS select * from [stdin];\n" "COMMIT;" @@ -188,8 +185,8 @@ def test_memory_save(tmpdir, extra_args): assert result.exit_code == 0 db = Database(save_to) assert list(db["stdin"].rows) == [ - {"rowid": 1, "id": 1, "name": "Cleo"}, - {"rowid": 2, "id": 2, "name": "Bants"}, + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Bants"}, ] diff --git a/tests/test_extract.py b/tests/test_extract.py index 9eae704..280c59c 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -126,12 +126,25 @@ def test_extract_rowid_table(fresh_db): fresh_db["tree"].extract(["common_name", "latin_name"]) assert fresh_db["tree"].schema == ( 'CREATE TABLE "tree" (\n' - " [rowid] INTEGER PRIMARY KEY,\n" " [name] TEXT,\n" " [common_name_latin_name_id] INTEGER,\n" " FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n" ")" ) + assert ( + fresh_db.execute( + """ + select + tree.name, + common_name_latin_name.common_name, + common_name_latin_name.latin_name + from tree + join common_name_latin_name + on tree.common_name_latin_name_id = common_name_latin_name.id + """ + ).fetchall() + == [("Tree 1", "Palm", "Arecaceae")] + ) def test_reuse_lookup_table(fresh_db): diff --git a/tests/test_transform.py b/tests/test_transform.py index b3ef009..19e447e 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -89,7 +89,9 @@ import pytest ], ) @pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) -def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys): +def test_transform_sql_table_with_primary_key( + fresh_db, params, expected_sql, use_pragma_foreign_keys +): captured = [] tracer = lambda sql, params: captured.append((sql, params)) dogs = fresh_db["dogs"] @@ -111,7 +113,77 @@ def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys): assert ("PRAGMA foreign_keys=1;", None) not in captured -def test_transform_sql_rowid_to_id(fresh_db): +@pytest.mark.parametrize( + "params,expected_sql", + [ + # Identity transform - nothing changes + ( + {}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + ], + ), + # Change column type + ( + {"types": {"age": int}}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] INTEGER\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + ], + ), + # Rename a column + ( + {"rename": {"age": "dog_age"}}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [dog_age] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + ], + ), + # Make ID a primary key + ( + {"pk": "id"}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + ], + ), + ], +) +@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) +def test_transform_sql_table_with_no_primary_key( + fresh_db, params, expected_sql, use_pragma_foreign_keys +): + captured = [] + tracer = lambda sql, params: captured.append((sql, params)) + dogs = fresh_db["dogs"] + if use_pragma_foreign_keys: + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) + sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}}) + assert sql == expected_sql + # Check that .transform() runs without exceptions: + with fresh_db.tracer(tracer): + dogs.transform(**params) + # If use_pragma_foreign_keys, check that we did the right thing + if use_pragma_foreign_keys: + assert ("PRAGMA foreign_keys=0;", None) in captured + assert captured[-2] == ("PRAGMA foreign_key_check;", None) + assert captured[-1] == ("PRAGMA foreign_keys=1;", None) + else: + assert ("PRAGMA foreign_keys=0;", None) not in captured + assert ("PRAGMA foreign_keys=1;", None) not in captured + + +def test_transform_sql_with_no_primary_key_to_primary_key_of_id(fresh_db): dogs = fresh_db["dogs"] dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) assert ( From 13e76b375ac3e3448df5d705ba65fadaaf9887d6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 09:01:39 -0700 Subject: [PATCH 0372/1004] Release 3.10 Refs #272, #274, #275, #276, #282, #284, #285 --- docs/changelog.rst | 68 ++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 879d19b..cb9ab6d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,74 @@ Changelog =========== +.. _3.10: + +3.10 (2021-06-19) +----------------- + +This release introduces the ``sqlite-utils memory`` command, which can be used to load CSV or JSON data into a temporary in-memory database and run SQL queries (including joins across multiple files) directly against that data. + +Also new: ``sqlite-utils insert --detect-types``, ``sqlite-utils dump``, ``table.use_rowid`` plus some smaller fixes. + +sqlite-utils memory +~~~~~~~~~~~~~~~~~~~ + +This example of ``sqlite-utils memory`` retrieves information about the all of the repositories in the `Dogsheep `__ organization on GitHub using `this JSON API `__, sorts them by their number of stars and outputs a table of the top five (using ``-t``):: + + $ curl -s 'https://api.github.com/users/dogsheep/repos' \ + | sqlite-utils memory - ' + select full_name, forks_count, stargazers_count + from stdin order by stargazers_count desc limit 5 + ' -t + full_name forks_count stargazers_count + --------------------------------- ------------- ------------------ + dogsheep/twitter-to-sqlite 12 225 + dogsheep/github-to-sqlite 14 139 + dogsheep/dogsheep-photos 5 116 + dogsheep/dogsheep.github.io 7 90 + dogsheep/healthkit-to-sqlite 4 85 + +The tool works against files on disk as well. This example joins data from two CSV files:: + + $ cat creatures.csv + species_id,name + 1,Cleo + 2,Bants + 2,Dori + 2,Azi + $ cat species.csv + id,species_name + 1,Dog + 2,Chicken + $ sqlite-utils memory species.csv creatures.csv ' + select * from creatures join species on creatures.species_id = species.id + ' + [{"species_id": 1, "name": "Cleo", "id": 1, "species_name": "Dog"}, + {"species_id": 2, "name": "Bants", "id": 2, "species_name": "Chicken"}, + {"species_id": 2, "name": "Dori", "id": 2, "species_name": "Chicken"}, + {"species_id": 2, "name": "Azi", "id": 2, "species_name": "Chicken"}] + +Here the ``species.csv`` file becomes the ``species`` table, the ``creatures.csv`` file becomes the ``creatures`` table and the output is JSON, the default output format. + +You can also use the ``--attach`` option to attach existing SQLite database files to the in-memory database, in order to join data from CSV or JSON directly against your existing tables. + +Full documentation of this new feature is available in :ref:`cli_memory`. (`#272 `__) + +sqlite-utils insert \-\-detect-types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`sqlite-utils insert ` command can be used to insert data from JSON, CSV or TSV files into a SQLite database file. The new ``--detect-types`` option (shortcut ``-d``), when used in conjunction with a CSV or TSV import, will automatically detect if columns in the file are integers or floating point numbers as opposed to treating everything as a text column and create the new table with the corresponding schema. See :ref:`cli_insert_csv_tsv` for details. (`#282 `__) + +Other changes +~~~~~~~~~~~~~ + +- **Bug fix**: ``table.transform()``, when run against a table without explicit primary keys, would incorrectly create a new version of the table with an explicit primary key column called ``rowid``. (`#284 `__) +- New ``table.use_rowid`` introspection property, see :ref:`python_api_introspection_use_rowid`. (`#285 `__) +- The new ``sqlite-utils dump file.db`` command outputs a SQL dump that can be used to recreate a database. (`#274 `__) +- ``-h`` now works as a shortcut for ``--help``, thanks Loren McIntyre. (`#276 `__) +- Now using `pytest-cov `__ and `Codecov `__ to track test coverage - currently at 96%. (`#275 `__) +- SQL errors that occur when using ``sqlite-utils query`` are now displayed as CLI errors. + .. _3.9.1: 3.9.1 (2021-06-12) diff --git a/setup.py b/setup.py index 9c73ff4..408055c 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.9.1" +VERSION = "3.10" def get_long_description(): From eb18b6e42c6d10aca6193204bc907490d2f56547 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 09:09:29 -0700 Subject: [PATCH 0373/1004] Disabling macos-latest for the moment GitHub seems to have run out of workers right now. --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9f05567..2ea0d33 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, windows-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From e71c41d39ba32785772bfbaf62aad4cc63839124 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 13:36:16 -0700 Subject: [PATCH 0374/1004] Fixed broken anchors --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cb9ab6d..e358994 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog =========== -.. _3.10: +.. _3_10: 3.10 (2021-06-19) ----------------- @@ -70,7 +70,7 @@ Other changes - Now using `pytest-cov `__ and `Codecov `__ to track test coverage - currently at 96%. (`#275 `__) - SQL errors that occur when using ``sqlite-utils query`` are now displayed as CLI errors. -.. _3.9.1: +.. _3_9_1: 3.9.1 (2021-06-12) ------------------ From dce9bb05b697e6f5caebf2f46dcd7cec83055bcb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 14:43:04 -0700 Subject: [PATCH 0375/1004] Really fix the anchors this time --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e358994..bbbd5f2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog =========== -.. _3_10: +.. _v3_10: 3.10 (2021-06-19) ----------------- @@ -70,7 +70,7 @@ Other changes - Now using `pytest-cov `__ and `Codecov `__ to track test coverage - currently at 96%. (`#275 `__) - SQL errors that occur when using ``sqlite-utils query`` are now displayed as CLI errors. -.. _3_9_1: +.. _v3_9_1: 3.9.1 (2021-06-12) ------------------ From 933be66eba1203a3287ebaacd69f694d12f6f0a0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Jun 2021 11:25:21 -0700 Subject: [PATCH 0376/1004] sqlite-utils memory --schema, closes #288 Also updated some rowid examples, closes #287 --- docs/cli.rst | 96 +++++++++++++++++++++------------------- sqlite_utils/cli.py | 8 +++- tests/test_cli_memory.py | 18 ++++++++ 3 files changed, 76 insertions(+), 46 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 8a061c3..a404333 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -326,24 +326,33 @@ Here the ``--attach trees trees.db`` option makes the ``trees.db`` database avai The CSV data that was piped into the script is available in the ``stdin`` table, so ``... where rowid in (select id from stdin)`` can be used to return rows from the ``trees`` table that match IDs that were piped in as CSV content. -.. _cli_memory_dump_save: +.. _cli_memory_schema_dump_save: -\-\-dump and \-\-save ---------------------- +\-\-schema, \-\-dump and \-\-save +--------------------------------- -You can dump out the SQL used for the temporary in-memory database, complete with all imported data, using the ``--dump`` option:: +To see the schema that will be created for a file or multiple files, use ``--schema``:: + + % sqlite-utils memory dogs.csv --schema + CREATE TABLE [dogs] ( + [id] INTEGER, + [age] INTEGER, + [name] TEXT + ); + CREATE VIEW t1 AS select * from [dogs]; + CREATE VIEW t AS select * from [dogs]; + +You can output SQL that will both create the tables and insert the full data used to populate the in-memory database using ``--dump``:: % sqlite-utils memory dogs.csv --dump BEGIN TRANSACTION; CREATE TABLE [dogs] ( - [rowid] TEXT, - [id] TEXT, - [dog_age] TEXT, + [id] INTEGER, + [age] INTEGER, [name] TEXT ); - INSERT INTO "dogs" VALUES('1','1','4','Cleo'); - INSERT INTO "dogs" VALUES('2','2','2','Pancakes'); - INSERT INTO "dogs" VALUES('3','2','3','Pancakes'); + INSERT INTO "dogs" VALUES('1','4','Cleo'); + INSERT INTO "dogs" VALUES('2','2','Pancakes'); CREATE VIEW t1 AS select * from [dogs]; CREATE VIEW t AS select * from [dogs]; COMMIT; @@ -354,7 +363,6 @@ Passing ``--save other.db`` will instead use that SQL to populate a new database These features are mainly intented as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`. - .. _cli_rows: Returning all rows in a table @@ -738,13 +746,12 @@ For example, given a ``creatures.csv`` file containing this:: The following command:: - $ sqlite-utils insert creatures.db creatures creatures.tsv --csv --detect-types + $ sqlite-utils insert creatures.db creatures creatures.csv --csv --detect-types Will produce this schema:: $ sqlite-utils schema creatures.db CREATE TABLE "creatures" ( - [rowid] INTEGER PRIMARY KEY, [name] TEXT, [age] INTEGER, [weight] FLOAT @@ -1108,45 +1115,44 @@ Here's a more complex example that makes use of these options. It converts `this --fk-column country_id \ --rename country_long name -After running the above, the command ``sqlite3 global.db .schema`` reveals the following schema: +After running the above, the command ``sqlite-utils schema global.db`` reveals the following schema: .. code-block:: sql CREATE TABLE [countries] ( - [id] INTEGER PRIMARY KEY, - [country] TEXT, - [name] TEXT + [id] INTEGER PRIMARY KEY, + [country] TEXT, + [name] TEXT + ); + CREATE TABLE "power_plants" ( + [country_id] INTEGER, + [name] TEXT, + [gppd_idnr] TEXT, + [capacity_mw] TEXT, + [latitude] TEXT, + [longitude] TEXT, + [primary_fuel] TEXT, + [other_fuel1] TEXT, + [other_fuel2] TEXT, + [other_fuel3] TEXT, + [commissioning_year] TEXT, + [owner] TEXT, + [source] TEXT, + [url] TEXT, + [geolocation_source] TEXT, + [wepp_id] TEXT, + [year_of_capacity_data] TEXT, + [generation_gwh_2013] TEXT, + [generation_gwh_2014] TEXT, + [generation_gwh_2015] TEXT, + [generation_gwh_2016] TEXT, + [generation_gwh_2017] TEXT, + [generation_data_source] TEXT, + [estimated_generation_gwh] TEXT, + FOREIGN KEY([country_id]) REFERENCES [countries]([id]) ); CREATE UNIQUE INDEX [idx_countries_country_name] ON [countries] ([country], [name]); - CREATE TABLE IF NOT EXISTS "power_plants" ( - [rowid] INTEGER PRIMARY KEY, - [country_id] INTEGER, - [name] TEXT, - [gppd_idnr] TEXT, - [capacity_mw] TEXT, - [latitude] TEXT, - [longitude] TEXT, - [primary_fuel] TEXT, - [other_fuel1] TEXT, - [other_fuel2] TEXT, - [other_fuel3] TEXT, - [commissioning_year] TEXT, - [owner] TEXT, - [source] TEXT, - [url] TEXT, - [geolocation_source] TEXT, - [wepp_id] TEXT, - [year_of_capacity_data] TEXT, - [generation_gwh_2013] TEXT, - [generation_gwh_2014] TEXT, - [generation_gwh_2015] TEXT, - [generation_gwh_2016] TEXT, - [generation_gwh_2017] TEXT, - [generation_data_source] TEXT, - [estimated_generation_gwh] TEXT, - FOREIGN KEY(country_id) REFERENCES countries(id) - ); .. _cli_create_view: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 32bd693..8043c98 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1173,6 +1173,7 @@ def query( is_flag=True, help="Treat all CSV/TSV columns as TEXT", ) +@click.option("--schema", is_flag=True, help="Show SQL schema for in-memory database") @click.option("--dump", is_flag=True, help="Dump SQL for in-memory database") @click.option( "--save", @@ -1196,6 +1197,7 @@ def memory( param, encoding, no_detect_types, + schema, dump, save, load_extension, @@ -1224,7 +1226,7 @@ def memory( """ db = sqlite_utils.Database(memory=True) # If --dump or --save used but no paths detected, assume SQL query is a path: - if (dump or save) and not paths: + if (dump or save or schema) and not paths: paths = [sql] sql = None for i, path in enumerate(paths): @@ -1262,6 +1264,10 @@ def memory( click.echo(line) return + if schema: + click.echo(db.schema) + return + if save: db2 = sqlite_utils.Database(save) for line in db.conn.iterdump(): diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index 6966d05..e42e1cb 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -174,6 +174,24 @@ def test_memory_dump(extra_args): ) +@pytest.mark.parametrize("extra_args", ([], ["select 1"])) +def test_memory_schema(extra_args): + result = CliRunner().invoke( + cli.cli, + ["memory", "-"] + extra_args + ["--schema"], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + assert result.output.strip() == ( + 'CREATE TABLE "stdin" (\n' + " [id] INTEGER,\n" + " [name] TEXT\n" + ");\n" + "CREATE VIEW t1 AS select * from [stdin];\n" + "CREATE VIEW t AS select * from [stdin];" + ) + + @pytest.mark.parametrize("extra_args", ([], ["select 1"])) def test_memory_save(tmpdir, extra_args): save_to = str(tmpdir / "save.db") From dbcba6c597fb199d3a7ea0dd732909ec86b616cf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Jun 2021 11:47:09 -0700 Subject: [PATCH 0377/1004] Added installation instructions, closes #286 --- docs/index.rst | 1 + docs/installation.rst | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 docs/installation.rst diff --git a/docs/index.rst b/docs/index.rst index 571020f..b1bfc0e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,6 +29,7 @@ Contents .. toctree:: :maxdepth: 3 + installation cli python-api changelog diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..099dd16 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,40 @@ +.. _installation: + +============== + Installation +============== + +``sqlite-utils`` is tested on Linux, macOS and Windows. + +.. _installation_homebrew: + +Using Homebrew +============== + +The :ref:`sqlite-utils commad-line tool ` can be installed on macOS using Homebrew:: + + brew install sqlite-utils + +If you have it installed and want to upgrade to the most recent release, you ran run:: + + brew upgrade sqlite-utils + +Then run ``sqlite-utils --version`` to confirm the installed version. + +.. _installation_pip: + +Using pip +========= + +The `sqlite-utils package `__ on PyPI includes both the :ref:`sqlite_utils Python library ` and the ``sqlite-utils`` command-line tool. You can install them using ``pip`` like so:: + + pip install sqlite-utils + +.. _installation_pipx: + +Using pipx +========== + +`pipx `__ is a tool for installing Python command-line applications in their own isolated environments. You can use ``pipx`` to install the ``sqlite-utils`` command-line tool like this:: + + pipx install sqlite-utils From a25a5845b8b4dd79a3c3ff681b7ddd1f9a608b58 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Jun 2021 11:50:48 -0700 Subject: [PATCH 0378/1004] Release 3.11 Refs #286, #287, #288 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index bbbd5f2..0c31e79 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v3_11: + +3.11 (2021-06-20) +----------------- + +- New ``sqlite-utils memory data.csv --schema`` option, for outputting the schema of the in-memory database generated from one or more files. See :ref:`cli_memory_schema_dump_save`. (`#288 `__) +- Added :ref:`installation instructions `. (`#286 `__) + .. _v3_10: 3.10 (2021-06-19) diff --git a/setup.py b/setup.py index 408055c..626409e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.10" +VERSION = "3.11" def get_long_description(): From adcd32a866dd8828002b9fdc77b4f674fa80c46e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Jun 2021 11:52:15 -0700 Subject: [PATCH 0379/1004] Re-enable publish testing on macos-latest --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2ea0d33..c0bd779 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 8cedc6a8b29180e68326f6b76f249d5e39e4b591 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Jun 2021 13:44:02 -0700 Subject: [PATCH 0380/1004] Typo fix Thanks, https://twitter.com/garrettc/status/1406705348648525830 --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 099dd16..aa3234d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -15,7 +15,7 @@ The :ref:`sqlite-utils commad-line tool ` can be installed on macOS using H brew install sqlite-utils -If you have it installed and want to upgrade to the most recent release, you ran run:: +If you have it installed and want to upgrade to the most recent release, you can run:: brew upgrade sqlite-utils From 9faeef230bf84c2f9b2859e5a4544f5ec50adf68 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Jun 2021 21:03:59 -0700 Subject: [PATCH 0381/1004] New db.query() method, refs #290 --- sqlite_utils/db.py | 8 ++++++-- tests/test_cli.py | 43 +++++++++++++++++++++---------------------- tests/test_create.py | 8 ++++---- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9da1b97..1a88dba 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -359,10 +359,14 @@ class Database: for table in tables ) - def execute_returning_dicts(self, sql, params=None): + def query(self, sql, params=None): cursor = self.execute(sql, params or tuple()) keys = [d[0] for d in cursor.description] - return [dict(zip(keys, row)) for row in cursor.fetchall()] + for row in cursor: + yield dict(zip(keys, row)) + + def execute_returning_dicts(self, sql, params=None): + return list(self.query(sql, params)) def resolve_foreign_keys(self, name, foreign_keys): # foreign_keys may be a list of strcolumn names, a list of ForeignKey tuples, diff --git a/tests/test_cli.py b/tests/test_cli.py index 8d59c0c..5028458 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -613,8 +613,8 @@ def test_insert_simple(tmpdir): open(json_path, "w").write(json.dumps({"name": "Cleo", "age": 4})) result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path]) assert 0 == result.exit_code - assert [{"age": 4, "name": "Cleo"}] == Database(db_path).execute_returning_dicts( - "select * from dogs" + assert [{"age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") ) db = Database(db_path) assert ["dogs"] == db.table_names() @@ -629,8 +629,8 @@ def test_insert_from_stdin(tmpdir): input=json.dumps({"name": "Cleo", "age": 4}), ) assert 0 == result.exit_code - assert [{"age": 4, "name": "Cleo"}] == Database(db_path).execute_returning_dicts( - "select * from dogs" + assert [{"age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") ) @@ -655,9 +655,9 @@ def test_insert_with_primary_key(db_path, tmpdir): cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] ) assert 0 == result.exit_code - assert [{"id": 1, "age": 4, "name": "Cleo"}] == Database( - db_path - ).execute_returning_dicts("select * from dogs") + assert [{"id": 1, "age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") + ) db = Database(db_path) assert ["id"] == db["dogs"].pks @@ -671,7 +671,7 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir): ) assert 0 == result.exit_code db = Database(db_path) - assert dogs == db.execute_returning_dicts("select * from dogs order by id") + assert dogs == list(db.query("select * from dogs order by id")) assert ["id"] == db["dogs"].pks @@ -687,7 +687,7 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): ) assert 0 == result.exit_code db = Database(db_path) - assert dogs == db.execute_returning_dicts("select * from dogs order by breed, id") + assert dogs == list(db.query("select * from dogs order by breed, id")) assert {"breed", "id"} == set(db["dogs"].pks) assert ( "CREATE TABLE [dogs] (\n" @@ -732,7 +732,7 @@ def test_insert_binary_base64(db_path): ) assert 0 == result.exit_code, result.output db = Database(db_path) - actual = db.execute_returning_dicts("select content from files") + actual = list(db.query("select content from files")) assert actual == [{"content": b"hello"}] @@ -747,7 +747,7 @@ def test_insert_newline_delimited(db_path): assert [ {"foo": "bar", "n": 1}, {"foo": "baz", "n": 2}, - ] == db.execute_returning_dicts("select foo, n from from_json_nl") + ] == list(db.query("select foo, n from from_json_nl")) def test_insert_ignore(db_path, tmpdir): @@ -766,9 +766,7 @@ def test_insert_ignore(db_path, tmpdir): ) assert 0 == result.exit_code, result.output # ... but it should actually have no effect - assert [{"id": 1, "name": "Cleo"}] == db.execute_returning_dicts( - "select * from dogs" - ) + assert [{"id": 1, "name": "Cleo"}] == list(db.query("select * from dogs")) @pytest.mark.parametrize( @@ -831,8 +829,9 @@ def test_insert_replace(db_path, tmpdir): ) assert 0 == result.exit_code, result.output assert 21 == db["dogs"].count - assert insert_replace_dogs == db.execute_returning_dicts( - "select * from dogs where id in (1, 2, 21) order by id" + assert ( + list(db.query("select * from dogs where id in (1, 2, 21) order by id")) + == insert_replace_dogs ) @@ -847,7 +846,7 @@ def test_insert_truncate(db_path): assert [ {"foo": "bar", "n": 1}, {"foo": "baz", "n": 2}, - ] == db.execute_returning_dicts("select foo, n from from_json_nl") + ] == list(db.query("select foo, n from from_json_nl")) # Truncate and insert new rows result = CliRunner().invoke( cli.cli, @@ -866,7 +865,7 @@ def test_insert_truncate(db_path): assert [ {"foo": "bam", "n": 3}, {"foo": "bat", "n": 4}, - ] == db.execute_returning_dicts("select foo, n from from_json_nl") + ] == list(db.query("select foo, n from from_json_nl")) def test_insert_alter(db_path, tmpdir): @@ -897,7 +896,7 @@ def test_insert_alter(db_path, tmpdir): {"foo": "bar", "n": 1, "baz": None}, {"foo": "baz", "n": 2, "baz": None}, {"foo": "bar", "baz": 5, "n": None}, - ] == db.execute_returning_dicts("select foo, n, baz from from_json_nl") + ] == list(db.query("select foo, n, baz from from_json_nl")) @pytest.mark.parametrize( @@ -1168,7 +1167,7 @@ def test_upsert(db_path, tmpdir): assert [ {"id": 1, "name": "Cleo", "age": 4}, {"id": 2, "name": "Nixie", "age": 4}, - ] == db.execute_returning_dicts("select * from dogs order by id") + ] == list(db.query("select * from dogs order by id")) def test_upsert_alter(db_path, tmpdir): @@ -1195,7 +1194,7 @@ def test_upsert_alter(db_path, tmpdir): assert 0 == result.exit_code assert [ {"id": 1, "name": "Cleo", "age": 5}, - ] == db.execute_returning_dicts("select * from dogs order by id") + ] == list(db.query("select * from dogs order by id")) @pytest.mark.parametrize( @@ -1549,7 +1548,7 @@ def test_query_update(db_path, args, expected): cli.cli, [db_path, "update dogs set age = 5 where name = 'Cleo'"] + args ) assert expected == result.output.strip() - assert db.execute_returning_dicts("select * from dogs") == [ + assert list(db.query("select * from dogs")) == [ {"id": 1, "age": 5, "name": "Cleo"}, ] diff --git a/tests/test_create.py b/tests/test_create.py index 926a7d1..2cf0b4d 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -827,8 +827,8 @@ def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db + [{"i": 101, "extra": "Should trigger ALTER"}], alter=True, ) - rows = fresh_db.execute_returning_dicts("select * from test where i = 101") - assert [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] == rows + rows = list(fresh_db.query("select * from test where i = 101")) + assert rows == [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] def test_insert_ignore(fresh_db): @@ -839,8 +839,8 @@ def test_insert_ignore(fresh_db): # Using ignore=True should cause our insert to be silently ignored fresh_db["test"].insert({"id": 1, "bar": 3}, pk="id", ignore=True) # Only one row, and it should be bar=2, not bar=3 - rows = fresh_db.execute_returning_dicts("select * from test") - assert [{"id": 1, "bar": 2}] == rows + rows = list(fresh_db.query("select * from test")) + assert rows == [{"id": 1, "bar": 2}] def test_insert_hash_id(fresh_db): From e5d7a2ba3d585303c8e1c861a09e8761ba63678f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 10:43:49 -0700 Subject: [PATCH 0382/1004] Tests for db.query() method, refs #290 --- tests/test_query.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/test_query.py diff --git a/tests/test_query.py b/tests/test_query.py new file mode 100644 index 0000000..fe79cc0 --- /dev/null +++ b/tests/test_query.py @@ -0,0 +1,17 @@ +import types + + +def test_query(fresh_db): + fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) + results = fresh_db.query("select * from dogs order by name desc") + assert isinstance(results, types.GeneratorType) + assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}] + + +def test_execute_returning_dicts(fresh_db): + # Like db.query() but returns a list, included for backwards compatibility + # see https://github.com/simonw/sqlite-utils/issues/290 + fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") + assert fresh_db.execute_returning_dicts("select * from test") == [ + {"id": 1, "bar": 2} + ] From 3805d1c9731d5355797271bfb292a1a91758db01 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 10:44:12 -0700 Subject: [PATCH 0383/1004] Removed duplicate vacuum() function, thanks mypy --- sqlite_utils/cli.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8043c98..0a0c0fe 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -260,17 +260,6 @@ def views( ) -@cli.command() -@click.argument( - "path", - type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), - required=True, -) -def vacuum(path): - """Run VACUUM against the database""" - sqlite_utils.Database(path).vacuum() - - @cli.command() @click.argument( "path", From 7b3fdf0fcd553ddf25b8d606b7fc34f9fd7979df Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 11:04:32 -0700 Subject: [PATCH 0384/1004] mypy annotations for rows_from_file(), run mypy in CI Refs #289, #279 --- .github/workflows/test.yml | 3 +++ sqlite_utils/cli.py | 2 +- sqlite_utils/db.py | 8 ++++---- sqlite_utils/utils.py | 31 ++++++++++++++++++------------- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f8a79ae..012db32 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,11 +26,14 @@ jobs: - name: Install dependencies run: | pip install -e '.[test]' + pip install mypy - name: Optionally install numpy if: matrix.numpy == 1 run: pip install numpy - name: Run tests run: | pytest + - name: run mypy + run: mypy sqlite_utils - name: Check formatting run: black . --check diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0a0c0fe..2de6bf3 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,6 +1,6 @@ import base64 import click -from click_default_group import DefaultGroup +from click_default_group import DefaultGroup # type: ignore from datetime import datetime import hashlib import pathlib diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1a88dba..acd1726 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -11,7 +11,7 @@ import json import os import pathlib import re -from sqlite_fts4 import rank_bm25 +from sqlite_fts4 import rank_bm25 # type: ignore import sys import textwrap import uuid @@ -39,14 +39,14 @@ USING\s+(?P\w+) # e.g. USING FTS5 ) try: - import pandas as pd + import pandas as pd # type: ignore except ImportError: pd = None try: - import numpy as np + import numpy as np # type: ignore except ImportError: - np = None + np = None # type: ignore Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 4dd5f21..4f3c819 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -5,17 +5,18 @@ import enum import io import json import os -from typing import Generator +from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type import click try: - import pysqlite3 as sqlite3 - import pysqlite3.dbapi2 + import pysqlite3 as sqlite3 # type: ignore + import pysqlite3.dbapi2 # type: ignore OperationalError = pysqlite3.dbapi2.OperationalError except ImportError: - import sqlite3 + # https://github.com/python/mypy/issues/1153#issuecomment-253842414 + import sqlite3 # type: ignore OperationalError = sqlite3.OperationalError @@ -143,12 +144,11 @@ class RowsFromFileBadJSON(RowsFromFileError): def rows_from_file( - fp, - format=None, - dialect=None, - encoding=None, - detect_types=False, -) -> Generator[dict, None, None]: + fp: BinaryIO, + format: Optional[Format] = None, + dialect: Optional[Type[csv.Dialect]] = None, + encoding: Optional[str] = None, +) -> Tuple[Iterable[dict], Format]: if format == Format.JSON: decoded = json.load(fp) if isinstance(decoded, dict): @@ -159,8 +159,13 @@ def rows_from_file( elif format == Format.NL: return (json.loads(line) for line in fp if line.strip()), Format.NL elif format == Format.CSV: - decoded_fp = io.TextIOWrapper(fp, encoding=encoding or "utf-8-sig") - return csv.DictReader(decoded_fp, dialect=dialect), Format.CSV + use_encoding: str = encoding or "utf-8-sig" + decoded_fp = io.TextIOWrapper(fp, encoding=use_encoding) + if dialect is not None: + reader = csv.DictReader(decoded_fp, dialect=dialect) + else: + reader = csv.DictReader(decoded_fp) + return reader, Format.CSV elif format == Format.TSV: return ( rows_from_file( @@ -170,7 +175,7 @@ def rows_from_file( ) elif format is None: # Detect the format, then call this recursively - buffered = io.BufferedReader(fp, buffer_size=4096) + buffered = io.BufferedReader(cast(io.RawIOBase, fp), buffer_size=4096) first_bytes = buffered.peek(2048).strip() if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"): # TODO: Detect newline-JSON From 8d1d8013899e110c03c50c1f66a7b9c0b51f8383 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 11:08:21 -0700 Subject: [PATCH 0385/1004] mypy tweaks, refs #289, #266, #37 --- .github/workflows/test.yml | 3 +-- setup.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 012db32..2e19e61 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,8 +25,7 @@ jobs: ${{ runner.os }}-pip- - name: Install dependencies run: | - pip install -e '.[test]' - pip install mypy + pip install -e '.[test,mypy]' - name: Optionally install numpy if: matrix.numpy == 1 run: pip install numpy diff --git a/setup.py b/setup.py index 626409e..a356f30 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ setup( extras_require={ "test": ["pytest", "black", "hypothesis"], "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], + "mypy": ["mypy", "types-click", "types-tabulate"], }, entry_points=""" [console_scripts] From 90e211e3e2f36d2ff911ecf1afe4470ff45c7c0d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 18:22:08 -0700 Subject: [PATCH 0386/1004] Now complies with flake8, refs #291 --- setup.cfg | 3 + sqlite_utils/db.py | 6 +- tests/test_analyze_tables.py | 5 +- tests/test_cli.py | 181 +++++++++++++++++++++++++++----- tests/test_cli_memory.py | 4 +- tests/test_constructor.py | 1 - tests/test_conversions.py | 3 - tests/test_create.py | 9 +- tests/test_enable_counts.py | 27 ++++- tests/test_extract.py | 2 +- tests/test_extracts.py | 2 +- tests/test_hypothesis.py | 1 + tests/test_introspect.py | 20 +++- tests/test_recreate.py | 2 +- tests/test_register_function.py | 4 +- tests/test_rows.py | 1 - tests/test_tracer.py | 35 +++++- tests/test_transform.py | 10 +- tests/test_upsert.py | 2 +- tests/test_wal.py | 1 - 20 files changed, 252 insertions(+), 67 deletions(-) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..7a88d6f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 160 +extend-ignore = E203 # for Black diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index acd1726..50fd496 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,5 +1,5 @@ from .utils import sqlite3, OperationalError, suggest_column_types, column_affinity -from collections import namedtuple, OrderedDict +from collections import namedtuple from collections.abc import Mapping import contextlib import datetime @@ -1116,8 +1116,6 @@ class Table(Queryable): ) ) table = table or "_".join(columns) - first_column = columns[0] - pks = self.pks lookup_table = self.db[table] fk_column = fk_column or "{}_id".format(table) magic_lookup_column = "{}_{}".format(fk_column, os.urandom(6).hex()) @@ -1236,7 +1234,7 @@ class Table(Queryable): fk_col_type = None if fk is not None: # fk must be a valid table - if not fk in self.db.table_names(): + if fk not in self.db.table_names(): raise AlterError("table '{}' does not exist".format(fk)) # if fk_col specified, must be a valid column if fk_col is not None: diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index f0af2ca..5795a7a 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -1,10 +1,8 @@ -from sqlite_utils.db import Database, ForeignKey, ColumnDetails +from sqlite_utils.db import Database, ColumnDetails from sqlite_utils import cli -from sqlite_utils.utils import OperationalError from click.testing import CliRunner import pytest import sqlite3 -import textwrap @pytest.fixture @@ -132,6 +130,7 @@ def test_analyze_table_save(db_to_analyze_path): result = CliRunner().invoke( cli.cli, ["analyze-tables", db_to_analyze_path, "--save"] ) + assert result.exit_code == 0 rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows) assert rows == [ { diff --git a/tests/test_cli.py b/tests/test_cli.py index 5028458..37a96a3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -410,7 +410,7 @@ def test_index_foreign_keys(db_path): def test_enable_fts(db_path): db = Database(db_path) - assert None == db["Gosh"].detect_fts() + assert db["Gosh"].detect_fts() is None result = CliRunner().invoke( cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] ) @@ -420,7 +420,7 @@ def test_enable_fts(db_path): # Table names with restricted chars are handled correctly. # colons and dots are restricted characters for table names. db["http://example.com"].create({"c1": str, "c2": str, "c3": str}) - assert None == db["http://example.com"].detect_fts() + assert db["http://example.com"].detect_fts() is None result = CliRunner().invoke( cli.cli, [ @@ -966,7 +966,23 @@ def test_query_json(db_path, sql, args, expected): assert expected == result.output.strip() -LOREM_IPSUM_COMPRESSED = b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8ef\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J\xb9\x97>i\xa9\x11W\xb13\xa5\xde\x96$\x13\xf3I\x9cu\xe8J\xda\xee$EcsI\x8e\x0b$\xea\xab\xf6L&u\xc4emI\xb3foFnT\xf83\xca\x93\xd8QZ\xa8\xf2\xbd1q\xd1\x87\xf3\x85>\x8c\xa4i\x8d\xdaTu\x7f\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\xfb\x8f\xef\x1b\x9b\x06\x83}" +LOREM_IPSUM_COMPRESSED = ( + b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8e" + b"f\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J" + b"\xb9\x97>i\xa9\x11W\xb13\xa5\xde\x96$\x13\xf3I\x9cu\xe8J\xda\xee$EcsI\x8e\x0b" + b"$\xea\xab\xf6L&u\xc4emI\xb3foFnT\xf83\xca\x93\xd8QZ\xa8\xf2\xbd1q\xd1\x87\xf3" + b"\x85>\x8c\xa4i\x8d\xdaTu\x7f\xf0\x81\x0f|\xe0\x03" + b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03" + b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03" + b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03" + b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\xfb\x8f\xef\x1b\x9b\x06\x83}" +) def test_query_json_binary(db_path): @@ -988,7 +1004,18 @@ def test_query_json_binary(db_path): "sz": 16984, "data": { "$base64": True, - "encoded": "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uIjnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3fiCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9", + "encoded": ( + ( + "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH" + "8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+" + "DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I" + "/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uI" + "jnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3f" + "iCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8" + "IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984A" + "Mf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9" + ) + ), }, } ] @@ -1157,17 +1184,17 @@ def test_upsert(db_path, tmpdir): {"id": 1, "age": 5}, {"id": 2, "age": 5}, ] - open(json_path, "w").write(json.dumps(insert_dogs)) + open(json_path, "w").write(json.dumps(upsert_dogs)) result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"], catch_exceptions=False, ) assert 0 == result.exit_code, result.output - assert [ - {"id": 1, "name": "Cleo", "age": 4}, - {"id": 2, "name": "Nixie", "age": 4}, - ] == list(db.query("select * from dogs order by id")) + assert list(db.query("select * from dogs order by id")) == [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Nixie", "age": 5}, + ] def test_upsert_alter(db_path, tmpdir): @@ -1596,47 +1623,112 @@ def test_add_foreign_keys(db_path): [ ( [], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--type", "age", "text"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] TEXT NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] TEXT NOT NULL DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--drop", "age"], - 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)', + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [name] TEXT\n" + ")" + ), ), ( ["--rename", "age", "age2", "--rename", "id", "pk"], - "CREATE TABLE \"dogs\" (\n [pk] INTEGER PRIMARY KEY,\n [age2] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [pk] INTEGER PRIMARY KEY,\n" + " [age2] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--not-null", "name"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT NOT NULL\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT NOT NULL\n" + ")" + ), ), ( ["--not-null-false", "age"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--pk", "name"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT PRIMARY KEY\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT PRIMARY KEY\n" + ")" + ), ), ( ["--pk-none"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--default", "name", "Turnip"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT DEFAULT 'Turnip'\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT DEFAULT 'Turnip'\n" + ")" + ), ), ( ["--default-none", "age"], - 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL,\n [name] TEXT\n)', + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER NOT NULL,\n" + " [name] TEXT\n" + ")" + ), ), ( ["-o", "name", "--column-order", "age", "-o", "id"], - "CREATE TABLE \"dogs\" (\n [name] TEXT,\n [age] INTEGER NOT NULL DEFAULT '1',\n [id] INTEGER PRIMARY KEY\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [name] TEXT,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [id] INTEGER PRIMARY KEY\n" + ")" + ), ), ], ) @@ -1685,9 +1777,13 @@ def test_transform_drop_foreign_key(db_path): print(result.output) assert result.exit_code == 0 schema = db["places"].schema - assert ( - schema - == 'CREATE TABLE "places" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [country] INTEGER,\n [city] INTEGER REFERENCES [city]([id])\n)' + assert schema == ( + 'CREATE TABLE "places" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [name] TEXT,\n" + " [country] INTEGER,\n" + " [city] INTEGER REFERENCES [city]([id])\n" + ")" ) @@ -1701,22 +1797,48 @@ _common_other_schema = ( [ ( [], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)', + ( + 'CREATE TABLE "trees" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [address] TEXT,\n" + " [species_id] INTEGER,\n" + " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n" + ")" + ), _common_other_schema, ), ( ["--table", "custom_table"], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_table_id] INTEGER,\n FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n)', + ( + 'CREATE TABLE "trees" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [address] TEXT,\n" + " [custom_table_id] INTEGER,\n" + " FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n" + ")" + ), "CREATE TABLE [custom_table] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)", ), ( ["--fk-column", "custom_fk"], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_fk] INTEGER,\n FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n)', + ( + 'CREATE TABLE "trees" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [address] TEXT,\n" + " [custom_fk] INTEGER,\n" + " FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n" + ")" + ), _common_other_schema, ), ( ["--rename", "name", "name2"], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)', + 'CREATE TABLE "trees" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [address] TEXT,\n" + " [species_id] INTEGER,\n" + " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n" + ")", "CREATE TABLE [species] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)", ), ], @@ -1902,7 +2024,10 @@ def test_indexes(tmpdir): ] -_TRIGGERS_EXPECTED = '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah AFTER INSERT ON articles\\nBEGIN\\n UPDATE counter SET count = count + 1;\\nEND"}]\n' +_TRIGGERS_EXPECTED = ( + '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah ' + 'AFTER INSERT ON articles\\nBEGIN\\n UPDATE counter SET count = count + 1;\\nEND"}]\n' +) @pytest.mark.parametrize( diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index e42e1cb..e465927 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -52,7 +52,7 @@ def test_memory_tsv(tmpdir, use_stdin): result = CliRunner().invoke( cli.cli, ["memory", path, "select * from {}".format(sql_from)], - input=data, + input=input, ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ @@ -102,7 +102,7 @@ def test_memory_json_nl(tmpdir, use_stdin): result = CliRunner().invoke( cli.cli, ["memory", path, "select * from {}".format(sql_from)], - input=data, + input=input, ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ diff --git a/tests/test_constructor.py b/tests/test_constructor.py index b3cd963..924df66 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -1,5 +1,4 @@ from sqlite_utils import Database -import pytest def test_recursive_triggers(): diff --git a/tests/test_conversions.py b/tests/test_conversions.py index ebe2a50..d70f5c8 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -1,6 +1,3 @@ -import pytest - - def test_insert_conversion(fresh_db): table = fresh_db["table"] table.insert({"foo": "bar"}, conversions={"foo": "upper(?)"}) diff --git a/tests/test_create.py b/tests/test_create.py index 2cf0b4d..ff36f90 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -2,7 +2,6 @@ from sqlite_utils.db import ( Index, Database, DescIndex, - ForeignKey, AlterError, NoObviousTable, ForeignKey, @@ -148,8 +147,8 @@ def test_create_table_with_not_null(fresh_db): ) def test_create_table_from_example(fresh_db, example, expected_columns): people_table = fresh_db["people"] - assert None == people_table.last_rowid - assert None == people_table.last_pk + assert people_table.last_rowid is None + assert people_table.last_pk is None people_table.insert(example) assert 1 == people_table.last_rowid assert 1 == people_table.last_pk @@ -515,7 +514,7 @@ def test_insert_row_alter_table( def test_insert_row_alter_table_invalid_column_characters(fresh_db): table = fresh_db["table"] - rowid = table.insert({"foo": "bar"}).last_pk + table.insert({"foo": "bar"}).last_pk with pytest.raises(AssertionError): table.insert({"foo": "baz", "new_col[abc]": 1.2}, alter=True) @@ -870,8 +869,6 @@ def test_works_with_pathlib_path(tmpdir): @pytest.mark.skipif(pd is None, reason="pandas and numpy are not installed") def test_create_table_numpy(fresh_db): - import numpy as np - df = pd.DataFrame({"col 1": range(3), "col 2": range(3)}) fresh_db["pandas"].insert_all(df.to_dict(orient="records")) assert [ diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index b70378e..7a52108 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -14,8 +14,31 @@ def test_enable_counts_specific_table(fresh_db): # Now enable counts foo.enable_counts() assert foo.triggers_dict == { - "foo_counts_insert": "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\nBEGIN\n INSERT OR REPLACE INTO [_counts]\n VALUES (\n 'foo',\n COALESCE(\n (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n 0\n ) + 1\n );\nEND", - "foo_counts_delete": "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\nBEGIN\n INSERT OR REPLACE INTO [_counts]\n VALUES (\n 'foo',\n COALESCE(\n (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n 0\n ) - 1\n );\nEND", + "foo_counts_insert": ( + "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\n" + "BEGIN\n" + " INSERT OR REPLACE INTO [_counts]\n" + " VALUES (\n 'foo',\n" + " COALESCE(\n" + " (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n" + " 0\n" + " ) + 1\n" + " );\n" + "END" + ), + "foo_counts_delete": ( + "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\n" + "BEGIN\n" + " INSERT OR REPLACE INTO [_counts]\n" + " VALUES (\n" + " 'foo',\n" + " COALESCE(\n" + " (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n" + " 0\n" + " ) - 1\n" + " );\n" + "END" + ), } assert fresh_db.table_names() == ["foo", "_counts"] assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}] diff --git a/tests/test_extract.py b/tests/test_extract.py index 280c59c..10b5b09 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import Index, InvalidColumns +from sqlite_utils.db import InvalidColumns import itertools import pytest diff --git a/tests/test_extracts.py b/tests/test_extracts.py index 0edd002..cca16ba 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import Index, ForeignKey +from sqlite_utils.db import Index import pytest diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 5759c5e..f12f865 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -2,6 +2,7 @@ from hypothesis import given import hypothesis.strategies as st import sqlite_utils + # SQLite integers are -(2^63) to 2^63 - 1 @given(st.integers(-9223372036854775808, 9223372036854775807)) def test_roundtrip_integers(integer): diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 6ca578c..cc33c46 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -33,7 +33,7 @@ def test_detect_fts(existing_db): assert "woo_fts" == existing_db["woo_fts"].detect_fts() assert "woo2_fts" == existing_db["woo2"].detect_fts() assert "woo2_fts" == existing_db["woo2_fts"].detect_fts() - assert None == existing_db["foo"].detect_fts() + assert existing_db["foo"].detect_fts() is None def test_tables(existing_db): @@ -175,9 +175,21 @@ def test_triggers_and_triggers_dict(fresh_db): (t.name, t.table) for t in fresh_db["authors"].triggers } expected_triggers = { - "authors_ai": "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND", - "authors_ad": "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\nEND", - "authors_au": "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND", + "authors_ai": ( + "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n" + " INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\n" + "END" + ), + "authors_ad": ( + "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n" + " INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n" + "END" + ), + "authors_au": ( + "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n" + " INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n" + " INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND" + ), } assert authors.triggers_dict == expected_triggers assert fresh_db["other"].triggers == [] diff --git a/tests/test_recreate.py b/tests/test_recreate.py index ddef115..504a0b8 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -15,7 +15,7 @@ def test_recreate_ignored_for_in_memory(): def test_recreate_not_allowed_for_connection(): conn = sqlite3.connect(":memory:") with pytest.raises(AssertionError): - db = Database(conn, recreate=True) + Database(conn, recreate=True) @pytest.mark.parametrize( diff --git a/tests/test_register_function.py b/tests/test_register_function.py index e6d977a..c99477e 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -55,14 +55,14 @@ def test_register_function_replace(fresh_db): # This will fail to replace the function: @fresh_db.register_function() - def one(): + def one(): # noqa: F811 return "two" assert "one" == fresh_db.execute("select one()").fetchone()[0] # This will replace it @fresh_db.register_function(replace=True) - def one(): + def one(): # noqa: F811 return "two" assert "two" == fresh_db.execute("select one()").fetchone()[0] diff --git a/tests/test_rows.py b/tests/test_rows.py index 3ac52f9..a8a4ca0 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -1,4 +1,3 @@ -from sqlite_utils.db import Index, View import pytest diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 094551a..d3ff22d 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -1,4 +1,3 @@ -import pytest from sqlite_utils import Database @@ -32,7 +31,9 @@ def test_tracer(): def test_with_tracer(): collected = [] - tracer = lambda sql, params: collected.append((sql, params)) + + def tracer(sql, params): + return collected.append((sql, params)) db = Database(memory=True) @@ -48,13 +49,39 @@ def test_with_tracer(): assert collected == [ ("select name from sqlite_master where type = 'view'", None), ( - "SELECT name FROM sqlite_master\n WHERE rootpage = 0\n AND (\n sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n OR (\n tbl_name = \"dogs\"\n AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n )\n )", + ( + "SELECT name FROM sqlite_master\n" + " WHERE rootpage = 0\n" + " AND (\n" + " sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n" + " OR (\n" + ' tbl_name = "dogs"\n' + " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" + " )\n" + " )" + ), None, ), ("select name from sqlite_master where type = 'view'", None), ("select sql from sqlite_master where name = ?", ("dogs_fts",)), ( - "with original as (\n select\n rowid,\n *\n from [dogs]\n)\nselect\n [original].*\nfrom\n [original]\n join [dogs_fts] on [original].rowid = [dogs_fts].rowid\nwhere\n [dogs_fts] match :query\norder by\n [dogs_fts].rank", + ( + "with original as (\n" + " select\n" + " rowid,\n" + " *\n" + " from [dogs]\n" + ")\n" + "select\n" + " [original].*\n" + "from\n" + " [original]\n" + " join [dogs_fts] on [original].rowid = [dogs_fts].rowid\n" + "where\n" + " [dogs_fts] match :query\n" + "order by\n" + " [dogs_fts].rank" + ), {"query": "Cleopaws"}, ), ] diff --git a/tests/test_transform.py b/tests/test_transform.py index 19e447e..06e5729 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -93,7 +93,10 @@ def test_transform_sql_table_with_primary_key( fresh_db, params, expected_sql, use_pragma_foreign_keys ): captured = [] - tracer = lambda sql, params: captured.append((sql, params)) + + def tracer(sql, params): + return captured.append((sql, params)) + dogs = fresh_db["dogs"] if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") @@ -163,7 +166,10 @@ def test_transform_sql_table_with_no_primary_key( fresh_db, params, expected_sql, use_pragma_foreign_keys ): captured = [] - tracer = lambda sql, params: captured.append((sql, params)) + + def tracer(sql, params): + return captured.append((sql, params)) + dogs = fresh_db["dogs"] if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 9b1990e..09bdacc 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -54,7 +54,7 @@ def test_upsert_compound_primary_key(fresh_db): ], pk=("species", "id"), ) - assert None == table.last_pk + assert table.last_pk is None table.upsert({"species": "dog", "id": 1, "age": 5}, pk=("species", "id")) assert ("dog", 1) == table.last_pk assert [ diff --git a/tests/test_wal.py b/tests/test_wal.py index 1303eed..23ca144 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -1,6 +1,5 @@ import pytest from sqlite_utils import Database -import sqlite3 @pytest.fixture From 02898bf7af4a4e484ecc8ec852d5fee98463277b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 18:23:12 -0700 Subject: [PATCH 0387/1004] Run flake8 in CI, refs #291 --- .github/workflows/test.yml | 4 +++- setup.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e19e61..5dc3fe1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: ${{ runner.os }}-pip- - name: Install dependencies run: | - pip install -e '.[test,mypy]' + pip install -e '.[test,mypy,flake8]' - name: Optionally install numpy if: matrix.numpy == 1 run: pip install numpy @@ -34,5 +34,7 @@ jobs: pytest - name: run mypy run: mypy sqlite_utils + - name: run flake8 + run: flake8 - name: Check formatting run: black . --check diff --git a/setup.py b/setup.py index a356f30..f5421ec 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ setup( "test": ["pytest", "black", "hypothesis"], "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], "mypy": ["mypy", "types-click", "types-tabulate"], + "flake8": ["flake8"], }, entry_points=""" [console_scripts] From 1fba60537dcac8be664de0e3ba2c66143cc996bd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 19:06:18 -0700 Subject: [PATCH 0388/1004] Try more aggressive noqa, refs #291 --- tests/test_register_function.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_register_function.py b/tests/test_register_function.py index c99477e..06f4857 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -55,14 +55,14 @@ def test_register_function_replace(fresh_db): # This will fail to replace the function: @fresh_db.register_function() - def one(): # noqa: F811 + def one(): # noqa return "two" assert "one" == fresh_db.execute("select one()").fetchone()[0] # This will replace it @fresh_db.register_function(replace=True) - def one(): # noqa: F811 + def one(): # noqa return "two" assert "two" == fresh_db.execute("select one()").fetchone()[0] From 93c7fd9868fed3193a1732b39bfac539e5812b0b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 19:08:52 -0700 Subject: [PATCH 0389/1004] Ignore this entire file, refs #291 --- tests/test_register_function.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_register_function.py b/tests/test_register_function.py index 06f4857..19af0b6 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -1,3 +1,4 @@ +# flake8: noqa import pytest import sys from unittest.mock import MagicMock From 33c9d0087983a99ba18cbe87fe92ea722caa499c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Jun 2021 15:54:38 -0700 Subject: [PATCH 0390/1004] Documentation for db.query(), closes #290 --- docs/python-api.rst | 77 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 85f0296..7f3cdd5 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1,8 +1,8 @@ .. _python_api: -============ - Python API -============ +============================= + sqlite_utils Python library +============================= .. contents:: :local: @@ -59,12 +59,11 @@ You can attach an additional database using the ``.attach()`` method, providing db = Database("first.db") db.attach("second", "second.db") # Now you can run queries like this one: - cursor = db.execute(""" + print(db.query(""" select * from table_in_first union all select * from second.table_in_second - """) - print(cursor.fetchall()) + """)) You can reference tables in the attached database using the alias value you passed to ``db.attach(alias, filepath)`` as a prefix, for example the ``second.table_in_second`` reference in the SQL query above. @@ -97,27 +96,77 @@ You can also turn on a tracer function temporarily for a block of code using the This example will print queries only for the duration of the ``with`` block. -.. _python_api_execute: +.. _python_api_executing_queries: Executing queries ================= -The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around ``.execute()`` and ``.executescript()`` on the underlying SQLite connection. These wrappers log to the tracer function if one has been registered. +The ``Database`` class offers several methods for directly executing SQL queries. + +.. _python_api_query: + +db.query(sql, params) +--------------------- + +The ``db.query(sql)`` function executes a SQL query and returns an iterator over Python dictionaries representing the resulting rows: + +.. code-block:: python + + db = Database(memory=True) + db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) + for row in db.query("select * from dogs"): + print(row) + # Outputs: + # {'name': 'Cleo'} + # {'name': 'Pancakes'} + +.. _python_api_execute: + +db.execute(sql, params) +----------------------- + +The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around ``.execute()`` and ``.executescript()`` on the underlying SQLite connection. These wrappers log to the :ref:`tracer function ` if one has been registered. + +``db.execute(sql)`` returns a `sqlite3.Cursor `__ that was used to execute the SQL. .. code-block:: python db = Database(memory=True) db["dogs"].insert({"name": "Cleo"}) - db.execute("update dogs set name = 'Cleopaws'") + cursor = db.execute("update dogs set name = 'Cleopaws'") + print(cursor.rowcount) + # Outputs the number of rows affected by the update + # In this case 2 -You can pass parameters as an optional second argument, using either a list or a dictionary. These will be correctly quoted and escaped. +Other cursor methods such as ``.fetchone()`` and ``.fetchall()`` are also available, see the `standard library documentation `__. + +.. _python_api_parameters: + +Passing parameters +------------------ + +Both ``db.query()`` and ``db.execute()`` accept an optional second argument for parameters to be passed to the SQL query. + +This can take the form of either a tuple/list or a dictionary, depending on the type of parameters used in the query. Values passed in this way will be correctly quoted and escaped, helping avoid XSS vulnerabilities. + +``?`` parameters in the SQL query can be filled in using a list: .. code-block:: python - # Using ? and a list: db.execute("update dogs set name = ?", ["Cleopaws"]) - # Or using :name and a dictionary: - db.execute("update dogs set name = :name", {"name": "Cleopaws"}) + # This will rename ALL dogs to be called "Cleopaws" + +Named parameters using ``:name`` can be filled using a dictionary: + +.. code-block:: python + + dog = next(db.query( + "select rowid, name from dogs where name = :name", + {"name": "Cleopaws"} + )) + # dog is now {'rowid': 1, 'name': 'Cleopaws'} + +In this example ``next()`` is used to retrieve the first result in the iterator returned by the ``db.query()`` method. .. _python_api_table: @@ -2222,7 +2271,7 @@ If you want to deliberately replace the registered function with a new implement Quoting strings for use in SQL ============================== -In almost all cases you should pass values to your SQL queries using the optional ``parameters`` argument to ``db.execute()``, as described in :ref:`python_api_execute`. +In almost all cases you should pass values to your SQL queries using the optional ``parameters`` argument to ``db.query()``, as described in :ref:`python_api_parameters`. If that option isn't relevant to your use-case you can to quote a string for use with SQLite using the ``db.quote()`` method, like so: From 747be6057d09a4e5d9d726e29d5cf99b10c59dea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Jun 2021 16:06:12 -0700 Subject: [PATCH 0391/1004] Added some more types, refs #266, #290 --- sqlite_utils/db.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 50fd496..ae87e27 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -14,6 +14,7 @@ import re from sqlite_fts4 import rank_bm25 # type: ignore import sys import textwrap +from typing import Generator, Iterable, Union, Optional, List import uuid SQLITE_MAX_VARS = 999 @@ -359,13 +360,17 @@ class Database: for table in tables ) - def query(self, sql, params=None): + def query( + self, sql: str, params: Optional[Union[Iterable, dict]] = None + ) -> Generator[dict, None, None]: cursor = self.execute(sql, params or tuple()) keys = [d[0] for d in cursor.description] for row in cursor: yield dict(zip(keys, row)) - def execute_returning_dicts(self, sql, params=None): + def execute_returning_dicts( + self, sql: str, params: Optional[Union[Iterable, dict]] = None + ) -> List[dict]: return list(self.query(sql, params)) def resolve_foreign_keys(self, name, foreign_keys): From fec6cd55cab7ee91046ca4ee278b90cd045a32c2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 25 Jun 2021 10:53:46 -0700 Subject: [PATCH 0392/1004] Contributing documentation, closes #292 --- docs/contributing.rst | 67 +++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 68 insertions(+) create mode 100644 docs/contributing.rst diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..ec1a4a7 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1,67 @@ +.. _contributing: + +============== + Contributing +============== + +To work on this library locally, first checkout the code. Then create a new virtual environment:: + + git clone git@github.com:simonw/sqlite-utils + cd sqlite-utils + python3 -mvenv venv + source venv/bin/activate + +Or if you are using ``pipenv``:: + + pipenv shell + +Within the virtual environment running ``sqlite-utils`` should run your locally editable version of the tool. You can use ``which sqlite-utils`` to confirm that you are running the version that lives in your virtual environment. + +.. _contributing_tests: + +Running the tests +================= + +To install the dependencies and test dependencies:: + + pip install -e '.[test]' + +To run the tests:: + + pytest + +.. _contributing_docs: + +Building the documentation +========================== + +To build the documentation, first install the documentation dependencies:: + + pip install -e '.[docs]' + +Then run ``make livehtml`` from the ``docs/`` directory to start a server on port 8000 that will serve the documentation and live-reload any time you make an edit to a ``.rst`` file:: + + cd docs + make livehtml + +.. _contributing_linting: + +Linting and formatting +====================== + +``sqlite-utils`` uses `Black `__ for code formatting, and `flake8 `__ and `mypy `__ for linting and type checking. + +Black is installed as part of ``pip install -e '.[test]'`` - you can then format your code by running it in the root of the project:: + + black . + +To install ``mypy`` and ``flake8`` run the following:: + + pip install -e '.[flake8,mypy]' + +Both commands can then be run in the root of the project like this:: + + flake8 + mypy sqlite_utils + +All three of these tools are run by our CI mechanism against every commit and pull request. diff --git a/docs/index.rst b/docs/index.rst index b1bfc0e..93b0bc0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,7 @@ Contents installation cli python-api + contributing changelog Take a look at `this script `_ for an example of this library in action. From 8981b9c1f3e4e3865924861ee63922f696078f6c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 25 Jun 2021 10:59:05 -0700 Subject: [PATCH 0393/1004] Release 3.12 Refs #290, #291, #292 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0c31e79..57bff4a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v3_12: + +3.12 (2021-06-25) +----------------- + +- New :ref:`db.query(sql, params) ` method, which executes a SQL query and returns the results as an iterator over Python dictionaries. (`#290 `__) +- This project now uses ``flake8`` and has started to use ``mypy``. (`#291 `__) +- New documentation on :ref:`contributing ` to this project. (`#292 `__) + .. _v3_11: 3.11 (2021-06-20) diff --git a/setup.py b/setup.py index f5421ec..02224b3 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.11" +VERSION = "3.12" def get_long_description(): From 8286a66413bc466db11b3b7e0e75826efbd7850e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Jun 2021 09:35:01 -0700 Subject: [PATCH 0394/1004] sqlite-utils memory --help now mentions --schema --- sqlite_utils/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 2de6bf3..8a4910f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1212,6 +1212,10 @@ def memory( cat animals.csv | sqlite-utils memory stdin:csv places.dat:nl \\ "select * from stdin where place_id in (select id from places)" + Use --schema to view the SQL schema of any imported files: + + \b + sqlite-utils memory animals.csv --schema """ db = sqlite_utils.Database(memory=True) # If --dump or --save used but no paths detected, assume SQL query is a path: From ab8d4aad0c42f905640981f6f24bc1e37205ae62 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 24 Jul 2021 15:08:36 -0700 Subject: [PATCH 0395/1004] sqlite-utils schema now takes optional tables, closes #299 --- docs/cli.rst | 5 ++++ sqlite_utils/cli.py | 10 ++++++-- tests/test_cli.py | 57 +++++++++++++++++++++++++++++++++------------ 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index a404333..d8993e4 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -525,6 +525,11 @@ The ``sqlite-utils schema`` command shows the full SQL schema for the database:: [name] TEXT ); +This will show the schema for every table and index in the database. To view the schema just for a specified subset of tables pass those as additional arguments:: + + $ sqlite-utils schema dogs.db dogs chickens + ... + .. _cli_analyze_tables: Analyzing tables diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8a4910f..b168ec0 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1550,15 +1550,21 @@ def indexes( type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), required=True, ) +@click.argument("tables", nargs=-1, required=False) @load_extension_option def schema( path, + tables, load_extension, ): - "Show full schema for this database" + "Show full schema for this database or for specified tables" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - click.echo(db.schema) + if tables: + for table in tables: + click.echo(db[table].schema) + else: + click.echo(db.schema) @cli.command() diff --git a/tests/test_cli.py b/tests/test_cli.py index 37a96a3..e84453a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2064,7 +2064,46 @@ def test_triggers(tmpdir, extra_args, expected): assert result.output == expected -def test_schema(tmpdir): +@pytest.mark.parametrize( + "options,expected", + ( + ( + [], + ( + "CREATE TABLE [dogs] (\n" + " [id] INTEGER,\n" + " [name] TEXT\n" + ");\n" + "CREATE TABLE [chickens] (\n" + " [id] INTEGER,\n" + " [name] TEXT,\n" + " [breed] TEXT\n" + ");\n" + "CREATE INDEX [idx_chickens_breed]\n" + " ON [chickens] ([breed]);\n" + ), + ), + ( + ["dogs"], + ("CREATE TABLE [dogs] (\n" " [id] INTEGER,\n" " [name] TEXT\n" ")\n"), + ), + ( + ["chickens", "dogs"], + ( + "CREATE TABLE [chickens] (\n" + " [id] INTEGER,\n" + " [name] TEXT,\n" + " [breed] TEXT\n" + ")\n" + "CREATE TABLE [dogs] (\n" + " [id] INTEGER,\n" + " [name] TEXT\n" + ")\n" + ), + ), + ), +) +def test_schema(tmpdir, options, expected): db_path = str(tmpdir / "test.db") db = Database(db_path) db["dogs"].create({"id": int, "name": str}) @@ -2072,23 +2111,11 @@ def test_schema(tmpdir): db["chickens"].create_index(["breed"]) result = CliRunner().invoke( cli.cli, - ["schema", db_path], + ["schema", db_path] + options, catch_exceptions=False, ) assert result.exit_code == 0 - assert result.output == ( - "CREATE TABLE [dogs] (\n" - " [id] INTEGER,\n" - " [name] TEXT\n" - ");\n" - "CREATE TABLE [chickens] (\n" - " [id] INTEGER,\n" - " [name] TEXT,\n" - " [breed] TEXT\n" - ");\n" - "CREATE INDEX [idx_chickens_breed]\n" - " ON [chickens] ([breed]);\n" - ) + assert result.output == expected def test_long_csv_column_value(tmpdir): From c7e8d72be9fe8fe0811f685a18eebc637662d41b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 24 Jul 2021 15:15:27 -0700 Subject: [PATCH 0396/1004] Release 3.13 Refs #299 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 57bff4a..9d17c1e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v3_13: + +3.13 (2021-07-24) +----------------- + +- ``sqlite-utils schema my.db table1 table2`` command now accepts optional table names. (`#299 `__) +- ``sqlite-utils memory --help`` now describes the ``--schema`` option. + .. _v3_12: 3.12 (2021-06-25) diff --git a/setup.py b/setup.py index 02224b3..eeee83b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.12" +VERSION = "3.13" def get_long_description(): From 5ec6686153e29ae10d4921a1ad4c841f192f20e2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 1 Aug 2021 21:47:39 -0700 Subject: [PATCH 0397/1004] sqlite-utils convert command and db[table].convert(...) method Closes #251, closes #302. --- docs/cli.rst | 106 +++++++++ docs/python-api.rst | 41 +++- setup.py | 10 +- sqlite_utils/cli.py | 137 +++++++++++- sqlite_utils/db.py | 111 +++++++++- sqlite_utils/recipes.py | 19 ++ sqlite_utils/utils.py | 19 +- tests/test_cli_convert.py | 441 ++++++++++++++++++++++++++++++++++++++ tests/test_convert.py | 77 +++++++ tests/test_docs.py | 33 ++- tests/test_recipes.py | 108 ++++++++++ 11 files changed, 1093 insertions(+), 9 deletions(-) create mode 100644 sqlite_utils/recipes.py create mode 100644 tests/test_cli_convert.py create mode 100644 tests/test_convert.py create mode 100644 tests/test_recipes.py diff --git a/docs/cli.rst b/docs/cli.rst index d8993e4..7bc7a3b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -920,6 +920,112 @@ The ``-`` argument indicates data should be read from standard input. The string When inserting data from standard input only the following column definitions are supported: ``name``, ``path``, ``content``, ``sha256``, ``md5`` and ``size``. +.. _cli_convert: + +Converting data in columns +========================== + +The ``convert`` command can be used to transform the data in a specified column - for example to parse a date string into an ISO timestamp, or to split a string of tags into a JSON array. + +The command accepts a database, table, one or more columns and a string of Python code to be executed against the values from those columns. The following example would replace the values in the ``headline`` column in the ``articles`` table with an upper-case version:: + + $ sqlite-utils convert content.db articles headline 'value.upper()' + +The Python code is passed as a string. Within that Python code the ``value`` variable will be the value of the current column. + +The code you provide will be compiled into a function that takes ``value`` as a single argument. If you break your function body into multiple lines the last line should be a ``return`` statement:: + + $ sqlite-utils convert content.db articles headline ' + value = str(value) + return value.upper()' + +You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options:: + + $ sqlite-utils convert content.db articles content \ + '"\n".join(textwrap.wrap(value, 10))' \ + --import=textwrap + +The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. + +.. _cli_convert_recipes: + +sqlite-utils convert recipes +---------------------------- + +Various built-in recipe functions are available for common operations. These are: + +``r.jsonsplit(value, delimiter=',', type=)`` + Convert a string like ``a,b,c`` into a JSON array ``["a", "b", "c"]`` + + The ``delimiter`` parameter can be used to specify a different delimiter. + + The ``type`` parameter can be set to ``float`` or ``int`` to produce a JSON array of different types, for example if the column's string value was ``1.2,3,4`` the following:: + + r.jsonsplit(value, type=float) + + Would produce an array like this: ``[1.2, 3.0, 4.5]`` + +``r.parsedate(value, dayfirst=False, yearfirst=False)`` + Parse a date and convert it to ISO date format: ``yyyy-mm-dd`` + + In the case of dates such as ``03/04/05`` U.S. ``MM/DD/YY`` format is assumed - you can use ``dayfirst=True`` or ``yearfirst=True`` to change how these ambiguous dates are interpreted. + +``r.parsedatetime(value, dayfirst=False, yearfirst=False)`` + Parse a datetime and convert it to ISO datetime format: ``yyyy-mm-ddTHH:MM:SS`` + +These recipes can be used in the code passed to ``sqlite-utils convert`` like this:: + + $ sqlite-utils convert my.db mytable mycolumn \ + 'r.jsonsplit(value, delimiter=":")' + +.. _cli_convert_output: + +Saving the result to a different column +--------------------------------------- + +The ``--output`` and ``--output-type`` options can be used to save the result of the conversion to a separate column, which will be created if that column does not already exist:: + + $ sqlite-utils convert content.db articles headline 'value.upper()' \ + --output headline_upper + +The type of the created column defaults to ``text``, but a different column type can be specified using ``--output-type``. This example will create a new floating point column called ``id_as_a_float`` with a copy of each item's ID increased by 0.5:: + + $ sqlite-utils convert content.db articles id 'float(value) + 0.5' \ + --output id_as_a_float \ + --output-type float + +You can drop the original column at the end of the operation by adding ``--drop``. + +.. _cli_convert_multi: + +Converting a column into multiple columns +----------------------------------------- + +Sometimes you may wish to convert a single column into multiple derived columns. For example, you may have a ``location`` column containing ``latitude,longitude`` values which you wish to split out into separate ``latitude`` and ``longitude`` columns. + +You can achieve this using the ``--multi`` option to ``sqlite-utils convert``. This option expects your Python code to return a Python dictionary: new columns well be created and populated for each of the keys in that dictionary. + +For the ``latitude,longitude`` example you would use the following:: + + $ sqlite-utils convert demo.db places location \ + 'bits = value.split(",") + return { + "latitude": float(bits[0]), + "longitude": float(bits[1]), + }' --multi + +The type of the returned values will be taken into account when creating the new columns. In this example, the resulting database schema will look like this: + +.. code-block:: sql + + CREATE TABLE [places] ( + [location] TEXT, + [latitude] FLOAT, + [longitude] FLOAT + ); + +The code function can also return ``None``, in which case its output will be ignored. You can drop the original column at the end of the operation by adding ``--drop``. + .. _cli_create_table: Creating tables diff --git a/docs/python-api.rst b/docs/python-api.rst index 7f3cdd5..7f490ab 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -702,7 +702,7 @@ You can delete all records in a table that match a specific WHERE statement usin >>> db = sqlite_utils.Database("dogs.db") >>> # Delete every dog with age less than 3 - >>> db["dogs"].delete_where("age < ?", [3]): + >>> db["dogs"].delete_where("age < ?", [3]) Calling ``table.delete_where()`` with no other arguments will delete every row in the table. @@ -736,6 +736,45 @@ An ``upsert_all()`` method is also available, which behaves like ``insert_all()` .. note:: ``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 `__ for details of this change. +.. _python_api_convert: + +Converting data in columns +========================== + +The ``table.convert(...)`` method can be used to apply a conversion function to the values in a column, either to update that column or to populate new columns. It is the Python library equivalent of the :ref:`sqlite-utils convert ` command. + +This feature works by registering a custom SQLite function that applies a Python transformation, then running a SQL query equivalent to ``UPDATE table SET column = convert_value(column);`` + +To transform a specific column to uppercase, you would use the following: + +.. code-block:: python + + db["dogs"].convert("name", lambda value: value.upper()) + +You can pass a list of columns, in which case the transformation will be applied to each one: + +.. code-block:: python + + db["dogs"].convert(["name", "twitter"], lambda value: value.upper()) + +To save the output to of the transformation to a different column, use the ``output=`` parameter: + +.. code-block:: python + + db["dogs"].convert("name", lambda value: value.upper(), output="name_upper") + +This will add the new column, if it does not already exist. You can pass ``output_type=int`` or some other type to control the type of the new column - otherwise it will default to text. + +If you want to drop the original column after saving the results in a separate output column, pass ``drop=True``. + +You can create multiple new columns from a single input column by passing ``multi=True`` and a conversion function that returns a Python dictionary. This example creates new ``upper`` and ``lower`` columns populated from the single ``title`` column: + +.. code-block:: python + + table.convert( + "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True + ) + .. _python_api_lookup_tables: Working with lookup tables diff --git a/setup.py b/setup.py index eeee83b..5009736 100644 --- a/setup.py +++ b/setup.py @@ -22,12 +22,18 @@ setup( version=VERSION, license="Apache License, Version 2.0", packages=find_packages(exclude=["tests", "tests.*"]), - install_requires=["sqlite-fts4", "click", "click-default-group", "tabulate"], + install_requires=[ + "sqlite-fts4", + "click", + "click-default-group", + "tabulate", + "dateutils", + ], setup_requires=["pytest-runner"], extras_require={ "test": ["pytest", "black", "hypothesis"], "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], - "mypy": ["mypy", "types-click", "types-tabulate"], + "mypy": ["mypy", "types-click", "types-tabulate", "types-python-dateutil"], "flake8": ["flake8"], }, entry_points=""" diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index b168ec0..e1b770f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -5,8 +5,10 @@ from datetime import datetime import hashlib import pathlib import sqlite_utils -from sqlite_utils.db import AlterError, DescIndex +from sqlite_utils.db import AlterError, BadMultiValues, DescIndex +from sqlite_utils import recipes import textwrap +import inspect import io import itertools import json @@ -1903,6 +1905,139 @@ def analyze_tables( click.echo(details) +def _generate_convert_help(): + help = textwrap.dedent( + """ + Convert columns using Python code you supply. For example: + + \b + $ sqlite-utils convert my.db mytable mycolumn \\ + '"\\n".join(textwrap.wrap(value, 10))' \\ + --import=textwrap + + "value" is a variable with the column value to be converted. + + The following common operations are available as recipe functions: + """ + ).strip() + recipe_names = [ + n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser") + ] + for name in recipe_names: + fn = getattr(recipes, name) + help += "\n\nr.{}{}\n\n {}".format( + name, str(inspect.signature(fn)), fn.__doc__ + ) + help += "\n\n" + help += textwrap.dedent( + """ + You can use these recipes like so: + + \b + $ sqlite-utils convert my.db mytable mycolumn \\ + 'r.jsonsplit(value, delimiter=":")' + """ + ).strip() + return help + + +@cli.command(help=_generate_convert_help()) +@click.argument( + "db_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table", type=str) +@click.argument("columns", type=str, nargs=-1, required=True) +@click.argument("code", type=str) +@click.option( + "--import", "imports", type=str, multiple=True, help="Python modules to import" +) +@click.option( + "--dry-run", is_flag=True, help="Show results of running this against first 10 rows" +) +@click.option( + "--multi", is_flag=True, help="Populate columns for keys in returned dictionary" +) +@click.option("--output", help="Optional separate column to populate with the output") +@click.option( + "--output-type", + help="Column type to use for the output column", + default="text", + type=click.Choice(["integer", "float", "blob", "text"]), +) +@click.option("--drop", is_flag=True, help="Drop original column afterwards") +@click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") +def convert( + db_path, + table, + columns, + code, + imports, + dry_run, + multi, + output, + output_type, + drop, + silent, +): + sqlite3.enable_callback_tracebacks(True) + db = sqlite_utils.Database(db_path) + if output is not None and len(columns) > 1: + raise click.ClickException("Cannot use --output with more than one column") + if multi and len(columns) > 1: + raise click.ClickException("Cannot use --multi with more than one column") + if drop and not (output or multi): + raise click.ClickException("--drop can only be used with --output or --multi") + # If single line and no 'return', add the return + if "\n" not in code and not code.strip().startswith("return "): + code = "return {}".format(code) + # Compile the code into a function body called fn(value) + new_code = ["def fn(value):"] + for line in code.split("\n"): + new_code.append(" {}".format(line)) + code_o = compile("\n".join(new_code), "", "exec") + locals = {} + globals = {"r": recipes, "recipes": recipes} + for import_ in imports: + globals[import_] = __import__(import_) + exec(code_o, globals, locals) + fn = locals["fn"] + if dry_run: + # Pull first 20 values for first column and preview them + db.conn.create_function("preview_transform", 1, lambda v: fn(v) if v else v) + sql = """ + select + [{column}] as value, + preview_transform([{column}]) as preview + from [{table}] limit 10 + """.format( + column=columns[0], table=table + ) + for row in db.conn.execute(sql).fetchall(): + click.echo(str(row[0])) + click.echo(" --- becomes:") + click.echo(str(row[1])) + click.echo() + else: + try: + db[table].convert( + columns, + fn, + output=output, + output_type=output_type, + drop=drop, + multi=multi, + show_progress=not silent, + ) + except BadMultiValues as e: + raise click.ClickException( + "When using --multi code must return a Python dictionary - returned: {}".format( + repr(e.values) + ) + ) + + def _render_common(title, values): if values is None: return "" diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ae87e27..eb714e5 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,4 +1,11 @@ -from .utils import sqlite3, OperationalError, suggest_column_types, column_affinity +from .utils import ( + sqlite3, + OperationalError, + suggest_column_types, + types_for_column_types, + column_affinity, + progressbar, +) from collections import namedtuple from collections.abc import Mapping import contextlib @@ -153,6 +160,13 @@ class DescIndex(str): pass +class BadMultiValues(Exception): + "With multi=True code must return a Python dictionary" + + def __init__(self, values): + self.values = values + + _COUNTS_TABLE_CREATE_SQL = """ CREATE TABLE IF NOT EXISTS [{}]( [table] TEXT PRIMARY KEY, @@ -1697,6 +1711,101 @@ class Table(Queryable): self.last_pk = pk_values[0] if len(pks) == 1 else pk_values return self + def convert( + self, + columns, + fn, + output=None, + output_type=None, + drop=False, + multi=False, + show_progress=False, + ): + if isinstance(columns, str): + columns = [columns] + + if multi: + return self._convert_multi( + columns[0], fn, drop=drop, show_progress=show_progress + ) + + if output is not None: + assert len(columns) == 1, "output= can only be used with a single column" + if output not in self.columns_dict: + self.add_column(output, output_type or "text") + + todo_count = self.count * len(columns) + with progressbar(length=todo_count, silent=not show_progress) as bar: + + def convert_value(v): + bar.update(1) + if not v: + return v + return fn(v) + + self.db.register_function(convert_value) + sql = "update [{table}] set {sets};".format( + table=self.name, + sets=", ".join( + [ + "[{output_column}] = convert_value([{column}])".format( + output_column=output or column, column=column + ) + for column in columns + ] + ), + ) + with self.db.conn: + self.db.execute(sql) + if drop: + self.transform(drop=columns) + return self + + def _convert_multi(self, column, fn, drop, show_progress): + # First we execute the function + pk_to_values = {} + new_column_types = {} + pks = [column.name for column in self.columns if column.is_pk] + if not pks: + pks = ["rowid"] + + with progressbar( + length=self.count, silent=not show_progress, label="1: Evaluating" + ) as bar: + for row in self.rows_where( + select=", ".join( + "[{}]".format(column_name) for column_name in (pks + [column]) + ) + ): + row_pk = tuple(row[pk] for pk in pks) + if len(row_pk) == 1: + row_pk = row_pk[0] + values = fn(row[column]) + if values is not None and not isinstance(values, dict): + raise BadMultiValues(values) + if values: + for key, value in values.items(): + new_column_types.setdefault(key, set()).add(type(value)) + pk_to_values[row_pk] = values + bar.update(1) + + # Add any new columns + columns_to_create = types_for_column_types(new_column_types) + for column_name, column_type in columns_to_create.items(): + if column_name not in self.columns_dict: + self.add_column(column_name, column_type) + + # Run the updates + with progressbar( + length=self.count, silent=not show_progress, label="2: Updating" + ) as bar: + with self.db.conn: + for pk, updates in pk_to_values.items(): + self.update(pk, updates) + bar.update(1) + if drop: + self.transform(drop=(column,)) + def build_insert_queries_and_params( self, extracts, diff --git a/sqlite_utils/recipes.py b/sqlite_utils/recipes.py new file mode 100644 index 0000000..6918661 --- /dev/null +++ b/sqlite_utils/recipes.py @@ -0,0 +1,19 @@ +from dateutil import parser +import json + + +def parsedate(value, dayfirst=False, yearfirst=False): + "Parse a date and convert it to ISO date format: yyyy-mm-dd" + return ( + parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).date().isoformat() + ) + + +def parsedatetime(value, dayfirst=False, yearfirst=False): + "Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS" + return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat() + + +def jsonsplit(value, delimiter=",", type=str): + 'Convert a string like a,b,c into a JSON array ["a", "b", "c"]' + return json.dumps([type(s.strip()) for s in value.split(delimiter)]) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 4f3c819..a781469 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -31,8 +31,11 @@ def suggest_column_types(records): for record in records: for key, value in record.items(): all_column_types.setdefault(key, set()).add(type(value)) - column_types = {} + return types_for_column_types(all_column_types) + +def types_for_column_types(all_column_types): + column_types = {} for key, types in all_column_types.items(): # Ignore null values if at least one other type present: if len(types) > 1: @@ -254,3 +257,17 @@ class ValueTracker: not_these.append(name) for key in not_these: del self.couldbe[key] + + +class NullProgressBar: + def update(self, value): + pass + + +@contextlib.contextmanager +def progressbar(silent=False, **kwargs): + if silent: + yield NullProgressBar() + else: + with click.progressbar(**kwargs) as bar: + yield bar diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py new file mode 100644 index 0000000..51634cd --- /dev/null +++ b/tests/test_cli_convert.py @@ -0,0 +1,441 @@ +from click.testing import CliRunner +from sqlite_utils import cli +import sqlite_utils +import json +import textwrap +import pathlib +import pytest + + +@pytest.fixture +def test_db_and_path(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["example"].insert_all( + [ + {"id": 1, "dt": "5th October 2019 12:04"}, + {"id": 2, "dt": "6th October 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ], + pk="id", + ) + return db, db_path + + +@pytest.fixture +def fresh_db_and_path(tmpdir): + db_path = str(pathlib.Path(tmpdir) / "data.db") + db = sqlite_utils.Database(db_path) + return db, db_path + + +@pytest.mark.parametrize( + "code", + [ + "return value.replace('October', 'Spooktober')", + # Return is optional: + "value.replace('October', 'Spooktober')", + ], +) +def test_convert_single_line(test_db_and_path, code): + db, db_path = test_db_and_path + result = CliRunner().invoke(cli.cli, ["convert", db_path, "example", "dt", code]) + assert 0 == result.exit_code, result.output + assert [ + {"id": 1, "dt": "5th Spooktober 2019 12:04"}, + {"id": 2, "dt": "6th Spooktober 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] == list(db["example"].rows) + + +def test_convert_multiple_lines(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "v = value.replace('October', 'Spooktober')\nreturn v.upper()", + ], + ) + assert 0 == result.exit_code, result.output + assert [ + {"id": 1, "dt": "5TH SPOOKTOBER 2019 12:04"}, + {"id": 2, "dt": "6TH SPOOKTOBER 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] == list(db["example"].rows) + + +def test_convert_import(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "return re.sub('O..', 'OXX', value)", + "--import", + "re", + ], + ) + assert 0 == result.exit_code, result.output + assert [ + {"id": 1, "dt": "5th OXXober 2019 12:04"}, + {"id": 2, "dt": "6th OXXober 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] == list(db["example"].rows) + + +def test_convert_dryrun(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "return re.sub('O..', 'OXX', value)", + "--import", + "re", + "--dry-run", + ], + ) + assert result.exit_code == 0 + assert result.output.strip() == ( + "5th October 2019 12:04\n" + " --- becomes:\n" + "5th OXXober 2019 12:04\n" + "\n" + "6th October 2019 00:05:06\n" + " --- becomes:\n" + "6th OXXober 2019 00:05:06\n" + "\n" + "\n" + " --- becomes:\n" + "\n" + "\n" + "None\n" + " --- becomes:\n" + "None" + ) + # But it should not have actually modified the table data + assert list(db["example"].rows) == [ + {"id": 1, "dt": "5th October 2019 12:04"}, + {"id": 2, "dt": "6th October 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] + + +@pytest.mark.parametrize("drop", (True, False)) +def test_convert_output_column(test_db_and_path, drop): + db, db_path = test_db_and_path + args = [ + "convert", + db_path, + "example", + "dt", + "value.replace('October', 'Spooktober')", + "--output", + "newcol", + ] + if drop: + args += ["--drop"] + result = CliRunner().invoke(cli.cli, args) + assert 0 == result.exit_code, result.output + expected = [ + { + "id": 1, + "dt": "5th October 2019 12:04", + "newcol": "5th Spooktober 2019 12:04", + }, + { + "id": 2, + "dt": "6th October 2019 00:05:06", + "newcol": "6th Spooktober 2019 00:05:06", + }, + {"id": 3, "dt": "", "newcol": ""}, + {"id": 4, "dt": None, "newcol": None}, + ] + if drop: + for row in expected: + del row["dt"] + assert list(db["example"].rows) == expected + + +@pytest.mark.parametrize( + "output_type,expected", + ( + ("text", [(1, "1"), (2, "2"), (3, "3"), (4, "4")]), + ("float", [(1, 1.0), (2, 2.0), (3, 3.0), (4, 4.0)]), + ("integer", [(1, 1), (2, 2), (3, 3), (4, 4)]), + (None, [(1, "1"), (2, "2"), (3, "3"), (4, "4")]), + ), +) +def test_convert_output_column_output_type(test_db_and_path, output_type, expected): + db, db_path = test_db_and_path + args = [ + "convert", + db_path, + "example", + "id", + "value", + "--output", + "new_id", + ] + if output_type: + args += ["--output-type", output_type] + result = CliRunner().invoke( + cli.cli, + args, + ) + assert 0 == result.exit_code, result.output + assert expected == list(db.execute("select id, new_id from example")) + + +@pytest.mark.parametrize( + "options,expected_error", + [ + ( + [ + "dt", + "id", + "value.replace('October', 'Spooktober')", + "--output", + "newcol", + ], + "Cannot use --output with more than one column", + ), + ( + [ + "dt", + "value.replace('October', 'Spooktober')", + "--output", + "newcol", + "--output-type", + "invalid", + ], + "Error: Invalid value for '--output-type'", + ), + ( + [ + "value.replace('October', 'Spooktober')", + ], + "Missing argument 'COLUMNS...'", + ), + ], +) +def test_convert_output_error(test_db_and_path, options, expected_error): + db_path = test_db_and_path[1] + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + ] + + options, + ) + assert result.exit_code != 0 + assert expected_error in result.output + + +@pytest.mark.parametrize("drop", (True, False)) +def test_convert_multi(fresh_db_and_path, drop): + db, db_path = fresh_db_and_path + db["creatures"].insert_all( + [ + {"id": 1, "name": "Simon"}, + {"id": 2, "name": "Cleo"}, + ], + pk="id", + ) + args = [ + "convert", + db_path, + "creatures", + "name", + "--multi", + '{"upper": value.upper(), "lower": value.lower()}', + ] + if drop: + args += ["--drop"] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 0, result.output + expected = [ + {"id": 1, "name": "Simon", "upper": "SIMON", "lower": "simon"}, + {"id": 2, "name": "Cleo", "upper": "CLEO", "lower": "cleo"}, + ] + if drop: + for row in expected: + del row["name"] + assert list(db["creatures"].rows) == expected + + +def test_convert_multi_complex_column_types(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["rows"].insert_all( + [ + {"id": 1}, + {"id": 2}, + {"id": 3}, + {"id": 4}, + ], + pk="id", + ) + code = textwrap.dedent( + """ + if value == 1: + return {"is_str": "", "is_float": 1.2, "is_int": None} + elif value == 2: + return {"is_float": 1, "is_int": 12} + elif value == 3: + return {"is_bytes": b"blah"} + """ + ) + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "rows", + "id", + "--multi", + code, + ], + ) + assert result.exit_code == 0, result.output + assert list(db["rows"].rows) == [ + {"id": 1, "is_str": "", "is_float": 1.2, "is_int": None, "is_bytes": None}, + {"id": 2, "is_str": None, "is_float": 1.0, "is_int": 12, "is_bytes": None}, + { + "id": 3, + "is_str": None, + "is_float": None, + "is_int": None, + "is_bytes": b"blah", + }, + {"id": 4, "is_str": None, "is_float": None, "is_int": None, "is_bytes": None}, + ] + assert db["rows"].schema == ( + "CREATE TABLE [rows] (\n" + " [id] INTEGER PRIMARY KEY\n" + ", [is_str] TEXT, [is_float] FLOAT, [is_int] INTEGER, [is_bytes] BLOB)" + ) + + +@pytest.mark.parametrize("delimiter", [None, ";", "-"]) +def test_recipe_jsonsplit(tmpdir, delimiter): + db_path = str(pathlib.Path(tmpdir) / "data.db") + db = sqlite_utils.Database(db_path) + db["example"].insert_all( + [ + {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])}, + {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])}, + ], + pk="id", + ) + code = "r.jsonsplit(value)" + if delimiter: + code = 'recipes.jsonsplit(value, delimiter="{}")'.format(delimiter) + args = ["convert", db_path, "example", "tags", code] + result = CliRunner().invoke(cli.cli, args) + assert 0 == result.exit_code, result.output + assert list(db["example"].rows) == [ + {"id": 1, "tags": '["foo", "bar"]'}, + {"id": 2, "tags": '["bar", "baz"]'}, + ] + + +@pytest.mark.parametrize( + "type,expected_array", + ( + (None, ["1", "2", "3"]), + ("float", [1.0, 2.0, 3.0]), + ("int", [1, 2, 3]), + ), +) +def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array): + db, db_path = fresh_db_and_path + db["example"].insert_all( + [ + {"id": 1, "records": "1,2,3"}, + ], + pk="id", + ) + code = "r.jsonsplit(value)" + if type: + code = "recipes.jsonsplit(value, type={})".format(type) + args = ["convert", db_path, "example", "records", code] + result = CliRunner().invoke(cli.cli, args) + assert 0 == result.exit_code, result.output + assert json.loads(db["example"].get(1)["records"]) == expected_array + + +@pytest.mark.parametrize("drop", (True, False)) +def test_recipe_jsonsplit_output(fresh_db_and_path, drop): + db, db_path = fresh_db_and_path + db["example"].insert_all( + [ + {"id": 1, "records": "1,2,3"}, + ], + pk="id", + ) + code = "r.jsonsplit(value)" + args = ["convert", db_path, "example", "records", code, "--output", "tags"] + if drop: + args += ["--drop"] + result = CliRunner().invoke(cli.cli, args) + assert 0 == result.exit_code, result.output + expected = { + "id": 1, + "records": "1,2,3", + "tags": '["1", "2", "3"]', + } + if drop: + del expected["records"] + assert db["example"].get(1) == expected + + +def test_cannot_use_drop_without_multi_or_output(fresh_db_and_path): + args = ["convert", fresh_db_and_path[1], "example", "records", "value", "--drop"] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 1, result.output + assert "Error: --drop can only be used with --output or --multi" in result.output + + +def test_cannot_use_multi_with_more_than_one_column(fresh_db_and_path): + args = [ + "convert", + fresh_db_and_path[1], + "example", + "records", + "othercol", + "value", + "--multi", + ] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 1, result.output + assert "Error: Cannot use --multi with more than one column" in result.output + + +def test_multi_with_bad_function(test_db_and_path): + args = [ + "convert", + test_db_and_path[1], + "example", + "dt", + "value.upper()", + "--multi", + ] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 1, result.output + assert "When using --multi code must return a Python dictionary" in result.output diff --git a/tests/test_convert.py b/tests/test_convert.py new file mode 100644 index 0000000..34e98f1 --- /dev/null +++ b/tests/test_convert.py @@ -0,0 +1,77 @@ +from sqlite_utils.db import BadMultiValues +import pytest + + +@pytest.mark.parametrize( + "columns,fn,expected", + ( + ( + "title", + lambda value: value.upper(), + {"title": "MIXED CASE", "abstract": "Abstract"}, + ), + ( + ["title", "abstract"], + lambda value: value.upper(), + {"title": "MIXED CASE", "abstract": "ABSTRACT"}, + ), + ), +) +def test_convert(fresh_db, columns, fn, expected): + table = fresh_db["table"] + table.insert({"title": "Mixed Case", "abstract": "Abstract"}) + table.convert(columns, fn) + assert list(table.rows) == [expected] + + +@pytest.mark.parametrize( + "drop,expected", + ( + (False, {"title": "Mixed Case", "other": "MIXED CASE"}), + (True, {"other": "MIXED CASE"}), + ), +) +def test_convert_output(fresh_db, drop, expected): + table = fresh_db["table"] + table.insert({"title": "Mixed Case"}) + table.convert("title", lambda v: v.upper(), output="other", drop=drop) + assert list(table.rows) == [expected] + + +def test_convert_output_multiple_column_error(fresh_db): + table = fresh_db["table"] + with pytest.raises(AssertionError) as excinfo: + table.convert(["title", "other"], lambda v: v, output="out") + assert "output= can only be used with a single column" in str(excinfo.value) + + +@pytest.mark.parametrize( + "type,expected", + ( + (int, {"other": 123}), + (float, {"other": 123.0}), + ), +) +def test_convert_output_type(fresh_db, type, expected): + table = fresh_db["table"] + table.insert({"number": "123"}) + table.convert("number", lambda v: v, output="other", output_type=type, drop=True) + assert list(table.rows) == [expected] + + +def test_convert_multi(fresh_db): + table = fresh_db["table"] + table.insert({"title": "Mixed Case"}) + table.convert( + "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True + ) + assert list(table.rows) == [ + {"title": "Mixed Case", "upper": "MIXED CASE", "lower": "mixed case"} + ] + + +def test_convert_multi_exception(fresh_db): + table = fresh_db["table"] + table.insert({"title": "Mixed Case"}) + with pytest.raises(BadMultiValues): + table.convert("title", lambda v: v.upper(), multi=True) diff --git a/tests/test_docs.py b/tests/test_docs.py index 87b685e..d760629 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,10 +1,12 @@ -from sqlite_utils import cli +from click.testing import CliRunner +from sqlite_utils import cli, recipes from pathlib import Path import pytest import re docs_path = Path(__file__).parent.parent / "docs" commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+) ") +recipes_re = re.compile(r"r\.(\w+)\(") @pytest.fixture(scope="session") @@ -17,11 +19,36 @@ def documented_commands(): } +@pytest.fixture(scope="session") +def documented_recipes(): + rst = (docs_path / "cli.rst").read_text() + return set(recipes_re.findall(rst)) + + @pytest.mark.parametrize("command", cli.cli.commands.keys()) def test_commands_are_documented(documented_commands, command): assert command in documented_commands @pytest.mark.parametrize("command", cli.cli.commands.values()) -def test_commands_have_docstrings(command): - assert command.__doc__, "{} is missing a docstring".format(command) +def test_commands_have_help(command): + assert command.help, "{} is missing its help".format(command) + + +def test_convert_help(): + result = CliRunner().invoke(cli.cli, ["convert", "--help"]) + assert result.exit_code == 0 + for expected in ( + "r.jsonsplit(value, ", + "r.parsedate(value, ", + "r.parsedatetime(value, ", + ): + assert expected in result.output + + +@pytest.mark.parametrize( + "recipe", + [n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")], +) +def test_recipes_are_documented(documented_recipes, recipe): + assert recipe in documented_recipes diff --git a/tests/test_recipes.py b/tests/test_recipes.py new file mode 100644 index 0000000..89240a2 --- /dev/null +++ b/tests/test_recipes.py @@ -0,0 +1,108 @@ +from sqlite_utils import recipes +import json +import pytest + + +@pytest.fixture +def dates_db(fresh_db): + fresh_db["example"].insert_all( + [ + {"id": 1, "dt": "5th October 2019 12:04"}, + {"id": 2, "dt": "6th October 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ], + pk="id", + ) + return fresh_db + + +def test_parsedate(dates_db): + dates_db["example"].convert("dt", recipes.parsedate) + assert list(dates_db["example"].rows) == [ + {"id": 1, "dt": "2019-10-05"}, + {"id": 2, "dt": "2019-10-06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] + + +def test_parsedatetime(dates_db): + dates_db["example"].convert("dt", recipes.parsedatetime) + assert list(dates_db["example"].rows) == [ + {"id": 1, "dt": "2019-10-05T12:04:00"}, + {"id": 2, "dt": "2019-10-06T00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] + + +@pytest.mark.parametrize( + "recipe,kwargs,expected", + ( + ("parsedate", {}, "2005-03-04"), + ("parsedate", {"dayfirst": True}, "2005-04-03"), + ("parsedatetime", {}, "2005-03-04T00:00:00"), + ("parsedatetime", {"dayfirst": True}, "2005-04-03T00:00:00"), + ), +) +def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected): + fresh_db["example"].insert_all( + [ + {"id": 1, "dt": "03/04/05"}, + ], + pk="id", + ) + fresh_db["example"].convert( + "dt", lambda value: getattr(recipes, recipe)(value, **kwargs) + ) + assert list(fresh_db["example"].rows) == [ + {"id": 1, "dt": expected}, + ] + + +@pytest.mark.parametrize("delimiter", [None, ";", "-"]) +def test_jsonsplit(fresh_db, delimiter): + fresh_db["example"].insert_all( + [ + {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])}, + {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])}, + ], + pk="id", + ) + fn = recipes.jsonsplit + if delimiter is not None: + + def fn(value): + return recipes.jsonsplit(value, delimiter=delimiter) + + fresh_db["example"].convert("tags", fn) + assert list(fresh_db["example"].rows) == [ + {"id": 1, "tags": '["foo", "bar"]'}, + {"id": 2, "tags": '["bar", "baz"]'}, + ] + + +@pytest.mark.parametrize( + "type,expected", + ( + (None, ["1", "2", "3"]), + (float, [1.0, 2.0, 3.0]), + (int, [1, 2, 3]), + ), +) +def test_jsonsplit_type(fresh_db, type, expected): + fresh_db["example"].insert_all( + [ + {"id": 1, "records": "1,2,3"}, + ], + pk="id", + ) + fn = recipes.jsonsplit + if type is not None: + + def fn(value): + return recipes.jsonsplit(value, type=type) + + fresh_db["example"].convert("records", fn) + assert json.loads(fresh_db["example"].get(1)["records"]) == expected From 4823aff4c33dd979bf61dcade2d3a6006d248372 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 1 Aug 2021 22:05:03 -0700 Subject: [PATCH 0398/1004] table.count_where() method, closes #305 --- docs/python-api.rst | 12 +++++++++++- sqlite_utils/db.py | 32 +++++++++++++++++++++++++------- tests/test_enable_counts.py | 2 +- tests/test_introspect.py | 9 ++++++++- 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 7f490ab..aba43ad 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -286,6 +286,16 @@ This method also accepts ``offset=`` and ``limit=`` arguments, for specifying an ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} +.. _python_api_rows_count_where: + +Counting rows +------------- + +To count the number of rows that would be returned by a where filter, use ``.count_where(where, where_args)``: + + >>> db["dogs"].count_where("age > ?", [1]): + 2 + .. _python_api_pks_and_rows_where: Listing rows with their primary keys @@ -1602,7 +1612,7 @@ The ``.count`` property shows the current number of rows (``select count(*) from >>> db["Street_Tree_List"].count 189144 -This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.execute_count()`` instead of accessing the property. +This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.count_where()`` instead of accessing the property. .. _python_api_introspection_columns: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index eb714e5..b5da058 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -682,14 +682,23 @@ class Queryable: self.db = db self.name = name + def count_where( + self, + where=None, + where_args=None, + ): + sql = "select count(*) from [{}]".format(self.name) + if where is not None: + sql += " where " + where + return self.db.execute(sql, where_args or []).fetchone()[0] + def execute_count(self): - return self.db.execute( - "select count(*) from [{}]".format(self.name) - ).fetchone()[0] + # Backwards compatibility, see https://github.com/simonw/sqlite-utils/issues/305#issuecomment-890713185 + return self.count_where() @property def count(self): - return self.execute_count() + return self.count_where() @property def rows(self): @@ -820,7 +829,7 @@ class Table(Queryable): counts = self.db.cached_counts([self.name]) if counts: return next(iter(counts.values())) - return self.execute_count() + return self.count_where() def exists(self): return self.name in self.db.table_names() @@ -1719,6 +1728,8 @@ class Table(Queryable): output_type=None, drop=False, multi=False, + where=None, + where_args=None, show_progress=False, ): if isinstance(columns, str): @@ -1726,7 +1737,12 @@ class Table(Queryable): if multi: return self._convert_multi( - columns[0], fn, drop=drop, show_progress=show_progress + columns[0], + fn, + drop=drop, + where=where, + where_args=where_args, + show_progress=show_progress, ) if output is not None: @@ -1761,7 +1777,9 @@ class Table(Queryable): self.transform(drop=columns) return self - def _convert_multi(self, column, fn, drop, show_progress): + def _convert_multi( + self, column, fn, drop, show_progress, where=None, where_args=None + ): # First we execute the function pk_to_values = {} new_column_types = {} diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index 7a52108..d724e80 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -132,7 +132,7 @@ def test_uses_counts_after_enable_counts(counts_db_path): assert db["foo"].count == 1 assert logged == [ ("select name from sqlite_master where type = 'view'", None), - ("select count(*) from [foo]", None), + ("select count(*) from [foo]", []), ] logged.clear() assert not db.use_counts_table diff --git a/tests/test_introspect.py b/tests/test_introspect.py index cc33c46..dce8afc 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -52,7 +52,14 @@ def test_views(fresh_db): def test_count(existing_db): - assert 3 == existing_db["foo"].count + assert existing_db["foo"].count == 3 + assert existing_db["foo"].count_where() == 3 + assert existing_db["foo"].execute_count() == 3 + + +def test_count_where(existing_db): + assert existing_db["foo"].count_where("text != ?", ["two"]) == 2 + assert existing_db["foo"].count_where("text != :t", {"t": "two"}) == 2 def test_columns(existing_db): From 69c7da5ec9698dabeb23379cc08d012b0cd8e6d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 11:33:56 -0700 Subject: [PATCH 0399/1004] Implemented .convert(..., where=, where_args=), refs #304 --- docs/python-api.rst | 8 ++++++++ sqlite_utils/db.py | 11 +++++++---- tests/test_convert.py | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index aba43ad..c372a62 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -785,6 +785,14 @@ You can create multiple new columns from a single input column by passing ``mult "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True ) +The ``.convert()`` method accepts optional ``where=`` and ``where_args=`` parameters which can be used to apply the conversion to a subset of rows specified by a where clause. Here's how to apply the conversion only to rows with an ``id`` that is higher than 20: + +.. code-block:: python + + table.convert("title", lambda v: v.upper(), where="id > :id", where_args={"id": 20}) + +These behave the same as the corresponding parameters to the :ref:`.rows_where() ` method, so you can use ``?`` placeholders and a list of values instead of ``:named`` placeholders with a dictionary. + .. _python_api_lookup_tables: Working with lookup tables diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b5da058..a21acc4 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1750,7 +1750,7 @@ class Table(Queryable): if output not in self.columns_dict: self.add_column(output, output_type or "text") - todo_count = self.count * len(columns) + todo_count = self.count_where(where, where_args) * len(columns) with progressbar(length=todo_count, silent=not show_progress) as bar: def convert_value(v): @@ -1760,7 +1760,7 @@ class Table(Queryable): return fn(v) self.db.register_function(convert_value) - sql = "update [{table}] set {sets};".format( + sql = "update [{table}] set {sets}{where};".format( table=self.name, sets=", ".join( [ @@ -1770,9 +1770,10 @@ class Table(Queryable): for column in columns ] ), + where=" where {}".format(where) if where is not None else "", ) with self.db.conn: - self.db.execute(sql) + self.db.execute(sql, where_args or []) if drop: self.transform(drop=columns) return self @@ -1793,7 +1794,9 @@ class Table(Queryable): for row in self.rows_where( select=", ".join( "[{}]".format(column_name) for column_name in (pks + [column]) - ) + ), + where=where, + where_args=where_args, ): row_pk = tuple(row[pk] for pk in pks) if len(row_pk) == 1: diff --git a/tests/test_convert.py b/tests/test_convert.py index 34e98f1..796a08f 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -24,6 +24,24 @@ def test_convert(fresh_db, columns, fn, expected): assert list(table.rows) == [expected] +@pytest.mark.parametrize( + "where,where_args", (("id > 1", None), ("id > :id", {"id": 1}), ("id > ?", [1])) +) +def test_convert_where(fresh_db, where, where_args): + table = fresh_db["table"] + table.insert_all( + [ + {"id": 1, "title": "One"}, + {"id": 2, "title": "Two"}, + ], + pk="id", + ) + table.convert( + "title", lambda value: value.upper(), where=where, where_args=where_args + ) + assert list(table.rows) == [{"id": 1, "title": "One"}, {"id": 2, "title": "TWO"}] + + @pytest.mark.parametrize( "drop,expected", ( @@ -70,6 +88,28 @@ def test_convert_multi(fresh_db): ] +def test_convert_multi_where(fresh_db): + table = fresh_db["table"] + table.insert_all( + [ + {"id": 1, "title": "One"}, + {"id": 2, "title": "Two"}, + ], + pk="id", + ) + table.convert( + "title", + lambda v: {"upper": v.upper(), "lower": v.lower()}, + multi=True, + where="id > ?", + where_args=[1], + ) + assert list(table.rows) == [ + {"id": 1, "lower": None, "title": "One", "upper": None}, + {"id": 2, "lower": "two", "title": "Two", "upper": "TWO"}, + ] + + def test_convert_multi_exception(fresh_db): table = fresh_db["table"] table.insert({"title": "Mixed Case"}) From d83b2568131f2b1cc01228419bb08c96d843d65d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 11:58:05 -0700 Subject: [PATCH 0400/1004] --where and -p options for sqlite-utils convert, closes #304 --- docs/cli.rst | 11 ++++++ sqlite_utils/cli.py | 26 +++++++++++-- tests/test_cli_convert.py | 78 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 111 insertions(+), 4 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 7bc7a3b..72c8bbf 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -945,6 +945,17 @@ You can specify Python modules that should be imported and made available to you '"\n".join(textwrap.wrap(value, 10))' \ --import=textwrap +The transformation will be applied to every row in the specified table. You can limit that to just rows that match a ``WHERE`` clause using ``--where``:: + + $ sqlite-utils convert content.db articles headline 'value.upper()' \ + --where "headline like '%cat%'" + +You can include named parameters in your where clause and populate them using one or more ``--param`` options:: + + $ sqlite-utils convert content.db articles headline 'value.upper()' \ + --where "headline like :like" \ + --param like '%cat%' + The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. .. _cli_convert_recipes: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index e1b770f..3f32413 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1959,6 +1959,14 @@ def _generate_convert_help(): @click.option( "--multi", is_flag=True, help="Populate columns for keys in returned dictionary" ) +@click.option("--where", help="Optional where clause") +@click.option( + "-p", + "--param", + multiple=True, + type=(str, str), + help="Named :parameters for where clause", +) @click.option("--output", help="Optional separate column to populate with the output") @click.option( "--output-type", @@ -1976,6 +1984,8 @@ def convert( imports, dry_run, multi, + where, + param, output, output_type, drop, @@ -1992,6 +2002,7 @@ def convert( # If single line and no 'return', add the return if "\n" not in code and not code.strip().startswith("return "): code = "return {}".format(code) + where_args = dict(param) if param else [] # Compile the code into a function body called fn(value) new_code = ["def fn(value):"] for line in code.split("\n"): @@ -2010,20 +2021,29 @@ def convert( select [{column}] as value, preview_transform([{column}]) as preview - from [{table}] limit 10 + from [{table}]{where} limit 10 """.format( - column=columns[0], table=table + column=columns[0], + table=table, + where=" where {}".format(where) if where is not None else "", ) - for row in db.conn.execute(sql).fetchall(): + for row in db.conn.execute(sql, where_args).fetchall(): click.echo(str(row[0])) click.echo(" --- becomes:") click.echo(str(row[1])) click.echo() + count = db[table].count_where( + where=where, + where_args=where_args, + ) + click.echo("Would affect {} row{}".format(count, "" if count == 1 else "s")) else: try: db[table].convert( columns, fn, + where=where, + where_args=where_args, output=output, output_type=output_type, drop=drop, diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 51634cd..8ce5203 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -124,7 +124,8 @@ def test_convert_dryrun(test_db_and_path): "\n" "None\n" " --- becomes:\n" - "None" + "None\n\n" + "Would affect 4 rows" ) # But it should not have actually modified the table data assert list(db["example"].rows) == [ @@ -133,6 +134,27 @@ def test_convert_dryrun(test_db_and_path): {"id": 3, "dt": ""}, {"id": 4, "dt": None}, ] + # Test with a where clause too + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "return re.sub('O..', 'OXX', value)", + "--import", + "re", + "--dry-run", + "--where", + "id = :id", + "-p", + "id", + "4", + ], + ) + assert result.exit_code == 0 + assert result.output.strip().split("\n")[-1] == "Would affect 1 row" @pytest.mark.parametrize("drop", (True, False)) @@ -439,3 +461,57 @@ def test_multi_with_bad_function(test_db_and_path): result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 1, result.output assert "When using --multi code must return a Python dictionary" in result.output + + +def test_convert_where(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "str(value).upper()", + "--where", + "id = :id", + "-p", + "id", + 2, + ], + ) + assert result.exit_code == 0, result.output + assert list(db["example"].rows) == [ + {"id": 1, "dt": "5th October 2019 12:04"}, + {"id": 2, "dt": "6TH OCTOBER 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] + + +def test_convert_where_multi(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["names"].insert_all( + [{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Bants"}], pk="id" + ) + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "names", + "name", + '{"upper": value.upper()}', + "--where", + "id = :id", + "-p", + "id", + 2, + "--multi", + ], + ) + assert 0 == result.exit_code, result.output + assert list(db["names"].rows) == [ + {"id": 1, "name": "Cleo", "upper": None}, + {"id": 2, "name": "Bants", "upper": "BANTS"}, + ] From 60dea99ef78c748dedabb6e5f50510a1750fecec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 12:12:16 -0700 Subject: [PATCH 0401/1004] --silent option for sqlite-utils insert-files, closes #301 --- sqlite_utils/cli.py | 16 ++++++++++++++-- sqlite_utils/utils.py | 13 ++++++++++--- tests/test_insert_files.py | 9 +++++++-- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 3f32413..bd063f6 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -21,6 +21,7 @@ from .utils import ( find_spatialite, sqlite3, decode_base64_values, + progressbar, rows_from_file, Format, TypeTracker, @@ -1744,9 +1745,20 @@ def extract( @click.option("--replace", is_flag=True, help="Replace files with matching primary key") @click.option("--upsert", is_flag=True, help="Upsert files with matching primary key") @click.option("--name", type=str, help="File name to use") +@click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") @load_extension_option def insert_files( - path, table, file_or_dir, column, pk, alter, replace, upsert, name, load_extension + path, + table, + file_or_dir, + column, + pk, + alter, + replace, + upsert, + name, + silent, + load_extension, ): """ Insert one or more files using BLOB columns in the specified table @@ -1783,7 +1795,7 @@ def insert_files( # Load all paths so we can show a progress bar paths_and_relative_paths = list(yield_paths_and_relative_paths()) - with click.progressbar(paths_and_relative_paths) as bar: + with progressbar(paths_and_relative_paths, silent=silent) as bar: def to_insert(): for path, relative_path in bar: diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index a781469..00a3c02 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -260,14 +260,21 @@ class ValueTracker: class NullProgressBar: + def __init__(self, *args): + self.args = args + + def __iter__(self): + yield from self.args[0] + def update(self, value): pass @contextlib.contextmanager -def progressbar(silent=False, **kwargs): +def progressbar(*args, **kwargs): + silent = kwargs.pop("silent") if silent: - yield NullProgressBar() + yield NullProgressBar(*args) else: - with click.progressbar(**kwargs) as bar: + with click.progressbar(*args, **kwargs) as bar: yield bar diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 1a11d2c..1e30a8d 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -2,9 +2,11 @@ from sqlite_utils import cli, Database from click.testing import CliRunner import os import pathlib +import pytest -def test_insert_files(): +@pytest.mark.parametrize("silent", (False, True)) +def test_insert_files(silent): runner = CliRunner() with runner.isolated_filesystem(): tmpdir = pathlib.Path(".") @@ -34,7 +36,10 @@ def test_insert_files(): cols += ["-c", "{}:{}".format(coltype, coltype)] result = runner.invoke( cli.cli, - ["insert-files", db_path, "files", str(tmpdir)] + cols + ["--pk", "path"], + ["insert-files", db_path, "files", str(tmpdir)] + + cols + + ["--pk", "path"] + + (["--silent"] if silent else []), catch_exceptions=False, ) assert result.exit_code == 0, result.stdout From 59032b00bb252c6d8cc43cbc490e0492423f61b1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 12:29:55 -0700 Subject: [PATCH 0402/1004] Fixed incorrect example in documentation --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 72c8bbf..32405c6 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -970,7 +970,7 @@ Various built-in recipe functions are available for common operations. These are The ``delimiter`` parameter can be used to specify a different delimiter. - The ``type`` parameter can be set to ``float`` or ``int`` to produce a JSON array of different types, for example if the column's string value was ``1.2,3,4`` the following:: + The ``type`` parameter can be set to ``float`` or ``int`` to produce a JSON array of different types, for example if the column's string value was ``1.2,3,4.5`` the following:: r.jsonsplit(value, type=float) From 723ee35344fa9f5e49dca578170cc5f5eb7223ce Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 14:18:01 -0700 Subject: [PATCH 0403/1004] Release 3.14 Refs #251, #301, #302, #303, #304, #305 --- docs/changelog.rst | 29 +++++++++++++++++++++++++++++ docs/cli.rst | 4 ++-- setup.py | 2 +- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9d17c1e..d9a125d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,35 @@ Changelog =========== +.. _v3_14: + +3.14 (2021-08-02) +----------------- + +This release introduces the new :ref:`sqlite-utils convert command ` (`#251 `__) and corresponding :ref:`table.convert(...) ` Python method (`#302 `__). These tools can be used to apply a Python conversion function to one or more columns of a table, either updating the column in place or using transformed data from that column to populate one or more other columns. + +This command-line example uses the Python standard library `textwrap module `__ to wrap the content of the ``content`` column in the ``articles`` table to 100 characters:: + + $ sqlite-utils convert content.db articles content \ + '"\n".join(textwrap.wrap(value, 100))' \ + --import=textwrap + +The same operation in Python code looks like this: + +.. code-block:: python + + import sqlite_utils, textwrap + + db = sqlite_utils.Database("content.db") + db["articles"].convert("content", lambda v: "\n".join(textwrap.wrap(v, 100))) + +See the full documentation for the :ref:`sqlite-utils convert command ` and the :ref:`table.convert(...) ` Python method for more details. + +Also in this release: + +- The new ``table.count_where(...)`` method, for counting rows in a table that match a specific SQL ``WHERE`` clause. (`#305 `__) +- New ``--silent`` option for the :ref:`sqlite-utils insert-files command ` to hide the terminal progress bar, consistent with the ``--silent`` option for ``sqlite-utils convert``. (`#301 `__) + .. _v3_13: 3.13 (2021-07-24) diff --git a/docs/cli.rst b/docs/cli.rst index 32405c6..3802963 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -939,10 +939,10 @@ The code you provide will be compiled into a function that takes ``value`` as a value = str(value) return value.upper()' -You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options:: +You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options. This example uses the ``textwrap`` module to wrap the ``content`` column at 100 characters:: $ sqlite-utils convert content.db articles content \ - '"\n".join(textwrap.wrap(value, 10))' \ + '"\n".join(textwrap.wrap(value, 100))' \ --import=textwrap The transformation will be applied to every row in the specified table. You can limit that to just rows that match a ``WHERE`` clause using ``--where``:: diff --git a/setup.py b/setup.py index 5009736..8aa9aeb 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.13" +VERSION = "3.14" def get_long_description(): From e83aef951bd3e8c179511faddb607239a5fa8682 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 14:29:00 -0700 Subject: [PATCH 0404/1004] New :issue: macro, closes #306 --- docs/changelog.rst | 286 ++++++++++++++++++++++----------------------- docs/conf.py | 6 +- 2 files changed, 148 insertions(+), 144 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d9a125d..e03dbd6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,7 @@ 3.14 (2021-08-02) ----------------- -This release introduces the new :ref:`sqlite-utils convert command ` (`#251 `__) and corresponding :ref:`table.convert(...) ` Python method (`#302 `__). These tools can be used to apply a Python conversion function to one or more columns of a table, either updating the column in place or using transformed data from that column to populate one or more other columns. +This release introduces the new :ref:`sqlite-utils convert command ` (:issue:`251`) and corresponding :ref:`table.convert(...) ` Python method (:issue:`302`). These tools can be used to apply a Python conversion function to one or more columns of a table, either updating the column in place or using transformed data from that column to populate one or more other columns. This command-line example uses the Python standard library `textwrap module `__ to wrap the content of the ``content`` column in the ``articles`` table to 100 characters:: @@ -28,15 +28,15 @@ See the full documentation for the :ref:`sqlite-utils convert command `__) -- New ``--silent`` option for the :ref:`sqlite-utils insert-files command ` to hide the terminal progress bar, consistent with the ``--silent`` option for ``sqlite-utils convert``. (`#301 `__) +- The new ``table.count_where(...)`` method, for counting rows in a table that match a specific SQL ``WHERE`` clause. (:issue:`305`) +- New ``--silent`` option for the :ref:`sqlite-utils insert-files command ` to hide the terminal progress bar, consistent with the ``--silent`` option for ``sqlite-utils convert``. (:issue:`301`) .. _v3_13: 3.13 (2021-07-24) ----------------- -- ``sqlite-utils schema my.db table1 table2`` command now accepts optional table names. (`#299 `__) +- ``sqlite-utils schema my.db table1 table2`` command now accepts optional table names. (:issue:`299`) - ``sqlite-utils memory --help`` now describes the ``--schema`` option. .. _v3_12: @@ -44,17 +44,17 @@ Also in this release: 3.12 (2021-06-25) ----------------- -- New :ref:`db.query(sql, params) ` method, which executes a SQL query and returns the results as an iterator over Python dictionaries. (`#290 `__) -- This project now uses ``flake8`` and has started to use ``mypy``. (`#291 `__) -- New documentation on :ref:`contributing ` to this project. (`#292 `__) +- New :ref:`db.query(sql, params) ` method, which executes a SQL query and returns the results as an iterator over Python dictionaries. (:issue:`290`) +- This project now uses ``flake8`` and has started to use ``mypy``. (:issue:`291`) +- New documentation on :ref:`contributing ` to this project. (:issue:`292`) .. _v3_11: 3.11 (2021-06-20) ----------------- -- New ``sqlite-utils memory data.csv --schema`` option, for outputting the schema of the in-memory database generated from one or more files. See :ref:`cli_memory_schema_dump_save`. (`#288 `__) -- Added :ref:`installation instructions `. (`#286 `__) +- New ``sqlite-utils memory data.csv --schema`` option, for outputting the schema of the in-memory database generated from one or more files. See :ref:`cli_memory_schema_dump_save`. (:issue:`288`) +- Added :ref:`installation instructions `. (:issue:`286`) .. _v3_10: @@ -107,21 +107,21 @@ Here the ``species.csv`` file becomes the ``species`` table, the ``creatures.csv You can also use the ``--attach`` option to attach existing SQLite database files to the in-memory database, in order to join data from CSV or JSON directly against your existing tables. -Full documentation of this new feature is available in :ref:`cli_memory`. (`#272 `__) +Full documentation of this new feature is available in :ref:`cli_memory`. (:issue:`272`) sqlite-utils insert \-\-detect-types ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The :ref:`sqlite-utils insert ` command can be used to insert data from JSON, CSV or TSV files into a SQLite database file. The new ``--detect-types`` option (shortcut ``-d``), when used in conjunction with a CSV or TSV import, will automatically detect if columns in the file are integers or floating point numbers as opposed to treating everything as a text column and create the new table with the corresponding schema. See :ref:`cli_insert_csv_tsv` for details. (`#282 `__) +The :ref:`sqlite-utils insert ` command can be used to insert data from JSON, CSV or TSV files into a SQLite database file. The new ``--detect-types`` option (shortcut ``-d``), when used in conjunction with a CSV or TSV import, will automatically detect if columns in the file are integers or floating point numbers as opposed to treating everything as a text column and create the new table with the corresponding schema. See :ref:`cli_insert_csv_tsv` for details. (:issue:`282`) Other changes ~~~~~~~~~~~~~ -- **Bug fix**: ``table.transform()``, when run against a table without explicit primary keys, would incorrectly create a new version of the table with an explicit primary key column called ``rowid``. (`#284 `__) -- New ``table.use_rowid`` introspection property, see :ref:`python_api_introspection_use_rowid`. (`#285 `__) -- The new ``sqlite-utils dump file.db`` command outputs a SQL dump that can be used to recreate a database. (`#274 `__) -- ``-h`` now works as a shortcut for ``--help``, thanks Loren McIntyre. (`#276 `__) -- Now using `pytest-cov `__ and `Codecov `__ to track test coverage - currently at 96%. (`#275 `__) +- **Bug fix**: ``table.transform()``, when run against a table without explicit primary keys, would incorrectly create a new version of the table with an explicit primary key column called ``rowid``. (:issue:`284`) +- New ``table.use_rowid`` introspection property, see :ref:`python_api_introspection_use_rowid`. (:issue:`285`) +- The new ``sqlite-utils dump file.db`` command outputs a SQL dump that can be used to recreate a database. (:issue:`274`) +- ``-h`` now works as a shortcut for ``--help``, thanks Loren McIntyre. (:issue:`276`) +- Now using `pytest-cov `__ and `Codecov `__ to track test coverage - currently at 96%. (:issue:`275`) - SQL errors that occur when using ``sqlite-utils query`` are now displayed as CLI errors. .. _v3_9_1: @@ -129,14 +129,14 @@ Other changes 3.9.1 (2021-06-12) ------------------ -- Fixed bug when using ``table.upsert_all()`` to create a table with only a single column that is treated as the primary key. (`#271 `__) +- Fixed bug when using ``table.upsert_all()`` to create a table with only a single column that is treated as the primary key. (:issue:`271`) .. _v3_9: 3.9 (2021-06-11) ---------------- -- New ``sqlite-utils schema`` command showing the full SQL schema for a database, see :ref:`Showing the schema (CLI)`. (`#268 `__) +- New ``sqlite-utils schema`` command showing the full SQL schema for a database, see :ref:`Showing the schema (CLI)`. (:issue:`268`) - ``db.schema`` introspection property exposing the same feature to the Python library, see :ref:`Showing the schema (Python library) `. .. _v3_8: @@ -144,22 +144,22 @@ Other changes 3.8 (2021-06-02) ---------------- -- New ``sqlite-utils indexes`` command to list indexes in a database, see :ref:`cli_indexes`. (`#263 `__) -- ``table.xindexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (`#261 `__) +- New ``sqlite-utils indexes`` command to list indexes in a database, see :ref:`cli_indexes`. (:issue:`263`) +- ``table.xindexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (:issue:`261`) .. _v3_7: 3.7 (2021-05-28) ---------------- -- New ``table.pks_and_rows_where()`` method returning ``(primary_key, row_dictionary)`` tuples - see :ref:`python_api_pks_and_rows_where`. (`#240 `__) -- Fixed bug with ``table.add_foreign_key()`` against columns containing spaces. (`#238 `__) -- ``table_or_view.drop(ignore=True)`` option for avoiding errors if the table or view does not exist. (`#237 `__) -- ``sqlite-utils drop-view --ignore`` and ``sqlite-utils drop-table --ignore`` options. (`#237 `__) -- Fixed a bug with inserts of nested JSON containing non-ascii strings - thanks, Dylan Wu. (`#257 `__) -- Suggest ``--alter`` if an error occurs caused by a missing column. (`#259 `__) -- Support creating indexes with columns in descending order, see :ref:`API documentation ` and :ref:`CLI documentation `. (`#260 `__) -- Correctly handle CSV files that start with a UTF-8 BOM. (`#250 `__) +- New ``table.pks_and_rows_where()`` method returning ``(primary_key, row_dictionary)`` tuples - see :ref:`python_api_pks_and_rows_where`. (:issue:`240`) +- Fixed bug with ``table.add_foreign_key()`` against columns containing spaces. (:issue:`238`) +- ``table_or_view.drop(ignore=True)`` option for avoiding errors if the table or view does not exist. (:issue:`237`) +- ``sqlite-utils drop-view --ignore`` and ``sqlite-utils drop-table --ignore`` options. (:issue:`237`) +- Fixed a bug with inserts of nested JSON containing non-ascii strings - thanks, Dylan Wu. (:issue:`257`) +- Suggest ``--alter`` if an error occurs caused by a missing column. (:issue:`259`) +- Support creating indexes with columns in descending order, see :ref:`API documentation ` and :ref:`CLI documentation `. (:issue:`260`) +- Correctly handle CSV files that start with a UTF-8 BOM. (:issue:`250`) .. _v3_6: @@ -168,73 +168,73 @@ Other changes This release adds the ability to execute queries joining data from more than one database file - similar to the cross database querying feature introduced in `Datasette 0.55 `__. -- The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation `. (`#113 `__) -- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (`#236 `__) +- The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation `. (:issue:`113`) +- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (:issue:`236`) .. _v3_5: 3.5 (2021-02-14) ---------------- -- ``sqlite-utils insert --sniff`` option for detecting the delimiter and quote character used by a CSV file, see :ref:`cli_insert_csv_tsv_delimiter`. (`#230 `__) -- The ``table.rows_where()``, ``table.search()`` and ``table.search_sql()`` methods all now take optional ``offset=`` and ``limit=`` arguments. (`#231 `__) -- New ``--no-headers`` option for ``sqlite-utils insert --csv`` to handle CSV files that are missing the header row, see :ref:`cli_insert_csv_tsv_no_header`. (`#228 `__) -- Fixed bug where inserting data with extra columns in subsequent chunks would throw an error. Thanks `@nieuwenhoven `__ for the fix. (`#234 `__) -- Fixed bug importing CSV files with columns containing more than 128KB of data. (`#229 `__) -- Test suite now runs in CI against Ubuntu, macOS and Windows. Thanks `@nieuwenhoven `__ for the Windows test fixes. (`#232 `__) +- ``sqlite-utils insert --sniff`` option for detecting the delimiter and quote character used by a CSV file, see :ref:`cli_insert_csv_tsv_delimiter`. (:issue:`230`) +- The ``table.rows_where()``, ``table.search()`` and ``table.search_sql()`` methods all now take optional ``offset=`` and ``limit=`` arguments. (:issue:`231`) +- New ``--no-headers`` option for ``sqlite-utils insert --csv`` to handle CSV files that are missing the header row, see :ref:`cli_insert_csv_tsv_no_header`. (:issue:`228`) +- Fixed bug where inserting data with extra columns in subsequent chunks would throw an error. Thanks `@nieuwenhoven `__ for the fix. (:issue:`234`) +- Fixed bug importing CSV files with columns containing more than 128KB of data. (:issue:`229`) +- Test suite now runs in CI against Ubuntu, macOS and Windows. Thanks `@nieuwenhoven `__ for the Windows test fixes. (:issue:`232`) .. _v3_4_1: 3.4.1 (2021-02-05) ------------------ -- Fixed a code import bug that slipped in to 3.4. (`#226 `__) +- Fixed a code import bug that slipped in to 3.4. (:issue:`226`) .. _v3_4: 3.4 (2021-02-05) ---------------- -- ``sqlite-utils insert --csv`` now accepts optional ``--delimiter`` and ``--quotechar`` options. See :ref:`cli_insert_csv_tsv_delimiter`. (`#223 `__) +- ``sqlite-utils insert --csv`` now accepts optional ``--delimiter`` and ``--quotechar`` options. See :ref:`cli_insert_csv_tsv_delimiter`. (:issue:`223`) .. _v3_3: 3.3 (2021-01-17) ---------------- -- The ``table.m2m()`` method now accepts an optional ``alter=True`` argument to specify that any missing columns should be added to the referenced table. See :ref:`python_api_m2m`. (`#222 `__) +- The ``table.m2m()`` method now accepts an optional ``alter=True`` argument to specify that any missing columns should be added to the referenced table. See :ref:`python_api_m2m`. (:issue:`222`) .. _v3_2_1: 3.2.1 (2021-01-12) ------------------ -- Fixed a bug where ``.add_missing_columns()`` failed to take case insensitive column names into account. (`#221 `__) +- Fixed a bug where ``.add_missing_columns()`` failed to take case insensitive column names into account. (:issue:`221`) .. _v3_2: 3.2 (2021-01-03) ---------------- -This release introduces a new mechanism for speeding up ``count(*)`` queries using cached table counts, stored in a ``_counts`` table and updated by triggers. This mechanism is described in :ref:`python_api_cached_table_counts`, and can be enabled using Python API methods or the new ``enable-counts`` CLI command. (`#212 `__) +This release introduces a new mechanism for speeding up ``count(*)`` queries using cached table counts, stored in a ``_counts`` table and updated by triggers. This mechanism is described in :ref:`python_api_cached_table_counts`, and can be enabled using Python API methods or the new ``enable-counts`` CLI command. (:issue:`212`) - ``table.enable_counts()`` method for enabling these triggers on a specific table. -- ``db.enable_counts()`` method for enabling triggers on every table in the database. (`#213 `__) -- New ``sqlite-utils enable-counts my.db`` command for enabling counts on all or specific tables, see :ref:`cli_enable_counts`. (`#214 `__) -- New ``sqlite-utils triggers`` command for listing the triggers defined for a database or specific tables, see :ref:`cli_triggers`. (`#218 `__) -- New ``db.use_counts_table`` property which, if ``True``, causes ``table.count`` to read from the ``_counts`` table. (`#215 `__) +- ``db.enable_counts()`` method for enabling triggers on every table in the database. (:issue:`213`) +- New ``sqlite-utils enable-counts my.db`` command for enabling counts on all or specific tables, see :ref:`cli_enable_counts`. (:issue:`214`) +- New ``sqlite-utils triggers`` command for listing the triggers defined for a database or specific tables, see :ref:`cli_triggers`. (:issue:`218`) +- New ``db.use_counts_table`` property which, if ``True``, causes ``table.count`` to read from the ``_counts`` table. (:issue:`215`) - ``table.has_counts_triggers`` property revealing if a table has been configured with the new ``_counts`` database triggers. -- ``db.reset_counts()`` method and ``sqlite-utils reset-counts`` command for resetting the values in the ``_counts`` table. (`#219 `__) -- The previously undocumented ``db.escape()`` method has been renamed to ``db.quote()`` and is now covered by the documentation: :ref:`python_api_quote`. (`#217 `__) -- New ``table.triggers_dict`` and ``db.triggers_dict`` introspection properties. (`#211 `__, `#216 `__) -- ``sqlite-utils insert`` now shows a more useful error message for invalid JSON. (`#206 `__) +- ``db.reset_counts()`` method and ``sqlite-utils reset-counts`` command for resetting the values in the ``_counts`` table. (:issue:`219`) +- The previously undocumented ``db.escape()`` method has been renamed to ``db.quote()`` and is now covered by the documentation: :ref:`python_api_quote`. (:issue:`217`) +- New ``table.triggers_dict`` and ``db.triggers_dict`` introspection properties. (:issue:`211`, :issue:`216`) +- ``sqlite-utils insert`` now shows a more useful error message for invalid JSON. (:issue:`206`) .. _v3_1_1: 3.1.1 (2021-01-01) ------------------ -- Fixed failing test caused by ``optimize`` sometimes creating larger database files. (`#209 `__) +- Fixed failing test caused by ``optimize`` sometimes creating larger database files. (:issue:`209`) - Documentation now lives on https://sqlite-utils.datasette.io/ - README now includes ``brew install sqlite-utils`` installation method. @@ -243,7 +243,7 @@ This release introduces a new mechanism for speeding up ``count(*)`` queries usi 3.1 (2020-12-12) ---------------- -- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (`#207 `__) +- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (:issue:`207`) - New ``table.analyze_column(column)`` Python method used by the ``analyze-tables`` command - see :ref:`python_api_analyze_column`. - The ``table.update()`` method now correctly handles values that should be stored as JSON. Thanks, Andreas Madsack. (`#204 `__) @@ -252,28 +252,28 @@ This release introduces a new mechanism for speeding up ``count(*)`` queries usi 3.0 (2020-11-08) ---------------- -This release introduces a new ``sqlite-utils search`` command for searching tables, see :ref:`cli_search`. (`#192 `__) +This release introduces a new ``sqlite-utils search`` command for searching tables, see :ref:`cli_search`. (:issue:`192`) -The ``table.search()`` method has been redesigned, see :ref:`python_api_fts_search`. (`#197 `__) +The ``table.search()`` method has been redesigned, see :ref:`python_api_fts_search`. (:issue:`197`) The release includes minor backwards-incompatible changes, hence the version bump to 3.0. Those changes, which should not affect most users, are: - The ``-c`` shortcut option for outputting CSV is no longer available. The full ``--csv`` option is required instead. - The ``-f`` shortcut for ``--fmt`` has also been removed - use ``--fmt``. -- The ``table.search()`` method now defaults to sorting by relevance, not sorting by ``rowid``. (`#198 `__) +- The ``table.search()`` method now defaults to sorting by relevance, not sorting by ``rowid``. (:issue:`198`) - The ``table.search()`` method now returns a generator over a list of Python dictionaries. It previously returned a list of tuples. Also in this release: -- The ``query``, ``tables``, ``rows`` and ``search`` CLI commands now accept a new ``--tsv`` option which outputs the results in TSV. (`#193 `__) -- A new ``table.virtual_table_using`` property reveals if a table is a virtual table, and returns the upper case type of virtual table (e.g. ``FTS4`` or ``FTS5``) if it is. It returns ``None`` if the table is not a virtual table. (`#196 `__) +- The ``query``, ``tables``, ``rows`` and ``search`` CLI commands now accept a new ``--tsv`` option which outputs the results in TSV. (:issue:`193`) +- A new ``table.virtual_table_using`` property reveals if a table is a virtual table, and returns the upper case type of virtual table (e.g. ``FTS4`` or ``FTS5``) if it is. It returns ``None`` if the table is not a virtual table. (:issue:`196`) - The new ``table.search_sql()`` method returns the SQL for searching a table, see :ref:`python_api_fts_search_sql`. -- ``sqlite-utils rows`` now accepts multiple optional ``-c`` parameters specifying the columns to return. (`#200 `__) +- ``sqlite-utils rows`` now accepts multiple optional ``-c`` parameters specifying the columns to return. (:issue:`200`) Changes since the 3.0a0 alpha release: - The ``sqlite-utils search`` command now defaults to returning every result, unless you add a ``--limit 20`` option. -- The ``sqlite-utils search -c`` and ``table.search(columns=[])`` options are now fully respected. (`#201 `__) +- The ``sqlite-utils search -c`` and ``table.search(columns=[])`` options are now fully respected. (:issue:`201`) .. _v2_23: @@ -281,30 +281,30 @@ Changes since the 3.0a0 alpha release: ----------------- - ``table.m2m(other_table, records)`` method now takes any iterable, not just a list or tuple. Thanks, Adam Wolf. (`#189 `__) -- ``sqlite-utils insert`` now displays a progress bar for CSV or TSV imports. (`#173 `__) -- New ``@db.register_function(deterministic=True)`` option for registering deterministic SQLite functions in Python 3.8 or higher. (`#191 `__) +- ``sqlite-utils insert`` now displays a progress bar for CSV or TSV imports. (:issue:`173`) +- New ``@db.register_function(deterministic=True)`` option for registering deterministic SQLite functions in Python 3.8 or higher. (:issue:`191`) .. _v2_22: 2.22 (2020-10-16) ----------------- -- New ``--encoding`` option for processing CSV and TSV files that use a non-utf-8 encoding, for both the ``insert`` and ``update`` commands. (`#182 `__) -- The ``--load-extension`` option is now available to many more commands. (`#137 `__) -- ``--load-extension=spatialite`` can be used to load SpatiaLite from common installation locations, if it is available. (`#136 `__) -- Tests now also run against Python 3.9. (`#184 `__) -- Passing ``pk=["id"]`` now has the same effect as passing ``pk="id"``. (`#181 `__) +- New ``--encoding`` option for processing CSV and TSV files that use a non-utf-8 encoding, for both the ``insert`` and ``update`` commands. (:issue:`182`) +- The ``--load-extension`` option is now available to many more commands. (:issue:`137`) +- ``--load-extension=spatialite`` can be used to load SpatiaLite from common installation locations, if it is available. (:issue:`136`) +- Tests now also run against Python 3.9. (:issue:`184`) +- Passing ``pk=["id"]`` now has the same effect as passing ``pk="id"``. (:issue:`181`) .. _v2_21: 2.21 (2020-09-24) ----------------- -- ``table.extract()`` and ``sqlite-utils extract`` now apply much, much faster - one example operation reduced from twelve minutes to just four seconds! (`#172 `__) +- ``table.extract()`` and ``sqlite-utils extract`` now apply much, much faster - one example operation reduced from twelve minutes to just four seconds! (:issue:`172`) - ``sqlite-utils extract`` no longer shows a progress bar, because it's fast enough not to need one. -- New ``column_order=`` option for ``table.transform()`` which can be used to alter the order of columns in a table. (`#175 `__) -- ``sqlite-utils transform --column-order=`` option (with a ``-o`` shortcut) for changing column order. (`#176 `__) -- The ``table.transform(drop_foreign_keys=)`` parameter and the ``sqlite-utils transform --drop-foreign-key`` option have changed. They now accept just the name of the column rather than requiring all three of the column, other table and other column. This is technically a backwards-incompatible change but I chose not to bump the major version number because the transform feature is so new. (`#177 `__) +- New ``column_order=`` option for ``table.transform()`` which can be used to alter the order of columns in a table. (:issue:`175`) +- ``sqlite-utils transform --column-order=`` option (with a ``-o`` shortcut) for changing column order. (:issue:`176`) +- The ``table.transform(drop_foreign_keys=)`` parameter and the ``sqlite-utils transform --drop-foreign-key`` option have changed. They now accept just the name of the column rather than requiring all three of the column, other table and other column. This is technically a backwards-incompatible change but I chose not to bump the major version number because the transform feature is so new. (:issue:`177`) - The table ``.disable_fts()``, ``.rebuild_fts()``, ``.delete()``, ``.delete_where()`` and ``.add_missing_columns()`` methods all now ``return self``, which means they can be chained together with other table operations. .. _v2_20: @@ -312,7 +312,7 @@ Changes since the 3.0a0 alpha release: 2.20 (2020-09-22) ----------------- -This release introduces two key new capabilities: **transform** (`#114 `__) and **extract** (`#42 `__). +This release introduces two key new capabilities: **transform** (:issue:`114`) and **extract** (:issue:`42`). Transform ~~~~~~~~~ @@ -333,7 +333,7 @@ The Python library :ref:`extract() documentation ` describes Other changes ~~~~~~~~~~~~~ -- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref:`python_api_register_function`. (`#162 `__) +- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref:`python_api_register_function`. (:issue:`162`) - The ``table.rows_where()`` method now accepts an optional ``select=`` argument for specifying which columns should be selected, see :ref:`python_api_rows`. .. _v2_19: @@ -341,31 +341,31 @@ Other changes 2.19 (2020-09-20) ----------------- -- New ``sqlite-utils add-foreign-keys`` command for :ref:`cli_add_foreign_keys`. (`#157 `__) -- New ``table.enable_fts(..., replace=True)`` argument for replacing an existing FTS table with a new configuration. (`#160 `__) -- New ``table.add_foreign_key(..., ignore=True)`` argument for ignoring a foreign key if it already exists. (`#112 `__) +- New ``sqlite-utils add-foreign-keys`` command for :ref:`cli_add_foreign_keys`. (:issue:`157`) +- New ``table.enable_fts(..., replace=True)`` argument for replacing an existing FTS table with a new configuration. (:issue:`160`) +- New ``table.add_foreign_key(..., ignore=True)`` argument for ignoring a foreign key if it already exists. (:issue:`112`) .. _v2_18: 2.18 (2020-09-08) ----------------- -- ``table.rebuild_fts()`` method for rebuilding a FTS index, see :ref:`python_api_fts_rebuild`. (`#155 `__) -- ``sqlite-utils rebuild-fts data.db`` command for rebuilding FTS indexes across all tables, or just specific tables. (`#155 `__) +- ``table.rebuild_fts()`` method for rebuilding a FTS index, see :ref:`python_api_fts_rebuild`. (:issue:`155`) +- ``sqlite-utils rebuild-fts data.db`` command for rebuilding FTS indexes across all tables, or just specific tables. (:issue:`155`) - ``table.optimize()`` method no longer deletes junk rows from the ``*_fts_docsize`` table. This was added in 2.17 but it turns out running ``table.rebuild_fts()`` is a better solution to this problem. -- Fixed a bug where rows with additional columns that are inserted after the first batch of records could cause an error due to breaking SQLite's maximum number of parameters. Thanks, Simon Wiles. (`#145 `__) +- Fixed a bug where rows with additional columns that are inserted after the first batch of records could cause an error due to breaking SQLite's maximum number of parameters. Thanks, Simon Wiles. (:issue:`145`) .. _v2_17: 2.17 (2020-09-07) ----------------- -This release handles a bug where replacing rows in FTS tables could result in growing numbers of unnecessary rows in the associated ``*_fts_docsize`` table. (`#149 `__) +This release handles a bug where replacing rows in FTS tables could result in growing numbers of unnecessary rows in the associated ``*_fts_docsize`` table. (:issue:`149`) -- ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (`#152 `__) -- ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (`#153 `__) -- New tracer method for tracking underlying SQL queries, see :ref:`python_api_tracing`. (`#150 `__) -- Neater indentation for schema SQL. (`#148 `__) +- ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (:issue:`152`) +- ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (:issue:`153`) +- New tracer method for tracking underlying SQL queries, see :ref:`python_api_tracing`. (:issue:`150`) +- Neater indentation for schema SQL. (:issue:`148`) - Documentation for ``sqlite_utils.AlterError`` exception thrown by in ``add_foreign_keys()``. .. _v2_16_1: @@ -373,23 +373,23 @@ This release handles a bug where replacing rows in FTS tables could result in gr 2.16.1 (2020-08-28) ------------------- -- ``insert_all(..., alter=True)`` now works for columns introduced after the first 100 records. Thanks, Simon Wiles! (`#139 `__) -- Continuous Integration is now powered by GitHub Actions. (`#143 `__) +- ``insert_all(..., alter=True)`` now works for columns introduced after the first 100 records. Thanks, Simon Wiles! (:issue:`139`) +- Continuous Integration is now powered by GitHub Actions. (:issue:`143`) .. _v2_16: 2.16 (2020-08-21) ----------------- -- ``--load-extension`` option for ``sqlite-utils query`` for loading SQLite extensions. (`#134 `__) -- New ``sqlite_utils.utils.find_spatialite()`` function for finding SpatiaLite in common locations. (`#135 `__) +- ``--load-extension`` option for ``sqlite-utils query`` for loading SQLite extensions. (:issue:`134`) +- New ``sqlite_utils.utils.find_spatialite()`` function for finding SpatiaLite in common locations. (:issue:`135`) .. _v2_15_1: 2.15.1 (2020-08-12) ------------------- -- Now available as a ``sdist`` package on PyPI in addition to a wheel. (`#133 `__) +- Now available as a ``sdist`` package on PyPI in addition to a wheel. (:issue:`133`) .. _v2_15: @@ -397,7 +397,7 @@ This release handles a bug where replacing rows in FTS tables could result in gr ----------------- - New ``db.enable_wal()`` and ``db.disable_wal()`` methods for enabling and disabling `Write-Ahead Logging `__ for a database file - see :ref:`python_api_wal` in the Python API documentation. -- Also ``sqlite-utils enable-wal file.db`` and ``sqlite-utils disable-wal file.db`` commands for doing the same thing on the command-line, see :ref:`WAL mode (CLI) `. (`#132 `__) +- Also ``sqlite-utils enable-wal file.db`` and ``sqlite-utils disable-wal file.db`` commands for doing the same thing on the command-line, see :ref:`WAL mode (CLI) `. (:issue:`132`) .. _v2_14_1: @@ -411,8 +411,8 @@ This release handles a bug where replacing rows in FTS tables could result in gr 2.14 (2020-08-01) ----------------- -- The :ref:`insert-files command ` can now read from standard input: ``cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg``. (`#127 `__) -- You can now specify a full-text search tokenizer using the new ``tokenize=`` parameter to :ref:`enable_fts() `. This means you can enable Porter stemming on a table by running ``db["articles"].enable_fts(["headline", "body"], tokenize="porter")``. (`#130 `__) +- The :ref:`insert-files command ` can now read from standard input: ``cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg``. (:issue:`127`) +- You can now specify a full-text search tokenizer using the new ``tokenize=`` parameter to :ref:`enable_fts() `. This means you can enable Porter stemming on a table by running ``db["articles"].enable_fts(["headline", "body"], tokenize="porter")``. (:issue:`130`) - You can also set a custom tokenizer using the :ref:`sqlite-utils enable-fts ` CLI command, via the new ``--tokenize`` option. .. _v2_13: @@ -420,7 +420,7 @@ This release handles a bug where replacing rows in FTS tables could result in gr 2.13 (2020-07-29) ----------------- -- ``memoryview`` and ``uuid.UUID`` objects are now supported. ``memoryview`` objects will be stored using ``BLOB`` and ``uuid.UUID`` objects will be stored using ``TEXT``. (`#128 `__) +- ``memoryview`` and ``uuid.UUID`` objects are now supported. ``memoryview`` objects will be stored using ``BLOB`` and ``uuid.UUID`` objects will be stored using ``TEXT``. (:issue:`128`) .. _v2_12: @@ -429,11 +429,11 @@ This release handles a bug where replacing rows in FTS tables could result in gr The theme of this release is better tools for working with binary data. The new ``insert-files`` command can be used to insert binary files directly into a database table, and other commands have been improved with better support for BLOB columns. -- ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (`#122 `__) -- ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (`#123 `__) -- JSON output now encodes BLOB values as special base64 objects - see :ref:`cli_query_json`. (`#125 `__) -- The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (`#126 `__) -- The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (`#124 `__) +- ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (:issue:`122`) +- ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (:issue:`123`) +- JSON output now encodes BLOB values as special base64 objects - see :ref:`cli_query_json`. (:issue:`125`) +- The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (:issue:`126`) +- The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (:issue:`124`) .. _v2_11: @@ -448,14 +448,14 @@ The theme of this release is better tools for working with binary data. The new 2.10.1 (2020-06-23) ------------------- -- Added documentation for the ``table.pks`` introspection property. (`#116 `__) +- Added documentation for the ``table.pks`` introspection property. (:issue:`116`) .. _v2_10: 2.10 (2020-06-12) ----------------- -- The ``sqlite-utils`` command now supports UPDATE/INSERT/DELETE in addition to SELECT. (`#115 `__) +- The ``sqlite-utils`` command now supports UPDATE/INSERT/DELETE in addition to SELECT. (:issue:`115`) .. _v2_9_1: @@ -469,77 +469,77 @@ The theme of this release is better tools for working with binary data. The new 2.9 (2020-05-10) ---------------- -- New ``sqlite-utils drop-table`` command, see :ref:`cli_drop_table`. (`#111 `__) +- New ``sqlite-utils drop-table`` command, see :ref:`cli_drop_table`. (:issue:`111`) - New ``sqlite-utils drop-view`` command, see :ref:`cli_drop_view`. -- Python ``decimal.Decimal`` objects are now stored as ``FLOAT``. (`#110 `__) +- Python ``decimal.Decimal`` objects are now stored as ``FLOAT``. (:issue:`110`) .. _v2_8: 2.8 (2020-05-03) ---------------- -- New ``sqlite-utils create-table`` command, see :ref:`cli_create_table`. (`#27 `__) -- New ``sqlite-utils create-view`` command, see :ref:`cli_create_view`. (`#107 `__) +- New ``sqlite-utils create-table`` command, see :ref:`cli_create_table`. (:issue:`27`) +- New ``sqlite-utils create-view`` command, see :ref:`cli_create_view`. (:issue:`107`) .. _v2_7.2: 2.7.2 (2020-05-02) ------------------ -- ``db.create_view(...)`` now has additional parameters ``ignore=True`` or ``replace=True``, see :ref:`python_api_create_view`. (`#106 `__) +- ``db.create_view(...)`` now has additional parameters ``ignore=True`` or ``replace=True``, see :ref:`python_api_create_view`. (:issue:`106`) .. _v2_7.1: 2.7.1 (2020-05-01) ------------------ -- New ``sqlite-utils views my.db`` command for listing views in a database, see :ref:`cli_views`. (`#105 `__) -- ``sqlite-utils tables`` (and ``views``) has a new ``--schema`` option which outputs the table/view schema, see :ref:`cli_tables`. (`#104 `__) -- Nested structures containing invalid JSON values (e.g. Python bytestrings) are now serialized using ``repr()`` instead of throwing an error. (`#102 `__) +- New ``sqlite-utils views my.db`` command for listing views in a database, see :ref:`cli_views`. (:issue:`105`) +- ``sqlite-utils tables`` (and ``views``) has a new ``--schema`` option which outputs the table/view schema, see :ref:`cli_tables`. (:issue:`104`) +- Nested structures containing invalid JSON values (e.g. Python bytestrings) are now serialized using ``repr()`` instead of throwing an error. (:issue:`102`) .. _v2_7: 2.7 (2020-04-17) ---------------- -- New ``columns=`` argument for the ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()`` methods, for over-riding the auto-detected types for columns and specifying additional columns that should be added when the table is created. See :ref:`python_api_custom_columns`. (`#100 `__) +- New ``columns=`` argument for the ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()`` methods, for over-riding the auto-detected types for columns and specifying additional columns that should be added when the table is created. See :ref:`python_api_custom_columns`. (:issue:`100`) .. _v2_6: 2.6 (2020-04-15) ---------------- -- New ``table.rows_where(..., order_by="age desc")`` argument, see :ref:`python_api_rows`. (`#76 `__) +- New ``table.rows_where(..., order_by="age desc")`` argument, see :ref:`python_api_rows`. (:issue:`76`) .. _v2_5: 2.5 (2020-04-12) ---------------- -- Panda's Timestamp is now stored as a SQLite TEXT column. Thanks, b0b5h4rp13! (`#96 `__) -- ``table.last_pk`` is now only available for inserts or upserts of a single record. (`#98 `__) -- New ``Database(filepath, recreate=True)`` parameter for deleting and recreating the database. (`#97 `__) +- Panda's Timestamp is now stored as a SQLite TEXT column. Thanks, b0b5h4rp13! (:issue:`96`) +- ``table.last_pk`` is now only available for inserts or upserts of a single record. (:issue:`98`) +- New ``Database(filepath, recreate=True)`` parameter for deleting and recreating the database. (:issue:`97`) .. _v2_4_4: 2.4.4 (2020-03-23) ------------------ -- Fixed bug where columns with only null values were not correctly created. (`#95 `__) +- Fixed bug where columns with only null values were not correctly created. (:issue:`95`) .. _v2_4_3: 2.4.3 (2020-03-23) ------------------ -- Column type suggestion code is no longer confused by null values. (`#94 `__) +- Column type suggestion code is no longer confused by null values. (:issue:`94`) .. _v2_4_2: 2.4.2 (2020-03-14) ------------------ -- ``table.column_dicts`` now works with all column types - previously it would throw errors on types other than ``TEXT``, ``BLOB``, ``INTEGER`` or ``FLOAT``. (`#92 `__) +- ``table.column_dicts`` now works with all column types - previously it would throw errors on types other than ``TEXT``, ``BLOB``, ``INTEGER`` or ``FLOAT``. (:issue:`92`) - Documentation for ``NotFoundError`` thrown by ``table.get(pk)`` - see :ref:`python_api_get`. .. _v2_4_1: @@ -547,45 +547,45 @@ The theme of this release is better tools for working with binary data. The new 2.4.1 (2020-03-01) ------------------ -- ``table.enable_fts()`` now works with columns that contain spaces. (`#90 `__) +- ``table.enable_fts()`` now works with columns that contain spaces. (:issue:`90`) .. _v2_4: 2.4 (2020-02-26) ---------------- -- ``table.disable_fts()`` can now be used to remove FTS tables and triggers that were created using ``table.enable_fts(...)``. (`#88 `__) -- The ``sqlite-utils disable-fts`` command can be used to remove FTS tables and triggers from the command-line. (`#88 `__) -- Trying to create table columns with square braces ([ or ]) in the name now raises an error. (`#86 `__) -- Subclasses of ``dict``, ``list`` and ``tuple`` are now detected as needing a JSON column. (`#87 `__) +- ``table.disable_fts()`` can now be used to remove FTS tables and triggers that were created using ``table.enable_fts(...)``. (:issue:`88`) +- The ``sqlite-utils disable-fts`` command can be used to remove FTS tables and triggers from the command-line. (:issue:`88`) +- Trying to create table columns with square braces ([ or ]) in the name now raises an error. (:issue:`86`) +- Subclasses of ``dict``, ``list`` and ``tuple`` are now detected as needing a JSON column. (:issue:`87`) .. _v2_3_1: 2.3.1 (2020-02-10) ------------------ -``table.create_index()`` now works for columns that contain spaces. (`#85 `__) +``table.create_index()`` now works for columns that contain spaces. (:issue:`85`) .. _v2_3: 2.3 (2020-02-08) ---------------- -``table.exists()`` is now a method, not a property. This was not a documented part of the API before so I'm considering this a non-breaking change. (`#83 `__) +``table.exists()`` is now a method, not a property. This was not a documented part of the API before so I'm considering this a non-breaking change. (:issue:`83`) .. _v2_2_1: 2.2.1 (2020-02-06) ------------------ -Fixed a bug where ``.upsert(..., hash_id="pk")`` threw an error (`#84 `__). +Fixed a bug where ``.upsert(..., hash_id="pk")`` threw an error (:issue:`84`). .. _v2_2: 2.2 (2020-02-01) ---------------- -New feature: ``sqlite_utils.suggest_column_types([records])`` returns the suggested column types for a list of records. See :ref:`python_api_suggest_column_types`. (`#81 `__). +New feature: ``sqlite_utils.suggest_column_types([records])`` returns the suggested column types for a list of records. See :ref:`python_api_suggest_column_types`. (:issue:`81`). This replaces the undocumented ``table.detect_column_types()`` method. @@ -601,7 +601,7 @@ New feature: ``conversions={...}`` can be passed to the ``.insert()`` family of 2.0.1 (2020-01-05) ------------------ -The ``.upsert()`` and ``.upsert_all()`` methods now raise a ``sqlite_utils.db.PrimaryKeyRequired`` exception if you call them without specifying the primary key column using ``pk=`` (`#73 `__). +The ``.upsert()`` and ``.upsert_all()`` methods now raise a ``sqlite_utils.db.PrimaryKeyRequired`` exception if you call them without specifying the primary key column using ``pk=`` (:issue:`73`). .. _v2: @@ -623,14 +623,14 @@ For full background on this change, see `issue #66 `__) +- Fixed error thrown when ``.insert_all()`` and ``.upsert_all()`` were called with empty lists (:issue:`52`) .. _v1_12: 1.12 (2019-11-04) ----------------- -Python library utilities for deleting records (`#62 `__) +Python library utilities for deleting records (:issue:`62`) - ``db["tablename"].delete(4)`` to delete by primary key, see :ref:`python_api_delete` - ``db["tablename"].delete_where("id > ?", [3])`` to delete by a where clause, see :ref:`python_api_delete_where` @@ -644,14 +644,14 @@ Option to create triggers to automatically keep FTS tables up-to-date with newly - ``sqlite-utils enable-fts ... --create-triggers`` - see :ref:`Configuring full-text search using the CLI ` - ``db["tablename"].enable_fts(..., create_triggers=True)`` - see :ref:`Configuring full-text search using the Python library ` -- Support for introspecting triggers for a database or table - see :ref:`python_api_introspection` (`#59 `__) +- Support for introspecting triggers for a database or table - see :ref:`python_api_introspection` (:issue:`59`) .. _v1_10: 1.10 (2019-08-23) ----------------- -Ability to introspect and run queries against views (`#54 `__) +Ability to introspect and run queries against views (:issue:`54`) - ``db.view_names()`` method and and ``db.views`` property - Separate ``View`` and ``Table`` classes, both subclassing new ``Queryable`` class @@ -664,21 +664,21 @@ See :ref:`python_api_views`. 1.9 (2019-08-04) ---------------- -- ``table.m2m(...)`` method for creating many-to-many relationships: :ref:`python_api_m2m` (`#23 `__) +- ``table.m2m(...)`` method for creating many-to-many relationships: :ref:`python_api_m2m` (:issue:`23`) .. _v1_8: 1.8 (2019-07-28) ---------------- -- ``table.update(pk, values)`` method: :ref:`python_api_update` (`#35 `__) +- ``table.update(pk, values)`` method: :ref:`python_api_update` (:issue:`35`) .. _v1_7_1: 1.7.1 (2019-07-28) ------------------ -- Fixed bug where inserting records with 11 columns in a batch of 100 triggered a "too many SQL variables" error (`#50 `__) +- Fixed bug where inserting records with 11 columns in a batch of 100 triggered a "too many SQL variables" error (:issue:`50`) - Documentation and tests for ``table.drop()`` method: :ref:`python_api_drop` .. _v1_7: @@ -688,8 +688,8 @@ See :ref:`python_api_views`. Support for lookup tables. -- New ``table.lookup({...})`` utility method for building and querying lookup tables - see :ref:`python_api_lookup_tables` (`#44 `__) -- New ``extracts=`` table configuration option, see :ref:`python_api_extracts` (`#46 `__) +- New ``table.lookup({...})`` utility method for building and querying lookup tables - see :ref:`python_api_lookup_tables` (:issue:`44`) +- New ``extracts=`` table configuration option, see :ref:`python_api_extracts` (:issue:`46`) - Use `pysqlite3 `__ if it is available, otherwise use ``sqlite3`` from the standard library - Table options can now be passed to the new ``db.table(name, **options)`` factory function in addition to being passed to ``insert_all(records, **options)`` and friends - see :ref:`python_api_table_configuration` - In-memory databases can now be created using ``db = Database(memory=True)`` @@ -699,19 +699,19 @@ Support for lookup tables. 1.6 (2019-07-18) ---------------- -- ``sqlite-utils insert`` can now accept TSV data via the new ``--tsv`` option (`#41 `__) +- ``sqlite-utils insert`` can now accept TSV data via the new ``--tsv`` option (:issue:`41`) .. _v1_5: 1.5 (2019-07-14) ---------------- -- Support for compound primary keys (`#36 `__) +- Support for compound primary keys (:issue:`36`) - Configure these using the CLI tool by passing ``--pk`` multiple times - In Python, pass a tuple of columns to the ``pk=(..., ...)`` argument: :ref:`python_api_compound_primary_keys` -- New ``table.get()`` method for retrieving a record by its primary key: :ref:`python_api_get` (`#39 `__) +- New ``table.get()`` method for retrieving a record by its primary key: :ref:`python_api_get` (:issue:`39`) .. _v1_4_1: @@ -725,14 +725,14 @@ Support for lookup tables. 1.4 (2019-06-30) ---------------- -- Added ``sqlite-utils index-foreign-keys`` command (:ref:`docs `) and ``db.index_foreign_keys()`` method (:ref:`docs `) (`#33 `__) +- Added ``sqlite-utils index-foreign-keys`` command (:ref:`docs `) and ``db.index_foreign_keys()`` method (:ref:`docs `) (:issue:`33`) .. _v1_3: 1.3 (2019-06-28) ---------------- -- New mechanism for adding multiple foreign key constraints at once: :ref:`db.add_foreign_keys() documentation ` (`#31 `__) +- New mechanism for adding multiple foreign key constraints at once: :ref:`db.add_foreign_keys() documentation ` (:issue:`31`) .. _v1_2_2: @@ -746,15 +746,15 @@ Support for lookup tables. 1.2.1 (2019-06-20) ------------------ -- Check the column exists before attempting to add a foreign key (`#29 `__) +- Check the column exists before attempting to add a foreign key (:issue:`29`) .. _v1_2: 1.2 (2019-06-12) ---------------- -- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by introspecting the database. See :ref:`python_api_add_foreign_key` for details. (`#25 `__) -- Ability to set ``NOT NULL`` constraints and ``DEFAULT`` values when creating tables (`#24 `__). Documentation: :ref:`Setting defaults and not null constraints (Python API) `, :ref:`Setting defaults and not null constraints (CLI) ` +- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by introspecting the database. See :ref:`python_api_add_foreign_key` for details. (:issue:`25`) +- Ability to set ``NOT NULL`` constraints and ``DEFAULT`` values when creating tables (:issue:`24`). Documentation: :ref:`Setting defaults and not null constraints (Python API) `, :ref:`Setting defaults and not null constraints (CLI) ` - Support for ``not_null_default=X`` / ``--not-null-default`` for setting a ``NOT NULL DEFAULT 'x'`` when adding a new column. Documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` .. _v1_1: @@ -762,8 +762,8 @@ Support for lookup tables. 1.1 (2019-05-28) ---------------- -- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key already exists (`#21 `__) - documentation: :ref:`Inserting data (Python API) `, :ref:`Inserting data (CLI) ` -- Ability to add a column that is a foreign key reference using ``fk=...`` / ``--fk`` (`#16 `__) - documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` +- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key already exists (:issue:`21`) - documentation: :ref:`Inserting data (Python API) `, :ref:`Inserting data (CLI) ` +- Ability to add a column that is a foreign key reference using ``fk=...`` / ``--fk`` (:issue:`16`) - documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` .. _v1_0_1: diff --git a/docs/conf.py b/docs/conf.py index 929a41e..b4c7f44 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,11 @@ from subprocess import Popen, PIPE # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = ["sphinx.ext.extlinks"] + +extlinks = { + "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"), +} # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] From ec50e5eebc502c85448a2d3db74985c3b0c630c5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 14:53:44 -0700 Subject: [PATCH 0405/1004] sqlite3.enable_callback_tracebacks(True) in docs, closes #300 --- docs/python-api.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index c372a62..7ff3727 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2323,6 +2323,18 @@ If you want to deliberately replace the registered function with a new implement def reverse_string(s): return s[::-1] +Exceptions that occur inside a user-defined function default to returning the following error:: + + Unexpected error: user-defined function raised exception + +You can cause ``sqlite3`` to return more useful errors, including the traceback from the custom function, by executing the following before your custom fuctions are executed: + +.. code-block:: python + + from sqlite_utils.utils import sqlite3 + + sqlite3.enable_callback_tracebacks(True) + .. _python_api_quote: Quoting strings for use in SQL From d83f624a3124ef9489014fc1f023646f082fdc55 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 15:39:16 -0700 Subject: [PATCH 0406/1004] Clarified documentation for convert recipes, refs #251 --- docs/cli.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 3802963..125dc87 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -986,6 +986,11 @@ Various built-in recipe functions are available for common operations. These are These recipes can be used in the code passed to ``sqlite-utils convert`` like this:: + $ sqlite-utils convert my.db mytable mycolumn \ + 'r.jsonsplit(value)' + +To use any of the documented parameters, do this:: + $ sqlite-utils convert my.db mytable mycolumn \ 'r.jsonsplit(value, delimiter=":")' From f7c8c78cd0916bd9bca043a665d7c1ec7ae5da7d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 23:05:45 -0700 Subject: [PATCH 0407/1004] Fixed typo: commad --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index aa3234d..f4f132e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -11,7 +11,7 @@ Using Homebrew ============== -The :ref:`sqlite-utils commad-line tool ` can be installed on macOS using Homebrew:: +The :ref:`sqlite-utils command-line tool ` can be installed on macOS using Homebrew:: brew install sqlite-utils From 991cf56ae2840aaefda2af828a5c40396d2506ca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Aug 2021 09:48:37 -0700 Subject: [PATCH 0408/1004] Check spelling with codespell, closes #307 --- .github/workflows/spellcheck.yml | 25 +++++++++++++++++++++++++ docs/cli.rst | 4 ++-- docs/codespell-ignore-words.txt | 1 + docs/python-api.rst | 4 ++-- setup.py | 2 +- 5 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/spellcheck.yml create mode 100644 docs/codespell-ignore-words.txt diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml new file mode 100644 index 0000000..d498e17 --- /dev/null +++ b/.github/workflows/spellcheck.yml @@ -0,0 +1,25 @@ +name: Check spelling in documentation + +on: [push, pull_request] + +jobs: + spellcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install dependencies + run: | + pip install -e '.[docs]' + - name: Check spelling + run: codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt diff --git a/docs/cli.rst b/docs/cli.rst index 125dc87..e766d73 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -361,7 +361,7 @@ Passing ``--save other.db`` will instead use that SQL to populate a new database % sqlite-utils memory dogs.csv --save dogs.db -These features are mainly intented as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`. +These features are mainly intended as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`. .. _cli_rows: @@ -617,7 +617,7 @@ The ``_analyze_tables_`` table has the following schema:: PRIMARY KEY ([table], [column]) ); -The ``most_common`` and ``least_common`` columns will contain nested JSON arrays of the most commond and least common values that look like this:: +The ``most_common`` and ``least_common`` columns will contain nested JSON arrays of the most common and least common values that look like this:: [ ["Del Libertador, Av", 5068], diff --git a/docs/codespell-ignore-words.txt b/docs/codespell-ignore-words.txt new file mode 100644 index 0000000..a625cde --- /dev/null +++ b/docs/codespell-ignore-words.txt @@ -0,0 +1 @@ +AddWordsToIgnoreHere diff --git a/docs/python-api.rst b/docs/python-api.rst index 7ff3727..70baf0c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1096,7 +1096,7 @@ Here's an example of this mechanism in action: ]) db["books"].add_foreign_key("author_id", "authors", "id") -The ``table.add_foreign_key(column, other_table, other_column)`` method takes the name of the column, the table that is being referenced and the key column within that other table. If you ommit the ``other_column`` argument the primary key from that table will be used automatically. If you omit the ``other_table`` argument the table will be guessed based on some simple rules: +The ``table.add_foreign_key(column, other_table, other_column)`` method takes the name of the column, the table that is being referenced and the key column within that other table. If you omit the ``other_column`` argument the primary key from that table will be used automatically. If you omit the ``other_table`` argument the table will be guessed based on some simple rules: - If the column is of format ``author_id``, look for tables called ``author`` or ``authors`` - If the column does not end in ``_id``, try looking for a table with the exact name of the column or that name with an added ``s`` @@ -2327,7 +2327,7 @@ Exceptions that occur inside a user-defined function default to returning the fo Unexpected error: user-defined function raised exception -You can cause ``sqlite3`` to return more useful errors, including the traceback from the custom function, by executing the following before your custom fuctions are executed: +You can cause ``sqlite3`` to return more useful errors, including the traceback from the custom function, by executing the following before your custom functions are executed: .. code-block:: python diff --git a/setup.py b/setup.py index 8aa9aeb..5b13427 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "test": ["pytest", "black", "hypothesis"], - "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], + "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], "mypy": ["mypy", "types-click", "types-tabulate", "types-python-dateutil"], "flake8": ["flake8"], }, From cff6afcc43bb96a1e028aca69b67f7d758820150 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Aug 2021 10:06:08 -0700 Subject: [PATCH 0409/1004] Run codespell against source code too, refs #307 --- .github/workflows/spellcheck.yml | 4 +++- docs/codespell-ignore-words.txt | 2 +- sqlite_utils/cli.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index d498e17..8a86cd2 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -22,4 +22,6 @@ jobs: run: | pip install -e '.[docs]' - name: Check spelling - run: codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + run: | + codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + codespell sqlite_utils --ignore-words docs/codespell-ignore-words.txt diff --git a/docs/codespell-ignore-words.txt b/docs/codespell-ignore-words.txt index a625cde..f8418c4 100644 --- a/docs/codespell-ignore-words.txt +++ b/docs/codespell-ignore-words.txt @@ -1 +1 @@ -AddWordsToIgnoreHere +doub diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index bd063f6..c4f8501 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -44,7 +44,7 @@ It's often worth trying: --encoding=latin-1 """.strip() -# Increase CSV field size limit to maximim possible +# Increase CSV field size limit to maximum possible # https://stackoverflow.com/a/15063941 field_size_limit = sys.maxsize From cc90745f4e8bb1ac57d8ee973863cfe00c2e4fe5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 4 Aug 2021 13:34:30 -0700 Subject: [PATCH 0410/1004] Started a Jupyter notebook tutorial, refs #308 --- docs/tutorial.ipynb | 1053 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1053 insertions(+) create mode 100644 docs/tutorial.ipynb diff --git a/docs/tutorial.ipynb b/docs/tutorial.ipynb new file mode 100644 index 0000000..aa22461 --- /dev/null +++ b/docs/tutorial.ipynb @@ -0,0 +1,1053 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "27ae18ec", + "metadata": {}, + "source": [ + "# The sqlite-utils tutorial\n", + "\n", + "[sqlite-utils](https://sqlite-utils.datasette.io/en/stable/python-api.html) is a Python library (and [command-line tool](https://sqlite-utils.datasette.io/en/stable/cli.html) for quickly creating and manipulating SQLite database files.\n", + "\n", + "This tutorial will show you how to use the Python library to manipulate data.\n", + "\n", + "## Installation\n", + "\n", + "To install the library, run:\n", + "\n", + " pip install sqlite-utils\n", + "\n", + "You can run this in a Jupyter notebook cell by executing:\n", + "\n", + " %pip install sqlite-utils\n", + " \n", + "Or use `pip install -U sqlite-utils` to ensure you have upgraded to the most recent version." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "bddee0d2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: sqlite_utils in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (3.14)\n", + "Requirement already satisfied: click-default-group in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (1.2.2)\n", + "Requirement already satisfied: sqlite-fts4 in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (1.0.1)\n", + "Requirement already satisfied: click in /Users/simon/Library/Python/3.9/lib/python/site-packages (from sqlite_utils) (7.1.2)\n", + "Requirement already satisfied: tabulate in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (0.8.7)\n", + "Requirement already satisfied: dateutils in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from sqlite_utils) (0.6.12)\n", + "Requirement already satisfied: python-dateutil in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from dateutils->sqlite_utils) (2.8.1)\n", + "Requirement already satisfied: pytz in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from dateutils->sqlite_utils) (2021.1)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from python-dateutil->dateutils->sqlite_utils) (1.16.0)\n", + "\u001b[33mWARNING: You are using pip version 21.1.1; however, version 21.2.2 is available.\n", + "You should consider upgrading via the '/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/bin/python3.9 -m pip install --upgrade pip' command.\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -U sqlite_utils" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "050e85a8", + "metadata": {}, + "outputs": [], + "source": [ + "import sqlite_utils" + ] + }, + { + "cell_type": "markdown", + "id": "348bcbfc", + "metadata": {}, + "source": [ + "You can use the library with a database file on disk by running:\n", + "\n", + " db = sqlite_utils.Database(\"path/to/my/database.db\")\n", + "\n", + "In this tutorial we will use an in-memory database. This is a quick way to try out new things, though you should note that when you close the notebook the data store in the in-memory database will be lost." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4b2aee7e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + ">" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db = sqlite_utils.Database(memory=True)\n", + "db" + ] + }, + { + "cell_type": "markdown", + "id": "1598ab43", + "metadata": {}, + "source": [ + "## Creating a table\n", + "\n", + "We are going to create a new table in our database called `creatures` by passing in a Python list of dictionaries.\n", + "\n", + "`db[name_of_table]` will access a database table object with that name.\n", + "\n", + "Inserting data into that table will create it if it does not already exist." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "4a0ac420", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db[\"creatures\"].insert_all([{\n", + " \"name\": \"Cleo\",\n", + " \"species\": \"dog\",\n", + " \"age\": 6\n", + "}, {\n", + " \"name\": \"Lila\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}, {\n", + " \"name\": \"Bants\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}])" + ] + }, + { + "cell_type": "markdown", + "id": "049d110b", + "metadata": {}, + "source": [ + "Let's grab a `table` reference to the new creatures table:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "8d84ad9c", + "metadata": {}, + "outputs": [], + "source": [ + "table = db[\"creatures\"]" + ] + }, + { + "cell_type": "markdown", + "id": "ffe45750", + "metadata": {}, + "source": [ + "`sqlite-utils` automatically creates a table schema that matches the keys and data types of the dictionaries that were passed to `.insert_all()`.\n", + "\n", + "We can see that schema using `table.schema`:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "136cee1e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE [creatures] (\n", + " [name] TEXT,\n", + " [species] TEXT,\n", + " [age] FLOAT\n", + ")\n" + ] + } + ], + "source": [ + "print(table.schema)" + ] + }, + { + "cell_type": "markdown", + "id": "9e5c3ae9", + "metadata": {}, + "source": [ + "## Accessing data\n", + "\n", + "The `table.rows` property lets us loop through the rows in the table, returning each one as a Python dictionary:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "f812914d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'name': 'Cleo', 'species': 'dog', 'age': 6.0}\n", + "{'name': 'Lila', 'species': 'chicken', 'age': 0.8}\n", + "{'name': 'Bants', 'species': 'chicken', 'age': 0.8}\n" + ] + } + ], + "source": [ + "for row in table.rows:\n", + " print(row)" + ] + }, + { + "cell_type": "markdown", + "id": "60bc6b2c", + "metadata": {}, + "source": [ + "The `db.query(sql)` method can be used to execute SQL queries and return the results as dictionaries:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "eaadd85f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'name': 'Bants', 'species': 'chicken', 'age': 0.8}]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select * from creatures\"))" + ] + }, + { + "cell_type": "markdown", + "id": "6614467b", + "metadata": {}, + "source": [ + "Or in a loop:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "88fdd52e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cleo is a dog\n", + "Lila is a chicken\n", + "Bants is a chicken\n" + ] + } + ], + "source": [ + "for row in db.query(\"select name, species from creatures\"):\n", + " print(f'{row[\"name\"]} is a {row[\"species\"]}')" + ] + }, + { + "cell_type": "markdown", + "id": "b81c031c", + "metadata": {}, + "source": [ + "### SQL parameters\n", + "\n", + "You can run a parameterized query using `?` as placeholders and passing a list of variables. The variables you pass will be correctly quoted, protecting your code from SQL injection vulnerabilities." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "267035d9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'Cleo', 'species': 'dog', 'age': 6.0}]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select * from creatures where age > ?\", [1.0]))" + ] + }, + { + "cell_type": "markdown", + "id": "87cb301b", + "metadata": {}, + "source": [ + "As an alternative to question marks we can use `:name` parameters and feed in the values using a dictionary:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "83be9a80", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'name': 'Bants', 'species': 'chicken', 'age': 0.8}]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select * from creatures where species = :species\", {\"species\": \"chicken\"}))" + ] + }, + { + "cell_type": "markdown", + "id": "5e5179cc", + "metadata": {}, + "source": [ + "### Primary keys\n", + "\n", + "When we created this table we did not specify a primary key. SQLite automatically creates a primary key called `rowid` if no other primary key is defined.\n", + "\n", + "We can run `select rowid, * from creatures` to see this hidden primary key:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "c9d963df", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'rowid': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'rowid': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'rowid': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8}]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select rowid, * from creatures\"))" + ] + }, + { + "cell_type": "markdown", + "id": "0f87cdfb", + "metadata": {}, + "source": [ + "We can also see that using `table.pks_and_rows_where()`:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "d365e405", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 {'rowid': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0}\n", + "2 {'rowid': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8}\n", + "3 {'rowid': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8}\n" + ] + } + ], + "source": [ + "for pk, row in table.pks_and_rows_where():\n", + " print(pk, row)" + ] + }, + { + "cell_type": "markdown", + "id": "5b0e9b74", + "metadata": {}, + "source": [ + "Let's recreate the table with our own primary key, which we will call `id`.\n", + "\n", + "`table.drop()` drops the table:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "568a0e29", + "metadata": {}, + "outputs": [], + "source": [ + "table.drop()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "13ebd3ab", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table" + ] + }, + { + "cell_type": "markdown", + "id": "522aa6d0", + "metadata": {}, + "source": [ + "We can see a list of tables in the database using `db.tables`:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "f3e62678", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db.tables" + ] + }, + { + "cell_type": "markdown", + "id": "6b80d523", + "metadata": {}, + "source": [ + "We'll create the table again, this time with an `id` column.\n", + "\n", + "We use `pk=\"id\"` to specify that the `id` column should be treated as the primary key for the table:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "c9ee8b9f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db[\"creatures\"].insert_all([{\n", + " \"id\": 1,\n", + " \"name\": \"Cleo\",\n", + " \"species\": \"dog\",\n", + " \"age\": 6\n", + "}, {\n", + " \"id\": 2,\n", + " \"name\": \"Lila\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}, {\n", + " \"id\": 3,\n", + " \"name\": \"Bants\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}], pk=\"id\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "523e01ab", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE [creatures] (\n", + " [id] INTEGER PRIMARY KEY,\n", + " [name] TEXT,\n", + " [species] TEXT,\n", + " [age] FLOAT\n", + ")\n" + ] + } + ], + "source": [ + "print(table.schema)" + ] + }, + { + "cell_type": "markdown", + "id": "811bea70", + "metadata": {}, + "source": [ + "## Inserting more records\n", + "\n", + "We can call `.insert_all()` again to insert more records. Let's add two more chickens." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "716df161", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.insert_all([{\n", + " \"id\": 4,\n", + " \"name\": \"Azi\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}, {\n", + " \"id\": 5,\n", + " \"name\": \"Snowy\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.9,\n", + "}], pk=\"id\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "4b1b2476", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'id': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 4, 'name': 'Azi', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 5, 'name': 'Snowy', 'species': 'chicken', 'age': 0.9}]" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(table.rows)" + ] + }, + { + "cell_type": "markdown", + "id": "2af4ae75", + "metadata": {}, + "source": [ + "Since the `id` column is an integer primary key, we can insert a record without specifying an ID and one will be automatically added.\n", + "\n", + "Since we are only adding one record we will use `.insert()` instead of `.insert_all()`." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "246c6dd5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.insert({\"name\": \"Blue\", \"species\": \"chicken\", \"age\": 0.9})" + ] + }, + { + "cell_type": "markdown", + "id": "d7c28e4d", + "metadata": {}, + "source": [ + "We can use `table.last_pk` to see the ID of the record we just added." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "de012e1e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "6" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.last_pk" + ] + }, + { + "cell_type": "markdown", + "id": "c38edaf4", + "metadata": {}, + "source": [ + "Here's the full list of rows again:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "7c27075e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'id': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 4, 'name': 'Azi', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 5, 'name': 'Snowy', 'species': 'chicken', 'age': 0.9},\n", + " {'id': 6, 'name': 'Blue', 'species': 'chicken', 'age': 0.9}]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(table.rows)" + ] + }, + { + "cell_type": "markdown", + "id": "64931bd0", + "metadata": {}, + "source": [ + "If you try to add a new record with an existing ID, you will get an `IntegrityError`:" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "36327794", + "metadata": {}, + "outputs": [ + { + "ename": "IntegrityError", + "evalue": "UNIQUE constraint failed: creatures.id", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mIntegrityError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"id\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m6\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"name\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"Red\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"species\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"chicken\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"age\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m0.9\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/sqlite_utils/db.py\u001b[0m in \u001b[0;36minsert\u001b[0;34m(self, record, pk, foreign_keys, column_order, not_null, defaults, hash_id, alter, ignore, replace, extracts, conversions, columns)\u001b[0m\n\u001b[1;32m 2027\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mDEFAULT\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2028\u001b[0m ):\n\u001b[0;32m-> 2029\u001b[0;31m return self.insert_all(\n\u001b[0m\u001b[1;32m 2030\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2031\u001b[0m \u001b[0mpk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpk\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/sqlite_utils/db.py\u001b[0m in \u001b[0;36minsert_all\u001b[0;34m(self, records, pk, foreign_keys, column_order, not_null, defaults, batch_size, hash_id, alter, ignore, replace, truncate, extracts, conversions, columns, upsert)\u001b[0m\n\u001b[1;32m 2143\u001b[0m \u001b[0mfirst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2145\u001b[0;31m self.insert_chunk(\n\u001b[0m\u001b[1;32m 2146\u001b[0m \u001b[0malter\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2147\u001b[0m \u001b[0mextracts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/sqlite_utils/db.py\u001b[0m in \u001b[0;36minsert_chunk\u001b[0;34m(self, alter, extracts, chunk, all_columns, hash_id, upsert, pk, conversions, num_records_processed, replace, ignore)\u001b[0m\n\u001b[1;32m 1955\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mqueries_and_params\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1956\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1957\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1958\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mOperationalError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1959\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0malter\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\" column\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/sqlite_utils/db.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, sql, parameters)\u001b[0m\n\u001b[1;32m 255\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tracer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 256\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mparameters\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 257\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 258\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 259\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mIntegrityError\u001b[0m: UNIQUE constraint failed: creatures.id" + ] + } + ], + "source": [ + "table.insert({\"id\": 6, \"name\": \"Red\", \"species\": \"chicken\", \"age\": 0.9})" + ] + }, + { + "cell_type": "markdown", + "id": "2e00692f", + "metadata": {}, + "source": [ + "You can use `replace=True` to replace the matching record with a new one:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "2be75589", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.insert({\"id\": 6, \"name\": \"Red\", \"species\": \"chicken\", \"age\": 0.9}, replace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "83281675", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'id': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 4, 'name': 'Azi', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 5, 'name': 'Snowy', 'species': 'chicken', 'age': 0.9},\n", + " {'id': 6, 'name': 'Red', 'species': 'chicken', 'age': 0.9}]" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(table.rows)" + ] + }, + { + "cell_type": "markdown", + "id": "d7122b76", + "metadata": {}, + "source": [ + "## Updating a record\n", + "\n", + "We will rename that row back to `Blue`, this time using the `table.update(pk, updates)` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "43df156d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.update(6, {\"name\": \"Blue\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "0b8f8422", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 6, 'name': 'Blue', 'species': 'chicken', 'age': 0.9}]" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select * from creatures where id = ?\", [6]))" + ] + }, + { + "cell_type": "markdown", + "id": "58142b86", + "metadata": {}, + "source": [ + "## Extracting one of the columns into another table\n", + "\n", + "Our current table has a `species` column with a string in it - let's pull that out into a separate table.\n", + "\n", + "We can do that using the [table.extract() method](https://sqlite-utils.datasette.io/en/stable/python-api.html#extracting-columns-into-a-separate-table)." + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "6ab69111", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.extract(\"species\")" + ] + }, + { + "cell_type": "markdown", + "id": "dca327b2", + "metadata": {}, + "source": [ + "We now have a new table called `species`, which we can see using the `db.tables` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "76e95b36", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[
,
]" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db.tables" + ] + }, + { + "cell_type": "markdown", + "id": "5ea43bf5", + "metadata": {}, + "source": [ + "Our creatures table has been modified - instead of a `species` column it now has `species_id` which is a foreign key to the new table:" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "c0438bff", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE \"creatures\" (\n", + " [id] INTEGER PRIMARY KEY,\n", + " [name] TEXT,\n", + " [species_id] INTEGER,\n", + " [age] FLOAT,\n", + " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n", + ")\n", + "[{'id': 1, 'name': 'Cleo', 'species_id': 1, 'age': 6.0}, {'id': 2, 'name': 'Lila', 'species_id': 2, 'age': 0.8}, {'id': 3, 'name': 'Bants', 'species_id': 2, 'age': 0.8}, {'id': 4, 'name': 'Azi', 'species_id': 2, 'age': 0.8}, {'id': 5, 'name': 'Snowy', 'species_id': 2, 'age': 0.9}, {'id': 6, 'name': 'Blue', 'species_id': 2, 'age': 0.9}]\n" + ] + } + ], + "source": [ + "print(db[\"creatures\"].schema)\n", + "print(list(db[\"creatures\"].rows))" + ] + }, + { + "cell_type": "markdown", + "id": "0452c201", + "metadata": {}, + "source": [ + "The new `species` table has been created and populated too:" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "5d38c3a8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE [species] (\n", + " [id] INTEGER PRIMARY KEY,\n", + " [species] TEXT\n", + ")\n", + "[{'id': 1, 'species': 'dog'}, {'id': 2, 'species': 'chicken'}]\n" + ] + } + ], + "source": [ + "print(db[\"species\"].schema)\n", + "print(list(db[\"species\"].rows))" + ] + }, + { + "cell_type": "markdown", + "id": "a0312d1e", + "metadata": {}, + "source": [ + "We can use a join SQL query to combine data from these two tables:" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "6734ed5d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 'Cleo', 'age': 6.0, 'species_id': 1, 'species': 'dog'},\n", + " {'id': 2, 'name': 'Lila', 'age': 0.8, 'species_id': 2, 'species': 'chicken'},\n", + " {'id': 3, 'name': 'Bants', 'age': 0.8, 'species_id': 2, 'species': 'chicken'},\n", + " {'id': 4, 'name': 'Azi', 'age': 0.8, 'species_id': 2, 'species': 'chicken'},\n", + " {'id': 5, 'name': 'Snowy', 'age': 0.9, 'species_id': 2, 'species': 'chicken'},\n", + " {'id': 6, 'name': 'Blue', 'age': 0.9, 'species_id': 2, 'species': 'chicken'}]" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"\"\"\n", + " select\n", + " creatures.id,\n", + " creatures.name,\n", + " creatures.age,\n", + " species.id as species_id,\n", + " species.species\n", + " from creatures\n", + " join species on creatures.species_id = species.id\n", + "\"\"\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c4802ac", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From f67327abf0a9f018e1764660e190c5bbf9556ec2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 14:44:03 -0700 Subject: [PATCH 0411/1004] sqlite-utils insert --flatten option, closes #310 --- docs/cli.rst | 43 +++++++++++++++++++++++++++++++++++ sqlite_utils/cli.py | 19 ++++++++++++++++ tests/test_cli.py | 55 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index e766d73..28bd3f7 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -722,6 +722,49 @@ This also means you pipe ``sqlite-utils`` together to easily create a new SQLite 207368,920 Kirkham St,37.760210314285,-122.47073935813 188702,1501 Evans Ave,37.7422086702947,-122.387293152263 +.. _cli_inserting_data_flatten: + +Flattening nested JSON objects +------------------------------ + +``sqlite-utils insert`` expects incoming data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table. + +If your data is nested you can use the `--flatten` object to create columns that are derived from the nested data. + +Consider this example document, in a file called ``log.json``:: + + { + "httpRequest": { + "latency": "0.112114537s", + "requestMethod": "GET", + "requestSize": "534", + "status": 200 + }, + "insertId": "6111722f000b5b4c4d4071e2", + "labels": { + "service": "datasette-io" + } + } + +Inserting this into a table using ``sqlite-utils insert logs.db log log.json`` will create a table with the following schema:: + + CREATE TABLE [logs] ( + [httpRequest] TEXT, + [insertId] TEXT, + [labels] TEXT + ); + +With the ``--flatten`` option columns will be created using ``topkey_nextkey`` column names - so running ``sqlite-utils insert logs.db log log.json --flatten`` will create the following schema instead:: + + CREATE TABLE [logs] ( + [httpRequest_latency] TEXT, + [httpRequest_requestMethod] TEXT, + [httpRequest_requestSize] TEXT, + [httpRequest_status] INTEGER, + [insertId] TEXT, + [labels_service] TEXT + ); + .. _cli_insert_csv_tsv: Inserting CSV or TSV data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c4f8501..2f32d0a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -643,6 +643,7 @@ def insert_upsert_options(fn): "--pk", help="Columns to use as the primary key, e.g. id", multiple=True ), click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), + click.option("--flatten", is_flag=True, help="Flatten nested JSON objets"), click.option("-c", "--csv", is_flag=True, help="Expect CSV"), click.option("--tsv", is_flag=True, help="Expect TSV"), click.option("--delimiter", help="Delimiter to use for CSV files"), @@ -697,6 +698,7 @@ def insert_upsert_implementation( json_file, pk, nl, + flatten, csv, tsv, delimiter, @@ -722,6 +724,8 @@ def insert_upsert_implementation( csv = True if (nl + csv + tsv) >= 2: raise click.ClickException("Use just one of --nl, --csv or --tsv") + if (csv or tsv) and flatten: + raise click.ClickException("--flatten cannot be used with --csv or --tsv") if encoding and not (csv or tsv): raise click.ClickException("--encoding must be used with --csv or --tsv") if pk and len(pk) == 1: @@ -766,6 +770,8 @@ def insert_upsert_implementation( raise click.ClickException( "Invalid JSON - use --csv for CSV or --tsv for TSV files" ) + if flatten: + docs = (dict(_flatten(doc)) for doc in docs) extra_kwargs = {"ignore": ignore, "replace": replace, "truncate": truncate} if not_null: @@ -790,6 +796,15 @@ def insert_upsert_implementation( db[table].transform(types=tracker.types) +def _flatten(d): + for key, value in d.items(): + if isinstance(value, dict): + for key2, value2 in _flatten(value): + yield key + "_" + key2, value2 + else: + yield key, value + + @cli.command() @insert_upsert_options @click.option( @@ -813,6 +828,7 @@ def insert( json_file, pk, nl, + flatten, csv, tsv, delimiter, @@ -844,6 +860,7 @@ def insert( json_file, pk, nl, + flatten, csv, tsv, delimiter, @@ -875,6 +892,7 @@ def upsert( json_file, pk, nl, + flatten, csv, tsv, batch_size, @@ -902,6 +920,7 @@ def upsert( json_file, pk, nl, + flatten, csv, tsv, delimiter, diff --git a/tests/test_cli.py b/tests/test_cli.py index e84453a..e6f6683 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -648,6 +648,34 @@ def test_insert_invalid_json_error(tmpdir): ) +def test_insert_json_flatten(tmpdir): + db_path = str(tmpdir / "flat.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-", "--flatten"], + input=json.dumps({"nested": {"data": 4}}), + ) + assert result.exit_code == 0 + assert list(Database(db_path).query("select * from items")) == [{"nested_data": 4}] + + +def test_insert_json_flatten_nl(tmpdir): + db_path = str(tmpdir / "flat.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-", "--flatten", "--nl"], + input="\n".join( + json.dumps(item) + for item in [{"nested": {"data": 4}}, {"nested": {"other": 3}}] + ), + ) + assert result.exit_code == 0 + assert list(Database(db_path).query("select * from items")) == [ + {"nested_data": 4, "nested_other": None}, + {"nested_data": None, "nested_other": 3}, + ] + + def test_insert_with_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dog.json") open(json_path, "w").write(json.dumps({"id": 1, "name": "Cleo", "age": 4})) @@ -1197,6 +1225,21 @@ def test_upsert(db_path, tmpdir): ] +def test_upsert_flatten(tmpdir): + db_path = str(tmpdir / "flat.db") + db = Database(db_path) + db["upsert_me"].insert({"id": 1, "name": "Example"}, pk="id") + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "upsert_me", "-", "--flatten", "--pk", "id", "--alter"], + input=json.dumps({"id": 1, "nested": {"two": 2}}), + ) + assert result.exit_code == 0 + assert list(db.query("select * from upsert_me")) == [ + {"id": 1, "name": "Example", "nested_two": 2} + ] + + def test_upsert_alter(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") db = Database(db_path) @@ -2249,3 +2292,15 @@ def test_insert_detect_types(tmpdir, option_or_env_var): _test() else: _test() + + +@pytest.mark.parametrize( + "input,expected", + ( + ({"foo": {"bar": 1}}, {"foo_bar": 1}), + ({"foo": {"bar": [1, 2, {"baz": 3}]}}, {"foo_bar": [1, 2, {"baz": 3}]}), + ({"foo": {"bar": 1, "baz": {"three": 3}}}, {"foo_bar": 1, "foo_baz_three": 3}), + ), +) +def test_flatten_helper(input, expected): + assert dict(cli._flatten(input)) == expected From 15758d02fd437004fd9f84c9d4a8bf49f0793e13 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 14:45:39 -0700 Subject: [PATCH 0412/1004] Fixed spelling of objects, refs #310 --- sqlite_utils/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 2f32d0a..79103bc 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -643,7 +643,7 @@ def insert_upsert_options(fn): "--pk", help="Columns to use as the primary key, e.g. id", multiple=True ), click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), - click.option("--flatten", is_flag=True, help="Flatten nested JSON objets"), + click.option("--flatten", is_flag=True, help="Flatten nested JSON objects"), click.option("-c", "--csv", is_flag=True, help="Expect CSV"), click.option("--tsv", is_flag=True, help="Expect TSV"), click.option("--delimiter", help="Delimiter to use for CSV files"), From 3fb1034e869090876cab0247146a312be993210f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 14:46:47 -0700 Subject: [PATCH 0413/1004] option, not object, refs #310 --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 28bd3f7..063b9e9 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -729,7 +729,7 @@ Flattening nested JSON objects ``sqlite-utils insert`` expects incoming data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table. -If your data is nested you can use the `--flatten` object to create columns that are derived from the nested data. +If your data is nested you can use the ``--flatten`` option to create columns that are derived from the nested data. Consider this example document, in a file called ``log.json``:: From 14f643d9e91f5557d5e46251dadac481f4b41021 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 15:25:52 -0700 Subject: [PATCH 0414/1004] Better error messages in CLI, closes #309 --- sqlite_utils/cli.py | 31 ++++++++++++++++++++++++++++--- tests/test_cli.py | 21 ++++++++++++++++++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 79103bc..0acb252 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -786,12 +786,25 @@ def insert_upsert_implementation( db[table].insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs ) - except sqlite3.OperationalError as e: - if e.args and "has no column named" in e.args[0]: + except Exception as e: + if ( + isinstance(e, sqlite3.OperationalError) + and e.args + and "has no column named" in e.args[0] + ): raise click.ClickException( "{}\n\nTry using --alter to add additional columns".format(e.args[0]) ) - raise + # If we can find sql= and params= arguments, show those + variables = _find_variables(e.__traceback__, ["sql", "params"]) + if "sql" in variables and "params" in variables: + raise click.ClickException( + "{}\n\nsql = {}\nparams={}".format( + str(e), variables["sql"], variables["params"] + ) + ) + else: + raise if tracker is not None: db[table].transform(types=tracker.types) @@ -805,6 +818,18 @@ def _flatten(d): yield key, value +def _find_variables(tb, vars): + to_find = list(vars) + found = {} + for var in to_find: + if var in tb.tb_frame.f_locals: + vars.remove(var) + found[var] = tb.tb_frame.f_locals[var] + if vars and tb.tb_next: + found.update(_find_variables(tb.tb_next, vars)) + return found + + @cli.command() @insert_upsert_options @click.option( diff --git a/tests/test_cli.py b/tests/test_cli.py index e6f6683..319bd40 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1256,7 +1256,11 @@ def test_upsert_alter(db_path, tmpdir): cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"] ) assert 1 == result.exit_code - assert "no such column: age" == str(result.exception) + assert ( + "Error: no such column: age\n\n" + "sql = UPDATE [dogs] SET [age] = ? WHERE [id] = ?\n" + "params=[5, 1]" + ) == result.output.strip() # Should succeed with --alter result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id", "--alter"] @@ -2304,3 +2308,18 @@ def test_insert_detect_types(tmpdir, option_or_env_var): ) def test_flatten_helper(input, expected): assert dict(cli._flatten(input)) == expected + + +def test_integer_overflow_error(tmpdir): + db_path = str(tmpdir / "test.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-"], + input=json.dumps({"bignumber": 34223049823094832094802398430298048240}), + ) + assert result.exit_code == 1 + assert result.output == ( + "Error: Python int too large to convert to SQLite INTEGER\n\n" + "sql = INSERT INTO [items] ([bignumber]) VALUES (?);\n" + "params=[34223049823094832094802398430298048240]\n" + ) From a6567ec507e235fd4d313c6b1570d5a4f45e4b86 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 15:33:33 -0700 Subject: [PATCH 0415/1004] Capture parameters= not params=, refs #309 --- sqlite_utils/cli.py | 10 +++++----- tests/test_cli.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0acb252..d94229d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -795,12 +795,12 @@ def insert_upsert_implementation( raise click.ClickException( "{}\n\nTry using --alter to add additional columns".format(e.args[0]) ) - # If we can find sql= and params= arguments, show those - variables = _find_variables(e.__traceback__, ["sql", "params"]) - if "sql" in variables and "params" in variables: + # If we can find sql= and parameters= arguments, show those + variables = _find_variables(e.__traceback__, ["sql", "parameters"]) + if "sql" in variables and "parameters" in variables: raise click.ClickException( - "{}\n\nsql = {}\nparams={}".format( - str(e), variables["sql"], variables["params"] + "{}\n\nsql = {}\nparameters = {}".format( + str(e), variables["sql"], variables["parameters"] ) ) else: diff --git a/tests/test_cli.py b/tests/test_cli.py index 319bd40..d4801ac 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1259,7 +1259,7 @@ def test_upsert_alter(db_path, tmpdir): assert ( "Error: no such column: age\n\n" "sql = UPDATE [dogs] SET [age] = ? WHERE [id] = ?\n" - "params=[5, 1]" + "parameters = [5, 1]" ) == result.output.strip() # Should succeed with --alter result = CliRunner().invoke( @@ -2321,5 +2321,5 @@ def test_integer_overflow_error(tmpdir): assert result.output == ( "Error: Python int too large to convert to SQLite INTEGER\n\n" "sql = INSERT INTO [items] ([bignumber]) VALUES (?);\n" - "params=[34223049823094832094802398430298048240]\n" + "parameters = [34223049823094832094802398430298048240]\n" ) From d5ef91212022ea3ed85258af605bdc28e5799ff1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 15:42:06 -0700 Subject: [PATCH 0416/1004] Release 3.15 Refs #309, #310 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e03dbd6..9a1669d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v3_15: + +3.15 (2021-08-09) +----------------- + +- ``sqlite-utils insert --flatten`` option for :ref:`flattening nested JSON objects ` to create tables with column names like ``topkey_nestedkey``. (:issue:`310`) +- Fixed several spelling mistakes in the documentation, spotted `using codespell `__. +- Errors that occur while using the ``sqlite-utils`` CLI tool now show the responsible SQL and query parameters, if possible. (:issue:`309`) + .. _v3_14: 3.14 (2021-08-02) diff --git a/setup.py b/setup.py index 5b13427..c8c58ac 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.14" +VERSION = "3.15" def get_long_description(): From 8757de84b27cedf494ee917ce2daf773d2c3f877 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 15:44:26 -0700 Subject: [PATCH 0417/1004] Link to stable docs, not latest --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4c738ab..bac6369 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # sqlite-utils [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/) -[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/latest/changelog.html) +[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/stable/changelog.html) [![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/) [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest) -[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.datasette.io/en/latest/?badge=latest) +[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=stable)](http://sqlite-utils.datasette.io/en/stable/?badge=stable) [![codecov](https://codecov.io/gh/simonw/sqlite-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/simonw/sqlite-utils) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE) From ee469e3122d6f5973ec2584c1580d930daca2e7c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 15:50:53 -0700 Subject: [PATCH 0418/1004] Corrected tiny mistake in --flatten examples --- docs/cli.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 063b9e9..0c08526 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -746,7 +746,7 @@ Consider this example document, in a file called ``log.json``:: } } -Inserting this into a table using ``sqlite-utils insert logs.db log log.json`` will create a table with the following schema:: +Inserting this into a table using ``sqlite-utils insert logs.db logs log.json`` will create a table with the following schema:: CREATE TABLE [logs] ( [httpRequest] TEXT, @@ -754,7 +754,7 @@ Inserting this into a table using ``sqlite-utils insert logs.db log log.json`` w [labels] TEXT ); -With the ``--flatten`` option columns will be created using ``topkey_nextkey`` column names - so running ``sqlite-utils insert logs.db log log.json --flatten`` will create the following schema instead:: +With the ``--flatten`` option columns will be created using ``topkey_nextkey`` column names - so running ``sqlite-utils insert logs.db logs log.json --flatten`` will create the following schema instead:: CREATE TABLE [logs] ( [httpRequest_latency] TEXT, From 6155da72c8939b5d9bdacb7853e5e8d1767ce1d5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 10 Aug 2021 16:09:28 -0700 Subject: [PATCH 0419/1004] Add reference page to documentation using Sphinx autodoc (#312) * Docstrings and type annotations for almost every method * New /reference API documentation page using Sphink autodoc * Custom Read The Docs config, to get autodoc working * Fix for #313 (add_foreign_keys() doesn't reject being called with a View) * Fixed #315 (.delete_where() returns [] when it should return self) --- .readthedocs.yaml | 12 + docs/Makefile | 2 +- docs/conf.py | 3 +- docs/index.rst | 1 + docs/python-api.rst | 2 +- docs/reference.rst | 69 ++++ sqlite_utils/db.py | 802 ++++++++++++++++++++++++++++++++++---------- 7 files changed, 711 insertions(+), 180 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 docs/reference.rst diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..ce66cbe --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,12 @@ +version: 2 + +sphinx: + configuration: docs/conf.py + +python: + version: "3.8" + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/docs/Makefile b/docs/Makefile index a279768..5578ae3 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -20,4 +20,4 @@ help: @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) livehtml: - sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) + sphinx-autobuild -a -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) --watch ../sqlite_utils diff --git a/docs/conf.py b/docs/conf.py index b4c7f44..1f5a158 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,8 @@ from subprocess import Popen, PIPE # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.extlinks"] +extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"] +autodoc_member_order = "bysource" extlinks = { "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"), diff --git a/docs/index.rst b/docs/index.rst index 93b0bc0..581f306 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,7 @@ Contents installation cli python-api + reference contributing changelog diff --git a/docs/python-api.rst b/docs/python-api.rst index 70baf0c..b793920 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -962,7 +962,7 @@ The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` metho The name of the column ``total_rows`` - The total number of rows in the table` + The total number of rows in the table ``num_null`` The number of rows for which this column is null diff --git a/docs/reference.rst b/docs/reference.rst new file mode 100644 index 0000000..e29f468 --- /dev/null +++ b/docs/reference.rst @@ -0,0 +1,69 @@ +=============== + API Reference +=============== + +.. contents:: :local: + +.. _reference_db_database: + +sqlite_utils.db.Database +======================== + +.. autoclass:: sqlite_utils.db.Database + :members: + :undoc-members: + :show-inheritance: + :special-members: __getitem__ + :exclude-members: use_counts_table, execute_returning_dicts, resolve_foreign_keys + +.. _reference_db_queryable: + +sqlite_utils.db.Queryable +========================= + +:ref:`Table ` and :ref:`View ` are both subclasses of ``Queryable``, providing access to the following methods: + +.. autoclass:: sqlite_utils.db.Queryable + :members: + :undoc-members: + :exclude-members: execute_count + +.. _reference_db_table: + +sqlite_utils.db.Table +===================== + +.. autoclass:: sqlite_utils.db.Table + :members: + :undoc-members: + :show-inheritance: + :exclude-members: guess_foreign_column, value_or_default, build_insert_queries_and_params, insert_chunk, add_missing_columns + +.. _reference_db_view: + +sqlite_utils.db.View +==================== + +.. autoclass:: sqlite_utils.db.View + :members: + :undoc-members: + :show-inheritance: + +.. _reference_db_other: + +Other +===== + +.. _reference_db_other_column: + +sqlite_utils.db.Column +---------------------- + +.. autoclass:: sqlite_utils.db.Column + +.. _reference_db_other_column_details: + +sqlite_utils.db.ColumnDetails +----------------------------- + +.. autoclass:: sqlite_utils.db.ColumnDetails diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a21acc4..c7dc832 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -21,7 +21,19 @@ import re from sqlite_fts4 import rank_bm25 # type: ignore import sys import textwrap -from typing import Generator, Iterable, Union, Optional, List +from typing import ( + cast, + Any, + Callable, + Dict, + Generator, + Iterable, + Union, + Optional, + List, + Set, + Tuple, +) import uuid SQLITE_MAX_VARS = 999 @@ -41,7 +53,7 @@ _virtual_table_using_re = re.compile( ) ) \s+(IF\s+NOT\s+EXISTS\s+)? # IF NOT EXISTS (optional) -USING\s+(?P\w+) # e.g. USING FTS5 +USING\s+(?P\w+) # for example USING FTS5 """, re.VERBOSE | re.IGNORECASE, ) @@ -49,7 +61,7 @@ USING\s+(?P\w+) # e.g. USING FTS5 try: import pandas as pd # type: ignore except ImportError: - pd = None + pd = None # type: ignore try: import numpy as np # type: ignore @@ -59,6 +71,28 @@ except ImportError: Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") ) +Column.__doc__ = """ +Describes a SQLite column returned by the :attr:`.Table.columns` property. + +``cid`` + Column index + +``name`` + Column name + +``type`` + Column type + +``notnull`` + Does the column have a ``not null` constraint + +``default_value`` + Default value for this column + +``is_pk`` + Is this column part of the primary key +""" + ColumnDetails = namedtuple( "ColumnDetails", ( @@ -72,6 +106,34 @@ ColumnDetails = namedtuple( "least_common", ), ) +ColumnDetails.__doc__ = """ +Summary information about a column, see :ref:`python_api_analyze_column`. + +``table`` + The name of the table + +``column`` + The name of the column + +``total_rows`` + The total number of rows in the table + +``num_null`` + The number of rows for which this column is null + +``num_blank`` + The number of rows for which this column is blank (the empty string) + +``num_distinct`` + The number of distinct values in this column + +``most_common`` + The ``N`` most common values as a list of ``(value, count)`` tuples`, or ``None`` if the table consists entirely of distinct values + +``least_common`` + The ``N`` least common values as a list of ``(value, count)`` tuples`, or ``None`` if the table is entirely distinct + or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) +""" ForeignKey = namedtuple( "ForeignKey", ("table", "column", "other_table", "other_column") ) @@ -83,7 +145,11 @@ XIndexColumn = namedtuple( Trigger = namedtuple("Trigger", ("name", "table", "sql")) -DEFAULT = object() +class Default: + pass + + +DEFAULT = Default() COLUMN_TYPE_MAPPING = { float: "FLOAT", @@ -133,26 +199,32 @@ if pd: class AlterError(Exception): + "Error altering table" pass class NoObviousTable(Exception): + "Could not tell which table this operation refers to" pass class BadPrimaryKey(Exception): + "Table does not have a single obvious primary key" pass class NotFoundError(Exception): + "Record not found" pass class PrimaryKeyRequired(Exception): + "Primary key needs to be specified" pass class InvalidColumns(Exception): + "Specified columns do not exist" pass @@ -176,17 +248,32 @@ CREATE TABLE IF NOT EXISTS [{}]( class Database: + """ + Wrapper for a SQLite database connection that adds a variety of useful utility methods. + + - ``filename_or_conn`` - String path to a file, or a ``pathlib.Path`` object, or a + ``sqlite3`` connection + - ``memory`` - set to ``True`` to create an in-memory database + - ``recreate`` - set to ``True`` to delete and recreate a file database (**dangerous**) + - ``recursive_triggers`` - defaults to ``True``, which sets ``PRAGMA recursive_triggers=on;`` - + set to ``False`` to avoid setting this pragma + - ``tracer`` - set a tracer function (``print`` works for this) which will be called with + ``sql, parameters`` every time a SQL query is executed + - ``use_counts_table`` - set to ``True`` to use a cached counts table, if available. See + :ref:`python_api_cached_table_counts`. + """ + _counts_table_name = "_counts" use_counts_table = False def __init__( self, filename_or_conn=None, - memory=False, - recreate=False, - recursive_triggers=True, - tracer=None, - use_counts_table=False, + memory: bool = False, + recreate: bool = False, + recursive_triggers: bool = True, + tracer: Callable = None, + use_counts_table: bool = False, ): assert (filename_or_conn is not None and not memory) or ( filename_or_conn is None and memory @@ -203,11 +290,24 @@ class Database: self._tracer = tracer if recursive_triggers: self.execute("PRAGMA recursive_triggers=on;") - self._registered_functions = set() + self._registered_functions: set = set() self.use_counts_table = use_counts_table @contextlib.contextmanager - def tracer(self, tracer=None): + def tracer(self, tracer: Callable = None): + """ + Context manager to temporarily set a tracer function - all executed SQL queries will + be passed to this. + + The tracer function should accept two arguments: ``sql`` and ``parameters`` + + Example usage:: + + with db.tracer(print): + db["creatures"].insert({"name": "Cleo"}) + + See :ref:`python_api_tracing`. + """ prev_tracer = self._tracer self._tracer = tracer or print try: @@ -215,13 +315,39 @@ class Database: finally: self._tracer = prev_tracer - def __getitem__(self, table_name): + def __getitem__(self, table_name: str) -> Union["Table", "View"]: + """ + ``db[table_name]`` returns a :class:`.Table` object for the table with the specified name. + If the table does not exist yet it will be created the first time data is inserted into it. + """ return self.table(table_name) def __repr__(self): return "".format(self.conn) - def register_function(self, fn=None, deterministic=None, replace=False): + def register_function( + self, fn: Callable = None, deterministic: bool = False, replace: bool = False + ): + """ + ``fn`` will be made available as a function within SQL, with the same name and number + of arguments. Can be used as a decorator:: + + @db.register + def upper(value): + return str(value).upper() + + The decorator can take arguments:: + + @db.register(deterministic=True, replace=True) + def upper(value): + return str(value).upper() + + - ``deterministic`` - set ``True`` for functions that always returns the same output for a given input + - ``replace`` - set ``True`` to replace an existing function with the same name - otherwise throw an error + + See :ref:`python_api_register_function`. + """ + def register(fn): name = fn.__name__ arity = len(inspect.signature(fn).parameters) @@ -240,9 +366,15 @@ class Database: register(fn) def register_fts4_bm25(self): + "Register the ``rank_bm25(match_info)`` function used for calculating relevance with SQLite FTS4." self.register_function(rank_bm25, deterministic=True) - def attach(self, alias, filepath): + def attach(self, alias: str, filepath: Union[str, pathlib.Path]): + """ + Attach another SQLite database file to this connection with the specified alias, equivalent to:: + + ATTACH DATABASE 'filepath.db' AS alias + """ attach_sql = """ ATTACH DATABASE '{}' AS [{}]; """.format( @@ -250,7 +382,19 @@ class Database: ).strip() self.execute(attach_sql) - def execute(self, sql, parameters=None): + def query( + self, sql: str, params: Optional[Union[Iterable, dict]] = None + ) -> Generator[dict, None, None]: + "Execute ``sql`` and return an iterable of dictionaries representing each row." + cursor = self.execute(sql, params or tuple()) + keys = [d[0] for d in cursor.description] + for row in cursor: + yield dict(zip(keys, row)) + + def execute( + self, sql: str, parameters: Optional[Union[Iterable, dict]] = None + ) -> sqlite3.Cursor: + "Execute SQL query and return a ``sqlite3.Cursor``." if self._tracer: self._tracer(sql, parameters) if parameters is not None: @@ -258,16 +402,19 @@ class Database: else: return self.conn.execute(sql) - def executescript(self, sql): + def executescript(self, sql: str) -> sqlite3.Cursor: + "Execute multiple SQL statements separated by ; and return the ``sqlite3.Cursor``." if self._tracer: self._tracer(sql, None) return self.conn.executescript(sql) - def table(self, table_name, **kwargs): + def table(self, table_name: str, **kwargs) -> Union["Table", "View"]: + "Return a table object, optionally configured with default options." klass = View if table_name in self.view_names() else Table return klass(self, table_name, **kwargs) - def quote(self, value): + def quote(self, value: str) -> str: + "Apply SQLite string quoting to a value, including wrappping it in single quotes." # Normally we would use .execute(sql, [params]) for escaping, but # occasionally that isn't available - most notable when we need # to include a "... DEFAULT 'value'" in a column definition. @@ -277,7 +424,8 @@ class Database: {"value": value}, ).fetchone()[0] - def table_names(self, fts4=False, fts5=False): + def table_names(self, fts4: bool = False, fts5: bool = False) -> List[str]: + "A list of string table names in this database." where = ["type = 'table'"] if fts4: where.append("sql like '%USING FTS4%'") @@ -286,7 +434,8 @@ class Database: sql = "select name from sqlite_master where {}".format(" AND ".join(where)) return [r[0] for r in self.execute(sql).fetchall()] - def view_names(self): + def view_names(self) -> List[str]: + "A list of string view names in this database." return [ r[0] for r in self.execute( @@ -295,15 +444,18 @@ class Database: ] @property - def tables(self): - return [self[name] for name in self.table_names()] + def tables(self) -> List["Table"]: + "A list of Table objects in this database." + return cast(List["Table"], [self[name] for name in self.table_names()]) @property - def views(self): - return [self[name] for name in self.view_names()] + def views(self) -> List["View"]: + "A list of View objects in this database." + return cast(List["View"], [self[name] for name in self.view_names()]) @property - def triggers(self): + def triggers(self) -> List[Trigger]: + "A list of ``(name, table_name, sql)`` tuples representing triggers in this database." return [ Trigger(*r) for r in self.execute( @@ -312,12 +464,13 @@ class Database: ] @property - def triggers_dict(self): - "Returns {trigger_name: sql} dictionary" + def triggers_dict(self) -> Dict[str, str]: + "A ``{trigger_name: sql}`` dictionary of triggers in this database." return {trigger.name: trigger.sql for trigger in self.triggers} @property - def schema(self): + def schema(self) -> str: + "SQL schema for this database" sqls = [] for row in self.execute( "select sql from sqlite_master where sql is not null" @@ -329,14 +482,17 @@ class Database: return "\n".join(sqls) @property - def journal_mode(self): + def journal_mode(self) -> str: + "Current ``journal_mode`` of this database." return self.execute("PRAGMA journal_mode;").fetchone()[0] def enable_wal(self): + "Set ``journal_mode`` to ``'wal'`` to enable Write-Ahead Log mode." if self.journal_mode != "wal": self.execute("PRAGMA journal_mode=wal;") def disable_wal(self): + "Set ``journal_mode`` back to ``'delete'`` to disable Write-Ahead Log mode." if self.journal_mode != "delete": self.execute("PRAGMA journal_mode=delete;") @@ -345,6 +501,10 @@ class Database: self.execute(_COUNTS_TABLE_CREATE_SQL.format(self._counts_table_name)) def enable_counts(self): + """ + Enable trigger-based count caching for every table in the database, see + :ref:`python_api_cached_table_counts`. + """ self._ensure_counts_table() for table in self.tables: if ( @@ -354,7 +514,11 @@ class Database: table.enable_counts() self.use_counts_table = True - def cached_counts(self, tables=None): + def cached_counts(self, tables: Optional[Iterable[str]] = None) -> Dict[str, int]: + """ + Return ``{table_name: count}`` dictionary of cached counts for specified tables, or + all tables if ``tables`` not provided. + """ sql = "select [table], count from {}".format(self._counts_table_name) if tables: sql += " where [table] in ({})".format(", ".join("?" for table in tables)) @@ -364,6 +528,7 @@ class Database: return {} def reset_counts(self): + "Re-calculate cached counts for tables." tables = [table for table in self.tables if table.has_counts_triggers] with self.conn: self._ensure_counts_table() @@ -374,14 +539,6 @@ class Database: for table in tables ) - def query( - self, sql: str, params: Optional[Union[Iterable, dict]] = None - ) -> Generator[dict, None, None]: - cursor = self.execute(sql, params or tuple()) - keys = [d[0] for d in cursor.description] - for row in cursor: - yield dict(zip(keys, row)) - def execute_returning_dicts( self, sql: str, params: Optional[Union[Iterable, dict]] = None ) -> List[dict]: @@ -430,16 +587,17 @@ class Database: def create_table_sql( self, - name, - columns, - pk=None, + name: str, + columns: Dict[str, Any], + pk: Optional[Any] = None, foreign_keys=None, column_order=None, not_null=None, defaults=None, hash_id=None, extracts=None, - ): + ) -> str: + "Returns the SQL ``CREATE TABLE`` statement for creating the specified table." foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) foreign_keys_by_column = {fk.column: fk for fk in foreign_keys} # any extracts will be treated as integer columns with a foreign key @@ -536,16 +694,21 @@ class Database: def create_table( self, - name, - columns, - pk=None, + name: str, + columns: Dict[str, Any], + pk: Optional[Any] = None, foreign_keys=None, column_order=None, not_null=None, defaults=None, hash_id=None, extracts=None, - ): + ) -> "Table": + """ + Create a table with the specified name and the specified ``{column_name: type}`` columns. + + See :ref:`python_api_explicit_create`. + """ sql = self.create_table_sql( name=name, columns=columns, @@ -558,7 +721,7 @@ class Database: extracts=extracts, ) self.execute(sql) - return self.table( + table = self.table( name, pk=pk, foreign_keys=foreign_keys, @@ -567,8 +730,17 @@ class Database: defaults=defaults, hash_id=hash_id, ) + return cast(Table, table) - def create_view(self, name, sql, ignore=False, replace=False): + def create_view( + self, name: str, sql: str, ignore: bool = False, replace: bool = False + ): + """ + Create a new SQL view with the specified name - ``sql`` should start with ``SELECT ...``. + + - ``ignore`` - set to ``True`` to do nothing if a view with this name already exists + - ``replace`` - set to ``True`` to do replace the view if one with this name already exists + """ assert not ( ignore and replace ), "Use one or the other of ignore/replace, not both" @@ -586,18 +758,28 @@ class Database: self.execute(create_sql) return self - def m2m_table_candidates(self, table, other_table): - "Returns potential m2m tables for arguments, based on FKs" + def m2m_table_candidates(self, table: str, other_table: str) -> List[str]: + """ + Given two table names returns the name of tables that could define a + many-to-many relationship between those two tables, based on having + foreign keys to both of the provided tables. + """ candidates = [] tables = {table, other_table} - for table in self.tables: + for table_obj in self.tables: # Does it have foreign keys to both table and other_table? - has_fks_to = {fk.other_table for fk in table.foreign_keys} + has_fks_to = {fk.other_table for fk in table_obj.foreign_keys} if has_fks_to.issuperset(tables): - candidates.append(table.name) + candidates.append(table_obj.name) return candidates - def add_foreign_keys(self, foreign_keys): + def add_foreign_keys(self, foreign_keys: Iterable[Tuple[str, str, str, str]]): + """ + See :ref:`python_api_add_foreign_keys`. + + ``foreign_keys`` should be a list of ``(table, column, other_table, other_column)`` + tuples, see :ref:`python_api_add_foreign_keys`. + """ # foreign_keys is a list of explicit 4-tuples assert all( len(fk) == 4 and isinstance(fk, (list, tuple)) for fk in foreign_keys @@ -609,7 +791,11 @@ class Database: for table, column, other_table, other_column in foreign_keys: if not self[table].exists(): raise AlterError("No such table: {}".format(table)) - if column not in self[table].columns_dict: + table_obj = self[table] + if not isinstance(table_obj, Table): + raise AlterError("Must be a table, not a view: {}".format(table)) + table_obj = cast(Table, table_obj) + if column not in table_obj.columns_dict: raise AlterError("No such column: {} in {}".format(column, table)) if not self[other_table].exists(): raise AlterError("No such other_table: {}".format(other_table)) @@ -623,7 +809,7 @@ class Database: # We will silently skip foreign keys that exist already if not any( fk - for fk in self[table].foreign_keys + for fk in table_obj.foreign_keys if fk.column == column and fk.other_table == other_table and fk.other_column == other_column @@ -633,7 +819,7 @@ class Database: ) # Construct SQL for use with "UPDATE sqlite_master SET sql = ? WHERE name = ?" - table_sql = {} + table_sql: Dict[str, str] = {} for table, column, other_table, other_column in foreign_keys_to_create: old_sql = table_sql.get(table, self[table].schema) extra_sql = ",\n FOREIGN KEY([{column}]) REFERENCES [{other_table}]([{other_column}])\n".format( @@ -661,6 +847,7 @@ class Database: self.vacuum() def index_foreign_keys(self): + "Create indexes for every foreign key column on every table in the database." for table_name in self.table_names(): table = self[table_name] existing_indexes = { @@ -671,11 +858,13 @@ class Database: table.create_index([fk.column]) def vacuum(self): + "Run a SQLite ``VACUUM`` against the database." self.execute("VACUUM;") class Queryable: - def exists(self): + def exists(self) -> bool: + "Does this table or view exist yet?" return False def __init__(self, db, name): @@ -684,9 +873,10 @@ class Queryable: def count_where( self, - where=None, - where_args=None, - ): + where: str = None, + where_args: Optional[Union[Iterable, dict]] = None, + ) -> int: + "Executes ``SELECT count(*) FROM table WHERE ...`` and returns a count." sql = "select count(*) from [{}]".format(self.name) if where is not None: sql += " where " + where @@ -697,24 +887,38 @@ class Queryable: return self.count_where() @property - def count(self): + def count(self) -> int: + "A count of the rows in this table or view." return self.count_where() @property - def rows(self): + def rows(self) -> Generator[dict, None, None]: + "Iterate over every dictionaries for each row in this table or view." return self.rows_where() def rows_where( self, - where=None, - where_args=None, - order_by=None, - select="*", - limit=None, - offset=None, - ): + where: str = None, + where_args: Optional[Union[Iterable, dict]] = None, + order_by: str = None, + select: str = "*", + limit: int = None, + offset: int = None, + ) -> Generator[dict, None, None]: + """ + Iterate over every row in this table or view that matches the specified where clause. + + - ``where`` - a SQL fragment to use as a ``WHERE`` clause, for example ``age > ?`` or ``age > :age``. + - ``where_args`` - a list of arguments (if using ``?``) or a dictionary (if using ``:age``). + - ``order_by`` - optional column or fragment of SQL to order by. + - ``select`` - optional comma-separated list of columns to select. + - ``limit`` - optional integer number of rows to limit to. + - ``offset`` - optional integer for SQL offset. + + Returns each row as a dictionary. See :ref:`python_api_rows` for more details. + """ if not self.exists(): - return [] + return sql = "select {} from [{}]".format(select, self.name) if where is not None: sql += " where " + where @@ -731,13 +935,13 @@ class Queryable: def pks_and_rows_where( self, - where=None, - where_args=None, - order_by=None, - limit=None, - offset=None, - ): - "Like .rows_where() but returns (pk, row) pairs - pk can be a single value or tuple" + where: str = None, + where_args: Optional[Union[Iterable, dict]] = None, + order_by: str = None, + limit: int = None, + offset: int = None, + ) -> Generator[Tuple[Any, Dict], None, None]: + "Like ``.rows_where()`` but returns ``(pk, row)`` pairs - ``pk`` can be a single value or tuple." column_names = [column.name for column in self.columns] pks = [column.name for column in self.columns if column.is_pk] if not pks: @@ -758,32 +962,37 @@ class Queryable: yield row_pk, row @property - def columns(self): + def columns(self) -> List["Column"]: + "List of :ref:`Columns ` representing the columns in this table or view." if not self.exists(): return [] rows = self.db.execute("PRAGMA table_info([{}])".format(self.name)).fetchall() return [Column(*row) for row in rows] @property - def columns_dict(self): - "Returns {column: python-type} dictionary" + def columns_dict(self) -> Dict[str, Any]: + "``{column_name: python-type}`` dictionary representing columns in this table or view." return {column.name: column_affinity(column.type) for column in self.columns} @property - def schema(self): + def schema(self) -> str: + "SQL schema for this table or view." return self.db.execute( "select sql from sqlite_master where name = ?", (self.name,) ).fetchone()[0] class Table(Queryable): - last_rowid = None - last_pk = None + "Tables should usually be initialized using the ``db.table(table_name)`` or ``db[table_name]`` methods." + #: The ``rowid`` of the last inserted, updated or selected row.` + last_rowid: Optional[int] = None + #: The primary key of the last inserted, updated or selected row.` + last_pk: Optional[Any] = None def __init__( self, - db, - name, + db: Database, + name: str, pk=None, foreign_keys=None, column_order=None, @@ -815,7 +1024,7 @@ class Table(Queryable): columns=columns, ) - def __repr__(self): + def __repr__(self) -> str: return "
".format( self.name, " (does not exist yet)" @@ -824,7 +1033,8 @@ class Table(Queryable): ) @property - def count(self): + def count(self) -> int: + "Count of the rows in this table - optionally from the table count cache, if configured." if self.db.use_counts_table: counts = self.db.cached_counts([self.name]) if counts: @@ -835,17 +1045,26 @@ class Table(Queryable): return self.name in self.db.table_names() @property - def pks(self): + def pks(self) -> List[str]: + "Primary key columns for this table." names = [column.name for column in self.columns if column.is_pk] if not names: names = ["rowid"] return names @property - def use_rowid(self): + def use_rowid(self) -> bool: + "Does this table use ``rowid`` for its primary key (no other primary keys are specified)?" return not any(column for column in self.columns if column.is_pk) - def get(self, pk_values): + def get(self, pk_values: Union[list, tuple, str, int]) -> dict: + """ + Return row (as dictionary) for the specified primary key. + + Primary key can be a single value, or a tuple for tables with a compound primary key. + + Raises ``NotFoundError`` if a matching row cannot be found. + """ if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] pks = self.pks @@ -867,7 +1086,8 @@ class Table(Queryable): raise NotFoundError @property - def foreign_keys(self): + def foreign_keys(self) -> List["ForeignKey"]: + "List of foreign keys defined on this table." fks = [] for row in self.db.execute( "PRAGMA foreign_key_list([{}])".format(self.name) @@ -885,15 +1105,16 @@ class Table(Queryable): return fks @property - def virtual_table_using(self): - "Returns type of virtual table or None if this is not a virtual table" + def virtual_table_using(self) -> Optional[str]: + "Type of virtual table, or ``None`` if this is not a virtual table." match = _virtual_table_using_re.match(self.schema) if match is None: return None return match.groupdict()["using"].upper() @property - def indexes(self): + def indexes(self) -> List[Index]: + "List of indexes defined on this table." sql = 'PRAGMA index_list("{}")'.format(self.name) indexes = [] for row in self.db.execute_returning_dicts(sql): @@ -916,7 +1137,8 @@ class Table(Queryable): return indexes @property - def xindexes(self): + def xindexes(self) -> List[XIndex]: + "List of indexes defined on this table using the more detailed ``XIndex`` format." sql = 'PRAGMA index_list("{}")'.format(self.name) indexes = [] for row in self.db.execute_returning_dicts(sql): @@ -934,7 +1156,8 @@ class Table(Queryable): return indexes @property - def triggers(self): + def triggers(self) -> List[Trigger]: + "List of triggers defined on this table." return [ Trigger(*r) for r in self.db.execute( @@ -945,8 +1168,8 @@ class Table(Queryable): ] @property - def triggers_dict(self): - "Returns {trigger_name: sql} dictionary" + def triggers_dict(self) -> Dict[str, str]: + "``{trigger_name: sql}`` dictionary of triggers defined on this table." return {trigger.name: trigger.sql for trigger in self.triggers} def create( @@ -959,7 +1182,12 @@ class Table(Queryable): defaults=None, hash_id=None, extracts=None, - ): + ) -> "Table": + """ + Create a table with the specified columns. + + See :ref:`python_api_explicit_create` for full details. + """ columns = {name: value for (name, value) in columns.items()} with self.db.conn: self.db.create_table( @@ -986,7 +1214,13 @@ class Table(Queryable): defaults=None, drop_foreign_keys=None, column_order=None, - ): + ) -> "Table": + """ + Apply an advanced alter table, including operations that are not supported by + ``ALTER TABLE`` in SQLite itself. + + See :ref:`python_api_transform` for full details. + """ assert self.exists(), "Cannot transform a table that doesn't exist yet" sqls = self.transform_sql( types=types, @@ -1027,7 +1261,8 @@ class Table(Queryable): drop_foreign_keys=None, column_order=None, tmp_suffix=None, - ): + ) -> List[str]: + "Returns a list of SQL statements that would be executed in order to apply this transformation." types = types or {} rename = rename or {} drop = drop or set() @@ -1133,7 +1368,18 @@ class Table(Queryable): ) return sqls - def extract(self, columns, table=None, fk_column=None, rename=None): + def extract( + self, + columns: Union[str, Iterable[str]], + table: Optional[str] = None, + fk_column: Optional[str] = None, + rename: Optional[Dict[str, str]] = None, + ) -> "Table": + """ + Extract specified columns into a separate table. + + See :ref:`python_api_extract` for details. + """ rename = rename or {} if isinstance(columns, str): columns = [columns] @@ -1225,7 +1471,24 @@ class Table(Queryable): self.add_foreign_key(fk_column, table, "id") return self - def create_index(self, columns, index_name=None, unique=False, if_not_exists=False): + def create_index( + self, + columns: Iterable[Union[str, DescIndex]], + index_name: Optional[str] = None, + unique: bool = False, + if_not_exists: bool = False, + ): + """ + Create an index on this table. + + - ``columns`` - a single columns or list of columns to index. These can be strings or, + to create an index using the column in descending order, ``db.DescIndex(column_name)`` objects. + - ``index_name`` - the name to use for the new index. Defaults to the column names joined on ``_``. + - ``unique`` - should the index be marked as unique, forcing unique values? + - ``if_not_exists`` - only create the index if one with that name does not already exist. + + See :ref:`python_api_create_index`. + """ if index_name is None: index_name = "idx_{}_{}".format( self.name.replace(" ", "_"), "_".join(columns) @@ -1257,8 +1520,9 @@ class Table(Queryable): return self def add_column( - self, col_name, col_type=None, fk=None, fk_col=None, not_null_default=None + self, col_name: str, col_type=None, fk=None, fk_col=None, not_null_default=None ): + "Add a column to this table. See :ref:`python_api_add_column`." fk_col_type = None if fk is not None: # fk must be a valid table @@ -1293,14 +1557,24 @@ class Table(Queryable): self.add_foreign_key(col_name, fk, fk_col) return self - def drop(self, ignore=False): + def drop(self, ignore: bool = False): + "Drop this table. ``ignore=True`` means errors will be ignored." try: self.db.execute("DROP TABLE [{}]".format(self.name)) except sqlite3.OperationalError: if not ignore: raise - def guess_foreign_table(self, column): + def guess_foreign_table(self, column: str) -> str: + """ + For a given column, suggest another table that might be referenced by this + column should it be used as a foreign key. + + For example, a column called ``tag_id`` or ``tag`` or ``tags`` might suggest + a ``tag`` table, if one exists. + + If no candidates can be found, raises a ``NoObviousTable`` exception. + """ column = column.lower() possibilities = [column] if column.endswith("_id"): @@ -1321,7 +1595,7 @@ class Table(Queryable): ) ) - def guess_foreign_column(self, other_table): + def guess_foreign_column(self, other_table: str): pks = [c for c in self.db[other_table].columns if c.is_pk] if len(pks) != 1: raise BadPrimaryKey( @@ -1331,8 +1605,20 @@ class Table(Queryable): return pks[0].name def add_foreign_key( - self, column, other_table=None, other_column=None, ignore=False + self, + column: str, + other_table: Optional[str] = None, + other_column: Optional[str] = None, + ignore: bool = False, ): + """ + Alter the schema to mark the specified column as a foreign key to another table. + + - ``column`` - the column to mark as a foreign key. + - ``other_table`` - the table it refers to - if omitted, will be guessed based on the column name. + - ``other_column`` - the column on the other table it - if omitted, will be guessed. + - ``ignore`` - set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError` will be raised. + """ # Ensure column exists if column not in self.columns_dict: raise AlterError("No such column: {}".format(column)) @@ -1369,6 +1655,11 @@ class Table(Queryable): return self def enable_counts(self): + """ + Set up triggers to update a cache of the count of rows in this table. + + See :ref:`python_api_cached_table_counts` for details. + """ sql = ( textwrap.dedent( """ @@ -1413,7 +1704,8 @@ class Table(Queryable): self.db.use_counts_table = True @property - def has_counts_triggers(self): + def has_counts_triggers(self) -> bool: + "Does this table have triggers setup to update cached counts?" trigger_names = { "{table}{counts_table}_{suffix}".format( counts_table=self.db._counts_table_name, table=self.name, suffix=suffix @@ -1424,13 +1716,23 @@ class Table(Queryable): def enable_fts( self, - columns, - fts_version="FTS5", - create_triggers=False, - tokenize=None, - replace=False, + columns: Iterable[str], + fts_version: str = "FTS5", + create_triggers: bool = False, + tokenize: Optional[str] = None, + replace: bool = False, ): - "Enables FTS on the specified columns." + """ + Enable SQLite full-text search against the specified columns. + + - ``columns`` - list of column names to include in the search index. + - ``fts_version`` - FTS version to use - defaults to ``FTS5`` but you may want ``FTS4`` for older SQLite versions. + - ``create_triggers`` - should triggers be created to keep the search index up-to-date? Defaults to ``False``. + - ``tokenize`` - custom SQLite tokenizer to use, for example ``"porter"`` to enable Porter stemming. + - ``replace`` - should any existing FTS index for this table be replaced by the new one? + + See :ref:`python_api_fts` for more details. + """ create_fts_sql = ( textwrap.dedent( """ @@ -1498,7 +1800,11 @@ class Table(Queryable): self.db.executescript(triggers) return self - def populate_fts(self, columns): + def populate_fts(self, columns: Iterable[str]) -> "Table": + """ + Update the associated SQLite full-text search index with the latest data from the + table for the specified columns. + """ sql = ( textwrap.dedent( """ @@ -1514,7 +1820,8 @@ class Table(Queryable): self.db.executescript(sql) return self - def disable_fts(self): + def disable_fts(self) -> "Table": + "Remove any full-text search index and related triggers configured for this table." fts_table = self.detect_fts() if fts_table: self.db[fts_table].drop() @@ -1539,6 +1846,7 @@ class Table(Queryable): return self def rebuild_fts(self): + "Run the ``rebuild`` operation against the associated full-text search index table." fts_table = self.detect_fts() if fts_table is None: # Assume this is itself an FTS table @@ -1550,7 +1858,7 @@ class Table(Queryable): ) return self - def detect_fts(self): + def detect_fts(self) -> Optional[str]: "Detect if table has a corresponding FTS virtual table and return it" sql = ( textwrap.dedent( @@ -1575,7 +1883,8 @@ class Table(Queryable): else: return rows[0][0] - def optimize(self): + def optimize(self) -> "Table": + "Run the ``optimize`` operation against the associated full-text search index table." fts_table = self.detect_fts() if fts_table is not None: self.db.execute( @@ -1587,7 +1896,8 @@ class Table(Queryable): ) return self - def search_sql(self, columns=None, order_by=None, limit=None, offset=None): + def search_sql(self, columns=None, order_by=None, limit=None, offset=None) -> str: + "Return SQL string that can be used to execute searches against this table." # Pick names for table and rank column that don't clash original = "original_" if self.name == "original" else "original" columns_sql = "*" @@ -1644,7 +1954,26 @@ class Table(Queryable): limit_offset=limit_offset.strip(), ).strip() - def search(self, q, order_by=None, columns=None, limit=None, offset=None): + def search( + self, + q: str, + order_by: Optional[str] = None, + columns: Optional[List[str]] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Generator[dict, None, None]: + """ + Execute a search against this table using SQLite full-text search, returning a sequence of + dictionaries for each row. + + - ``q`` - words to search for + - ``order_by`` - defaults to order by rank, or specify a column here. + - ``columns`` - list of columns to return, defaults to all columns. + - ``limit`` - optional integer limit for returned rows. + - ``offset`` - optional integer SQL offset. + + See :ref:`python_api_fts_search`. + """ cursor = self.db.execute( self.search_sql( order_by=order_by, @@ -1661,7 +1990,8 @@ class Table(Queryable): def value_or_default(self, key, value): return self._defaults[key] if value is DEFAULT else value - def delete(self, pk_values): + def delete(self, pk_values: Union[list, tuple, str, int, float]) -> "Table": + "Delete row matching the specified primary key." if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] self.get(pk_values) @@ -1673,16 +2003,37 @@ class Table(Queryable): self.db.execute(sql, pk_values) return self - def delete_where(self, where=None, where_args=None): + def delete_where( + self, where: str = None, where_args: Optional[Union[Iterable, dict]] = None + ) -> "Table": + "Delete rows matching specified where clause, or delete all rows in the table." if not self.exists(): - return [] + return self sql = "delete from [{}]".format(self.name) if where is not None: sql += " where " + where self.db.execute(sql, where_args or []) return self - def update(self, pk_values, updates=None, alter=False, conversions=None): + def update( + self, + pk_values: Union[list, tuple, str, int, float], + updates: Optional[dict] = None, + alter: bool = False, + conversions: Optional[dict] = None, + ) -> "Table": + """ + Execute a SQL ``UPDATE`` against the specified row. + + - ``pk_values`` - the primary key of an individual record - can be a tuple if the + table has a compound primary key. + - ``updates`` - a dictionary mapping columns to their updated values. + - ``alter``` - set to ``True`` to add any missing columns. + - ``conversions`` - optional dictionary of SQL functions to apply during the update, for example + ``{"mycolumn": "upper(?)"}``. + + See :ref:`python_api_update`. + """ updates = updates or {} conversions = conversions or {} if not isinstance(pk_values, (list, tuple)): @@ -1722,16 +2073,34 @@ class Table(Queryable): def convert( self, - columns, - fn, - output=None, - output_type=None, - drop=False, - multi=False, - where=None, - where_args=None, - show_progress=False, + columns: Union[str, List[str]], + fn: Callable, + output: Optional[str] = None, + output_type: Optional[Any] = None, + drop: bool = False, + multi: bool = False, + where: Optional[str] = None, + where_args: Optional[Union[Iterable, dict]] = None, + show_progress: bool = False, ): + """ + Apply conversion function ``fn`` to every value in the specified columns. + + - ``columns`` - a single column or list of string column names to convert. + - ``fn`` - a callable that takes a single argument, ``value``, and returns it converted. + - ``output`` - optional string column name to write the results to (defaults to the input column). + - ``output_type`` - if the output column needs to be created, this is the type that will be used + for the new column. + - ``drop`` - boolean, should the original column be dropped once the conversion is complete? + - ``multi`` - boolean, if ``True`` the return value of ``fn(value)`` will be expected to be a + dictionary, and new columns will be created for each key of that dictionary. + - ``where`` - a SQL fragment to use as a ``WHERE`` clause to limit the rows to which the conversion + is applied, for example ``age > ?`` or ``age > :age``. + - ``where_args`` - a list of arguments (if using ``?``) or a dictionary (if using ``:age``). + - ``show_progress`` - boolean, should a progress bar be displayed? + + See :ref:`python_api_convert`. + """ if isinstance(columns, str): columns = [columns] @@ -2012,20 +2381,51 @@ class Table(Queryable): def insert( self, - record, + record: Dict[str, Any], pk=DEFAULT, foreign_keys=DEFAULT, - column_order=DEFAULT, - not_null=DEFAULT, - defaults=DEFAULT, - hash_id=DEFAULT, - alter=DEFAULT, - ignore=DEFAULT, - replace=DEFAULT, - extracts=DEFAULT, - conversions=DEFAULT, - columns=DEFAULT, - ): + column_order: Optional[Union[List[str], Default]] = DEFAULT, + not_null: Optional[Union[Set[str], Default]] = DEFAULT, + defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT, + hash_id: Optional[Union[str, Default]] = DEFAULT, + alter: Optional[Union[bool, Default]] = DEFAULT, + ignore: Optional[Union[bool, Default]] = DEFAULT, + replace: Optional[Union[bool, Default]] = DEFAULT, + extracts: Optional[Union[Dict[str, str], List[str], Default]] = DEFAULT, + conversions: Optional[Union[Dict[str, str], Default]] = DEFAULT, + columns: Optional[Union[Dict[str, Any], Default]] = DEFAULT, + ) -> "Table": + """ + Insert a single record into the table. The table will be created with a schema that matches + the inserted record if it does not already exist, see :ref:`python_api_creating_tables`. + + - ``record`` - required: a dictionary representing the record to be inserted. + + The other parameters are optional, and mostly influence how the new table will be created if + that table does not exist yet. + + Each of them defaults to ``DEFAULT``, which indicates that the default setting for the current + ``Table`` object (specified in the table constructor) should be used. + + - ``pk`` - if creating the table, which column should be the primary key. + - ``foreign_keys`` - see :ref:`python_api_foreign_keys`. + - ``column_order`` - optional list of strings specifying a full or partial column order + to use when creating the table. + - ``not_null`` - optional set of strings specifying columns that should be ``NOT NULL``. + - ``defaults`` - optional dictionary specifying default values for specific columns. + - ``hash_id`` - optional name of a column to create and use as a primary key, where the + value of thet primary key will be derived as a SHA1 hash of the other column values + in the record. ``hash_id="id"`` is a common column name used for this. + - ``alter`` - boolean, should any missing columns be added automatically? + - ``ignore`` - boolean, if a record already exists with this primary key, ignore this insert. + - ``replace`` - boolean, if a record already exists with this primary key, replace it with this new record. + - ``extracts`` - a list of columns to extract to other tables, or a dictionary that maps + ``{column_name: other_table_name}``. See :ref:`python_api_extracts`. + - ``conversions`` - dictionary specifying SQL conversion functions to be applied to the data while it + is being inserted, for example ``{"name": "upper(?)"}``. See :ref:`python_api_conversions`. + - ``columns`` - dictionary over-riding the detected types used for the columns, for example + ``{"age": int, "weight": float}``. + """ return self.insert_all( [record], pk=pk, @@ -2060,11 +2460,10 @@ class Table(Queryable): conversions=DEFAULT, columns=DEFAULT, upsert=False, - ): + ) -> "Table": """ - Like .insert() but takes a list of records and ensures that the table - that it creates (if table does not exist) has columns for ALL of that - data + Like ``.insert()`` but takes a list of records and ensures that the table + that it creates (if table does not exist) has columns for ALL of that data. """ pk = self.value_or_default("pk", pk) foreign_keys = self.value_or_default("foreign_keys", foreign_keys) @@ -2089,7 +2488,7 @@ class Table(Queryable): assert not ( ignore and replace ), "Use either ignore=True or replace=True, not both" - all_columns = None + all_columns = [] first = True num_records_processed = 0 # We can only handle a max of 999 variables in a SQL insert, so @@ -2127,10 +2526,10 @@ class Table(Queryable): hash_id=hash_id, extracts=extracts, ) - all_columns = set() + all_columns_set = set() for record in chunk: - all_columns.update(record.keys()) - all_columns = list(sorted(all_columns)) + all_columns_set.update(record.keys()) + all_columns = list(sorted(all_columns_set)) if hash_id: all_columns.insert(0, hash_id) else: @@ -2171,7 +2570,13 @@ class Table(Queryable): extracts=DEFAULT, conversions=DEFAULT, columns=DEFAULT, - ): + ) -> "Table": + """ + Like ``.insert()`` but performs an ``UPSERT``, where records are inserted if they do + not exist and updated if they DO exist, based on matching against their primary key. + + See :ref:`python_api_upsert`. + """ return self.upsert_all( [record], pk=pk, @@ -2200,7 +2605,10 @@ class Table(Queryable): extracts=DEFAULT, conversions=DEFAULT, columns=DEFAULT, - ): + ) -> "Table": + """ + Like ``.upsert()`` but can be applied to a list of records. + """ return self.insert_all( records, pk=pk, @@ -2217,7 +2625,7 @@ class Table(Queryable): upsert=True, ) - def add_missing_columns(self, records): + def add_missing_columns(self, records: Iterable[Dict[str, Any]]) -> "Table": needed_columns = suggest_column_types(records) current_columns = {c.lower() for c in self.columns_dict} for col_name, col_type in needed_columns.items(): @@ -2225,7 +2633,20 @@ class Table(Queryable): self.add_column(col_name, col_type) return self - def lookup(self, column_values): + def lookup(self, column_values: Dict[str, Any]): + """ + Create or populate a lookup table with the specified values. + + ``db["Species"].lookup({"name": "Palm"})`` will create a table called ``Species`` + (if one does not already exist) with two columns: ``id`` and ``name``. It will + set up a unique constraint on the ``name`` column to guarantee it will not + contain duplicate rows. + + It well then inserts a new row with the ``name`` set to ``Palm`` and return the + new integer primary key value. + + See :ref:`python_api_lookup_tables` for more details. + """ # lookups is a dictionary - all columns will be used for a unique index assert isinstance(column_values, dict) if self.exists(): @@ -2250,15 +2671,38 @@ class Table(Queryable): def m2m( self, - other_table, - record_or_iterable=None, - pk=DEFAULT, - lookup=None, - m2m_table=None, - alter=False, + other_table: Union[str, "Table"], + record_or_iterable: Optional[ + Union[Iterable[Dict[str, Any]], Dict[str, Any]] + ] = None, + pk: Optional[Union[Any, Default]] = DEFAULT, + lookup: Optional[Dict[str, Any]] = None, + m2m_table: Optional[str] = None, + alter: bool = False, ): + """ + After inserting a record in a table, create one or more records in some other + table and then create many-to-many records linking the original record and the + newly created records together. + + For example:: + + db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id").m2m( + "humans", {"id": 1, "name": "Natalie"}, pk="id" + ) + See :ref:`python_api_m2m` for details. + + - ``other_table`` - the name of the table to insert the new records into. + - ``record_or_iterable`` - a single dictionary record to insert, or a list of records. + - ``pk`` - the primary key to use if creating ``other_table``. + - ``lookup`` - same dictionary as for ``.lookup()``, to create a many-to-many lookup table. + - ``m2m_table`` - the string name to use for the many-to-many table, defaults to creating + this automatically based on the names of the two tables. + - ``alter``` - set to ``True`` to add any missing columns on ``other_table`` if that table + already exists. + """ if isinstance(other_table, str): - other_table = self.db.table(other_table, pk=pk) + other_table = cast(Table, self.db.table(other_table, pk=pk)) our_id = self.last_pk if lookup is not None: assert record_or_iterable is None, "Provide lookup= or record, not both" @@ -2282,20 +2726,19 @@ class Table(Queryable): else: # If not, create a new table m2m_table_name = m2m_table or "{}_{}".format(*tables) - m2m_table = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) + m2m_table_obj = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) if lookup is None: # if records is only one record, put the record in a list - records = ( - [record_or_iterable] - if isinstance(record_or_iterable, Mapping) - else record_or_iterable - ) + if isinstance(record_or_iterable, Mapping): + records = [record_or_iterable] + else: + records = cast(List, record_or_iterable) # Ensure each record exists in other table for record in records: id = other_table.insert( - record, pk=pk, replace=True, alter=alter + cast(dict, record), pk=pk, replace=True, alter=alter ).last_pk - m2m_table.insert( + m2m_table_obj.insert( { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, @@ -2304,7 +2747,7 @@ class Table(Queryable): ) else: id = other_table.lookup(lookup) - m2m_table.insert( + m2m_table_obj.insert( { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, @@ -2314,8 +2757,13 @@ class Table(Queryable): return self def analyze_column( - self, column, common_limit=10, value_truncate=None, total_rows=None - ): + self, column: str, common_limit: int = 10, value_truncate=None, total_rows=None + ) -> "ColumnDetails": + """ + Return statistics about the specified column. + + See :ref:`python_api_analyze_column`. + """ db = self.db table = self.name if total_rows is None: From 86fc9fb5c8073af8e20acc6af25974b89ec4720a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 10 Aug 2021 16:51:59 -0700 Subject: [PATCH 0420/1004] Release 3.15.1 Refs #311, #312, #313, #315 --- docs/changelog.rst | 10 ++++++++++ docs/reference.rst | 2 ++ setup.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9a1669d..da6e746 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,16 @@ Changelog =========== +.. _v3_15.1: + +3.15.1 (2021-08-10) +------------------- + +- Python library now includes type annotations on almost all of the methods, plus detailed docstrings describing each one. (:issue:`311`) +- New :ref:`reference` documentation page, powered by those docstrings. +- Fixed bug where ``.add_foreign_keys()`` failed to raise an error if called against a ``View``. (:issue:`313`) +- Fixed bug where ``.delete_where()`` returned a ``[]`` instead of returning ``self`` if called against a non-existant table. (:issue:`315`) + .. _v3_15: 3.15 (2021-08-09) diff --git a/docs/reference.rst b/docs/reference.rst index e29f468..8331b1e 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -1,3 +1,5 @@ +.. _reference: + =============== API Reference =============== diff --git a/setup.py b/setup.py index c8c58ac..65fe6f7 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.15" +VERSION = "3.15.1" def get_long_description(): From bde372525734bd41d94251675141422b0fd56bda Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 10 Aug 2021 16:55:12 -0700 Subject: [PATCH 0421/1004] Fixed spelling existent --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index da6e746..ab424ed 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ - Python library now includes type annotations on almost all of the methods, plus detailed docstrings describing each one. (:issue:`311`) - New :ref:`reference` documentation page, powered by those docstrings. - Fixed bug where ``.add_foreign_keys()`` failed to raise an error if called against a ``View``. (:issue:`313`) -- Fixed bug where ``.delete_where()`` returned a ``[]`` instead of returning ``self`` if called against a non-existant table. (:issue:`315`) +- Fixed bug where ``.delete_where()`` returned a ``[]`` instead of returning ``self`` if called against a non-existent table. (:issue:`315`) .. _v3_15: From 3091bec4f7bab85c94fe2879a36c96474e152230 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 11 Aug 2021 04:54:00 -0700 Subject: [PATCH 0422/1004] Don't show inheritance for Database class --- docs/reference.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/reference.rst b/docs/reference.rst index 8331b1e..ea1203e 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -14,7 +14,6 @@ sqlite_utils.db.Database .. autoclass:: sqlite_utils.db.Database :members: :undoc-members: - :show-inheritance: :special-members: __getitem__ :exclude-members: use_counts_table, execute_returning_dicts, resolve_foreign_keys From af89c5f8513ad6c4228e5f8c8b6c9b5c98c12f63 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 11 Aug 2021 04:56:54 -0700 Subject: [PATCH 0423/1004] How to create a Database instance --- sqlite_utils/db.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c7dc832..7ba4ecb 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -251,6 +251,13 @@ class Database: """ Wrapper for a SQLite database connection that adds a variety of useful utility methods. + To create an instance:: + + # create data.db file, or open existing: + db = Database("data.db") + # Create an in-memory database: + dB = Database(memory=True) + - ``filename_or_conn`` - String path to a file, or a ``pathlib.Path`` object, or a ``sqlite3`` connection - ``memory`` - set to ``True`` to create an in-memory database From 6de0a5d46a00a66d827c32deaca5cbd0ad2103ad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 11 Aug 2021 05:03:07 -0700 Subject: [PATCH 0424/1004] Typo fix --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 7ba4ecb..55067d0 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -746,7 +746,7 @@ class Database: Create a new SQL view with the specified name - ``sql`` should start with ``SELECT ...``. - ``ignore`` - set to ``True`` to do nothing if a view with this name already exists - - ``replace`` - set to ``True`` to do replace the view if one with this name already exists + - ``replace`` - set to ``True`` to replace the view if one with this name already exists """ assert not ( ignore and replace From b966c44ef81bc6acbc4be95942afcf33b31e876f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 13 Aug 2021 04:32:40 -0700 Subject: [PATCH 0425/1004] Minor markup fix --- sqlite_utils/db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 55067d0..ec399fa 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -991,9 +991,9 @@ class Queryable: class Table(Queryable): "Tables should usually be initialized using the ``db.table(table_name)`` or ``db[table_name]`` methods." - #: The ``rowid`` of the last inserted, updated or selected row.` + #: The ``rowid`` of the last inserted, updated or selected row. last_rowid: Optional[int] = None - #: The primary key of the last inserted, updated or selected row.` + #: The primary key of the last inserted, updated or selected row. last_pk: Optional[Any] = None def __init__( From 7ee7b628e101863c73c2a95911bd2213de00fb1f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 13 Aug 2021 22:10:47 -0700 Subject: [PATCH 0426/1004] Fixed some rogue backticks, closes #316 --- sqlite_utils/db.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ec399fa..520008f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -84,7 +84,7 @@ Describes a SQLite column returned by the :attr:`.Table.columns` property. Column type ``notnull`` - Does the column have a ``not null` constraint + Does the column have a ``not null`` constraint ``default_value`` Default value for this column @@ -128,10 +128,10 @@ Summary information about a column, see :ref:`python_api_analyze_column`. The number of distinct values in this column ``most_common`` - The ``N`` most common values as a list of ``(value, count)`` tuples`, or ``None`` if the table consists entirely of distinct values + The ``N`` most common values as a list of ``(value, count)`` tuples, or ``None`` if the table consists entirely of distinct values ``least_common`` - The ``N`` least common values as a list of ``(value, count)`` tuples`, or ``None`` if the table is entirely distinct + The ``N`` least common values as a list of ``(value, count)`` tuples, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) """ ForeignKey = namedtuple( @@ -1624,7 +1624,7 @@ class Table(Queryable): - ``column`` - the column to mark as a foreign key. - ``other_table`` - the table it refers to - if omitted, will be guessed based on the column name. - ``other_column`` - the column on the other table it - if omitted, will be guessed. - - ``ignore`` - set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError` will be raised. + - ``ignore`` - set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. """ # Ensure column exists if column not in self.columns_dict: @@ -2035,7 +2035,7 @@ class Table(Queryable): - ``pk_values`` - the primary key of an individual record - can be a tuple if the table has a compound primary key. - ``updates`` - a dictionary mapping columns to their updated values. - - ``alter``` - set to ``True`` to add any missing columns. + - ``alter`` - set to ``True`` to add any missing columns. - ``conversions`` - optional dictionary of SQL functions to apply during the update, for example ``{"mycolumn": "upper(?)"}``. @@ -2705,7 +2705,7 @@ class Table(Queryable): - ``lookup`` - same dictionary as for ``.lookup()``, to create a many-to-many lookup table. - ``m2m_table`` - the string name to use for the many-to-many table, defaults to creating this automatically based on the names of the two tables. - - ``alter``` - set to ``True`` to add any missing columns on ``other_table`` if that table + - ``alter`` - set to ``True`` to add any missing columns on ``other_table`` if that table already exists. """ if isinstance(other_table, str): From 7a19822ac9ee24be2fbb4c2326a0bf2f3d2d9c4d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 17 Aug 2021 08:42:02 -0700 Subject: [PATCH 0427/1004] Updated tagline --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 581f306..91e7431 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,7 +13,7 @@ .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg :target: https://github.com/simonw/sqlite-utils/blob/main/LICENSE -*Python utility functions for manipulating SQLite databases* +*CLI tool and Python utility functions for manipulating SQLite databases* This library and command-line utility helps create SQLite databases from an existing collection of data. From 1fe73c898b44695052f1a9ca832818d50cecf662 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 11:31:20 -0700 Subject: [PATCH 0428/1004] Remove link to older code example --- docs/index.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 91e7431..0629e0e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,5 +35,3 @@ Contents reference contributing changelog - -Take a look at `this script `_ for an example of this library in action. From 53fec0d8639d2a66e322d05e1fcc8f34caa57815 Mon Sep 17 00:00:00 2001 From: Mark Neumann Date: Wed, 18 Aug 2021 19:43:11 +0100 Subject: [PATCH 0429/1004] db.quote_fts() method, thanks Mark Neumann Refs #296, closes #246. --- sqlite_utils/db.py | 21 +++++++++++++++++++++ tests/test_fts.py | 11 +++++++++++ 2 files changed, 32 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 520008f..f51dba6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -38,6 +38,8 @@ import uuid SQLITE_MAX_VARS = 999 +_quote_fts_re = re.compile(r'\s+|(".*?")') + _virtual_table_using_re = re.compile( r""" ^ # Start of string @@ -431,6 +433,25 @@ class Database: {"value": value}, ).fetchone()[0] + def quote_fts(self, query: str) -> str: + "Escape special characters in a SQLite full-text search query" + # NOTE: This is not a query validator for FTS. Sqlite has + # a well defined query syntax here: + # https://www2.sqlite.org/fts5.html#full_text_query_syntax + # but this function just aggressively quotes strings + # to ensure that they are valid. In particular, passing + # queries which make use of the query syntax will be incorrect, + # e.g 'NEAR(one, two, 3)'. + + # If query has unbalanced ", add one at end + if query.count('"') % 2: + query += '"' + bits = _quote_fts_re.split(query) + bits = [b for b in bits if b and b != '""'] + return " ".join( + '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits + ) + def table_names(self, fts4: bool = False, fts5: bool = False) -> List[str]: "A list of string table names in this database." where = ["type = 'table'"] diff --git a/tests/test_fts.py b/tests/test_fts.py index a3efa54..defff9d 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -501,3 +501,14 @@ def test_search_sql(kwargs, fts, expected): db["books"].enable_fts(["title", "author"], fts_version=fts) sql = db["books"].search_sql(**kwargs) assert sql == expected + +def test_quote_fts_query(fresh_db): + + table = fresh_db["searchable"] + table.insert_all(search_records) + table.enable_fts(["text", "country"]) + + query = "cat's" + result = fresh_db.quote_fts(query) + # Executing query does not crash. + list(table.search(result)) From e6b10227919c167288990ba6151adb63ea1c143b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 11:55:19 -0700 Subject: [PATCH 0430/1004] Fix markup warning in docstring --- sqlite_utils/db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f51dba6..306ff18 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2718,6 +2718,7 @@ class Table(Queryable): db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id").m2m( "humans", {"id": 1, "name": "Natalie"}, pk="id" ) + See :ref:`python_api_m2m` for details. - ``other_table`` - the name of the table to insert the new records into. From 1fa5a12a4952b02341e2a59ba75aabb740518ecb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 11:55:50 -0700 Subject: [PATCH 0431/1004] Documentation for db.quote_fts(), refs #246 --- docs/python-api.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index b793920..0a0e1df 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1909,6 +1909,18 @@ To remove the FTS tables and triggers you created, use the ``disable_fts()`` tab db["dogs"].disable_fts() +.. _python_api_quote_fts: + +Quoting characters for use in search +------------------------------------ + +SQLite supports `advanced search query syntax `__. In some situations you may wish to disable this, since characters such as ``.`` may have special meaning that causes errors when searching for strings provided by your users. + +The ``db.quote_fts(query)`` method returns the query with SQLite full-text search quoting applied such that the query should be safe to use in a search:: + + db.quote_fts("Search term.") + # Returns: '"Search" "term."' + .. _python_api_fts_search: Searching with table.search() From f0fd19267f937a067c4b6f2eb195bcf96fece5a4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 11:56:36 -0700 Subject: [PATCH 0432/1004] Black/flake8, refs #246 --- tests/test_fts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_fts.py b/tests/test_fts.py index defff9d..7a414da 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -502,6 +502,7 @@ def test_search_sql(kwargs, fts, expected): sql = db["books"].search_sql(**kwargs) assert sql == expected + def test_quote_fts_query(fresh_db): table = fresh_db["searchable"] From 8ae77a6961fed94ef2c9cc81fcfc7c81d222d9a2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 12:55:53 -0700 Subject: [PATCH 0433/1004] table.search(quote=True) parameter, refs #296 --- docs/python-api.rst | 14 ++++++++++++-- sqlite_utils/db.py | 16 ++++++++++++---- tests/test_fts.py | 37 +++++++++++++++++++++++++++++++------ 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 0a0e1df..20d080d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1843,8 +1843,15 @@ The ``.has_counts_triggers`` property shows if a table has been configured with .. _python_api_fts: -Enabling full-text search -========================= +Full-text search +================ + +SQLite includes bundled extensions that implement `powerful full-text search `__. + +.. _python_api_fts_enable: + +Enabling full-text search for a table +------------------------------------- You can enable full-text search on a table using ``.enable_fts(columns)``: @@ -1947,6 +1954,9 @@ The ``.search()`` method also accepts the following optional parameters: ``offset`` integer Offset to use along side the limit parameter. +``quote`` bool + Apply :ref:`FTS quoting rules ` to the search query, disabling advanced query syntax in a way that avoids surprising errors. + To return just the title and published columns for three matches for ``"dog"`` ordered by ``published`` with the most recent first, use the following: .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 306ff18..e4c1d5c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1924,7 +1924,13 @@ class Table(Queryable): ) return self - def search_sql(self, columns=None, order_by=None, limit=None, offset=None) -> str: + def search_sql( + self, + columns: Optional[Iterable[str]] = None, + order_by: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> str: "Return SQL string that can be used to execute searches against this table." # Pick names for table and rank column that don't clash original = "original_" if self.name == "original" else "original" @@ -1986,19 +1992,21 @@ class Table(Queryable): self, q: str, order_by: Optional[str] = None, - columns: Optional[List[str]] = None, + columns: Optional[Iterable[str]] = None, limit: Optional[int] = None, offset: Optional[int] = None, + quote: bool = False, ) -> Generator[dict, None, None]: """ Execute a search against this table using SQLite full-text search, returning a sequence of dictionaries for each row. - - ``q`` - words to search for + - ``q`` - terms to search for - ``order_by`` - defaults to order by rank, or specify a column here. - ``columns`` - list of columns to return, defaults to all columns. - ``limit`` - optional integer limit for returned rows. - ``offset`` - optional integer SQL offset. + - ``quote`` - apply quoting to disable any special characters in the search query See :ref:`python_api_fts_search`. """ @@ -2009,7 +2017,7 @@ class Table(Queryable): limit=limit, offset=offset, ), - {"query": q}, + {"query": self.db.quote_fts(q) if quote else q}, ) columns = [c[0] for c in cursor.description] for row in cursor: diff --git a/tests/test_fts.py b/tests/test_fts.py index 7a414da..6aa6fbe 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -503,13 +503,38 @@ def test_search_sql(kwargs, fts, expected): assert sql == expected -def test_quote_fts_query(fresh_db): - +@pytest.mark.parametrize( + "input,expected", + ( + ("dog", '"dog"'), + ("cat,", '"cat,"'), + ("cat's", '"cat\'s"'), + ("dog.", '"dog."'), + ("cat dog", '"cat" "dog"'), + # If a phrase is already double quoted, leave it so + ('"cat dog"', '"cat dog"'), + ('"cat dog" fish', '"cat dog" "fish"'), + # Sensibly handle unbalanced double quotes + ('cat"', '"cat"'), + ('"cat dog" "fish', '"cat dog" "fish"'), + ), +) +def test_quote_fts_query(fresh_db, input, expected): table = fresh_db["searchable"] table.insert_all(search_records) table.enable_fts(["text", "country"]) - - query = "cat's" - result = fresh_db.quote_fts(query) + quoted = fresh_db.quote_fts(input) + assert quoted == expected # Executing query does not crash. - list(table.search(result)) + list(table.search(quoted)) + + +def test_search_quote(fresh_db): + table = fresh_db["searchable"] + table.insert_all(search_records) + table.enable_fts(["text", "country"]) + query = "cat's" + with pytest.raises(sqlite3.OperationalError): + list(table.search(query)) + # No exception with quote=True + list(table.search(query, quote=True)) From ccf128cd6df57f9db1900f043aaa540928f9c844 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 13:10:44 -0700 Subject: [PATCH 0434/1004] sqlite-utils search --quote option, closes #296 --- docs/cli.rst | 2 ++ sqlite_utils/cli.py | 42 +++++++++++++++++++++++++++--------------- tests/test_cli.py | 19 +++++++++++++++++++ 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 0c08526..6a96a82 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1519,6 +1519,8 @@ By default it shows the most relevant matches first. You can specify a different # Sort by created in descending order $ sqlite-utils search mydb.db documents searchterm -o 'created desc' +SQLite `advanced search syntax `__ is enabled by default. To run a search with automatic quoting use the ``--quote`` option. + You can specify a subset of columns to be returned using the ``-c`` option one or more times:: $ sqlite-utils search mydb.db documents searchterm -c title -c created diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index d94229d..4b39a5e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1375,6 +1375,7 @@ def _execute_query( @click.option( "--sql", "show_sql", is_flag=True, help="Show SQL query that would be run" ) +@click.option("--quote", is_flag=True, help="Apply FTS quoting rules to search term") @output_options @load_extension_option @click.pass_context @@ -1385,6 +1386,7 @@ def search( q, order, show_sql, + quote, column, limit, nl, @@ -1420,21 +1422,31 @@ def search( if show_sql: click.echo(sql) return - ctx.invoke( - query, - path=path, - sql=sql, - nl=nl, - arrays=arrays, - csv=csv, - tsv=tsv, - no_headers=no_headers, - table=table, - fmt=fmt, - json_cols=json_cols, - param=[("query", q)], - load_extension=load_extension, - ) + if quote: + q = db.quote_fts(q) + try: + ctx.invoke( + query, + path=path, + sql=sql, + nl=nl, + arrays=arrays, + csv=csv, + tsv=tsv, + no_headers=no_headers, + table=table, + fmt=fmt, + json_cols=json_cols, + param=[("query", q)], + load_extension=load_extension, + ) + except click.ClickException as e: + if "malformed MATCH expression" in str(e) or "unterminated string" in str(e): + raise click.ClickException( + "{}\n\nTry running this again with the --quote option".format(str(e)) + ) + else: + raise @cli.command() diff --git a/tests/test_cli.py b/tests/test_cli.py index d4801ac..58347cd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1994,6 +1994,25 @@ def test_search(tmpdir, fts, extra_arg, expected): assert result.output.replace("\r", "") == expected +def test_search_quote(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["creatures"].insert({"name": "dog."}).enable_fts(["name"]) + # Without --quote should return an error + error_result = CliRunner().invoke(cli.cli, ["search", db_path, "creatures", 'dog"']) + assert error_result.exit_code == 1 + assert error_result.output == ( + "Error: unterminated string\n\n" + "Try running this again with the --quote option\n" + ) + # With --quote it should work + result = CliRunner().invoke( + cli.cli, ["search", db_path, "creatures", 'dog"', "--quote"] + ) + assert result.exit_code == 0 + assert result.output.strip() == '[{"rowid": 1, "name": "dog."}]' + + def test_indexes(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) From 61b60f58cef1820d113da8740f7f46d4914fb95b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 13:13:52 -0700 Subject: [PATCH 0435/1004] Nice capitalization of API reference --- docs/reference.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference.rst b/docs/reference.rst index ea1203e..4638026 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -1,7 +1,7 @@ .. _reference: =============== - API Reference + API reference =============== .. contents:: :local: From 7e2dcbbbea7efdd66f24838d1fe88e44e2e29dfe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 13:18:54 -0700 Subject: [PATCH 0436/1004] Fixed bug with --no-headers --tsv, closes #295 --- sqlite_utils/cli.py | 2 +- tests/test_cli.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4b39a5e..476c08a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -720,7 +720,7 @@ def insert_upsert_implementation( ): db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - if delimiter or quotechar or sniff or no_headers: + if (delimiter or quotechar or sniff or no_headers) and not tsv: csv = True if (nl + csv + tsv) >= 2: raise click.ClickException("Use just one of --nl, --csv or --tsv") diff --git a/tests/test_cli.py b/tests/test_cli.py index 58347cd..a6ddca0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2205,25 +2205,27 @@ def test_long_csv_column_value(tmpdir): @pytest.mark.parametrize( - "args", + "args,tsv", ( - ["--csv", "--no-headers"], - ["--no-headers"], + (["--csv", "--no-headers"], False), + (["--no-headers"], False), + (["--tsv", "--no-headers"], True), ), ) -def test_csv_import_no_headers(tmpdir, args): +def test_import_no_headers(tmpdir, args, tsv): db_path = str(tmpdir / "test.db") csv_path = str(tmpdir / "test.csv") csv_file = open(csv_path, "w") - csv_file.write("Cleo,Dog,5\n") - csv_file.write("Tracy,Spider,7\n") + sep = "\t" if tsv else "," + csv_file.write("Cleo{sep}Dog{sep}5\n".format(sep=sep)) + csv_file.write("Tracy{sep}Spider{sep}7\n".format(sep=sep)) csv_file.close() result = CliRunner().invoke( cli.cli, ["insert", db_path, "creatures", csv_path] + args, catch_exceptions=False, ) - assert result.exit_code == 0 + assert result.exit_code == 0, result.output db = Database(db_path) schema = db["creatures"].schema assert schema == ( From 7479933bc4f708e9063d959c9d6fd3700ed6cc93 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 14:40:02 -0700 Subject: [PATCH 0437/1004] More sqlite-utils memory examples in README closes #294 --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index bac6369..9890e61 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Python CLI utility and library for manipulating SQLite databases. ## Some feature highlights - [Pipe JSON](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-json-data) (or [CSV or TSV](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-csv-or-tsv-data)) directly into a new SQLite database file, automatically creating a table with the appropriate schema +- [Run in-memory SQL queries](https://sqlite-utils.datasette.io/en/stable/cli.html#querying-data-directly-using-an-in-memory-database), including joins, directly against data in CSV, TSV or JSON files and view the results. - [Configure SQLite full-text search](https://sqlite-utils.datasette.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance - Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as dropping columns - [Extract columns](https://sqlite-utils.datasette.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data @@ -32,6 +33,14 @@ Or if you use [Homebrew](https://brew.sh/) for macOS: Now you can do things with the CLI utility like this: + $ sqlite-utils memory dogs.csv "select * from t" + [{"id": 1, "age": 4, "name": "Cleo"}, + {"id": 2, "age": 2, "name": "Pancakes"}] + + $ sqlite-utils insert dogs.db dogs dogs.csv --csv + [{"id": 1, "age": 4, "name": "Cleo"}, + {"id": 2, "age": 2, "name": "Pancakes"}] + $ sqlite-utils tables dogs.db --counts [{"table": "dogs", "count": 2}] From c62363ebdcd088c7f6e00c4c8096057c194b0de5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 14:48:05 -0700 Subject: [PATCH 0438/1004] Run mypy against tests/ too, refs #37 --- .github/workflows/test.yml | 2 +- setup.py | 2 +- sqlite_utils/db.py | 2 +- tests/test_create.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5dc3fe1..32be2e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: run: | pytest - name: run mypy - run: mypy sqlite_utils + run: mypy sqlite_utils tests - name: run flake8 run: flake8 - name: Check formatting diff --git a/setup.py b/setup.py index 65fe6f7..0d3e456 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ setup( extras_require={ "test": ["pytest", "black", "hypothesis"], "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], - "mypy": ["mypy", "types-click", "types-tabulate", "types-python-dateutil"], + "mypy": ["mypy", "types-click", "types-tabulate", "types-python-dateutil", "data-science-types"], "flake8": ["flake8"], }, entry_points=""" diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e4c1d5c..4677733 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -197,7 +197,7 @@ if np: # If pandas is available, add more types if pd: - COLUMN_TYPE_MAPPING.update({pd.Timestamp: "TEXT"}) + COLUMN_TYPE_MAPPING.update({pd.Timestamp: "TEXT"}) # type: ignore class AlterError(Exception): diff --git a/tests/test_create.py b/tests/test_create.py index ff36f90..58dda95 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -20,9 +20,9 @@ import uuid from .utils import collapse_whitespace try: - import pandas as pd + import pandas as pd # type: ignore except ImportError: - pd = None + pd = None # type: ignore def test_create_table(fresh_db): From 282e81362ae34b134abab3d774963d6b2a57a1be Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 14:55:37 -0700 Subject: [PATCH 0439/1004] Applied Black plus some extra type hints --- setup.py | 8 +++++++- sqlite_utils/db.py | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 0d3e456..71d9cd4 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,13 @@ setup( extras_require={ "test": ["pytest", "black", "hypothesis"], "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], - "mypy": ["mypy", "types-click", "types-tabulate", "types-python-dateutil", "data-science-types"], + "mypy": [ + "mypy", + "types-click", + "types-tabulate", + "types-python-dateutil", + "data-science-types", + ], "flake8": ["flake8"], }, entry_points=""" diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 4677733..893c3b2 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -277,7 +277,7 @@ class Database: def __init__( self, - filename_or_conn=None, + filename_or_conn: Union[str, pathlib.Path, sqlite3.Connection] = None, memory: bool = False, recreate: bool = False, recursive_triggers: bool = True, @@ -331,7 +331,7 @@ class Database: """ return self.table(table_name) - def __repr__(self): + def __repr__(self) -> str: return "".format(self.conn) def register_function( @@ -1069,7 +1069,7 @@ class Table(Queryable): return next(iter(counts.values())) return self.count_where() - def exists(self): + def exists(self) -> bool: return self.name in self.db.table_names() @property From c79737bb4f04d7e0eda3c440fed0c35169a04d24 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 15:25:18 -0700 Subject: [PATCH 0440/1004] Type signatures for .create_table() and .create_table_sql() and .create() and Table.__init__ Closes #314 --- sqlite_utils/db.py | 104 +++++++++++++++++++++++++------------------ tests/test_tracer.py | 1 + 2 files changed, 62 insertions(+), 43 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 893c3b2..93d25b2 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -147,6 +147,15 @@ XIndexColumn = namedtuple( Trigger = namedtuple("Trigger", ("name", "table", "sql")) +ForeignKeysType = Union[ + Iterable[str], + Iterable[ForeignKey], + Iterable[Tuple[str, str]], + Iterable[Tuple[str, str, str]], + Iterable[Tuple[str, str, str, str]], +] + + class Default: pass @@ -572,18 +581,22 @@ class Database: ) -> List[dict]: return list(self.query(sql, params)) - def resolve_foreign_keys(self, name, foreign_keys): - # foreign_keys may be a list of strcolumn names, a list of ForeignKey tuples, + def resolve_foreign_keys( + self, name: str, foreign_keys: ForeignKeysType + ) -> List[ForeignKey]: + # foreign_keys may be a list of column names, a list of ForeignKey tuples, # a list of tuple-pairs or a list of tuple-triples. We want to turn # it into a list of ForeignKey tuples + table = cast(Table, self[name]) if all(isinstance(fk, ForeignKey) for fk in foreign_keys): - return foreign_keys + return cast(List[ForeignKey], foreign_keys) if all(isinstance(fk, str) for fk in foreign_keys): # It's a list of columns fks = [] for column in foreign_keys: - other_table = self[name].guess_foreign_table(column) - other_column = self[name].guess_foreign_column(other_table) + column = cast(str, column) + other_table = table.guess_foreign_table(column) + other_column = table.guess_foreign_column(other_table) fks.append(ForeignKey(name, column, other_table, other_column)) return fks assert all( @@ -596,6 +609,7 @@ class Database: 3, ), "foreign_keys= should be a list of tuple pairs or triples" if len(tuple_or_list) == 3: + tuple_or_list = cast(Tuple[str, str, str], tuple_or_list) fks.append( ForeignKey( name, tuple_or_list[0], tuple_or_list[1], tuple_or_list[2] @@ -608,7 +622,7 @@ class Database: name, tuple_or_list[0], tuple_or_list[1], - self[name].guess_foreign_column(tuple_or_list[1]), + table.guess_foreign_column(tuple_or_list[1]), ) ) return fks @@ -618,12 +632,12 @@ class Database: name: str, columns: Dict[str, Any], pk: Optional[Any] = None, - foreign_keys=None, - column_order=None, - not_null=None, - defaults=None, - hash_id=None, - extracts=None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Iterable[str] = None, + defaults: Optional[Dict[str, Any]] = None, + hash_id: Optional[Any] = None, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, ) -> str: "Returns the SQL ``CREATE TABLE`` statement for creating the specified table." foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) @@ -656,9 +670,11 @@ class Database: validate_column_names(columns.keys()) column_items = list(columns.items()) if column_order is not None: - column_items.sort( - key=lambda p: column_order.index(p[0]) if p[0] in column_order else 999 - ) + + def sort_key(p): + return column_order.index(p[0]) if p[0] in column_order else 999 + + column_items.sort(key=sort_key) if hash_id: column_items.insert(0, (hash_id, str)) pk = hash_id @@ -725,12 +741,12 @@ class Database: name: str, columns: Dict[str, Any], pk: Optional[Any] = None, - foreign_keys=None, - column_order=None, - not_null=None, - defaults=None, - hash_id=None, - extracts=None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Iterable[str] = None, + defaults: Optional[Dict[str, Any]] = None, + hash_id: Optional[Any] = None, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, ) -> "Table": """ Create a table with the specified name and the specified ``{column_name: type}`` columns. @@ -1021,19 +1037,19 @@ class Table(Queryable): self, db: Database, name: str, - pk=None, - foreign_keys=None, - column_order=None, - not_null=None, - defaults=None, - batch_size=100, - hash_id=None, - alter=False, - ignore=False, - replace=False, - extracts=None, - conversions=None, - columns=None, + pk: Optional[Any] = None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Iterable[str] = None, + defaults: Optional[Dict[str, Any]] = None, + batch_size: int = 100, + hash_id: Optional[Any] = None, + alter: bool = False, + ignore: bool = False, + replace: bool = False, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, + conversions: Optional[dict] = None, + columns: Optional[Union[Dict[str, Any]]] = None, ): super().__init__(db, name) self._defaults = dict( @@ -1202,14 +1218,14 @@ class Table(Queryable): def create( self, - columns, - pk=None, - foreign_keys=None, - column_order=None, - not_null=None, - defaults=None, - hash_id=None, - extracts=None, + columns: Dict[str, Any], + pk: Optional[Any] = None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Iterable[str] = None, + defaults: Optional[Dict[str, Any]] = None, + hash_id: Optional[Any] = None, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, ) -> "Table": """ Create a table with the specified columns. @@ -2914,7 +2930,9 @@ def _hash(record): ).hexdigest() -def resolve_extracts(extracts): +def resolve_extracts( + extracts: Optional[Union[Dict[str, str], List[str], Tuple[str]]] +) -> dict: if extracts is None: extracts = {} if isinstance(extracts, (list, tuple)): diff --git a/tests/test_tracer.py b/tests/test_tracer.py index d3ff22d..618d1e6 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -13,6 +13,7 @@ def test_tracer(): ("PRAGMA recursive_triggers=on;", None), ("select name from sqlite_master where type = 'view'", None), ("select name from sqlite_master where type = 'table'", None), + ("select name from sqlite_master where type = 'view'", None), ("CREATE TABLE [dogs] (\n [name] TEXT\n);\n ", None), ("select name from sqlite_master where type = 'view'", None), ("INSERT INTO [dogs] ([name]) VALUES (?);", ["Cleopaws"]), From 5912878d62ef7de6fa3b9274aed8d98243ff5e56 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 15:36:32 -0700 Subject: [PATCH 0441/1004] Release 3.16 Refs #37, #246, #294, #295, #296, #314, #316 --- docs/changelog.rst | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ab424ed..7ff467d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,18 @@ Changelog =========== +.. _v3_16: + +3.16 (2021-08-18) +----------------- + +- Type signatures added to more methods, including ``table.resolve_foreign_keys()``, ``db.create_table_sql()``, ``db.create_table()`` and ``table.create()``. (:issue:`314`) +- New ``db.quote_fts(value)`` method, see :ref:`python_api_quote_fts` - thanks, Mark Neumann. (:issue:`246`) +- ``table.search()`` now accepts an optional ``quote=True`` parameter. (:issue:`296`) +- CLI command ``sqlite-utils search`` now accepts a ``--quote`` option. (:issue:`296`) +- Fixed bug where ``--no-headers`` and ``--tsv`` options to :ref:`sqlite-utils insert ` could not be used together. (:issue:`295`) +- Various small improvements to :ref:`reference` documentation. + .. _v3_15.1: 3.15.1 (2021-08-10) diff --git a/setup.py b/setup.py index 71d9cd4..6d138b4 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.15.1" +VERSION = "3.16" def get_long_description(): From ddfdff657f34126c0b4c6f8361c2ca9e5d30c336 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 16:01:00 -0700 Subject: [PATCH 0442/1004] Fixed incorrecte output example --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 9890e61..cfb2582 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,7 @@ Now you can do things with the CLI utility like this: {"id": 2, "age": 2, "name": "Pancakes"}] $ sqlite-utils insert dogs.db dogs dogs.csv --csv - [{"id": 1, "age": 4, "name": "Cleo"}, - {"id": 2, "age": 2, "name": "Pancakes"}] + [####################################] 100% $ sqlite-utils tables dogs.db --counts [{"table": "dogs", "count": 2}] From b30f725d982309eb26ef0b985aadc0064df8e8f9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 16:02:07 -0700 Subject: [PATCH 0443/1004] Small improvement to example --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cfb2582..6b88947 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,9 @@ Now you can do things with the CLI utility like this: $ sqlite-utils tables dogs.db --counts [{"table": "dogs", "count": 2}] - $ sqlite-utils dogs.db "select * from dogs" - [{"id": 1, "age": 4, "name": "Cleo"}, - {"id": 2, "age": 2, "name": "Pancakes"}] + $ sqlite-utils dogs.db "select id, name from dogs" + [{"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Pancakes"}] $ sqlite-utils dogs.db "select * from dogs" --csv id,age,name From d7b1024d3a9e092c030237410219a8ae376a4799 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 16:02:55 -0700 Subject: [PATCH 0444/1004] Corrected stdin example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b88947..9345286 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Or for data in a CSV file: `sqlite-utils memory` lets you import CSV or JSON data into an in-memory database and run SQL queries against it in a single command: - $ cat dogs.csv | sqlite-utils memory - "select name, age from dogs" + $ cat dogs.csv | sqlite-utils memory - "select name, age from stdin" See the [full CLI documentation](https://sqlite-utils.datasette.io/en/stable/cli.html) for comprehensive coverage of many more commands. From 9258f4bd8450c951900de998a7bf81ca9b45a014 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 22 Aug 2021 08:44:25 -0700 Subject: [PATCH 0445/1004] sqlite-utils memory --analyze, closes #320 --- docs/cli.rst | 33 ++++++++++++++++++++++++++++++--- sqlite_utils/cli.py | 18 ++++++++++++++++-- tests/test_cli_memory.py | 21 +++++++++++++++++++++ 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 6a96a82..3c06b81 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -328,10 +328,10 @@ The CSV data that was piped into the script is available in the ``stdin`` table, .. _cli_memory_schema_dump_save: -\-\-schema, \-\-dump and \-\-save ---------------------------------- +\-\-schema, \-\-analyze, \-\-dump and \-\-save +---------------------------------------------- -To see the schema that will be created for a file or multiple files, use ``--schema``:: +To see the in-memory datbase schema that would be used for a file or for multiple files, use ``--schema``:: % sqlite-utils memory dogs.csv --schema CREATE TABLE [dogs] ( @@ -342,6 +342,33 @@ To see the schema that will be created for a file or multiple files, use ``--sch CREATE VIEW t1 AS select * from [dogs]; CREATE VIEW t AS select * from [dogs]; +You can run the equivalent of the :ref:`analyze-tables ` command using ``--analyze``:: + + % sqlite-utils memory dogs.csv --analyze + dogs.id: (1/3) + + Total rows: 2 + Null rows: 0 + Blank rows: 0 + + Distinct values: 2 + + dogs.name: (2/3) + + Total rows: 2 + Null rows: 0 + Blank rows: 0 + + Distinct values: 2 + + dogs.age: (3/3) + + Total rows: 2 + Null rows: 0 + Blank rows: 0 + + Distinct values: 2 + You can output SQL that will both create the tables and insert the full data used to populate the in-memory database using ``--dump``:: % sqlite-utils memory dogs.csv --dump diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 476c08a..58cda49 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1216,6 +1216,11 @@ def query( type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), help="Save in-memory database to this file", ) +@click.option( + "--analyze", + is_flag=True, + help="Analyze resulting tables and output results", +) @load_extension_option def memory( paths, @@ -1236,6 +1241,7 @@ def memory( schema, dump, save, + analyze, load_extension, ): """Execute SQL query against an in-memory database, optionally populated by imported data @@ -1265,8 +1271,8 @@ def memory( sqlite-utils memory animals.csv --schema """ db = sqlite_utils.Database(memory=True) - # If --dump or --save used but no paths detected, assume SQL query is a path: - if (dump or save or schema) and not paths: + # If --dump or --save or --analyze used but no paths detected, assume SQL query is a path: + if (dump or save or schema or analyze) and not paths: paths = [sql] sql = None for i, path in enumerate(paths): @@ -1299,6 +1305,10 @@ def memory( if not db[view_name].exists(): db.create_view(view_name, "select * from [{}]".format(csv_table)) + if analyze: + _analyze(db, tables=None, columns=None, save=False) + return + if dump: for line in db.conn.iterdump(): click.echo(line) @@ -1922,6 +1932,10 @@ def analyze_tables( "Analyze the columns in one or more tables" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) + _analyze(db, tables, columns, save) + + +def _analyze(db, tables, columns, save): if not tables: tables = db.table_names() todo = [] diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index e465927..08566e3 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -220,3 +220,24 @@ def test_memory_no_detect_types(option): {"id": "1", "name": "Cleo", "weight": "45.5"}, {"id": "2", "name": "Bants", "weight": "3.5"}, ] + + +def test_memory_analyze(): + result = CliRunner().invoke( + cli.cli, + ["memory", "-", "--analyze"], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + assert result.output == ( + "stdin.id: (1/2)\n\n" + " Total rows: 2\n" + " Null rows: 0\n" + " Blank rows: 0\n\n" + " Distinct values: 2\n\n" + "stdin.name: (2/2)\n\n" + " Total rows: 2\n" + " Null rows: 0\n" + " Blank rows: 0\n\n" + " Distinct values: 2\n\n" + ) From 49a010c93d90bc68ce1c6fff7639927248912b54 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Aug 2021 16:31:13 -0700 Subject: [PATCH 0446/1004] Ability to insert file contents as text, in addition to blob (#321) --- docs/cli.rst | 18 ++++++++----- sqlite_utils/cli.py | 45 +++++++++++++++++++++++++++---- tests/test_insert_files.py | 54 +++++++++++++++++++++++++++++++++++--- 3 files changed, 101 insertions(+), 16 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 3c06b81..0cc20dd 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -331,7 +331,7 @@ The CSV data that was piped into the script is available in the ``stdin`` table, \-\-schema, \-\-analyze, \-\-dump and \-\-save ---------------------------------------------- -To see the in-memory datbase schema that would be used for a file or for multiple files, use ``--schema``:: +To see the in-memory database schema that would be used for a file or for multiple files, use ``--schema``:: % sqlite-utils memory dogs.csv --schema CREATE TABLE [dogs] ( @@ -909,12 +909,10 @@ The command will fail if you reference columns that do not exist on the table. T .. _cli_insert_files: -Inserting binary data from files -================================ +Inserting data from files +========================= -SQLite ``BLOB`` columns can be used to store binary content. It can be useful to insert the contents of files into a SQLite table. - -The ``insert-files`` command can be used to insert the content of files, along with their metadata. +The ``insert-files`` command can be used to insert the content of files, along with their metadata, into a SQLite table. Here's an example that inserts all of the GIF files in the current directory into a ``gifs.db`` database, placing the file contents in an ``images`` table:: @@ -932,6 +930,8 @@ By default this command will create a table with the following schema:: [size] INTEGER ); +Content will be treated as binary by default and stored in a ``BLOB`` column. You can use the ``--text`` option to store that content in a ``TEXT`` column instead. + You can customize the schema using one or more ``-c`` options. For a table schema that includes just the path, MD5 hash and last modification time of the file, you would use this:: $ sqlite-utils insert-files gifs.db images *.gif -c path -c md5 -c mtime --pk=path @@ -944,6 +944,8 @@ This will result in the following schema:: [mtime] FLOAT ); +Note that there's no ``content`` column here at all - if you specify custom columns using ``-c`` you need to include ``-c content`` to create that column. + You can change the name of one of these columns using a ``-c colname:coldef`` parameter. To rename the ``mtime`` column to ``last_modified`` you would use this:: $ sqlite-utils insert-files gifs.db images *.gif \ @@ -967,6 +969,8 @@ The full list of column definitions you can use is as follows: The permission bits of the file, as an integer - you may want to convert this to octal ``content`` The binary file contents, which will be stored as a BLOB +``content_text`` + The text file contents, which will be stored as TEXT ``mtime`` The modification time of the file, as floating point seconds since the Unix epoch ``ctime`` @@ -988,7 +992,7 @@ You can insert data piped from standard input like this:: The ``-`` argument indicates data should be read from standard input. The string passed using the ``--name`` option will be used for the file name and path values. -When inserting data from standard input only the following column definitions are supported: ``name``, ``path``, ``content``, ``sha256``, ``md5`` and ``size``. +When inserting data from standard input only the following column definitions are supported: ``name``, ``path``, ``content``, ``content_text``, ``sha256``, ``md5`` and ``size``. .. _cli_convert: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 58cda49..10a11c6 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1811,6 +1811,11 @@ def extract( @click.option("--replace", is_flag=True, help="Replace files with matching primary key") @click.option("--upsert", is_flag=True, help="Upsert files with matching primary key") @click.option("--name", type=str, help="File name to use") +@click.option("--text", is_flag=True, help="Store file content as TEXT, not BLOB") +@click.option( + "--encoding", + help="Character encoding for input, defaults to utf-8", +) @click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") @load_extension_option def insert_files( @@ -1823,6 +1828,8 @@ def insert_files( replace, upsert, name, + text, + encoding, silent, load_extension, ): @@ -1842,7 +1849,10 @@ def insert_files( --pk name """ if not column: - column = ["path:path", "content:content", "size:size"] + if text: + column = ["path:path", "content_text:content_text", "size:size"] + else: + column = ["path:path", "content:content", "size:size"] if not pk: pk = "path" @@ -1866,7 +1876,16 @@ def insert_files( def to_insert(): for path, relative_path in bar: row = {} - lookups = FILE_COLUMNS + # content_text is special case as it considers 'encoding' + + def _content_text(p): + resolved = p.resolve() + try: + return resolved.read_text(encoding=encoding) + except UnicodeDecodeError as e: + raise UnicodeDecodeErrorForPath(e, resolved) + + lookups = dict(FILE_COLUMNS, content_text=_content_text) if path == "-": stdin_data = sys.stdin.buffer.read() # We only support a subset of columns for this case @@ -1874,6 +1893,9 @@ def insert_files( "name": lambda p: name or "-", "path": lambda p: name or "-", "content": lambda p: stdin_data, + "content_text": lambda p: stdin_data.decode( + encoding or "utf-8" + ), "sha256": lambda p: hashlib.sha256(stdin_data).hexdigest(), "md5": lambda p: hashlib.md5(stdin_data).hexdigest(), "size": lambda p: len(stdin_data), @@ -1899,9 +1921,16 @@ def insert_files( db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - with db.conn: - db[table].insert_all( - to_insert(), pk=pk, alter=alter, replace=replace, upsert=upsert + try: + with db.conn: + db[table].insert_all( + to_insert(), pk=pk, alter=alter, replace=replace, upsert=upsert + ) + except UnicodeDecodeErrorForPath as e: + raise click.ClickException( + UNICODE_ERROR.format( + "Could not read file '{}' as text\n\n{}".format(e.path, e.exception) + ) ) @@ -2149,6 +2178,12 @@ def _render_common(title, values): return "\n".join(lines) +class UnicodeDecodeErrorForPath(Exception): + def __init__(self, exception, path): + self.exception = exception + self.path = path + + FILE_COLUMNS = { "name": lambda p: p.name, "path": lambda p: str(p), diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 1e30a8d..86ee4e3 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -3,6 +3,7 @@ from click.testing import CliRunner import os import pathlib import pytest +import sys @pytest.mark.parametrize("silent", (False, True)) @@ -23,6 +24,7 @@ def test_insert_files(silent): "md5", "mode", "content", + "content_text", "mtime", "ctime", "mtime_int", @@ -52,6 +54,7 @@ def test_insert_files(silent): ) assert { "content": b"This is file one", + "content_text": "This is file one", "md5": "556dfb57fce9ca301f914e2273adf354", "name": "one.txt", "path": "one.txt", @@ -60,6 +63,7 @@ def test_insert_files(silent): }.items() <= one.items() assert { "content": b"Two is shorter", + "content_text": "Two is shorter", "md5": "f86f067b083af1911043eb215e74ac70", "name": "two.txt", "path": "two.txt", @@ -68,6 +72,7 @@ def test_insert_files(silent): }.items() <= two.items() assert { "content": b"Three is nested", + "content_text": "Three is nested", "md5": "12580f341781f5a5b589164d3cd39523", "name": "three.txt", "path": os.path.join("nested", "three.txt"), @@ -84,24 +89,65 @@ def test_insert_files(silent): "mtime_iso": str, "mode": int, "fullpath": str, + "content": bytes, + "content_text": str, } for colname, expected_type in expected_types.items(): for row in (one, two, three): assert isinstance(row[colname], expected_type) -def test_insert_files_stdin(): +@pytest.mark.parametrize( + "use_text,encoding,input,expected", + ( + (False, None, "hello world", b"hello world"), + (True, None, "hello world", "hello world"), + (False, None, b"S\xe3o Paulo", b"S\xe3o Paulo"), + (True, "latin-1", b"S\xe3o Paulo", "S\xe3o Paulo"), + ), +) +def test_insert_files_stdin(use_text, encoding, input, expected): runner = CliRunner() with runner.isolated_filesystem(): tmpdir = pathlib.Path(".") db_path = str(tmpdir / "files.db") + args = ["insert-files", db_path, "files", "-", "--name", "stdin-name"] + if use_text: + args += ["--text"] + if encoding is not None: + args += ["--encoding", encoding] result = runner.invoke( cli.cli, - ["insert-files", db_path, "files", "-", "--name", "stdin-name"], + args, catch_exceptions=False, - input="hello world", + input=input, ) assert result.exit_code == 0, result.stdout db = Database(db_path) row = list(db["files"].rows)[0] - assert {"path": "stdin-name", "content": b"hello world", "size": 11} == row + key = "content" + if use_text: + key = "content_text" + assert {"path": "stdin-name", key: expected}.items() <= row.items() + + +@pytest.mark.skipif( + sys.platform.startswith("win"), + reason="Windows has a different way of handling default encodings", +) +def test_insert_files_bad_text_encoding_error(): + runner = CliRunner() + with runner.isolated_filesystem(): + tmpdir = pathlib.Path(".") + latin = tmpdir / "latin.txt" + latin.write_bytes(b"S\xe3o Paulo") + db_path = str(tmpdir / "files.db") + result = runner.invoke( + cli.cli, + ["insert-files", db_path, "files", str(latin), "--text"], + catch_exceptions=False, + ) + assert result.exit_code == 1, result.output + assert result.output.strip().startswith( + "Error: Could not read file '{}' as text".format(str(latin.resolve())) + ) From 77c240df56068341561e95e4a412cbfa24dc5bc7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Aug 2021 16:39:49 -0700 Subject: [PATCH 0447/1004] Release 3.17 Refs #319, #320 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7ff467d..6144b33 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v3_17: + +3.17 (2021-08-24) +----------------- + +- The :ref:`sqlite-utils memory ` command has a new ``--analyze`` option, which runs the equivalent of the :ref:`analyze-tables ` command directly against the in-memory database created from the incoming CSV or JSON data. (:issue:`320`) +- :ref:`sqlite-utils insert-files ` now has the ability to insert file contents in to ``TEXT`` columns in addition to the default ``BLOB``. Pass the ``--text`` option or use ``content_text`` as a column specifier. (:issue:`319`) + .. _v3_16: 3.16 (2021-08-18) diff --git a/setup.py b/setup.py index 6d138b4..e12f494 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.16" +VERSION = "3.17" def get_long_description(): From 7427a9137f60de961b6331d0922a3f03da0d1890 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 22 Sep 2021 13:20:04 -0700 Subject: [PATCH 0448/1004] Output [] in JSON mode if no rows, closes #328 --- sqlite_utils/cli.py | 3 +++ tests/test_cli.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 10a11c6..5217965 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2225,6 +2225,9 @@ def output_rows(iterator, headers, nl, arrays, json_cols): ) yield line first = False + if first: + # We didn't output any rows, so yield the empty list + yield "[]" def maybe_json(value): diff --git a/tests/test_cli.py b/tests/test_cli.py index a6ddca0..47920c6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -994,6 +994,13 @@ def test_query_json(db_path, sql, args, expected): assert expected == result.output.strip() +def test_query_json_empty(db_path): + result = CliRunner().invoke( + cli.cli, [db_path, "select * from sqlite_master where 0"] + ) + assert result.output.strip() == "[]" + + LOREM_IPSUM_COMPRESSED = ( b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8e" b"f\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J" @@ -2098,7 +2105,11 @@ _TRIGGERS_EXPECTED = ( @pytest.mark.parametrize( "extra_args,expected", - [([], _TRIGGERS_EXPECTED), (["articles"], _TRIGGERS_EXPECTED), (["counter"], "")], + [ + ([], _TRIGGERS_EXPECTED), + (["articles"], _TRIGGERS_EXPECTED), + (["counter"], "[]\n"), + ], ) def test_triggers(tmpdir, extra_args, expected): db_path = str(tmpdir / "test.db") From c1b26eed03f60c3e317550053a3832b7ad62e588 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 22 Sep 2021 13:45:37 -0700 Subject: [PATCH 0449/1004] sqlite-utils memory handles files with same filename, closes #325 --- docs/cli.rst | 4 ++++ sqlite_utils/cli.py | 8 +++++++- tests/test_cli_memory.py | 24 ++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 0cc20dd..d2a4d84 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -281,6 +281,10 @@ The in-memory tables will be named after the files without their extensions. The $ sqlite-utils memory example.csv "select * from t" +If two files have the same name they will be assigned a numeric suffix:: + + $ sqlite-utils memory foo/data.csv bar/data.csv "select * from data_2" + To read from standard input, use either ``-`` or ``stdin`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name:: $ cat example.csv | sqlite-utils memory - "select * from stdin" diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 5217965..b29314e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1275,6 +1275,7 @@ def memory( if (dump or save or schema or analyze) and not paths: paths = [sql] sql = None + stem_counts = {} for i, path in enumerate(paths): # Path may have a :format suffix if ":" in path and path.rsplit(":", 1)[-1].upper() in Format.__members__: @@ -1287,7 +1288,12 @@ def memory( csv_table = "stdin" else: csv_path = pathlib.Path(path) - csv_table = csv_path.stem + stem = csv_path.stem + if stem_counts.get(stem): + csv_table = "{}_{}".format(stem, stem_counts[stem]) + else: + csv_table = stem + stem_counts[stem] = stem_counts.get(stem, 1) + 1 csv_fp = csv_path.open("rb") rows, format_used = rows_from_file(csv_fp, format=format, encoding=encoding) tracker = None diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index 08566e3..a8b3a18 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -241,3 +241,27 @@ def test_memory_analyze(): " Blank rows: 0\n\n" " Distinct values: 2\n\n" ) + + +def test_memory_two_files_with_same_stem(tmpdir): + (tmpdir / "one").mkdir() + (tmpdir / "two").mkdir() + one = tmpdir / "one" / "data.csv" + two = tmpdir / "two" / "data.csv" + one.write_text("id,name\n1,Cleo\n2,Bants", encoding="utf-8") + two.write_text("id,name\n3,Blue\n4,Lila", encoding="utf-8") + result = CliRunner().invoke(cli.cli, ["memory", str(one), str(two), "", "--schema"]) + assert result.exit_code == 0 + assert result.output == ( + 'CREATE TABLE "data" (\n' + " [id] INTEGER,\n" + " [name] TEXT\n" + ");\n" + "CREATE VIEW t1 AS select * from [data];\n" + "CREATE VIEW t AS select * from [data];\n" + 'CREATE TABLE "data_2" (\n' + " [id] INTEGER,\n" + " [name] TEXT\n" + ");\n" + "CREATE VIEW t2 AS select * from [data_2];\n" + ) From 54191d4dc114d7dc21e849b48a4d5ae4f9e601ca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 22 Sep 2021 13:49:36 -0700 Subject: [PATCH 0450/1004] Release 3.17.1 Refs #325, #328 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6144b33..0eadaa1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v3_17.1: + +3.17.1 (2021-09-22) +------------------- + +- :ref:`sqlite-utils memory ` now works if files passed to it share the same file name. (:issue:`325`) +- :ref:`sqlite-utils query ` now returns ``[]`` in JSON mode if no rows are returned. (:issue:`328`) + .. _v3_17: 3.17 (2021-08-24) diff --git a/setup.py b/setup.py index e12f494..f1bb056 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.17" +VERSION = "3.17.1" def get_long_description(): From 718a8f61bcaed39c04d5d223104056213f8c8516 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 2 Oct 2021 09:54:39 -0700 Subject: [PATCH 0451/1004] Clarified description of --quote --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index d2a4d84..7a127fe 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1554,7 +1554,7 @@ By default it shows the most relevant matches first. You can specify a different # Sort by created in descending order $ sqlite-utils search mydb.db documents searchterm -o 'created desc' -SQLite `advanced search syntax `__ is enabled by default. To run a search with automatic quoting use the ``--quote`` option. +SQLite `advanced search syntax `__ is enabled by default. To run a search with automatic quoting applied to the terms to avoid them being potentially interpreted as advanced search syntax use the ``--quote`` option. You can specify a subset of columns to be returned using the ``-c`` option one or more times:: From fda4dad23a0494890267fbe8baf179e2b56ee914 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Oct 2021 15:25:05 -0700 Subject: [PATCH 0452/1004] Test against Python 3.10 (#330) * Test against Python 3.10 * Added 3.10 to classifiers * Test on Python 3.10 before publish --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 2 +- setup.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c0bd779..59c8a23 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] os: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 32be2e0..86b2531 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest] steps: diff --git a/setup.py b/setup.py index f1bb056..091c94a 100644 --- a/setup.py +++ b/setup.py @@ -67,5 +67,6 @@ setup( "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], ) From 92aa5c9c5d26b0889c8c3d97c76a908d5f8af211 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Nov 2021 12:50:22 -0800 Subject: [PATCH 0453/1004] Fixed typo --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 93d25b2..ab7252a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2694,7 +2694,7 @@ class Table(Queryable): set up a unique constraint on the ``name`` column to guarantee it will not contain duplicate rows. - It well then inserts a new row with the ``name`` set to ``Palm`` and return the + It will then insert a new row with the ``name`` set to ``Palm`` and return the new integer primary key value. See :ref:`python_api_lookup_tables` for more details. From e8d958109ee290cfa1b44ef7a39629bb50ab673e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Nov 2021 14:49:19 -0800 Subject: [PATCH 0454/1004] create_index(..., find_unique_name=True) option, refs #335 --- docs/python-api.rst | 4 ++- sqlite_utils/db.py | 59 +++++++++++++++++++++++++++++++------------- tests/test_create.py | 17 +++++++++++++ 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 20d080d..8932656 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2141,7 +2141,9 @@ You can create an index on a table using the ``.create_index(columns)`` method. db["dogs"].create_index(["is_good_dog"]) -By default the index will be named ``idx_{table-name}_{columns}`` - if you want to customize the name of the created index you can pass the ``index_name`` parameter: +By default the index will be named ``idx_{table-name}_{columns}``. If you pass ``find_unique_name=True`` and the automatically derived name already exists, an available name will be found by incrementing a suffix number, for example ``idx_items_title_2``. + +You can customize the name of the created index by passing the ``index_name`` parameter: .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ab7252a..1e75861 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -899,7 +899,7 @@ class Database: } for fk in table.foreign_keys: if fk.column not in existing_indexes: - table.create_index([fk.column]) + table.create_index([fk.column], find_unique_name=True) def vacuum(self): "Run a SQLite ``VACUUM`` against the database." @@ -1521,6 +1521,7 @@ class Table(Queryable): index_name: Optional[str] = None, unique: bool = False, if_not_exists: bool = False, + find_unique_name: bool = False, ): """ Create an index on this table. @@ -1530,6 +1531,8 @@ class Table(Queryable): - ``index_name`` - the name to use for the new index. Defaults to the column names joined on ``_``. - ``unique`` - should the index be marked as unique, forcing unique values? - ``if_not_exists`` - only create the index if one with that name does not already exist. + - ``find_unique_name`` - if ``index_name`` is not provided and the automatically derived name + already exists, keep incrementing a suffix number to find an available name. See :ref:`python_api_create_index`. """ @@ -1544,23 +1547,45 @@ class Table(Queryable): else: fmt = "[{}]" columns_sql.append(fmt.format(column)) - sql = ( - textwrap.dedent( - """ - CREATE {unique}INDEX {if_not_exists}[{index_name}] - ON [{table_name}] ({columns}); - """ + + suffix = None + while True: + sql = ( + textwrap.dedent( + """ + CREATE {unique}INDEX {if_not_exists}[{index_name}] + ON [{table_name}] ({columns}); + """ + ) + .strip() + .format( + index_name="{}_{}".format(index_name, suffix) + if suffix + else index_name, + table_name=self.name, + columns=", ".join(columns_sql), + unique="UNIQUE " if unique else "", + if_not_exists="IF NOT EXISTS " if if_not_exists else "", + ) ) - .strip() - .format( - index_name=index_name, - table_name=self.name, - columns=", ".join(columns_sql), - unique="UNIQUE " if unique else "", - if_not_exists="IF NOT EXISTS " if if_not_exists else "", - ) - ) - self.db.execute(sql) + try: + self.db.execute(sql) + break + except OperationalError as e: + # find_unique_name=True - try again if 'index ... already exists' + arg = e.args[0] + if ( + find_unique_name + and arg.startswith("index ") + and arg.endswith(" already exists") + ): + if suffix is None: + suffix = 2 + else: + suffix += 1 + continue + else: + raise e return self def add_column( diff --git a/tests/test_create.py b/tests/test_create.py index 58dda95..8d5b023 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -4,6 +4,7 @@ from sqlite_utils.db import ( DescIndex, AlterError, NoObviousTable, + OperationalError, ForeignKey, Table, View, @@ -752,6 +753,22 @@ def test_create_index_desc(fresh_db): ) +def test_create_index_find_unique_name(): + db = Database(memory=True) + table = db["t"] + table.insert({"id": 1}) + table.create_index(["id"]) + # Without find_unique_name should error + with pytest.raises(OperationalError, match="index idx_t_id already exists"): + table.create_index(["id"]) + # With find_unique_name=True it should work + table.create_index(["id"], find_unique_name=True) + table.create_index(["id"], find_unique_name=True) + # Should have three now + index_names = {idx.name for idx in table.indexes} + assert index_names == {"idx_t_id", "idx_t_id_2", "idx_t_id_3"} + + @pytest.mark.parametrize( "data_structure", ( From 13195d8747764df3952ed1117e0fd2152f1899e7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Nov 2021 14:55:42 -0800 Subject: [PATCH 0455/1004] Test demonstrating fix for #335 --- tests/test_create.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/test_create.py b/tests/test_create.py index 8d5b023..acaeaee 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -475,6 +475,18 @@ def test_index_foreign_keys(fresh_db): assert [["breed_id"]] == [i.columns for i in fresh_db["dogs"].indexes] +def test_index_foreign_keys_if_index_name_is_already_used(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/335 + test_add_foreign_key_guess_table(fresh_db) + # Add index with a name that will conflict with index_foreign_keys() + fresh_db["dogs"].create_index(["name"], index_name="idx_dogs_breed_id") + fresh_db.index_foreign_keys() + assert {(idx.name, tuple(idx.columns)) for idx in fresh_db["dogs"].indexes} == { + ("idx_dogs_breed_id_2", ("breed_id",)), + ("idx_dogs_breed_id", ("name",)), + } + + @pytest.mark.parametrize( "extra_data,expected_new_columns", [ @@ -753,9 +765,8 @@ def test_create_index_desc(fresh_db): ) -def test_create_index_find_unique_name(): - db = Database(memory=True) - table = db["t"] +def test_create_index_find_unique_name(fresh_db): + table = fresh_db["t"] table.insert({"id": 1}) table.create_index(["id"]) # Without find_unique_name should error From 12b8c9de256ba907d4fa8e134bf9ce9bc012302e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Nov 2021 15:05:00 -0800 Subject: [PATCH 0456/1004] sqlite-utils memory --flatten, closes #332 --- docs/cli.rst | 2 +- sqlite_utils/cli.py | 4 ++++ tests/test_cli_memory.py | 24 ++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 7a127fe..a096fde 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -758,7 +758,7 @@ This also means you pipe ``sqlite-utils`` together to easily create a new SQLite Flattening nested JSON objects ------------------------------ -``sqlite-utils insert`` expects incoming data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table. +``sqlite-utils insert`` and ``sqlite-utils memory`` both expect incoming JSON data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table. If your data is nested you can use the ``--flatten`` option to create columns that are derived from the nested data. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index b29314e..cd0be82 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1190,6 +1190,7 @@ def query( multiple=True, help="Additional databases to attach - specify alias and filepath", ) +@click.option("--flatten", is_flag=True, help="Flatten nested JSON objects") @output_options @click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") @click.option( @@ -1226,6 +1227,7 @@ def memory( paths, sql, attach, + flatten, nl, arrays, csv, @@ -1300,6 +1302,8 @@ def memory( if format_used in (Format.CSV, Format.TSV) and not no_detect_types: tracker = TypeTracker() rows = tracker.wrap(rows) + if flatten: + rows = (dict(_flatten(row)) for row in rows) db[csv_table].insert_all(rows, alter=True) if tracker is not None: db[csv_table].transform(types=tracker.types) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index a8b3a18..aee74e7 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -222,6 +222,30 @@ def test_memory_no_detect_types(option): ] +def test_memory_flatten(): + result = CliRunner().invoke( + cli.cli, + ["memory", "-", "select * from stdin", "--flatten"], + input=json.dumps( + { + "httpRequest": { + "latency": "0.112114537s", + "requestMethod": "GET", + }, + "insertId": "6111722f000b5b4c4d4071e2", + } + ), + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + { + "httpRequest_latency": "0.112114537s", + "httpRequest_requestMethod": "GET", + "insertId": "6111722f000b5b4c4d4071e2", + } + ] + + def test_memory_analyze(): result = CliRunner().invoke( cli.cli, From 73e214a9760c5dc32ed3c5429cb04d4d471ce014 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Nov 2021 15:21:04 -0800 Subject: [PATCH 0457/1004] py.typed file so mypy picks up the types, closes #331 --- setup.py | 3 ++- sqlite_utils/py.typed | 0 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 sqlite_utils/py.typed diff --git a/setup.py b/setup.py index 091c94a..47c927d 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ setup( version=VERSION, license="Apache License, Version 2.0", packages=find_packages(exclude=["tests", "tests.*"]), + package_data={"sqlite_utils": ["py.typed"]}, install_requires=[ "sqlite-fts4", "click", @@ -46,7 +47,6 @@ setup( [console_scripts] sqlite-utils=sqlite_utils.cli:cli """, - tests_require=["sqlite-utils[test]"], url="https://github.com/simonw/sqlite-utils", project_urls={ "Documentation": "https://sqlite-utils.datasette.io/en/stable/", @@ -69,4 +69,5 @@ setup( "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", ], + zip_safe=False, # For mypy and py.typed ) diff --git a/sqlite_utils/py.typed b/sqlite_utils/py.typed new file mode 100644 index 0000000..e69de29 From adea5bc3965c80684f219b12299f708f2f422ca1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Nov 2021 15:24:15 -0800 Subject: [PATCH 0458/1004] flake8 fix, refs #331 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 47c927d..ff36a13 100644 --- a/setup.py +++ b/setup.py @@ -69,5 +69,6 @@ setup( "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", ], - zip_safe=False, # For mypy and py.typed + # Needed to bundle py.typed so mypy can see it: + zip_safe=False, ) From bc4c42d68879c710c851dba3c98deda96ca6caa8 Mon Sep 17 00:00:00 2001 From: Denys Pavlov Date: Sun, 14 Nov 2021 18:25:40 -0500 Subject: [PATCH 0459/1004] Use python-dateutil package instead of dateutils (#324) --- docs/tutorial.ipynb | 6 ++---- setup.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/tutorial.ipynb b/docs/tutorial.ipynb index aa22461..179f010 100644 --- a/docs/tutorial.ipynb +++ b/docs/tutorial.ipynb @@ -39,10 +39,8 @@ "Requirement already satisfied: sqlite-fts4 in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (1.0.1)\n", "Requirement already satisfied: click in /Users/simon/Library/Python/3.9/lib/python/site-packages (from sqlite_utils) (7.1.2)\n", "Requirement already satisfied: tabulate in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (0.8.7)\n", - "Requirement already satisfied: dateutils in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from sqlite_utils) (0.6.12)\n", - "Requirement already satisfied: python-dateutil in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from dateutils->sqlite_utils) (2.8.1)\n", - "Requirement already satisfied: pytz in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from dateutils->sqlite_utils) (2021.1)\n", - "Requirement already satisfied: six>=1.5 in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from python-dateutil->dateutils->sqlite_utils) (1.16.0)\n", + "Requirement already satisfied: python-dateutil in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-package (from sqlite-utils) (2.8.1)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-package (from python-dateutil->sqlite-utils) (1.16.0)\n", "\u001b[33mWARNING: You are using pip version 21.1.1; however, version 21.2.2 is available.\n", "You should consider upgrading via the '/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/bin/python3.9 -m pip install --upgrade pip' command.\u001b[0m\n", "Note: you may need to restart the kernel to use updated packages.\n" diff --git a/setup.py b/setup.py index ff36a13..f40dcfe 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( "click", "click-default-group", "tabulate", - "dateutils", + "python-dateutil", ], setup_requires=["pytest-runner"], extras_require={ From 271b894af52eb6437ae6cd84eba9867ad8dd43f6 Mon Sep 17 00:00:00 2001 From: Mina Rizk Date: Mon, 15 Nov 2021 02:27:40 +0200 Subject: [PATCH 0460/1004] Map dict to TEXT Thanks, @minaeid90 --- sqlite_utils/db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1e75861..591a286 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -167,6 +167,7 @@ COLUMN_TYPE_MAPPING = { int: "INTEGER", bool: "INTEGER", str: "TEXT", + dict: "TEXT", bytes.__class__: "BLOB", bytes: "BLOB", memoryview: "BLOB", @@ -185,6 +186,7 @@ COLUMN_TYPE_MAPPING = { "integer": "INTEGER", "float": "FLOAT", "blob": "BLOB", + } # If numpy is available, add more types if np: From 84007dffa8fd2fcdf4ff24abe6ee90c01c3d08ae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Nov 2021 16:28:53 -0800 Subject: [PATCH 0461/1004] Applied Black, refs #322, #328 --- sqlite_utils/db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 591a286..9667641 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -186,7 +186,6 @@ COLUMN_TYPE_MAPPING = { "integer": "INTEGER", "float": "FLOAT", "blob": "BLOB", - } # If numpy is available, add more types if np: From 9cda5b070f885a7995f0c307bcc4f45f0812994a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Nov 2021 16:36:00 -0800 Subject: [PATCH 0462/1004] Handle dict/tuple/list mapping to TEXT, closes #338 --- sqlite_utils/db.py | 2 ++ tests/test_create.py | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9667641..828ef68 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -168,6 +168,8 @@ COLUMN_TYPE_MAPPING = { bool: "INTEGER", str: "TEXT", dict: "TEXT", + tuple: "TEXT", + list: "TEXT", bytes.__class__: "BLOB", bytes: "BLOB", memoryview: "BLOB", diff --git a/tests/test_create.py b/tests/test_create.py index acaeaee..c9568d9 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1040,3 +1040,30 @@ def test_create_with_nested_bytes(fresh_db): ) def test_quote(fresh_db, input, expected): assert fresh_db.quote(input) == expected + + +@pytest.mark.parametrize( + "columns,expected_sql_middle", + ( + ( + {"id": int}, + "[id] INTEGER", + ), + ( + {"col": dict}, + "[col] TEXT", + ), + ( + {"col": tuple}, + "[col] TEXT", + ), + ( + {"col": list}, + "[col] TEXT", + ), + ), +) +def test_create_table_sql(fresh_db, columns, expected_sql_middle): + sql = fresh_db.create_table_sql("t", columns) + middle = sql.split("(")[1].split(")")[0].strip() + assert middle == expected_sql_middle From 54a2269e91ce72b059618662ed133a85f3d42e4a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Nov 2021 18:01:56 -0800 Subject: [PATCH 0463/1004] Optional second argument to .lookup() to populate extra columns, closes #339 --- docs/python-api.rst | 10 ++++++++++ sqlite_utils/db.py | 34 +++++++++++++++++++++++----------- tests/test_lookup.py | 20 ++++++++++++++++++++ 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 8932656..cc236a2 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -824,6 +824,16 @@ If you pass in a dictionary with multiple values, both values will be used to in }) }) +The ``.lookup()`` method has an optional second argument which can be used to populate other columns in the table but only if the row does not exist yet. These columns will not be included in the unique index. + +To create a species record with a note on when it was first seen, you can use this: + +.. code-block:: python + + db["Species"].lookup({"name": "Palm"}, {"first_seen": "2021-03-04"}) + +The first time this is called the record will be created for ``name="Palm"``. Any subsequent calls with that name will ignore the second argument, even if it includes different values. + .. _python_api_extracts: Populating lookup tables automatically during insert/upsert diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 828ef68..ecf14be 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2713,7 +2713,11 @@ class Table(Queryable): self.add_column(col_name, col_type) return self - def lookup(self, column_values: Dict[str, Any]): + def lookup( + self, + lookup_values: Dict[str, Any], + extra_values: Optional[Dict[str, Any]] = None, + ): """ Create or populate a lookup table with the specified values. @@ -2725,28 +2729,36 @@ class Table(Queryable): It will then insert a new row with the ``name`` set to ``Palm`` and return the new integer primary key value. + An optional second argument can be provided with more ``name: value`` pairs to + be included only if the record is being created for the first time. These will + be ignored on subsequent lookup calls for records that already exist. + See :ref:`python_api_lookup_tables` for more details. """ - # lookups is a dictionary - all columns will be used for a unique index - assert isinstance(column_values, dict) + assert isinstance(lookup_values, dict) + if extra_values is not None: + assert isinstance(extra_values, dict) + combined_values = dict(lookup_values) + if extra_values is not None: + combined_values.update(extra_values) if self.exists(): - self.add_missing_columns([column_values]) + self.add_missing_columns([combined_values]) unique_column_sets = [set(i.columns) for i in self.indexes] - if set(column_values.keys()) not in unique_column_sets: - self.create_index(column_values.keys(), unique=True) - wheres = ["[{}] = ?".format(column) for column in column_values] + if set(lookup_values.keys()) not in unique_column_sets: + self.create_index(lookup_values.keys(), unique=True) + wheres = ["[{}] = ?".format(column) for column in lookup_values] rows = list( self.rows_where( - " and ".join(wheres), [value for _, value in column_values.items()] + " and ".join(wheres), [value for _, value in lookup_values.items()] ) ) try: return rows[0]["id"] except IndexError: - return self.insert(column_values, pk="id").last_pk + return self.insert(combined_values, pk="id").last_pk else: - pk = self.insert(column_values, pk="id").last_pk - self.create_index(column_values.keys(), unique=True) + pk = self.insert(combined_values, pk="id").last_pk + self.create_index(lookup_values.keys(), unique=True) return pk def m2m( diff --git a/tests/test_lookup.py b/tests/test_lookup.py index ac8e1c2..348ef92 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -66,3 +66,23 @@ def test_lookup_fails_if_constraint_cannot_be_added(fresh_db): # This will fail because the name column is not unique with pytest.raises(Exception, match="UNIQUE constraint failed"): species.lookup({"name": "Palm"}) + + +def test_lookup_with_extra_values(fresh_db): + species = fresh_db["species"] + id = species.lookup({"name": "Palm", "type": "Tree"}, {"first_seen": "2020-01-01"}) + assert species.get(id) == { + "id": 1, + "name": "Palm", + "type": "Tree", + "first_seen": "2020-01-01", + } + # A subsequent lookup() should ignore the second dictionary + id2 = species.lookup({"name": "Palm", "type": "Tree"}, {"first_seen": "2021-02-02"}) + assert id2 == id + assert species.get(id2) == { + "id": 1, + "name": "Palm", + "type": "Tree", + "first_seen": "2020-01-01", + } From fb9d61754a9088a4efafce490db01e2999dea2d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Nov 2021 18:19:28 -0800 Subject: [PATCH 0464/1004] Better type signature for hash_id, closes #341 --- sqlite_utils/db.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ecf14be..039f9da 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -639,7 +639,7 @@ class Database: column_order: Optional[List[str]] = None, not_null: Iterable[str] = None, defaults: Optional[Dict[str, Any]] = None, - hash_id: Optional[Any] = None, + hash_id: Optional[str] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, ) -> str: "Returns the SQL ``CREATE TABLE`` statement for creating the specified table." @@ -748,7 +748,7 @@ class Database: column_order: Optional[List[str]] = None, not_null: Iterable[str] = None, defaults: Optional[Dict[str, Any]] = None, - hash_id: Optional[Any] = None, + hash_id: Optional[str] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, ) -> "Table": """ @@ -1046,7 +1046,7 @@ class Table(Queryable): not_null: Iterable[str] = None, defaults: Optional[Dict[str, Any]] = None, batch_size: int = 100, - hash_id: Optional[Any] = None, + hash_id: Optional[str] = None, alter: bool = False, ignore: bool = False, replace: bool = False, @@ -1227,7 +1227,7 @@ class Table(Queryable): column_order: Optional[List[str]] = None, not_null: Iterable[str] = None, defaults: Optional[Dict[str, Any]] = None, - hash_id: Optional[Any] = None, + hash_id: Optional[str] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, ) -> "Table": """ From ffb54427d3c5944ea4ed83d138d3917309cc5242 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Nov 2021 18:56:35 -0800 Subject: [PATCH 0465/1004] insert now replaces square braces in column name with underscore, closes #341 --- sqlite_utils/db.py | 14 +++++++++++++- tests/test_create.py | 8 +------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 039f9da..fd58c40 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2571,6 +2571,8 @@ class Table(Queryable): all_columns = [] first = True num_records_processed = 0 + # Fix up any records with square braces in the column names + records = fix_square_braces(records) # We can only handle a max of 999 variables in a SQL insert, so # we need to adjust the batch_size down if we have too many cols records = iter(records) @@ -2618,7 +2620,6 @@ class Table(Queryable): column for column in record if column not in all_columns ] - validate_column_names(all_columns) first = False self.insert_chunk( @@ -2986,3 +2987,14 @@ def validate_column_names(columns): assert ( "[" not in column and "]" not in column ), "'[' and ']' cannot be used in column names" + + +def fix_square_braces(records: Iterable[Dict[str, Any]]): + for record in records: + if any("[" in key or "]" in key for key in record.keys()): + yield { + key.replace("[", "_").replace("]", "_"): value + for key, value in record.items() + } + else: + yield record diff --git a/tests/test_create.py b/tests/test_create.py index c9568d9..187d20a 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -144,6 +144,7 @@ def test_create_table_with_not_null(fresh_db): [{"name": "memoryview", "type": "BLOB"}], ), ({"uuid": uuid.uuid4()}, [{"name": "uuid", "type": "TEXT"}]), + ({"foo[bar]": 1}, [{"name": "foo_bar_", "type": "INTEGER"}]), ), ) def test_create_table_from_example(fresh_db, example, expected_columns): @@ -525,13 +526,6 @@ def test_insert_row_alter_table( ] -def test_insert_row_alter_table_invalid_column_characters(fresh_db): - table = fresh_db["table"] - table.insert({"foo": "bar"}).last_pk - with pytest.raises(AssertionError): - table.insert({"foo": "baz", "new_col[abc]": 1.2}, alter=True) - - def test_add_missing_columns_case_insensitive(fresh_db): table = fresh_db["foo"] table.insert({"id": 1, "name": "Cleo"}, pk="id") From 3b8abe608796e99e4ffc5f3f4597a85e605c0e9b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Nov 2021 19:15:23 -0800 Subject: [PATCH 0466/1004] Release 3.18 Refs #324, #329, #330, #331, #332, #335, #338, #339 --- docs/changelog.rst | 14 ++++++++++++++ setup.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0eadaa1..0c8f901 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,20 @@ Changelog =========== +.. _v3_18: + +3.18 (2021-11-14) +----------------- + +- The ``table.lookup()`` method now has an optional second argument which can be used to populate columns only the first time the record is created, see :ref:`python_api_lookup_tables`. (:issue:`339`) +- ``sqlite-utils memory`` now has a ``--flatten`` option for :ref:`flattening nested JSON objects ` into separate columns, consistent with ``sqlite-utils insert``. (:issue:`332`) +- ``table.create_index(..., find_unique_name=True)`` parameter, which finds an available name for the created index even if the default name has already been taken. This means that ``index-foreign-keys`` will work even if one of the indexes it tries to create clashes with an existing index name. (:issue:`335`) +- Added ``py.typed`` to the module, so `mypy `__ should now correctly pick up the type annotations. Thanks, Andreas Longo. (:issue:`331`) +- Now depends on ``python-dateutil`` instead of depending on ``dateutils``. Thanks, Denys Pavlov. (:issue:`324`) +- ``table.create()`` (see :ref:`python_api_explicit_create`) now handles ``dict``, ``list`` and ``tuple`` types, mapping them to ``TEXT`` columns in SQLite so that they can be stored encoded as JSON. (:issue:`338`) +- Inserted data with square braces in the column names (for example a CSV file containing a ``item[price]``) column now have the braces converted to underscores: ``item_price_``. Previously such columns would be rejected with an error. (:issue:`329`) +- Now also tested against Python 3.10. (`#330 `__) + .. _v3_17.1: 3.17.1 (2021-09-22) diff --git a/setup.py b/setup.py index f40dcfe..2147697 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.17.1" +VERSION = "3.18" def get_long_description(): From 93b21c230a6ae33e5a4904f042fa513796689bce Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Nov 2021 23:26:50 -0800 Subject: [PATCH 0467/1004] Extra parameters for .lookup(), passed to .insert() - closes #342 --- docs/python-api.rst | 11 ++++++++ sqlite_utils/db.py | 38 +++++++++++++++++++++++--- tests/test_lookup.py | 65 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 4 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index cc236a2..1c0dc60 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -834,6 +834,17 @@ To create a species record with a note on when it was first seen, you can use th The first time this is called the record will be created for ``name="Palm"``. Any subsequent calls with that name will ignore the second argument, even if it includes different values. +``.lookup()`` also accepts keyword arguments, which are passed through to the :ref:`insert() method ` and can be used to influence the shape of the created table. Supported parameters are: + +- ``pk`` - which defaults to ``id`` +- ``foreign_keys`` +- ``column_order`` +- ``not_null`` +- ``defaults`` +- ``extracts`` +- ``conversions`` +- ``columns`` + .. _python_api_extracts: Populating lookup tables automatically during insert/upsert diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index fd58c40..aebca89 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2556,7 +2556,7 @@ class Table(Queryable): ignore = self.value_or_default("ignore", ignore) replace = self.value_or_default("replace", replace) extracts = self.value_or_default("extracts", extracts) - conversions = self.value_or_default("conversions", conversions) + conversions = self.value_or_default("conversions", conversions) or {} columns = self.value_or_default("columns", columns) if upsert and (not pk and not hash_id): @@ -2718,6 +2718,14 @@ class Table(Queryable): self, lookup_values: Dict[str, Any], extra_values: Optional[Dict[str, Any]] = None, + pk: Optional[str] = "id", + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Optional[Set[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, + conversions: Optional[Dict[str, str]] = None, + columns: Optional[Dict[str, Any]] = None, ): """ Create or populate a lookup table with the specified values. @@ -2735,6 +2743,8 @@ class Table(Queryable): be ignored on subsequent lookup calls for records that already exist. See :ref:`python_api_lookup_tables` for more details. + + All other keyword arguments are passed through to ``.insert()``. """ assert isinstance(lookup_values, dict) if extra_values is not None: @@ -2754,11 +2764,31 @@ class Table(Queryable): ) ) try: - return rows[0]["id"] + return rows[0][pk] except IndexError: - return self.insert(combined_values, pk="id").last_pk + return self.insert( + combined_values, + pk=pk, + foreign_keys=foreign_keys, + column_order=column_order, + not_null=not_null, + defaults=defaults, + extracts=extracts, + conversions=conversions, + columns=columns, + ).last_pk else: - pk = self.insert(combined_values, pk="id").last_pk + pk = self.insert( + combined_values, + pk=pk, + foreign_keys=foreign_keys, + column_order=column_order, + not_null=not_null, + defaults=defaults, + extracts=extracts, + conversions=conversions, + columns=columns, + ).last_pk self.create_index(lookup_values.keys(), unique=True) return pk diff --git a/tests/test_lookup.py b/tests/test_lookup.py index 348ef92..31be414 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -86,3 +86,68 @@ def test_lookup_with_extra_values(fresh_db): "type": "Tree", "first_seen": "2020-01-01", } + + +def test_lookup_with_extra_insert_parameters(fresh_db): + other_table = fresh_db["other_table"] + other_table.insert({"id": 1, "name": "Name"}, pk="id") + species = fresh_db["species"] + id = species.lookup( + {"name": "Palm", "type": "Tree"}, + { + "first_seen": "2020-01-01", + "make_not_null": 1, + "fk_to_other": 1, + "default_is_dog": "cat", + "extract_this": "This is extracted", + "convert_to_upper": "upper", + "make_this_integer": "2", + "this_at_front": 1, + }, + pk="renamed_id", + foreign_keys=(("fk_to_other", "other_table", "id"),), + column_order=("this_at_front",), + not_null={"make_not_null"}, + defaults={"default_is_dog": "dog"}, + extracts=["extract_this"], + conversions={"convert_to_upper": "upper(?)"}, + columns={"make_this_integer": int}, + ) + assert species.schema == ( + "CREATE TABLE [species] (\n" + " [renamed_id] INTEGER PRIMARY KEY,\n" + " [this_at_front] INTEGER,\n" + " [name] TEXT,\n" + " [type] TEXT,\n" + " [first_seen] TEXT,\n" + " [make_not_null] INTEGER NOT NULL,\n" + " [fk_to_other] INTEGER REFERENCES [other_table]([id]),\n" + " [default_is_dog] TEXT DEFAULT 'dog',\n" + " [extract_this] INTEGER REFERENCES [extract_this]([id]),\n" + " [convert_to_upper] TEXT,\n" + " [make_this_integer] INTEGER\n" + ")" + ) + assert species.get(id) == { + "renamed_id": id, + "this_at_front": 1, + "name": "Palm", + "type": "Tree", + "first_seen": "2020-01-01", + "make_not_null": 1, + "fk_to_other": 1, + "default_is_dog": "cat", + "extract_this": 1, + "convert_to_upper": "UPPER", + "make_this_integer": 2, + } + assert species.indexes == [ + Index( + seq=0, + name="idx_species_name_type", + unique=1, + origin="c", + partial=0, + columns=["name", "type"], + ) + ] From 8f386a0d300d1b1c76132bb75972b755049fb742 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Nov 2021 23:27:41 -0800 Subject: [PATCH 0468/1004] Release 3.19a0 Refs #342 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2147697..820c871 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.18" +VERSION = "3.19a0" def get_long_description(): From 33176ad47b9757f40ea016e7b8ec328229e60a74 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 19 Nov 2021 00:09:16 -0800 Subject: [PATCH 0469/1004] Run pytest with colors Tip from https://twitter.com/cjolowicz/status/1461266663681187841 --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 86b2531..fec8cfa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,9 @@ name: Test on: [push] +env: + FORCE_COLOR: 1 + jobs: test: runs-on: ${{ matrix.os }} @@ -31,7 +34,7 @@ jobs: run: pip install numpy - name: Run tests run: | - pytest + pytest -v - name: run mypy run: mypy sqlite_utils tests - name: run flake8 From 126703706ea153f63e6134ad14e5712e4bbcb8ae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 20 Nov 2021 20:40:47 -0800 Subject: [PATCH 0470/1004] Release 3.19 Refs #342 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0c8f901..a30e7cf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v3_19: + +3.19 (2021-11-20) +----------------- + +- The :ref:`table.lookup() method ` now accepts keyword arguments that match those on the underlying ``table.insert()`` method: ``foreign_keys=``, ``column_order=``, ``not_null=``, ``defaults=``, ``extracts=``, ``conversions=`` and ``columns=``. You can also now pass ``pk=`` to specify a different column name to use for the primary key. (:issue:`342`) + .. _v3_18: 3.18 (2021-11-14) diff --git a/setup.py b/setup.py index 820c871..7198027 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.19a0" +VERSION = "3.19" def get_long_description(): From e3f108e0f339e3d87ce48541bbca8f891bfaf040 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 14:19:30 -0800 Subject: [PATCH 0471/1004] db.supports_strict and table.strict properties, refs #344 --- docs/python-api.rst | 16 ++++++++++++++-- sqlite_utils/db.py | 21 +++++++++++++++++++++ tests/test_introspect.py | 38 ++++++++++++++++++++++++++++---------- 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 1c0dc60..5d391ee 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1609,8 +1609,8 @@ A more useful example: if you are working with `SpatiaLite `__. + +:: + + >>> db["ny_times_us_counties"].strict + False + .. _python_api_introspection_indexes: .indexes diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index aebca89..fea1051 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -18,6 +18,7 @@ import json import os import pathlib import re +import secrets from sqlite_fts4 import rank_bm25 # type: ignore import sys import textwrap @@ -521,6 +522,19 @@ class Database: sqls.append(sql) return "\n".join(sqls) + @property + def supports_strict(self): + try: + table_name = "t{}".format(secrets.token_hex(16)) + with self.conn: + self.conn.execute( + "create table {} (name text) strict".format(table_name) + ) + self.conn.execute("drop table {}".format(table_name)) + return True + except Exception as e: + return False + @property def journal_mode(self) -> str: "Current ``journal_mode`` of this database." @@ -1219,6 +1233,13 @@ class Table(Queryable): "``{trigger_name: sql}`` dictionary of triggers defined on this table." return {trigger.name: trigger.sql for trigger in self.triggers} + @property + def strict(self) -> bool: + "Is this a STRICT table?" + table_suffix = self.schema.split(")")[-1].strip().upper() + table_options = [bit.strip() for bit in table_suffix.split(",")] + return "STRICT" in table_options + def create( self, columns: Dict[str, Any], diff --git a/tests/test_introspect.py b/tests/test_introspect.py index dce8afc..28a5009 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -252,15 +252,33 @@ def test_has_counts_triggers(fresh_db): ), ], ) -def test_virtual_table_using(sql, expected_name, expected_using): - db = Database(memory=True) - db.execute(sql) - assert db[expected_name].virtual_table_using == expected_using +def test_virtual_table_using(fresh_db, sql, expected_name, expected_using): + fresh_db.execute(sql) + assert fresh_db[expected_name].virtual_table_using == expected_using -def test_use_rowid(): - db = Database(memory=True) - db["rowid_table"].insert({"name": "Cleo"}) - db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id") - assert db["rowid_table"].use_rowid - assert not db["regular_table"].use_rowid +def test_use_rowid(fresh_db): + fresh_db["rowid_table"].insert({"name": "Cleo"}) + fresh_db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id") + assert fresh_db["rowid_table"].use_rowid + assert not fresh_db["regular_table"].use_rowid + + +@pytest.mark.skipif( + not Database(memory=True).supports_strict, + reason="Needs SQLite version that supports strict", +) +@pytest.mark.parametrize( + "create_table,expected_strict", + ( + ("create table t (id integer) strict", True), + ("create table t (id integer) STRICT", True), + ("create table t (id integer primary key) StriCt, WITHOUT ROWID", True), + ("create table t (id integer primary key) WITHOUT ROWID", False), + ("create table t (id integer)", False), + ), +) +def test_table_strict(fresh_db, create_table, expected_strict): + fresh_db.execute(create_table) + table = fresh_db["t"] + assert table.strict == expected_strict From 1f8178f7e41f64965195c1320d310032d783a8b1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 14:29:46 -0800 Subject: [PATCH 0472/1004] Fix flake8 error, refs #344, #345 --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index fea1051..a2c40e1 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -532,7 +532,7 @@ class Database: ) self.conn.execute("drop table {}".format(table_name)) return True - except Exception as e: + except: return False @property From 213a0ff177f23a35f3b235386366ff132eb879f1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 14:34:40 -0800 Subject: [PATCH 0473/1004] Really fix flake8 error, refs #344, #345 --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a2c40e1..0ff056c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -532,7 +532,7 @@ class Database: ) self.conn.execute("drop table {}".format(table_name)) return True - except: + except Exception: return False @property From e328db8eba1fbf29a69eda95dfec861954f9e771 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 10 Dec 2021 13:12:38 -0800 Subject: [PATCH 0474/1004] Improved schema example for sqlite-utils extract --- docs/cli.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index a096fde..1e3b330 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1288,12 +1288,13 @@ This would produce the following schema: [TreeAddress] TEXT, [species_id] INTEGER, FOREIGN KEY(species_id) REFERENCES species(id) - ) - + ); CREATE TABLE [species] ( [id] INTEGER PRIMARY KEY, [species] TEXT - ) + ); + CREATE UNIQUE INDEX [idx_species_species] + ON [species] ([species]); The command takes the following options: From a3df483c803ea6e45cf878025aa8a59d2c62f67e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 10 Dec 2021 16:01:02 -0800 Subject: [PATCH 0475/1004] sqlite-utils convert db table column -, refs #353 --- docs/cli.rst | 10 ++++++++++ sqlite_utils/cli.py | 5 +++++ tests/test_cli_convert.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 1e3b330..1185c1a 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1023,6 +1023,16 @@ You can specify Python modules that should be imported and made available to you '"\n".join(textwrap.wrap(value, 100))' \ --import=textwrap +Use a CODE value of `-` to read from standard input: + + $ cat mycode.py | sqlite-utils convert content.db articles headline - + +Where `mycode.py` contains a fragment of Python code that looks like this: + +```python +return value.upper() +``` + The transformation will be applied to every row in the specified table. You can limit that to just rows that match a ``WHERE`` clause using ``--where``:: $ sqlite-utils convert content.db articles headline 'value.upper()' \ diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cd0be82..0345d55 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2038,6 +2038,8 @@ def _generate_convert_help(): "value" is a variable with the column value to be converted. + Use "-" for CODE to read Python code from standard input. + The following common operations are available as recipe functions: """ ).strip() @@ -2120,6 +2122,9 @@ def convert( raise click.ClickException("Cannot use --multi with more than one column") if drop and not (output or multi): raise click.ClickException("--drop can only be used with --output or --multi") + if code == "-": + # Read code from standard input + code = sys.stdin.read() # If single line and no 'return', add the return if "\n" not in code and not code.strip().startswith("return "): code = "return {}".format(code) diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 8ce5203..2c77c0c 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -515,3 +515,36 @@ def test_convert_where_multi(fresh_db_and_path): {"id": 1, "name": "Cleo", "upper": None}, {"id": 2, "name": "Bants", "upper": "BANTS"}, ] + + +def test_convert_code_standard_input(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "names", + "name", + "-", + ], + input="value.upper()", + ) + assert 0 == result.exit_code, result.output + assert list(db["names"].rows) == [ + {"id": 1, "name": "CLEO"}, + ] + + +def test_convert_hyphen_workaround(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + result = CliRunner().invoke( + cli.cli, + ["convert", db_path, "names", "name", '"-"'], + ) + assert 0 == result.exit_code, result.output + assert list(db["names"].rows) == [ + {"id": 1, "name": "-"}, + ] From 7a43af232e4bc00bd227307665163614e225948b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 10 Dec 2021 16:11:22 -0800 Subject: [PATCH 0476/1004] Support nested imports, closes #351 --- docs/cli.rst | 6 ++++++ sqlite_utils/cli.py | 2 +- tests/test_cli_convert.py | 21 +++++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 1185c1a..6a39c1e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1023,6 +1023,12 @@ You can specify Python modules that should be imported and made available to you '"\n".join(textwrap.wrap(value, 100))' \ --import=textwrap +This supports nested imports as well, for example to use `ElementTree `__:: + + $ sqlite-utils convert content.db articles content \ + 'xml.etree.ElementTree.fromstring(value).attrib["title"]' \ + --import=xml.etree.ElementTree + Use a CODE value of `-` to read from standard input: $ cat mycode.py | sqlite-utils convert content.db articles headline - diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0345d55..c19339d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2137,7 +2137,7 @@ def convert( locals = {} globals = {"r": recipes, "recipes": recipes} for import_ in imports: - globals[import_] = __import__(import_) + globals[import_.split(".")[0]] = __import__(import_) exec(code_o, globals, locals) fn = locals["fn"] if dry_run: diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 2c77c0c..8755600 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -93,6 +93,27 @@ def test_convert_import(test_db_and_path): ] == list(db["example"].rows) +def test_convert_import_nested(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["example"].insert({"xml": ''}) + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "xml", + 'xml.etree.ElementTree.fromstring(value).attrib["name"]', + "--import", + "xml.etree.ElementTree", + ], + ) + assert 0 == result.exit_code, result.output + assert [ + {"xml": "Cleo"}, + ] == list(db["example"].rows) + + def test_convert_dryrun(test_db_and_path): db, db_path = test_db_and_path result = CliRunner().invoke( From 500a35ad4d91c8a6232134ce9406efec11bedff8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 10 Dec 2021 16:49:28 -0800 Subject: [PATCH 0477/1004] Also support def convert(value), closes #355 Plus added custom syntax error display --- docs/cli.rst | 19 +++++++---- sqlite_utils/cli.py | 22 +++++-------- sqlite_utils/utils.py | 28 ++++++++++++++++ tests/test_cli_convert.py | 69 +++++++++++++++++++++++---------------- 4 files changed, 91 insertions(+), 47 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 6a39c1e..aa11286 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1029,17 +1029,24 @@ This supports nested imports as well, for example to use `ElementTree ", "exec") - locals = {} - globals = {"r": recipes, "recipes": recipes} - for import_ in imports: - globals[import_.split(".")[0]] = __import__(import_) - exec(code_o, globals, locals) - fn = locals["fn"] + try: + fn = _compile_code(code, imports) + except SyntaxError as e: + raise click.ClickException( + "Syntax error in code:\n\n{}\n\n{}".format( + textwrap.indent(e.text.strip(), " "), e.msg + ) + ) if dry_run: # Pull first 20 values for first column and preview them db.conn.create_function("preview_transform", 1, lambda v: fn(v) if v else v) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 00a3c02..fe71b73 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -5,6 +5,7 @@ import enum import io import json import os +from . import recipes from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type import click @@ -278,3 +279,30 @@ def progressbar(*args, **kwargs): else: with click.progressbar(*args, **kwargs) as bar: yield bar + + +def _compile_code(code, imports): + locals = {} + globals = {"r": recipes, "recipes": recipes} + # If user defined a convert() function, return that + try: + exec(code, globals, locals) + return locals["convert"] + except (SyntaxError, NameError, KeyError, TypeError): + pass + + # Try compiling their code as a function instead + + # If single line and no 'return', add the return + if "\n" not in code and not code.strip().startswith("return "): + code = "return {}".format(code) + + new_code = ["def fn(value):"] + for line in code.split("\n"): + new_code.append(" {}".format(line)) + code_o = compile("\n".join(new_code), "", "exec") + + for import_ in imports: + globals[import_.split(".")[0]] = __import__(import_) + exec(code_o, globals, locals) + return locals["fn"] diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 8755600..481bebd 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -35,39 +35,52 @@ def fresh_db_and_path(tmpdir): "return value.replace('October', 'Spooktober')", # Return is optional: "value.replace('October', 'Spooktober')", + # Multiple lines are supported: + "v = value.replace('October', 'Spooktober')\nreturn v", + # Can also define a convert() function + "def convert(value): return value.replace('October', 'Spooktober')", + # ... with imports + "import re\n\ndef convert(value): return value.replace('October', 'Spooktober')", ], ) -def test_convert_single_line(test_db_and_path, code): - db, db_path = test_db_and_path - result = CliRunner().invoke(cli.cli, ["convert", db_path, "example", "dt", code]) - assert 0 == result.exit_code, result.output - assert [ - {"id": 1, "dt": "5th Spooktober 2019 12:04"}, - {"id": 2, "dt": "6th Spooktober 2019 00:05:06"}, - {"id": 3, "dt": ""}, - {"id": 4, "dt": None}, - ] == list(db["example"].rows) - - -def test_convert_multiple_lines(test_db_and_path): - db, db_path = test_db_and_path +def test_convert_code(fresh_db_and_path, code): + db, db_path = fresh_db_and_path + db["t"].insert({"text": "October"}) result = CliRunner().invoke( - cli.cli, - [ - "convert", - db_path, - "example", - "dt", - "v = value.replace('October', 'Spooktober')\nreturn v.upper()", - ], + cli.cli, ["convert", db_path, "t", "text", code], catch_exceptions=False ) assert 0 == result.exit_code, result.output - assert [ - {"id": 1, "dt": "5TH SPOOKTOBER 2019 12:04"}, - {"id": 2, "dt": "6TH SPOOKTOBER 2019 00:05:06"}, - {"id": 3, "dt": ""}, - {"id": 4, "dt": None}, - ] == list(db["example"].rows) + value = list(db["t"].rows)[0]["text"] + assert value == "Spooktober" + + +@pytest.mark.parametrize( + "bad_code,expected_error", + [ + ( + "def foo(value)", + """Error: Syntax error in code: + + return def foo(value) + +invalid syntax""", + ), + ( + "$", + """Error: Syntax error in code: + + return $ + +invalid syntax""", + ), + ], +) +def test_convert_code_errors(fresh_db_and_path, bad_code, expected_error): + db, db_path = fresh_db_and_path + db["t"].insert({"text": "October"}) + result = CliRunner().invoke(cli.cli, ["convert", db_path, "t", "text", bad_code]) + assert 1 == result.exit_code + assert result.output.strip() == expected_error.strip() def test_convert_import(test_db_and_path): From 3b2a7c0e5bfc05b18eddb40dabb71dee9a333a15 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 10 Dec 2021 16:56:50 -0800 Subject: [PATCH 0478/1004] Refactor test for #149, spitting it from other rebuild test Also refs #354 --- tests/test_cli.py | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 47920c6..26ef004 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -556,8 +556,7 @@ def test_optimize(db_path, tables): assert 0 == result.exit_code -@pytest.mark.parametrize("tables", ([], ["fts4_table"], ["fts5_table"])) -def test_rebuild_fts(db_path, tables): +def test_rebuild_fts_fixes_docsize_error(db_path): db = Database(db_path, recursive_triggers=False) records = [ { @@ -568,43 +567,22 @@ def test_rebuild_fts(db_path, tables): for i in range(10000) ] with db.conn: - db["fts4_table"].insert_all(records, pk="c1") db["fts5_table"].insert_all(records, pk="c1") - db["fts4_table"].enable_fts( - ["c1", "c2", "c3"], fts_version="FTS4", create_triggers=True - ) db["fts5_table"].enable_fts( ["c1", "c2", "c3"], fts_version="FTS5", create_triggers=True ) # Search should work - assert list(db["fts4_table"].search("verb1")) assert list(db["fts5_table"].search("verb1")) - # Deleting _fts_segments to break FTS4 - with db.conn: - db["fts4_table_fts_segments"].delete_where() - # Now this should error: - with pytest.raises(sqlite3.DatabaseError): - list(db["fts4_table"].search("verb1")) # Replicate docsize error from this issue for FTS5 # https://github.com/simonw/sqlite-utils/issues/149 assert db["fts5_table_fts_docsize"].count == 10000 db["fts5_table"].insert_all(records, replace=True) assert db["fts5_table"].count == 10000 assert db["fts5_table_fts_docsize"].count == 20000 - # Running rebuild-fts should fix both issues - print(["rebuild-fts", db_path] + tables) - result = CliRunner().invoke(cli.cli, ["rebuild-fts", db_path] + tables) + # Running rebuild-fts should fix this + result = CliRunner().invoke(cli.cli, ["rebuild-fts", db_path, "fts5_table"]) assert 0 == result.exit_code - fixed_tables = tables or ["fts4_table", "fts5_table"] - if "fts4_table" in fixed_tables: - assert list(db["fts4_table"].search("verb1")) - else: - with pytest.raises(sqlite3.DatabaseError): - list(db["fts4_table"].search("verb1")) - if "fts5_table" in fixed_tables: - assert db["fts5_table_fts_docsize"].count == 10000 - else: - assert db["fts5_table_fts_docsize"].count == 20000 + assert db["fts5_table_fts_docsize"].count == 10000 def test_insert_simple(tmpdir): From ee13f98c2c7ca3b819bd0fc55da3108cb6a6434a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 10 Dec 2021 16:59:37 -0800 Subject: [PATCH 0479/1004] Better test for rebuild, refs #354 --- tests/test_fts.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/tests/test_fts.py b/tests/test_fts.py index 6aa6fbe..0ae2a52 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -254,13 +254,12 @@ def test_disable_fts(fresh_db, create_triggers): assert ["searchable"] == fresh_db.table_names() -@pytest.mark.parametrize("table_to_fix", ["searchable", "searchable_fts"]) -def test_rebuild_fts(fresh_db, table_to_fix): +def test_rebuild_fts(fresh_db): table = fresh_db["searchable"] table.insert(search_records[0]) table.enable_fts(["text", "country"]) # Run a search - rows = list(table.search("tanuki")) + rows = list(table.search("are")) assert len(rows) == 1 assert { "rowid": 1, @@ -268,21 +267,14 @@ def test_rebuild_fts(fresh_db, table_to_fix): "country": "Japan", "not_searchable": "foo", }.items() <= rows[0].items() - # Delete from searchable_fts_data - fresh_db["searchable_fts_data"].delete_where() - # This should have broken the index - with pytest.raises(sqlite3.DatabaseError): - list(table.search("tanuki")) + # Insert another record + table.insert(search_records[1]) + # This should NOT show up in a searchs + assert len(list(table.search("are"))) == 1 # Running rebuild_fts() should fix it - fresh_db[table_to_fix].rebuild_fts() - rows2 = list(table.search("tanuki")) - assert len(rows2) == 1 - assert { - "rowid": 1, - "text": "tanuki are running tricksters", - "country": "Japan", - "not_searchable": "foo", - }.items() <= rows2[0].items() + table.rebuild_fts() + rows2 = list(table.search("are")) + assert len(rows2) == 2 @pytest.mark.parametrize("invalid_table", ["does_not_exist", "not_searchable"]) From f3fd8613113d21d44238a6ec54b375f5aa72c4e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Dec 2021 12:43:12 -0800 Subject: [PATCH 0480/1004] Removed unneccessary pytest-runner, closes #357 --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 7198027..3cc7b0e 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ setup( "tabulate", "python-dateutil", ], - setup_requires=["pytest-runner"], extras_require={ "test": ["pytest", "black", "hypothesis"], "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], From 9e286cc6d2edc14ee7f7263450b11cfdc8f72157 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Jan 2022 17:32:40 -0800 Subject: [PATCH 0481/1004] New help for --lines and --all and --convert and --import, refs #356 --- docs/cli.rst | 2 +- sqlite_utils/cli.py | 98 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index aa11286..6482c46 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1029,7 +1029,7 @@ This supports nested imports as well, for example to use `ElementTree Date: Wed, 5 Jan 2022 17:57:03 -0800 Subject: [PATCH 0482/1004] Refactored sqlite-utils insert tests into test_cli_insert.py --- tests/conftest.py | 14 ++ tests/test_cli.py | 334 --------------------------------------- tests/test_cli_insert.py | 324 +++++++++++++++++++++++++++++++++++++ 3 files changed, 338 insertions(+), 334 deletions(-) create mode 100644 tests/test_cli_insert.py diff --git a/tests/conftest.py b/tests/conftest.py index 4541689..621d67e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,12 @@ from sqlite_utils import Database +from sqlite_utils.utils import sqlite3 import pytest +CREATE_TABLES = """ +create table Gosh (c1 text, c2 text, c3 text); +create table Gosh2 (c1 text, c2 text, c3 text); +""" + @pytest.fixture def fresh_db(): @@ -19,3 +25,11 @@ def existing_db(): """ ) return database + + +@pytest.fixture +def db_path(tmpdir): + path = str(tmpdir / "test.db") + db = sqlite3.connect(path) + db.executescript(CREATE_TABLES) + return path diff --git a/tests/test_cli.py b/tests/test_cli.py index 26ef004..c4c50a9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,20 +11,6 @@ import textwrap from .utils import collapse_whitespace -CREATE_TABLES = """ -create table Gosh (c1 text, c2 text, c3 text); -create table Gosh2 (c1 text, c2 text, c3 text); -""" - - -@pytest.fixture -def db_path(tmpdir): - path = str(tmpdir / "test.db") - db = sqlite3.connect(path) - db.executescript(CREATE_TABLES) - return path - - @pytest.mark.parametrize( "options", ( @@ -585,326 +571,6 @@ def test_rebuild_fts_fixes_docsize_error(db_path): assert db["fts5_table_fts_docsize"].count == 10000 -def test_insert_simple(tmpdir): - json_path = str(tmpdir / "dog.json") - db_path = str(tmpdir / "dogs.db") - open(json_path, "w").write(json.dumps({"name": "Cleo", "age": 4})) - result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path]) - assert 0 == result.exit_code - assert [{"age": 4, "name": "Cleo"}] == list( - Database(db_path).query("select * from dogs") - ) - db = Database(db_path) - assert ["dogs"] == db.table_names() - assert [] == db["dogs"].indexes - - -def test_insert_from_stdin(tmpdir): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "dogs", "-"], - input=json.dumps({"name": "Cleo", "age": 4}), - ) - assert 0 == result.exit_code - assert [{"age": 4, "name": "Cleo"}] == list( - Database(db_path).query("select * from dogs") - ) - - -def test_insert_invalid_json_error(tmpdir): - db_path = str(tmpdir / "dogs.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "dogs", "-"], - input="name,age\nCleo,4", - ) - assert result.exit_code == 1 - assert ( - result.output - == "Error: Invalid JSON - use --csv for CSV or --tsv for TSV files\n" - ) - - -def test_insert_json_flatten(tmpdir): - db_path = str(tmpdir / "flat.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "items", "-", "--flatten"], - input=json.dumps({"nested": {"data": 4}}), - ) - assert result.exit_code == 0 - assert list(Database(db_path).query("select * from items")) == [{"nested_data": 4}] - - -def test_insert_json_flatten_nl(tmpdir): - db_path = str(tmpdir / "flat.db") - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "items", "-", "--flatten", "--nl"], - input="\n".join( - json.dumps(item) - for item in [{"nested": {"data": 4}}, {"nested": {"other": 3}}] - ), - ) - assert result.exit_code == 0 - assert list(Database(db_path).query("select * from items")) == [ - {"nested_data": 4, "nested_other": None}, - {"nested_data": None, "nested_other": 3}, - ] - - -def test_insert_with_primary_key(db_path, tmpdir): - json_path = str(tmpdir / "dog.json") - open(json_path, "w").write(json.dumps({"id": 1, "name": "Cleo", "age": 4})) - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] - ) - assert 0 == result.exit_code - assert [{"id": 1, "age": 4, "name": "Cleo"}] == list( - Database(db_path).query("select * from dogs") - ) - db = Database(db_path) - assert ["id"] == db["dogs"].pks - - -def test_insert_multiple_with_primary_key(db_path, tmpdir): - json_path = str(tmpdir / "dogs.json") - dogs = [{"id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21)] - open(json_path, "w").write(json.dumps(dogs)) - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] - ) - assert 0 == result.exit_code - db = Database(db_path) - assert dogs == list(db.query("select * from dogs order by id")) - assert ["id"] == db["dogs"].pks - - -def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): - json_path = str(tmpdir / "dogs.json") - dogs = [ - {"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3} - for i in range(1, 21) - ] - open(json_path, "w").write(json.dumps(dogs)) - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--pk", "breed"] - ) - assert 0 == result.exit_code - db = Database(db_path) - assert dogs == list(db.query("select * from dogs order by breed, id")) - assert {"breed", "id"} == set(db["dogs"].pks) - assert ( - "CREATE TABLE [dogs] (\n" - " [breed] TEXT,\n" - " [id] INTEGER,\n" - " [name] TEXT,\n" - " [age] INTEGER,\n" - " PRIMARY KEY ([id], [breed])\n" - ")" - ) == db["dogs"].schema - - -def test_insert_not_null_default(db_path, tmpdir): - json_path = str(tmpdir / "dogs.json") - dogs = [ - {"id": i, "name": "Cleo {}".format(i), "age": i + 3, "score": 10} - for i in range(1, 21) - ] - open(json_path, "w").write(json.dumps(dogs)) - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "dogs", json_path, "--pk", "id"] - + ["--not-null", "name", "--not-null", "age"] - + ["--default", "score", "5", "--default", "age", "1"], - ) - assert 0 == result.exit_code - db = Database(db_path) - assert ( - "CREATE TABLE [dogs] (\n" - " [id] INTEGER PRIMARY KEY,\n" - " [name] TEXT NOT NULL,\n" - " [age] INTEGER NOT NULL DEFAULT '1',\n" - " [score] INTEGER DEFAULT '5'\n)" - ) == db["dogs"].schema - - -def test_insert_binary_base64(db_path): - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "files", "-"], - input=r'{"content": {"$base64": true, "encoded": "aGVsbG8="}}', - ) - assert 0 == result.exit_code, result.output - db = Database(db_path) - actual = list(db.query("select content from files")) - assert actual == [{"content": b"hello"}] - - -def test_insert_newline_delimited(db_path): - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "from_json_nl", "-", "--nl"], - input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', - ) - assert 0 == result.exit_code, result.output - db = Database(db_path) - assert [ - {"foo": "bar", "n": 1}, - {"foo": "baz", "n": 2}, - ] == list(db.query("select foo, n from from_json_nl")) - - -def test_insert_ignore(db_path, tmpdir): - db = Database(db_path) - db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") - json_path = str(tmpdir / "dogs.json") - open(json_path, "w").write(json.dumps([{"id": 1, "name": "Bailey"}])) - # Should raise error without --ignore - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] - ) - assert 0 != result.exit_code, result.output - # If we use --ignore it should run OK - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--ignore"] - ) - assert 0 == result.exit_code, result.output - # ... but it should actually have no effect - assert [{"id": 1, "name": "Cleo"}] == list(db.query("select * from dogs")) - - -@pytest.mark.parametrize( - "content,options", - [ - ("foo\tbar\tbaz\n1\t2\tcat,dog", ["--tsv"]), - ('foo,bar,baz\n1,2,"cat,dog"', ["--csv"]), - ('foo;bar;baz\n1;2;"cat,dog"', ["--csv", "--delimiter", ";"]), - # --delimiter implies --csv: - ('foo;bar;baz\n1;2;"cat,dog"', ["--delimiter", ";"]), - ("foo,bar,baz\n1,2,|cat,dog|", ["--csv", "--quotechar", "|"]), - ("foo,bar,baz\n1,2,|cat,dog|", ["--quotechar", "|"]), - ], -) -def test_insert_csv_tsv(content, options, db_path, tmpdir): - db = Database(db_path) - file_path = str(tmpdir / "insert.csv-tsv") - open(file_path, "w").write(content) - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "data", file_path] + options, - catch_exceptions=False, - ) - assert 0 == result.exit_code - assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows) - - -@pytest.mark.parametrize( - "options", - ( - ["--tsv", "--nl"], - ["--tsv", "--csv"], - ["--csv", "--nl"], - ["--csv", "--nl", "--tsv"], - ), -) -def test_only_allow_one_of_nl_tsv_csv(options, db_path, tmpdir): - file_path = str(tmpdir / "insert.csv-tsv") - open(file_path, "w").write("foo") - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "data", file_path] + options - ) - assert 0 != result.exit_code - assert "Error: Use just one of --nl, --csv or --tsv" == result.output.strip() - - -def test_insert_replace(db_path, tmpdir): - test_insert_multiple_with_primary_key(db_path, tmpdir) - json_path = str(tmpdir / "insert-replace.json") - db = Database(db_path) - assert 20 == db["dogs"].count - insert_replace_dogs = [ - {"id": 1, "name": "Insert replaced 1", "age": 4}, - {"id": 2, "name": "Insert replaced 2", "age": 4}, - {"id": 21, "name": "Fresh insert 21", "age": 6}, - ] - open(json_path, "w").write(json.dumps(insert_replace_dogs)) - result = CliRunner().invoke( - cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--replace"] - ) - assert 0 == result.exit_code, result.output - assert 21 == db["dogs"].count - assert ( - list(db.query("select * from dogs where id in (1, 2, 21) order by id")) - == insert_replace_dogs - ) - - -def test_insert_truncate(db_path): - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "from_json_nl", "-", "--nl", "--batch-size=1"], - input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', - ) - assert 0 == result.exit_code, result.output - db = Database(db_path) - assert [ - {"foo": "bar", "n": 1}, - {"foo": "baz", "n": 2}, - ] == list(db.query("select foo, n from from_json_nl")) - # Truncate and insert new rows - result = CliRunner().invoke( - cli.cli, - [ - "insert", - db_path, - "from_json_nl", - "-", - "--nl", - "--truncate", - "--batch-size=1", - ], - input='{"foo": "bam", "n": 3}\n{"foo": "bat", "n": 4}', - ) - assert 0 == result.exit_code, result.output - assert [ - {"foo": "bam", "n": 3}, - {"foo": "bat", "n": 4}, - ] == list(db.query("select foo, n from from_json_nl")) - - -def test_insert_alter(db_path, tmpdir): - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "from_json_nl", "-", "--nl"], - input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', - ) - assert 0 == result.exit_code, result.output - # Should get an error with incorrect shaped additional data - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "from_json_nl", "-", "--nl"], - input='{"foo": "bar", "baz": 5}', - ) - assert 0 != result.exit_code, result.output - # If we run it again with --alter it should work correctly - result = CliRunner().invoke( - cli.cli, - ["insert", db_path, "from_json_nl", "-", "--nl", "--alter"], - input='{"foo": "bar", "baz": 5}', - ) - assert 0 == result.exit_code, result.output - # Soundness check the database itself - db = Database(db_path) - assert {"foo": str, "n": int, "baz": int} == db["from_json_nl"].columns_dict - assert [ - {"foo": "bar", "n": 1, "baz": None}, - {"foo": "baz", "n": 2, "baz": None}, - {"foo": "bar", "baz": 5, "n": None}, - ] == list(db.query("select foo, n, baz from from_json_nl")) - - @pytest.mark.parametrize( "format,expected", [ diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py new file mode 100644 index 0000000..b01b2cf --- /dev/null +++ b/tests/test_cli_insert.py @@ -0,0 +1,324 @@ +from sqlite_utils import cli, Database +from click.testing import CliRunner +import json +import pytest + + +def test_insert_simple(tmpdir): + json_path = str(tmpdir / "dog.json") + db_path = str(tmpdir / "dogs.db") + open(json_path, "w").write(json.dumps({"name": "Cleo", "age": 4})) + result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path]) + assert 0 == result.exit_code + assert [{"age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") + ) + db = Database(db_path) + assert ["dogs"] == db.table_names() + assert [] == db["dogs"].indexes + + +def test_insert_from_stdin(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", "-"], + input=json.dumps({"name": "Cleo", "age": 4}), + ) + assert 0 == result.exit_code + assert [{"age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") + ) + + +def test_insert_invalid_json_error(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", "-"], + input="name,age\nCleo,4", + ) + assert result.exit_code == 1 + assert ( + result.output + == "Error: Invalid JSON - use --csv for CSV or --tsv for TSV files\n" + ) + + +def test_insert_json_flatten(tmpdir): + db_path = str(tmpdir / "flat.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-", "--flatten"], + input=json.dumps({"nested": {"data": 4}}), + ) + assert result.exit_code == 0 + assert list(Database(db_path).query("select * from items")) == [{"nested_data": 4}] + + +def test_insert_json_flatten_nl(tmpdir): + db_path = str(tmpdir / "flat.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-", "--flatten", "--nl"], + input="\n".join( + json.dumps(item) + for item in [{"nested": {"data": 4}}, {"nested": {"other": 3}}] + ), + ) + assert result.exit_code == 0 + assert list(Database(db_path).query("select * from items")) == [ + {"nested_data": 4, "nested_other": None}, + {"nested_data": None, "nested_other": 3}, + ] + + +def test_insert_with_primary_key(db_path, tmpdir): + json_path = str(tmpdir / "dog.json") + open(json_path, "w").write(json.dumps({"id": 1, "name": "Cleo", "age": 4})) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert 0 == result.exit_code + assert [{"id": 1, "age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") + ) + db = Database(db_path) + assert ["id"] == db["dogs"].pks + + +def test_insert_multiple_with_primary_key(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + dogs = [{"id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21)] + open(json_path, "w").write(json.dumps(dogs)) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert 0 == result.exit_code + db = Database(db_path) + assert dogs == list(db.query("select * from dogs order by id")) + assert ["id"] == db["dogs"].pks + + +def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + dogs = [ + {"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3} + for i in range(1, 21) + ] + open(json_path, "w").write(json.dumps(dogs)) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--pk", "breed"] + ) + assert 0 == result.exit_code + db = Database(db_path) + assert dogs == list(db.query("select * from dogs order by breed, id")) + assert {"breed", "id"} == set(db["dogs"].pks) + assert ( + "CREATE TABLE [dogs] (\n" + " [breed] TEXT,\n" + " [id] INTEGER,\n" + " [name] TEXT,\n" + " [age] INTEGER,\n" + " PRIMARY KEY ([id], [breed])\n" + ")" + ) == db["dogs"].schema + + +def test_insert_not_null_default(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + dogs = [ + {"id": i, "name": "Cleo {}".format(i), "age": i + 3, "score": 10} + for i in range(1, 21) + ] + open(json_path, "w").write(json.dumps(dogs)) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", json_path, "--pk", "id"] + + ["--not-null", "name", "--not-null", "age"] + + ["--default", "score", "5", "--default", "age", "1"], + ) + assert 0 == result.exit_code + db = Database(db_path) + assert ( + "CREATE TABLE [dogs] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [name] TEXT NOT NULL,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [score] INTEGER DEFAULT '5'\n)" + ) == db["dogs"].schema + + +def test_insert_binary_base64(db_path): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "files", "-"], + input=r'{"content": {"$base64": true, "encoded": "aGVsbG8="}}', + ) + assert 0 == result.exit_code, result.output + db = Database(db_path) + actual = list(db.query("select content from files")) + assert actual == [{"content": b"hello"}] + + +def test_insert_newline_delimited(db_path): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_json_nl", "-", "--nl"], + input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', + ) + assert 0 == result.exit_code, result.output + db = Database(db_path) + assert [ + {"foo": "bar", "n": 1}, + {"foo": "baz", "n": 2}, + ] == list(db.query("select foo, n from from_json_nl")) + + +def test_insert_ignore(db_path, tmpdir): + db = Database(db_path) + db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + json_path = str(tmpdir / "dogs.json") + open(json_path, "w").write(json.dumps([{"id": 1, "name": "Bailey"}])) + # Should raise error without --ignore + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert 0 != result.exit_code, result.output + # If we use --ignore it should run OK + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--ignore"] + ) + assert 0 == result.exit_code, result.output + # ... but it should actually have no effect + assert [{"id": 1, "name": "Cleo"}] == list(db.query("select * from dogs")) + + +@pytest.mark.parametrize( + "content,options", + [ + ("foo\tbar\tbaz\n1\t2\tcat,dog", ["--tsv"]), + ('foo,bar,baz\n1,2,"cat,dog"', ["--csv"]), + ('foo;bar;baz\n1;2;"cat,dog"', ["--csv", "--delimiter", ";"]), + # --delimiter implies --csv: + ('foo;bar;baz\n1;2;"cat,dog"', ["--delimiter", ";"]), + ("foo,bar,baz\n1,2,|cat,dog|", ["--csv", "--quotechar", "|"]), + ("foo,bar,baz\n1,2,|cat,dog|", ["--quotechar", "|"]), + ], +) +def test_insert_csv_tsv(content, options, db_path, tmpdir): + db = Database(db_path) + file_path = str(tmpdir / "insert.csv-tsv") + open(file_path, "w").write(content) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "data", file_path] + options, + catch_exceptions=False, + ) + assert 0 == result.exit_code + assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows) + + +@pytest.mark.parametrize( + "options", + ( + ["--tsv", "--nl"], + ["--tsv", "--csv"], + ["--csv", "--nl"], + ["--csv", "--nl", "--tsv"], + ), +) +def test_only_allow_one_of_nl_tsv_csv(options, db_path, tmpdir): + file_path = str(tmpdir / "insert.csv-tsv") + open(file_path, "w").write("foo") + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "data", file_path] + options + ) + assert 0 != result.exit_code + assert "Error: Use just one of --nl, --csv or --tsv" == result.output.strip() + + +def test_insert_replace(db_path, tmpdir): + test_insert_multiple_with_primary_key(db_path, tmpdir) + json_path = str(tmpdir / "insert-replace.json") + db = Database(db_path) + assert 20 == db["dogs"].count + insert_replace_dogs = [ + {"id": 1, "name": "Insert replaced 1", "age": 4}, + {"id": 2, "name": "Insert replaced 2", "age": 4}, + {"id": 21, "name": "Fresh insert 21", "age": 6}, + ] + open(json_path, "w").write(json.dumps(insert_replace_dogs)) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--replace"] + ) + assert 0 == result.exit_code, result.output + assert 21 == db["dogs"].count + assert ( + list(db.query("select * from dogs where id in (1, 2, 21) order by id")) + == insert_replace_dogs + ) + + +def test_insert_truncate(db_path): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_json_nl", "-", "--nl", "--batch-size=1"], + input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', + ) + assert 0 == result.exit_code, result.output + db = Database(db_path) + assert [ + {"foo": "bar", "n": 1}, + {"foo": "baz", "n": 2}, + ] == list(db.query("select foo, n from from_json_nl")) + # Truncate and insert new rows + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "from_json_nl", + "-", + "--nl", + "--truncate", + "--batch-size=1", + ], + input='{"foo": "bam", "n": 3}\n{"foo": "bat", "n": 4}', + ) + assert 0 == result.exit_code, result.output + assert [ + {"foo": "bam", "n": 3}, + {"foo": "bat", "n": 4}, + ] == list(db.query("select foo, n from from_json_nl")) + + +def test_insert_alter(db_path, tmpdir): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_json_nl", "-", "--nl"], + input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', + ) + assert 0 == result.exit_code, result.output + # Should get an error with incorrect shaped additional data + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_json_nl", "-", "--nl"], + input='{"foo": "bar", "baz": 5}', + ) + assert 0 != result.exit_code, result.output + # If we run it again with --alter it should work correctly + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_json_nl", "-", "--nl", "--alter"], + input='{"foo": "bar", "baz": 5}', + ) + assert 0 == result.exit_code, result.output + # Soundness check the database itself + db = Database(db_path) + assert {"foo": str, "n": int, "baz": int} == db["from_json_nl"].columns_dict + assert [ + {"foo": "bar", "n": 1, "baz": None}, + {"foo": "baz", "n": 2, "baz": None}, + {"foo": "bar", "baz": 5, "n": None}, + ] == list(db.query("select foo, n, baz from from_json_nl")) From f1569c9f7fc063ddf2f1ca91d5f1798afa9d0262 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Jan 2022 18:10:10 -0800 Subject: [PATCH 0483/1004] Implemented sqlite-utils insert --lines --- docs/cli.rst | 9 +++++++++ sqlite_utils/cli.py | 2 ++ tests/test_cli_insert.py | 15 +++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 6482c46..f93fc57 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -876,6 +876,15 @@ If your file does not include this row, you can use the ``--no-headers`` option If you do this, the table will be created with column names called ``untitled_1`` and ``untitled_2`` and so on. You can then rename them using the ``sqlite-utils transform ... --rename`` command, see :ref:`cli_transform_table`. +.. _cli_insert_lines: + +Inserting newline-delimited data +================================ + +If you have an unstructured file you can insert its contents into a table with a single ``line`` column containing each line from the file using ``--lines``. This can be useful if you intend to further analyze those lines using SQL string functions or :ref:`sqlite-utils convert `:: + + $ sqlite-utils insert logs.db loglines logfile.log --lines + .. _cli_insert_replace: Insert-replacing data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 319dee1..6d3f8eb 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -783,6 +783,8 @@ def insert_upsert_implementation( if detect_types: tracker = TypeTracker() docs = tracker.wrap(docs) + elif lines: + docs = ({"line": line.strip()} for line in decoded) elif convert: fn = _compile_code(convert, imports) docs = (fn(line) for line in decoded) diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index b01b2cf..2922a2f 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -322,3 +322,18 @@ def test_insert_alter(db_path, tmpdir): {"foo": "baz", "n": 2, "baz": None}, {"foo": "bar", "baz": 5, "n": None}, ] == list(db.query("select foo, n, baz from from_json_nl")) + + +def test_insert_lines(db_path): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_lines", "-", "--lines"], + input='First line\nSecond line\n{"foo": "baz"}', + ) + assert 0 == result.exit_code, result.output + db = Database(db_path) + assert [ + {"line": "First line"}, + {"line": "Second line"}, + {"line": '{"foo": "baz"}'}, + ] == list(db.query("select line from from_lines")) From e66299c6eda3091557504526aaf0f64fb321cb35 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Jan 2022 18:16:51 -0800 Subject: [PATCH 0484/1004] Implemented and documented sqlite-utils insert --all --- docs/cli.rst | 26 +++++++++++++++++++++++--- sqlite_utils/cli.py | 2 ++ tests/test_cli_insert.py | 13 +++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index f93fc57..5bf02d2 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -876,15 +876,35 @@ If your file does not include this row, you can use the ``--no-headers`` option If you do this, the table will be created with column names called ``untitled_1`` and ``untitled_2`` and so on. You can then rename them using the ``sqlite-utils transform ... --rename`` command, see :ref:`cli_transform_table`. -.. _cli_insert_lines: +.. _cli_insert_unstructured: -Inserting newline-delimited data -================================ +Inserting unstructured data with \-\-lines and \-\-all +====================================================== If you have an unstructured file you can insert its contents into a table with a single ``line`` column containing each line from the file using ``--lines``. This can be useful if you intend to further analyze those lines using SQL string functions or :ref:`sqlite-utils convert `:: $ sqlite-utils insert logs.db loglines logfile.log --lines +This will produce the following schema: + +.. code-block:: sql + + CREATE TABLE [loglines] ( + [line] TEXT + ); + +You can also insert the entire contents of the file into a single column called ``all`` using ``--all``:: + + $ sqlite-utils insert content.db content file.txt --all + +The schema here will be: + +.. code-block:: sql + + CREATE TABLE [content] ( + [all] TEXT + ); + .. _cli_insert_replace: Insert-replacing data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 6d3f8eb..a5b979f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -785,6 +785,8 @@ def insert_upsert_implementation( docs = tracker.wrap(docs) elif lines: docs = ({"line": line.strip()} for line in decoded) + elif all: + docs = ({"all": decoded.read()},) elif convert: fn = _compile_code(convert, imports) docs = (fn(line) for line in decoded) diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 2922a2f..063c43f 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -337,3 +337,16 @@ def test_insert_lines(db_path): {"line": "Second line"}, {"line": '{"foo": "baz"}'}, ] == list(db.query("select line from from_lines")) + + +def test_insert_all(db_path): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "from_all", "-", "--all"], + input='First line\nSecond line\n{"foo": "baz"}', + ) + assert 0 == result.exit_code, result.output + db = Database(db_path) + assert [{"all": 'First line\nSecond line\n{"foo": "baz"}'}] == list( + db.query("select [all] from from_all") + ) From 2e4847e493a03d95f827ddfaa698c052e3b231a8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Jan 2022 21:44:04 -0800 Subject: [PATCH 0485/1004] Implemented --convert for different things, renamed --all to --text --- docs/cli.rst | 10 +++--- sqlite_utils/cli.py | 41 +++++++++++++++-------- sqlite_utils/utils.py | 6 ++-- tests/test_cli_insert.py | 72 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 104 insertions(+), 25 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 5bf02d2..8ce2215 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -878,8 +878,8 @@ If you do this, the table will be created with column names called ``untitled_1` .. _cli_insert_unstructured: -Inserting unstructured data with \-\-lines and \-\-all -====================================================== +Inserting unstructured data with \-\-lines and \-\-text +======================================================= If you have an unstructured file you can insert its contents into a table with a single ``line`` column containing each line from the file using ``--lines``. This can be useful if you intend to further analyze those lines using SQL string functions or :ref:`sqlite-utils convert `:: @@ -893,16 +893,16 @@ This will produce the following schema: [line] TEXT ); -You can also insert the entire contents of the file into a single column called ``all`` using ``--all``:: +You can also insert the entire contents of the file into a single column called ``text`` using ``--text``:: - $ sqlite-utils insert content.db content file.txt --all + $ sqlite-utils insert content.db content file.txt --text The schema here will be: .. code-block:: sql CREATE TABLE [content] ( - [all] TEXT + [text] TEXT ); .. _cli_insert_replace: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index a5b979f..5b83dc5 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -657,7 +657,9 @@ def insert_upsert_options(fn): help="Treat each line as a single value called 'line'", ), click.option( - "--all", is_flag=True, help="Treat input as a single value called 'all'" + "--text", + is_flag=True, + help="Treat input as a single value called 'text'", ), click.option("--convert", help="Python code to convert each item"), click.option( @@ -723,7 +725,7 @@ def insert_upsert_implementation( csv, tsv, lines, - all, + text, convert, imports, delimiter, @@ -785,11 +787,8 @@ def insert_upsert_implementation( docs = tracker.wrap(docs) elif lines: docs = ({"line": line.strip()} for line in decoded) - elif all: - docs = ({"all": decoded.read()},) - elif convert: - fn = _compile_code(convert, imports) - docs = (fn(line) for line in decoded) + elif text: + docs = ({"text": decoded.read()},) else: try: if nl: @@ -805,6 +804,20 @@ def insert_upsert_implementation( if flatten: docs = (dict(_flatten(doc)) for doc in docs) + if convert: + variable = "row" + if lines: + variable = "line" + elif text: + variable = "text" + fn = _compile_code(convert, imports, variable=variable) + if lines: + docs = (fn(doc["line"]) for doc in docs) + elif text: + docs = (fn(doc["text"]) for doc in docs) + else: + docs = (fn(doc) for doc in docs) + extra_kwargs = {"ignore": ignore, "replace": replace, "truncate": truncate} if not_null: extra_kwargs["not_null"] = set(not_null) @@ -812,8 +825,10 @@ def insert_upsert_implementation( extra_kwargs["defaults"] = dict(default) if upsert: extra_kwargs["upsert"] = upsert + # Apply {"$base64": true, ...} decoding, if needed docs = (decode_base64_values(doc) for doc in docs) + try: db[table].insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs @@ -889,7 +904,7 @@ def insert( csv, tsv, lines, - all, + text, convert, imports, delimiter, @@ -918,7 +933,7 @@ def insert( - Use --nl for newline-delimited JSON objects - Use --csv or --tsv for comma-separated or tab-separated input - Use --lines to write each incoming line to a column called "line" - - Use --all to write the entire input to a column called "all" + - Use --text to write the entire input to a column called "text" You can also use --convert to pass a fragment of Python code that will be used to convert each input. @@ -927,7 +942,7 @@ def insert( imported row, and can return a modified row. If you are using --lines your code will be passed a "line" variable, - and for --all an "all" variable. + and for --text an "text" variable. """ try: insert_upsert_implementation( @@ -940,7 +955,7 @@ def insert( csv, tsv, lines, - all, + text, convert, imports, delimiter, @@ -976,7 +991,7 @@ def upsert( csv, tsv, lines, - all, + text, convert, imports, batch_size, @@ -1008,7 +1023,7 @@ def upsert( csv, tsv, lines, - all, + text, convert, imports, delimiter, diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index fe71b73..caec976 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -281,14 +281,14 @@ def progressbar(*args, **kwargs): yield bar -def _compile_code(code, imports): +def _compile_code(code, imports, variable="value"): locals = {} globals = {"r": recipes, "recipes": recipes} # If user defined a convert() function, return that try: exec(code, globals, locals) return locals["convert"] - except (SyntaxError, NameError, KeyError, TypeError): + except (AttributeError, SyntaxError, NameError, KeyError, TypeError): pass # Try compiling their code as a function instead @@ -297,7 +297,7 @@ def _compile_code(code, imports): if "\n" not in code and not code.strip().startswith("return "): code = "return {}".format(code) - new_code = ["def fn(value):"] + new_code = ["def fn({}):".format(variable)] for line in code.split("\n"): new_code.append(" {}".format(line)) code_o = compile("\n".join(new_code), "", "exec") diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 063c43f..bb9aa9b 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -339,14 +339,78 @@ def test_insert_lines(db_path): ] == list(db.query("select line from from_lines")) -def test_insert_all(db_path): +def test_insert_text(db_path): result = CliRunner().invoke( cli.cli, - ["insert", db_path, "from_all", "-", "--all"], + ["insert", db_path, "from_text", "-", "--text"], input='First line\nSecond line\n{"foo": "baz"}', ) assert 0 == result.exit_code, result.output db = Database(db_path) - assert [{"all": 'First line\nSecond line\n{"foo": "baz"}'}] == list( - db.query("select [all] from from_all") + assert [{"text": 'First line\nSecond line\n{"foo": "baz"}'}] == list( + db.query("select text from from_text") ) + + +@pytest.mark.parametrize( + "options,input", + ( + ([], '[{"id": "1", "name": "Bob"}, {"id": "2", "name": "Cat"}]'), + (["--csv"], "id,name\n1,Bob\n2,Cat"), + (["--nl"], '{"id": "1", "name": "Bob"}\n{"id": "2", "name": "Cat"}'), + ), +) +def test_insert_convert_json_csv_jsonnl(db_path, options, input): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "rows", "-", "--convert", '{**row, **{"extra": 1}}'] + + options, + input=input, + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + rows = list(db.query("select id, name, extra from rows")) + assert rows == [ + {"id": "1", "name": "Bob", "extra": 1}, + {"id": "2", "name": "Cat", "extra": 1}, + ] + + +def test_insert_convert_text(db_path): + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "text", + "-", + "--text", + "--convert", + '{"text": text.upper()}', + ], + input="This is text\nwill be upper now", + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + rows = list(db.query("select [text] from [text]")) + assert rows == [{"text": "THIS IS TEXT\nWILL BE UPPER NOW"}] + + +def test_insert_convert_lines(db_path): + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "all", + "-", + "--lines", + "--convert", + '{"line": line.upper()}', + ], + input="This is text\nwill be upper now", + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + rows = list(db.query("select [line] from [all]")) + assert rows == [{"line": "THIS IS TEXT"}, {"line": "WILL BE UPPER NOW"}] From 413f8ed754e38d7b190de888c85fe8438336cb11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Jan 2022 22:19:52 -0800 Subject: [PATCH 0486/1004] --convert --text for iterators, docs for --convert --- docs/cli.rst | 118 +++++++++++++++++++++++++++++++++++---- sqlite_utils/cli.py | 11 +++- tests/test_cli_insert.py | 20 +++++++ 3 files changed, 136 insertions(+), 13 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 8ce2215..6d687ec 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -905,6 +905,95 @@ The schema here will be: [text] TEXT ); +.. _cli_insert_convert: + +Applying conversions while inserting data +========================================= + +The ``--convert`` option can be used to apply a Python conversion function to imported data before it is inserted into the database. It works in a similar way to :ref:`sqlite-utils convert `. + +Your Python function will be passed a dictionary called ``row`` for each item that is being imported. You can modify that dictionary and return it - or return a fresh dictionary - to change the data that will be inserted. + +Given a JSON file called ``dogs.json`` containing this: + +.. code-block:: json + + [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Pancakes"} + ] + +The following command will insert that data and add an ``is_good`` column set to ``1`` for each dog:: + + $ sqlite-utils insert dogs.db dogs dogs.json --convert ' + row["is_good"] = 1 + return row' + +The ``--convert`` option also works with the ``--csv``, ``--tsv`` and ``--nl`` insert options. + +As with ``sqlite-utils convert`` you can use ``--import`` to import additional Python modules, see :ref:`cli_convert_import` for details. + +.. _cli_insert_convert_lines: + +\-\-convert with \-\-lines +-------------------------- + +Things work slightly differently when combined with the ``--lines`` or ``--text`` options. + +With ``--lines``, instead of being passed a ``row`` dictionary your function will be passed a ``line`` string representing each line of the input. Given a file called ``access.log`` containing the following:: + + INFO: 127.0.0.1:60581 - GET / HTTP/1.1 200 OK + INFO: 127.0.0.1:60581 - GET /foo/-/static/app.css?cead5a HTTP/1.1 200 OK + +You could convert it into structured data like so:: + + $ sqlite-utils insert logs.db loglines access.log --convert ' + type, ip, _, verb, path, _, status, _ = line.split() + return { + "type": type, + "ip": ip, + "verb": verb, + "path": path, + "status": status, + }' --lines + +The resulting table would look like this: + +====== =============== ====== ============================ ======== +type ip verb path status +====== =============== ====== ============================ ======== +INFO: 127.0.0.1:60581 GET / 200 +INFO: 127.0.0.1:60581 GET /foo/-/static/app.css?cead5a 200 +====== =============== ====== ============================ ======== + +.. _cli_insert_convert_text: + +\-\-convert with \-\-text +------------------------- + +With ``--text`` the entire input to the command will be made available to the function as a variable called ``text``. + +The function can return a single dictionary which will be inserted as a single row, or it can return a list or iterator of dictionaries, each of which will be inserted. + +Here's how to use ``--convert`` and ``--text`` to insert one record per word in the input:: + + $ echo 'A bunch of words' | sqlite-utils insert words.db words - \ + --text --convert '({"word": w} for w in text.split())' + +The result looks like this:: + + $ sqlite-utils dump words.db + BEGIN TRANSACTION; + CREATE TABLE [words] ( + [word] TEXT + ); + INSERT INTO "words" VALUES('A'); + INSERT INTO "words" VALUES('bunch'); + INSERT INTO "words" VALUES('of'); + INSERT INTO "words" VALUES('words'); + COMMIT; + + .. _cli_insert_replace: Insert-replacing data @@ -1046,18 +1135,6 @@ The code you provide will be compiled into a function that takes ``value`` as a value = str(value) return value.upper()' -You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options. This example uses the ``textwrap`` module to wrap the ``content`` column at 100 characters:: - - $ sqlite-utils convert content.db articles content \ - '"\n".join(textwrap.wrap(value, 100))' \ - --import=textwrap - -This supports nested imports as well, for example to use `ElementTree `__:: - - $ sqlite-utils convert content.db articles content \ - 'xml.etree.ElementTree.fromstring(value).attrib["title"]' \ - --import=xml.etree.ElementTree - Your code will be automatically wrapped in a function, but you can also define a function called ``convert(value)`` which will be called, if available:: $ sqlite-utils convert content.db articles headline ' @@ -1088,6 +1165,23 @@ You can include named parameters in your where clause and populate them using on The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. +.. _cli_convert_import: + +Importing additional modules +---------------------------- + +You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options. This example uses the ``textwrap`` module to wrap the ``content`` column at 100 characters:: + + $ sqlite-utils convert content.db articles content \ + '"\n".join(textwrap.wrap(value, 100))' \ + --import=textwrap + +This supports nested imports as well, for example to use `ElementTree `__:: + + $ sqlite-utils convert content.db articles content \ + 'xml.etree.ElementTree.fromstring(value).attrib["title"]' \ + --import=xml.etree.ElementTree + .. _cli_convert_recipes: sqlite-utils convert recipes diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 5b83dc5..ab27761 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -814,7 +814,16 @@ def insert_upsert_implementation( if lines: docs = (fn(doc["line"]) for doc in docs) elif text: - docs = (fn(doc["text"]) for doc in docs) + # Special case: this is allowed to be an iterable + text_value = list(docs)[0]["text"] + fn_return = fn(text_value) + if isinstance(fn_return, dict): + docs = [fn_return] + else: + try: + docs = iter(fn_return) + except TypeError: + raise click.ClickException("--convert must return dict or iterator") else: docs = (fn(doc) for doc in docs) diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index bb9aa9b..085f322 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -396,6 +396,26 @@ def test_insert_convert_text(db_path): assert rows == [{"text": "THIS IS TEXT\nWILL BE UPPER NOW"}] +def test_insert_convert_text_returning_iterator(db_path): + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "text", + "-", + "--text", + "--convert", + '({"word": w} for w in text.split())', + ], + input="A bunch of words", + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + rows = list(db.query("select [word] from [text]")) + assert rows == [{"word": "A"}, {"word": "bunch"}, {"word": "of"}, {"word": "words"}] + + def test_insert_convert_lines(db_path): result = CliRunner().invoke( cli.cli, From a7b29bfaa99c34dc40414b4ad12bff9b78d70427 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Jan 2022 22:28:29 -0800 Subject: [PATCH 0487/1004] Fixed bug with sqlite-utils upsert --detect-types, closes #362 --- sqlite_utils/cli.py | 1 + tests/test_cli.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index ab27761..624f47e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1045,6 +1045,7 @@ def upsert( not_null=not_null, default=default, encoding=encoding, + detect_types=detect_types, load_extension=load_extension, silent=silent, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index c4c50a9..b789b1e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1974,6 +1974,24 @@ def test_insert_detect_types(tmpdir, option_or_env_var): _test() +@pytest.mark.parametrize("option", ("-d", "--detect-types")) +def test_upsert_detect_types(tmpdir, option): + db_path = str(tmpdir / "test.db") + data = "id,name,age,weight\n1,Cleo,6,45.5\n2,Dori,1,3.5" + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "creatures", "-", "--csv", "--pk", "id"] + [option], + catch_exceptions=False, + input=data, + ) + assert result.exit_code == 0 + db = Database(db_path) + assert list(db["creatures"].rows) == [ + {"id": 1, "name": "Cleo", "age": 6, "weight": 45.5}, + {"id": 2, "name": "Dori", "age": 1, "weight": 3.5}, + ] + + @pytest.mark.parametrize( "input,expected", ( From 3d464893ee442c179a8e5015ffd7577f34f01adc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Jan 2022 22:55:35 -0800 Subject: [PATCH 0488/1004] Release 3.20 Refs #344, #353, #356, #362 --- docs/changelog.rst | 14 ++++++++++++++ setup.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a30e7cf..f14870b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,20 @@ Changelog =========== +.. _v3_20: + +3.20 (2022-01-05) +----------------- + +- ``sqlite-utils insert ... --lines`` to insert the lines from a file into a table with a single ``line`` column, see :ref:`cli_insert_unstructured`. +- ``sqlite-utils insert ... --text`` to insert the contents of the file into a table with a single ``text`` column and a single row. +- ``sqlite-utils insert ... --convert`` allows a Python function to be provided that will be used to convert each row that is being inserted into the database. See :ref:`cli_insert_convert`, including details on special behavior when combined with ``--lines`` and ``--text``. (:issue:`356`) +- ``sqlite-utils convert`` now accepts a code value of ``-`` to read code from standard input. (:issue:`353`) +- ``sqlite-utils convert`` also now accepts code that defines a named ``convert(value)`` function, see :ref:`cli_convert`. +- ``db.supports_strict`` property showing if the database connection supports `SQLite strict tables `__. +- ``table.strict`` property (see :ref:`python_api_introspection_strict`) indicating if the table uses strict mode. (:issue:`344`) +- Fixed bug where ``sqlite-utils upsert ... --detect-types`` ignored the ``--detect-types`` option. (:issue:`362`) + .. _v3_19: 3.19 (2021-11-20) diff --git a/setup.py b/setup.py index 3cc7b0e..1c05f3e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.19" +VERSION = "3.20" def get_long_description(): From 6e46b9913411682f3a3ec66f4d58886c1db8654b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 6 Jan 2022 10:01:35 -0800 Subject: [PATCH 0489/1004] Renamed ip to source in example code --- docs/cli.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 6d687ec..0fe6669 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -948,10 +948,10 @@ With ``--lines``, instead of being passed a ``row`` dictionary your function wil You could convert it into structured data like so:: $ sqlite-utils insert logs.db loglines access.log --convert ' - type, ip, _, verb, path, _, status, _ = line.split() + type, source, _, verb, path, _, status, _ = line.split() return { "type": type, - "ip": ip, + "source": source, "verb": verb, "path": path, "status": status, @@ -960,7 +960,7 @@ You could convert it into structured data like so:: The resulting table would look like this: ====== =============== ====== ============================ ======== -type ip verb path status +type source verb path status ====== =============== ====== ============================ ======== INFO: 127.0.0.1:60581 GET / 200 INFO: 127.0.0.1:60581 GET /foo/-/static/app.css?cead5a 200 From a8f9cc6f64f299830834428509940d448b82b4ed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jan 2022 13:16:34 -0800 Subject: [PATCH 0490/1004] Add test for chunks(), refs #364 --- sqlite_utils/db.py | 7 +------ sqlite_utils/utils.py | 7 +++++++ tests/test_utils.py | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0ff056c..dfc4723 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,4 +1,5 @@ from .utils import ( + chunks, sqlite3, OperationalError, suggest_column_types, @@ -2995,12 +2996,6 @@ class View(Queryable): ) -def chunks(sequence, size): - iterator = iter(sequence) - for item in iterator: - yield itertools.chain([item], itertools.islice(iterator, size - 1)) - - def jsonify_if_needed(value): if isinstance(value, decimal.Decimal): return float(value) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index caec976..3b4769a 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -3,6 +3,7 @@ import contextlib import csv import enum import io +import itertools import json import os from . import recipes @@ -306,3 +307,9 @@ def _compile_code(code, imports, variable="value"): globals[import_.split(".")[0]] = __import__(import_) exec(code_o, globals, locals) return locals["fn"] + + +def chunks(sequence, size): + iterator = iter(sequence) + for item in iterator: + yield itertools.chain([item], itertools.islice(iterator, size - 1)) diff --git a/tests/test_utils.py b/tests/test_utils.py index a44b2e0..8630e28 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -25,3 +25,18 @@ def test_decode_base64_values(input, expected, should_be_is): def test_find_spatialite(): spatialite = utils.find_spatialite() assert spatialite is None or isinstance(spatialite, str) + + +@pytest.mark.parametrize( + "size,expected", + ( + (1, [["a"], ["b"], ["c"], ["d"]]), + (2, [["a", "b"], ["c", "d"]]), + (3, [["a", "b", "c"], ["d"]]), + (4, [["a", "b", "c", "d"]]), + ), +) +def test_chunks(size, expected): + input = ["a", "b", "c", "d"] + chunks = list(map(list, utils.chunks(input, size))) + assert chunks == expected From 539f5ccd90371fa87f946018f8b77d55929e06db Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jan 2022 18:33:00 -0800 Subject: [PATCH 0491/1004] Support 'python -m sqlite_utils', closes #368 Refs #364 --- docs/cli.rst | 2 ++ sqlite_utils/__main__.py | 4 ++++ tests/test_cli.py | 11 +++++++++++ 3 files changed, 17 insertions(+) create mode 100644 sqlite_utils/__main__.py diff --git a/docs/cli.rst b/docs/cli.rst index 0fe6669..e9d2218 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -6,6 +6,8 @@ The ``sqlite-utils`` command-line tool can be used to manipulate SQLite databases in a number of different ways. +Once installed, the tool should be available as ``sqlite-utils``. It can also be run using ``python -m sqlite_utils``. + .. contents:: :local: .. _cli_query: diff --git a/sqlite_utils/__main__.py b/sqlite_utils/__main__.py new file mode 100644 index 0000000..98dcca0 --- /dev/null +++ b/sqlite_utils/__main__.py @@ -0,0 +1,4 @@ +from .cli import cli + +if __name__ == "__main__": + cli() diff --git a/tests/test_cli.py b/tests/test_cli.py index b789b1e..336ce7b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,8 @@ from sqlite_utils import cli, Database from sqlite_utils.db import Index, ForeignKey from click.testing import CliRunner +import subprocess +import sys from unittest import mock import json import os @@ -2017,3 +2019,12 @@ def test_integer_overflow_error(tmpdir): "sql = INSERT INTO [items] ([bignumber]) VALUES (?);\n" "parameters = [34223049823094832094802398430298048240]\n" ) + + +def test_python_dash_m(): + "Tool can be run using python -m sqlite_utils" + result = subprocess.run( + [sys.executable, "-m", "sqlite_utils", "--help"], capture_output=True + ) + assert result.returncode == 0 + assert b"Commands for interacting with a SQLite database" in result.stdout From e0c476bc380744680c8b7675c24fb0e9f5ec6dcd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jan 2022 18:37:53 -0800 Subject: [PATCH 0492/1004] Fix test for Python 3.6, refs #368 --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 336ce7b..7afe9ca 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2024,7 +2024,7 @@ def test_integer_overflow_error(tmpdir): def test_python_dash_m(): "Tool can be run using python -m sqlite_utils" result = subprocess.run( - [sys.executable, "-m", "sqlite_utils", "--help"], capture_output=True + [sys.executable, "-m", "sqlite_utils", "--help"], stdout=subprocess.PIPE ) assert result.returncode == 0 assert b"Commands for interacting with a SQLite database" in result.stdout From 148e9c7aeea2486b0562814b82f152506bfb0dd5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 09:48:48 -0800 Subject: [PATCH 0493/1004] Use cog to maintain --fmt list, closes #373 --- .github/workflows/test.yml | 3 +++ docs/cli.rst | 36 +++++++++++++++++++++++++++++++++++- docs/contributing.rst | 4 ++++ setup.py | 2 +- 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fec8cfa..1f92538 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,3 +41,6 @@ jobs: run: flake8 - name: Check formatting run: black . --check + - name: Check if cog needs to be run + run: | + cog --check README.md docs/*.rst diff --git a/docs/cli.rst b/docs/cli.rst index e9d2218..497e00e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -176,7 +176,41 @@ You can use the ``--fmt`` option to specify different table formats, for example 2 2 Pancakes ==== ===== ======== -For a full list of table format options, run ``sqlite-utils query --help``. +Available ``--fmt`` options are: + +.. [[[cog + import tabulate + cog.out("\n" + "\n".join('- ``{}``'.format(t) for t in tabulate.tabulate_formats) + "\n\n") +.. ]]] + +- ``fancy_grid`` +- ``fancy_outline`` +- ``github`` +- ``grid`` +- ``html`` +- ``jira`` +- ``latex`` +- ``latex_booktabs`` +- ``latex_longtable`` +- ``latex_raw`` +- ``mediawiki`` +- ``moinmoin`` +- ``orgtbl`` +- ``pipe`` +- ``plain`` +- ``presto`` +- ``pretty`` +- ``psql`` +- ``rst`` +- ``simple`` +- ``textile`` +- ``tsv`` +- ``unsafehtml`` +- ``youtrack`` + +.. [[[end]]] + +This list can also be found by running ``sqlite-utils query --help``. .. _cli_query_raw: diff --git a/docs/contributing.rst b/docs/contributing.rst index ec1a4a7..c2d20ba 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -44,6 +44,10 @@ Then run ``make livehtml`` from the ``docs/`` directory to start a server on por cd docs make livehtml +The `cog `__ tool is used to maintain portions of the documentation. You can run it like so:: + + cog -r docs/*.rst + .. _contributing_linting: Linting and formatting diff --git a/setup.py b/setup.py index 1c05f3e..901725d 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ setup( "python-dateutil", ], extras_require={ - "test": ["pytest", "black", "hypothesis"], + "test": ["pytest", "black", "hypothesis", "cogapp"], "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], "mypy": [ "mypy", From b8c134059e89f0fa040b84fb7d0bda25b9a52759 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 10:07:48 -0800 Subject: [PATCH 0494/1004] --fmt now implies --table, closes #374 --- docs/cli.rst | 6 ++---- sqlite_utils/cli.py | 13 ++++++++----- tests/test_cli.py | 21 ++++++++++++++++----- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 497e00e..f56e382 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -168,7 +168,7 @@ You can use the ``--table`` option (or ``-t`` shortcut) to output query results You can use the ``--fmt`` option to specify different table formats, for example ``rst`` for reStructuredText:: - $ sqlite-utils dogs.db "select * from dogs" --table --fmt rst + $ sqlite-utils dogs.db "select * from dogs" --fmt rst ==== ===== ======== id age name ==== ===== ======== @@ -182,7 +182,6 @@ Available ``--fmt`` options are: import tabulate cog.out("\n" + "\n".join('- ``{}``'.format(t) for t in tabulate.tabulate_formats) + "\n\n") .. ]]] - - ``fancy_grid`` - ``fancy_outline`` - ``github`` @@ -207,7 +206,6 @@ Available ``--fmt`` options are: - ``tsv`` - ``unsafehtml`` - ``youtrack`` - .. [[[end]]] This list can also be found by running ``sqlite-utils query --help``. @@ -289,7 +287,7 @@ It takes all of the same output formatting options as :ref:`sqlite-utils query < $ sqlite-utils memory 'select sqlite_version()' --csv sqlite_version() 3.35.5 - $ sqlite-utils memory 'select sqlite_version()' --table --fmt grid + $ sqlite-utils memory 'select sqlite_version()' --fmt grid +--------------------+ | sqlite_version() | +====================+ diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 624f47e..2bd0278 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -81,7 +81,6 @@ def output_options(fn): help="Table format - one of {}".format( ", ".join(tabulate.tabulate_formats) ), - default="simple", ), click.option( "--json-cols", @@ -192,8 +191,8 @@ def tables( row.append(db[name].schema) yield row - if table: - print(tabulate.tabulate(_iter(), headers=headers, tablefmt=fmt)) + if table or fmt: + print(tabulate.tabulate(_iter(), headers=headers, tablefmt=fmt or "simple")) elif csv or tsv: writer = csv_std.writer(sys.stdout, dialect="excel-tab" if tsv else "excel") if not no_headers: @@ -1456,8 +1455,12 @@ def _execute_query( sys.stdout.buffer.write(data) else: sys.stdout.write(str(data)) - elif table: - print(tabulate.tabulate(list(cursor), headers=headers, tablefmt=fmt)) + elif fmt or table: + print( + tabulate.tabulate( + list(cursor), headers=headers, tablefmt=fmt or "simple" + ) + ) elif csv or tsv: writer = csv_std.writer(sys.stdout, dialect="excel-tab" if tsv else "excel") if not no_headers: diff --git a/tests/test_cli.py b/tests/test_cli.py index 7afe9ca..5a9ca65 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -114,10 +114,10 @@ def test_tables_schema(db_path): @pytest.mark.parametrize( - "fmt,expected", + "options,expected", [ ( - "simple", + ["--fmt", "simple"], ( "c1 c2 c3\n" "----- ----- ----------\n" @@ -128,7 +128,18 @@ def test_tables_schema(db_path): ), ), ( - "rst", + ["-t"], + ( + "c1 c2 c3\n" + "----- ----- ----------\n" + "verb0 noun0 adjective0\n" + "verb1 noun1 adjective1\n" + "verb2 noun2 adjective2\n" + "verb3 noun3 adjective3" + ), + ), + ( + ["--fmt", "rst"], ( "===== ===== ==========\n" "c1 c2 c3\n" @@ -142,7 +153,7 @@ def test_tables_schema(db_path): ), ], ) -def test_output_table(db_path, fmt, expected): +def test_output_table(db_path, options, expected): db = Database(db_path) with db.conn: db["rows"].insert_all( @@ -155,7 +166,7 @@ def test_output_table(db_path, fmt, expected): for i in range(4) ] ) - result = CliRunner().invoke(cli.cli, ["rows", db_path, "rows", "-t", "--fmt", fmt]) + result = CliRunner().invoke(cli.cli, ["rows", db_path, "rows"] + options) assert 0 == result.exit_code assert expected == result.output.strip() From 22c8d10dd343476e8b7b9af3366fae4c8353dd2c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 12:06:02 -0800 Subject: [PATCH 0495/1004] --convert function can now modify row in place, closes #371 --- docs/cli.rst | 4 +--- sqlite_utils/cli.py | 8 ++------ sqlite_utils/utils.py | 24 +++++++++++++++++------- tests/test_cli_convert.py | 32 ++++++++++---------------------- tests/test_cli_insert.py | 19 +++++++++++++++++++ 5 files changed, 49 insertions(+), 38 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index f56e382..233a4c0 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -959,9 +959,7 @@ Given a JSON file called ``dogs.json`` containing this: The following command will insert that data and add an ``is_good`` column set to ``1`` for each dog:: - $ sqlite-utils insert dogs.db dogs dogs.json --convert ' - row["is_good"] = 1 - return row' + $ sqlite-utils insert dogs.db dogs dogs.json --convert 'row["is_good"] = 1` The ``--convert`` option also works with the ``--csv``, ``--tsv`` and ``--nl`` insert options. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 2bd0278..212a039 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -824,7 +824,7 @@ def insert_upsert_implementation( except TypeError: raise click.ClickException("--convert must return dict or iterator") else: - docs = (fn(doc) for doc in docs) + docs = (fn(doc) or doc for doc in docs) extra_kwargs = {"ignore": ignore, "replace": replace, "truncate": truncate} if not_null: @@ -2225,11 +2225,7 @@ def convert( try: fn = _compile_code(code, imports) except SyntaxError as e: - raise click.ClickException( - "Syntax error in code:\n\n{}\n\n{}".format( - textwrap.indent(e.text.strip(), " "), e.msg - ) - ) + raise click.ClickException(str(e)) if dry_run: # Pull first 20 values for first column and preview them db.conn.create_function("preview_transform", 1, lambda v: fn(v) if v else v) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 3b4769a..0777f30 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -293,15 +293,25 @@ def _compile_code(code, imports, variable="value"): pass # Try compiling their code as a function instead - - # If single line and no 'return', add the return + body_variants = [code] + # If single line and no 'return', try adding the return if "\n" not in code and not code.strip().startswith("return "): - code = "return {}".format(code) + body_variants.insert(0, "return {}".format(code)) - new_code = ["def fn({}):".format(variable)] - for line in code.split("\n"): - new_code.append(" {}".format(line)) - code_o = compile("\n".join(new_code), "", "exec") + code_o = None + for variant in body_variants: + new_code = ["def fn({}):".format(variable)] + for line in variant.split("\n"): + new_code.append(" {}".format(line)) + try: + code_o = compile("\n".join(new_code), "", "exec") + break + except SyntaxError: + # Try another variant, e.g. for 'return row["column"] = 1' + continue + + if code_o is None: + raise SyntaxError("Could not compile code") for import_ in imports: globals[import_.split(".")[0]] = __import__(import_) diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 481bebd..ec8110f 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -55,32 +55,20 @@ def test_convert_code(fresh_db_and_path, code): @pytest.mark.parametrize( - "bad_code,expected_error", - [ - ( - "def foo(value)", - """Error: Syntax error in code: - - return def foo(value) - -invalid syntax""", - ), - ( - "$", - """Error: Syntax error in code: - - return $ - -invalid syntax""", - ), - ], + "bad_code", + ( + "def foo(value)", + "$", + ), ) -def test_convert_code_errors(fresh_db_and_path, bad_code, expected_error): +def test_convert_code_errors(fresh_db_and_path, bad_code): db, db_path = fresh_db_and_path db["t"].insert({"text": "October"}) - result = CliRunner().invoke(cli.cli, ["convert", db_path, "t", "text", bad_code]) + result = CliRunner().invoke( + cli.cli, ["convert", db_path, "t", "text", bad_code], catch_exceptions=False + ) assert 1 == result.exit_code - assert result.output.strip() == expected_error.strip() + assert result.output == "Error: Could not compile code\n" def test_convert_import(test_db_and_path): diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 085f322..9c218cc 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -434,3 +434,22 @@ def test_insert_convert_lines(db_path): db = Database(db_path) rows = list(db.query("select [line] from [all]")) assert rows == [{"line": "THIS IS TEXT"}, {"line": "WILL BE UPPER NOW"}] + + +def test_insert_convert_row_modifying_in_place(db_path): + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "rows", + "-", + "--convert", + 'row["is_chicken"] = True', + ], + input='{"name": "Azi"}', + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + rows = list(db.query("select name, is_chicken from rows")) + assert rows == [{"name": "Azi", "is_chicken": 1}] From 49a54ffb2fb6d3b73522c96c2bf9fc722e99d036 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 12:08:03 -0800 Subject: [PATCH 0496/1004] Fix for cog error Should help tests pass for #374, #371 --- docs/cli.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 233a4c0..0346d99 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -182,6 +182,7 @@ Available ``--fmt`` options are: import tabulate cog.out("\n" + "\n".join('- ``{}``'.format(t) for t in tabulate.tabulate_formats) + "\n\n") .. ]]] + - ``fancy_grid`` - ``fancy_outline`` - ``github`` @@ -206,6 +207,7 @@ Available ``--fmt`` options are: - ``tsv`` - ``unsafehtml`` - ``youtrack`` + .. [[[end]]] This list can also be found by running ``sqlite-utils query --help``. From c9ecd0d6a32d4518c9b92bcc08183a10268d52d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 12:16:52 -0800 Subject: [PATCH 0497/1004] stem and suffix columns for insert-files, closes #372 --- docs/cli.rst | 4 ++++ sqlite_utils/cli.py | 2 ++ tests/test_insert_files.py | 18 ++++++++++++++---- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 0346d99..b92277b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1141,6 +1141,10 @@ The full list of column definitions you can use is as follows: The creation time is an ISO timestamp ``size`` The integer size of the file in bytes +``stem`` + The filename without the extension - for ``file.txt.gz`` this would be ``file.txt`` +``extension`` + The file extension - for ``file.txt.gz`` this would be ``.gz`` You can insert data piped from standard input like this:: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 212a039..1bc797e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2300,6 +2300,8 @@ FILE_COLUMNS = { "mtime_iso": lambda p: datetime.utcfromtimestamp(p.stat().st_mtime).isoformat(), "ctime_iso": lambda p: datetime.utcfromtimestamp(p.stat().st_ctime).isoformat(), "size": lambda p: p.stat().st_size, + "stem": lambda p: p.stem, + "suffix": lambda p: p.suffix, } diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 86ee4e3..d7af28e 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -15,7 +15,7 @@ def test_insert_files(silent): (tmpdir / "one.txt").write_text("This is file one", "utf-8") (tmpdir / "two.txt").write_text("Two is shorter", "utf-8") (tmpdir / "nested").mkdir() - (tmpdir / "nested" / "three.txt").write_text("Three is nested", "utf-8") + (tmpdir / "nested" / "three.zz.txt").write_text("Three is nested", "utf-8") coltypes = ( "name", "path", @@ -32,6 +32,8 @@ def test_insert_files(silent): "mtime_iso", "ctime_iso", "size", + "suffix", + "stem", ) cols = [] for coltype in coltypes: @@ -50,7 +52,7 @@ def test_insert_files(silent): one, two, three = ( rows_by_path["one.txt"], rows_by_path["two.txt"], - rows_by_path[os.path.join("nested", "three.txt")], + rows_by_path[os.path.join("nested", "three.zz.txt")], ) assert { "content": b"This is file one", @@ -60,6 +62,8 @@ def test_insert_files(silent): "path": "one.txt", "sha256": "e34138f26b5f7368f298b4e736fea0aad87ddec69fbd04dc183b20f4d844bad5", "size": 16, + "stem": "one", + "suffix": ".txt", }.items() <= one.items() assert { "content": b"Two is shorter", @@ -69,15 +73,19 @@ def test_insert_files(silent): "path": "two.txt", "sha256": "9368988ed16d4a2da0af9db9b686d385b942cb3ffd4e013f43aed2ec041183d9", "size": 14, + "stem": "two", + "suffix": ".txt", }.items() <= two.items() assert { "content": b"Three is nested", "content_text": "Three is nested", "md5": "12580f341781f5a5b589164d3cd39523", - "name": "three.txt", - "path": os.path.join("nested", "three.txt"), + "name": "three.zz.txt", + "path": os.path.join("nested", "three.zz.txt"), "sha256": "6dd45aaaaa6b9f96af19363a92c8fca5d34791d3c35c44eb19468a6a862cc8cd", "size": 15, + "stem": "three.zz", + "suffix": ".txt", }.items() <= three.items() # Assert the other int/str/float columns exist and are of the right types expected_types = { @@ -91,6 +99,8 @@ def test_insert_files(silent): "fullpath": str, "content": bytes, "content_text": str, + "stem": str, + "suffix": str, } for colname, expected_type in expected_types.items(): for row in (one, two, three): From f08fe6fd4d5df4fe1e638118707c98e1add80caf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 12:18:31 -0800 Subject: [PATCH 0498/1004] Fixed error in docs: it's suffix not extension, refs #372 --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index b92277b..e07c73d 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1143,7 +1143,7 @@ The full list of column definitions you can use is as follows: The integer size of the file in bytes ``stem`` The filename without the extension - for ``file.txt.gz`` this would be ``file.txt`` -``extension`` +``suffix`` The file extension - for ``file.txt.gz`` this would be ``.gz`` You can insert data piped from standard input like this:: From 1d64cd2e5b402ff957f9be2d9bb490d313c73989 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 12:33:16 -0800 Subject: [PATCH 0499/1004] sqlite-utils create-database command, closes #348 --- docs/cli.rst | 13 +++++++++++++ sqlite_utils/cli.py | 17 +++++++++++++++++ tests/test_cli.py | 18 ++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index e07c73d..36ff029 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -699,6 +699,19 @@ The ``most_common`` and ``least_common`` columns will contain nested JSON arrays ["Condarco", 4288] ] +.. _cli_create_database: + +Creating an empty database +========================== + +You can create a new empty database file using the ``create-database`` command:: + + $ sqlite-utils create-database empty.db + +To enable :ref:`cli_wal` on the newly created database add the ``--enable-wal`` option:: + + $ sqlite-utils create-database empty.db --enable-wal + .. _cli_inserting_data: Inserting JSON data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 1bc797e..315b162 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1052,6 +1052,23 @@ def upsert( raise click.ClickException(UNICODE_ERROR.format(ex)) +@cli.command(name="create-database") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.option( + "--enable-wal", is_flag=True, help="Enable WAL mode on the created database" +) +def create_table(path, enable_wal): + "Create a new empty database file." + db = sqlite_utils.Database(path) + if enable_wal: + db.enable_wal() + db.vacuum() + + @cli.command(name="create-table") @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index 5a9ca65..ca5c820 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2039,3 +2039,21 @@ def test_python_dash_m(): ) assert result.returncode == 0 assert b"Commands for interacting with a SQLite database" in result.stdout + + +@pytest.mark.parametrize("enable_wal", (False, True)) +def test_create_database(tmpdir, enable_wal): + db_path = tmpdir / "test.db" + assert not db_path.exists() + args = ["create-database", str(db_path)] + if enable_wal: + args.append("--enable-wal") + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 0, result.output + assert db_path.exists() + assert db_path.read_binary()[:16] == b"SQLite format 3\x00" + db = Database(str(db_path)) + if enable_wal: + assert db.journal_mode == "wal" + else: + assert db.journal_mode == "delete" From 2f8879235afc6a06a8ae25ded1b2fe289ad8c3a6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 12:39:14 -0800 Subject: [PATCH 0500/1004] Renamed function to fix lint error, refs #348 --- sqlite_utils/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 315b162..1e31088 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1061,7 +1061,7 @@ def upsert( @click.option( "--enable-wal", is_flag=True, help="Enable WAL mode on the created database" ) -def create_table(path, enable_wal): +def create_database(path, enable_wal): "Create a new empty database file." db = sqlite_utils.Database(path) if enable_wal: From d2a79d200f9071a86027365fa2a576865b71064f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 20:12:39 -0800 Subject: [PATCH 0501/1004] --nl now ignores blank lines, closes #376 --- sqlite_utils/cli.py | 2 +- tests/test_cli_insert.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 1e31088..b235068 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -791,7 +791,7 @@ def insert_upsert_implementation( else: try: if nl: - docs = (json.loads(line) for line in decoded) + docs = (json.loads(line) for line in decoded if line.strip()) else: docs = json.load(decoded) if isinstance(docs, dict): diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 9c218cc..c40dd85 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -165,7 +165,7 @@ def test_insert_newline_delimited(db_path): result = CliRunner().invoke( cli.cli, ["insert", db_path, "from_json_nl", "-", "--nl"], - input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', + input='{"foo": "bar", "n": 1}\n\n{"foo": "baz", "n": 2}', ) assert 0 == result.exit_code, result.output db = Database(db_path) From cfb3f1235848d000ba8609bf84e634bf56ac8291 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 20:39:58 -0800 Subject: [PATCH 0502/1004] Only buffer input if --sniff, closes #364 --- sqlite_utils/cli.py | 13 ++++++++++--- tests/test_cli_insert.py | 41 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index b235068..05a7bcc 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -757,13 +757,20 @@ def insert_upsert_implementation( if pk and len(pk) == 1: pk = pk[0] encoding = encoding or "utf-8-sig" - buffered = io.BufferedReader(file, buffer_size=4096) - decoded = io.TextIOWrapper(buffered, encoding=encoding) + + # The --sniff option needs us to buffer the file to peek ahead + sniff_buffer = None + if sniff: + sniff_buffer = io.BufferedReader(file, buffer_size=4096) + decoded = io.TextIOWrapper(sniff_buffer, encoding=encoding) + else: + decoded = io.TextIOWrapper(file, encoding=encoding) + tracker = None if csv or tsv: if sniff: # Read first 2048 bytes and use that to detect - first_bytes = buffered.peek(2048) + first_bytes = sniff_buffer.peek(2048) dialect = csv_std.Sniffer().sniff(first_bytes.decode(encoding, "ignore")) else: dialect = "excel-tab" if tsv else "excel" diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index c40dd85..675e9eb 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -2,6 +2,9 @@ from sqlite_utils import cli, Database from click.testing import CliRunner import json import pytest +import subprocess +import sys +import time def test_insert_simple(tmpdir): @@ -453,3 +456,41 @@ def test_insert_convert_row_modifying_in_place(db_path): db = Database(db_path) rows = list(db.query("select name, is_chicken from rows")) assert rows == [{"name": "Azi", "is_chicken": 1}] + + +def test_insert_streaming_batch_size_1(db_path): + # https://github.com/simonw/sqlite-utils/issues/364 + # Streaming with --batch-size 1 should commit on each record + # Can't use CliRunner().invoke() here bacuse we need to + # run assertions in between writing to process stdin + # First, create the DB with WAL mode enabled + CliRunner().invoke(cli.cli, ["create-database", db_path, "--enable-wal"]) + # Now start streaming rows to insert --nl + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "sqlite_utils", + "insert", + db_path, + "rows", + "-", + "--nl", + "--batch-size", + "1", + ], + stdin=subprocess.PIPE, + stdout=sys.stdout, + ) + proc.stdin.write(b'{"name": "Azi"}\n') + proc.stdin.flush() + # Without this delay the data wasn't yet visible + time.sleep(0.2) + assert list(Database(db_path)["rows"].rows) == [{"name": "Azi"}] + proc.stdin.write(b'{"name": "Suna"}\n') + proc.stdin.flush() + time.sleep(0.2) + assert list(Database(db_path)["rows"].rows) == [{"name": "Azi"}, {"name": "Suna"}] + proc.stdin.close() + proc.wait() + assert proc.returncode == 0 From e6ae643497803e51379f82881f4df2c734ef97f3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 20:41:00 -0800 Subject: [PATCH 0503/1004] Did not need WAL after all, refs #364 --- tests/test_cli_insert.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 675e9eb..3ef6ffe 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -463,9 +463,6 @@ def test_insert_streaming_batch_size_1(db_path): # Streaming with --batch-size 1 should commit on each record # Can't use CliRunner().invoke() here bacuse we need to # run assertions in between writing to process stdin - # First, create the DB with WAL mode enabled - CliRunner().invoke(cli.cli, ["create-database", db_path, "--enable-wal"]) - # Now start streaming rows to insert --nl proc = subprocess.Popen( [ sys.executable, From 046e5246c9698a6fc9901ca265ae47c68fcf5d13 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 20:51:07 -0800 Subject: [PATCH 0504/1004] Longer delay to hopefully get test to pass, refs #364 --- tests/test_cli_insert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 3ef6ffe..346bb38 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -482,11 +482,11 @@ def test_insert_streaming_batch_size_1(db_path): proc.stdin.write(b'{"name": "Azi"}\n') proc.stdin.flush() # Without this delay the data wasn't yet visible - time.sleep(0.2) + time.sleep(0.4) assert list(Database(db_path)["rows"].rows) == [{"name": "Azi"}] proc.stdin.write(b'{"name": "Suna"}\n') proc.stdin.flush() - time.sleep(0.2) + time.sleep(0.4) assert list(Database(db_path)["rows"].rows) == [{"name": "Azi"}, {"name": "Suna"}] proc.stdin.close() proc.wait() From b6dad08a8389736b7e960cfe9bc719cfc21a98f5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 9 Jan 2022 21:04:51 -0800 Subject: [PATCH 0505/1004] Keep trying up to ten times, refs #364 --- tests/test_cli_insert.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 346bb38..efa55bd 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -481,13 +481,22 @@ def test_insert_streaming_batch_size_1(db_path): ) proc.stdin.write(b'{"name": "Azi"}\n') proc.stdin.flush() - # Without this delay the data wasn't yet visible - time.sleep(0.4) - assert list(Database(db_path)["rows"].rows) == [{"name": "Azi"}] + + def try_until(expected): + tries = 0 + while True: + rows = list(Database(db_path)["rows"].rows) + if rows == expected: + return + tries += 1 + if tries > 10: + assert False, "Expected {}, got {}".format(expected, rows) + time.sleep(tries * 0.1) + + try_until([{"name": "Azi"}]) proc.stdin.write(b'{"name": "Suna"}\n') proc.stdin.flush() - time.sleep(0.4) - assert list(Database(db_path)["rows"].rows) == [{"name": "Azi"}, {"name": "Suna"}] + try_until([{"name": "Azi"}, {"name": "Suna"}]) proc.stdin.close() proc.wait() assert proc.returncode == 0 From 541f64ddb0513cd8fe7a84abc8ee218e36ef9ca6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Jan 2022 11:48:38 -0800 Subject: [PATCH 0506/1004] db.analyze() and table.analyze() methods, refs #366 --- docs/python-api.rst | 27 ++++++++++++++++++++++++++ sqlite_utils/db.py | 11 +++++++++++ tests/test_analyze.py | 45 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 tests/test_analyze.py diff --git a/docs/python-api.rst b/docs/python-api.rst index 5d391ee..791b9c5 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2204,6 +2204,33 @@ You can create a unique index by passing ``unique=True``: Use ``if_not_exists=True`` to do nothing if an index with that name already exists. +.. _python_api_analyze: + +Optimizing index usage with ANALYZE +=================================== + +The `SQLite ANALYZE command `__ can be used to build a table of statistics which the query planner can then use to make better decisions about which indexes to use for a given query. + +You should run ``ANALYZE`` if your database is large and you do not think your indexes are being efficiently used. + +To run ``ANALYZE`` against every index in a database, use this: + +.. code-block:: python + + db.analyze() + +To run it just against a specific named index, pass the name of the index to that method: + +.. code-block:: python + + db.analyze("idx_countries_country_name") + +To run against all indexes attached to a specific table, you can either pass the table name to ``db.analyze(...)`` or you can call the method directly on the table, like this: + +.. code-block:: python + + db["dogs"].analyze() + .. _python_api_vacuum: Vacuum diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index dfc4723..1348b4a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -923,6 +923,13 @@ class Database: "Run a SQLite ``VACUUM`` against the database." self.execute("VACUUM;") + def analyze(self, name=None): + "Run ``ANALYZE`` against the entire database or a named table or index." + sql = "ANALYZE" + if name is not None: + sql += " [{}]".format(name) + self.execute(sql) + class Queryable: def exists(self) -> bool: @@ -2902,6 +2909,10 @@ class Table(Queryable): ) return self + def analyze(self): + "Run ANALYZE against this table" + self.db.analyze(self.name) + def analyze_column( self, column: str, common_limit: int = 10, value_truncate=None, total_rows=None ) -> "ColumnDetails": diff --git a/tests/test_analyze.py b/tests/test_analyze.py new file mode 100644 index 0000000..a47c8be --- /dev/null +++ b/tests/test_analyze.py @@ -0,0 +1,45 @@ +import pytest + + +@pytest.fixture +def db(fresh_db): + fresh_db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db["one_index"].create_index(["name"]) + fresh_db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") + fresh_db["two_indexes"].create_index(["name"]) + fresh_db["two_indexes"].create_index(["species"]) + return fresh_db + + +def test_analyze_whole_database(db): + assert set(db.table_names()) == {"one_index", "two_indexes"} + db.analyze() + assert set(db.table_names()) == {"one_index", "two_indexes", "sqlite_stat1"} + assert list(db["sqlite_stat1"].rows) == [ + {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, + {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, + {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}, + ] + + +@pytest.mark.parametrize("method", ("db_method_with_name", "table_method")) +def test_analyze_one_table(db, method): + assert set(db.table_names()) == {"one_index", "two_indexes"} + if method == "db_method_with_name": + db.analyze("one_index") + elif method == "table_method": + db["one_index"].analyze() + + assert set(db.table_names()) == {"one_index", "two_indexes", "sqlite_stat1"} + assert list(db["sqlite_stat1"].rows) == [ + {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"} + ] + + +def test_analyze_index_by_name(db): + assert set(db.table_names()) == {"one_index", "two_indexes"} + db.analyze("idx_two_indexes_species") + assert set(db.table_names()) == {"one_index", "two_indexes", "sqlite_stat1"} + assert list(db["sqlite_stat1"].rows) == [ + {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, + ] From 0d10402f7b0428c6bb275a106b628298c6d0201d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Jan 2022 12:00:24 -0800 Subject: [PATCH 0507/1004] table.create_index(..., analyze=True), refs #378 --- docs/python-api.rst | 2 ++ sqlite_utils/db.py | 12 +++++++++--- tests/test_create.py | 8 ++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 791b9c5..66e6100 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2204,6 +2204,8 @@ You can create a unique index by passing ``unique=True``: Use ``if_not_exists=True`` to do nothing if an index with that name already exists. +Pass ``analyze=True`` to run ``ANALYZE`` against the new index after creating it. + .. _python_api_analyze: Optimizing index usage with ANALYZE diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1348b4a..6a68c1b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1554,6 +1554,7 @@ class Table(Queryable): unique: bool = False, if_not_exists: bool = False, find_unique_name: bool = False, + analyze: bool = False, ): """ Create an index on this table. @@ -1565,6 +1566,7 @@ class Table(Queryable): - ``if_not_exists`` - only create the index if one with that name does not already exist. - ``find_unique_name`` - if ``index_name`` is not provided and the automatically derived name already exists, keep incrementing a suffix number to find an available name. + - ``analyze`` - run ``ANALYZE`` against this index after creating it. See :ref:`python_api_create_index`. """ @@ -1581,7 +1583,11 @@ class Table(Queryable): columns_sql.append(fmt.format(column)) suffix = None + created_index_name = None while True: + created_index_name = ( + "{}_{}".format(index_name, suffix) if suffix else index_name + ) sql = ( textwrap.dedent( """ @@ -1591,9 +1597,7 @@ class Table(Queryable): ) .strip() .format( - index_name="{}_{}".format(index_name, suffix) - if suffix - else index_name, + index_name=created_index_name, table_name=self.name, columns=", ".join(columns_sql), unique="UNIQUE " if unique else "", @@ -1618,6 +1622,8 @@ class Table(Queryable): continue else: raise e + if analyze: + self.db.analyze(created_index_name) return self def add_column( diff --git a/tests/test_create.py b/tests/test_create.py index 187d20a..f5d9a12 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -774,6 +774,14 @@ def test_create_index_find_unique_name(fresh_db): assert index_names == {"idx_t_id", "idx_t_id_2", "idx_t_id_3"} +def test_create_index_analyze(fresh_db): + dogs = fresh_db["dogs"] + assert "sqlite_stat1" not in fresh_db.table_names() + dogs.insert({"name": "Cleo", "twitter": "cleopaws"}) + dogs.create_index(["name"], analyze=True) + assert "sqlite_stat1" in fresh_db.table_names() + + @pytest.mark.parametrize( "data_structure", ( From 0142c2a3c2772cc370c734e7e6049e8cc2343a5f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Jan 2022 12:02:08 -0800 Subject: [PATCH 0508/1004] Improved test_create_index_analyze test, refs #378 --- tests/test_create.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_create.py b/tests/test_create.py index f5d9a12..1906f10 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -780,6 +780,9 @@ def test_create_index_analyze(fresh_db): dogs.insert({"name": "Cleo", "twitter": "cleopaws"}) dogs.create_index(["name"], analyze=True) assert "sqlite_stat1" in fresh_db.table_names() + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "dogs", "idx": "idx_dogs_name", "stat": "1 1"} + ] @pytest.mark.parametrize( From ab392157f7c89e1596b480649e2f7195f838da29 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Jan 2022 17:00:34 -0800 Subject: [PATCH 0509/1004] analyze=True for insert_all/upsert_all, refs #378 --- docs/python-api.rst | 5 +++-- sqlite_utils/db.py | 8 ++++++++ tests/test_create.py | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 66e6100..a8fd786 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -638,8 +638,9 @@ The function can accept an iterator or generator of rows and will commit them ac You can skip inserting any records that have a primary key that already exists using ``ignore=True``. This works with both ``.insert({...}, ignore=True)`` and ``.insert_all([...], ignore=True)``. -You can delete all the existing rows in the table before inserting the new -records using ``truncate=True``. This is useful if you want to replace the data in the table. +You can delete all the existing rows in the table before inserting the new records using ``truncate=True``. This is useful if you want to replace the data in the table. + +Pass ``analyze=True`` to run ``ANALYZE`` against the table after inserting the new records. .. _python_api_insert_replace: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6a68c1b..6d3bcf9 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2575,10 +2575,13 @@ class Table(Queryable): conversions=DEFAULT, columns=DEFAULT, upsert=False, + analyze=False, ) -> "Table": """ Like ``.insert()`` but takes a list of records and ensures that the table that it creates (if table does not exist) has columns for ALL of that data. + + Use ``analyze=True`` to run ``ANALYZE`` after the insert has completed. """ pk = self.value_or_default("pk", pk) foreign_keys = self.value_or_default("foreign_keys", foreign_keys) @@ -2671,6 +2674,9 @@ class Table(Queryable): ignore, ) + if analyze: + self.analyze() + return self def upsert( @@ -2721,6 +2727,7 @@ class Table(Queryable): extracts=DEFAULT, conversions=DEFAULT, columns=DEFAULT, + analyze=False, ) -> "Table": """ Like ``.upsert()`` but can be applied to a list of records. @@ -2739,6 +2746,7 @@ class Table(Queryable): conversions=conversions, columns=columns, upsert=True, + analyze=analyze, ) def add_missing_columns(self, records: Iterable[Dict[str, Any]]) -> "Table": diff --git a/tests/test_create.py b/tests/test_create.py index 1906f10..82c2c72 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1028,6 +1028,23 @@ def test_insert_all_single_column(fresh_db): assert table.pks == ["name"] +@pytest.mark.parametrize("method_name", ("insert_all", "upsert_all")) +def test_insert_all_analyze(fresh_db, method_name): + table = fresh_db["table"] + table.insert_all([{"id": 1, "name": "Cleo"}], pk="id") + assert "sqlite_stat1" not in fresh_db.table_names() + table.create_index(["name"], analyze=True) + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "table", "idx": "idx_table_name", "stat": "1 1"} + ] + method = getattr(table, method_name) + method([{"id": 2, "name": "Suna"}], pk="id", analyze=True) + assert "sqlite_stat1" in fresh_db.table_names() + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "table", "idx": "idx_table_name", "stat": "2 1"} + ] + + def test_create_with_a_null_column(fresh_db): record = {"name": "Name", "description": None} fresh_db["t"].insert(record) From 389cbd57924da5886a7700c6802d55a934523a29 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Jan 2022 17:08:05 -0800 Subject: [PATCH 0510/1004] delete_where(analyze=True), closes #378 --- docs/python-api.rst | 2 ++ sqlite_utils/db.py | 17 +++++++++++++++-- tests/test_delete.py | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index a8fd786..c79e5bb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -717,6 +717,8 @@ You can delete all records in a table that match a specific WHERE statement usin Calling ``table.delete_where()`` with no other arguments will delete every row in the table. +Pass ``analyze=True`` to run ``ANALYZE`` against the table after deleting the rows. + .. _python_api_upsert: Upserting data diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6d3bcf9..e40eb12 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2119,15 +2119,28 @@ class Table(Queryable): return self def delete_where( - self, where: str = None, where_args: Optional[Union[Iterable, dict]] = None + self, + where: str = None, + where_args: Optional[Union[Iterable, dict]] = None, + analyze: bool = False, ) -> "Table": - "Delete rows matching specified where clause, or delete all rows in the table." + """ + Delete rows matching the specified where clause, or delete all rows in the table. + + - ``where`` - a SQL fragment to use as a ``WHERE`` clause, for example ``age > ?`` or ``age > :age``. + - ``where_args`` - a list of arguments (if using ``?``) or a dictionary (if using ``:age``). + - ``analyze`` - set to ``True`` to run ``ANALYZE`` after the rows have been deleted. + + See :ref:`python_api_delete_where`. + """ if not self.exists(): return self sql = "delete from [{}]".format(self.name) if where is not None: sql += " where " + where self.db.execute(sql, where_args or []) + if analyze: + self.analyze() return self def update( diff --git a/tests/test_delete.py b/tests/test_delete.py index 1198d06..f057749 100644 --- a/tests/test_delete.py +++ b/tests/test_delete.py @@ -30,3 +30,17 @@ def test_delete_where_all(fresh_db): assert 10 == table.count table.delete_where() assert 0 == table.count + + +def test_delete_where_analyze(fresh_db): + table = fresh_db["table"] + table.insert_all(({"id": i, "i": i} for i in range(10)), pk="id") + table.create_index(["i"], analyze=True) + assert "sqlite_stat1" in fresh_db.table_names() + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "table", "idx": "idx_table_i", "stat": "10 1"} + ] + table.delete_where("id > ?", [5], analyze=True) + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "table", "idx": "idx_table_i", "stat": "6 1"} + ] From e0ef9288fede5cba5698c5206f55c98363ca456e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Jan 2022 17:24:17 -0800 Subject: [PATCH 0511/1004] sqlite-utils analyze command, refs #379 --- docs/cli.rst | 17 +++++++++++++++++ docs/python-api.rst | 2 +- sqlite_utils/cli.py | 20 ++++++++++++++++++++ tests/test_cli.py | 38 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 36ff029..da6a562 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1800,6 +1800,23 @@ If the ``_counts`` table ever becomes out-of-sync with the actual table counts y $ sqlite-utils reset-counts mydb.db +.. _cli_analyze: + +Optimizing index usage with ANALYZE +=================================== + +The `SQLite ANALYZE command `__ builds a table of statistics which the query planner can use to make better decisions about which indexes to use for a given query. + +You should run ``ANALYZE`` if your database is large and you do not think your indexes are being efficiently used. + +To run ``ANALYZE`` against every index in a database, use this:: + + $ sqlite-utils analyze mydb.db + +You can run it against specific tables, or against specific named indexes, by passing them as optional arguments:: + + $ sqlite-utils analyze mydb.db mytable idx_mytable_name + .. _cli_vacuum: Vacuum diff --git a/docs/python-api.rst b/docs/python-api.rst index c79e5bb..6b0d542 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2214,7 +2214,7 @@ Pass ``analyze=True`` to run ``ANALYZE`` against the new index after creating it Optimizing index usage with ANALYZE =================================== -The `SQLite ANALYZE command `__ can be used to build a table of statistics which the query planner can then use to make better decisions about which indexes to use for a given query. +The `SQLite ANALYZE command `__ builds a table of statistics which the query planner can use to make better decisions about which indexes to use for a given query. You should run ``ANALYZE`` if your database is large and you do not think your indexes are being efficiently used. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 05a7bcc..cbc091c 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -304,6 +304,26 @@ def rebuild_fts(path, tables, load_extension): db[table].rebuild_fts() +@cli.command() +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("names", nargs=-1) +def analyze(path, names): + """Run ANALYZE against the whole database, or against specific named indexes and tables""" + db = sqlite_utils.Database(path) + try: + if names: + for name in names: + db.analyze(name) + else: + db.analyze() + except sqlite3.OperationalError as e: + raise click.ClickException(e) + + @cli.command() @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index ca5c820..b4163c4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2057,3 +2057,41 @@ def test_create_database(tmpdir, enable_wal): assert db.journal_mode == "wal" else: assert db.journal_mode == "delete" + + +@pytest.mark.parametrize( + "options,expected", + ( + ( + [], + [ + {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, + {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, + {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}, + ], + ), + ( + ["one_index"], + [ + {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}, + ], + ), + ( + ["idx_two_indexes_name"], + [ + {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, + ], + ), + ), +) +def test_analyze(tmpdir, options, expected): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id") + db["one_index"].create_index(["name"]) + db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") + db["two_indexes"].create_index(["name"]) + db["two_indexes"].create_index(["species"]) + result = CliRunner().invoke(cli.cli, ["analyze", db_path] + options) + assert result.exit_code == 0 + assert list(db["sqlite_stat1"].rows) == expected From 1b84c175b455ece931c728e25f3df859c1ad2fdc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Jan 2022 17:36:41 -0800 Subject: [PATCH 0512/1004] --analyze option for create-index, insert, update commands, closes #379, closes #365 --- docs/cli.rst | 6 ++++++ sqlite_utils/cli.py | 32 +++++++++++++++++++++++++++++--- tests/test_cli.py | 25 +++++++++++++++++++++++++ tests/test_cli_insert.py | 14 ++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index da6a562..8b33206 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -761,6 +761,8 @@ You can delete all the existing rows in the table before inserting the new recor $ sqlite-utils insert dogs.db dogs dogs.json --truncate +You can add the ``--analyze`` option to run ``ANALYZE`` against the table after the rows have been inserted. + .. _cli_inserting_data_binary: Inserting binary data @@ -1697,6 +1699,8 @@ This will create an index on that table on ``(col1, col2 desc, col3)``. If your column names are already prefixed with a hyphen you'll need to manually execute a ``CREATE INDEX`` SQL statement to add indexes to them rather than using this tool. +Add the ``--analyze`` option to run ``ANALYZE`` against the index after it has been created. + .. _cli_fts: Configuring full-text search @@ -1817,6 +1821,8 @@ You can run it against specific tables, or against specific named indexes, by pa $ sqlite-utils analyze mydb.db mytable idx_mytable_name +You can also run ``ANALYZE`` as part of another command using the ``--analyze`` option. This is supported by the ``create-index``, ``insert`` and ``upsert`` commands. + .. _cli_vacuum: Vacuum diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cbc091c..187795b 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -490,8 +490,15 @@ def index_foreign_keys(path, load_extension): default=False, is_flag=True, ) +@click.option( + "--analyze", + help="Run ANALYZE after creating the index", + is_flag=True, +) @load_extension_option -def create_index(path, table, column, name, unique, if_not_exists, load_extension): +def create_index( + path, table, column, name, unique, if_not_exists, analyze, load_extension +): """ Add an index to the specified table covering the specified columns. Use "sqlite-utils create-index mydb -- -column" to specify descending @@ -506,7 +513,11 @@ def create_index(path, table, column, name, unique, if_not_exists, load_extensio col = DescIndex(col[1:]) columns.append(col) db[table].create_index( - columns, index_name=name, unique=unique, if_not_exists=if_not_exists + columns, + index_name=name, + unique=unique, + if_not_exists=if_not_exists, + analyze=analyze, ) @@ -726,6 +737,11 @@ def insert_upsert_options(fn): envvar="SQLITE_UTILS_DETECT_TYPES", help="Detect types for columns in CSV/TSV data", ), + click.option( + "--analyze", + is_flag=True, + help="Run ANALYZE at the end of this operation", + ), load_extension_option, click.option("--silent", is_flag=True, help="Do not show progress bar"), ) @@ -761,6 +777,7 @@ def insert_upsert_implementation( default=None, encoding=None, detect_types=None, + analyze=False, load_extension=None, silent=False, ): @@ -853,7 +870,12 @@ def insert_upsert_implementation( else: docs = (fn(doc) or doc for doc in docs) - extra_kwargs = {"ignore": ignore, "replace": replace, "truncate": truncate} + extra_kwargs = { + "ignore": ignore, + "replace": replace, + "truncate": truncate, + "analyze": analyze, + } if not_null: extra_kwargs["not_null"] = set(not_null) if default: @@ -950,6 +972,7 @@ def insert( alter, encoding, detect_types, + analyze, load_extension, silent, ignore, @@ -1005,6 +1028,7 @@ def insert( truncate=truncate, encoding=encoding, detect_types=detect_types, + analyze=analyze, load_extension=load_extension, silent=silent, not_null=not_null, @@ -1039,6 +1063,7 @@ def upsert( default, encoding, detect_types, + analyze, load_extension, silent, ): @@ -1072,6 +1097,7 @@ def upsert( default=default, encoding=encoding, detect_types=detect_types, + analyze=analyze, load_extension=load_extension, silent=silent, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index b4163c4..5f095a6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -224,6 +224,17 @@ def test_create_index(db_path): ) +def test_create_index_analyze(db_path): + db = Database(db_path) + assert "sqlite_stat1" not in db.table_names() + assert [] == db["Gosh"].indexes + result = CliRunner().invoke( + cli.cli, ["create-index", db_path, "Gosh", "c1", "--analyze"] + ) + assert result.exit_code == 0 + assert "sqlite_stat1" in db.table_names() + + def test_create_index_desc(db_path): db = Database(db_path) assert [] == db["Gosh"].indexes @@ -889,6 +900,20 @@ def test_upsert(db_path, tmpdir): ] +def test_upsert_analyze(db_path, tmpdir): + db = Database(db_path) + db["rows"].insert({"id": 1, "foo": "x", "n": 3}, pk="id") + db["rows"].create_index(["n"]) + assert "sqlite_stat1" not in db.table_names() + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "rows", "-", "--nl", "--analyze", "--pk", "id"], + input='{"id": 2, "foo": "bar", "n": 1}', + ) + assert 0 == result.exit_code, result.output + assert "sqlite_stat1" in db.table_names() + + def test_upsert_flatten(tmpdir): db_path = str(tmpdir / "flat.db") db = Database(db_path) diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index efa55bd..0d6b482 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -327,6 +327,20 @@ def test_insert_alter(db_path, tmpdir): ] == list(db.query("select foo, n, baz from from_json_nl")) +def test_insert_analyze(db_path): + db = Database(db_path) + db["rows"].insert({"foo": "x", "n": 3}) + db["rows"].create_index(["n"]) + assert "sqlite_stat1" not in db.table_names() + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "rows", "-", "--nl", "--analyze"], + input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', + ) + assert 0 == result.exit_code, result.output + assert "sqlite_stat1" in db.table_names() + + def test_insert_lines(db_path): result = CliRunner().invoke( cli.cli, From 129141572f249ea290e2a075437e2ebaad215859 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Jan 2022 18:10:54 -0800 Subject: [PATCH 0513/1004] `sqlite-utils bulk` command * sqlite-utils bulk command, closes #375 * Refactor import_options and insert_upsert_options, refs #377 * Tests for sqlite-utils bulk, refs #377 * Documentation for sqlite-utils bulk, refs #377 --- docs/cli.rst | 30 ++++++++ sqlite_utils/cli.py | 167 ++++++++++++++++++++++++++++++----------- tests/test_cli_bulk.py | 57 ++++++++++++++ 3 files changed, 211 insertions(+), 43 deletions(-) create mode 100644 tests/test_cli_bulk.py diff --git a/docs/cli.rst b/docs/cli.rst index 8b33206..913a9ed 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1078,6 +1078,36 @@ The command will fail if you reference columns that do not exist on the table. T .. note:: ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 `__ for details of this change. + +.. _cli_bulk: + +Executing SQL in bulk +===================== + +If you have a JSON, newline-delimited JSON, CSV or TSV file you can execute a bulk SQL query using each of the records in that file using the ``sqlite-utils bulk`` command. + +The command takes the database file, the SQL to be executed and the file containing records to be used when evaluating the SQL query. + +The SQL query should include ``:named`` parameters that match the keys in the records. + +For example, given a ``chickens.csv`` CSV file containing the following:: + + id,name + 1,Blue + 2,Snowy + 3,Azi + 4,Lila + 5,Suna + 6,Cardi + +You could insert those rows into a pre-created ``chickens`` table like so:: + + $ sqlite-utils bulk chickens.db \ + 'insert into chickens (id, name) values (:id, :name)' \ + chickens.csv --csv + +This command takes the same options as the ``sqlite-utils insert`` command - so it defaults to expecting JSON but can accept other formats using ``--csv`` or ``--tsv`` or ``--nl`` or other options described above. + .. _cli_insert_files: Inserting data from files diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 187795b..0dd7f71 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -660,6 +660,50 @@ def reset_counts(path, load_extension): db.reset_counts() +_import_options = ( + click.option( + "--flatten", + is_flag=True, + help='Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1}', + ), + click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), + click.option("-c", "--csv", is_flag=True, help="Expect CSV input"), + click.option("--tsv", is_flag=True, help="Expect TSV input"), + click.option( + "--lines", + is_flag=True, + help="Treat each line as a single value called 'line'", + ), + click.option( + "--text", + is_flag=True, + help="Treat input as a single value called 'text'", + ), + click.option("--convert", help="Python code to convert each item"), + click.option( + "--import", + "imports", + type=str, + multiple=True, + help="Python modules to import", + ), + click.option("--delimiter", help="Delimiter to use for CSV files"), + click.option("--quotechar", help="Quote character to use for CSV/TSV"), + click.option("--sniff", is_flag=True, help="Detect delimiter and quote character"), + click.option("--no-headers", is_flag=True, help="CSV file has no header row"), + click.option( + "--encoding", + help="Character encoding for input, defaults to utf-8", + ), +) + + +def import_options(fn): + for decorator in reversed(_import_options): + fn = decorator(fn) + return fn + + def insert_upsert_options(fn): for decorator in reversed( ( @@ -673,40 +717,9 @@ def insert_upsert_options(fn): click.option( "--pk", help="Columns to use as the primary key, e.g. id", multiple=True ), - click.option( - "--flatten", - is_flag=True, - help='Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1}', - ), - click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), - click.option("-c", "--csv", is_flag=True, help="Expect CSV input"), - click.option("--tsv", is_flag=True, help="Expect TSV input"), - click.option( - "--lines", - is_flag=True, - help="Treat each line as a single value called 'line'", - ), - click.option( - "--text", - is_flag=True, - help="Treat input as a single value called 'text'", - ), - click.option("--convert", help="Python code to convert each item"), - click.option( - "--import", - "imports", - type=str, - multiple=True, - help="Python modules to import", - ), - click.option("--delimiter", help="Delimiter to use for CSV files"), - click.option("--quotechar", help="Quote character to use for CSV/TSV"), - click.option( - "--sniff", is_flag=True, help="Detect delimiter and quote character" - ), - click.option( - "--no-headers", is_flag=True, help="CSV file has no header row" - ), + ) + + _import_options + + ( click.option( "--batch-size", type=int, default=100, help="Commit every X records" ), @@ -726,10 +739,6 @@ def insert_upsert_options(fn): type=(str, str), help="Default value that should be set for a column", ), - click.option( - "--encoding", - help="Character encoding for input, defaults to utf-8", - ), click.option( "-d", "--detect-types", @@ -767,6 +776,7 @@ def insert_upsert_implementation( quotechar, sniff, no_headers, + encoding, batch_size, alter, upsert, @@ -775,11 +785,11 @@ def insert_upsert_implementation( truncate=False, not_null=None, default=None, - encoding=None, detect_types=None, analyze=False, load_extension=None, silent=False, + bulk_sql=None, ): db = sqlite_utils.Database(path) _load_extensions(db, load_extension) @@ -886,6 +896,12 @@ def insert_upsert_implementation( # Apply {"$base64": true, ...} decoding, if needed docs = (decode_base64_values(doc) for doc in docs) + # For bulk_sql= we use cursor.executemany() instead + if bulk_sql: + with db.conn: + db.conn.cursor().executemany(bulk_sql, docs) + return + try: db[table].insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs @@ -968,9 +984,9 @@ def insert( quotechar, sniff, no_headers, + encoding, batch_size, alter, - encoding, detect_types, analyze, load_extension, @@ -1020,13 +1036,13 @@ def insert( quotechar, sniff, no_headers, + encoding, batch_size, alter=alter, upsert=False, ignore=ignore, replace=replace, truncate=truncate, - encoding=encoding, detect_types=detect_types, analyze=analyze, load_extension=load_extension, @@ -1058,10 +1074,10 @@ def upsert( quotechar, sniff, no_headers, + encoding, alter, not_null, default, - encoding, detect_types, analyze, load_extension, @@ -1090,12 +1106,12 @@ def upsert( quotechar, sniff, no_headers, + encoding, batch_size, alter=alter, upsert=True, not_null=not_null, default=default, - encoding=encoding, detect_types=detect_types, analyze=analyze, load_extension=load_extension, @@ -1105,6 +1121,71 @@ def upsert( raise click.ClickException(UNICODE_ERROR.format(ex)) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("sql") +@click.argument("file", type=click.File("rb"), required=True) +@import_options +@load_extension_option +def bulk( + path, + file, + sql, + flatten, + nl, + csv, + tsv, + lines, + text, + convert, + imports, + delimiter, + quotechar, + sniff, + no_headers, + encoding, + load_extension, +): + """ + Execute parameterized SQL against the provided list of documents. + """ + try: + insert_upsert_implementation( + path=path, + table=None, + file=file, + pk=None, + flatten=flatten, + nl=nl, + csv=csv, + tsv=tsv, + lines=lines, + text=text, + convert=convert, + imports=imports, + delimiter=delimiter, + quotechar=quotechar, + sniff=sniff, + no_headers=no_headers, + encoding=encoding, + batch_size=1, + alter=False, + upsert=False, + not_null=set(), + default={}, + detect_types=False, + load_extension=load_extension, + silent=False, + bulk_sql=sql, + ) + except (sqlite3.OperationalError, sqlite3.IntegrityError) as e: + raise click.ClickException(str(e)) + + @cli.command(name="create-database") @click.argument( "path", diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py new file mode 100644 index 0000000..37cf60b --- /dev/null +++ b/tests/test_cli_bulk.py @@ -0,0 +1,57 @@ +from click.testing import CliRunner +from sqlite_utils import cli, Database +import pathlib +import pytest + + +@pytest.fixture +def test_db_and_path(tmpdir): + db_path = str(pathlib.Path(tmpdir) / "data.db") + db = Database(db_path) + db["example"].insert_all( + [ + {"id": 1, "name": "One"}, + {"id": 2, "name": "Two"}, + ], + pk="id", + ) + return db, db_path + + +def test_cli_bulk(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "bulk", + db_path, + "insert into example (id, name) values (:id, :name)", + "-", + "--nl", + ], + input='{"id": 3, "name": "Three"}\n{"id": 4, "name": "Four"}\n', + ) + assert result.exit_code == 0, result.output + assert [ + {"id": 1, "name": "One"}, + {"id": 2, "name": "Two"}, + {"id": 3, "name": "Three"}, + {"id": 4, "name": "Four"}, + ] == list(db["example"].rows) + + +def test_cli_bulk_error(test_db_and_path): + _, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "bulk", + db_path, + "insert into example (id, name) value (:id, :name)", + "-", + "--nl", + ], + input='{"id": 3, "name": "Three"}', + ) + assert result.exit_code == 1 + assert result.output == 'Error: near "value": syntax error\n' From 7c637b11805adc3d3970076a7ba6afe8e34b371e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 10 Jan 2022 18:33:48 -0800 Subject: [PATCH 0514/1004] Release 3.21 Refs #348, #364, #366, #368, #371, #372, #374, #375, #376, #379 Closes #380 --- docs/changelog.rst | 21 +++++++++++++++++++++ setup.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f14870b..9b6b303 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,27 @@ Changelog =========== +.. _v3_21: + +3.21 (2022-01-10) +----------------- + +CLI and Python library improvements to help run `ANALYZE `__ after creating indexes or inserting rows, to gain better performance from the SQLite query planner when it runs against indexes. + +Three new CLI commands: ``create-database``, ``analyze`` and ``bulk``. + +- New ``sqlite-utils create-database`` command for creating new empty database files. (:issue:`348`) +- New Python methods for running ``ANALYZE`` against a database, table or index: ``db.analyze()`` and ``table.analyze()``, see :ref:`python_api_analyze`. (:issue:`366`) +- New :ref:`sqlite-utils analyze command ` for running ``ANALYZE`` using the CLI. (:issue:`379`) +- The ``create-index``, ``insert`` and ``update`` commands now have a new ``--analyze`` option for running ``ANALYZE`` after the command has completed. (:issue:`379`) +- New :ref:`sqlite-utils bulk command ` which can import records in the same way as ``sqlite-utils insert`` (from JSON, CSV or TSV) and use them to bulk execute a parametrized SQL query. (:issue:`375`) +- The CLI tool can now also be run using ``python -m sqlite_utils``. (:issue:`368`) +- Using ``--fmt`` now implies ``--table``, so you don't need to pass both options. (:issue:`374`) +- The ``--convert`` function applied to rows can now modify the row in place. (:issue:`371`) +- The :ref:`insert-files command ` supports two new columns: ``stem`` and ``suffix``. (:issue:`372`) +- The ``--nl`` import option now ignores blank lines in the input. (:issue:`376`) +- Fixed bug where streaming input to the ``insert`` command with ``--batch-size 1`` would appear to only commit after several rows had been ingested, due to unnecessary input buffering. (:issue:`364`) + .. _v3_20: 3.20 (2022-01-05) diff --git a/setup.py b/setup.py index 901725d..4c514e0 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.20" +VERSION = "3.21" def get_long_description(): From 2448e45ddbc039a8acad49ea2af6f72dc14bcb3e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Jan 2022 10:06:50 -0800 Subject: [PATCH 0515/1004] upsert command, not update command --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9b6b303..af3a1e8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,7 +14,7 @@ Three new CLI commands: ``create-database``, ``analyze`` and ``bulk``. - New ``sqlite-utils create-database`` command for creating new empty database files. (:issue:`348`) - New Python methods for running ``ANALYZE`` against a database, table or index: ``db.analyze()`` and ``table.analyze()``, see :ref:`python_api_analyze`. (:issue:`366`) - New :ref:`sqlite-utils analyze command ` for running ``ANALYZE`` using the CLI. (:issue:`379`) -- The ``create-index``, ``insert`` and ``update`` commands now have a new ``--analyze`` option for running ``ANALYZE`` after the command has completed. (:issue:`379`) +- The ``create-index``, ``insert`` and ``upsert`` commands now have a new ``--analyze`` option for running ``ANALYZE`` after the command has completed. (:issue:`379`) - New :ref:`sqlite-utils bulk command ` which can import records in the same way as ``sqlite-utils insert`` (from JSON, CSV or TSV) and use them to bulk execute a parametrized SQL query. (:issue:`375`) - The CLI tool can now also be run using ``python -m sqlite_utils``. (:issue:`368`) - Using ``--fmt`` now implies ``--table``, so you don't need to pass both options. (:issue:`374`) From 5737a3aab4c32cabc05583a552905489eb76294c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Jan 2022 11:18:35 -0800 Subject: [PATCH 0516/1004] Link to annotated release notes --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index af3a1e8..615dc7c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,8 @@ CLI and Python library improvements to help run `ANALYZE `__. + - New ``sqlite-utils create-database`` command for creating new empty database files. (:issue:`348`) - New Python methods for running ``ANALYZE`` against a database, table or index: ``db.analyze()`` and ``table.analyze()``, see :ref:`python_api_analyze`. (:issue:`366`) - New :ref:`sqlite-utils analyze command ` for running ``ANALYZE`` using the CLI. (:issue:`379`) From 5f38c8160138702810698249be27a3c71023b9e4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Jan 2022 11:20:34 -0800 Subject: [PATCH 0517/1004] Fixed typo --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 913a9ed..d2fd9e7 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -976,7 +976,7 @@ Given a JSON file called ``dogs.json`` containing this: The following command will insert that data and add an ``is_good`` column set to ``1`` for each dog:: - $ sqlite-utils insert dogs.db dogs dogs.json --convert 'row["is_good"] = 1` + $ sqlite-utils insert dogs.db dogs dogs.json --convert 'row["is_good"] = 1' The ``--convert`` option also works with the ``--csv``, ``--tsv`` and ``--nl`` insert options. From 1d44b0cc2784c94aed1bcf350225cd86ee1aa7e5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Jan 2022 13:43:39 -0800 Subject: [PATCH 0518/1004] CLI reference page, maintained by cog, closes #383 --- docs/cli-reference.rst | 1051 ++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 1052 insertions(+) create mode 100644 docs/cli-reference.rst diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst new file mode 100644 index 0000000..76630bb --- /dev/null +++ b/docs/cli-reference.rst @@ -0,0 +1,1051 @@ +.. _cli_reference: + +=============== + CLI reference +=============== + +This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. + +.. contents:: :local: + +.. [[[cog + from sqlite_utils import cli + from click.testing import CliRunner + import textwrap + commands = list(cli.cli.commands.keys()) + go_first = [ + "query", "memory", "insert", "upsert", "bulk", "search", "transform", "extract", + "schema", "insert-files", "analyze-tables", "convert", "tables", "views", "rows", + "triggers", "indexes", "create-database", "create-table", "create-index", + "enable-fts", "populate-fts", "rebuild-fts", "disable-fts" + ] + refs = { + "query": "cli_query", + "memory": "cli_memory", + "insert": ["cli_inserting_data", "cli_insert_csv_tsv"], + "upsert": "cli_upsert", + "tables": "cli_tables", + "views": "cli_views", + "optimize": "cli_optimize", + "rows": "cli_rows", + "triggers": "cli_triggers", + "indexes": "cli_indexes", + "enable-fts": "cli_fts", + "analyze": "cli_analyze", + "vacuum": "cli_vacuum", + "dump": "cli_dump", + "add-column": "cli_add_column", + "add-foreign-key": "cli_add_foreign_key", + "add-foreign-keys": "cli_add_foreign_keys", + "index-foreign-keys": "cli_index_foreign_keys", + "create-index": "cli_create_index", + "enable-wal": "cli_wal", + "enable-counts": "cli_enable_counts", + "bulk": "cli_bulk", + "create-database": "cli_create_database", + "create-table": "cli_create_table", + "drop-table": "cli_drop_table", + "create-view": "cli_create_view", + "drop-view": "cli_drop_view", + "search": "cli_search", + "transform": "cli_transform_table", + "extract": "cli_extract", + "schema": "cli_schema", + "insert-files": "cli_insert_files", + "analyze-tables": "cli_analyze_tables", + "convert": "cli_convert", + } + commands.sort(key = lambda command: go_first.index(command) if command in go_first else 999) + for command in commands: + cog.out(command + "\n") + cog.out(("=" * len(command)) + "\n\n") + if command in refs: + command_refs = refs[command] + if isinstance(command_refs, str): + command_refs = [command_refs] + cog.out( + "See {}.\n\n".format( + ", ".join(":ref:`{}`".format(c) for c in command_refs) + ) + ) + cog.out("::\n\n") + result = CliRunner().invoke(cli.cli, [command, "--help"]) + output = result.output.replace("Usage: cli ", "Usage: sqlite-utils ") + cog.out(textwrap.indent(output, ' ')) + cog.out("\n\n") +.. ]]] +query +===== + +See :ref:`cli_query`. + +:: + + Usage: sqlite-utils query [OPTIONS] PATH SQL + + Execute SQL query and return the results as JSON + + Options: + --attach ... Additional databases to attach - specify alias and + filepath + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit CSV headers + -t, --table Output as a table + --fmt TEXT Table format - one of fancy_grid, fancy_outline, + github, grid, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, moinmoin, + orgtbl, pipe, plain, presto, pretty, psql, rst, + simple, textile, tsv, unsafehtml, youtrack + --json-cols Detect JSON cols and output them as JSON, not + escaped strings + -r, --raw Raw output, first column of first row + -p, --param ... Named :parameters for SQL query + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +memory +====== + +See :ref:`cli_memory`. + +:: + + Usage: sqlite-utils memory [OPTIONS] [PATHS]... SQL + + Execute SQL query against an in-memory database, optionally populated by + imported data + + To import data from CSV, TSV or JSON files pass them on the command-line: + + sqlite-utils memory one.csv two.json \ + "select * from one join two on one.two_id = two.id" + + For data piped into the tool from standard input, use "-" or "stdin": + + cat animals.csv | sqlite-utils memory - \ + "select * from stdin where species = 'dog'" + + The format of the data will be automatically detected. You can specify the + format explicitly using :json, :csv, :tsv or :nl (for newline-delimited JSON) + - for example: + + cat animals.csv | sqlite-utils memory stdin:csv places.dat:nl \ + "select * from stdin where place_id in (select id from places)" + + Use --schema to view the SQL schema of any imported files: + + sqlite-utils memory animals.csv --schema + + Options: + --attach ... Additional databases to attach - specify alias and + filepath + --flatten Flatten nested JSON objects, so {"foo": {"bar": + 1}} becomes {"foo_bar": 1} + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit CSV headers + -t, --table Output as a table + --fmt TEXT Table format - one of fancy_grid, fancy_outline, + github, grid, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, moinmoin, + orgtbl, pipe, plain, presto, pretty, psql, rst, + simple, textile, tsv, unsafehtml, youtrack + --json-cols Detect JSON cols and output them as JSON, not + escaped strings + -r, --raw Raw output, first column of first row + -p, --param ... Named :parameters for SQL query + --encoding TEXT Character encoding for CSV input, defaults to + utf-8 + -n, --no-detect-types Treat all CSV/TSV columns as TEXT + --schema Show SQL schema for in-memory database + --dump Dump SQL for in-memory database + --save FILE Save in-memory database to this file + --analyze Analyze resulting tables and output results + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +insert +====== + +See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`. + +:: + + Usage: sqlite-utils insert [OPTIONS] PATH TABLE FILE + + Insert records from FILE into a table, creating the table if it does not + already exist. + + By default the input is expected to be a JSON array of objects. Or: + + - Use --nl for newline-delimited JSON objects + - Use --csv or --tsv for comma-separated or tab-separated input + - Use --lines to write each incoming line to a column called "line" + - Use --text to write the entire input to a column called "text" + + You can also use --convert to pass a fragment of Python code that will be used + to convert each input. + + Your Python code will be passed a "row" variable representing the imported + row, and can return a modified row. + + If you are using --lines your code will be passed a "line" variable, and for + --text an "text" variable. + + Options: + --pk TEXT Columns to use as the primary key, e.g. id + --flatten Flatten nested JSON objects, so {"a": {"b": 1}} + becomes {"a_b": 1} + --nl Expect newline-delimited JSON + -c, --csv Expect CSV input + --tsv Expect TSV input + --lines Treat each line as a single value called 'line' + --text Treat input as a single value called 'text' + --convert TEXT Python code to convert each item + --import TEXT Python modules to import + --delimiter TEXT Delimiter to use for CSV files + --quotechar TEXT Quote character to use for CSV/TSV + --sniff Detect delimiter and quote character + --no-headers CSV file has no header row + --encoding TEXT Character encoding for input, defaults to utf-8 + --batch-size INTEGER Commit every X records + --alter Alter existing table to add any missing columns + --not-null TEXT Columns that should be created as NOT NULL + --default ... Default value that should be set for a column + -d, --detect-types Detect types for columns in CSV/TSV data + --analyze Run ANALYZE at the end of this operation + --load-extension TEXT SQLite extensions to load + --silent Do not show progress bar + --ignore Ignore records if pk already exists + --replace Replace records if pk already exists + --truncate Truncate table before inserting records, if table + already exists + -h, --help Show this message and exit. + + +upsert +====== + +See :ref:`cli_upsert`. + +:: + + Usage: sqlite-utils upsert [OPTIONS] PATH TABLE FILE + + Upsert records based on their primary key. Works like 'insert' but if an + incoming record has a primary key that matches an existing record the existing + record will be updated. + + Options: + --pk TEXT Columns to use as the primary key, e.g. id + --flatten Flatten nested JSON objects, so {"a": {"b": 1}} + becomes {"a_b": 1} + --nl Expect newline-delimited JSON + -c, --csv Expect CSV input + --tsv Expect TSV input + --lines Treat each line as a single value called 'line' + --text Treat input as a single value called 'text' + --convert TEXT Python code to convert each item + --import TEXT Python modules to import + --delimiter TEXT Delimiter to use for CSV files + --quotechar TEXT Quote character to use for CSV/TSV + --sniff Detect delimiter and quote character + --no-headers CSV file has no header row + --encoding TEXT Character encoding for input, defaults to utf-8 + --batch-size INTEGER Commit every X records + --alter Alter existing table to add any missing columns + --not-null TEXT Columns that should be created as NOT NULL + --default ... Default value that should be set for a column + -d, --detect-types Detect types for columns in CSV/TSV data + --analyze Run ANALYZE at the end of this operation + --load-extension TEXT SQLite extensions to load + --silent Do not show progress bar + -h, --help Show this message and exit. + + +bulk +==== + +See :ref:`cli_bulk`. + +:: + + Usage: sqlite-utils bulk [OPTIONS] PATH SQL FILE + + Execute parameterized SQL against the provided list of documents. + + Options: + --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes + {"a_b": 1} + --nl Expect newline-delimited JSON + -c, --csv Expect CSV input + --tsv Expect TSV input + --lines Treat each line as a single value called 'line' + --text Treat input as a single value called 'text' + --convert TEXT Python code to convert each item + --import TEXT Python modules to import + --delimiter TEXT Delimiter to use for CSV files + --quotechar TEXT Quote character to use for CSV/TSV + --sniff Detect delimiter and quote character + --no-headers CSV file has no header row + --encoding TEXT Character encoding for input, defaults to utf-8 + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +search +====== + +See :ref:`cli_search`. + +:: + + Usage: sqlite-utils search [OPTIONS] PATH DBTABLE Q + + Execute a full-text search against this table + + Options: + -o, --order TEXT Order by ('column' or 'column desc') + -c, --column TEXT Columns to return + --limit INTEGER Number of rows to return - defaults to everything + --sql Show SQL query that would be run + --quote Apply FTS quoting rules to search term + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit CSV headers + -t, --table Output as a table + --fmt TEXT Table format - one of fancy_grid, fancy_outline, + github, grid, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, moinmoin, + orgtbl, pipe, plain, presto, pretty, psql, rst, simple, + textile, tsv, unsafehtml, youtrack + --json-cols Detect JSON cols and output them as JSON, not escaped + strings + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +transform +========= + +See :ref:`cli_transform_table`. + +:: + + Usage: sqlite-utils transform [OPTIONS] PATH TABLE + + Transform a table beyond the capabilities of ALTER TABLE + + Options: + --type ... Change column type to INTEGER, TEXT, FLOAT or BLOB + --drop TEXT Drop this column + --rename ... Rename this column to X + -o, --column-order TEXT Reorder columns + --not-null TEXT Set this column to NOT NULL + --not-null-false TEXT Remove NOT NULL from this column + --pk TEXT Make this column the primary key + --pk-none Remove primary key (convert to rowid table) + --default ... Set default value for this column + --default-none TEXT Remove default from this column + --drop-foreign-key TEXT Drop this foreign key constraint + --sql Output SQL without executing it + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +extract +======= + +See :ref:`cli_extract`. + +:: + + Usage: sqlite-utils extract [OPTIONS] PATH TABLE COLUMNS... + + Extract one or more columns into a separate table + + Options: + --table TEXT Name of the other table to extract columns to + --fk-column TEXT Name of the foreign key column to add to the table + --rename ... Rename this column in extracted table + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +schema +====== + +See :ref:`cli_schema`. + +:: + + Usage: sqlite-utils schema [OPTIONS] PATH [TABLES]... + + Show full schema for this database or for specified tables + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +insert-files +============ + +See :ref:`cli_insert_files`. + +:: + + Usage: sqlite-utils insert-files [OPTIONS] PATH TABLE FILE_OR_DIR... + + Insert one or more files using BLOB columns in the specified table + + Example usage: + + sqlite-utils insert-files pics.db images *.gif \ + -c name:name \ + -c content:content \ + -c content_hash:sha256 \ + -c created:ctime_iso \ + -c modified:mtime_iso \ + -c size:size \ + --pk name + + Options: + -c, --column TEXT Column definitions for the table + --pk TEXT Column to use as primary key + --alter Alter table to add missing columns + --replace Replace files with matching primary key + --upsert Upsert files with matching primary key + --name TEXT File name to use + --text Store file content as TEXT, not BLOB + --encoding TEXT Character encoding for input, defaults to utf-8 + -s, --silent Don't show a progress bar + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +analyze-tables +============== + +See :ref:`cli_analyze_tables`. + +:: + + Usage: sqlite-utils analyze-tables [OPTIONS] PATH [TABLES]... + + Analyze the columns in one or more tables + + Options: + -c, --column TEXT Specific columns to analyze + --save Save results to _analyze_tables table + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +convert +======= + +See :ref:`cli_convert`. + +:: + + Usage: sqlite-utils convert [OPTIONS] DB_PATH TABLE COLUMNS... CODE + + Convert columns using Python code you supply. For example: + + $ sqlite-utils convert my.db mytable mycolumn \ + '"\n".join(textwrap.wrap(value, 10))' \ + --import=textwrap + + "value" is a variable with the column value to be converted. + + Use "-" for CODE to read Python code from standard input. + + The following common operations are available as recipe functions: + + r.jsonsplit(value, delimiter=',', type=) + + Convert a string like a,b,c into a JSON array ["a", "b", "c"] + + r.parsedate(value, dayfirst=False, yearfirst=False) + + Parse a date and convert it to ISO date format: yyyy-mm-dd + + r.parsedatetime(value, dayfirst=False, yearfirst=False) + + Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS + + You can use these recipes like so: + + $ sqlite-utils convert my.db mytable mycolumn \ + 'r.jsonsplit(value, delimiter=":")' + + Options: + --import TEXT Python modules to import + --dry-run Show results of running this against first 10 + rows + --multi Populate columns for keys in returned + dictionary + --where TEXT Optional where clause + -p, --param ... Named :parameters for where clause + --output TEXT Optional separate column to populate with the + output + --output-type [integer|float|blob|text] + Column type to use for the output column + --drop Drop original column afterwards + -s, --silent Don't show a progress bar + -h, --help Show this message and exit. + + +tables +====== + +See :ref:`cli_tables`. + +:: + + Usage: sqlite-utils tables [OPTIONS] PATH + + List the tables in the database + + Options: + --fts4 Just show FTS4 enabled tables + --fts5 Just show FTS5 enabled tables + --counts Include row counts per table + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit CSV headers + -t, --table Output as a table + --fmt TEXT Table format - one of fancy_grid, fancy_outline, + github, grid, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, moinmoin, + orgtbl, pipe, plain, presto, pretty, psql, rst, simple, + textile, tsv, unsafehtml, youtrack + --json-cols Detect JSON cols and output them as JSON, not escaped + strings + --columns Include list of columns for each table + --schema Include schema for each table + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +views +===== + +See :ref:`cli_views`. + +:: + + Usage: sqlite-utils views [OPTIONS] PATH + + List the views in the database + + Options: + --counts Include row counts per view + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit CSV headers + -t, --table Output as a table + --fmt TEXT Table format - one of fancy_grid, fancy_outline, + github, grid, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, moinmoin, + orgtbl, pipe, plain, presto, pretty, psql, rst, simple, + textile, tsv, unsafehtml, youtrack + --json-cols Detect JSON cols and output them as JSON, not escaped + strings + --columns Include list of columns for each view + --schema Include schema for each view + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +rows +==== + +See :ref:`cli_rows`. + +:: + + Usage: sqlite-utils rows [OPTIONS] PATH DBTABLE + + Output all rows in the specified table + + Options: + -c, --column TEXT Columns to return + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit CSV headers + -t, --table Output as a table + --fmt TEXT Table format - one of fancy_grid, fancy_outline, + github, grid, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, moinmoin, + orgtbl, pipe, plain, presto, pretty, psql, rst, simple, + textile, tsv, unsafehtml, youtrack + --json-cols Detect JSON cols and output them as JSON, not escaped + strings + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +triggers +======== + +See :ref:`cli_triggers`. + +:: + + Usage: sqlite-utils triggers [OPTIONS] PATH [TABLES]... + + Show triggers configured in this database + + Options: + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit CSV headers + -t, --table Output as a table + --fmt TEXT Table format - one of fancy_grid, fancy_outline, + github, grid, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, moinmoin, + orgtbl, pipe, plain, presto, pretty, psql, rst, simple, + textile, tsv, unsafehtml, youtrack + --json-cols Detect JSON cols and output them as JSON, not escaped + strings + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +indexes +======= + +See :ref:`cli_indexes`. + +:: + + Usage: sqlite-utils indexes [OPTIONS] PATH [TABLES]... + + Show indexes for this database + + Options: + --aux Include auxiliary columns + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit CSV headers + -t, --table Output as a table + --fmt TEXT Table format - one of fancy_grid, fancy_outline, + github, grid, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, moinmoin, + orgtbl, pipe, plain, presto, pretty, psql, rst, simple, + textile, tsv, unsafehtml, youtrack + --json-cols Detect JSON cols and output them as JSON, not escaped + strings + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +create-database +=============== + +See :ref:`cli_create_database`. + +:: + + Usage: sqlite-utils create-database [OPTIONS] PATH + + Create a new empty database file. + + Options: + --enable-wal Enable WAL mode on the created database + -h, --help Show this message and exit. + + +create-table +============ + +See :ref:`cli_create_table`. + +:: + + Usage: sqlite-utils create-table [OPTIONS] PATH TABLE COLUMNS... + + Add a table with the specified columns. Columns should be specified using + name, type pairs, for example: + + sqlite-utils create-table my.db people \ + id integer \ + name text \ + height float \ + photo blob --pk id + + Options: + --pk TEXT Column to use as primary key + --not-null TEXT Columns that should be created as NOT NULL + --default ... Default value that should be set for a column + --fk ... Column, other table, other column to set as a + foreign key + --ignore If table already exists, do nothing + --replace If table already exists, replace it + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +create-index +============ + +See :ref:`cli_create_index`. + +:: + + Usage: sqlite-utils create-index [OPTIONS] PATH TABLE COLUMN... + + Add an index to the specified table covering the specified columns. Use + "sqlite-utils create-index mydb -- -column" to specify descending order for a + column. + + Options: + --name TEXT Explicit name for the new index + --unique Make this a unique index + --if-not-exists Ignore if index already exists + --analyze Run ANALYZE after creating the index + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +enable-fts +========== + +See :ref:`cli_fts`. + +:: + + Usage: sqlite-utils enable-fts [OPTIONS] PATH TABLE COLUMN... + + Enable full-text search for specific table and columns + + Options: + --fts4 Use FTS4 + --fts5 Use FTS5 + --tokenize TEXT Tokenizer to use, e.g. porter + --create-triggers Create triggers to update the FTS tables when the + parent table changes. + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +populate-fts +============ + +:: + + Usage: sqlite-utils populate-fts [OPTIONS] PATH TABLE COLUMN... + + Re-populate full-text search for specific table and columns + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +rebuild-fts +=========== + +:: + + Usage: sqlite-utils rebuild-fts [OPTIONS] PATH [TABLES]... + + Rebuild all or specific full-text search tables + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +disable-fts +=========== + +:: + + Usage: sqlite-utils disable-fts [OPTIONS] PATH TABLE + + Disable full-text search for specific table + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +optimize +======== + +See :ref:`cli_optimize`. + +:: + + Usage: sqlite-utils optimize [OPTIONS] PATH [TABLES]... + + Optimize all full-text search tables and then run VACUUM - should shrink the + database file + + Options: + --no-vacuum Don't run VACUUM + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +analyze +======= + +See :ref:`cli_analyze`. + +:: + + Usage: sqlite-utils analyze [OPTIONS] PATH [NAMES]... + + Run ANALYZE against the whole database, or against specific named indexes and + tables + + Options: + -h, --help Show this message and exit. + + +vacuum +====== + +See :ref:`cli_vacuum`. + +:: + + Usage: sqlite-utils vacuum [OPTIONS] PATH + + Run VACUUM against the database + + Options: + -h, --help Show this message and exit. + + +dump +==== + +See :ref:`cli_dump`. + +:: + + Usage: sqlite-utils dump [OPTIONS] PATH + + Output a SQL dump of the schema and full contents of the database + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +add-column +========== + +See :ref:`cli_add_column`. + +:: + + Usage: sqlite-utils add-column [OPTIONS] PATH TABLE COL_NAME + [[integer|float|blob|text|INTEGER|FLOAT|BLOB|TEXT]] + + Add a column to the specified table + + Options: + --fk TEXT Table to reference as a foreign key + --fk-col TEXT Referenced column on that foreign key table - if + omitted will automatically use the primary key + --not-null-default TEXT Add NOT NULL DEFAULT 'TEXT' constraint + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +add-foreign-key +=============== + +See :ref:`cli_add_foreign_key`. + +:: + + Usage: sqlite-utils add-foreign-key [OPTIONS] PATH TABLE COLUMN [OTHER_TABLE] + [OTHER_COLUMN] + + Add a new foreign key constraint to an existing table. Example usage: + + $ sqlite-utils add-foreign-key my.db books author_id authors id + + WARNING: Could corrupt your database! Back up your database file first. + + Options: + --ignore If foreign key already exists, do nothing + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +add-foreign-keys +================ + +See :ref:`cli_add_foreign_keys`. + +:: + + Usage: sqlite-utils add-foreign-keys [OPTIONS] PATH [FOREIGN_KEY]... + + Add multiple new foreign key constraints to a database. Example usage: + + sqlite-utils add-foreign-keys my.db \ + books author_id authors id \ + authors country_id countries id + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +index-foreign-keys +================== + +See :ref:`cli_index_foreign_keys`. + +:: + + Usage: sqlite-utils index-foreign-keys [OPTIONS] PATH + + Ensure every foreign key column has an index on it. + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +enable-wal +========== + +See :ref:`cli_wal`. + +:: + + Usage: sqlite-utils enable-wal [OPTIONS] PATH... + + Enable WAL for database files + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +disable-wal +=========== + +:: + + Usage: sqlite-utils disable-wal [OPTIONS] PATH... + + Disable WAL for database files + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +enable-counts +============= + +See :ref:`cli_enable_counts`. + +:: + + Usage: sqlite-utils enable-counts [OPTIONS] PATH [TABLES]... + + Configure triggers to update a _counts table with row counts + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +reset-counts +============ + +:: + + Usage: sqlite-utils reset-counts [OPTIONS] PATH + + Reset calculated counts in the _counts table + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +drop-table +========== + +See :ref:`cli_drop_table`. + +:: + + Usage: sqlite-utils drop-table [OPTIONS] PATH TABLE + + Drop the specified table + + Options: + --ignore + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +create-view +=========== + +See :ref:`cli_create_view`. + +:: + + Usage: sqlite-utils create-view [OPTIONS] PATH VIEW SELECT + + Create a view for the provided SELECT query + + Options: + --ignore If view already exists, do nothing + --replace If view already exists, replace it + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +drop-view +========= + +See :ref:`cli_drop_view`. + +:: + + Usage: sqlite-utils drop-view [OPTIONS] PATH VIEW + + Drop the specified view + + Options: + --ignore + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +.. [[[end]]] diff --git a/docs/index.rst b/docs/index.rst index 0629e0e..49bf8b1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -33,5 +33,6 @@ Contents cli python-api reference + cli-reference contributing changelog From 324ebc31308752004fe5f7e4941fc83706c5539c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Jan 2022 15:19:29 -0800 Subject: [PATCH 0519/1004] sqlite-utils rows --limit and --offset options, closes #381 --- docs/cli-reference.rst | 2 ++ docs/cli.rst | 2 ++ sqlite_utils/cli.py | 19 ++++++++++++++++++- tests/test_cli.py | 8 ++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 76630bb..90e3f7d 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -585,6 +585,8 @@ See :ref:`cli_rows`. Options: -c, --column TEXT Columns to return + --limit INTEGER Number of rows to return - defaults to everything + --offset INTEGER SQL offset to use --nl Output newline-delimited JSON --arrays Output rows as arrays instead of objects --csv Output CSV diff --git a/docs/cli.rst b/docs/cli.rst index d2fd9e7..dda1007 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -449,6 +449,8 @@ You can use the ``-c`` option to specify a subset of columns to return:: [{"age": 4, "name": "Cleo"}, {"age": 2, "name": "Pancakes"}] +Use ``--limit N`` to only return the first ``N`` rows. Use ``--offset N`` to return rows starting from the specified offset. + .. _cli_tables: Listing tables diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0dd7f71..9c768b0 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1723,6 +1723,16 @@ def search( ) @click.argument("dbtable") @click.option("-c", "--column", type=str, multiple=True, help="Columns to return") +@click.option( + "--limit", + type=int, + help="Number of rows to return - defaults to everything", +) +@click.option( + "--offset", + type=int, + help="SQL offset to use", +) @output_options @load_extension_option @click.pass_context @@ -1731,6 +1741,8 @@ def rows( path, dbtable, column, + limit, + offset, nl, arrays, csv, @@ -1745,10 +1757,15 @@ def rows( columns = "*" if column: columns = ", ".join("[{}]".format(c) for c in column) + sql = "select {} from [{}]".format(columns, dbtable) + if limit: + sql += " limit {}".format(limit) + if offset: + sql += " offset {}".format(offset) ctx.invoke( query, path=path, - sql="select {} from [{}]".format(columns, dbtable), + sql=sql, nl=nl, arrays=arrays, csv=csv, diff --git a/tests/test_cli.py b/tests/test_cli.py index 5f095a6..1db0849 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -851,6 +851,14 @@ def test_query_memory_does_not_create_file(tmpdir): ["--nl", "-c", "age", "-c", "name"], '{"age": 4, "name": "Cleo"}\n{"age": 2, "name": "Pancakes"}', ), + ( + ["-c", "name", "--limit", "1"], + '[{"name": "Cleo"}]', + ), + ( + ["-c", "name", "--limit", "1", "--offset", "1"], + '[{"name": "Pancakes"}]', + ), ], ) def test_rows(db_path, args, expected): From 3b632f0a7eda0aff444ea67a78f5003797b286c5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Jan 2022 15:32:43 -0800 Subject: [PATCH 0520/1004] sqlite-utils rows --where and -p options, closes #382 --- docs/cli-reference.rst | 38 ++++++++++++++++++++------------------ docs/cli.rst | 10 ++++++++++ sqlite_utils/cli.py | 13 +++++++++++++ tests/test_cli.py | 14 ++++++++++++++ 4 files changed, 57 insertions(+), 18 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 90e3f7d..d4acc6a 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -584,24 +584,26 @@ See :ref:`cli_rows`. Output all rows in the specified table Options: - -c, --column TEXT Columns to return - --limit INTEGER Number of rows to return - defaults to everything - --offset INTEGER SQL offset to use - --nl Output newline-delimited JSON - --arrays Output rows as arrays instead of objects - --csv Output CSV - --tsv Output TSV - --no-headers Omit CSV headers - -t, --table Output as a table - --fmt TEXT Table format - one of fancy_grid, fancy_outline, - github, grid, html, jira, latex, latex_booktabs, - latex_longtable, latex_raw, mediawiki, moinmoin, - orgtbl, pipe, plain, presto, pretty, psql, rst, simple, - textile, tsv, unsafehtml, youtrack - --json-cols Detect JSON cols and output them as JSON, not escaped - strings - --load-extension TEXT SQLite extensions to load - -h, --help Show this message and exit. + -c, --column TEXT Columns to return + --where TEXT Optional where clause + -p, --param ... Named :parameters for where clause + --limit INTEGER Number of rows to return - defaults to everything + --offset INTEGER SQL offset to use + --nl Output newline-delimited JSON + --arrays Output rows as arrays instead of objects + --csv Output CSV + --tsv Output TSV + --no-headers Omit CSV headers + -t, --table Output as a table + --fmt TEXT Table format - one of fancy_grid, fancy_outline, + github, grid, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, moinmoin, + orgtbl, pipe, plain, presto, pretty, psql, rst, + simple, textile, tsv, unsafehtml, youtrack + --json-cols Detect JSON cols and output them as JSON, not + escaped strings + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. triggers diff --git a/docs/cli.rst b/docs/cli.rst index dda1007..733eee8 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -449,6 +449,16 @@ You can use the ``-c`` option to specify a subset of columns to return:: [{"age": 4, "name": "Cleo"}, {"age": 2, "name": "Pancakes"}] +You can filter rows using a where clause with the ``--where`` option:: + + $ sqlite-utils rows dogs.db dogs -c name --where 'name = "Cleo"' + [{"name": "Cleo"}] + +Or pass named parameters using ``--where`` in combination with ``-p``:: + + $ sqlite-utils rows dogs.db dogs -c name --where 'name = :name' -p name Cleo + [{"name": "Cleo"}] + Use ``--limit N`` to only return the first ``N`` rows. Use ``--offset N`` to return rows starting from the specified offset. .. _cli_tables: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 9c768b0..9f1331d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1723,6 +1723,14 @@ def search( ) @click.argument("dbtable") @click.option("-c", "--column", type=str, multiple=True, help="Columns to return") +@click.option("--where", help="Optional where clause") +@click.option( + "-p", + "--param", + multiple=True, + type=(str, str), + help="Named :parameters for where clause", +) @click.option( "--limit", type=int, @@ -1741,6 +1749,8 @@ def rows( path, dbtable, column, + where, + param, limit, offset, nl, @@ -1758,6 +1768,8 @@ def rows( if column: columns = ", ".join("[{}]".format(c) for c in column) sql = "select {} from [{}]".format(columns, dbtable) + if where: + sql += " where " + where if limit: sql += " limit {}".format(limit) if offset: @@ -1773,6 +1785,7 @@ def rows( no_headers=no_headers, table=table, fmt=fmt, + param=param, json_cols=json_cols, load_extension=load_extension, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 1db0849..60c95b0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -851,6 +851,7 @@ def test_query_memory_does_not_create_file(tmpdir): ["--nl", "-c", "age", "-c", "name"], '{"age": 4, "name": "Cleo"}\n{"age": 2, "name": "Pancakes"}', ), + # --limit and --offset ( ["-c", "name", "--limit", "1"], '[{"name": "Cleo"}]', @@ -859,6 +860,19 @@ def test_query_memory_does_not_create_file(tmpdir): ["-c", "name", "--limit", "1", "--offset", "1"], '[{"name": "Pancakes"}]', ), + # --where + ( + ["-c", "name", "--where", "id = 1"], + '[{"name": "Cleo"}]', + ), + ( + ["-c", "name", "--where", "id = :id", "-p", "id", "1"], + '[{"name": "Cleo"}]', + ), + ( + ["-c", "name", "--where", "id = :id", "--param", "id", "1"], + '[{"name": "Cleo"}]', + ), ], ) def test_rows(db_path, args, expected): From 74586d3cb26fa3cc3412721985ecdc1864c2a31d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Jan 2022 15:44:48 -0800 Subject: [PATCH 0521/1004] Release 3.22 Refs #381, #382, #383 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 615dc7c..0af3c04 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v3_22: + +3.22 (2022-01-11) +----------------- + +- New :ref:`cli_reference` documentation page, listing the output of ``--help`` for every one of the CLI commands. (:issue:`383`) +- ``sqlite-utils rows`` now has ``--limit`` and ``--offset`` options for paginating through data. (:issue:`381`) +- ``sqlite-utils rows`` now has ``--where`` and ``-p`` options for filtering the table using a ``WHERE`` query, see :ref:`cli_rows`. (:issue:`382`) + .. _v3_21: 3.21 (2022-01-10) diff --git a/setup.py b/setup.py index 4c514e0..e0aa54b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.21" +VERSION = "3.22" def get_long_description(): From 7fdff5019d7c9d609fb00b5c7fd64bcde029e4c8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 18 Jan 2022 18:15:21 -0800 Subject: [PATCH 0522/1004] Link to article from contributing, closes #386 --- docs/contributing.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index c2d20ba..7ac1ad8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -4,6 +4,15 @@ Contributing ============== +Development of ``sqlite-utils`` takes place in the `sqlite-utils GitHub repository `__. + +All improvements to the software should start with an issue. Read `How I build a feature `__ for a detailed description of the recommended process for building bug fixes or enhancements. + +.. _contributing_checkout: + +Obtaining the code +================== + To work on this library locally, first checkout the code. Then create a new virtual environment:: git clone git@github.com:simonw/sqlite-utils From 3091e6b6e9befe306310d2e5a484ffd88c0200bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 Jan 2022 20:06:40 -0800 Subject: [PATCH 0523/1004] Clearer help for --drop-foreign-key --- docs/cli-reference.rst | 2 +- sqlite_utils/cli.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index d4acc6a..546be1b 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -356,7 +356,7 @@ See :ref:`cli_transform_table`. --pk-none Remove primary key (convert to rowid table) --default ... Set default value for this column --default-none TEXT Remove default from this column - --drop-foreign-key TEXT Drop this foreign key constraint + --drop-foreign-key TEXT Drop foreign key constraint for this column --sql Output SQL without executing it --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 9f1331d..4f950ad 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1964,7 +1964,7 @@ def schema( "--drop-foreign-key", type=str, multiple=True, - help="Drop this foreign key constraint", + help="Drop foreign key constraint for this column", ) @click.option("--sql", is_flag=True, help="Output SQL without executing it") @load_extension_option From 82ea42ffeedd5c80570b1e6f16124dd80f8f4a1b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 Jan 2022 20:12:32 -0800 Subject: [PATCH 0524/1004] Added missing docstring for db.supports_strict --- sqlite_utils/db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e40eb12..06e7557 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -524,7 +524,8 @@ class Database: return "\n".join(sqls) @property - def supports_strict(self): + def supports_strict(self) -> bool: + "Does this database support STRICT mode?" try: table_name = "t{}".format(secrets.token_hex(16)) with self.conn: From 8d51ae48ab084284681d597b436be2112650a3b9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Jan 2022 17:35:26 -0800 Subject: [PATCH 0525/1004] Getting started section for Python library, closes #387 --- docs/python-api.rst | 59 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 6b0d542..42dfe94 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -6,6 +6,65 @@ .. contents:: :local: +.. _python_api_getting_started: + +Getting started +=============== + +Here's how to create a new SQLite database file containing a new ``chickens`` table, populated with four records: + +.. code-block:: python + + from sqlite_utils import Database + + db = Database("chickens.db") + db["chickens"].insert_all([{ + "name": "Azi", + "color": "blue", + }, { + "name": "Lila", + "color": "blue", + }, { + "name": "Suna", + "color": "gold", + }, { + "name": "Cardi", + "color": "black", + }]) + +You can loop through those rows like this: + +.. code-block:: python + + for row in db["chickens"].rows: + print(row) + +Which outputs the following:: + + {'name': 'Azi', 'color': 'blue'} + {'name': 'Lila', 'color': 'blue'} + {'name': 'Suna', 'color': 'gold'} + {'name': 'Cardi', 'color': 'black'} + +To run a SQL query, use :ref:`db.query() `: + +.. code-block:: python + + for row in db.query(""" + select color, count(*) + from chickens group by color + order by count(*) desc + """): + print(row) + +Which outputs:: + + {'color': 'blue', 'count(*)': 2} + {'color': 'gold', 'count(*)': 1} + {'color': 'black', 'count(*)': 1} + +.. _python_api_connect: + Connecting to or creating a database ==================================== From b3efb292127036d710ae3cb63daa36cd7a4d7d0c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Jan 2022 17:48:57 -0800 Subject: [PATCH 0526/1004] SQLite can drop columns now It gained that ability in 3.35.0 in 2021-03-12: https://www.sqlite.org/changes.html#version_3_35_0 --- README.md | 2 +- docs/python-api.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9345286..9debfcd 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Python CLI utility and library for manipulating SQLite databases. - [Pipe JSON](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-json-data) (or [CSV or TSV](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-csv-or-tsv-data)) directly into a new SQLite database file, automatically creating a table with the appropriate schema - [Run in-memory SQL queries](https://sqlite-utils.datasette.io/en/stable/cli.html#querying-data-directly-using-an-in-memory-database), including joins, directly against data in CSV, TSV or JSON files and view the results. - [Configure SQLite full-text search](https://sqlite-utils.datasette.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance -- Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as dropping columns +- Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as changing the type of a column - [Extract columns](https://sqlite-utils.datasette.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data Read more on my blog: [ diff --git a/docs/python-api.rst b/docs/python-api.rst index 42dfe94..6b703fe 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1247,7 +1247,7 @@ Pass ``ignore=True`` if you want to ignore the error caused by the table or view Transforming a table ==================== -The SQLite ``ALTER TABLE`` statement is limited. It can add columns and rename tables, but it cannot drop columns, change column types, change ``NOT NULL`` status or change the primary key for a table. +The SQLite ``ALTER TABLE`` statement is limited. It can add and drop columns and rename tables, but it cannot change column types, change ``NOT NULL`` status or change the primary key for a table. The ``table.transform()`` method can do all of these things, by implementing a multi-step pattern `described in the SQLite documentation `__: From 89c01103ec0b684b6f871694f77fc49d0cb57f98 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Jan 2022 18:00:16 -0800 Subject: [PATCH 0527/1004] Custom layout template for docs Adds plausible analytics, closes #389 Implements banner on latest page, closes #388 --- docs/_templates/layout.html | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 docs/_templates/layout.html diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 0000000..150edd0 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,39 @@ +{%- extends "!layout.html" %} + +{% block htmltitle %} +{{ super() }} + +{% endblock %} + +{% block footer %} +{{ super() }} + +{% endblock %} From 25d8c820de195039106b736440a5c4e3f72cd8b6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Jan 2022 18:06:02 -0800 Subject: [PATCH 0528/1004] Correct domain for Plausible, refs #389 --- docs/_templates/layout.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 150edd0..cecf66e 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -2,7 +2,7 @@ {% block htmltitle %} {{ super() }} - + {% endblock %} {% block footer %} From 6e85a4bbbefced11501a8e215d0847addc159199 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Jan 2022 18:56:44 -0800 Subject: [PATCH 0529/1004] Added examples to more --help output, refs #384 --- docs/cli-reference.rst | 100 +++++++++++++++++++++++++++-------- sqlite_utils/cli.py | 117 +++++++++++++++++++++++++++++++++-------- 2 files changed, 171 insertions(+), 46 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 546be1b..868cc3b 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -85,6 +85,12 @@ See :ref:`cli_query`. Execute SQL query and return the results as JSON + Example: + + sqlite-utils data.db \ + "select * from chickens where age > :age" \ + -p age 1 + Options: --attach ... Additional databases to attach - specify alias and filepath @@ -93,7 +99,7 @@ See :ref:`cli_query`. --csv Output CSV --tsv Output TSV --no-headers Omit CSV headers - -t, --table Output as a table + -t, --table Output as a formatted table --fmt TEXT Table format - one of fancy_grid, fancy_outline, github, grid, html, jira, latex, latex_booktabs, latex_longtable, latex_raw, mediawiki, moinmoin, @@ -150,7 +156,7 @@ See :ref:`cli_memory`. --csv Output CSV --tsv Output TSV --no-headers Omit CSV headers - -t, --table Output as a table + -t, --table Output as a formatted table --fmt TEXT Table format - one of fancy_grid, fancy_outline, github, grid, html, jira, latex, latex_booktabs, latex_longtable, latex_raw, mediawiki, moinmoin, @@ -183,7 +189,11 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`. Insert records from FILE into a table, creating the table if it does not already exist. - By default the input is expected to be a JSON array of objects. Or: + Example: + + echo '{"name": "Lila"}' | sqlite-utils insert data.db chickens - + + By default the input is expected to be a JSON object or array of objects. - Use --nl for newline-delimited JSON objects - Use --csv or --tsv for comma-separated or tab-separated input @@ -243,6 +253,15 @@ See :ref:`cli_upsert`. incoming record has a primary key that matches an existing record the existing record will be updated. + The --pk option is required. + + Example: + + echo '[ + {"id": 1, "name": "Lila"}, + {"id": 2, "name": "Suna"} + ]' | sqlite-utils upsert data.db chickens - --pk id + Options: --pk TEXT Columns to use as the primary key, e.g. id --flatten Flatten nested JSON objects, so {"a": {"b": 1}} @@ -281,6 +300,15 @@ See :ref:`cli_bulk`. Execute parameterized SQL against the provided list of documents. + Example: + + echo '[ + {"id": 1, "name": "Lila2"}, + {"id": 2, "name": "Suna2"} + ]' | sqlite-utils bulk data.db ' + update chickens set name = :name where id = :id + ' - + Options: --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} @@ -311,6 +339,10 @@ See :ref:`cli_search`. Execute a full-text search against this table + Example: + + sqlite-utils search data.db chickens lila + Options: -o, --order TEXT Order by ('column' or 'column desc') -c, --column TEXT Columns to return @@ -322,7 +354,7 @@ See :ref:`cli_search`. --csv Output CSV --tsv Output TSV --no-headers Omit CSV headers - -t, --table Output as a table + -t, --table Output as a formatted table --fmt TEXT Table format - one of fancy_grid, fancy_outline, github, grid, html, jira, latex, latex_booktabs, latex_longtable, latex_raw, mediawiki, moinmoin, @@ -345,6 +377,12 @@ See :ref:`cli_transform_table`. Transform a table beyond the capabilities of ALTER TABLE + Example: + + sqlite-utils transform mydb.db mytable \ + --drop column1 \ + --rename column2 column_renamed + Options: --type ... Change column type to INTEGER, TEXT, FLOAT or BLOB --drop TEXT Drop this column @@ -373,6 +411,10 @@ See :ref:`cli_extract`. Extract one or more columns into a separate table + Example: + + sqlite-utils extract trees.db Street_Trees species + Options: --table TEXT Name of the other table to extract columns to --fk-column TEXT Name of the foreign key column to add to the table @@ -392,6 +434,10 @@ See :ref:`cli_schema`. Show full schema for this database or for specified tables + Example: + + sqlite-utils schema trees.db + Options: --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. @@ -408,16 +454,16 @@ See :ref:`cli_insert_files`. Insert one or more files using BLOB columns in the specified table - Example usage: + Example: - sqlite-utils insert-files pics.db images *.gif \ - -c name:name \ - -c content:content \ - -c content_hash:sha256 \ - -c created:ctime_iso \ - -c modified:mtime_iso \ - -c size:size \ - --pk name + sqlite-utils insert-files pics.db images *.gif \ + -c name:name \ + -c content:content \ + -c content_hash:sha256 \ + -c created:ctime_iso \ + -c modified:mtime_iso \ + -c size:size \ + --pk name Options: -c, --column TEXT Column definitions for the table @@ -444,6 +490,10 @@ See :ref:`cli_analyze_tables`. Analyze the columns in one or more tables + Example: + + sqlite-utils analyze-tables data.db trees + Options: -c, --column TEXT Specific columns to analyze --save Save results to _analyze_tables table @@ -462,9 +512,9 @@ See :ref:`cli_convert`. Convert columns using Python code you supply. For example: - $ sqlite-utils convert my.db mytable mycolumn \ - '"\n".join(textwrap.wrap(value, 10))' \ - --import=textwrap + sqlite-utils convert my.db mytable mycolumn \ + '"\n".join(textwrap.wrap(value, 10))' \ + --import=textwrap "value" is a variable with the column value to be converted. @@ -486,8 +536,8 @@ See :ref:`cli_convert`. You can use these recipes like so: - $ sqlite-utils convert my.db mytable mycolumn \ - 'r.jsonsplit(value, delimiter=":")' + sqlite-utils convert my.db mytable mycolumn \ + 'r.jsonsplit(value, delimiter=":")' Options: --import TEXT Python modules to import @@ -517,6 +567,10 @@ See :ref:`cli_tables`. List the tables in the database + Example: + + sqlite-utils tables trees.db + Options: --fts4 Just show FTS4 enabled tables --fts5 Just show FTS5 enabled tables @@ -526,7 +580,7 @@ See :ref:`cli_tables`. --csv Output CSV --tsv Output TSV --no-headers Omit CSV headers - -t, --table Output as a table + -t, --table Output as a formatted table --fmt TEXT Table format - one of fancy_grid, fancy_outline, github, grid, html, jira, latex, latex_booktabs, latex_longtable, latex_raw, mediawiki, moinmoin, @@ -558,7 +612,7 @@ See :ref:`cli_views`. --csv Output CSV --tsv Output TSV --no-headers Omit CSV headers - -t, --table Output as a table + -t, --table Output as a formatted table --fmt TEXT Table format - one of fancy_grid, fancy_outline, github, grid, html, jira, latex, latex_booktabs, latex_longtable, latex_raw, mediawiki, moinmoin, @@ -594,7 +648,7 @@ See :ref:`cli_rows`. --csv Output CSV --tsv Output TSV --no-headers Omit CSV headers - -t, --table Output as a table + -t, --table Output as a formatted table --fmt TEXT Table format - one of fancy_grid, fancy_outline, github, grid, html, jira, latex, latex_booktabs, latex_longtable, latex_raw, mediawiki, moinmoin, @@ -623,7 +677,7 @@ See :ref:`cli_triggers`. --csv Output CSV --tsv Output TSV --no-headers Omit CSV headers - -t, --table Output as a table + -t, --table Output as a formatted table --fmt TEXT Table format - one of fancy_grid, fancy_outline, github, grid, html, jira, latex, latex_booktabs, latex_longtable, latex_raw, mediawiki, moinmoin, @@ -653,7 +707,7 @@ See :ref:`cli_indexes`. --csv Output CSV --tsv Output TSV --no-headers Omit CSV headers - -t, --table Output as a table + -t, --table Output as a formatted table --fmt TEXT Table format - one of fancy_grid, fancy_outline, github, grid, html, jira, latex, latex_booktabs, latex_longtable, latex_raw, mediawiki, moinmoin, diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4f950ad..42fc8e8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -75,7 +75,9 @@ def output_options(fn): click.option("--csv", is_flag=True, help="Output CSV"), click.option("--tsv", is_flag=True, help="Output TSV"), click.option("--no-headers", is_flag=True, help="Omit CSV headers"), - click.option("-t", "--table", is_flag=True, help="Output as a table"), + click.option( + "-t", "--table", is_flag=True, help="Output as a formatted table" + ), click.option( "--fmt", help="Table format - one of {}".format( @@ -161,7 +163,13 @@ def tables( load_extension, views=False, ): - """List the tables in the database""" + """List the tables in the database + + Example: + + \b + sqlite-utils tables trees.db + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) headers = ["view" if views else "table"] @@ -1001,7 +1009,11 @@ def insert( Insert records from FILE into a table, creating the table if it does not already exist. - By default the input is expected to be a JSON array of objects. Or: + Example: + + echo '{"name": "Lila"}' | sqlite-utils insert data.db chickens - + + By default the input is expected to be a JSON object or array of objects. \b - Use --nl for newline-delimited JSON objects @@ -1087,6 +1099,16 @@ def upsert( Upsert records based on their primary key. Works like 'insert' but if an incoming record has a primary key that matches an existing record the existing record will be updated. + + The --pk option is required. + + Example: + + \b + echo '[ + {"id": 1, "name": "Lila"}, + {"id": 2, "name": "Suna"} + ]' | sqlite-utils upsert data.db chickens - --pk id """ try: insert_upsert_implementation( @@ -1152,6 +1174,16 @@ def bulk( ): """ Execute parameterized SQL against the provided list of documents. + + Example: + + \b + echo '[ + {"id": 1, "name": "Lila2"}, + {"id": 2, "name": "Suna2"} + ]' | sqlite-utils bulk data.db ' + update chickens set name = :name where id = :id + ' - """ try: insert_upsert_implementation( @@ -1402,7 +1434,15 @@ def query( param, load_extension, ): - "Execute SQL query and return the results as JSON" + """Execute SQL query and return the results as JSON + + Example: + + \b + sqlite-utils data.db \\ + "select * from chickens where age > :age" \\ + -p age 1 + """ db = sqlite_utils.Database(path) for alias, attach_path in attach: db.attach(alias, attach_path) @@ -1665,7 +1705,12 @@ def search( json_cols, load_extension, ): - "Execute a full-text search against this table" + """Execute a full-text search against this table + + Example: + + sqlite-utils search data.db chickens lila + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) # Check table exists @@ -1912,7 +1957,13 @@ def schema( tables, load_extension, ): - "Show full schema for this database or for specified tables" + """Show full schema for this database or for specified tables + + Example: + + \b + sqlite-utils schema trees.db + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) if tables: @@ -1985,7 +2036,15 @@ def transform( sql, load_extension, ): - "Transform a table beyond the capabilities of ALTER TABLE" + """Transform a table beyond the capabilities of ALTER TABLE + + Example: + + \b + sqlite-utils transform mydb.db mytable \\ + --drop column1 \\ + --rename column2 column_renamed + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) types = {} @@ -2060,7 +2119,13 @@ def extract( rename, load_extension, ): - "Extract one or more columns into a separate table" + """Extract one or more columns into a separate table + + Example: + + \b + sqlite-utils extract trees.db Street_Trees species + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) kwargs = dict( @@ -2122,17 +2187,17 @@ def insert_files( """ Insert one or more files using BLOB columns in the specified table - Example usage: + Example: \b - sqlite-utils insert-files pics.db images *.gif \\ - -c name:name \\ - -c content:content \\ - -c content_hash:sha256 \\ - -c created:ctime_iso \\ - -c modified:mtime_iso \\ - -c size:size \\ - --pk name + sqlite-utils insert-files pics.db images *.gif \\ + -c name:name \\ + -c content:content \\ + -c content_hash:sha256 \\ + -c created:ctime_iso \\ + -c modified:mtime_iso \\ + -c size:size \\ + --pk name """ if not column: if text: @@ -2244,7 +2309,13 @@ def analyze_tables( save, load_extension, ): - "Analyze the columns in one or more tables" + """Analyze the columns in one or more tables + + Example: + + \b + sqlite-utils analyze-tables data.db trees + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) _analyze(db, tables, columns, save) @@ -2308,9 +2379,9 @@ def _generate_convert_help(): Convert columns using Python code you supply. For example: \b - $ sqlite-utils convert my.db mytable mycolumn \\ - '"\\n".join(textwrap.wrap(value, 10))' \\ - --import=textwrap + sqlite-utils convert my.db mytable mycolumn \\ + '"\\n".join(textwrap.wrap(value, 10))' \\ + --import=textwrap "value" is a variable with the column value to be converted. @@ -2333,8 +2404,8 @@ def _generate_convert_help(): You can use these recipes like so: \b - $ sqlite-utils convert my.db mytable mycolumn \\ - 'r.jsonsplit(value, delimiter=":")' + sqlite-utils convert my.db mytable mycolumn \\ + 'r.jsonsplit(value, delimiter=":")' """ ).strip() return help From 4c6023452cbfc0c112cfc2a940ed40d22e8d36c9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Jan 2022 19:14:59 -0800 Subject: [PATCH 0530/1004] Examples in --help for remaining commands, closes #384 --- docs/cli-reference.rst | 135 ++++++++++++++++++++++---- sqlite_utils/cli.py | 216 ++++++++++++++++++++++++++++++++++------- 2 files changed, 297 insertions(+), 54 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 868cc3b..583f7e2 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -605,6 +605,10 @@ See :ref:`cli_views`. List the views in the database + Example: + + sqlite-utils views trees.db + Options: --counts Include row counts per view --nl Output newline-delimited JSON @@ -637,6 +641,10 @@ See :ref:`cli_rows`. Output all rows in the specified table + Example: + + sqlite-utils rows trees.db Trees + Options: -c, --column TEXT Columns to return --where TEXT Optional where clause @@ -671,6 +679,10 @@ See :ref:`cli_triggers`. Show triggers configured in this database + Example: + + sqlite-utils triggers trees.db + Options: --nl Output newline-delimited JSON --arrays Output rows as arrays instead of objects @@ -698,7 +710,11 @@ See :ref:`cli_indexes`. Usage: sqlite-utils indexes [OPTIONS] PATH [TABLES]... - Show indexes for this database + Show indexes for the whole database or specific tables + + Example: + + sqlite-utils indexes trees.db Trees Options: --aux Include auxiliary columns @@ -728,7 +744,11 @@ See :ref:`cli_create_database`. Usage: sqlite-utils create-database [OPTIONS] PATH - Create a new empty database file. + Create a new empty database file + + Example: + + sqlite-utils create-database trees.db Options: --enable-wal Enable WAL mode on the created database @@ -747,11 +767,11 @@ See :ref:`cli_create_table`. Add a table with the specified columns. Columns should be specified using name, type pairs, for example: - sqlite-utils create-table my.db people \ - id integer \ - name text \ - height float \ - photo blob --pk id + sqlite-utils create-table my.db people \ + id integer \ + name text \ + height float \ + photo blob --pk id Options: --pk TEXT Column to use as primary key @@ -774,9 +794,15 @@ See :ref:`cli_create_index`. Usage: sqlite-utils create-index [OPTIONS] PATH TABLE COLUMN... - Add an index to the specified table covering the specified columns. Use - "sqlite-utils create-index mydb -- -column" to specify descending order for a - column. + Add an index to the specified table for the specified columns + + Example: + + sqlite-utils create-index chickens.db chickens name + + To create an index in descending order: + + sqlite-utils create-index chickens.db chickens -- -name Options: --name TEXT Explicit name for the new index @@ -796,7 +822,11 @@ See :ref:`cli_fts`. Usage: sqlite-utils enable-fts [OPTIONS] PATH TABLE COLUMN... - Enable full-text search for specific table and columns + Enable full-text search for specific table and columns" + + Example: + + sqlite-utils enable-fts chickens.db chickens name Options: --fts4 Use FTS4 @@ -817,6 +847,10 @@ populate-fts Re-populate full-text search for specific table and columns + Example: + + sqlite-utils populate-fts chickens.db chickens name + Options: --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. @@ -831,6 +865,10 @@ rebuild-fts Rebuild all or specific full-text search tables + Example: + + sqlite-utils rebuild-fts chickens.db chickens + Options: --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. @@ -845,6 +883,10 @@ disable-fts Disable full-text search for specific table + Example: + + sqlite-utils disable-fts chickens.db chickens + Options: --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. @@ -862,6 +904,10 @@ See :ref:`cli_optimize`. Optimize all full-text search tables and then run VACUUM - should shrink the database file + Example: + + sqlite-utils optimize chickens.db + Options: --no-vacuum Don't run VACUUM --load-extension TEXT SQLite extensions to load @@ -880,6 +926,10 @@ See :ref:`cli_analyze`. Run ANALYZE against the whole database, or against specific named indexes and tables + Example: + + sqlite-utils analyze chickens.db + Options: -h, --help Show this message and exit. @@ -895,6 +945,10 @@ See :ref:`cli_vacuum`. Run VACUUM against the database + Example: + + sqlite-utils vacuum chickens.db + Options: -h, --help Show this message and exit. @@ -910,6 +964,10 @@ See :ref:`cli_dump`. Output a SQL dump of the schema and full contents of the database + Example: + + sqlite-utils dump chickens.db + Options: --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. @@ -927,6 +985,10 @@ See :ref:`cli_add_column`. Add a column to the specified table + Example: + + sqlite-utils add-column chickens.db chickens weight float + Options: --fk TEXT Table to reference as a foreign key --fk-col TEXT Referenced column on that foreign key table - if @@ -946,9 +1008,11 @@ See :ref:`cli_add_foreign_key`. Usage: sqlite-utils add-foreign-key [OPTIONS] PATH TABLE COLUMN [OTHER_TABLE] [OTHER_COLUMN] - Add a new foreign key constraint to an existing table. Example usage: + Add a new foreign key constraint to an existing table - $ sqlite-utils add-foreign-key my.db books author_id authors id + Example: + + sqlite-utils add-foreign-key my.db books author_id authors id WARNING: Could corrupt your database! Back up your database file first. @@ -967,11 +1031,13 @@ See :ref:`cli_add_foreign_keys`. Usage: sqlite-utils add-foreign-keys [OPTIONS] PATH [FOREIGN_KEY]... - Add multiple new foreign key constraints to a database. Example usage: + Add multiple new foreign key constraints to a database - sqlite-utils add-foreign-keys my.db \ - books author_id authors id \ - authors country_id countries id + Example: + + sqlite-utils add-foreign-keys my.db \ + books author_id authors id \ + authors country_id countries id Options: --load-extension TEXT SQLite extensions to load @@ -987,7 +1053,11 @@ See :ref:`cli_index_foreign_keys`. Usage: sqlite-utils index-foreign-keys [OPTIONS] PATH - Ensure every foreign key column has an index on it. + Ensure every foreign key column has an index on it + + Example: + + sqlite-utils index-foreign-keys chickens.db Options: --load-extension TEXT SQLite extensions to load @@ -1005,6 +1075,10 @@ See :ref:`cli_wal`. Enable WAL for database files + Example: + + sqlite-utils enable-wal chickens.db + Options: --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. @@ -1019,6 +1093,10 @@ disable-wal Disable WAL for database files + Example: + + sqlite-utils disable-wal chickens.db + Options: --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. @@ -1035,6 +1113,10 @@ See :ref:`cli_enable_counts`. Configure triggers to update a _counts table with row counts + Example: + + sqlite-utils enable-counts chickens.db + Options: --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. @@ -1049,6 +1131,10 @@ reset-counts Reset calculated counts in the _counts table + Example: + + sqlite-utils reset-counts chickens.db + Options: --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. @@ -1065,6 +1151,10 @@ See :ref:`cli_drop_table`. Drop the specified table + Example: + + sqlite-utils drop-table chickens.db chickens + Options: --ignore --load-extension TEXT SQLite extensions to load @@ -1082,6 +1172,11 @@ See :ref:`cli_create_view`. Create a view for the provided SELECT query + Example: + + sqlite-utils create-view chickens.db heavy_chickens \ + 'select * from chickens where weight > 3' + Options: --ignore If view already exists, do nothing --replace If view already exists, replace it @@ -1100,6 +1195,10 @@ See :ref:`cli_drop_view`. Drop the specified view + Example: + + sqlite-utils drop-view chickens.db heavy_chickens + Options: --ignore --load-extension TEXT SQLite extensions to load diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 42fc8e8..350f9e3 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -250,7 +250,13 @@ def views( schema, load_extension, ): - """List the views in the database""" + """List the views in the database + + Example: + + \b + sqlite-utils views trees.db + """ tables.callback( path=path, fts4=False, @@ -281,7 +287,13 @@ def views( @click.option("--no-vacuum", help="Don't run VACUUM", default=False, is_flag=True) @load_extension_option def optimize(path, tables, no_vacuum, load_extension): - """Optimize all full-text search tables and then run VACUUM - should shrink the database file""" + """Optimize all full-text search tables and then run VACUUM - should shrink the database file + + Example: + + \b + sqlite-utils optimize chickens.db + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) if not tables: @@ -302,7 +314,13 @@ def optimize(path, tables, no_vacuum, load_extension): @click.argument("tables", nargs=-1) @load_extension_option def rebuild_fts(path, tables, load_extension): - """Rebuild all or specific full-text search tables""" + """Rebuild all or specific full-text search tables + + Example: + + \b + sqlite-utils rebuild-fts chickens.db chickens + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) if not tables: @@ -320,7 +338,13 @@ def rebuild_fts(path, tables, load_extension): ) @click.argument("names", nargs=-1) def analyze(path, names): - """Run ANALYZE against the whole database, or against specific named indexes and tables""" + """Run ANALYZE against the whole database, or against specific named indexes and tables + + Example: + + \b + sqlite-utils analyze chickens.db + """ db = sqlite_utils.Database(path) try: if names: @@ -339,7 +363,13 @@ def analyze(path, names): required=True, ) def vacuum(path): - """Run VACUUM against the database""" + """Run VACUUM against the database + + Example: + + \b + sqlite-utils vacuum chickens.db + """ sqlite_utils.Database(path).vacuum() @@ -351,7 +381,13 @@ def vacuum(path): ) @load_extension_option def dump(path, load_extension): - """Output a SQL dump of the schema and full contents of the database""" + """Output a SQL dump of the schema and full contents of the database + + Example: + + \b + sqlite-utils dump chickens.db + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) for line in db.conn.iterdump(): @@ -392,7 +428,13 @@ def dump(path, load_extension): def add_column( path, table, col_name, col_type, fk, fk_col, not_null_default, load_extension ): - "Add a column to the specified table" + """Add a column to the specified table + + Example: + + \b + sqlite-utils add-column chickens.db chickens weight float + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) db[table].add_column( @@ -420,9 +462,11 @@ def add_foreign_key( path, table, column, other_table, other_column, ignore, load_extension ): """ - Add a new foreign key constraint to an existing table. Example usage: + Add a new foreign key constraint to an existing table - $ sqlite-utils add-foreign-key my.db books author_id authors id + Example: + + sqlite-utils add-foreign-key my.db books author_id authors id WARNING: Could corrupt your database! Back up your database file first. """ @@ -444,12 +488,14 @@ def add_foreign_key( @load_extension_option def add_foreign_keys(path, foreign_key, load_extension): """ - Add multiple new foreign key constraints to a database. Example usage: + Add multiple new foreign key constraints to a database + + Example: \b - sqlite-utils add-foreign-keys my.db \\ - books author_id authors id \\ - authors country_id countries id + sqlite-utils add-foreign-keys my.db \\ + books author_id authors id \\ + authors country_id countries id """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) @@ -475,7 +521,12 @@ def add_foreign_keys(path, foreign_key, load_extension): @load_extension_option def index_foreign_keys(path, load_extension): """ - Ensure every foreign key column has an index on it. + Ensure every foreign key column has an index on it + + Example: + + \b + sqlite-utils index-foreign-keys chickens.db """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) @@ -508,9 +559,17 @@ def create_index( path, table, column, name, unique, if_not_exists, analyze, load_extension ): """ - Add an index to the specified table covering the specified columns. - Use "sqlite-utils create-index mydb -- -column" to specify descending - order for a column. + Add an index to the specified table for the specified columns + + Example: + + \b + sqlite-utils create-index chickens.db chickens name + + To create an index in descending order: + + \b + sqlite-utils create-index chickens.db chickens -- -name """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) @@ -550,7 +609,13 @@ def create_index( def enable_fts( path, table, column, fts4, fts5, tokenize, create_triggers, load_extension ): - "Enable full-text search for specific table and columns" + """Enable full-text search for specific table and columns" + + Example: + + \b + sqlite-utils enable-fts chickens.db chickens name + """ fts_version = "FTS5" if fts4 and fts5: click.echo("Can only use one of --fts4 or --fts5", err=True) @@ -578,7 +643,13 @@ def enable_fts( @click.argument("column", nargs=-1, required=True) @load_extension_option def populate_fts(path, table, column, load_extension): - "Re-populate full-text search for specific table and columns" + """Re-populate full-text search for specific table and columns + + Example: + + \b + sqlite-utils populate-fts chickens.db chickens name + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) db[table].populate_fts(column) @@ -593,7 +664,13 @@ def populate_fts(path, table, column, load_extension): @click.argument("table") @load_extension_option def disable_fts(path, table, load_extension): - "Disable full-text search for specific table" + """Disable full-text search for specific table + + Example: + + \b + sqlite-utils disable-fts chickens.db chickens + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) db[table].disable_fts() @@ -608,7 +685,13 @@ def disable_fts(path, table, load_extension): ) @load_extension_option def enable_wal(path, load_extension): - "Enable WAL for database files" + """Enable WAL for database files + + Example: + + \b + sqlite-utils enable-wal chickens.db + """ for path_ in path: db = sqlite_utils.Database(path_) _load_extensions(db, load_extension) @@ -624,7 +707,13 @@ def enable_wal(path, load_extension): ) @load_extension_option def disable_wal(path, load_extension): - "Disable WAL for database files" + """Disable WAL for database files + + Example: + + \b + sqlite-utils disable-wal chickens.db + """ for path_ in path: db = sqlite_utils.Database(path_) _load_extensions(db, load_extension) @@ -640,7 +729,13 @@ def disable_wal(path, load_extension): @click.argument("tables", nargs=-1) @load_extension_option def enable_counts(path, tables, load_extension): - "Configure triggers to update a _counts table with row counts" + """Configure triggers to update a _counts table with row counts + + Example: + + \b + sqlite-utils enable-counts chickens.db + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) if not tables: @@ -662,7 +757,13 @@ def enable_counts(path, tables, load_extension): ) @load_extension_option def reset_counts(path, load_extension): - "Reset calculated counts in the _counts table" + """Reset calculated counts in the _counts table + + Example: + + \b + sqlite-utils reset-counts chickens.db + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) db.reset_counts() @@ -1228,7 +1329,13 @@ def bulk( "--enable-wal", is_flag=True, help="Enable WAL mode on the created database" ) def create_database(path, enable_wal): - "Create a new empty database file." + """Create a new empty database file + + Example: + + \b + sqlite-utils create-database trees.db + """ db = sqlite_utils.Database(path) if enable_wal: db.enable_wal() @@ -1280,11 +1387,11 @@ def create_table( name, type pairs, for example: \b - sqlite-utils create-table my.db people \\ - id integer \\ - name text \\ - height float \\ - photo blob --pk id + sqlite-utils create-table my.db people \\ + id integer \\ + name text \\ + height float \\ + photo blob --pk id """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) @@ -1329,7 +1436,13 @@ def create_table( @click.option("--ignore", is_flag=True) @load_extension_option def drop_table(path, table, ignore, load_extension): - "Drop the specified table" + """Drop the specified table + + Example: + + \b + sqlite-utils drop-table chickens.db chickens + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) try: @@ -1358,7 +1471,14 @@ def drop_table(path, table, ignore, load_extension): ) @load_extension_option def create_view(path, view, select, ignore, replace, load_extension): - "Create a view for the provided SELECT query" + """Create a view for the provided SELECT query + + Example: + + \b + sqlite-utils create-view chickens.db heavy_chickens \\ + 'select * from chickens where weight > 3' + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) # Does view already exist? @@ -1386,7 +1506,13 @@ def create_view(path, view, select, ignore, replace, load_extension): @click.option("--ignore", is_flag=True) @load_extension_option def drop_view(path, view, ignore, load_extension): - "Drop the specified view" + """Drop the specified view + + Example: + + \b + sqlite-utils drop-view chickens.db heavy_chickens + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) try: @@ -1808,7 +1934,13 @@ def rows( json_cols, load_extension, ): - "Output all rows in the specified table" + """Output all rows in the specified table + + Example: + + \b + sqlite-utils rows trees.db Trees + """ columns = "*" if column: columns = ", ".join("[{}]".format(c) for c in column) @@ -1860,7 +1992,13 @@ def triggers( json_cols, load_extension, ): - "Show triggers configured in this database" + """Show triggers configured in this database + + Example: + + \b + sqlite-utils triggers trees.db + """ sql = "select name, tbl_name as [table], sql from sqlite_master where type = 'trigger'" if tables: quote = sqlite_utils.Database(memory=True).quote @@ -1909,7 +2047,13 @@ def indexes( json_cols, load_extension, ): - "Show indexes for this database" + """Show indexes for the whole database or specific tables + + Example: + + \b + sqlite-utils indexes trees.db Trees + """ sql = """ select sqlite_master.name as "table", From 2b20957b1869b5a2213960d2e3a67188f42d2a2f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Jan 2022 19:19:39 -0800 Subject: [PATCH 0531/1004] Better validation for upsert --pk, closes #390 --- docs/cli-reference.rst | 3 +- sqlite_utils/cli.py | 118 +++++++++++++++++++++-------------------- tests/test_cli.py | 17 ++++++ 3 files changed, 79 insertions(+), 59 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 583f7e2..eee907e 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -253,8 +253,6 @@ See :ref:`cli_upsert`. incoming record has a primary key that matches an existing record the existing record will be updated. - The --pk option is required. - Example: echo '[ @@ -264,6 +262,7 @@ See :ref:`cli_upsert`. Options: --pk TEXT Columns to use as the primary key, e.g. id + [required] --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 350f9e3..65a108f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -813,59 +813,65 @@ def import_options(fn): return fn -def insert_upsert_options(fn): - for decorator in reversed( - ( - click.argument( - "path", - type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), - required=True, - ), - click.argument("table"), - click.argument("file", type=click.File("rb"), required=True), - click.option( - "--pk", help="Columns to use as the primary key, e.g. id", multiple=True - ), - ) - + _import_options - + ( - click.option( - "--batch-size", type=int, default=100, help="Commit every X records" - ), - click.option( - "--alter", - is_flag=True, - help="Alter existing table to add any missing columns", - ), - click.option( - "--not-null", - multiple=True, - help="Columns that should be created as NOT NULL", - ), - click.option( - "--default", - multiple=True, - type=(str, str), - help="Default value that should be set for a column", - ), - click.option( - "-d", - "--detect-types", - is_flag=True, - envvar="SQLITE_UTILS_DETECT_TYPES", - help="Detect types for columns in CSV/TSV data", - ), - click.option( - "--analyze", - is_flag=True, - help="Run ANALYZE at the end of this operation", - ), - load_extension_option, - click.option("--silent", is_flag=True, help="Do not show progress bar"), - ) - ): - fn = decorator(fn) - return fn +def insert_upsert_options(*, require_pk=False): + def inner(fn): + for decorator in reversed( + ( + click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, + ), + click.argument("table"), + click.argument("file", type=click.File("rb"), required=True), + click.option( + "--pk", + help="Columns to use as the primary key, e.g. id", + multiple=True, + required=require_pk, + ), + ) + + _import_options + + ( + click.option( + "--batch-size", type=int, default=100, help="Commit every X records" + ), + click.option( + "--alter", + is_flag=True, + help="Alter existing table to add any missing columns", + ), + click.option( + "--not-null", + multiple=True, + help="Columns that should be created as NOT NULL", + ), + click.option( + "--default", + multiple=True, + type=(str, str), + help="Default value that should be set for a column", + ), + click.option( + "-d", + "--detect-types", + is_flag=True, + envvar="SQLITE_UTILS_DETECT_TYPES", + help="Detect types for columns in CSV/TSV data", + ), + click.option( + "--analyze", + is_flag=True, + help="Run ANALYZE at the end of this operation", + ), + load_extension_option, + click.option("--silent", is_flag=True, help="Do not show progress bar"), + ) + ): + fn = decorator(fn) + return fn + + return inner def insert_upsert_implementation( @@ -1060,7 +1066,7 @@ def _find_variables(tb, vars): @cli.command() -@insert_upsert_options +@insert_upsert_options() @click.option( "--ignore", is_flag=True, default=False, help="Ignore records if pk already exists" ) @@ -1168,7 +1174,7 @@ def insert( @cli.command() -@insert_upsert_options +@insert_upsert_options(require_pk=True) def upsert( path, table, @@ -1201,8 +1207,6 @@ def upsert( an incoming record has a primary key that matches an existing record the existing record will be updated. - The --pk option is required. - Example: \b diff --git a/tests/test_cli.py b/tests/test_cli.py index 60c95b0..21b7945 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -922,6 +922,23 @@ def test_upsert(db_path, tmpdir): ] +def test_upsert_pk_required(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + db = Database(db_path) + insert_dogs = [ + {"id": 1, "name": "Cleo", "age": 4}, + {"id": 2, "name": "Nixie", "age": 4}, + ] + open(json_path, "w").write(json.dumps(insert_dogs)) + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "dogs", json_path], + catch_exceptions=False, + ) + assert result.exit_code == 2 + assert "Error: Missing option '--pk'" in result.output + + def test_upsert_analyze(db_path, tmpdir): db = Database(db_path) db["rows"].insert({"id": 1, "foo": "x", "n": 3}, pk="id") From be1e89da5fa6f28b8910610aa9f2b95f1fe3168b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Jan 2022 19:22:06 -0800 Subject: [PATCH 0532/1004] Fixed flake8 errors --- sqlite_utils/cli.py | 2 +- tests/test_cli.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 65a108f..07a55b3 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -489,7 +489,7 @@ def add_foreign_key( def add_foreign_keys(path, foreign_key, load_extension): """ Add multiple new foreign key constraints to a database - + Example: \b diff --git a/tests/test_cli.py b/tests/test_cli.py index 21b7945..244029d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -924,7 +924,6 @@ def test_upsert(db_path, tmpdir): def test_upsert_pk_required(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") - db = Database(db_path) insert_dogs = [ {"id": 1, "name": "Cleo", "age": 4}, {"id": 2, "name": "Nixie", "age": 4}, From a9fca7efa4184fbb2a65ca1275c326950ed9d3c1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Jan 2022 19:28:30 -0800 Subject: [PATCH 0533/1004] Release 3.22.1 Refs #384, #387, #389 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0af3c04..f9cca0c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v3_22_1: + +3.22.1 (2022-01-25) +------------------- + +- All commands now include example usage in their ``--help`` - see :ref:`cli_reference`. (:issue:`384`) +- Python library documentation has a new :ref:`python_api_getting_started` section. (:issue:`387`) +- Documentation now uses `Plausible analytics `__. (:issue:`389`) + .. _v3_22: 3.22 (2022-01-11) diff --git a/setup.py b/setup.py index e0aa54b..e3a31d4 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.22" +VERSION = "3.22.1" def get_long_description(): From d1d2a8e6fa95d8daf11973f747578602d08e4962 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Jan 2022 10:15:23 -0800 Subject: [PATCH 0534/1004] sqlite-utils bulk --batch-size option, closes #392 --- docs/cli-reference.rst | 1 + docs/cli.rst | 2 ++ sqlite_utils/cli.py | 16 ++++++++++++---- tests/test_cli_bulk.py | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index eee907e..89318d7 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -309,6 +309,7 @@ See :ref:`cli_bulk`. ' - Options: + --batch-size INTEGER Commit every X records --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON diff --git a/docs/cli.rst b/docs/cli.rst index 733eee8..3ceffce 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1120,6 +1120,8 @@ You could insert those rows into a pre-created ``chickens`` table like so:: This command takes the same options as the ``sqlite-utils insert`` command - so it defaults to expecting JSON but can accept other formats using ``--csv`` or ``--tsv`` or ``--nl`` or other options described above. +By default all of the SQL queries will be executed in a single transaction. To commit every 20 records, use ``--batch-size 20``. + .. _cli_insert_files: Inserting data from files diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 07a55b3..771d432 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -18,6 +18,7 @@ import csv as csv_std import tabulate from .utils import ( _compile_code, + chunks, file_progress, find_spatialite, sqlite3, @@ -1013,8 +1014,13 @@ def insert_upsert_implementation( # For bulk_sql= we use cursor.executemany() instead if bulk_sql: - with db.conn: - db.conn.cursor().executemany(bulk_sql, docs) + if batch_size: + doc_chunks = chunks(docs, batch_size) + else: + doc_chunks = [docs] + for doc_chunk in doc_chunks: + with db.conn: + db.conn.cursor().executemany(bulk_sql, doc_chunk) return try: @@ -1256,12 +1262,14 @@ def upsert( ) @click.argument("sql") @click.argument("file", type=click.File("rb"), required=True) +@click.option("--batch-size", type=int, default=100, help="Commit every X records") @import_options @load_extension_option def bulk( path, - file, sql, + file, + batch_size, flatten, nl, csv, @@ -1309,7 +1317,7 @@ def bulk( sniff=sniff, no_headers=no_headers, encoding=encoding, - batch_size=1, + batch_size=batch_size, alter=False, upsert=False, not_null=set(), diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py index 37cf60b..c5dc9da 100644 --- a/tests/test_cli_bulk.py +++ b/tests/test_cli_bulk.py @@ -1,7 +1,11 @@ +from itertools import count from click.testing import CliRunner from sqlite_utils import cli, Database import pathlib import pytest +import subprocess +import sys +import time @pytest.fixture @@ -40,6 +44,41 @@ def test_cli_bulk(test_db_and_path): ] == list(db["example"].rows) +def test_cli_bulk_batch_size(test_db_and_path): + db, db_path = test_db_and_path + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "sqlite_utils", + "bulk", + db_path, + "insert into example (id, name) values (:id, :name)", + "-", + "--nl", + "--batch-size", + "2", + ], + stdin=subprocess.PIPE, + stdout=sys.stdout, + ) + # Writing one record should not commit + proc.stdin.write(b'{"id": 3, "name": "Three"}\n\n') + proc.stdin.flush() + time.sleep(1) + assert db["example"].count == 2 + + # Writing another should trigger a commit: + proc.stdin.write(b'{"id": 4, "name": "Four"}\n\n') + proc.stdin.flush() + time.sleep(1) + assert db["example"].count == 4 + + proc.stdin.close() + proc.wait() + assert proc.returncode == 0 + + def test_cli_bulk_error(test_db_and_path): _, db_path = test_db_and_path result = CliRunner().invoke( From d1e9f09c06f29f27eca5c1a06a75072e28a79f0d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Jan 2022 10:23:48 -0800 Subject: [PATCH 0535/1004] Removed unneccessary import, refs #392 --- tests/test_cli_bulk.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py index c5dc9da..6d01952 100644 --- a/tests/test_cli_bulk.py +++ b/tests/test_cli_bulk.py @@ -1,4 +1,3 @@ -from itertools import count from click.testing import CliRunner from sqlite_utils import cli, Database import pathlib From 6663d28952491aca2c8dcf586a301fb4791b5f69 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 30 Jan 2022 07:17:20 -0800 Subject: [PATCH 0536/1004] SQL injection, not XSS --- docs/python-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 6b703fe..ac835d4 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -206,7 +206,7 @@ Passing parameters Both ``db.query()`` and ``db.execute()`` accept an optional second argument for parameters to be passed to the SQL query. -This can take the form of either a tuple/list or a dictionary, depending on the type of parameters used in the query. Values passed in this way will be correctly quoted and escaped, helping avoid XSS vulnerabilities. +This can take the form of either a tuple/list or a dictionary, depending on the type of parameters used in the query. Values passed in this way will be correctly quoted and escaped, helping avoid SQL injection vulnerabilities. ``?`` parameters in the SQL query can be filled in using a list: From feb01c1ddd2ba0a3c01518b6856520470d649bae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 30 Jan 2022 07:22:39 -0800 Subject: [PATCH 0537/1004] Fixed duplicated example --- docs/python-api.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index ac835d4..6efec95 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -424,7 +424,6 @@ The ``db.schema`` property returns the full SQL schema for the database as a str >>> db = sqlite_utils.Database("dogs.db") >>> print(db.schema) - >>> print(db.schema) CREATE TABLE "dogs" ( [id] INTEGER PRIMARY KEY, [name] TEXT From a6da26a856c966598b2275b12558e65d3e61a682 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 30 Jan 2022 07:24:13 -0800 Subject: [PATCH 0538/1004] Simplified example --- docs/python-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 6efec95..72177c8 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -441,7 +441,7 @@ The easiest way to create a new table is to insert a record into it: from sqlite_utils import Database import sqlite3 - db = Database(sqlite3.connect("/tmp/dogs.db")) + db = Database("dogs.db") dogs = db["dogs"] dogs.insert({ "name": "Cleo", From 44cbddff8ab6526f20f608e4d76592422af757bd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Feb 2022 14:21:38 -0800 Subject: [PATCH 0539/1004] Run tests against Python 3.11-dev Refs #394 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f92538..59f7532 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11-dev"] numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest] steps: From b2ab08e048228c3938b973dee12adb18729ebe39 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 13:07:00 -0800 Subject: [PATCH 0540/1004] Don't test main against 3.11-dev yet It breaks on Windows. Refs #394 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 59f7532..1f92538 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11-dev"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest] steps: From 813b6d07ab97209435924311fda94a7fd377bd73 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 14:07:32 -0800 Subject: [PATCH 0541/1004] Much improved insert-replace documentation, refs #393 --- docs/python-api.rst | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 72177c8..df9449f 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -705,7 +705,31 @@ Pass ``analyze=True`` to run ``ANALYZE`` against the table after inserting the n Insert-replacing data ===================== -If you want to insert a record or replace an existing record with the same primary key, using the ``replace=True`` argument to ``.insert()`` or ``.insert_all()``:: +If you try to insert data using a primary key that already exists, the ``.insert()`` or ``.insert_all()`` method will raise a ``sqlite3.IntegrityError`` exception. + +This example that catches that exception: + +.. code-block:: python + + from sqlite_utils.utils import sqlite3 + + try: + db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + except sqlite3.IntegrityError: + print("Record already exists with that primary key") + +Importing from ``sqlite_utils.utils.sqlite3`` ensures your code continues to work even if you are using the ``pysqlite3`` library instead of the Python standard library ``sqlite3`` module. + +Use the ``ignore=True`` parameter to ignore this error: + +.. code-block:: python + + # This fails silently if a record with id=1 already exists + db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id", ignore=True) + +To replace any existing records that have a matching primary key, use the ``replace=True`` parameter to ``.insert()`` or ``.insert_all()``: + +.. code-block:: python db["dogs"].insert_all([{ "id": 1, @@ -722,7 +746,7 @@ If you want to insert a record or replace an existing record with the same prima }], pk="id", replace=True) .. note:: - Prior to sqlite-utils 2.x the ``.upsert()`` and ``.upsert_all()`` methods did this. See :ref:`python_api_upsert` for the new behaviour of those methods in 2.x. + Prior to sqlite-utils 2.0 the ``.upsert()`` and ``.upsert_all()`` methods worked the same way as ``.insert(replace=True)`` does today. See :ref:`python_api_upsert` for the new behaviour of those methods introduced in 2.0. .. _python_api_update: From 7d928f83085fb285f294dbdaeb93fd94a44d5d44 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 14:11:25 -0800 Subject: [PATCH 0542/1004] Better insert-replace CLI documentation, refs #393 --- docs/cli.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 3ceffce..d8aed91 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1060,15 +1060,13 @@ The result looks like this:: Insert-replacing data ===================== -Insert-replacing works exactly like inserting, with the exception that if your data has a primary key that matches an already existing record that record will be replaced with the new data. +The ``--replace`` option to ``insert`` causes any existing records with the same primary key to be replaced entirely by the new records. -After running the above ``dogs.json`` example, try running this:: +To replace a dog with in ID of 2 with a new record, run the following:: $ echo '{"id": 2, "name": "Pancakes", "age": 3}' | \ sqlite-utils insert dogs.db dogs - --pk=id --replace -This will replace the record for id=2 (Pancakes) with a new record with an updated age. - .. _cli_upsert: Upserting data @@ -1083,7 +1081,7 @@ For example:: $ echo '{"id": 2, "age": 4}' | \ sqlite-utils upsert dogs.db dogs - --pk=id -This will update the dog with id=2 to have an age of 4, creating a new record (with a null name) if one does not exist. If a row DOES exist the name will be left as-is. +This will update the dog with an ID of 2 to have an age of 4, creating a new record (with a null name) if one does not exist. If a row DOES exist the name will be left as-is. The command will fail if you reference columns that do not exist on the table. To automatically create missing columns, use the ``--alter`` option. From 9dcb099905c4a2246e3487be3289642161991864 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 14:51:25 -0800 Subject: [PATCH 0543/1004] Better error messages for --convert, closes #363 --- sqlite_utils/cli.py | 11 +++++++++++ tests/test_cli_insert.py | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 771d432..8b1b5a4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1009,6 +1009,9 @@ def insert_upsert_implementation( if upsert: extra_kwargs["upsert"] = upsert + # docs should all be dictionaries + docs = (verify_is_dict(doc) for doc in docs) + # Apply {"$base64": true, ...} decoding, if needed docs = (decode_base64_values(doc) for doc in docs) @@ -2760,6 +2763,14 @@ def json_binary(value): raise TypeError +def verify_is_dict(doc): + if not isinstance(doc, dict): + raise click.ClickException( + "Rows must all be dictionaries, got: {}".format(repr(doc)[:1000]) + ) + return doc + + def _load_extensions(db, load_extension): if load_extension: db.conn.enable_load_extension(True) diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 0d6b482..45ad362 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -472,6 +472,32 @@ def test_insert_convert_row_modifying_in_place(db_path): assert rows == [{"name": "Azi", "is_chicken": 1}] +@pytest.mark.parametrize( + "options,expected_error", + ( + ( + ["--text", "--convert", "1"], + "Error: --convert must return dict or iterator\n", + ), + (["--convert", "1"], "Error: Rows must all be dictionaries, got: 1\n"), + ), +) +def test_insert_convert_error_messages(db_path, options, expected_error): + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "rows", + "-", + ] + + options, + input='{"name": "Azi"}', + ) + assert result.exit_code == 1 + assert result.output == expected_error + + def test_insert_streaming_batch_size_1(db_path): # https://github.com/simonw/sqlite-utils/issues/364 # Streaming with --batch-size 1 should commit on each record From ee11274fcb1c00f32c95f2ef2924d5349538eb4d Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Fri, 4 Feb 2022 00:55:09 -0500 Subject: [PATCH 0544/1004] New spatialite helper methods, closes #79 - db.init_spatialite() - table.add_geometry_column() - table.create_spatial_index() Co-authored-by: Simon Willison --- docs/python-api.rst | 57 ++++++++++++-------- sqlite_utils/cli.py | 1 + sqlite_utils/db.py | 119 ++++++++++++++++++++++++++++++++++++++++++ sqlite_utils/utils.py | 35 ++++++++++--- tests/test_gis.py | 83 +++++++++++++++++++++++++++++ tests/test_utils.py | 5 -- 6 files changed, 267 insertions(+), 33 deletions(-) create mode 100644 tests/test_gis.py diff --git a/docs/python-api.rst b/docs/python-api.rst index df9449f..4da350b 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -352,7 +352,7 @@ Counting rows To count the number of rows that would be returned by a where filter, use ``.count_where(where, where_args)``: - >>> db["dogs"].count_where("age > ?", [1]): + >>> db["dogs"].count_where("age > ?", [1]) 2 .. _python_api_pks_and_rows_where: @@ -2422,26 +2422,6 @@ For example: # [thumbnail] BLOB # ) -.. _find_spatialite: - -Finding SpatiaLite -================== - -The ``find_spatialite()`` function searches for the `SpatiaLite `__ SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found. - -You can use it in code like this: - -.. code-block:: python - - from sqlite_utils import Database - from sqlite_utils.utils import find_spatialite - - db = Database("mydb.db") - spatialite = find_spatialite() - if spatialite: - db.conn.enable_load_extension(True) - db.conn.load_extension(spatialite) - .. _python_api_register_function: Registering custom SQL functions @@ -2522,3 +2502,38 @@ If that option isn't relevant to your use-case you can to quote a string for use "'hello'" >>> db.quote("hello'this'has'quotes") "'hello''this''has''quotes'" + +.. _python_api_gis: + +Spatialite helpers +================== + +`SpatiaLite `__ is a geographic extension to SQLite (similar to PostgreSQL + PostGIS). Using requires finding, loading and initializing the extension, adding geometry columns to existing tables and optionally creating spatial indexes. The utilities here help streamline that setup. + +.. _python_api_gis_init_spatialite: + +Initialize Spatialite +--------------------- + +.. automethod:: sqlite_utils.db.Database.init_spatialite + +.. _python_api_gis_find_spatialite: + +Finding Spatialite +------------------ + +.. autofunction:: sqlite_utils.utils.find_spatialite + +.. _python_api_gis_add_geometry_column: + +Adding geometry columns +----------------------- + +.. automethod:: sqlite_utils.db.Table.add_geometry_column + +.. _python_api_gis_create_spatial_index: + +Creating a spatial index +------------------------ + +.. automethod:: sqlite_utils.db.Table.create_spatial_index diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8b1b5a4..b137d9b 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -29,6 +29,7 @@ from .utils import ( TypeTracker, ) + CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB") diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 06e7557..4ab8862 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -6,6 +6,7 @@ from .utils import ( types_for_column_types, column_affinity, progressbar, + find_spatialite, ) from collections import namedtuple from collections.abc import Mapping @@ -931,6 +932,46 @@ class Database: sql += " [{}]".format(name) self.execute(sql) + def init_spatialite(self, path: str = None) -> bool: + """ + The ``init_spatialite`` method will load and initialize the SpatiaLite extension. + The ``path`` argument should be an absolute path to the compiled extension, which + can be found using ``find_spatialite``. + + Returns true if SpatiaLite was successfully initialized. + + .. code-block:: python + + from sqlite_utils.db import Database + from sqlite_utils.utils import find_spatialite + + db = Database("mydb.db") + db.init_spatialite(find_spatialite()) + + If you've installed SpatiaLite somewhere unexpected (for testing an alternate version, for example) + you can pass in an absolute path: + + .. code-block:: python + + from sqlite_utils.db import Database + from sqlite_utils.utils import find_spatialite + + db = Database("mydb.db") + db.init_spatialite("./local/mod_spatialite.dylib") + + """ + if path is None: + path = find_spatialite() + + self.conn.enable_load_extension(True) + self.conn.load_extension(path) + # Initialize SpatiaLite if not yet initialized + if "spatial_ref_sys" in self.table_names(): + return False + cursor = self.execute("select InitSpatialMetadata(1)") + result = cursor.fetchone() + return result and bool(result[0]) + class Queryable: def exists(self) -> bool: @@ -3012,6 +3053,84 @@ class Table(Queryable): least_common, ) + def add_geometry_column( + self, + column_name: str, + geometry_type: str, + srid: int = 4326, + coord_dimension: str = "XY", + not_null: bool = False, + ) -> bool: + """ + In SpatiaLite, a geometry column can only be added to an existing table. + To do so, use ``table.add_geometry_column``, passing in a geometry type. + + By default, this will add a nullable column using + `SRID 4326 `__. This can + be customized using the ``column_name``, ``srid`` and ``not_null`` arguments. + + Returns True if the column was successfully added, False if not. + + .. code-block:: python + + from sqlite_utils.db import Database + from sqlite_utils.utils import find_spatialite + + db = Database("mydb.db") + db.init_spatialite(find_spatialite()) + + # the table must exist before adding a geometry column + table = db["locations"].create({"name": str}) + table.add_geometry_column("geometry", "POINT") + + """ + cursor = self.db.execute( + "SELECT AddGeometryColumn(?, ?, ?, ?, ?, ?);", + [ + self.name, + column_name, + srid, + geometry_type, + coord_dimension, + int(not_null), + ], + ) + + result = cursor.fetchone() + return result and bool(result[0]) + + def create_spatial_index(self, column_name) -> bool: + """ + A spatial index allows for significantly faster bounding box queries. + To create one, use ``create_spatial_index`` with the name of an existing geometry column. + + Returns ``True`` if the index was successfully created, ``False`` if not. Calling this + function if an index already exists is a no-op. + + .. code-block:: python + + # assuming SpatiaLite is loaded, create the table, add the column + table = db["locations"].create({"name": str}) + table.add_geometry_column("geometry", "POINT") + + # now we can index it + table.create_spatial_index("geometry") + + # the spatial index is a virtual table, which we can inspect + print(db["idx_locations_geometry"].schema) + # outputs: + # CREATE VIRTUAL TABLE "idx_locations_geometry" USING rtree(pkid, xmin, xmax, ymin, ymax) + + """ + if f"idx_{self.name}_{column_name}" in self.db.table_names(): + return False + + cursor = self.db.execute( + "select CreateSpatialIndex(?, ?)", [self.name, column_name] + ) + result = cursor.fetchone() + return result and bool(result[0]) + class View(Queryable): def exists(self): diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 0777f30..c2a7c91 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -22,12 +22,40 @@ except ImportError: OperationalError = sqlite3.OperationalError + SPATIALITE_PATHS = ( "/usr/lib/x86_64-linux-gnu/mod_spatialite.so", "/usr/local/lib/mod_spatialite.dylib", ) +def find_spatialite() -> str: + """ + The ``find_spatialite()`` function searches for the `SpatiaLite `__ SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found. + + You can use it in code like this: + + .. code-block:: python + + from sqlite_utils import Database + from sqlite_utils.utils import find_spatialite + + db = Database("mydb.db") + spatialite = find_spatialite() + if spatialite: + db.conn.enable_load_extension(True) + db.conn.load_extension(spatialite) + + # or use with db.init_spatialite like this + db.init_spatialite(find_spatialite()) + + """ + for path in SPATIALITE_PATHS: + if os.path.exists(path): + return path + return None + + def suggest_column_types(records): all_column_types = {} for record in records: @@ -96,13 +124,6 @@ def decode_base64_values(doc): return dict(doc, **{k: base64.b64decode(doc[k]["encoded"]) for k in to_fix}) -def find_spatialite(): - for path in SPATIALITE_PATHS: - if os.path.exists(path): - return path - return None - - class UpdateWrapper: def __init__(self, wrapped, update): self._wrapped = wrapped diff --git a/tests/test_gis.py b/tests/test_gis.py new file mode 100644 index 0000000..3bf0d5f --- /dev/null +++ b/tests/test_gis.py @@ -0,0 +1,83 @@ +import pytest +from sqlite_utils.utils import find_spatialite +from sqlite_utils.db import Database +from sqlite_utils.utils import sqlite3 + +pytestmark = [ + pytest.mark.skipif( + not find_spatialite(), reason="Could not find SpatiaLite extension" + ), + pytest.mark.skipif( + not hasattr(sqlite3.Connection, "enable_load_extension"), + reason="sqlite3.Connection missing enable_load_extension", + ), +] + + +def test_find_spatialite(): + spatialite = find_spatialite() + assert spatialite is None or isinstance(spatialite, str) + + +def test_init_spatialite(): + db = Database(memory=True) + spatialite = find_spatialite() + db.init_spatialite(spatialite) + assert "spatial_ref_sys" in db.table_names() + + +def test_add_geometry_column(): + db = Database(memory=True) + spatialite = find_spatialite() + db.init_spatialite(spatialite) + + # create a table first + table = db.create_table("locations", {"id": str, "properties": str}) + table.add_geometry_column( + column_name="geometry", + geometry_type="Point", + srid=4326, + coord_dimension=2, + ) + + assert db["geometry_columns"].get(["locations", "geometry"]) == { + "f_table_name": "locations", + "f_geometry_column": "geometry", + "geometry_type": 1, # point + "coord_dimension": 2, + "srid": 4326, + "spatial_index_enabled": 0, + } + + +def test_create_spatial_index(): + db = Database(memory=True) + spatialite = find_spatialite() + assert db.init_spatialite(spatialite) + + # create a table, add a geometry column with default values + table = db.create_table("locations", {"id": str, "properties": str}) + assert table.add_geometry_column("geometry", "Point") + + # index it + assert table.create_spatial_index("geometry") + + assert "idx_locations_geometry" in db.table_names() + + +def test_double_create_spatial_index(): + db = Database(memory=True) + spatialite = find_spatialite() + db.init_spatialite(spatialite) + + # create a table, add a geometry column with default values + table = db.create_table("locations", {"id": str, "properties": str}) + table.add_geometry_column("geometry", "Point") + + # index it, return True + assert table.create_spatial_index("geometry") + + assert "idx_locations_geometry" in db.table_names() + + # call it again, return False + assert not table.create_spatial_index("geometry") diff --git a/tests/test_utils.py b/tests/test_utils.py index 8630e28..66ce413 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -22,11 +22,6 @@ def test_decode_base64_values(input, expected, should_be_is): assert actual == expected -def test_find_spatialite(): - spatialite = utils.find_spatialite() - assert spatialite is None or isinstance(spatialite, str) - - @pytest.mark.parametrize( "size,expected", ( From 4a2a3e2fd0d5534f446b3f1fee34cb165e4d86d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 21:56:34 -0800 Subject: [PATCH 0545/1004] Install SpatiaLite in tests To run tests for #79, #385 --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f92538..2d66dbb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,6 +32,8 @@ jobs: - name: Optionally install numpy if: matrix.numpy == 1 run: pip install numpy + - name: Install SpatiaLite + run: sudo apt-get install libsqlite3-mod-spatialite - name: Run tests run: | pytest -v From cea25c28bad0bd30cf375e2d0d5113f23ab84e0c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 21:59:59 -0800 Subject: [PATCH 0546/1004] Capitalization of SpatiaLite --- docs/python-api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 4da350b..5106307 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2512,14 +2512,14 @@ Spatialite helpers .. _python_api_gis_init_spatialite: -Initialize Spatialite +Initialize SpatiaLite --------------------- .. automethod:: sqlite_utils.db.Database.init_spatialite .. _python_api_gis_find_spatialite: -Finding Spatialite +Finding SpatiaLite ------------------ .. autofunction:: sqlite_utils.utils.find_spatialite From 749418728448abbbfa6305ad18152951a6721670 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 22:05:12 -0800 Subject: [PATCH 0547/1004] Only install SpatiaLite on Ubuntu, refs #395 For tests added to #79 --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d66dbb..3bd21f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,6 +33,7 @@ jobs: if: matrix.numpy == 1 run: pip install numpy - name: Install SpatiaLite + if: matrix.os == "ubuntu-latest" run: sudo apt-get install libsqlite3-mod-spatialite - name: Run tests run: | From e46798959e10e4674b2a58a9c2f227c0a2deca1d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 22:06:23 -0800 Subject: [PATCH 0548/1004] Looks like Actions if: clauses prefer single quotes Refs #395, #79 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3bd21f1..a8274e2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: if: matrix.numpy == 1 run: pip install numpy - name: Install SpatiaLite - if: matrix.os == "ubuntu-latest" + if: matrix.os == 'ubuntu-latest' run: sudo apt-get install libsqlite3-mod-spatialite - name: Run tests run: | From 0fe0f476a73ddbb3fea879bdb6bfef3ba4b97768 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 22:10:09 -0800 Subject: [PATCH 0549/1004] Fix for mypy error, closes #396 Should help tests pass for #395 and #79 --- sqlite_utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index c2a7c91..b719b01 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -29,7 +29,7 @@ SPATIALITE_PATHS = ( ) -def find_spatialite() -> str: +def find_spatialite() -> Optional[str]: """ The ``find_spatialite()`` function searches for the `SpatiaLite `__ SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found. From 482fcc0da7c5127ce5bc6765b63663b9c5a87f91 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 22:13:17 -0800 Subject: [PATCH 0550/1004] Fix for flake8, refs #79 --- sqlite_utils/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index b719b01..e3386d8 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -31,7 +31,8 @@ SPATIALITE_PATHS = ( def find_spatialite() -> Optional[str]: """ - The ``find_spatialite()`` function searches for the `SpatiaLite `__ SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found. + The ``find_spatialite()`` function searches for the `SpatiaLite `__ + SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found. You can use it in code like this: From 44894c6f6c854bb8d5c79cb349aa39526cf56ee2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 22:31:13 -0800 Subject: [PATCH 0551/1004] Fix warning about duplicate object description --- docs/python-api.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 5106307..cc34505 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2505,7 +2505,7 @@ If that option isn't relevant to your use-case you can to quote a string for use .. _python_api_gis: -Spatialite helpers +SpatiaLite helpers ================== `SpatiaLite `__ is a geographic extension to SQLite (similar to PostgreSQL + PostGIS). Using requires finding, loading and initializing the extension, adding geometry columns to existing tables and optionally creating spatial indexes. The utilities here help streamline that setup. @@ -2516,6 +2516,7 @@ Initialize SpatiaLite --------------------- .. automethod:: sqlite_utils.db.Database.init_spatialite + :noindex: .. _python_api_gis_find_spatialite: @@ -2530,6 +2531,7 @@ Adding geometry columns ----------------------- .. automethod:: sqlite_utils.db.Table.add_geometry_column + :noindex: .. _python_api_gis_create_spatial_index: @@ -2537,3 +2539,4 @@ Creating a spatial index ------------------------ .. automethod:: sqlite_utils.db.Table.create_spatial_index + :noindex: From 20fe3b8abf49f8f7d73ad0f5610d2a62541fd907 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 22:32:57 -0800 Subject: [PATCH 0552/1004] Fixed RST warning about empty line --- docs/cli-reference.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 89318d7..15b7b8c 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -56,6 +56,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. "convert": "cli_convert", } commands.sort(key = lambda command: go_first.index(command) if command in go_first else 999) + cog.out("\n") for command in commands: cog.out(command + "\n") cog.out(("=" * len(command)) + "\n\n") @@ -74,6 +75,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. cog.out(textwrap.indent(output, ' ')) cog.out("\n\n") .. ]]] + query ===== From 088d89982299f8136e608fa2b6c30e9529adc714 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Feb 2022 22:41:46 -0800 Subject: [PATCH 0553/1004] Release 3.23 Refs #79, #363, #392, #393, #395, #396 --- docs/changelog.rst | 13 +++++++++++++ setup.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f9cca0c..7f62c62 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,19 @@ Changelog =========== +.. _v3_23: + +3.23 (2022-02-03) +----------------- + +This release introduces four new utility methods for working with `SpatiaLite `__. Thanks, Chris Amico. (`#330 `__) + +- ``sqlite_utils.utils.find_spatialite()`` :ref:`finds the location of the SpatiaLite module ` on disk. +- ``db.init_spatialite()`` :ref:`initializes SpatiaLite ` for the given database. +- ``table.add_geometry_column(...)`` :ref:`adds a geometry column ` to an existing table. +- ``table.create_spatial_index(...)`` :ref:`creates a spatial index ` for a column. +- ``sqlite-utils batch`` now accepts a ``--batch-size`` option. (:issue:`392`) + .. _v3_22_1: 3.22.1 (2022-01-25) diff --git a/setup.py b/setup.py index e3a31d4..6d96533 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.22.1" +VERSION = "3.23" def get_long_description(): From 79b5b58354c35823ebf63cc19ffdfa603ee88d65 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 5 Feb 2022 17:19:39 -0800 Subject: [PATCH 0554/1004] Basic test for db[t].create(...), refs #397 --- tests/test_create.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_create.py b/tests/test_create.py index 82c2c72..d946d99 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1089,3 +1089,28 @@ def test_create_table_sql(fresh_db, columns, expected_sql_middle): sql = fresh_db.create_table_sql("t", columns) middle = sql.split("(")[1].split(")")[0].strip() assert middle == expected_sql_middle + + +def test_create(fresh_db): + fresh_db["t"].create( + { + "id": int, + "text": str, + "float": float, + "integer": int, + "bytes": bytes, + }, + pk="id", + column_order=("id", "float"), + not_null=("float", "integer"), + defaults={"integer": 0}, + ) + assert fresh_db["t"].schema == ( + "CREATE TABLE [t] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [float] FLOAT NOT NULL,\n" + " [text] TEXT,\n" + " [integer] INTEGER NOT NULL DEFAULT 0,\n" + " [bytes] BLOB\n" + ")" + ) From aa2490311369697adbdbef4185b334e6730c762e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 5 Feb 2022 17:28:53 -0800 Subject: [PATCH 0555/1004] Create table if_not_exists=True argument, closes #397 --- docs/python-api.rst | 15 ++++++++++++++- sqlite_utils/db.py | 12 ++++++++++-- tests/test_create.py | 9 +++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index cc34505..039a00a 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -522,7 +522,9 @@ This will create a table with the following schema: Explicitly creating a table --------------------------- -You can directly create a new table without inserting any data into it using the ``.create()`` method:: +You can directly create a new table without inserting any data into it using the ``.create()`` method: + +.. code-block:: python db["cats"].create({ "id": int, @@ -534,6 +536,17 @@ The first argument here is a dictionary specifying the columns you would like to This method takes optional arguments ``pk=``, ``column_order=``, ``foreign_keys=``, ``not_null=set()`` and ``defaults=dict()`` - explained below. +A ``sqlite_utils.utils.sqlite3.OperationalError`` will be raised if a table of that name already exists. + +To do nothing if the table already exists, add ``if_not_exists=True``: + +.. code-block:: python + + db["cats"].create({ + "id": int, + "name": str, + }, pk="id", if_not_exists=True) + .. _python_api_compound_primary_keys: Compound primary keys diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 4ab8862..1927f41 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -658,6 +658,7 @@ class Database: defaults: Optional[Dict[str, Any]] = None, hash_id: Optional[str] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, + if_not_exists: bool = False, ) -> str: "Returns the SQL ``CREATE TABLE`` statement for creating the specified table." foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) @@ -748,11 +749,14 @@ class Database: pks=", ".join(["[{}]".format(p) for p in pk]) ) columns_sql = ",\n".join(column_defs) - sql = """CREATE TABLE [{table}] ( + sql = """CREATE TABLE {if_not_exists}[{table}] ( {columns_sql}{extra_pk} ); """.format( - table=name, columns_sql=columns_sql, extra_pk=extra_pk + if_not_exists="IF NOT EXISTS " if if_not_exists else "", + table=name, + columns_sql=columns_sql, + extra_pk=extra_pk, ) return sql @@ -767,6 +771,7 @@ class Database: defaults: Optional[Dict[str, Any]] = None, hash_id: Optional[str] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, + if_not_exists: bool = False, ) -> "Table": """ Create a table with the specified name and the specified ``{column_name: type}`` columns. @@ -783,6 +788,7 @@ class Database: defaults=defaults, hash_id=hash_id, extracts=extracts, + if_not_exists=if_not_exists, ) self.execute(sql) table = self.table( @@ -1300,6 +1306,7 @@ class Table(Queryable): defaults: Optional[Dict[str, Any]] = None, hash_id: Optional[str] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, + if_not_exists: bool = False, ) -> "Table": """ Create a table with the specified columns. @@ -1318,6 +1325,7 @@ class Table(Queryable): defaults=defaults, hash_id=hash_id, extracts=extracts, + if_not_exists=if_not_exists, ) return self diff --git a/tests/test_create.py b/tests/test_create.py index d946d99..35844a1 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1114,3 +1114,12 @@ def test_create(fresh_db): " [bytes] BLOB\n" ")" ) + + +def test_create_if_not_exists(fresh_db): + fresh_db["t"].create({"id": int}) + # This should error + with pytest.raises(sqlite3.OperationalError): + fresh_db["t"].create({"id": int}) + # This should not + fresh_db["t"].create({"id": int}, if_not_exists=True) From fea8c9bcc509bcae75e99ae8870f520103b9aa58 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 5 Feb 2022 18:03:21 -0800 Subject: [PATCH 0556/1004] Improved SpatiaLite example, closes #401 --- docs/python-api.rst | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 039a00a..149c3a2 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1672,28 +1672,21 @@ A more useful example: if you are working with `SpatiaLite Date: Tue, 8 Feb 2022 11:33:41 -0800 Subject: [PATCH 0557/1004] Adding a primary key to a rowid table, closes #403 --- docs/cli.rst | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index d8aed91..e75adec 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1507,6 +1507,43 @@ If you want to see the SQL that will be executed to make the change without actu DROP TABLE [roadside_attractions]; ALTER TABLE [roadside_attractions_new_4033a60276b9] RENAME TO [roadside_attractions]; +.. _cli_transform_table_add_primary_key_to_rowid: + +Adding a primary key to a rowid table +------------------------------------- + +SQLite tables that are created without an explicit primary key are created as `rowid tables `__. They still have a numeric primary key which is available in the ``rowid`` column, but that column is not included in the output of ``select *``. Here's an example:: + + % echo '[{"name": "Azi"}, {"name": "Suna"}]' | \ + sqlite-utils insert chickens.db chickens - + % sqlite-utils schema chickens.db + CREATE TABLE [chickens] ( + [name] TEXT + ); + % sqlite-utils chickens.db 'select * from chickens' + [{"name": "Azi"}, + {"name": "Suna"}] + % sqlite-utils chickens.db 'select rowid, * from chickens' + [{"rowid": 1, "name": "Azi"}, + {"rowid": 2, "name": "Suna"}] + +You can use ``sqlite-utils transform ... --pk id`` to add a primary key column called ``id`` to the table. The primary key will be created as an ``INTEGER PRIMARY KEY`` and the existing ``rowid`` values will be copied across to it. It will automatically increment as new rows are added to the table:: + + % sqlite-utils transform chickens.db chickens --pk id + % sqlite-utils schema chickens.db + CREATE TABLE "chickens" ( + [id] INTEGER PRIMARY KEY, + [name] TEXT + ); + % sqlite-utils chickens.db 'select * from chickens' + [{"id": 1, "name": "Azi"}, + {"id": 2, "name": "Suna"}] + % echo '{"name": "Cardi"}' | sqlite-utils insert chickens.db chickens - + % sqlite-utils chickens.db 'select * from chickens' + [{"id": 1, "name": "Azi"}, + {"id": 2, "name": "Suna"}, + {"id": 3, "name": "Cardi"}] + .. _cli_extract: Extracting columns into a separate table From 79a5ece62ecfad5fb64da42c54ad110e822350d4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Feb 2022 22:54:40 -0800 Subject: [PATCH 0558/1004] Add --convert example to sqlite-utils insert --help, closes #404 --- docs/cli-reference.rst | 11 +++++++++++ sqlite_utils/cli.py | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 15b7b8c..5b7c6fe 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -208,6 +208,17 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`. Your Python code will be passed a "row" variable representing the imported row, and can return a modified row. + This example uses just the name, latitude and longitude columns from a CSV + file, converting name to upper case and latitude and longitude to floating + point numbers: + + sqlite-utils insert plants.db plants plants.csv --csv --convert ' + return { + "name": row["name"].upper(), + "latitude": float(row["latitude"]), + "longitude": float(row["longitude"]), + }' + If you are using --lines your code will be passed a "line" variable, and for --text an "text" variable. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index b137d9b..2ad7aff 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1144,6 +1144,18 @@ def insert( Your Python code will be passed a "row" variable representing the imported row, and can return a modified row. + This example uses just the name, latitude and longitude columns from + a CSV file, converting name to upper case and latitude and longitude + to floating point numbers: + + \b + sqlite-utils insert plants.db plants plants.csv --csv --convert ' + return { + "name": row["name"].upper(), + "latitude": float(row["latitude"]), + "longitude": float(row["longitude"]), + }' + If you are using --lines your code will be passed a "line" variable, and for --text an "text" variable. """ From 7142dbd58d25d54720c8396bd35990fd1387ba77 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Feb 2022 22:57:21 -0800 Subject: [PATCH 0559/1004] Fixed typo in --help --- docs/cli-reference.rst | 2 +- sqlite_utils/cli.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 5b7c6fe..abfacc2 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -220,7 +220,7 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`. }' If you are using --lines your code will be passed a "line" variable, and for - --text an "text" variable. + --text a "text" variable. Options: --pk TEXT Columns to use as the primary key, e.g. id diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 2ad7aff..0da3817 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1157,7 +1157,7 @@ def insert( }' If you are using --lines your code will be passed a "line" variable, - and for --text an "text" variable. + and for --text a "text" variable. """ try: insert_upsert_implementation( From e7f040106b5f5a892ebd984f19b21c605e87c142 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Feb 2022 23:03:04 -0800 Subject: [PATCH 0560/1004] Add an example of --text too, refs #404 --- docs/cli-reference.rst | 12 ++++++++++-- sqlite_utils/cli.py | 7 +++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index abfacc2..3359d5b 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -22,7 +22,9 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. refs = { "query": "cli_query", "memory": "cli_memory", - "insert": ["cli_inserting_data", "cli_insert_csv_tsv"], + "insert": [ + "cli_inserting_data", "cli_insert_csv_tsv", "cli_insert_unstructured", "cli_insert_convert" + ], "upsert": "cli_upsert", "tables": "cli_tables", "views": "cli_views", @@ -182,7 +184,7 @@ See :ref:`cli_memory`. insert ====== -See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`. +See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstructured`, :ref:`cli_insert_convert`. :: @@ -222,6 +224,12 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`. If you are using --lines your code will be passed a "line" variable, and for --text a "text" variable. + When using --text your function can return an iterator of rows to insert. This + example inserts one record per word in the input: + + echo 'A bunch of words' | sqlite-utils insert words.db words - \ + --text --convert '({"word": w} for w in text.split())' + Options: --pk TEXT Columns to use as the primary key, e.g. id --flatten Flatten nested JSON objects, so {"a": {"b": 1}} diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0da3817..9e0289f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1158,6 +1158,13 @@ def insert( If you are using --lines your code will be passed a "line" variable, and for --text a "text" variable. + + When using --text your function can return an iterator of rows to + insert. This example inserts one record per word in the input: + + \b + echo 'A bunch of words' | sqlite-utils insert words.db words - \\ + --text --convert '({"word": w} for w in text.split())' """ try: insert_upsert_implementation( From a692c56659c3563b26dcdc9e3534d63ecc26e180 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Tue, 15 Feb 2022 19:58:07 -0500 Subject: [PATCH 0561/1004] Add SpatiaLite helpers to CLI (#407) * Add SpatiaLite CLI helpers * Add docs for spaitalite helpers * Fix flake8 issues and add more detail on spatial types * Run cog and add some help text. * Use SpatiaLite when calculating coverage, refs #407 Co-authored-by: Simon Willison --- .github/workflows/test-coverage.yml | 2 + docs/cli-reference.rst | 51 ++++++++- docs/cli.rst | 39 +++++++ sqlite_utils/cli.py | 127 +++++++++++++++++++++- tests/test_cli.py | 29 ----- tests/test_gis.py | 157 +++++++++++++++++++++++++++- 6 files changed, 370 insertions(+), 35 deletions(-) diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 4b99cc5..0681f66 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -24,6 +24,8 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- + - name: Install SpatiaLite + run: sudo apt-get install libsqlite3-mod-spatialite - name: Install Python dependencies run: | python -m pip install --upgrade pip diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 3359d5b..5497259 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -772,8 +772,10 @@ See :ref:`cli_create_database`. sqlite-utils create-database trees.db Options: - --enable-wal Enable WAL mode on the created database - -h, --help Show this message and exit. + --enable-wal Enable WAL mode on the created database + --init-spatialite Enable SpatiaLite on the created database + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. create-table @@ -1226,4 +1228,49 @@ See :ref:`cli_drop_view`. -h, --help Show this message and exit. +add-geometry-column +=================== + +:: + + Usage: sqlite-utils add-geometry-column [OPTIONS] DB_PATH TABLE COLUMN_NAME + + Add a SpatiaLite geometry column to an existing table. Requires SpatiaLite + extension. + + By default, this command will try to load the SpatiaLite extension from usual + paths. To load it from a specific path, use --load-extension. + + Options: + -t, --type [POINT|LINESTRING|POLYGON|MULTIPOINT|MULTILINESTRING|MULTIPOLYGON|GEOMETRYCOLLECTION|GEOMETRY] + Specify a geometry type for this column. + [default: GEOMETRY] + --srid INTEGER Spatial Reference ID. See + https://spatialreference.org for details on + specific projections. [default: 4326] + --dimensions TEXT Coordinate dimensions. Use XYZ for three- + dimensional geometries. + --not-null Add a NOT NULL constraint. + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + +create-spatial-index +==================== + +:: + + Usage: sqlite-utils create-spatial-index [OPTIONS] DB_PATH TABLE COLUMN_NAME + + Create a spatial index on a SpatiaLite geometry column. The table and geometry + column must already exist before trying to add a spatial index. + + By default, this command will try to load the SpatiaLite extension from usual + paths. To load it from a specific path, use --load-extension. + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + .. [[[end]]] diff --git a/docs/cli.rst b/docs/cli.rst index e75adec..cadcbf1 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -724,6 +724,14 @@ To enable :ref:`cli_wal` on the newly created database add the ``--enable-wal`` $ sqlite-utils create-database empty.db --enable-wal +To enable SpatiaLite metadata on a newly created database, add the ``--init-spatialite`` flag:: + + $ sqlite-utils create-database empty.db --init-spatialite + +That will look for SpatiaLite in a set of predictable locations. To load it from somewhere else, use the ``--load-extension`` option:: + + $ sqlite-utils create-database empty.db --init-spatialite --load-extension /path/to/spatialite.so + .. _cli_inserting_data: Inserting JSON data @@ -1975,3 +1983,34 @@ Since `SpatiaLite `__ is com $ sqlite-utils memory "select spatialite_version()" --load-extension=spatialite [{"spatialite_version()": "4.3.0a"}] + + +SpatiaLite helpers +================== + +`SpatiaLite `_ adds geographic capability to SQLite (similar to how PostGIS builds on PostgreSQL). The `SpatiaLite cookbook `_ is a good resource for learning what's possible with it. + +You can convert an existing table to a geographic table by adding a geometry column, use the `sqlite-utils add-geometry-column` command:: + + $ sqlite-utils add-geometry-column spatial.db locations geometry --type POLYGON --srid 4326 + +The table (``locations`` in the example above) must already exist before adding a geometry column. Use ``sqlite-utils create-table`` first, then ``add-geometry-column``. + +Use the ``--type`` option to specify a geometry type. By default, ``add-geometry-column`` uses a generic ``GEOMETRY``, which will work with any type, though it may not be supported by some desktop GIS applications. + +Eight (case-insensitive) types are allowed: + + * POINT + * LINESTRING + * POLYGON + * MULTIPOINT + * MULTILINESTRING + * MULTIPOLYGON + * GEOMETRYCOLLECTION + * GEOMETRY + +Once you have a geometry column, you can speed up bounding box queries by adding a spatial index:: + + $ sqlite-utils create-spatial-index spatial.db locations geometry + +See the `SpatiaLite Cookbook `_ for examples of how to use a spatial index. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 9e0289f..8255b56 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1363,7 +1363,11 @@ def bulk( @click.option( "--enable-wal", is_flag=True, help="Enable WAL mode on the created database" ) -def create_database(path, enable_wal): +@click.option( + "--init-spatialite", is_flag=True, help="Enable SpatiaLite on the created database" +) +@load_extension_option +def create_database(path, enable_wal, init_spatialite, load_extension): """Create a new empty database file Example: @@ -1374,6 +1378,15 @@ def create_database(path, enable_wal): db = sqlite_utils.Database(path) if enable_wal: db.enable_wal() + + # load spatialite or another extension from a custom location + if load_extension: + _load_extensions(db, load_extension) + + # load spatialite from expected locations and initialize metadata + if init_spatialite: + db.init_spatialite() + db.vacuum() @@ -2544,7 +2557,7 @@ def _analyze(db, tables, columns, save): total=len(todo), most_common_rendered=most_common_rendered, least_common_rendered=least_common_rendered, - **column_details._asdict() + **column_details._asdict(), ) ) + "\n" @@ -2701,6 +2714,116 @@ def convert( ) +@cli.command("add-geometry-column") +@click.argument( + "db_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table", type=str) +@click.argument("column_name", type=str) +@click.option( + "-t", + "--type", + "geometry_type", + type=click.Choice( + [ + "POINT", + "LINESTRING", + "POLYGON", + "MULTIPOINT", + "MULTILINESTRING", + "MULTIPOLYGON", + "GEOMETRYCOLLECTION", + "GEOMETRY", + ], + case_sensitive=False, + ), + default="GEOMETRY", + help="Specify a geometry type for this column.", + show_default=True, +) +@click.option( + "--srid", + type=int, + default=4326, + show_default=True, + help="Spatial Reference ID. See https://spatialreference.org for details on specific projections.", +) +@click.option( + "--dimensions", + "coord_dimension", + type=str, + default="XY", + help="Coordinate dimensions. Use XYZ for three-dimensional geometries.", +) +@click.option("--not-null", "not_null", is_flag=True, help="Add a NOT NULL constraint.") +@load_extension_option +def add_geometry_column( + db_path, + table, + column_name, + geometry_type, + srid, + coord_dimension, + not_null, + load_extension, +): + """Add a SpatiaLite geometry column to an existing table. Requires SpatiaLite extension. + \n\n + By default, this command will try to load the SpatiaLite extension from usual paths. + To load it from a specific path, use --load-extension.""" + db = sqlite_utils.Database(db_path) + if not db[table].exists(): + raise click.ClickException( + "You must create a table before adding a geometry column" + ) + + # load spatialite, one way or another + if load_extension: + _load_extensions(db, load_extension) + db.init_spatialite() + + if db[table].add_geometry_column( + column_name, geometry_type, srid, coord_dimension, not_null + ): + click.echo(f"Added {geometry_type} column {column_name} to {table}") + + +@cli.command("create-spatial-index") +@click.argument( + "db_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table", type=str) +@click.argument("column_name", type=str) +@load_extension_option +def create_spatial_index(db_path, table, column_name, load_extension): + """Create a spatial index on a SpatiaLite geometry column. + The table and geometry column must already exist before trying to add a spatial index. + \n\n + By default, this command will try to load the SpatiaLite extension from usual paths. + To load it from a specific path, use --load-extension.""" + db = sqlite_utils.Database(db_path) + if not db[table].exists(): + raise click.ClickException( + "You must create a table and add a geometry column before creating a spatial index" + ) + + # load spatialite + if load_extension: + _load_extensions(db, load_extension) + db.init_spatialite() + + if column_name not in db[table].columns_dict: + raise click.ClickException( + "You must add a geometry column before creating a spatial index" + ) + + db[table].create_spatial_index(column_name) + + def _render_common(title, values): if values is None: return "" diff --git a/tests/test_cli.py b/tests/test_cli.py index 244029d..1922941 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,7 +7,6 @@ from unittest import mock import json import os import pytest -from sqlite_utils.utils import sqlite3, find_spatialite import textwrap from .utils import collapse_whitespace @@ -792,34 +791,6 @@ def test_query_raw(db_path, content, is_binary): assert result.output == str(content) -@pytest.mark.skipif(not find_spatialite(), reason="Could not find SpatiaLite extension") -@pytest.mark.skipif( - not hasattr(sqlite3.Connection, "enable_load_extension"), - reason="sqlite3.Connection missing enable_load_extension", -) -@pytest.mark.parametrize("use_spatialite_shortcut", [True, False]) -def test_query_load_extension(use_spatialite_shortcut): - # Without --load-extension: - result = CliRunner().invoke(cli.cli, [":memory:", "select spatialite_version()"]) - assert result.exit_code == 1 - assert "no such function: spatialite_version" in result.output - # With --load-extension: - if use_spatialite_shortcut: - load_extension = "spatialite" - else: - load_extension = find_spatialite() - result = CliRunner().invoke( - cli.cli, - [ - ":memory:", - "select spatialite_version()", - "--load-extension={}".format(load_extension), - ], - ) - assert result.exit_code == 0, result.stdout - assert ["spatialite_version()"] == list(json.loads(result.output)[0].keys()) - - def test_query_memory_does_not_create_file(tmpdir): owd = os.getcwd() try: diff --git a/tests/test_gis.py b/tests/test_gis.py index 3bf0d5f..3b1fbf1 100644 --- a/tests/test_gis.py +++ b/tests/test_gis.py @@ -1,7 +1,10 @@ +import json import pytest -from sqlite_utils.utils import find_spatialite + +from click.testing import CliRunner +from sqlite_utils.cli import cli from sqlite_utils.db import Database -from sqlite_utils.utils import sqlite3 +from sqlite_utils.utils import find_spatialite, sqlite3 pytestmark = [ pytest.mark.skipif( @@ -14,6 +17,7 @@ pytestmark = [ ] +# python API tests def test_find_spatialite(): spatialite = find_spatialite() assert spatialite is None or isinstance(spatialite, str) @@ -81,3 +85,152 @@ def test_double_create_spatial_index(): # call it again, return False assert not table.create_spatial_index("geometry") + + +# cli tests +@pytest.mark.parametrize("use_spatialite_shortcut", [True, False]) +def test_query_load_extension(use_spatialite_shortcut): + # Without --load-extension: + result = CliRunner().invoke(cli, [":memory:", "select spatialite_version()"]) + assert result.exit_code == 1 + assert "no such function: spatialite_version" in result.output + # With --load-extension: + if use_spatialite_shortcut: + load_extension = "spatialite" + else: + load_extension = find_spatialite() + result = CliRunner().invoke( + cli, + [ + ":memory:", + "select spatialite_version()", + "--load-extension={}".format(load_extension), + ], + ) + assert result.exit_code == 0, result.stdout + assert ["spatialite_version()"] == list(json.loads(result.output)[0].keys()) + + +def test_cli_create_spatialite(tmpdir): + # sqlite-utils create test.db --init-spatialite + db_path = tmpdir / "created.db" + result = CliRunner().invoke( + cli, ["create-database", str(db_path), "--init-spatialite"] + ) + + assert 0 == result.exit_code + assert db_path.exists() + assert db_path.read_binary()[:16] == b"SQLite format 3\x00" + + db = Database(str(db_path)) + assert "spatial_ref_sys" in db.table_names() + + +def test_cli_add_geometry_column(tmpdir): + # create a rowid table with one column + db_path = tmpdir / "spatial.db" + db = Database(str(db_path)) + db.init_spatialite() + + table = db["locations"].create({"name": str}) + + result = CliRunner().invoke( + cli, + [ + "add-geometry-column", + str(db_path), + table.name, + "geometry", + "--type", + "POINT", + ], + ) + + assert 0 == result.exit_code + + assert db["geometry_columns"].get(["locations", "geometry"]) == { + "f_table_name": "locations", + "f_geometry_column": "geometry", + "geometry_type": 1, # point + "coord_dimension": 2, + "srid": 4326, + "spatial_index_enabled": 0, + } + + +def test_cli_add_geometry_column_options(tmpdir): + # create a rowid table with one column + db_path = tmpdir / "spatial.db" + db = Database(str(db_path)) + db.init_spatialite() + table = db["locations"].create({"name": str}) + + result = CliRunner().invoke( + cli, + [ + "add-geometry-column", + str(db_path), + table.name, + "geometry", + "-t", + "POLYGON", + "--srid", + "3857", # https://epsg.io/3857 + "--not-null", + ], + ) + + assert 0 == result.exit_code + + assert db["geometry_columns"].get(["locations", "geometry"]) == { + "f_table_name": "locations", + "f_geometry_column": "geometry", + "geometry_type": 3, # polygon + "coord_dimension": 2, + "srid": 3857, + "spatial_index_enabled": 0, + } + + column = table.columns[1] + assert column.notnull + + +def test_cli_add_geometry_column_invalid_type(tmpdir): + # create a rowid table with one column + db_path = tmpdir / "spatial.db" + db = Database(str(db_path)) + db.init_spatialite() + + table = db["locations"].create({"name": str}) + + result = CliRunner().invoke( + cli, + [ + "add-geometry-column", + str(db_path), + table.name, + "geometry", + "--type", + "NOT-A-TYPE", + ], + ) + + assert 2 == result.exit_code + + +def test_cli_create_spatial_index(tmpdir): + # create a rowid table with one column + db_path = tmpdir / "spatial.db" + db = Database(str(db_path)) + db.init_spatialite() + + table = db["locations"].create({"name": str}) + table.add_geometry_column("geometry", "POINT") + + result = CliRunner().invoke( + cli, ["create-spatial-index", str(db_path), table.name, "geometry"] + ) + + assert 0 == result.exit_code + + assert "idx_locations_geometry" in db.table_names() From 3e5a4f60cc07e38113e522e5f1d09db35626affc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Feb 2022 17:06:49 -0800 Subject: [PATCH 0562/1004] Tweaked SpatiaLite CLI docs, refs #398 --- docs/cli-reference.rst | 6 ++++++ docs/cli.rst | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 5497259..19b714b 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -56,6 +56,8 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. "insert-files": "cli_insert_files", "analyze-tables": "cli_analyze_tables", "convert": "cli_convert", + "add-geometry-column": "cli_spatialite", + "create-spatial-index": "cli_spatialite_indexes", } commands.sort(key = lambda command: go_first.index(command) if command in go_first else 999) cog.out("\n") @@ -1231,6 +1233,8 @@ See :ref:`cli_drop_view`. add-geometry-column =================== +See :ref:`cli_spatialite`. + :: Usage: sqlite-utils add-geometry-column [OPTIONS] DB_PATH TABLE COLUMN_NAME @@ -1258,6 +1262,8 @@ add-geometry-column create-spatial-index ==================== +See :ref:`cli_spatialite_indexes`. + :: Usage: sqlite-utils create-spatial-index [OPTIONS] DB_PATH TABLE COLUMN_NAME diff --git a/docs/cli.rst b/docs/cli.rst index cadcbf1..8ec3f9e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1984,13 +1984,14 @@ Since `SpatiaLite `__ is com $ sqlite-utils memory "select spatialite_version()" --load-extension=spatialite [{"spatialite_version()": "4.3.0a"}] +.. _cli_spatialite: SpatiaLite helpers ================== `SpatiaLite `_ adds geographic capability to SQLite (similar to how PostGIS builds on PostgreSQL). The `SpatiaLite cookbook `_ is a good resource for learning what's possible with it. -You can convert an existing table to a geographic table by adding a geometry column, use the `sqlite-utils add-geometry-column` command:: +You can convert an existing table to a geographic table by adding a geometry column, use the ``sqlite-utils add-geometry-column`` command:: $ sqlite-utils add-geometry-column spatial.db locations geometry --type POLYGON --srid 4326 @@ -2009,6 +2010,11 @@ Eight (case-insensitive) types are allowed: * GEOMETRYCOLLECTION * GEOMETRY +.. _cli_spatialite_indexes: + +Adding spatial indexes +---------------------- + Once you have a geometry column, you can speed up bounding box queries by adding a spatial index:: $ sqlite-utils create-spatial-index spatial.db locations geometry From 8f528ed2b13c309c9efb1ee6e1150ab3fce11d89 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Feb 2022 17:21:07 -0800 Subject: [PATCH 0563/1004] Fix ReST warning --- docs/cli.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 8ec3f9e..03689c1 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1989,7 +1989,7 @@ Since `SpatiaLite `__ is com SpatiaLite helpers ================== -`SpatiaLite `_ adds geographic capability to SQLite (similar to how PostGIS builds on PostgreSQL). The `SpatiaLite cookbook `_ is a good resource for learning what's possible with it. +`SpatiaLite `_ adds geographic capability to SQLite (similar to how PostGIS builds on PostgreSQL). The `SpatiaLite cookbook `__ is a good resource for learning what's possible with it. You can convert an existing table to a geographic table by adding a geometry column, use the ``sqlite-utils add-geometry-column`` command:: @@ -2019,4 +2019,4 @@ Once you have a geometry column, you can speed up bounding box queries by adding $ sqlite-utils create-spatial-index spatial.db locations geometry -See the `SpatiaLite Cookbook `_ for examples of how to use a spatial index. +See this `SpatiaLite Cookbook recipe `__ for examples of how to use a spatial index. From 4bc06a243774ca8d8e04ad6592e895d3a7a0300b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Feb 2022 17:21:25 -0800 Subject: [PATCH 0564/1004] memory_name= feature, closes #405 --- docs/python-api.rst | 6 ++++++ sqlite_utils/db.py | 15 ++++++++++++--- tests/test_constructor.py | 7 +++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 149c3a2..fa814b3 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -98,6 +98,12 @@ If you want to create an in-memory database, you can do so like this: db = Database(memory=True) +You can also create a named in-memory database. Unlike regular memory databases these can be accessed by multiple threads, provided at least one reference to the database still exists. `del db` will clear the database from memory. + +.. code-block:: python + + db = Database(memory_name="my_shared_database") + Connections use ``PRAGMA recursive_triggers=on`` by default. If you don't want to use `recursive triggers `__ you can turn them off using: .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1927f41..1bae5ad 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -278,6 +278,7 @@ class Database: - ``filename_or_conn`` - String path to a file, or a ``pathlib.Path`` object, or a ``sqlite3`` connection - ``memory`` - set to ``True`` to create an in-memory database + - ``memory_name`` - creates a named in-memory database that can be shared across multiple connections. - ``recreate`` - set to ``True`` to delete and recreate a file database (**dangerous**) - ``recursive_triggers`` - defaults to ``True``, which sets ``PRAGMA recursive_triggers=on;`` - set to ``False`` to avoid setting this pragma @@ -294,15 +295,23 @@ class Database: self, filename_or_conn: Union[str, pathlib.Path, sqlite3.Connection] = None, memory: bool = False, + memory_name: str = None, recreate: bool = False, recursive_triggers: bool = True, tracer: Callable = None, use_counts_table: bool = False, ): - assert (filename_or_conn is not None and not memory) or ( - filename_or_conn is None and memory + assert (filename_or_conn is not None and (not memory and not memory_name)) or ( + filename_or_conn is None and (memory or memory_name) ), "Either specify a filename_or_conn or pass memory=True" - if memory or filename_or_conn == ":memory:": + if memory_name: + uri = "file:{}?mode=memory&cache=shared".format(memory_name) + self.conn = sqlite3.connect( + uri, + uri=True, + check_same_thread=False, + ) + elif memory or filename_or_conn == ":memory:": self.conn = sqlite3.connect(":memory:") elif isinstance(filename_or_conn, (str, pathlib.Path)): if recreate and os.path.exists(filename_or_conn): diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 924df66..36ab1bc 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -9,3 +9,10 @@ def test_recursive_triggers(): def test_recursive_triggers_off(): db = Database(memory=True, recursive_triggers=False) assert not db.execute("PRAGMA recursive_triggers").fetchone()[0] + + +def test_memory_name(): + db1 = Database(memory_name="shared") + db2 = Database(memory_name="shared") + db1["dogs"].insert({"name": "Cleo"}) + assert list(db2["dogs"].rows) == [{"name": "Cleo"}] From 757f103ae2a8b3803ceea89a412cf78b269f9e75 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Feb 2022 17:39:13 -0800 Subject: [PATCH 0565/1004] Release 3.24 Refs ##397, #398, #401, #403, #404, #405, #407 --- docs/changelog.rst | 15 +++++++++++++++ setup.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7f62c62..a2a024b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,21 @@ Changelog =========== +.. _v3_24: + +3.24 (2022-02-15) +----------------- + +- SpatiaLite helpers for the ``sqlite-utils`` command-line tool - thanks, Chris Amico. (:issue:`398`) + + - :ref:`sqlite-utils create-database ` ``--init-spatialite`` option for initializing SpatiaLite on a newly created database. + - :ref:`sqlite-utils add-geometry-column ` command for adding geometry columns. + - :ref:`sqlite-utils create-spatial-index ` command for adding spatial indexes. + +- ``db[table].create(..., if_not_exists=True)`` option for :ref:`creating a table ` only if it does not already exist. (:issue:`397`) +- ``Database(memory_name="my_shared_database")`` parameter for creating a :ref:`named in-memory database ` that can be shared between multiple connections. (:issue:`405`) +- Documentation now describes :ref:`how to add a primary key to a rowid table ` using ``sqlite-utils transform``. (:issue:`403`) + .. _v3_23: 3.23 (2022-02-03) diff --git a/setup.py b/setup.py index 6d96533..9db7db1 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.23" +VERSION = "3.24" def get_long_description(): From 7a098aa0c5e8beef6ccc55c866cf7792af5fcf43 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 17 Feb 2022 07:39:54 -0800 Subject: [PATCH 0566/1004] Link to my blog series --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9debfcd..6f81fc1 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,12 @@ Python CLI utility and library for manipulating SQLite databases. ## Some feature highlights - [Pipe JSON](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-json-data) (or [CSV or TSV](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-csv-or-tsv-data)) directly into a new SQLite database file, automatically creating a table with the appropriate schema -- [Run in-memory SQL queries](https://sqlite-utils.datasette.io/en/stable/cli.html#querying-data-directly-using-an-in-memory-database), including joins, directly against data in CSV, TSV or JSON files and view the results. +- [Run in-memory SQL queries](https://sqlite-utils.datasette.io/en/stable/cli.html#querying-data-directly-using-an-in-memory-database), including joins, directly against data in CSV, TSV or JSON files and view the results - [Configure SQLite full-text search](https://sqlite-utils.datasette.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance - Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as changing the type of a column - [Extract columns](https://sqlite-utils.datasette.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data -Read more on my blog: [ -sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) and other [entries tagged sqliteutils](https://simonwillison.net/tags/sqliteutils/). +Read more on my blog, in this series of posts on [New features in sqlite-utils](https://simonwillison.net/series/sqlite-utils-features/) and other [entries tagged sqliteutils](https://simonwillison.net/tags/sqliteutils/). ## Installation From b6c9dfce0ba27eb5fb6bc2221044798420f861c4 Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Tue, 1 Mar 2022 21:05:29 +0000 Subject: [PATCH 0567/1004] Correct spelling mistakes (found with codespell) (#410) --- tests/test_cli.py | 6 +++--- tests/test_fts.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 1922941..e2e4aa2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1218,7 +1218,7 @@ def test_drop_table_error(): ) assert 1 == result.exit_code assert 'Error: Table "t2" does not exist' == result.output.strip() - # Using --ignore supresses that error + # Using --ignore suppresses that error result = runner.invoke( cli.cli, ["drop-table", "test.db", "t2", "--ignore"], @@ -1259,7 +1259,7 @@ def test_drop_view_error(): ) assert 1 == result.exit_code assert 'Error: View "t2" does not exist' == result.output.strip() - # Using --ignore supresses that error + # Using --ignore suppresses that error result = runner.invoke( cli.cli, ["drop-view", "test.db", "t2", "--ignore"], @@ -2014,7 +2014,7 @@ def test_insert_detect_types(tmpdir, option_or_env_var): ] if option_or_env_var is None: - # Use environemnt variable instead of option + # Use environment variable instead of option with mock.patch.dict(os.environ, {"SQLITE_UTILS_DETECT_TYPES": "1"}): _test() else: diff --git a/tests/test_fts.py b/tests/test_fts.py index 0ae2a52..24980f7 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -269,7 +269,7 @@ def test_rebuild_fts(fresh_db): }.items() <= rows[0].items() # Insert another record table.insert(search_records[1]) - # This should NOT show up in a searchs + # This should NOT show up in searches assert len(list(table.search("are"))) == 1 # Running rebuild_fts() should fix it table.rebuild_fts() From 931b1e151320535acf0a899c7d403d71b5199f6a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Mar 2022 16:00:51 -0800 Subject: [PATCH 0568/1004] .insert(hash_id_columns=) parameter, closes #343 --- docs/python-api.rst | 13 ++++++++++- docs/reference.rst | 10 ++++++++ sqlite_utils/db.py | 53 +++++++++++++++++++++++++++++++++---------- sqlite_utils/utils.py | 31 ++++++++++++++++++++++++- tests/test_create.py | 28 ++++++++++++++++++++++- tests/test_upsert.py | 24 ++++++++++++++++++++ tests/test_utils.py | 14 ++++++++++++ 7 files changed, 158 insertions(+), 15 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index fa814b3..e8c4bda 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -633,7 +633,7 @@ You can set default values for these methods by accessing the table through the # Now you can call .insert() like so: table.insert({"id": 1, "name": "Tracy", "score": 5}) -The configuration options that can be specified in this way are ``pk``, ``foreign_keys``, ``column_order``, ``not_null``, ``defaults``, ``batch_size``, ``hash_id``, ``alter``, ``ignore``, ``replace``, ``extracts``, ``conversions``, ``columns``. These are all documented below. +The configuration options that can be specified in this way are ``pk``, ``foreign_keys``, ``column_order``, ``not_null``, ``defaults``, ``batch_size``, ``hash_id``, ``hash_id_columns``, ``alter``, ``ignore``, ``replace``, ``extracts``, ``conversions``, ``columns``. These are all documented below. .. _python_api_defaults_not_null: @@ -1594,6 +1594,17 @@ If you are going to use that ID straight away, you can access it using ``last_pk }, hash_id="id").last_pk # dog_id is now "f501265970505d9825d8d9f590bfab3519fb20b1" +The hash will be created using all of the column values. To create a hash using a subset of the columns, pass the ``hash_id_columns=`` parameter:: + + db["dogs"].upsert( + {"name": "Cleo", "twitter": "cleopaws", "age": 7}, + hash_id_columns=("name", "twitter") + ) + +The `hash_id=` parameter is optional if you specify ``hash_id_columns=`` - it will default to putting the hash in a column called ``id``. + +You can manually calculate these hashes using the :ref:`hash_record(record, keys=...) ` utility function. + .. _python_api_create_view: Creating views diff --git a/docs/reference.rst b/docs/reference.rst index 4638026..5bd609f 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -68,3 +68,13 @@ sqlite_utils.db.ColumnDetails ----------------------------- .. autoclass:: sqlite_utils.db.ColumnDetails + +sqlite_utils.utils +================== + +.. _reference_utils_hash_record: + +sqlite_utils.utils.hash_record +------------------------------ + +.. autofunction:: sqlite_utils.utils.hash_record diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1bae5ad..70b452c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,5 +1,6 @@ from .utils import ( chunks, + hash_record, sqlite3, OperationalError, suggest_column_types, @@ -13,7 +14,6 @@ from collections.abc import Mapping import contextlib import datetime import decimal -import hashlib import inspect import itertools import json @@ -663,13 +663,16 @@ class Database: pk: Optional[Any] = None, foreign_keys: Optional[ForeignKeysType] = None, column_order: Optional[List[str]] = None, - not_null: Iterable[str] = None, + not_null: Optional[Iterable[str]] = None, defaults: Optional[Dict[str, Any]] = None, hash_id: Optional[str] = None, + hash_id_columns: Optional[Iterable[str]] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, if_not_exists: bool = False, ) -> str: "Returns the SQL ``CREATE TABLE`` statement for creating the specified table." + if hash_id_columns and (hash_id is None): + hash_id = "id" foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) foreign_keys_by_column = {fk.column: fk for fk in foreign_keys} # any extracts will be treated as integer columns with a foreign key @@ -779,6 +782,7 @@ class Database: not_null: Iterable[str] = None, defaults: Optional[Dict[str, Any]] = None, hash_id: Optional[str] = None, + hash_id_columns: Optional[Iterable[str]] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, if_not_exists: bool = False, ) -> "Table": @@ -796,6 +800,7 @@ class Database: not_null=not_null, defaults=defaults, hash_id=hash_id, + hash_id_columns=hash_id_columns, extracts=extracts, if_not_exists=if_not_exists, ) @@ -808,6 +813,7 @@ class Database: not_null=not_null, defaults=defaults, hash_id=hash_id, + hash_id_columns=hash_id_columns, ) return cast(Table, table) @@ -1126,12 +1132,13 @@ class Table(Queryable): defaults: Optional[Dict[str, Any]] = None, batch_size: int = 100, hash_id: Optional[str] = None, + hash_id_columns: Optional[Iterable[str]] = None, alter: bool = False, ignore: bool = False, replace: bool = False, extracts: Optional[Union[Dict[str, str], List[str]]] = None, conversions: Optional[dict] = None, - columns: Optional[Union[Dict[str, Any]]] = None, + columns: Optional[Dict[str, Any]] = None, ): super().__init__(db, name) self._defaults = dict( @@ -1142,6 +1149,7 @@ class Table(Queryable): defaults=defaults, batch_size=batch_size, hash_id=hash_id, + hash_id_columns=hash_id_columns, alter=alter, ignore=ignore, replace=replace, @@ -1314,6 +1322,7 @@ class Table(Queryable): not_null: Iterable[str] = None, defaults: Optional[Dict[str, Any]] = None, hash_id: Optional[str] = None, + hash_id_columns: Optional[Iterable[str]] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, if_not_exists: bool = False, ) -> "Table": @@ -1333,6 +1342,7 @@ class Table(Queryable): not_null=not_null, defaults=defaults, hash_id=hash_id, + hash_id_columns=hash_id_columns, extracts=extracts, if_not_exists=if_not_exists, ) @@ -2389,6 +2399,7 @@ class Table(Queryable): chunk, all_columns, hash_id, + hash_id_columns, upsert, pk, conversions, @@ -2400,12 +2411,19 @@ class Table(Queryable): # .execute() method - but some of them may be replaced by # new primary keys if we are extracting any columns. values = [] + if hash_id_columns and hash_id is None: + hash_id = "id" extracts = resolve_extracts(extracts) for record in chunk: record_values = [] for key in all_columns: value = jsonify_if_needed( - record.get(key, None if key != hash_id else _hash(record)) + record.get( + key, + None + if key != hash_id + else hash_record(record, hash_id_columns), + ) ) if key in extracts: extract_table = extracts[key] @@ -2486,6 +2504,7 @@ class Table(Queryable): chunk, all_columns, hash_id, + hash_id_columns, upsert, pk, conversions, @@ -2498,6 +2517,7 @@ class Table(Queryable): chunk, all_columns, hash_id, + hash_id_columns, upsert, pk, conversions, @@ -2527,6 +2547,7 @@ class Table(Queryable): first_half, all_columns, hash_id, + hash_id_columns, upsert, pk, conversions, @@ -2541,6 +2562,7 @@ class Table(Queryable): second_half, all_columns, hash_id, + hash_id_columns, upsert, pk, conversions, @@ -2575,6 +2597,7 @@ class Table(Queryable): not_null: Optional[Union[Set[str], Default]] = DEFAULT, defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT, hash_id: Optional[Union[str, Default]] = DEFAULT, + hash_id_columns: Optional[Iterable[str]] = DEFAULT, alter: Optional[Union[bool, Default]] = DEFAULT, ignore: Optional[Union[bool, Default]] = DEFAULT, replace: Optional[Union[bool, Default]] = DEFAULT, @@ -2621,6 +2644,7 @@ class Table(Queryable): not_null=not_null, defaults=defaults, hash_id=hash_id, + hash_id_columns=hash_id_columns, alter=alter, ignore=ignore, replace=replace, @@ -2639,6 +2663,7 @@ class Table(Queryable): defaults=DEFAULT, batch_size=DEFAULT, hash_id=DEFAULT, + hash_id_columns=DEFAULT, alter=DEFAULT, ignore=DEFAULT, replace=DEFAULT, @@ -2662,6 +2687,7 @@ class Table(Queryable): defaults = self.value_or_default("defaults", defaults) batch_size = self.value_or_default("batch_size", batch_size) hash_id = self.value_or_default("hash_id", hash_id) + hash_id_columns = self.value_or_default("hash_id_columns", hash_id_columns) alter = self.value_or_default("alter", alter) ignore = self.value_or_default("ignore", ignore) replace = self.value_or_default("replace", replace) @@ -2669,9 +2695,14 @@ class Table(Queryable): conversions = self.value_or_default("conversions", conversions) or {} columns = self.value_or_default("columns", columns) + if hash_id_columns and hash_id is None: + hash_id = "id" + if upsert and (not pk and not hash_id): raise PrimaryKeyRequired("upsert() requires a pk") assert not (hash_id and pk), "Use either pk= or hash_id=" + if hash_id_columns and (hash_id is None): + hash_id = "id" if hash_id: pk = hash_id @@ -2716,6 +2747,7 @@ class Table(Queryable): not_null=not_null, defaults=defaults, hash_id=hash_id, + hash_id_columns=hash_id_columns, extracts=extracts, ) all_columns_set = set() @@ -2738,6 +2770,7 @@ class Table(Queryable): chunk, all_columns, hash_id, + hash_id_columns, upsert, pk, conversions, @@ -2760,6 +2793,7 @@ class Table(Queryable): not_null=DEFAULT, defaults=DEFAULT, hash_id=DEFAULT, + hash_id_columns=DEFAULT, alter=DEFAULT, extracts=DEFAULT, conversions=DEFAULT, @@ -2779,6 +2813,7 @@ class Table(Queryable): not_null=not_null, defaults=defaults, hash_id=hash_id, + hash_id_columns=hash_id_columns, alter=alter, extracts=extracts, conversions=conversions, @@ -2795,6 +2830,7 @@ class Table(Queryable): defaults=DEFAULT, batch_size=DEFAULT, hash_id=DEFAULT, + hash_id_columns=DEFAULT, alter=DEFAULT, extracts=DEFAULT, conversions=DEFAULT, @@ -2813,6 +2849,7 @@ class Table(Queryable): defaults=defaults, batch_size=batch_size, hash_id=hash_id, + hash_id_columns=hash_id_columns, alter=alter, extracts=extracts, conversions=conversions, @@ -3184,14 +3221,6 @@ def jsonify_if_needed(value): return value -def _hash(record): - return hashlib.sha1( - json.dumps(record, separators=(",", ":"), sort_keys=True, default=repr).encode( - "utf8" - ) - ).hexdigest() - - def resolve_extracts( extracts: Optional[Union[Dict[str, str], List[str], Tuple[str]]] ) -> dict: diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index e3386d8..4ec8b9f 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -2,12 +2,13 @@ import base64 import contextlib import csv import enum +import hashlib import io import itertools import json import os from . import recipes -from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type +from typing import Dict, cast, BinaryIO, Iterable, Optional, Tuple, Type import click @@ -345,3 +346,31 @@ def chunks(sequence, size): iterator = iter(sequence) for item in iterator: yield itertools.chain([item], itertools.islice(iterator, size - 1)) + + +def hash_record(record: Dict, keys: Optional[Iterable[str]] = None): + """ + ``record`` should be a Python dictionary. Returns a sha1 hash of the + keys and values in that record. + + If ``keys=`` is provided, uses just those keys to generate the hash. + + Example usage:: + + from sqlite_utils.utils import hash_record + + hashed = hash_record({"name": "Cleo", "twitter": "CleoPaws"}) + # Or with the keys= option: + hashed = hash_record( + {"name": "Cleo", "twitter": "CleoPaws", "age": 7}, + keys=("name", "twitter") + ) + """ + to_hash = record + if keys is not None: + to_hash = {key: record[key] for key in keys} + return hashlib.sha1( + json.dumps(to_hash, separators=(",", ":"), sort_keys=True, default=repr).encode( + "utf8" + ) + ).hexdigest() diff --git a/tests/test_create.py b/tests/test_create.py index 35844a1..c3871e1 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -9,7 +9,7 @@ from sqlite_utils.db import ( Table, View, ) -from sqlite_utils.utils import sqlite3 +from sqlite_utils.utils import hash_record, sqlite3 import collections import datetime import decimal @@ -888,6 +888,32 @@ def test_insert_hash_id(fresh_db): assert 1 == dogs.count +@pytest.mark.parametrize("use_table_factory", [True, False]) +def test_insert_hash_id_columns(fresh_db, use_table_factory): + if use_table_factory: + dogs = fresh_db.table("dogs", hash_id_columns=("name", "twitter")) + insert_kwargs = {} + else: + dogs = fresh_db["dogs"] + insert_kwargs = dict(hash_id_columns=("name", "twitter")) + + id = dogs.insert( + {"name": "Cleo", "twitter": "cleopaws", "age": 5}, + **insert_kwargs, + ).last_pk + expected_hash = hash_record({"name": "Cleo", "twitter": "cleopaws"}) + assert id == expected_hash + assert dogs.count == 1 + # Insert replacing a second time should not create a new row + id2 = dogs.insert( + {"name": "Cleo", "twitter": "cleopaws", "age": 6}, + **insert_kwargs, + replace=True, + ).last_pk + assert id2 == expected_hash + assert dogs.count == 1 + + def test_vacuum(fresh_db): fresh_db["data"].insert({"foo": "foo", "bar": "bar"}) fresh_db.vacuum() diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 09bdacc..cef8af0 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -45,6 +45,30 @@ def test_upsert_with_hash_id(fresh_db): assert "a5e744d0164540d33b1d7ea616c28f2fa97e754a" == table.last_pk +@pytest.mark.parametrize("hash_id", (None, "custom_id")) +def test_upsert_with_hash_id_columns(fresh_db, hash_id): + table = fresh_db["table"] + table.upsert({"a": 1, "b": 2, "c": 3}, hash_id=hash_id, hash_id_columns=("a", "b")) + assert list(table.rows) == [ + { + hash_id or "id": "4acc71e0547112eb432f0a36fb1924c4a738cb49", + "a": 1, + "b": 2, + "c": 3, + } + ] + assert table.last_pk == "4acc71e0547112eb432f0a36fb1924c4a738cb49" + table.upsert({"a": 1, "b": 2, "c": 4}, hash_id=hash_id, hash_id_columns=("a", "b")) + assert list(table.rows) == [ + { + hash_id or "id": "4acc71e0547112eb432f0a36fb1924c4a738cb49", + "a": 1, + "b": 2, + "c": 4, + } + ] + + def test_upsert_compound_primary_key(fresh_db): table = fresh_db["table"] table.upsert_all( diff --git a/tests/test_utils.py b/tests/test_utils.py index 66ce413..e76e97a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -35,3 +35,17 @@ def test_chunks(size, expected): input = ["a", "b", "c", "d"] chunks = list(map(list, utils.chunks(input, size))) assert chunks == expected + + +def test_hash_record(): + expected = "d383e7c0ba88f5ffcdd09be660de164b3847401a" + assert utils.hash_record({"name": "Cleo", "twitter": "CleoPaws"}) == expected + assert ( + utils.hash_record( + {"name": "Cleo", "twitter": "CleoPaws", "age": 7}, keys=("name", "twitter") + ) + == expected + ) + assert ( + utils.hash_record({"name": "Cleo", "twitter": "CleoPaws", "age": 7}) != expected + ) From 521921b849003ed3742338f76f9d47ff3d95eaf3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Mar 2022 16:05:11 -0800 Subject: [PATCH 0569/1004] Fixed mypy error, refs #343 --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 70b452c..9e00366 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2597,7 +2597,7 @@ class Table(Queryable): not_null: Optional[Union[Set[str], Default]] = DEFAULT, defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT, hash_id: Optional[Union[str, Default]] = DEFAULT, - hash_id_columns: Optional[Iterable[str]] = DEFAULT, + hash_id_columns: Optional[Union[Iterable[str], Default]] = DEFAULT, alter: Optional[Union[bool, Default]] = DEFAULT, ignore: Optional[Union[bool, Default]] = DEFAULT, replace: Optional[Union[bool, Default]] = DEFAULT, From d25cdd37a3b7d1b277b399106473fa368b72635a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Mar 2022 16:24:27 -0800 Subject: [PATCH 0570/1004] db.sqlite_version property and fix for deterministic=True on SQLite 3.8.3 Closes #408 --- docs/python-api.rst | 10 ++++++++++ sqlite_utils/db.py | 12 +++++++++++- tests/test_constructor.py | 9 +++++++++ tests/test_register_function.py | 19 +++++++++++++++++-- 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index e8c4bda..6c35ef3 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1717,6 +1717,16 @@ A more useful example: if you are working with `SpatiaLite >> db.sqlite_version + (3, 36, 0) + .. _python_api_introspection: Introspecting tables and views diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9e00366..3bc528f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -387,7 +387,11 @@ class Database: if not replace and (name, arity) in self._registered_functions: return fn kwargs = {} - if deterministic and sys.version_info >= (3, 8): + if ( + deterministic + and sys.version_info >= (3, 8) + and self.sqlite_version >= (3, 8, 3) + ): kwargs["deterministic"] = True self.conn.create_function(name, arity, fn, **kwargs) self._registered_functions.add((name, arity)) @@ -547,6 +551,12 @@ class Database: except Exception: return False + @property + def sqlite_version(self) -> Tuple[int, ...]: + "Version of SQLite, as a tuple of integers e.g. (3, 36, 0)" + row = self.execute("select sqlite_version()").fetchall()[0] + return tuple(map(int, row[0].split("."))) + @property def journal_mode(self) -> str: "Current ``journal_mode`` of this database." diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 36ab1bc..d8c8d72 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -16,3 +16,12 @@ def test_memory_name(): db2 = Database(memory_name="shared") db1["dogs"].insert({"name": "Cleo"}) assert list(db2["dogs"].rows) == [{"name": "Cleo"}] + + +def test_sqlite_version(): + db = Database(memory=True) + version = db.sqlite_version + assert isinstance(version, tuple) + as_string = ".".join(map(str, version)) + actual = next(db.query("select sqlite_version() as v"))["v"] + assert actual == as_string diff --git a/tests/test_register_function.py b/tests/test_register_function.py index 19af0b6..d86c7b6 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -34,16 +34,31 @@ def test_register_function_deterministic(fresh_db): @pytest.mark.skipif( sys.version_info < (3, 8), reason="deterministic=True was added in Python 3.8" ) -def test_register_function_deterministic_registered(fresh_db): +@pytest.mark.parametrize( + "fake_sqlite_version,should_use_deterministic", + ( + ("3.36.0", True), + ("3.8.3", True), + ("3.8.2", False), + ), +) +def test_register_function_deterministic_registered( + fresh_db, fake_sqlite_version, should_use_deterministic +): fresh_db.conn = MagicMock() fresh_db.conn.create_function = MagicMock() + fresh_db.conn.execute().fetchall.return_value = [(fake_sqlite_version,)] @fresh_db.register_function(deterministic=True) def to_lower_2(s): return s.lower() + expected_kwargs = {} + if should_use_deterministic: + expected_kwargs = dict(deterministic=True) + fresh_db.conn.create_function.assert_called_with( - "to_lower_2", 1, to_lower_2, deterministic=True + "to_lower_2", 1, to_lower_2, **expected_kwargs ) From b2b04aec0119a07f796652565966e6c910062eeb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Mar 2022 22:34:34 -0800 Subject: [PATCH 0571/1004] Release 3.25 Refs #343, #408 --- docs/changelog.rst | 10 ++++++++++ setup.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a2a024b..5d2667e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,16 @@ Changelog =========== +.. _v3_25: + +3.25 (2022-03-01) +----------------- + +- New ``hash_id_columns=`` parameter for creating a primary key that's a hash of the content of specific columns - see :ref:`python_api_hash` for details. (:issue:`343`) +- New :ref:`db.sqlite_version ` property, returning a tuple of integers representing the version of SQLite, for example ``(3, 38, 0)``. +- Fixed a bug where :ref:`register_function(deterministic=True) ` caused errors on versions of SQLite prior to 3.8.3. (:issue:`408`) +- New documented :ref:`hash_record(record, keys=...) ` function. + .. _v3_24: 3.24 (2022-02-15) diff --git a/setup.py b/setup.py index 9db7db1..62f8f2a 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.24" +VERSION = "3.25" def get_long_description(): From 7f56f90d3030a4cf1d57a73e21e06843d4855e63 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Mar 2022 23:01:07 -0800 Subject: [PATCH 0572/1004] Fixed rST mistake --- docs/python-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 6c35ef3..bd0265b 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1601,7 +1601,7 @@ The hash will be created using all of the column values. To create a hash using hash_id_columns=("name", "twitter") ) -The `hash_id=` parameter is optional if you specify ``hash_id_columns=`` - it will default to putting the hash in a column called ``id``. +The ``hash_id=`` parameter is optional if you specify ``hash_id_columns=`` - it will default to putting the hash in a column called ``id``. You can manually calculate these hashes using the :ref:`hash_record(record, keys=...) ` utility function. From 26e6d2622c57460a24ffdd0128bbaac051d51a5f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Mar 2022 09:38:34 -0800 Subject: [PATCH 0573/1004] Use :param x: for docstring comments, refs #413 --- docs/conf.py | 1 + sqlite_utils/db.py | 452 ++++++++++++++++++++++++++++++------------ sqlite_utils/utils.py | 3 + 3 files changed, 327 insertions(+), 129 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1f5a158..03c13d6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,6 +32,7 @@ from subprocess import Popen, PIPE # ones. extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"] autodoc_member_order = "bysource" +autodoc_typehints = "description" extlinks = { "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"), diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3bc528f..5f83ca1 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -275,17 +275,17 @@ class Database: # Create an in-memory database: dB = Database(memory=True) - - ``filename_or_conn`` - String path to a file, or a ``pathlib.Path`` object, or a + :param filename_or_conn: String path to a file, or a ``pathlib.Path`` object, or a ``sqlite3`` connection - - ``memory`` - set to ``True`` to create an in-memory database - - ``memory_name`` - creates a named in-memory database that can be shared across multiple connections. - - ``recreate`` - set to ``True`` to delete and recreate a file database (**dangerous**) - - ``recursive_triggers`` - defaults to ``True``, which sets ``PRAGMA recursive_triggers=on;`` - + :param memory: set to ``True`` to create an in-memory database + :param memory_name: creates a named in-memory database that can be shared across multiple connections + :param recreate: set to ``True`` to delete and recreate a file database (**dangerous**) + :param recursive_triggers: defaults to ``True``, which sets ``PRAGMA recursive_triggers=on;`` - set to ``False`` to avoid setting this pragma - - ``tracer`` - set a tracer function (``print`` works for this) which will be called with + :param tracer: set a tracer function (``print`` works for this) which will be called with ``sql, parameters`` every time a SQL query is executed - - ``use_counts_table`` - set to ``True`` to use a cached counts table, if available. See - :ref:`python_api_cached_table_counts`. + :param use_counts_table: set to ``True`` to use a cached counts table, if available. See + :ref:`python_api_cached_table_counts` """ _counts_table_name = "_counts" @@ -340,6 +340,8 @@ class Database: db["creatures"].insert({"name": "Cleo"}) See :ref:`python_api_tracing`. + + :param tracer: Callable accepting ``sql`` and ``parameters`` arguments """ prev_tracer = self._tracer self._tracer = tracer or print @@ -352,6 +354,8 @@ class Database: """ ``db[table_name]`` returns a :class:`.Table` object for the table with the specified name. If the table does not exist yet it will be created the first time data is inserted into it. + + :param table_name: The name of the table """ return self.table(table_name) @@ -375,10 +379,11 @@ class Database: def upper(value): return str(value).upper() - - ``deterministic`` - set ``True`` for functions that always returns the same output for a given input - - ``replace`` - set ``True`` to replace an existing function with the same name - otherwise throw an error - See :ref:`python_api_register_function`. + + :param fn: Function to register + :param deterministic: set ``True`` for functions that always returns the same output for a given input + :param replace: set ``True`` to replace an existing function with the same name - otherwise throw an error """ def register(fn): @@ -411,6 +416,9 @@ class Database: Attach another SQLite database file to this connection with the specified alias, equivalent to:: ATTACH DATABASE 'filepath.db' AS alias + + :param alias: Alias name to use + :param filepath: Path to SQLite database file on disk """ attach_sql = """ ATTACH DATABASE '{}' AS [{}]; @@ -422,7 +430,13 @@ class Database: def query( self, sql: str, params: Optional[Union[Iterable, dict]] = None ) -> Generator[dict, None, None]: - "Execute ``sql`` and return an iterable of dictionaries representing each row." + """ + Execute ``sql`` and return an iterable of dictionaries representing each row. + + :param sql: SQL query to execute + :param params: Parameters to use in that query - an iterable for ``where id = ?`` + parameters, or a dictionary for ``where id = :id`` + """ cursor = self.execute(sql, params or tuple()) keys = [d[0] for d in cursor.description] for row in cursor: @@ -431,7 +445,13 @@ class Database: def execute( self, sql: str, parameters: Optional[Union[Iterable, dict]] = None ) -> sqlite3.Cursor: - "Execute SQL query and return a ``sqlite3.Cursor``." + """ + Execute SQL query and return a ``sqlite3.Cursor``. + + :param sql: SQL query to execute + :param parameters: Parameters to use in that query - an iterable for ``where id = ?`` + parameters, or a dictionary for ``where id = :id`` + """ if self._tracer: self._tracer(sql, parameters) if parameters is not None: @@ -440,18 +460,30 @@ class Database: return self.conn.execute(sql) def executescript(self, sql: str) -> sqlite3.Cursor: - "Execute multiple SQL statements separated by ; and return the ``sqlite3.Cursor``." + """ + Execute multiple SQL statements separated by ; and return the ``sqlite3.Cursor``. + + :param sql: SQL to execute + """ if self._tracer: self._tracer(sql, None) return self.conn.executescript(sql) def table(self, table_name: str, **kwargs) -> Union["Table", "View"]: - "Return a table object, optionally configured with default options." + """ + Return a table object, optionally configured with default options. + + :param table_name: Name of the table + """ klass = View if table_name in self.view_names() else Table return klass(self, table_name, **kwargs) def quote(self, value: str) -> str: - "Apply SQLite string quoting to a value, including wrappping it in single quotes." + """ + Apply SQLite string quoting to a value, including wrappping it in single quotes. + + :param value: String to quote + """ # Normally we would use .execute(sql, [params]) for escaping, but # occasionally that isn't available - most notable when we need # to include a "... DEFAULT 'value'" in a column definition. @@ -462,16 +494,19 @@ class Database: ).fetchone()[0] def quote_fts(self, query: str) -> str: - "Escape special characters in a SQLite full-text search query" - # NOTE: This is not a query validator for FTS. Sqlite has - # a well defined query syntax here: - # https://www2.sqlite.org/fts5.html#full_text_query_syntax - # but this function just aggressively quotes strings - # to ensure that they are valid. In particular, passing - # queries which make use of the query syntax will be incorrect, - # e.g 'NEAR(one, two, 3)'. + """ + Escape special characters in a SQLite full-text search query. - # If query has unbalanced ", add one at end + This works by surrounding each token within the query with double + quotes, in order to avoid words like ``NOT`` and ``OR`` having + special meaning as defined by the FTS query syntax here: + + https://www.sqlite.org/fts5.html#full_text_query_syntax + + If the query has unbalanced ``"`` characters, adds one at end. + + :param query: String to escape + """ if query.count('"') % 2: query += '"' bits = _quote_fts_re.split(query) @@ -481,7 +516,12 @@ class Database: ) def table_names(self, fts4: bool = False, fts5: bool = False) -> List[str]: - "A list of string table names in this database." + """ + List of string table names in this database. + + :param fts4: Only return tables that are part of FTS4 indexes + :param fts5: Only return tables that are part of FTS5 indexes + """ where = ["type = 'table'"] if fts4: where.append("sql like '%USING FTS4%'") @@ -491,7 +531,7 @@ class Database: return [r[0] for r in self.execute(sql).fetchall()] def view_names(self) -> List[str]: - "A list of string view names in this database." + "List of string view names in this database." return [ r[0] for r in self.execute( @@ -501,17 +541,17 @@ class Database: @property def tables(self) -> List["Table"]: - "A list of Table objects in this database." + "List of Table objects in this database." return cast(List["Table"], [self[name] for name in self.table_names()]) @property def views(self) -> List["View"]: - "A list of View objects in this database." + "List of View objects in this database." return cast(List["View"], [self[name] for name in self.view_names()]) @property def triggers(self) -> List[Trigger]: - "A list of ``(name, table_name, sql)`` tuples representing triggers in this database." + "List of ``(name, table_name, sql)`` tuples representing triggers in this database." return [ Trigger(*r) for r in self.execute( @@ -526,7 +566,7 @@ class Database: @property def schema(self) -> str: - "SQL schema for this database" + "SQL schema for this database." sqls = [] for row in self.execute( "select sql from sqlite_master where sql is not null" @@ -553,22 +593,28 @@ class Database: @property def sqlite_version(self) -> Tuple[int, ...]: - "Version of SQLite, as a tuple of integers e.g. (3, 36, 0)" + "Version of SQLite, as a tuple of integers for example ``(3, 36, 0)``." row = self.execute("select sqlite_version()").fetchall()[0] return tuple(map(int, row[0].split("."))) @property def journal_mode(self) -> str: - "Current ``journal_mode`` of this database." + """ + Current ``journal_mode`` of this database. + + https://www.sqlite.org/pragma.html#pragma_journal_mode + """ return self.execute("PRAGMA journal_mode;").fetchone()[0] def enable_wal(self): - "Set ``journal_mode`` to ``'wal'`` to enable Write-Ahead Log mode." + """ + Sets ``journal_mode`` to ``'wal'`` to enable Write-Ahead Log mode. + """ if self.journal_mode != "wal": self.execute("PRAGMA journal_mode=wal;") def disable_wal(self): - "Set ``journal_mode`` back to ``'delete'`` to disable Write-Ahead Log mode." + "Sets ``journal_mode`` back to ``'delete'`` to disable Write-Ahead Log mode." if self.journal_mode != "delete": self.execute("PRAGMA journal_mode=delete;") @@ -594,6 +640,8 @@ class Database: """ Return ``{table_name: count}`` dictionary of cached counts for specified tables, or all tables if ``tables`` not provided. + + :param tables: Subset list of tables to return counts for. """ sql = "select [table], count from {}".format(self._counts_table_name) if tables: @@ -680,7 +728,21 @@ class Database: extracts: Optional[Union[Dict[str, str], List[str]]] = None, if_not_exists: bool = False, ) -> str: - "Returns the SQL ``CREATE TABLE`` statement for creating the specified table." + """ + Returns the SQL ``CREATE TABLE`` statement for creating the specified table. + + :param name: Name of table + :param columns: Dictionary mapping column names to their types, for example ``{"name": str, "age": int}`` + :param pk: String name of column to use as a primary key, or a tuple of strings for a compound primary key covering multiple columns + :param foreign_keys: List of foreign key definitions for this table + :param column_order: List specifying which columns should come first + :param not_null: List of columns that should be created as ``NOT NULL`` + :param defaults: Dictionary specifying default values for columns + :param hash_id: Name of column to be used as a primary key containing a hash of the other columns + :param hash_id_columns: List of columns to be used when calculating the hash ID for a row + :param extracts: List or dictionary of columns to be extracted during inserts, see :ref:`python_api_extracts` + :param if_not_exists: Use ``CREATE TABLE IF NOT EXISTS`` + """ if hash_id_columns and (hash_id is None): hash_id = "id" foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) @@ -800,6 +862,18 @@ class Database: Create a table with the specified name and the specified ``{column_name: type}`` columns. See :ref:`python_api_explicit_create`. + + :param name: Name of table + :param columns: Dictionary mapping column names to their types, for example ``{"name": str, "age": int}`` + :param pk: String name of column to use as a primary key, or a tuple of strings for a compound primary key covering multiple columns + :param foreign_keys: List of foreign key definitions for this table + :param column_order: List specifying which columns should come first + :param not_null: List of columns that should be created as ``NOT NULL`` + :param defaults: Dictionary specifying default values for columns + :param hash_id: Name of column to be used as a primary key containing a hash of the other columns + :param hash_id_columns: List of columns to be used when calculating the hash ID for a row + :param extracts: List or dictionary of columns to be extracted during inserts, see :ref:`python_api_extracts` + :param if_not_exists: Use ``CREATE TABLE IF NOT EXISTS`` """ sql = self.create_table_sql( name=name, @@ -833,8 +907,10 @@ class Database: """ Create a new SQL view with the specified name - ``sql`` should start with ``SELECT ...``. - - ``ignore`` - set to ``True`` to do nothing if a view with this name already exists - - ``replace`` - set to ``True`` to replace the view if one with this name already exists + :param name: Name of the view + :param sql: SQL ``SELECT`` query to use for this view. + :param ignore: Set to ``True`` to do nothing if a view with this name already exists + :param replace: Set to ``True`` to replace the view if one with this name already exists """ assert not ( ignore and replace @@ -858,6 +934,9 @@ class Database: Given two table names returns the name of tables that could define a many-to-many relationship between those two tables, based on having foreign keys to both of the provided tables. + + :param table: Table name + :param other_table: Other table name """ candidates = [] tables = {table, other_table} @@ -872,8 +951,8 @@ class Database: """ See :ref:`python_api_add_foreign_keys`. - ``foreign_keys`` should be a list of ``(table, column, other_table, other_column)`` - tuples, see :ref:`python_api_add_foreign_keys`. + :param foreign_keys: A list of ``(table, column, other_table, other_column)`` + tuples """ # foreign_keys is a list of explicit 4-tuples assert all( @@ -957,7 +1036,11 @@ class Database: self.execute("VACUUM;") def analyze(self, name=None): - "Run ``ANALYZE`` against the entire database or a named table or index." + """ + Run ``ANALYZE`` against the entire database or a named table or index. + + :param name: Run ``ANALYZE`` against this specific named table or index + """ sql = "ANALYZE" if name is not None: sql += " [{}]".format(name) @@ -969,7 +1052,7 @@ class Database: The ``path`` argument should be an absolute path to the compiled extension, which can be found using ``find_spatialite``. - Returns true if SpatiaLite was successfully initialized. + Returns ``True`` if SpatiaLite was successfully initialized. .. code-block:: python @@ -990,6 +1073,7 @@ class Database: db = Database("mydb.db") db.init_spatialite("./local/mod_spatialite.dylib") + :param path: Path to SpatiaLite module on disk """ if path is None: path = find_spatialite() @@ -1018,7 +1102,13 @@ class Queryable: where: str = None, where_args: Optional[Union[Iterable, dict]] = None, ) -> int: - "Executes ``SELECT count(*) FROM table WHERE ...`` and returns a count." + """ + Executes ``SELECT count(*) FROM table WHERE ...`` and returns a count. + + :param where: SQL where fragment to use, for example ``id > ?`` + :param where_args: Parameters to use with that fragment - an iterable for ``id > ?`` + parameters, or a dictionary for ``id > :id`` + """ sql = "select count(*) from [{}]".format(self.name) if where is not None: sql += " where " + where @@ -1050,14 +1140,15 @@ class Queryable: """ Iterate over every row in this table or view that matches the specified where clause. - - ``where`` - a SQL fragment to use as a ``WHERE`` clause, for example ``age > ?`` or ``age > :age``. - - ``where_args`` - a list of arguments (if using ``?``) or a dictionary (if using ``:age``). - - ``order_by`` - optional column or fragment of SQL to order by. - - ``select`` - optional comma-separated list of columns to select. - - ``limit`` - optional integer number of rows to limit to. - - ``offset`` - optional integer for SQL offset. - Returns each row as a dictionary. See :ref:`python_api_rows` for more details. + + :param where: SQL where fragment to use, for example ``id > ?`` + :param where_args: Parameters to use with that fragment - an iterable for ``id > ?`` + parameters, or a dictionary for ``id > :id`` + :param order_by: Column or fragment of SQL to order by + :param select: Comma-separated list of columns to select - defaults to ``*`` + :param limit: Integer number of rows to limit to + :param offset: Integer for SQL offset """ if not self.exists(): return @@ -1083,7 +1174,17 @@ class Queryable: limit: int = None, offset: int = None, ) -> Generator[Tuple[Any, Dict], None, None]: - "Like ``.rows_where()`` but returns ``(pk, row)`` pairs - ``pk`` can be a single value or tuple." + """ + Like ``.rows_where()`` but returns ``(pk, row)`` pairs - ``pk`` can be a single value or tuple. + + :param where: SQL where fragment to use, for example ``id > ?`` + :param where_args: Parameters to use with that fragment - an iterable for ``id > ?`` + parameters, or a dictionary for ``id > :id`` + :param order_by: Column or fragment of SQL to order by + :param select: Comma-separated list of columns to select - defaults to ``*`` + :param limit: Integer number of rows to limit to + :param offset: Integer for SQL offset + """ column_names = [column.name for column in self.columns] pks = [column.name for column in self.columns if column.is_pk] if not pks: @@ -1205,9 +1306,9 @@ class Table(Queryable): """ Return row (as dictionary) for the specified primary key. - Primary key can be a single value, or a tuple for tables with a compound primary key. + Raises ``sqlite_utils.db.NotFoundError`` if a matching row cannot be found. - Raises ``NotFoundError`` if a matching row cannot be found. + :param pk_values: A single value, or a tuple of values for tables that have a compound primary key """ if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] @@ -1340,6 +1441,17 @@ class Table(Queryable): Create a table with the specified columns. See :ref:`python_api_explicit_create` for full details. + + :param columns: Dictionary mapping column names to their types, for example ``{"name": str, "age": int}`` + :param pk: String name of column to use as a primary key, or a tuple of strings for a compound primary key covering multiple columns + :param foreign_keys: List of foreign key definitions for this table + :param column_order: List specifying which columns should come first + :param not_null: List of columns that should be created as ``NOT NULL`` + :param defaults: Dictionary specifying default values for columns + :param hash_id: Name of column to be used as a primary key containing a hash of the other columns + :param hash_id_columns: List of columns to be used when calculating the hash ID for a row + :param extracts: List or dictionary of columns to be extracted during inserts, see :ref:`python_api_extracts` + :param if_not_exists: Use ``CREATE TABLE IF NOT EXISTS`` """ columns = {name: value for (name, value) in columns.items()} with self.db.conn: @@ -1361,20 +1473,30 @@ class Table(Queryable): def transform( self, *, - types=None, - rename=None, - drop=None, - pk=DEFAULT, - not_null=None, - defaults=None, - drop_foreign_keys=None, - column_order=None, + types: Optional[dict] = None, + rename: Optional[dict] = None, + drop: Optional[Iterable] = None, + pk: Optional[Any] = None, + not_null: Optional[Set[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + drop_foreign_keys: Optional[Iterable] = None, + column_order: Optional[List[str]] = None, ) -> "Table": """ Apply an advanced alter table, including operations that are not supported by ``ALTER TABLE`` in SQLite itself. See :ref:`python_api_transform` for full details. + + :param types: Columns that should have their type changed, for example ``{"weight": float}`` + :param rename: Columns to rename, for example ``{"headline": "title"}`` + :param drop: Columns to drop + :param pk: New primary key for the table + :param not_null: Columns to set as ``NOT NULL`` + :param defaults: Default values for columns + :param drop_foreign_keys: Names of columns that should have their foreign key constraints removed + :param column_order: List of strings specifying a full or partial column order + to use when creating the table. """ assert self.exists(), "Cannot transform a table that doesn't exist yet" sqls = self.transform_sql( @@ -1417,7 +1539,19 @@ class Table(Queryable): column_order=None, tmp_suffix=None, ) -> List[str]: - "Returns a list of SQL statements that would be executed in order to apply this transformation." + """ + Return a list of SQL statements that should be executed in order to apply this transformation. + + :param types: Columns that should have their type changed, for example ``{"weight": float}`` + :param rename: Columns to rename, for example ``{"headline": "title"}`` + :param drop: Columns to drop + :param pk: New primary key for the table + :param not_null: Columns to set as ``NOT NULL`` + :param defaults: Default values for columns + :param drop_foreign_keys: Names of columns that should have their foreign key constraints removed + :param column_order: List of strings specifying a full or partial column order + to use when creating the table. + """ types = types or {} rename = rename or {} drop = drop or set() @@ -1534,6 +1668,11 @@ class Table(Queryable): Extract specified columns into a separate table. See :ref:`python_api_extract` for details. + + :param columns: Single column or list of columns that should be extracted + :param table: Name of table in which the new records should be created + :param fk_column: Name of the foreign key column to populate in the original table + :param rename: Dictionary of columns that should be renamed when populating the new table """ rename = rename or {} if isinstance(columns, str): @@ -1638,14 +1777,14 @@ class Table(Queryable): """ Create an index on this table. - - ``columns`` - a single columns or list of columns to index. These can be strings or, + :param columns: A single columns or list of columns to index. These can be strings or, to create an index using the column in descending order, ``db.DescIndex(column_name)`` objects. - - ``index_name`` - the name to use for the new index. Defaults to the column names joined on ``_``. - - ``unique`` - should the index be marked as unique, forcing unique values? - - ``if_not_exists`` - only create the index if one with that name does not already exist. - - ``find_unique_name`` - if ``index_name`` is not provided and the automatically derived name + :param index_name: The name to use for the new index. Defaults to the column names joined on ``_``. + :param unique: Should the index be marked as unique, forcing unique values? + :param if_not_exists: Only create the index if one with that name does not already exist. + :param find_unique_name: If ``index_name`` is not provided and the automatically derived name already exists, keep incrementing a suffix number to find an available name. - - ``analyze`` - run ``ANALYZE`` against this index after creating it. + :param analyze: Run ``ANALYZE`` against this index after creating it. See :ref:`python_api_create_index`. """ @@ -1706,9 +1845,22 @@ class Table(Queryable): return self def add_column( - self, col_name: str, col_type=None, fk=None, fk_col=None, not_null_default=None + self, + col_name: str, + col_type: Optional[Any] = None, + fk: Optional[str] = None, + fk_col: Optional[str] = None, + not_null_default: Optional[Any] = None, ): - "Add a column to this table. See :ref:`python_api_add_column`." + """ + Add a column to this table. See :ref:`python_api_add_column`. + + :param col_name: Name of the new column + :param col_type: Column type - a Python type such as ``str`` or a SQLite type string such as ``"BLOB"`` + :param fk: Name of a table that this column should be a foreign key reference to + :param fk_col: Column in the foreign key table that this should reference + :param not_null_default: Set this column to ``not null`` and give it this default value + """ fk_col_type = None if fk is not None: # fk must be a valid table @@ -1744,7 +1896,11 @@ class Table(Queryable): return self def drop(self, ignore: bool = False): - "Drop this table. ``ignore=True`` means errors will be ignored." + """ + Drop this table. + + :param ignore: Set to ``True`` to ignore the error if the table does not exist + """ try: self.db.execute("DROP TABLE [{}]".format(self.name)) except sqlite3.OperationalError: @@ -1760,6 +1916,8 @@ class Table(Queryable): a ``tag`` table, if one exists. If no candidates can be found, raises a ``NoObviousTable`` exception. + + :param column: Name of column """ column = column.lower() possibilities = [column] @@ -1800,10 +1958,10 @@ class Table(Queryable): """ Alter the schema to mark the specified column as a foreign key to another table. - - ``column`` - the column to mark as a foreign key. - - ``other_table`` - the table it refers to - if omitted, will be guessed based on the column name. - - ``other_column`` - the column on the other table it - if omitted, will be guessed. - - ``ignore`` - set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. + :param column: The column to mark as a foreign key. + :param other_table: The table it refers to - if omitted, will be guessed based on the column name. + :param other_column: The column on the other table it - if omitted, will be guessed. + :param ignore: Set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. """ # Ensure column exists if column not in self.columns_dict: @@ -1911,13 +2069,13 @@ class Table(Queryable): """ Enable SQLite full-text search against the specified columns. - - ``columns`` - list of column names to include in the search index. - - ``fts_version`` - FTS version to use - defaults to ``FTS5`` but you may want ``FTS4`` for older SQLite versions. - - ``create_triggers`` - should triggers be created to keep the search index up-to-date? Defaults to ``False``. - - ``tokenize`` - custom SQLite tokenizer to use, for example ``"porter"`` to enable Porter stemming. - - ``replace`` - should any existing FTS index for this table be replaced by the new one? - See :ref:`python_api_fts` for more details. + + :param columns: List of column names to include in the search index. + :param fts_version: FTS version to use - defaults to ``FTS5`` but you may want ``FTS4`` for older SQLite versions. + :param create_triggers: Should triggers be created to keep the search index up-to-date? Defaults to ``False``. + :param tokenize: Custom SQLite tokenizer to use, for example ``"porter"`` to enable Porter stemming. + :param replace: Should any existing FTS index for this table be replaced by the new one? """ create_fts_sql = ( textwrap.dedent( @@ -1990,6 +2148,8 @@ class Table(Queryable): """ Update the associated SQLite full-text search index with the latest data from the table for the specified columns. + + :param columns: Columns to populate the data for """ sql = ( textwrap.dedent( @@ -2089,7 +2249,14 @@ class Table(Queryable): limit: Optional[int] = None, offset: Optional[int] = None, ) -> str: - "Return SQL string that can be used to execute searches against this table." + """ " + Return SQL string that can be used to execute searches against this table. + + :param columns: Columns to search against + :param order_by: Column or SQL expression to sort by + :param limit: SQL limit + :param offset: SQL offset + """ # Pick names for table and rank column that don't clash original = "original_" if self.name == "original" else "original" columns_sql = "*" @@ -2159,12 +2326,12 @@ class Table(Queryable): Execute a search against this table using SQLite full-text search, returning a sequence of dictionaries for each row. - - ``q`` - terms to search for - - ``order_by`` - defaults to order by rank, or specify a column here. - - ``columns`` - list of columns to return, defaults to all columns. - - ``limit`` - optional integer limit for returned rows. - - ``offset`` - optional integer SQL offset. - - ``quote`` - apply quoting to disable any special characters in the search query + :param q: Terms to search for + :param order_by: Defaults to order by rank, or specify a column here. + :param columns: List of columns to return, defaults to all columns. + :param limit: Optional integer limit for returned rows. + :param offset: Optional integer SQL offset. + :param quote: Apply quoting to disable any special characters in the search query See :ref:`python_api_fts_search`. """ @@ -2185,7 +2352,11 @@ class Table(Queryable): return self._defaults[key] if value is DEFAULT else value def delete(self, pk_values: Union[list, tuple, str, int, float]) -> "Table": - "Delete row matching the specified primary key." + """ + Delete row matching the specified primary key. + + :param pk_values: A single value, or a tuple of values for tables that have a compound primary key + """ if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] self.get(pk_values) @@ -2206,11 +2377,12 @@ class Table(Queryable): """ Delete rows matching the specified where clause, or delete all rows in the table. - - ``where`` - a SQL fragment to use as a ``WHERE`` clause, for example ``age > ?`` or ``age > :age``. - - ``where_args`` - a list of arguments (if using ``?``) or a dictionary (if using ``:age``). - - ``analyze`` - set to ``True`` to run ``ANALYZE`` after the rows have been deleted. - See :ref:`python_api_delete_where`. + + :param where: SQL where fragment to use, for example ``id > ?`` + :param where_args: Parameters to use with that fragment - an iterable for ``id > ?`` + parameters, or a dictionary for ``id > :id`` + :param analyze: Set to ``True`` to run ``ANALYZE`` after the rows have been deleted. """ if not self.exists(): return self @@ -2232,14 +2404,14 @@ class Table(Queryable): """ Execute a SQL ``UPDATE`` against the specified row. - - ``pk_values`` - the primary key of an individual record - can be a tuple if the - table has a compound primary key. - - ``updates`` - a dictionary mapping columns to their updated values. - - ``alter`` - set to ``True`` to add any missing columns. - - ``conversions`` - optional dictionary of SQL functions to apply during the update, for example - ``{"mycolumn": "upper(?)"}``. - See :ref:`python_api_update`. + + :param pk_values: The primary key of an individual record - can be a tuple if the + table has a compound primary key. + :param updates: A dictionary mapping columns to their updated values. + :param alter: Set to ``True`` to add any missing columns. + :param conversions: Optional dictionary of SQL functions to apply during the update, for example + ``{"mycolumn": "upper(?)"}``. """ updates = updates or {} conversions = conversions or {} @@ -2293,18 +2465,18 @@ class Table(Queryable): """ Apply conversion function ``fn`` to every value in the specified columns. - - ``columns`` - a single column or list of string column names to convert. - - ``fn`` - a callable that takes a single argument, ``value``, and returns it converted. - - ``output`` - optional string column name to write the results to (defaults to the input column). - - ``output_type`` - if the output column needs to be created, this is the type that will be used + :param columns: A single column or list of string column names to convert. + :param fn: A callable that takes a single argument, ``value``, and returns it converted. + :param output: Optional string column name to write the results to (defaults to the input column). + :param output_type: If the output column needs to be created, this is the type that will be used for the new column. - - ``drop`` - boolean, should the original column be dropped once the conversion is complete? - - ``multi`` - boolean, if ``True`` the return value of ``fn(value)`` will be expected to be a + :param drop: Should the original column be dropped once the conversion is complete? + :param multi: If ``True`` the return value of ``fn(value)`` will be expected to be a dictionary, and new columns will be created for each key of that dictionary. - - ``where`` - a SQL fragment to use as a ``WHERE`` clause to limit the rows to which the conversion + :param where: SQL fragment to use as a ``WHERE`` clause to limit the rows to which the conversion is applied, for example ``age > ?`` or ``age > :age``. - - ``where_args`` - a list of arguments (if using ``?``) or a dictionary (if using ``:age``). - - ``show_progress`` - boolean, should a progress bar be displayed? + :param where_args: List of arguments (if using ``?``) or a dictionary (if using ``:age``). + :param show_progress: Should a progress bar be displayed? See :ref:`python_api_convert`. """ @@ -2627,23 +2799,24 @@ class Table(Queryable): Each of them defaults to ``DEFAULT``, which indicates that the default setting for the current ``Table`` object (specified in the table constructor) should be used. - - ``pk`` - if creating the table, which column should be the primary key. - - ``foreign_keys`` - see :ref:`python_api_foreign_keys`. - - ``column_order`` - optional list of strings specifying a full or partial column order + :param record: Dictionary record to be inserted + :param pk: If creating the table, which column should be the primary key. + :param foreign_keys: See :ref:`python_api_foreign_keys`. + :param column_order: List of strings specifying a full or partial column order to use when creating the table. - - ``not_null`` - optional set of strings specifying columns that should be ``NOT NULL``. - - ``defaults`` - optional dictionary specifying default values for specific columns. - - ``hash_id`` - optional name of a column to create and use as a primary key, where the + :param not_null: Set of strings specifying columns that should be ``NOT NULL``. + :param defaults: Dictionary specifying default values for specific columns. + :param hash_id: Name of a column to create and use as a primary key, where the value of thet primary key will be derived as a SHA1 hash of the other column values in the record. ``hash_id="id"`` is a common column name used for this. - - ``alter`` - boolean, should any missing columns be added automatically? - - ``ignore`` - boolean, if a record already exists with this primary key, ignore this insert. - - ``replace`` - boolean, if a record already exists with this primary key, replace it with this new record. - - ``extracts`` - a list of columns to extract to other tables, or a dictionary that maps + :param alter: Boolean, should any missing columns be added automatically? + :param ignore: Boolean, if a record already exists with this primary key, ignore this insert. + :param replace: Boolean, if a record already exists with this primary key, replace it with this new record. + :param extracts: A list of columns to extract to other tables, or a dictionary that maps ``{column_name: other_table_name}``. See :ref:`python_api_extracts`. - - ``conversions`` - dictionary specifying SQL conversion functions to be applied to the data while it + :param conversions: Dictionary specifying SQL conversion functions to be applied to the data while it is being inserted, for example ``{"name": "upper(?)"}``. See :ref:`python_api_conversions`. - - ``columns`` - dictionary over-riding the detected types used for the columns, for example + :param columns: Dictionary over-riding the detected types used for the columns, for example ``{"age": int, "weight": float}``. """ return self.insert_all( @@ -2904,9 +3077,12 @@ class Table(Queryable): be included only if the record is being created for the first time. These will be ignored on subsequent lookup calls for records that already exist. + All other keyword arguments are passed through to ``.insert()``. + See :ref:`python_api_lookup_tables` for more details. - All other keyword arguments are passed through to ``.insert()``. + :param lookup_values: Dictionary specifying column names and values to use for the lookup + :param extra_values: Additional column values to be used only if creating a new record """ assert isinstance(lookup_values, dict) if extra_values is not None: @@ -2978,13 +3154,13 @@ class Table(Queryable): See :ref:`python_api_m2m` for details. - - ``other_table`` - the name of the table to insert the new records into. - - ``record_or_iterable`` - a single dictionary record to insert, or a list of records. - - ``pk`` - the primary key to use if creating ``other_table``. - - ``lookup`` - same dictionary as for ``.lookup()``, to create a many-to-many lookup table. - - ``m2m_table`` - the string name to use for the many-to-many table, defaults to creating + :param other_table: The name of the table to insert the new records into. + :param record_or_iterable: A single dictionary record to insert, or a list of records. + :param pk: The primary key to use if creating ``other_table``. + :param lookup: Same dictionary as for ``.lookup()``, to create a many-to-many lookup table. + :param m2m_table: The string name to use for the many-to-many table, defaults to creating this automatically based on the names of the two tables. - - ``alter`` - set to ``True`` to add any missing columns on ``other_table`` if that table + :param alter: Set to ``True`` to add any missing columns on ``other_table`` if that table already exists. """ if isinstance(other_table, str): @@ -3053,6 +3229,11 @@ class Table(Queryable): Return statistics about the specified column. See :ref:`python_api_analyze_column`. + + :param column: Column to analyze + :param common_limit: Show this many column values + :param value_truncate: Truncate display of common values to this many characters + :param total_rows: Optimization - pass the total number of rows in the table to save running a fresh ``count(*)`` query """ db = self.db table = self.name @@ -3133,7 +3314,7 @@ class Table(Queryable): `SRID 4326 `__. This can be customized using the ``column_name``, ``srid`` and ``not_null`` arguments. - Returns True if the column was successfully added, False if not. + Returns ``True`` if the column was successfully added, ``False`` if not. .. code-block:: python @@ -3147,6 +3328,11 @@ class Table(Queryable): table = db["locations"].create({"name": str}) table.add_geometry_column("geometry", "POINT") + :param column_name: Name of column to add + :param geometry_type: Type of geometry column, for example ``"GEOMETRY"`` or ``"POINT" or ``"POLYGON"`` + :param srid: Integer SRID, defaults to 4326 for WGS84 + :param coord_dimension: Dimensions to use, defaults to ``"XY"`` - set to ``"XYZ"`` to work in three dimensions + :param not_null: Should the column be ``NOT NULL`` """ cursor = self.db.execute( "SELECT AddGeometryColumn(?, ?, ?, ?, ?, ?);", @@ -3185,6 +3371,7 @@ class Table(Queryable): # outputs: # CREATE VIRTUAL TABLE "idx_locations_geometry" USING rtree(pkid, xmin, xmax, ymin, ymax) + :param column_name: Geometry column to create the spatial index against """ if f"idx_{self.name}_{column_name}" in self.db.table_names(): return False @@ -3206,6 +3393,12 @@ class View(Queryable): ) def drop(self, ignore=False): + """ + Drop this view. + + :param ignore: Set to ``True`` to ignore the error if the view does not exist + """ + try: self.db.execute("DROP VIEW [{}]".format(self.name)) except sqlite3.OperationalError: @@ -3213,6 +3406,7 @@ class View(Queryable): raise def enable_fts(self, *args, **kwargs): + "``enable_fts()`` is supported on tables but not on views." raise NotImplementedError( "enable_fts() is supported on tables but not on views" ) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 4ec8b9f..d35cc05 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -365,6 +365,9 @@ def hash_record(record: Dict, keys: Optional[Iterable[str]] = None): {"name": "Cleo", "twitter": "CleoPaws", "age": 7}, keys=("name", "twitter") ) + + :param record: Record to generate a hash for + :param keys: Subset of keys to use for that hash """ to_hash = record if keys is not None: From 6fd7c138e2dd76cc91f99f6fe2f80636642652de Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Mar 2022 09:54:17 -0800 Subject: [PATCH 0574/1004] Fixed .transform() method which I broke in #413 --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 5f83ca1..4739fc9 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1476,7 +1476,7 @@ class Table(Queryable): types: Optional[dict] = None, rename: Optional[dict] = None, drop: Optional[Iterable] = None, - pk: Optional[Any] = None, + pk: Optional[Any] = DEFAULT, not_null: Optional[Set[str]] = None, defaults: Optional[Dict[str, Any]] = None, drop_foreign_keys: Optional[Iterable] = None, From 40b76f6f56e4a00da023396999a25989c83d91a6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Mar 2022 10:08:20 -0800 Subject: [PATCH 0575/1004] Release 3.25.1 Refs #413 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 62f8f2a..d562d40 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.25" +VERSION = "3.25.1" def get_long_description(): From 9388edf57aa15719095e3cf0952c1653cd070c9b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Mar 2022 10:34:37 -0800 Subject: [PATCH 0576/1004] Changelog item for 3.25.1 Refs #413, #414 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5d2667e..916b9df 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _v3_25_1: + +3.25.1 (2022-03-11) +------------------- + +- Improved display of type information and parameters in the :ref:`API reference documentation `. (:issue:`413`) + .. _v3_25: 3.25 (2022-03-01) From 433813612ff9b4b501739fd7543bef0040dd51fe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Mar 2022 13:44:07 -0800 Subject: [PATCH 0577/1004] Move sqls=[] closer to where it is populated --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 4739fc9..3ffed9f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1570,7 +1570,6 @@ class Table(Queryable): new_column_pairs.append((new_name, type_)) copy_from_to[name] = new_name - sqls = [] if pk is DEFAULT: pks_renamed = tuple( rename.get(p.name) or p.name for p in self.columns if p.is_pk @@ -1624,6 +1623,7 @@ class Table(Queryable): if column_order is not None: column_order = [rename.get(col) or col for col in column_order] + sqls = [] sqls.append( self.db.create_table_sql( new_table_name, From 878d5f5cea3455b4d135a9664ccad6b673354812 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Mar 2022 21:01:35 -0700 Subject: [PATCH 0578/1004] errors=r.SET_NULL/r.IGNORE options for parsedate/parsedatetime, closes #416 --- docs/cli-reference.rst | 20 +++++++++++---- docs/cli.rst | 12 +++++++-- sqlite_utils/cli.py | 10 +++++--- sqlite_utils/recipes.py | 57 +++++++++++++++++++++++++++++++++-------- tests/test_docs.py | 8 +++++- tests/test_recipes.py | 23 +++++++++++++++++ 6 files changed, 109 insertions(+), 21 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 19b714b..c3f7a24 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -547,15 +547,25 @@ See :ref:`cli_convert`. r.jsonsplit(value, delimiter=',', type=) - Convert a string like a,b,c into a JSON array ["a", "b", "c"] + Convert a string like a,b,c into a JSON array ["a", "b", "c"] - r.parsedate(value, dayfirst=False, yearfirst=False) + r.parsedate(value, dayfirst=False, yearfirst=False, errors=None) - Parse a date and convert it to ISO date format: yyyy-mm-dd + Parse a date and convert it to ISO date format: yyyy-mm-dd +  + - dayfirst=True: treat xx as the day in xx/yy/zz + - yearfirst=True: treat xx as the year in xx/yy/zz + - errors=r.IGNORE to ignore values that cannot be parsed + - errors=r.SET_NULL to set values that cannot be parsed to null - r.parsedatetime(value, dayfirst=False, yearfirst=False) + r.parsedatetime(value, dayfirst=False, yearfirst=False, errors=None) - Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS + Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS +  + - dayfirst=True: treat xx as the day in xx/yy/zz + - yearfirst=True: treat xx as the year in xx/yy/zz + - errors=r.IGNORE to ignore values that cannot be parsed + - errors=r.SET_NULL to set values that cannot be parsed to null You can use these recipes like so: diff --git a/docs/cli.rst b/docs/cli.rst index 03689c1..8bd3650 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1303,12 +1303,20 @@ Various built-in recipe functions are available for common operations. These are Would produce an array like this: ``[1.2, 3.0, 4.5]`` -``r.parsedate(value, dayfirst=False, yearfirst=False)`` +``r.parsedate(value, dayfirst=False, yearfirst=False, errors=None)`` Parse a date and convert it to ISO date format: ``yyyy-mm-dd`` In the case of dates such as ``03/04/05`` U.S. ``MM/DD/YY`` format is assumed - you can use ``dayfirst=True`` or ``yearfirst=True`` to change how these ambiguous dates are interpreted. -``r.parsedatetime(value, dayfirst=False, yearfirst=False)`` + Use the ``errors=`` parameter to specify what should happen if a value cannot be parsed. + + By default, if any value cannot be parsed an error will be occurred and all values will be left as they were. + + Set ``errors=r.IGNORE`` to ignore any values that cannot be parsed, leaving them unchanged. + + Set ``errors=r.SET_NULL`` to set any values that cannot be parsed to ``null``. + +``r.parsedatetime(value, dayfirst=False, yearfirst=False, errors=None)`` Parse a datetime and convert it to ISO datetime format: ``yyyy-mm-ddTHH:MM:SS`` These recipes can be used in the code passed to ``sqlite-utils convert`` like this:: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8255b56..0cf0468 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2583,12 +2583,16 @@ def _generate_convert_help(): """ ).strip() recipe_names = [ - n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser") + n + for n in dir(recipes) + if not n.startswith("_") + and n not in ("json", "parser") + and callable(getattr(recipes, n)) ] for name in recipe_names: fn = getattr(recipes, name) - help += "\n\nr.{}{}\n\n {}".format( - name, str(inspect.signature(fn)), fn.__doc__ + help += "\n\nr.{}{}\n\n\b{}".format( + name, str(inspect.signature(fn)), fn.__doc__.rstrip() ) help += "\n\n" help += textwrap.dedent( diff --git a/sqlite_utils/recipes.py b/sqlite_utils/recipes.py index 6918661..ac41954 100644 --- a/sqlite_utils/recipes.py +++ b/sqlite_utils/recipes.py @@ -1,19 +1,56 @@ from dateutil import parser import json - -def parsedate(value, dayfirst=False, yearfirst=False): - "Parse a date and convert it to ISO date format: yyyy-mm-dd" - return ( - parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).date().isoformat() - ) +IGNORE = object() +SET_NULL = object() -def parsedatetime(value, dayfirst=False, yearfirst=False): - "Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS" - return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat() +def parsedate(value, dayfirst=False, yearfirst=False, errors=None): + """ + Parse a date and convert it to ISO date format: yyyy-mm-dd + \b + - dayfirst=True: treat xx as the day in xx/yy/zz + - yearfirst=True: treat xx as the year in xx/yy/zz + - errors=r.IGNORE to ignore values that cannot be parsed + - errors=r.SET_NULL to set values that cannot be parsed to null + """ + try: + return ( + parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst) + .date() + .isoformat() + ) + except parser.ParserError: + if errors is IGNORE: + return value + elif errors is SET_NULL: + return None + else: + raise + + +def parsedatetime(value, dayfirst=False, yearfirst=False, errors=None): + """ + Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS + \b + - dayfirst=True: treat xx as the day in xx/yy/zz + - yearfirst=True: treat xx as the year in xx/yy/zz + - errors=r.IGNORE to ignore values that cannot be parsed + - errors=r.SET_NULL to set values that cannot be parsed to null + """ + try: + return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat() + except parser.ParserError: + if errors is IGNORE: + return value + elif errors is SET_NULL: + return None + else: + raise def jsonsplit(value, delimiter=",", type=str): - 'Convert a string like a,b,c into a JSON array ["a", "b", "c"]' + """ + Convert a string like a,b,c into a JSON array ["a", "b", "c"] + """ return json.dumps([type(s.strip()) for s in value.split(delimiter)]) diff --git a/tests/test_docs.py b/tests/test_docs.py index d760629..7f2e3ed 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -48,7 +48,13 @@ def test_convert_help(): @pytest.mark.parametrize( "recipe", - [n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")], + [ + n + for n in dir(recipes) + if not n.startswith("_") + and n not in ("json", "parser") + and callable(getattr(recipes, n)) + ], ) def test_recipes_are_documented(documented_recipes, recipe): assert recipe in documented_recipes diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 89240a2..afb3e1a 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -1,4 +1,5 @@ from sqlite_utils import recipes +from sqlite_utils.utils import sqlite3 import json import pytest @@ -61,6 +62,28 @@ def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected): ] +@pytest.mark.parametrize("fn", ("parsedate", "parsedatetime")) +@pytest.mark.parametrize("errors", (None, recipes.SET_NULL, recipes.IGNORE)) +def test_dateparse_errors(fresh_db, fn, errors): + fresh_db["example"].insert_all( + [ + {"id": 1, "dt": "invalid"}, + ], + pk="id", + ) + if errors is None: + # Should raise an error + with pytest.raises(sqlite3.OperationalError): + fresh_db["example"].convert("dt", lambda value: getattr(recipes, fn)(value)) + else: + fresh_db["example"].convert( + "dt", lambda value: getattr(recipes, fn)(value, errors=errors) + ) + rows = list(fresh_db["example"].rows) + expected = [{"id": 1, "dt": None if errors is recipes.SET_NULL else "invalid"}] + assert rows == expected + + @pytest.mark.parametrize("delimiter", [None, ";", "-"]) def test_jsonsplit(fresh_db, delimiter): fresh_db["example"].insert_all( From 751ab205ac1f6bcd1b31449d2aca4734abca16c8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Mar 2022 21:18:18 -0700 Subject: [PATCH 0579/1004] Fix for --multi combined with --dry-run, closes #415 --- sqlite_utils/cli.py | 5 ++++- tests/test_cli_convert.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0cf0468..b2a0440 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2676,7 +2676,10 @@ def convert( raise click.ClickException(str(e)) if dry_run: # Pull first 20 values for first column and preview them - db.conn.create_function("preview_transform", 1, lambda v: fn(v) if v else v) + preview = lambda v: fn(v) if v else v + if multi: + preview = lambda v: json.dumps(fn(v), default=repr) if v else v + db.conn.create_function("preview_transform", 1, preview) sql = """ select [{column}] as value, diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index ec8110f..10b4563 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -179,6 +179,42 @@ def test_convert_dryrun(test_db_and_path): assert result.output.strip().split("\n")[-1] == "Would affect 1 row" +def test_convert_multi_dryrun(test_db_and_path): + db_path = test_db_and_path[1] + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "{'foo': 'bar', 'baz': 1}", + "--dry-run", + "--multi", + ], + ) + assert result.exit_code == 0 + assert result.output.strip() == ( + "5th October 2019 12:04\n" + " --- becomes:\n" + '{"foo": "bar", "baz": 1}\n' + "\n" + "6th October 2019 00:05:06\n" + " --- becomes:\n" + '{"foo": "bar", "baz": 1}\n' + "\n" + "\n" + " --- becomes:\n" + "\n" + "\n" + "None\n" + " --- becomes:\n" + "None\n" + "\n" + "Would affect 4 rows" + ) + + @pytest.mark.parametrize("drop", (True, False)) def test_convert_output_column(test_db_and_path, drop): db, db_path = test_db_and_path From 93fa79d30b1531bea281d0eb6b925c4e61bc1aa6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Mar 2022 21:22:09 -0700 Subject: [PATCH 0580/1004] Ignore flake8 lambda errors, refs #415 --- sqlite_utils/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index b2a0440..05ff9d3 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2676,9 +2676,11 @@ def convert( raise click.ClickException(str(e)) if dry_run: # Pull first 20 values for first column and preview them - preview = lambda v: fn(v) if v else v + preview = lambda v: fn(v) if v else v # noqa: E731 if multi: - preview = lambda v: json.dumps(fn(v), default=repr) if v else v + preview = ( + lambda v: json.dumps(fn(v), default=repr) if v else v + ) # noqa: E731 db.conn.create_function("preview_transform", 1, preview) sql = """ select From 396f80fcc60da8dd844577114f7920830a2e5403 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Thu, 24 Mar 2022 17:01:43 -0400 Subject: [PATCH 0581/1004] Ignore common generated files (#419) Thanks, @eyeseast --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 75fa28e..a224f01 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,7 @@ venv .coverage .schema .vscode +.hypothesis +Pipfile +Pipfile.lock +pyproject.toml From 0b7b80bd40fe86e4d66a04c9f607d94991c45c0b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 25 Mar 2022 13:07:29 -0700 Subject: [PATCH 0582/1004] Document the convert() with initialization pattern, closes #420 --- docs/cli.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 8bd3650..5a03402 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1002,6 +1002,8 @@ The ``--convert`` option also works with the ``--csv``, ``--tsv`` and ``--nl`` i As with ``sqlite-utils convert`` you can use ``--import`` to import additional Python modules, see :ref:`cli_convert_import` for details. +You can also pass code that runs some initialization steps and defines a ``convert(value)`` function, see :ref:`cli_convert_complex`. + .. _cli_insert_convert_lines: \-\-convert with \-\-lines @@ -1285,6 +1287,27 @@ This supports nested imports as well, for example to use `ElementTree Date: Fri, 25 Mar 2022 14:17:10 -0700 Subject: [PATCH 0583/1004] Clarified support for newline-delimited JSON, closes #417 --- docs/cli.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 5a03402..190d285 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -805,12 +805,21 @@ You can insert binary data into a BLOB column by first encoding it using base64 Inserting newline-delimited JSON -------------------------------- -You can also import newline-delimited JSON using the ``--nl`` option. Since `Datasette `__ can export newline-delimited JSON, you can combine the two tools like so:: +You can also import `newline-delimited JSON `__ using the ``--nl`` option:: + + $ echo '{"id": 1, "name": "Cleo"} + {"id": 2, "name": "Suna"}' | sqlite-utils insert creatures.db creatures - --nl + +Newline-delimited JSON consists of full JSON objects separated by newlines. + +If you are processing data using ``jq`` you can use the ``jq -c`` option to output valid newline-delimited JSON. + +Since `Datasette `__ can export newline-delimited JSON, you can combine the Datasette and ``sqlite-utils`` like so:: $ curl -L "https://latest.datasette.io/fixtures/facetable.json?_shape=array&_nl=on" \ | sqlite-utils insert nl-demo.db facetable - --pk=id --nl -This also means you pipe ``sqlite-utils`` together to easily create a new SQLite database file containing the results of a SQL query against another database:: +You can also pipe ``sqlite-utils`` together to create a new SQLite database file containing the results of a SQL query against another database:: $ sqlite-utils sf-trees.db \ "select TreeID, qAddress, Latitude, Longitude from Street_Tree_List" --nl \ From 6f3ae864f1a521caa1b2a48d714d627ab8e9e188 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Apr 2022 15:31:37 -0700 Subject: [PATCH 0584/1004] Better support check for deterministic=True, closes #425 Bug first discovered in #421 --- sqlite_utils/db.py | 19 ++++++++----- tests/test_register_function.py | 48 ++++++++++++++++++--------------- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3ffed9f..ac1022b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -392,13 +392,18 @@ class Database: if not replace and (name, arity) in self._registered_functions: return fn kwargs = {} - if ( - deterministic - and sys.version_info >= (3, 8) - and self.sqlite_version >= (3, 8, 3) - ): - kwargs["deterministic"] = True - self.conn.create_function(name, arity, fn, **kwargs) + registered = False + if deterministic: + # Try this, but fall back if sqlite3.NotSupportedError + try: + self.conn.create_function( + name, arity, fn, **dict(kwargs, deterministic=True) + ) + registered = True + except sqlite3.NotSupportedError: + pass + if not registered: + self.conn.create_function(name, arity, fn, **kwargs) self._registered_functions.add((name, arity)) return fn diff --git a/tests/test_register_function.py b/tests/test_register_function.py index d86c7b6..615f38f 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -1,7 +1,8 @@ # flake8: noqa import pytest import sys -from unittest.mock import MagicMock +from unittest.mock import MagicMock, call +from sqlite_utils.utils import sqlite3 def test_register_function(fresh_db): @@ -31,36 +32,41 @@ def test_register_function_deterministic(fresh_db): assert result == "bob" -@pytest.mark.skipif( - sys.version_info < (3, 8), reason="deterministic=True was added in Python 3.8" -) -@pytest.mark.parametrize( - "fake_sqlite_version,should_use_deterministic", - ( - ("3.36.0", True), - ("3.8.3", True), - ("3.8.2", False), - ), -) -def test_register_function_deterministic_registered( - fresh_db, fake_sqlite_version, should_use_deterministic -): +def test_register_function_deterministic_tries_again_if_exception_raised(fresh_db): fresh_db.conn = MagicMock() fresh_db.conn.create_function = MagicMock() - fresh_db.conn.execute().fetchall.return_value = [(fake_sqlite_version,)] @fresh_db.register_function(deterministic=True) def to_lower_2(s): return s.lower() - expected_kwargs = {} - if should_use_deterministic: - expected_kwargs = dict(deterministic=True) - fresh_db.conn.create_function.assert_called_with( - "to_lower_2", 1, to_lower_2, **expected_kwargs + "to_lower_2", 1, to_lower_2, deterministic=True ) + first = True + + def side_effect(*args, **kwargs): + # Raise exception only first time this is called + nonlocal first + if first: + first = False + raise sqlite3.NotSupportedError() + + # But if sqlite3.NotSupportedError is raised, it tries again + fresh_db.conn.create_function.reset_mock() + fresh_db.conn.create_function.side_effect = side_effect + + @fresh_db.register_function(deterministic=True) + def to_lower_3(s): + return s.lower() + + # Should have been called once with deterministic=True and once without + assert fresh_db.conn.create_function.call_args_list == [ + call("to_lower_3", 1, to_lower_3, deterministic=True), + call("to_lower_3", 1, to_lower_3), + ] + def test_register_function_replace(fresh_db): @fresh_db.register_function() From 4433eafff7a09bcf6e9752e86bb5ffec23d6db25 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Apr 2022 15:35:57 -0700 Subject: [PATCH 0585/1004] Fix for register() on Python 3.7, refs #425 --- sqlite_utils/db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ac1022b..dcbc2b6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -400,7 +400,8 @@ class Database: name, arity, fn, **dict(kwargs, deterministic=True) ) registered = True - except sqlite3.NotSupportedError: + except (sqlite3.NotSupportedError, TypeError): + # TypeError is Python 3.7 "function takes at most 3 arguments" pass if not registered: self.conn.create_function(name, arity, fn, **kwargs) From 0e60f3c80cd6df5c177d8405afc54d014addebd0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Apr 2022 15:39:48 -0700 Subject: [PATCH 0586/1004] Better error message if table has no columns, closes #424 --- sqlite_utils/db.py | 1 + tests/test_create.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index dcbc2b6..82156b9 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -768,6 +768,7 @@ class Database: # Soundness check not_null, and defaults if provided not_null = not_null or set() defaults = defaults or {} + assert columns, "Tables must have at least one column" assert all( n in columns for n in not_null ), "not_null set {} includes items not in columns {}".format( diff --git a/tests/test_create.py b/tests/test_create.py index c3871e1..60181de 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1149,3 +1149,9 @@ def test_create_if_not_exists(fresh_db): fresh_db["t"].create({"id": int}) # This should not fresh_db["t"].create({"id": int}, if_not_exists=True) + + +def test_create_if_no_columns(fresh_db): + with pytest.raises(AssertionError) as error: + fresh_db["t"].create({}) + assert error.value.args[0] == "Tables must have at least one column" From cbaad1f153b7a2be50223576afd61fb4e68de2f7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Apr 2022 15:40:57 -0700 Subject: [PATCH 0587/1004] Removed unused sys import, refs #425 --- sqlite_utils/db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 82156b9..cbf59d6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -22,7 +22,6 @@ import pathlib import re import secrets from sqlite_fts4 import rank_bm25 # type: ignore -import sys import textwrap from typing import ( cast, From 6915fbcce24d03d7e7fbcb901c18be84b5568a9d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Apr 2022 15:51:48 -0700 Subject: [PATCH 0588/1004] Release 3.26 Refs #415, #416, #420, #421, #425 --- docs/changelog.rst | 10 ++++++++++ setup.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 916b9df..ecccfb4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,16 @@ Changelog =========== +.. _v3_26: + +3.26 (2022-04-13) +----------------- + +- New ``errors=r.IGNORE/r.SET_NULL`` parameter for the ``r.parsedatetime()`` and ``r.parsedate()`` :ref:`convert recipes `. (:issue:`416`) +- Fixed a bug where ``--multi`` could not be used in combination with ``--dry-run`` for the :ref:`convert ` command. (:issue:`415`) +- New documentation: :ref:`cli_convert_complex`. (:issue:`420`) +- More robust detection for whether or not ``deterministic=True`` is supported. (:issue:`425`) + .. _v3_25_1: 3.25.1 (2022-03-11) diff --git a/setup.py b/setup.py index d562d40..5152d54 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.25.1" +VERSION = "3.26" def get_long_description(): From e3a14c33a033b0c2fc00f2470666caaf9027e446 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Apr 2022 17:21:04 -0700 Subject: [PATCH 0589/1004] Run tests against pull requests --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a8274e2..75ef594 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,6 @@ name: Test -on: [push] +on: [push, pull_request] env: FORCE_COLOR: 1 From ed6fd516082e8cc83b199798f62dd67728a6974f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 May 2022 11:05:00 -0700 Subject: [PATCH 0590/1004] Depend on click-default-group-wheel (#429) To get this to work with Pyodide. Refs: https://github.com/simonw/click-default-group-wheel/issues/3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5152d54..218cc49 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ setup( install_requires=[ "sqlite-fts4", "click", - "click-default-group", + "click-default-group-wheel", "tabulate", "python-dateutil", ], From 56571775a15159d66bf6e5af971a45757e841f96 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 May 2022 11:14:29 -0700 Subject: [PATCH 0591/1004] Release 3.26.1 Refs #429 --- docs/changelog.rst | 20 +++++++++++++++++++- setup.py | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ecccfb4..8733e7c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,24 @@ Changelog =========== +.. _v3_26_1: + +3.26.1 (2022-05-02) +------------------- + +- Now depends on `click-default-group-wheel `__, a pure Python wheel package. This means you can install and use this package with `Pyodide `__, which can run Python entirely in your broswer using WebAssembly. (`#429 `__) + + Try that out using the `Pyodide REPL `__: + + .. code-block:: python + + >>> import micropip + >>> await micropip.install("sqlite-utils") + >>> import sqlite_utils + >>> db = sqlite_utils.Database(memory=True) + >>> list(db.query("select 3 * 5")) + [{'3 * 5': 15}] + .. _v3_26: 3.26 (2022-04-13) @@ -49,7 +67,7 @@ 3.23 (2022-02-03) ----------------- -This release introduces four new utility methods for working with `SpatiaLite `__. Thanks, Chris Amico. (`#330 `__) +This release introduces four new utility methods for working with `SpatiaLite `__. Thanks, Chris Amico. (`#385 `__) - ``sqlite_utils.utils.find_spatialite()`` :ref:`finds the location of the SpatiaLite module ` on disk. - ``db.init_spatialite()`` :ref:`initializes SpatiaLite ` for the given database. diff --git a/setup.py b/setup.py index 218cc49..8c5ed1b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.26" +VERSION = "3.26.1" def get_long_description(): From 841ad44bacaff05ec79ef78166d12e80c82ba6d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 May 2022 11:17:19 -0700 Subject: [PATCH 0592/1004] Fixed typo --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8733e7c..f2f97a7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,7 @@ 3.26.1 (2022-05-02) ------------------- -- Now depends on `click-default-group-wheel `__, a pure Python wheel package. This means you can install and use this package with `Pyodide `__, which can run Python entirely in your broswer using WebAssembly. (`#429 `__) +- Now depends on `click-default-group-wheel `__, a pure Python wheel package. This means you can install and use this package with `Pyodide `__, which can run Python entirely in your browser using WebAssembly. (`#429 `__) Try that out using the `Pyodide REPL `__: From 397183debd9329a2ddbcabe7a181278f042952ad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 20 May 2022 14:51:30 -0700 Subject: [PATCH 0593/1004] Switch docs theme to Furo, refs #435 --- .../layout.html => _static/js/custom.js} | 18 +----------------- docs/_templates/base.html | 6 ++++++ docs/cli-reference.rst | 1 + docs/cli.rst | 1 + docs/conf.py | 16 +++------------- docs/python-api.rst | 1 + docs/reference.rst | 1 + setup.py | 2 +- 8 files changed, 15 insertions(+), 31 deletions(-) rename docs/{_templates/layout.html => _static/js/custom.js} (67%) create mode 100644 docs/_templates/base.html diff --git a/docs/_templates/layout.html b/docs/_static/js/custom.js similarity index 67% rename from docs/_templates/layout.html rename to docs/_static/js/custom.js index cecf66e..8786865 100644 --- a/docs/_templates/layout.html +++ b/docs/_static/js/custom.js @@ -1,13 +1,3 @@ -{%- extends "!layout.html" %} - -{% block htmltitle %} -{{ super() }} - -{% endblock %} - -{% block footer %} -{{ super() }} - -{% endblock %} diff --git a/docs/_templates/base.html b/docs/_templates/base.html new file mode 100644 index 0000000..a9e670e --- /dev/null +++ b/docs/_templates/base.html @@ -0,0 +1,6 @@ +{%- extends "!base.html" %} + +{% block site_meta %} +{{ super() }} + +{% endblock %} \ No newline at end of file diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c3f7a24..7d73e12 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -7,6 +7,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. .. contents:: :local: + :class: this-will-duplicate-information-and-it-is-still-useful-here .. [[[cog from sqlite_utils import cli diff --git a/docs/cli.rst b/docs/cli.rst index 190d285..2d2ea76 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -9,6 +9,7 @@ The ``sqlite-utils`` command-line tool can be used to manipulate SQLite database Once installed, the tool should be available as ``sqlite-utils``. It can also be run using ``python -m sqlite_utils``. .. contents:: :local: + :class: this-will-duplicate-information-and-it-is-still-useful-here .. _cli_query: diff --git a/docs/conf.py b/docs/conf.py index 03c13d6..7244d2d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -94,7 +94,8 @@ todo_include_todos = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_rtd_theme" +html_theme = "furo" +html_title = "sqlite-utils" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -107,18 +108,7 @@ html_theme = "sphinx_rtd_theme" # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - "**": [ - "relations.html", # needs 'show_related': True theme option to display - "searchbox.html", - ] -} - +html_js_files = ["js/custom.js"] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/docs/python-api.rst b/docs/python-api.rst index bd0265b..82ec11c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -5,6 +5,7 @@ ============================= .. contents:: :local: + :class: this-will-duplicate-information-and-it-is-still-useful-here .. _python_api_getting_started: diff --git a/docs/reference.rst b/docs/reference.rst index 5bd609f..2896bbf 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -5,6 +5,7 @@ =============== .. contents:: :local: + :class: this-will-duplicate-information-and-it-is-still-useful-here .. _reference_db_database: diff --git a/setup.py b/setup.py index 8c5ed1b..51830e3 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( ], extras_require={ "test": ["pytest", "black", "hypothesis", "cogapp"], - "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], + "docs": ["furo", "sphinx-autobuild", "codespell"], "mypy": [ "mypy", "types-click", From 2238e9baf94d1d2af794d6cb064dbac098abd3f3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 20 May 2022 14:57:26 -0700 Subject: [PATCH 0594/1004] sphinx-copybutton extension, closes #436 --- docs/conf.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 7244d2d..996a499 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,7 @@ from subprocess import Popen, PIPE # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"] +extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc", "sphinx_copybutton"] autodoc_member_order = "bysource" autodoc_typehints = "description" diff --git a/setup.py b/setup.py index 51830e3..eafb1ce 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( ], extras_require={ "test": ["pytest", "black", "hypothesis", "cogapp"], - "docs": ["furo", "sphinx-autobuild", "codespell"], + "docs": ["furo", "sphinx-autobuild", "codespell", "sphinx-copybutton"], "mypy": [ "mypy", "types-click", From 59be60c471fd7a2c4be7f75e8911163e618ff5ca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 20 May 2022 14:57:49 -0700 Subject: [PATCH 0595/1004] Update copyright dates on docs --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 996a499..1b1c4f1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,7 +52,7 @@ master_doc = "index" # General information about the project. project = "sqlite-utils" -copyright = "2018-2021, Simon Willison" +copyright = "2018-2022, Simon Willison" author = "Simon Willison" # The version info for the project you're documenting, acts as replacement for From 9fedfc69d7239ac49900051e1c48ee9cdd470d9e Mon Sep 17 00:00:00 2001 From: Yuri Date: Mon, 30 May 2022 17:32:41 -0400 Subject: [PATCH 0596/1004] docs to dogs (#437) --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 2d2ea76..1fc5a00 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -884,7 +884,7 @@ Inserting CSV or TSV data If your data is in CSV format, you can insert it using the ``--csv`` option:: - $ sqlite-utils insert dogs.db dogs docs.csv --csv + $ sqlite-utils insert dogs.db dogs dogs.csv --csv For tab-delimited data, use ``--tsv``:: From 7ddf5300886a32d6daf60cf1d71efe492b65c87e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 13 Jun 2022 08:22:59 -0700 Subject: [PATCH 0597/1004] A less potentially confusing parameter name --- docs/cli.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 1fc5a00..9169fba 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1275,8 +1275,8 @@ The conversion will be applied to every row in the specified table. You can limi You can include named parameters in your where clause and populate them using one or more ``--param`` options:: $ sqlite-utils convert content.db articles headline 'value.upper()' \ - --where "headline like :like" \ - --param like '%cat%' + --where "headline like :query" \ + --param query '%cat%' The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. From d379f430f8bc00e5177d38097c9ab2919152ee76 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 08:14:02 -0700 Subject: [PATCH 0598/1004] rows_from_file(... ignore_extras: bool, restkey: str), refs #440 --- sqlite_utils/utils.py | 45 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index d35cc05..611ff30 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -171,12 +171,43 @@ class RowsFromFileBadJSON(RowsFromFileError): pass +class RowError(Exception): + pass + + +def _extra_key_strategy( + reader: Iterable[dict], + ignore_extras: Optional[bool] = False, + restkey: Optional[str] = None, +): + # Logic for handling CSV rows with more values than there are headings + for row in reader: + # DictReader adds a 'None' key with extra row values + if None not in row: + yield row + elif ignore_extras: + row.pop(None) + yield row + elif not restkey: + extras = row.pop(None) + raise RowError( + "Row {} contained these extra values: {}".format(row, extras) + ) + else: + row[restkey] = row.pop(None) + yield row + + def rows_from_file( fp: BinaryIO, format: Optional[Format] = None, dialect: Optional[Type[csv.Dialect]] = None, encoding: Optional[str] = None, + ignore_extras: Optional[bool] = False, + restkey: Optional[str] = None, ) -> Tuple[Iterable[dict], Format]: + if ignore_extras and restkey: + raise ValueError("Cannot use ignore_extras= and restkey= together") if format == Format.JSON: decoded = json.load(fp) if isinstance(decoded, dict): @@ -193,12 +224,13 @@ def rows_from_file( reader = csv.DictReader(decoded_fp, dialect=dialect) else: reader = csv.DictReader(decoded_fp) - return reader, Format.CSV + return _extra_key_strategy(reader, ignore_extras, restkey), Format.CSV elif format == Format.TSV: + reader = rows_from_file( + fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding + )[0] return ( - rows_from_file( - fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding - )[0], + _extra_key_strategy(reader, ignore_extras, restkey), Format.TSV, ) elif format is None: @@ -212,9 +244,12 @@ def rows_from_file( dialect = csv.Sniffer().sniff( first_bytes.decode(encoding or "utf-8-sig", "ignore") ) - return rows_from_file( + rows, _ = rows_from_file( buffered, format=Format.CSV, dialect=dialect, encoding=encoding ) + # Make sure we return the format we detected + format = Format.TSV if dialect.delimiter == "\t" else Format.CSV + return _extra_key_strategy(rows, ignore_extras, restkey), format else: raise RowsFromFileError("Bad format") From ad96bd18c37a789b0505dbef0057557c7415b133 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 08:14:38 -0700 Subject: [PATCH 0599/1004] Tests for rows_from_file, refs #440 --- tests/test_rows_from_file.py | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/test_rows_from_file.py diff --git a/tests/test_rows_from_file.py b/tests/test_rows_from_file.py new file mode 100644 index 0000000..5575b34 --- /dev/null +++ b/tests/test_rows_from_file.py @@ -0,0 +1,46 @@ +from sqlite_utils.utils import rows_from_file, Format, RowError +from io import BytesIO +import pytest + + +@pytest.mark.parametrize( + "input,expected_format", + ( + (b"id,name\n1,Cleo", Format.CSV), + (b"id\tname\n1\tCleo", Format.TSV), + (b'[{"id": "1", "name": "Cleo"}]', Format.JSON), + ), +) +def test_rows_from_file_detect_format(input, expected_format): + rows, format = rows_from_file(BytesIO(input)) + assert format == expected_format + rows_list = list(rows) + assert rows_list == [{"id": "1", "name": "Cleo"}] + + +@pytest.mark.parametrize( + "ignore_extras,restkey,expected", + ( + (True, None, [{"id": "1", "name": "Cleo"}]), + (False, "_rest", [{"id": "1", "name": "Cleo", "_rest": ["oops"]}]), + # expected of None means expect an error: + (False, False, None), + ), +) +def test_rows_from_file_extra_fields_strategies(ignore_extras, restkey, expected): + try: + rows, format = rows_from_file( + BytesIO(b"id,name\r\n1,Cleo,oops"), + format=Format.CSV, + ignore_extras=ignore_extras, + restkey=restkey, + ) + list_rows = list(rows) + except RowError: + if expected is None: + # This is fine, + return + else: + # We did not expect an error + raise + assert list_rows == expected From 19efee2746d4afdb67a7225b6972aa5aa7bbb1b7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 08:39:08 -0700 Subject: [PATCH 0600/1004] mypy fixes, refs #440 --- sqlite_utils/utils.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 611ff30..6eb6712 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -8,7 +8,7 @@ import itertools import json import os from . import recipes -from typing import Dict, cast, BinaryIO, Iterable, Optional, Tuple, Type +from typing import Dict, cast, BinaryIO, Iterable, Optional, Tuple, Type, Union import click @@ -176,25 +176,27 @@ class RowError(Exception): def _extra_key_strategy( - reader: Iterable[dict], + reader: Union[Iterable[dict], csv.DictReader[str]], ignore_extras: Optional[bool] = False, restkey: Optional[str] = None, -): +) -> Iterable[dict]: # Logic for handling CSV rows with more values than there are headings for row in reader: # DictReader adds a 'None' key with extra row values if None not in row: yield row elif ignore_extras: - row.pop(None) + # ignoring row.pop(none) because of this issue: + # https://github.com/simonw/sqlite-utils/issues/440#issuecomment-1155358637 + row.pop(None) # type: ignore yield row elif not restkey: - extras = row.pop(None) + extras = row.pop(None) # type: ignore raise RowError( "Row {} contained these extra values: {}".format(row, extras) ) else: - row[restkey] = row.pop(None) + row[restkey] = row.pop(None) # type: ignore yield row @@ -226,11 +228,11 @@ def rows_from_file( reader = csv.DictReader(decoded_fp) return _extra_key_strategy(reader, ignore_extras, restkey), Format.CSV elif format == Format.TSV: - reader = rows_from_file( + rows = rows_from_file( fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding )[0] return ( - _extra_key_strategy(reader, ignore_extras, restkey), + _extra_key_strategy(rows, ignore_extras, restkey), Format.TSV, ) elif format is None: From d9c715a2fc0e4ccc0dd8e50ae68686a09f92eda8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 09:07:57 -0700 Subject: [PATCH 0601/1004] One more typing fix, refs #440 --- sqlite_utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 6eb6712..eb80293 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -176,7 +176,7 @@ class RowError(Exception): def _extra_key_strategy( - reader: Union[Iterable[dict], csv.DictReader[str]], + reader: Iterable[dict], ignore_extras: Optional[bool] = False, restkey: Optional[str] = None, ) -> Iterable[dict]: From f142bb1212f98c1cb9ff72a3161351c5c8d1d281 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 09:14:57 -0700 Subject: [PATCH 0602/1004] flake8 fix, refs #440 --- sqlite_utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index eb80293..8bbb1d2 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -8,7 +8,7 @@ import itertools import json import os from . import recipes -from typing import Dict, cast, BinaryIO, Iterable, Optional, Tuple, Type, Union +from typing import Dict, cast, BinaryIO, Iterable, Optional, Tuple, Type import click From ce670e2d44327f9e73452aba30da632902f6a937 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 13:12:32 -0700 Subject: [PATCH 0603/1004] Docs for rows_from_file Closes #440, closes #443 --- docs/python-api.rst | 10 ++++++++ sqlite_utils/utils.py | 58 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 82ec11c..9fe6d67 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2539,6 +2539,16 @@ If that option isn't relevant to your use-case you can to quote a string for use >>> db.quote("hello'this'has'quotes") "'hello''this''has''quotes'" +.. _python_api_rows_from_file: + +Reading rows from a file +======================== + +The ``sqlite_utils.utils.rows_from_file()`` helper function can read rows (a sequence of dictionaries) from CSV, TSV, JSON or newline-delimited JSON files. + +.. autofunction:: sqlite_utils.utils.rows_from_file + :noindex: + .. _python_api_gis: SpatiaLite helpers diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 8bbb1d2..141356a 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -178,7 +178,7 @@ class RowError(Exception): def _extra_key_strategy( reader: Iterable[dict], ignore_extras: Optional[bool] = False, - restkey: Optional[str] = None, + extras_key: Optional[str] = None, ) -> Iterable[dict]: # Logic for handling CSV rows with more values than there are headings for row in reader: @@ -190,13 +190,13 @@ def _extra_key_strategy( # https://github.com/simonw/sqlite-utils/issues/440#issuecomment-1155358637 row.pop(None) # type: ignore yield row - elif not restkey: + elif not extras_key: extras = row.pop(None) # type: ignore raise RowError( "Row {} contained these extra values: {}".format(row, extras) ) else: - row[restkey] = row.pop(None) # type: ignore + row[extras_key] = row.pop(None) # type: ignore yield row @@ -206,10 +206,50 @@ def rows_from_file( dialect: Optional[Type[csv.Dialect]] = None, encoding: Optional[str] = None, ignore_extras: Optional[bool] = False, - restkey: Optional[str] = None, + extras_key: Optional[str] = None, ) -> Tuple[Iterable[dict], Format]: - if ignore_extras and restkey: - raise ValueError("Cannot use ignore_extras= and restkey= together") + """ + Load a sequence of dictionaries from a file-like object containing one of four different formats. + + .. code-block:: python + + from sqlite_utils.utils import rows_from_file + import io + + rows, format = rows_from_file(io.StringIO("id,name\\n1,Cleo"))) + print(list(rows), format) + # Outputs [{'id': '1', 'name': 'Cleo'}] Format.CSV + + This defaults to attempting to automatically detect the format of the data, or you can pass in an + explicit format using the format= option. + + Returns a tuple of ``(rows_generator, format_used)`` where ``rows_generator`` can be iterated over + to return dictionaries, while ``format_used`` is a value from the ``sqlite_utils.utils.Format`` enum: + + .. code-block:: python + + class Format(enum.Enum): + CSV = 1 + TSV = 2 + JSON = 3 + NL = 4 + + If a CSV or TSV file includes rows with more fields than are declared in the header a + ``sqlite_utils.utils.RowError`` exception will be raised when you loop over the generator. + + You can instead ignore the extra data by passing ``ignore_extras=True``. + + Or pass ``extras_key="rest"`` to put those additional values in a list in a key called ``rest``. + + :param fp: a file-like object containing binary data + :param format: the format to use - omit this to detect the format + :param dialect: the CSV dialect to use - omit this to detect the dialect + :param encoding: the character encoding to use when reading CSV/TSV data + :param ignore_extras: ignore any extra fields on rows + :param extras_key: put any extra fields in a list with this key + """ + if ignore_extras and extras_key: + raise ValueError("Cannot use ignore_extras= and extras_key= together") if format == Format.JSON: decoded = json.load(fp) if isinstance(decoded, dict): @@ -226,13 +266,13 @@ def rows_from_file( reader = csv.DictReader(decoded_fp, dialect=dialect) else: reader = csv.DictReader(decoded_fp) - return _extra_key_strategy(reader, ignore_extras, restkey), Format.CSV + return _extra_key_strategy(reader, ignore_extras, extras_key), Format.CSV elif format == Format.TSV: rows = rows_from_file( fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding )[0] return ( - _extra_key_strategy(rows, ignore_extras, restkey), + _extra_key_strategy(rows, ignore_extras, extras_key), Format.TSV, ) elif format is None: @@ -251,7 +291,7 @@ def rows_from_file( ) # Make sure we return the format we detected format = Format.TSV if dialect.delimiter == "\t" else Format.CSV - return _extra_key_strategy(rows, ignore_extras, restkey), format + return _extra_key_strategy(rows, ignore_extras, extras_key), format else: raise RowsFromFileError("Bad format") From 0cee77b1764f7dff029eb0ea1e857e5b69c591ee Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 14:14:10 -0700 Subject: [PATCH 0604/1004] Update test for renamed restkey, refs #440, #443 --- sqlite_utils/cli.py | 13 ++----------- sqlite_utils/utils.py | 19 +++++++++++++++++++ tests/test_rows_from_file.py | 6 +++--- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 05ff9d3..86eddfb 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -6,6 +6,7 @@ import hashlib import pathlib import sqlite_utils from sqlite_utils.db import AlterError, BadMultiValues, DescIndex +from sqlite_utils.utils import maximize_csv_field_size_limit from sqlite_utils import recipes import textwrap import inspect @@ -46,17 +47,7 @@ If you do not know the encoding, running 'file filename.csv' may tell you. It's often worth trying: --encoding=latin-1 """.strip() - -# Increase CSV field size limit to maximum possible -# https://stackoverflow.com/a/15063941 -field_size_limit = sys.maxsize - -while True: - try: - csv_std.field_size_limit(field_size_limit) - break - except OverflowError: - field_size_limit = int(field_size_limit / 10) +maximize_csv_field_size_limit() def output_options(fn): diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 141356a..d2ccc5f 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -7,6 +7,7 @@ import io import itertools import json import os +import sys from . import recipes from typing import Dict, cast, BinaryIO, Iterable, Optional, Tuple, Type @@ -29,6 +30,24 @@ SPATIALITE_PATHS = ( "/usr/local/lib/mod_spatialite.dylib", ) +# Mainly so we can restore it if needed in the tests: +ORIGINAL_CSV_FIELD_SIZE_LIMIT = csv.field_size_limit() + + +def maximize_csv_field_size_limit(): + """ + Increase the CSV field size limit to the maximum possible. + """ + # https://stackoverflow.com/a/15063941 + field_size_limit = sys.maxsize + + while True: + try: + csv.field_size_limit(field_size_limit) + break + except OverflowError: + field_size_limit = int(field_size_limit / 10) + def find_spatialite() -> Optional[str]: """ diff --git a/tests/test_rows_from_file.py b/tests/test_rows_from_file.py index 5575b34..fdc7a03 100644 --- a/tests/test_rows_from_file.py +++ b/tests/test_rows_from_file.py @@ -19,7 +19,7 @@ def test_rows_from_file_detect_format(input, expected_format): @pytest.mark.parametrize( - "ignore_extras,restkey,expected", + "ignore_extras,extras_key,expected", ( (True, None, [{"id": "1", "name": "Cleo"}]), (False, "_rest", [{"id": "1", "name": "Cleo", "_rest": ["oops"]}]), @@ -27,13 +27,13 @@ def test_rows_from_file_detect_format(input, expected_format): (False, False, None), ), ) -def test_rows_from_file_extra_fields_strategies(ignore_extras, restkey, expected): +def test_rows_from_file_extra_fields_strategies(ignore_extras, extras_key, expected): try: rows, format = rows_from_file( BytesIO(b"id,name\r\n1,Cleo,oops"), format=Format.CSV, ignore_extras=ignore_extras, - restkey=restkey, + extras_key=extras_key, ) list_rows = list(rows) except RowError: From 9d1bac4a99fd7f9ef2a6cb44242a00e1faa408e4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 14:23:54 -0700 Subject: [PATCH 0605/1004] Test for maximize_csv_field_size_limit, refs #442 --- tests/test_utils.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index e76e97a..d397176 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,6 @@ from sqlite_utils import utils +import csv +import io import pytest @@ -49,3 +51,23 @@ def test_hash_record(): assert ( utils.hash_record({"name": "Cleo", "twitter": "CleoPaws", "age": 7}) != expected ) + + +def test_maximize_csv_field_size_limit(): + # Reset to default in case other tests have changed it + csv.field_size_limit(utils.ORIGINAL_CSV_FIELD_SIZE_LIMIT) + long_value = "a" * 131073 + long_csv = "id,text\n1,{}".format(long_value) + fp = io.BytesIO(long_csv.encode("utf-8")) + # Using rows_from_file should error + with pytest.raises(csv.Error): + rows, _ = utils.rows_from_file(fp, utils.Format.CSV) + list(rows) + # But if we call maximize_csv_field_size_limit() first it should be OK: + utils.maximize_csv_field_size_limit() + fp2 = io.BytesIO(long_csv.encode("utf-8")) + rows2, _ = utils.rows_from_file(fp2, utils.Format.CSV) + rows_list2 = list(rows2) + assert len(rows_list2) == 1 + assert rows_list2[0]["id"] == "1" + assert rows_list2[0]["text"] == long_value From 0b6aba696dd07814d97f563de6ad1d5daab01fd9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 14:31:45 -0700 Subject: [PATCH 0606/1004] Documentation for maximize_csv_field_size_limit, closes #442 --- docs/python-api.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 9fe6d67..ca98317 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2549,6 +2549,37 @@ The ``sqlite_utils.utils.rows_from_file()`` helper function can read rows (a seq .. autofunction:: sqlite_utils.utils.rows_from_file :noindex: +.. _python_api_maximize_csv_field_size_limit: + +Setting the maximum CSV field size limit +======================================== + +Sometimes when working with CSV files that include extremely long fields you may see an error that looks like this:: + + _csv.Error: field larger than field limit (131072) + +The Python standard library ``csv`` module enforces a field size limit. You can increase that limit using the ``csv.field_size_limit(new_limit)`` method (`documented here `__) but if you don't want to pick a new level you may instead want to increase it to the maximum possible. + +The maximum possible value for this is not documented, and varies between systems. + +Calling ``sqlite_utils.utils.maximize_csv_field_size_limit()`` will set the value to the highest possible for the current system: + +.. code-block:: python + + from sqlite_utils.utils import maximize_csv_field_size_limit + + maximize_csv_field_size_limit() + + +If you need to reset to the original value after calling this function you can do so like this: + +.. code-block:: python + + from sqlite_utils.utils import ORIGINAL_CSV_FIELD_SIZE_LIMIT + import csv + + csv.field_size_limit(ORIGINAL_CSV_FIELD_SIZE_LIMIT) + .. _python_api_gis: SpatiaLite helpers From 1b09538bc6c1fda773590f3e600993ef06591041 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 14:54:35 -0700 Subject: [PATCH 0607/1004] where= and where_args= parameters to search() and search_sql() Closes #441 --- docs/python-api.rst | 12 +++++-- sqlite_utils/db.py | 20 ++++++++++-- tests/test_fts.py | 76 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index ca98317..1a8c989 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2108,10 +2108,16 @@ The ``.search()`` method also accepts the following optional parameters: ``offset`` integer Offset to use along side the limit parameter. +``where`` string + Extra SQL fragment for the WHERE clause + +``where_args`` dictionary + Arguments to use for ``:param`` placeholders in the extra WHERE clause + ``quote`` bool Apply :ref:`FTS quoting rules ` to the search query, disabling advanced query syntax in a way that avoids surprising errors. -To return just the title and published columns for three matches for ``"dog"`` ordered by ``published`` with the most recent first, use the following: +To return just the title and published columns for three matches for ``"dog"`` where the ``id`` is greater than 10 ordered by ``published`` with the most recent first, use the following: .. code-block:: python @@ -2119,6 +2125,8 @@ To return just the title and published columns for three matches for ``"dog"`` o "dog", order_by="published desc", limit=3, + where="id > :min_id", + where_args={"min_id": 10}, columns=["title", "published"] ): print(article) @@ -2128,7 +2136,7 @@ To return just the title and published columns for three matches for ``"dog"`` o Building SQL queries with table.search_sql() -------------------------------------------- -You can generate the SQL query that would be used for a search using the ``table.search_sql()`` method. It takes the same arguments as ``table.search()`` with the exception of the search query itself, since the returned SQL includes a parameter that can be used for the search. +You can generate the SQL query that would be used for a search using the ``table.search_sql()`` method. It takes the same arguments as ``table.search()``, with the exception of the search query and the ``where_args`` parameter, since those should be provided when the returned SQL is executed. .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index cbf59d6..7a06304 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2254,6 +2254,7 @@ class Table(Queryable): order_by: Optional[str] = None, limit: Optional[int] = None, offset: Optional[int] = None, + where: Optional[str] = None, ) -> str: """ " Return SQL string that can be used to execute searches against this table. @@ -2262,6 +2263,7 @@ class Table(Queryable): :param order_by: Column or SQL expression to sort by :param limit: SQL limit :param offset: SQL offset + :param where: Extra SQL fragment for the WHERE clause """ # Pick names for table and rank column that don't clash original = "original_" if self.name == "original" else "original" @@ -2283,7 +2285,7 @@ class Table(Queryable): select rowid, {columns} - from [{dbtable}] + from [{dbtable}]{where_clause} ) select {columns_with_prefix} @@ -2311,6 +2313,7 @@ class Table(Queryable): limit_offset += " offset {}".format(offset) return sql.format( dbtable=self.name, + where_clause="\n where {}".format(where) if where else "", original=original, columns=columns_sql, columns_with_prefix=columns_with_prefix_sql, @@ -2326,6 +2329,8 @@ class Table(Queryable): columns: Optional[Iterable[str]] = None, limit: Optional[int] = None, offset: Optional[int] = None, + where: str = None, + where_args: Optional[Union[Iterable, dict]] = None, quote: bool = False, ) -> Generator[dict, None, None]: """ @@ -2337,18 +2342,29 @@ class Table(Queryable): :param columns: List of columns to return, defaults to all columns. :param limit: Optional integer limit for returned rows. :param offset: Optional integer SQL offset. + :param where: Extra SQL fragment for the WHERE clause + :param where_args: Arguments to use for :param placeholders in the extra WHERE clause :param quote: Apply quoting to disable any special characters in the search query See :ref:`python_api_fts_search`. """ + args = {"query": self.db.quote_fts(q) if quote else q} + if where_args and "query" in where_args: + raise ValueError( + "'query' is a reserved key and cannot be passed to where_args for .search()" + ) + if where_args: + args.update(where_args) + cursor = self.db.execute( self.search_sql( order_by=order_by, columns=columns, limit=limit, offset=offset, + where=where, ), - {"query": self.db.quote_fts(q) if quote else q}, + args, ) columns = [c[0] for c in cursor.description] for row in cursor: diff --git a/tests/test_fts.py b/tests/test_fts.py index 24980f7..53a7bc2 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -94,6 +94,38 @@ def test_search_limit_offset(fresh_db): ) +@pytest.mark.parametrize("fts_version", ("FTS4", "FTS5")) +def test_search_where(fresh_db, fts_version): + table = fresh_db["t"] + table.insert_all(search_records) + table.enable_fts(["text", "country"], fts_version=fts_version) + results = list( + table.search("are", where="country = :country", where_args={"country": "Japan"}) + ) + assert results == [ + { + "rowid": 1, + "text": "tanuki are running tricksters", + "country": "Japan", + "not_searchable": "foo", + } + ] + + +def test_search_where_args_disallows_query(fresh_db): + table = fresh_db["t"] + with pytest.raises(ValueError) as ex: + list( + table.search( + "x", where="author = :query", where_args={"query": "not allowed"} + ) + ) + assert ( + ex.value.args[0] + == "'query' is a reserved key and cannot be passed to where_args for .search()" + ) + + def test_enable_fts_table_names_containing_spaces(fresh_db): table = fresh_db["test"] table.insert({"column with spaces": "in its name"}) @@ -415,6 +447,28 @@ def test_enable_fts_error_message_on_views(): "limit 10" ), ), + ( + {"where": "author = :author"}, + "FTS5", + ( + "with original as (\n" + " select\n" + " rowid,\n" + " *\n" + " from [books]\n" + " where author = :author\n" + ")\n" + "select\n" + " [original].*\n" + "from\n" + " [original]\n" + " join [books_fts] on [original].rowid = [books_fts].rowid\n" + "where\n" + " [books_fts] match :query\n" + "order by\n" + " [books_fts].rank" + ), + ), ( {"columns": ["title"]}, "FTS4", @@ -480,6 +534,28 @@ def test_enable_fts_error_message_on_views(): "limit 2" ), ), + ( + {"where": "author = :author"}, + "FTS4", + ( + "with original as (\n" + " select\n" + " rowid,\n" + " *\n" + " from [books]\n" + " where author = :author\n" + ")\n" + "select\n" + " [original].*\n" + "from\n" + " [original]\n" + " join [books_fts] on [original].rowid = [books_fts].rowid\n" + "where\n" + " [books_fts] match :query\n" + "order by\n" + " rank_bm25(matchinfo([books_fts], 'pcnalx'))" + ), + ), ], ) def test_search_sql(kwargs, fts, expected): From b8af3b96f5c72317cc8783dc296a94f6719987d9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 16:24:06 -0700 Subject: [PATCH 0608/1004] Fix bug with detect_fts() and similar table names, closes #434 --- sqlite_utils/db.py | 22 ++++++++++++---------- tests/test_introspect.py | 17 +++++++++++++++++ tests/test_tracer.py | 15 ++++++++++----- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 7a06304..c835fbd 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2212,24 +2212,26 @@ class Table(Queryable): def detect_fts(self) -> Optional[str]: "Detect if table has a corresponding FTS virtual table and return it" - sql = ( - textwrap.dedent( - """ + sql = textwrap.dedent( + """ SELECT name FROM sqlite_master WHERE rootpage = 0 AND ( - sql LIKE '%VIRTUAL TABLE%USING FTS%content=%{table}%' + sql LIKE :like + OR sql LIKE :like2 OR ( - tbl_name = "{table}" + tbl_name = :table AND sql LIKE '%VIRTUAL TABLE%USING FTS%' ) ) """ - ) - .strip() - .format(table=self.name) - ) - rows = self.db.execute(sql).fetchall() + ).strip() + args = { + "like": "%VIRTUAL TABLE%USING FTS%content=[{}]%".format(self.name), + "like2": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), + "table": self.name, + } + rows = self.db.execute(sql, args).fetchall() if len(rows) == 0: return None else: diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 28a5009..fe9542b 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -36,6 +36,23 @@ def test_detect_fts(existing_db): assert existing_db["foo"].detect_fts() is None +@pytest.mark.parametrize("reverse_order", (True, False)) +def test_detect_fts_similar_tables(fresh_db, reverse_order): + # https://github.com/simonw/sqlite-utils/issues/434 + table1, table2 = ("demo", "demo2") + if reverse_order: + table1, table2 = table2, table1 + + fresh_db[table1].insert({"title": "Hello"}).enable_fts( + ["title"], fts_version="FTS4" + ) + fresh_db[table2].insert({"title": "Hello"}).enable_fts( + ["title"], fts_version="FTS4" + ) + assert fresh_db[table1].detect_fts() == "{}_fts".format(table1) + assert fresh_db[table2].detect_fts() == "{}_fts".format(table2) + + def test_tables(existing_db): assert 1 == len(existing_db.tables) assert "foo" == existing_db.tables[0].name diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 618d1e6..2d76c74 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -54,14 +54,19 @@ def test_with_tracer(): "SELECT name FROM sqlite_master\n" " WHERE rootpage = 0\n" " AND (\n" - " sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n" + " sql LIKE :like\n" + " OR sql LIKE :like2\n" " OR (\n" - ' tbl_name = "dogs"\n' + " tbl_name = :table\n" " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" " )\n" - " )" - ), - None, + " )", + { + "like": "%VIRTUAL TABLE%USING FTS%content=[dogs]%", + "like2": '%VIRTUAL TABLE%USING FTS%content="dogs"%', + "table": "dogs", + }, + ) ), ("select name from sqlite_master where type = 'view'", None), ("select sql from sqlite_master where name = ?", ("dogs_fts",)), From 679d6081198a2b658d6578f24bc3446750395ecf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 21:29:34 -0700 Subject: [PATCH 0609/1004] Release 3.27 Refs #434, #435, #436, #440, #441, #442, #443 --- docs/changelog.rst | 13 +++++++++++++ setup.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f2f97a7..5c9bc18 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,19 @@ Changelog =========== +.. _v3_27: + +3.27 (2022-06-14) +----------------- + +- Documentation now uses the `Furo `__ Sphinx theme. (:issue:`435`) +- Code examples in documentation now have a "copy to clipboard" button. (:issue:`436`) +- ``sqlite_utils.utils.utils.rows_from_file()`` is now a documented API, see :ref:`python_api_rows_from_file`. (:issue:`443`) +- ``rows_from_file()`` has two new parameters to help handle CSV files with rows that contain more values than are listed in that CSV file's headings: ``ignore_extrasTrue`` and ``extras_key="name-of-key"``. (:issue:`440`) +- ``sqlite_utils.utils.maximize_csv_field_size_limit()`` helper function for increasing the field size limit for reading CSV files to its maximum, see :ref:`python_api_maximize_csv_field_size_limit`. (:issue:`442`) +- ``table.search(where=, where_args=)`` parameters for adding additional ``WHERE`` clauses to a search query. The ``where=`` parameter is available on ``table.search_sql(...)`` as well. See :ref:`python_api_fts_search`. (:issue:`441`) +- Fixed bug where ``table.detect_fts()`` and other search-related functions could fail if two FTS-enabled tables had names that were prefixes of each other. (:issue:`434`) + .. _v3_26_1: 3.26.1 (2022-05-02) diff --git a/setup.py b/setup.py index eafb1ce..a416570 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.26.1" +VERSION = "3.27" def get_long_description(): From 2d84577202e227670ad434fcf3dd9786a2df0819 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jun 2022 21:41:16 -0700 Subject: [PATCH 0610/1004] Fix tiny typo in 3.27 release notes --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5c9bc18..d7b48f1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ - Documentation now uses the `Furo `__ Sphinx theme. (:issue:`435`) - Code examples in documentation now have a "copy to clipboard" button. (:issue:`436`) - ``sqlite_utils.utils.utils.rows_from_file()`` is now a documented API, see :ref:`python_api_rows_from_file`. (:issue:`443`) -- ``rows_from_file()`` has two new parameters to help handle CSV files with rows that contain more values than are listed in that CSV file's headings: ``ignore_extrasTrue`` and ``extras_key="name-of-key"``. (:issue:`440`) +- ``rows_from_file()`` has two new parameters to help handle CSV files with rows that contain more values than are listed in that CSV file's headings: ``ignore_extras=True`` and ``extras_key="name-of-key"``. (:issue:`440`) - ``sqlite_utils.utils.maximize_csv_field_size_limit()`` helper function for increasing the field size limit for reading CSV files to its maximum, see :ref:`python_api_maximize_csv_field_size_limit`. (:issue:`442`) - ``table.search(where=, where_args=)`` parameters for adding additional ``WHERE`` clauses to a search query. The ``where=`` parameter is available on ``table.search_sql(...)`` as well. See :ref:`python_api_fts_search`. (:issue:`441`) - Fixed bug where ``table.detect_fts()`` and other search-related functions could fail if two FTS-enabled tables had names that were prefixes of each other. (:issue:`434`) From 3fbe8a784cc2f3fa0bfa8612fec9752ff9068a2b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 18 Jun 2022 20:30:24 -0700 Subject: [PATCH 0611/1004] Link to annotated release notes for 3.27 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d7b48f1..b284eb8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,8 @@ 3.27 (2022-06-14) ----------------- +See also `the annotated release notes `__ for this release. + - Documentation now uses the `Furo `__ Sphinx theme. (:issue:`435`) - Code examples in documentation now have a "copy to clipboard" button. (:issue:`436`) - ``sqlite_utils.utils.utils.rows_from_file()`` is now a documented API, see :ref:`python_api_rows_from_file`. (:issue:`443`) From c80971d28ab03c703f2d39cfaa6123ea8a249711 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Jun 2022 12:22:27 -0700 Subject: [PATCH 0612/1004] sqlite_utils.utils.rows_from_file in reference.html, refs #443 --- docs/reference.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/reference.rst b/docs/reference.rst index 2896bbf..6603d99 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -79,3 +79,10 @@ sqlite_utils.utils.hash_record ------------------------------ .. autofunction:: sqlite_utils.utils.hash_record + +.. _reference_utils_rows_from_file: + +sqlite_utils.utils.rows_from_file +--------------------------------- + +.. autofunction:: sqlite_utils.utils.rows_from_file From 773f2b6b20622bb986984a1c3161d5b3aaa1046b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Jun 2022 12:46:49 -0700 Subject: [PATCH 0613/1004] Documented TypeTracker, closes #445 --- docs/python-api.rst | 59 +++++++++++++++++++++++++++++++++++++++++++ docs/reference.rst | 8 ++++++ sqlite_utils/utils.py | 33 ++++++++++++++++++++++-- 3 files changed, 98 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 1a8c989..2a87610 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2588,6 +2588,65 @@ If you need to reset to the original value after calling this function you can d csv.field_size_limit(ORIGINAL_CSV_FIELD_SIZE_LIMIT) +.. _python_api_typetracker: + +Detecting column types using TypeTracker +======================================== + +Sometimes you may find yourself working with data that lacks type information - data from a CSV file for example. + +The ``TypeTracker`` class can be used to try to automatically identify the most likely types for data that is initially represented as strings. + +Consider this example: + +.. code-block:: python + + import csv, io + + csv_file = io.StringIO("id,name\n1,Cleo\n2,Cardi") + rows = list(csv.DictReader(csv_file)) + + # rows is now this: + # [{'id': '1', 'name': 'Cleo'}, {'id': '2', 'name': 'Cardi'}] + +If we insert this data directly into a table we will get a schema that is entirely ``TEXT`` columns: + +.. code-block:: python + + from sqlite_utils import Database + + db = Database(memory=True) + db["creatures"].insert_all(rows) + print(db.schema) + # Outputs: + # CREATE TABLE [creatures] ( + # [id] TEXT, + # [name] TEXT + # ); + +We can detect the best column types using a ``TypeTracker`` instance: + +.. code-block:: python + + from sqlite_utils.utils import TypeTracker + + tracker = TypeTracker() + db["creatures2"].insert_all(tracker.wrap(rows)) + print(tracker.types) + # Outputs {'id': 'integer', 'name': 'text'} + +We can then apply those types to our new table using the :ref:`table.transform() ` method: + +.. code-block:: python + + db["creatures2"].transform(types=tracker.types) + print(db["creatures2"].schema) + # Outputs: + # CREATE TABLE [creatures2] ( + # [id] INTEGER, + # [name] TEXT + # ); + .. _python_api_gis: SpatiaLite helpers diff --git a/docs/reference.rst b/docs/reference.rst index 6603d99..abd918a 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -86,3 +86,11 @@ sqlite_utils.utils.rows_from_file --------------------------------- .. autofunction:: sqlite_utils.utils.rows_from_file + +.. _reference_utils_typetracker: + +sqlite_utils.utils.TypeTracker +------------------------------ + +.. autoclass:: sqlite_utils.utils.TypeTracker + :members: wrap, types diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index d2ccc5f..98ceffd 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -316,10 +316,35 @@ def rows_from_file( class TypeTracker: + """ + Wrap an iterator of dictionaries and keep track of which SQLite column + types are the most likely fit for each of their keys. + + Example usage: + + .. code-block:: python + + from sqlite_utils.utils import TypeTracker + import sqlite_utils + + db = sqlite_utils.Database(memory=True) + tracker = TypeTracker() + rows = [{"id": "1", "name": "Cleo", "id": "2", "name": "Cardi"}] + db["creatures"].insert_all(tracker.wrap(rows)) + print(tracker.types) + # Outputs {'id': 'integer', 'name': 'text'} + db["creatures"].transform(types=tracker.types) + """ def __init__(self): self.trackers = {} - def wrap(self, iterator): + def wrap(self, iterator: Iterable[dict]) -> Iterable[dict]: + """ + Use this to loop through an existing iterator, tracking the column types + as part of the iteration. + + :param iterator: The iterator to wrap + """ for row in iterator: for key, value in row.items(): tracker = self.trackers.setdefault(key, ValueTracker()) @@ -327,7 +352,11 @@ class TypeTracker: yield row @property - def types(self): + def types(self) -> Dict[str, str]: + """ + A dictionary mapping column names to their detected types. This can be passed + to the ``db[table_name].transform(types=tracker.types)`` method. + """ return {key: tracker.guessed_type for key, tracker in self.trackers.items()} From 8a9fe6498faf783a1fdeb1793e661ad194a05267 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Jun 2022 12:50:15 -0700 Subject: [PATCH 0614/1004] Applied Black, refs #445 --- sqlite_utils/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 98ceffd..377f82a 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -335,6 +335,7 @@ class TypeTracker: # Outputs {'id': 'integer', 'name': 'text'} db["creatures"].transform(types=tracker.types) """ + def __init__(self): self.trackers = {} From 3ddacb7bdc5a949786e341151e34c5fcf9050033 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Jun 2022 12:54:46 -0700 Subject: [PATCH 0615/1004] Justfile for tests and linting, closes #446 --- Justfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 Justfile diff --git a/Justfile b/Justfile new file mode 100644 index 0000000..b069682 --- /dev/null +++ b/Justfile @@ -0,0 +1,12 @@ +@default: test lint + +@test *options: + pipenv run pytest {{options}} + +@lint: + pipenv run black . --check + pipenv run flake8 + pipenv run mypy sqlite_utils tests + +@black: + pipenv run black . From 575431149400fcccb87d69ac7325d81d97686ef6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Jun 2022 08:00:17 -0700 Subject: [PATCH 0616/1004] Only syntax highlight if a code-block is used Refs #447 --- docs/conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 1b1c4f1..d9ebc4b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -85,6 +85,9 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" +# Only syntax highlight of code-block is used: +highlight_language = "none" + # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False From 2768effe078285982fbaae81d2884444b5a682ad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Jun 2022 12:24:23 -0700 Subject: [PATCH 0617/1004] Run cog using just as well, refs #446 --- Justfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Justfile b/Justfile index b069682..aec0f44 100644 --- a/Justfile +++ b/Justfile @@ -7,6 +7,10 @@ pipenv run black . --check pipenv run flake8 pipenv run mypy sqlite_utils tests + cog --check README.md docs/*.rst + +@cog: + cog -r README.md docs/*.rst @black: pipenv run black . From ba8cf54908bc90f0775c4b15efe5e18327796402 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 21 Jun 2022 12:28:01 -0700 Subject: [PATCH 0618/1004] Add documentation strings to Justfile, ref #446 --- Justfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Justfile b/Justfile index aec0f44..52e6190 100644 --- a/Justfile +++ b/Justfile @@ -1,16 +1,21 @@ +# Run tests and linters @default: test lint +# Run pytest with supplied options @test *options: pipenv run pytest {{options}} +# Run linters: black, flake8, mypy, cog @lint: pipenv run black . --check pipenv run flake8 pipenv run mypy sqlite_utils tests cog --check README.md docs/*.rst +# Rebuild docs with cog @cog: cog -r README.md docs/*.rst +# Apply Black @black: pipenv run black . From 9a6495fbef897247551de733fbee204b19ea9385 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jun 2022 23:38:45 +0000 Subject: [PATCH 0619/1004] Configure GitPod --- .gitpod.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitpod.yml diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 0000000..fe8ba1e --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,2 @@ +tasks: +- init: pip install '.[test]' \ No newline at end of file From 2dca2210d9bbaa990c8f1243e98b8c2bfc961a4c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jun 2022 23:39:23 +0000 Subject: [PATCH 0620/1004] Ignore build in .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a224f01..32032d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .venv +build *.db __pycache__/ *.py[cod] From 42440d6345c242ee39778045e29143fb550bd2c2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jun 2022 23:41:13 +0000 Subject: [PATCH 0621/1004] Use parametrize for FTS test --- tests/test_fts.py | 52 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/test_fts.py b/tests/test_fts.py index 53a7bc2..477d599 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -180,32 +180,32 @@ def test_populate_fts_escape_table_names(fresh_db): ] == list(table.search("usa")) -def test_fts_tokenize(fresh_db): - for fts_version in ("4", "5"): - table_name = "searchable_{}".format(fts_version) - table = fresh_db[table_name] - table.insert_all(search_records) - # Test without porter stemming - table.enable_fts( - ["text", "country"], - fts_version="FTS{}".format(fts_version), - ) - assert [] == list(table.search("bite")) - # Test WITH stemming - table.disable_fts() - table.enable_fts( - ["text", "country"], - fts_version="FTS{}".format(fts_version), - tokenize="porter", - ) - rows = list(table.search("bite", order_by="rowid")) - assert len(rows) == 1 - assert { - "rowid": 2, - "text": "racoons are biting trash pandas", - "country": "USA", - "not_searchable": "bar", - }.items() <= rows[0].items() +@pytest.mark.parametrize("fts_version", ("4", "5")) +def test_fts_tokenize(fresh_db, fts_version): + table_name = "searchable_{}".format(fts_version) + table = fresh_db[table_name] + table.insert_all(search_records) + # Test without porter stemming + table.enable_fts( + ["text", "country"], + fts_version="FTS{}".format(fts_version), + ) + assert [] == list(table.search("bite")) + # Test WITH stemming + table.disable_fts() + table.enable_fts( + ["text", "country"], + fts_version="FTS{}".format(fts_version), + tokenize="porter", + ) + rows = list(table.search("bite", order_by="rowid")) + assert len(rows) == 1 + assert { + "rowid": 2, + "text": "racoons are biting trash pandas", + "country": "USA", + "not_searchable": "bar", + }.items() <= rows[0].items() def test_optimize_fts(fresh_db): From b366e68deb0780048a23610c279552f8529d4726 Mon Sep 17 00:00:00 2001 From: David <1690072+davidleejy@users.noreply.github.com> Date: Sat, 16 Jul 2022 05:21:36 +0800 Subject: [PATCH 0622/1004] table.duplicate(new_table_name) feature, closes #449 Thanks, @davidleejy --- sqlite_utils/db.py | 15 +++++++++++++++ tests/test_duplicate.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tests/test_duplicate.py diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c835fbd..5d4d979 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1476,6 +1476,21 @@ class Table(Queryable): ) return self + def duplicate(self, name_new: str) -> "Table": + """ + Duplicate this table in this database. + + :param name_new: Name of new table. + """ + assert self.exists() + with self.db.conn: + sql = "CREATE TABLE [{new_table}] AS SELECT * FROM [{table}];".format( + new_table=name_new, + table=self.name, + ) + self.db.execute(sql) + return self.db[name_new] + def transform( self, *, diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py new file mode 100644 index 0000000..0324b00 --- /dev/null +++ b/tests/test_duplicate.py @@ -0,0 +1,36 @@ +import datetime + + +def test_duplicate(fresh_db): + # Create table using native Sqlite statement: + fresh_db.execute( + """CREATE TABLE [table1] ( + [text_col] TEXT, + [real_col] REAL, + [int_col] INTEGER, + [bool_col] INTEGER, + [datetime_col] TEXT)""" + ) + # Insert one row of mock data: + dt = datetime.datetime.now() + data = { + "text_col": "Cleo", + "real_col": 3.14, + "int_col": -255, + "bool_col": True, + "datetime_col": str(dt), + } + table1 = fresh_db["table1"] + row_id = table1.insert(data).last_rowid + # Duplicate table: + table2 = table1.duplicate("table2") + # Ensure data integrity: + assert data == table2.get(row_id) + # Ensure schema integrity: + assert [ + {"name": "text_col", "type": "TEXT"}, + {"name": "real_col", "type": "REAL"}, + {"name": "int_col", "type": "INT"}, + {"name": "bool_col", "type": "INT"}, + {"name": "datetime_col", "type": "TEXT"}, + ] == [{"name": col.name, "type": col.type} for col in table2.columns] From 4af4762521fe3e1ced7fcade67eaeabf41213aab Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Jul 2022 14:23:57 -0700 Subject: [PATCH 0623/1004] test_duplicate_fails_if_table_does_not_exist, refs #449 --- tests/test_duplicate.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py index 0324b00..f56556d 100644 --- a/tests/test_duplicate.py +++ b/tests/test_duplicate.py @@ -1,4 +1,5 @@ import datetime +import pytest def test_duplicate(fresh_db): @@ -34,3 +35,8 @@ def test_duplicate(fresh_db): {"name": "bool_col", "type": "INT"}, {"name": "datetime_col", "type": "TEXT"}, ] == [{"name": col.name, "type": col.type} for col in table2.columns] + + +def test_duplicate_fails_if_table_does_not_exist(fresh_db): + with pytest.raises(AssertionError): + fresh_db["not_a_table"].duplicate("duplicated") From da030d49fd12bbae931871c54a49caccc604f558 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Jul 2022 14:29:52 -0700 Subject: [PATCH 0624/1004] Documentation for .duplicate(), refs #449 --- docs/python-api.rst | 13 +++++++++++++ sqlite_utils/db.py | 10 +++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 2a87610..4d6676a 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -678,6 +678,19 @@ Here's an example that uses these features: # [score] INTEGER NOT NULL DEFAULT 1 # ) +.. _python_api_duplicate: + +Duplicating tables +================== + +The ``table.duplicate()`` method creates a copy of the table, copying both the table schema and all of the rows in that table: + +.. code-block:: python + + db["authors"].duplicate("authors_copy") + +The new ``authors_copy`` table will now contain a duplicate copy of the data from ``authors``. + .. _python_api_bulk_inserts: Bulk inserts diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 5d4d979..3a27cbe 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1476,20 +1476,20 @@ class Table(Queryable): ) return self - def duplicate(self, name_new: str) -> "Table": + def duplicate(self, new_name: str) -> "Table": """ - Duplicate this table in this database. + Create a duplicate of this table, copying across the schema and all row data. - :param name_new: Name of new table. + :param new_name: Name of the new table """ assert self.exists() with self.db.conn: sql = "CREATE TABLE [{new_table}] AS SELECT * FROM [{table}];".format( - new_table=name_new, + new_table=new_name, table=self.name, ) self.db.execute(sql) - return self.db[name_new] + return self.db[new_name] def transform( self, From c710ade6443d45ca2f581cdf975c7990436ab8fc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Jul 2022 14:45:14 -0700 Subject: [PATCH 0625/1004] sqlite-utils duplicate command, closes #454, refs #449 Also made it so .duplicate() raises new NoTable exception rather than raising an AssertionError if the source table does not exist. --- docs/cli-reference.rst | 14 ++++++++++++++ docs/cli.rst | 9 +++++++++ docs/python-api.rst | 2 ++ sqlite_utils/cli.py | 23 ++++++++++++++++++++++- sqlite_utils/db.py | 9 ++++++++- tests/test_cli.py | 23 +++++++++++++++++++++++ tests/test_duplicate.py | 3 ++- 7 files changed, 80 insertions(+), 3 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 7d73e12..b1a652e 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -1176,6 +1176,20 @@ reset-counts -h, --help Show this message and exit. +duplicate +========= + +:: + + Usage: sqlite-utils duplicate [OPTIONS] PATH TABLE NEW_TABLE + + Create a duplicate of this table, copying across the schema and all row data. + + Options: + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. + + drop-table ========== diff --git a/docs/cli.rst b/docs/cli.rst index 9169fba..e126881 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1476,6 +1476,15 @@ You can specify foreign key relationships between the tables you are creating us If a table with the same name already exists, you will get an error. You can choose to silently ignore this error with ``--ignore``, or you can replace the existing table with a new, empty table using ``--replace``. +.. _cli_duplicate_table: + +Duplicating tables +================== + +The ``duplicate`` command duplicates a table - creating a new table with the same schema and a copy of all of the rows:: + + $ sqlite-utils duplicate books.db authors authors_copy + .. _cli_drop_table: Dropping tables diff --git a/docs/python-api.rst b/docs/python-api.rst index 4d6676a..464afb2 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -691,6 +691,8 @@ The ``table.duplicate()`` method creates a copy of the table, copying both the t The new ``authors_copy`` table will now contain a duplicate copy of the data from ``authors``. +This method raises ``sqlite_utils.db.NoTable`` if the table does not exist. + .. _python_api_bulk_inserts: Bulk inserts diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 86eddfb..0af947d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -5,7 +5,7 @@ from datetime import datetime import hashlib import pathlib import sqlite_utils -from sqlite_utils.db import AlterError, BadMultiValues, DescIndex +from sqlite_utils.db import AlterError, BadMultiValues, DescIndex, NoTable from sqlite_utils.utils import maximize_csv_field_size_limit from sqlite_utils import recipes import textwrap @@ -1465,6 +1465,27 @@ def create_table( ) +@cli.command(name="duplicate") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table") +@click.argument("new_table") +@load_extension_option +def create_table(path, table, new_table, load_extension): + """ + Create a duplicate of this table, copying across the schema and all row data. + """ + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + try: + db[table].duplicate(new_table) + except NoTable: + raise click.ClickException('Table "{}" does not exist'.format(table)) + + @cli.command(name="drop-table") @click.argument( "path", diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3a27cbe..4755ea2 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,3 +1,4 @@ +from ast import Not from .utils import ( chunks, hash_record, @@ -224,6 +225,11 @@ class NoObviousTable(Exception): pass +class NoTable(Exception): + "Specified table does not exist" + pass + + class BadPrimaryKey(Exception): "Table does not have a single obvious primary key" pass @@ -1482,7 +1488,8 @@ class Table(Queryable): :param new_name: Name of the new table """ - assert self.exists() + if not self.exists(): + raise NoTable(f"Table {self.name} does not exist") with self.db.conn: sql = "CREATE TABLE [{new_table}] AS SELECT * FROM [{table}];".format( new_table=new_name, diff --git a/tests/test_cli.py b/tests/test_cli.py index e2e4aa2..a2ce739 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2129,3 +2129,26 @@ def test_analyze(tmpdir, options, expected): result = CliRunner().invoke(cli.cli, ["analyze", db_path] + options) assert result.exit_code == 0 assert list(db["sqlite_stat1"].rows) == expected + + +def test_duplicate_table(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["one"].insert({"id": 1, "name": "Cleo"}, pk="id") + # First try a non-existent table + result_error = CliRunner().invoke( + cli.cli, + ["duplicate", db_path, "missing", "two"], + catch_exceptions=False, + ) + assert result_error.exit_code == 1 + assert result_error.output == 'Error: Table "missing" does not exist\n' + # Now try for a table that exists + result = CliRunner().invoke( + cli.cli, + ["duplicate", db_path, "one", "two"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert db["one"].columns_dict == db["two"].columns_dict + assert list(db["one"].rows) == list(db["two"].rows) diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py index f56556d..87996ef 100644 --- a/tests/test_duplicate.py +++ b/tests/test_duplicate.py @@ -1,3 +1,4 @@ +from sqlite_utils.db import NoTable import datetime import pytest @@ -38,5 +39,5 @@ def test_duplicate(fresh_db): def test_duplicate_fails_if_table_does_not_exist(fresh_db): - with pytest.raises(AssertionError): + with pytest.raises(NoTable): fresh_db["not_a_table"].duplicate("duplicated") From 015c6634644a11e5a3ca9d7dafe22ba62b87f2dd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Jul 2022 14:59:06 -0700 Subject: [PATCH 0626/1004] Fixed lint errors, refs #454 --- sqlite_utils/cli.py | 2 +- sqlite_utils/db.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0af947d..c940b30 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1474,7 +1474,7 @@ def create_table( @click.argument("table") @click.argument("new_table") @load_extension_option -def create_table(path, table, new_table, load_extension): +def duplicate(path, table, new_table, load_extension): """ Create a duplicate of this table, copying across the schema and all row data. """ diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 4755ea2..a2a30f9 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,4 +1,3 @@ -from ast import Not from .utils import ( chunks, hash_record, From e10536c7f59abbb785f092bf83c4ab94c00e31a3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Jul 2022 14:59:30 -0700 Subject: [PATCH 0627/1004] utils.chunks() is now a documented API, closes #451 --- docs/reference.rst | 7 +++++++ sqlite_utils/utils.py | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/reference.rst b/docs/reference.rst index abd918a..66b9d38 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -94,3 +94,10 @@ sqlite_utils.utils.TypeTracker .. autoclass:: sqlite_utils.utils.TypeTracker :members: wrap, types + +.. _reference_utils_chunks: + +sqlite_utils.utils.chunks +------------------------- + +.. autofunction:: sqlite_utils.utils.chunks diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 377f82a..64bba50 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -468,7 +468,13 @@ def _compile_code(code, imports, variable="value"): return locals["fn"] -def chunks(sequence, size): +def chunks(sequence: Iterable, size: int) -> Iterable[Iterable]: + """ + Iterate over chunks of the sequence of the given size. + + :param sequence: Any Python iterator + :param size: The size of each chunk + """ iterator = iter(sequence) for item in iterator: yield itertools.chain([item], itertools.islice(iterator, size - 1)) From 9dd4cf891d9f4565019e030ddc712507ac87b998 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Jul 2022 15:09:07 -0700 Subject: [PATCH 0628/1004] Improve CLI help for drop-table and drop-view, refs #450 --- docs/cli-reference.rst | 4 ++-- sqlite_utils/cli.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index b1a652e..6ef7052 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -1206,7 +1206,7 @@ See :ref:`cli_drop_table`. sqlite-utils drop-table chickens.db chickens Options: - --ignore + --ignore If table does not exist, do nothing --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. @@ -1250,7 +1250,7 @@ See :ref:`cli_drop_view`. sqlite-utils drop-view chickens.db heavy_chickens Options: - --ignore + --ignore If view does not exist, do nothing --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c940b30..09e441c 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1493,7 +1493,7 @@ def duplicate(path, table, new_table, load_extension): required=True, ) @click.argument("table") -@click.option("--ignore", is_flag=True) +@click.option("--ignore", is_flag=True, help="If table does not exist, do nothing") @load_extension_option def drop_table(path, table, ignore, load_extension): """Drop the specified table @@ -1563,7 +1563,7 @@ def create_view(path, view, select, ignore, replace, load_extension): required=True, ) @click.argument("view") -@click.option("--ignore", is_flag=True) +@click.option("--ignore", is_flag=True, help="If view does not exist, do nothing") @load_extension_option def drop_view(path, view, ignore, load_extension): """Drop the specified view From 40b6947255540b8cf0639b87824ea8568ec6863c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Jul 2022 15:20:26 -0700 Subject: [PATCH 0629/1004] enable-fts --replace option, refs #450 Also fixed up some sqlite3.OperationalError imports. --- docs/cli-reference.rst | 1 + sqlite_utils/cli.py | 36 +++++++++++++++++++++++------------- tests/test_cli.py | 25 +++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 6ef7052..09eb0be 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -870,6 +870,7 @@ See :ref:`cli_fts`. --tokenize TEXT Tokenizer to use, e.g. porter --create-triggers Create triggers to update the FTS tables when the parent table changes. + --replace Replace existing FTS configuration if it exists --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 09e441c..cf00898 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -18,6 +18,7 @@ import sys import csv as csv_std import tabulate from .utils import ( + OperationalError, _compile_code, chunks, file_progress, @@ -345,7 +346,7 @@ def analyze(path, names): db.analyze(name) else: db.analyze() - except sqlite3.OperationalError as e: + except OperationalError as e: raise click.ClickException(e) @@ -598,9 +599,14 @@ def create_index( default=False, is_flag=True, ) +@click.option( + "--replace", + is_flag=True, + help="Replace existing FTS configuration if it exists", +) @load_extension_option def enable_fts( - path, table, column, fts4, fts5, tokenize, create_triggers, load_extension + path, table, column, fts4, fts5, tokenize, create_triggers, replace, load_extension ): """Enable full-text search for specific table and columns" @@ -618,12 +624,16 @@ def enable_fts( db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - db[table].enable_fts( - column, - fts_version=fts_version, - tokenize=tokenize, - create_triggers=create_triggers, - ) + try: + db[table].enable_fts( + column, + fts_version=fts_version, + tokenize=tokenize, + create_triggers=create_triggers, + replace=replace, + ) + except OperationalError as ex: + raise click.ClickException(ex) @cli.command(name="populate-fts") @@ -1024,7 +1034,7 @@ def insert_upsert_implementation( ) except Exception as e: if ( - isinstance(e, sqlite3.OperationalError) + isinstance(e, OperationalError) and e.args and "has no column named" in e.args[0] ): @@ -1341,7 +1351,7 @@ def bulk( silent=False, bulk_sql=sql, ) - except (sqlite3.OperationalError, sqlite3.IntegrityError) as e: + except (OperationalError, sqlite3.IntegrityError) as e: raise click.ClickException(str(e)) @@ -1507,7 +1517,7 @@ def drop_table(path, table, ignore, load_extension): _load_extensions(db, load_extension) try: db[table].drop(ignore=ignore) - except sqlite3.OperationalError: + except OperationalError: raise click.ClickException('Table "{}" does not exist'.format(table)) @@ -1577,7 +1587,7 @@ def drop_view(path, view, ignore, load_extension): _load_extensions(db, load_extension) try: db[view].drop(ignore=ignore) - except sqlite3.OperationalError: + except OperationalError: raise click.ClickException('View "{}" does not exist'.format(view)) @@ -1818,7 +1828,7 @@ def _execute_query( with db.conn: try: cursor = db.execute(sql, dict(param)) - except sqlite3.OperationalError as e: + except OperationalError as e: raise click.ClickException(str(e)) if cursor.description is None: # This was an update/insert diff --git a/tests/test_cli.py b/tests/test_cli.py index a2ce739..01cc9bb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -455,6 +455,31 @@ def test_enable_fts(db_path): db["http://example.com"].drop() +def test_enable_fts_replace(db_path): + db = Database(db_path) + assert db["Gosh"].detect_fts() is None + result = CliRunner().invoke( + cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] + ) + assert result.exit_code == 0 + assert "Gosh_fts" == db["Gosh"].detect_fts() + assert db["Gosh_fts"].columns_dict == {"c1": str} + + # This should throw an error + result2 = CliRunner().invoke( + cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] + ) + assert result2.exit_code == 1 + assert result2.output == "Error: table [Gosh_fts] already exists\n" + + # This should work + result3 = CliRunner().invoke( + cli.cli, ["enable-fts", db_path, "Gosh", "c2", "--fts4", "--replace"] + ) + assert result3.exit_code == 0 + assert db["Gosh_fts"].columns_dict == {"c2": str} + + def test_enable_fts_with_triggers(db_path): Database(db_path)["Gosh"].insert_all([{"c1": "baz"}]) exit_code = ( From 2c77f4467efcd540ccd57d438cdebece541b90ac Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Jul 2022 15:25:49 -0700 Subject: [PATCH 0630/1004] Add --ignore to create-index as alias of --if-not-exists, refs #450 --- docs/cli-reference.rst | 12 ++++++------ sqlite_utils/cli.py | 1 + tests/test_cli.py | 13 ++++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 09eb0be..f03348f 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -841,12 +841,12 @@ See :ref:`cli_create_index`. sqlite-utils create-index chickens.db chickens -- -name Options: - --name TEXT Explicit name for the new index - --unique Make this a unique index - --if-not-exists Ignore if index already exists - --analyze Run ANALYZE after creating the index - --load-extension TEXT SQLite extensions to load - -h, --help Show this message and exit. + --name TEXT Explicit name for the new index + --unique Make this a unique index + --if-not-exists, --ignore Ignore if index already exists + --analyze Run ANALYZE after creating the index + --load-extension TEXT SQLite extensions to load + -h, --help Show this message and exit. enable-fts diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cf00898..999658d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -539,6 +539,7 @@ def index_foreign_keys(path, load_extension): @click.option("--unique", help="Make this a unique index", default=False, is_flag=True) @click.option( "--if-not-exists", + "--ignore", help="Ignore if index already exists", default=False, is_flag=True, diff --git a/tests/test_cli.py b/tests/test_cli.py index 01cc9bb..e98ffc1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -214,13 +214,12 @@ def test_create_index(db_path): ] == db["Gosh2"].indexes # Trying to create the same index should fail assert 0 != CliRunner().invoke(cli.cli, create_index_unique_args).exit_code - # ... unless we use --if-not-exists - assert ( - 0 - == CliRunner() - .invoke(cli.cli, create_index_unique_args + ["--if-not-exists"]) - .exit_code - ) + # ... unless we use --if-not-exists or --ignore + for option in ("--if-not-exists", "--ignore"): + assert ( + CliRunner().invoke(cli.cli, create_index_unique_args + [option]).exit_code + == 0 + ) def test_create_index_analyze(db_path): From 5fa823f03ff2117020ae7fd56198ca7d50130574 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Jul 2022 15:31:37 -0700 Subject: [PATCH 0631/1004] add-column --ignore option, refs #450 --- docs/cli-reference.rst | 1 + sqlite_utils/cli.py | 25 +++++++++++++++++++++---- tests/test_cli.py | 13 +++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index f03348f..6d6f837 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -1031,6 +1031,7 @@ See :ref:`cli_add_column`. --fk-col TEXT Referenced column on that foreign key table - if omitted will automatically use the primary key --not-null-default TEXT Add NOT NULL DEFAULT 'TEXT' constraint + --ignore If column already exists, do nothing --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 999658d..de223af 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -418,9 +418,22 @@ def dump(path, load_extension): required=False, help="Add NOT NULL DEFAULT 'TEXT' constraint", ) +@click.option( + "--ignore", + is_flag=True, + help="If column already exists, do nothing", +) @load_extension_option def add_column( - path, table, col_name, col_type, fk, fk_col, not_null_default, load_extension + path, + table, + col_name, + col_type, + fk, + fk_col, + not_null_default, + ignore, + load_extension, ): """Add a column to the specified table @@ -431,9 +444,13 @@ def add_column( """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - db[table].add_column( - col_name, col_type, fk=fk, fk_col=fk_col, not_null_default=not_null_default - ) + try: + db[table].add_column( + col_name, col_type, fk=fk, fk_col=fk_col, not_null_default=not_null_default + ) + except OperationalError as ex: + if not ignore: + raise click.ClickException(str(ex)) @cli.command(name="add-foreign-key") diff --git a/tests/test_cli.py b/tests/test_cli.py index e98ffc1..3694fd7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -271,6 +271,19 @@ def test_add_column(db_path, col_name, col_type, expected_schema): assert expected_schema == collapse_whitespace(db["dogs"].schema) +@pytest.mark.parametrize("ignore", (True, False)) +def test_add_column_ignore(db_path, ignore): + db = Database(db_path) + db.create_table("dogs", {"name": str}) + args = ["add-column", db_path, "dogs", "name"] + (["--ignore"] if ignore else []) + result = CliRunner().invoke(cli.cli, args) + if ignore: + assert result.exit_code == 0 + else: + assert result.exit_code == 1 + assert result.output == "Error: duplicate column name: name\n" + + def test_add_column_not_null_default(db_path): db = Database(db_path) db.create_table("dogs", {"name": str}) From b9a89a0f2c3559989efe65f25a6e1f8fa76fe8b0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Jul 2022 15:35:58 -0700 Subject: [PATCH 0632/1004] duplicate --ignore option, refs #450 --- docs/cli-reference.rst | 1 + sqlite_utils/cli.py | 6 ++++-- tests/test_cli.py | 7 +++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 6d6f837..86225aa 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -1188,6 +1188,7 @@ duplicate Create a duplicate of this table, copying across the schema and all row data. Options: + --ignore If table does not exist, do nothing --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index de223af..fe57fbe 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1501,8 +1501,9 @@ def create_table( ) @click.argument("table") @click.argument("new_table") +@click.option("--ignore", is_flag=True, help="If table does not exist, do nothing") @load_extension_option -def duplicate(path, table, new_table, load_extension): +def duplicate(path, table, new_table, ignore, load_extension): """ Create a duplicate of this table, copying across the schema and all row data. """ @@ -1511,7 +1512,8 @@ def duplicate(path, table, new_table, load_extension): try: db[table].duplicate(new_table) except NoTable: - raise click.ClickException('Table "{}" does not exist'.format(table)) + if not ignore: + raise click.ClickException('Table "{}" does not exist'.format(table)) @cli.command(name="drop-table") diff --git a/tests/test_cli.py b/tests/test_cli.py index 3694fd7..061a578 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2180,6 +2180,13 @@ def test_duplicate_table(tmpdir): ) assert result_error.exit_code == 1 assert result_error.output == 'Error: Table "missing" does not exist\n' + # And check --ignore works + result_error2 = CliRunner().invoke( + cli.cli, + ["duplicate", db_path, "missing", "two", "--ignore"], + catch_exceptions=False, + ) + assert result_error2.exit_code == 0 # Now try for a table that exists result = CliRunner().invoke( cli.cli, From 855bce8c3823718def13e0b8928c58bf857e41b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Jul 2022 15:56:01 -0700 Subject: [PATCH 0633/1004] Release 3.28 Refs #441, #443, #445, #449, #450, #454 --- docs/changelog.rst | 13 +++++++++++++ setup.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b284eb8..fc2b4e9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,19 @@ Changelog =========== +.. _v3_28: + +3.28 (2022-07-15) +----------------- + +- New :ref:`table.duplicate(new_name) ` method for creating a copy of a table with a matching schema and row contents. Thanks, `David `__. (:issue:`449`) +- New ``sqlite-utils duplicate data.db table_name new_name`` CLI command for :ref:`cli_duplicate_table`. (:issue:`454`) +- ``sqlite_utils.utils.rows_from_file()`` is now a :ref:`documented API `. It can be used to read a sequence of dictionaries from a file-like object containing CSV, TSV, JSON or newline-delimited JSON. It can be passed an explicit format or can attempt to detect the format automatically. (:issue:`443`) +- ``sqlite_utils.utils.TypeTracker`` is now a documented API for detecting the likely column types for a sequence of string rows, see :ref:`python_api_typetracker`. (:issue:`445`) +- ``sqlite_utils.utils.chunks()`` is now a documented API for :ref:`splitting an iterator into chunks `. (:issue:`441`) +- ``sqlite-utils enable-fts`` now has a ``--replace`` option for replacing the existing FTS configuration for a table. (:issue:`450`) +- The ``create-index``, ``add-column`` and ``duplicate`` commands all now take a ``--ignore`` option for ignoring errors should the database not be in the right state for them to operate. (:issue:`450`) + .. _v3_27: 3.27 (2022-06-14) diff --git a/setup.py b/setup.py index a416570..019d44e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.27" +VERSION = "3.28" def get_long_description(): From 9e6cceac1c0e086429e2d308b700e59cc53a1991 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 Jul 2022 16:09:53 -0700 Subject: [PATCH 0634/1004] Fixed incorrect issue number --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fc2b4e9..b8f35c5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,7 +11,7 @@ - New ``sqlite-utils duplicate data.db table_name new_name`` CLI command for :ref:`cli_duplicate_table`. (:issue:`454`) - ``sqlite_utils.utils.rows_from_file()`` is now a :ref:`documented API `. It can be used to read a sequence of dictionaries from a file-like object containing CSV, TSV, JSON or newline-delimited JSON. It can be passed an explicit format or can attempt to detect the format automatically. (:issue:`443`) - ``sqlite_utils.utils.TypeTracker`` is now a documented API for detecting the likely column types for a sequence of string rows, see :ref:`python_api_typetracker`. (:issue:`445`) -- ``sqlite_utils.utils.chunks()`` is now a documented API for :ref:`splitting an iterator into chunks `. (:issue:`441`) +- ``sqlite_utils.utils.chunks()`` is now a documented API for :ref:`splitting an iterator into chunks `. (:issue:`451`) - ``sqlite-utils enable-fts`` now has a ``--replace`` option for replacing the existing FTS configuration for a table. (:issue:`450`) - The ``create-index``, ``add-column`` and ``duplicate`` commands all now take a ``--ignore`` option for ignoring errors should the database not be in the right state for them to operate. (:issue:`450`) From 77ca051d4f5ddbd42fd6250749efac7ea85ea094 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 Jul 2022 10:57:50 -0700 Subject: [PATCH 0635/1004] Link to installation instructions (#457) --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index e126881..e3b1294 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -6,7 +6,7 @@ The ``sqlite-utils`` command-line tool can be used to manipulate SQLite databases in a number of different ways. -Once installed, the tool should be available as ``sqlite-utils``. It can also be run using ``python -m sqlite_utils``. +Once :ref:`installed ` the tool should be available as ``sqlite-utils``. It can also be run using ``python -m sqlite_utils``. .. contents:: :local: :class: this-will-duplicate-information-and-it-is-still-useful-here From bfbe69646e67322dde328b07109f77c68463dc71 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 Jul 2022 17:07:23 -0700 Subject: [PATCH 0636/1004] Fixed incorrect register_function code example --- sqlite_utils/db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a2a30f9..ca1f90e 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -373,13 +373,13 @@ class Database: ``fn`` will be made available as a function within SQL, with the same name and number of arguments. Can be used as a decorator:: - @db.register + @db.register_function def upper(value): return str(value).upper() The decorator can take arguments:: - @db.register(deterministic=True, replace=True) + @db.register_function(deterministic=True, replace=True) def upper(value): return str(value).upper() From 1491b66dd7439dd87cd5cd4c4684f46eb3c5751b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 Jul 2022 17:13:49 -0700 Subject: [PATCH 0637/1004] register_function(name=...) argument, closes #458 --- docs/python-api.rst | 10 ++++++++++ sqlite_utils/db.py | 17 +++++++++++------ tests/test_register_function.py | 9 +++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 464afb2..07f2bf6 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2513,6 +2513,16 @@ You can also use the method as a function decorator like so: print(db.execute('select reverse_string("hello")').fetchone()[0]) +By default, the name of the Python function will be used as the name of the SQL function. You can customize this with the ``name=`` keyword argument: + +.. code-block:: python + + @db.register_function(name="rev") + def reverse_string(s): + return "".join(reversed(list(s))) + + print(db.execute('select rev("hello")').fetchone()[0]) + Python 3.8 added the ability to register `deterministic SQLite functions `__, allowing you to indicate that a function will return the exact same result for any given inputs and hence allowing SQLite to apply some performance optimizations. You can mark a function as deterministic using ``deterministic=True``, like this: .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ca1f90e..0533bd1 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -367,7 +367,11 @@ class Database: return "".format(self.conn) def register_function( - self, fn: Callable = None, deterministic: bool = False, replace: bool = False + self, + fn: Callable = None, + deterministic: bool = False, + replace: bool = False, + name: Optional[str] = None, ): """ ``fn`` will be made available as a function within SQL, with the same name and number @@ -388,12 +392,13 @@ class Database: :param fn: Function to register :param deterministic: set ``True`` for functions that always returns the same output for a given input :param replace: set ``True`` to replace an existing function with the same name - otherwise throw an error + :param name: name of the SQLite function - if not specified, the Python function name will be used """ def register(fn): - name = fn.__name__ + fn_name = name or fn.__name__ arity = len(inspect.signature(fn).parameters) - if not replace and (name, arity) in self._registered_functions: + if not replace and (fn_name, arity) in self._registered_functions: return fn kwargs = {} registered = False @@ -401,15 +406,15 @@ class Database: # Try this, but fall back if sqlite3.NotSupportedError try: self.conn.create_function( - name, arity, fn, **dict(kwargs, deterministic=True) + fn_name, arity, fn, **dict(kwargs, deterministic=True) ) registered = True except (sqlite3.NotSupportedError, TypeError): # TypeError is Python 3.7 "function takes at most 3 arguments" pass if not registered: - self.conn.create_function(name, arity, fn, **kwargs) - self._registered_functions.add((name, arity)) + self.conn.create_function(fn_name, arity, fn, **kwargs) + self._registered_functions.add((fn_name, arity)) return fn if fn is None: diff --git a/tests/test_register_function.py b/tests/test_register_function.py index 615f38f..5169a67 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -14,6 +14,15 @@ def test_register_function(fresh_db): assert result == "olleh" +def test_register_function_custom_name(fresh_db): + @fresh_db.register_function(name="revstr") + def reverse_string(s): + return "".join(reversed(list(s))) + + result = fresh_db.execute('select revstr("hello")').fetchone()[0] + assert result == "olleh" + + def test_register_function_multiple_arguments(fresh_db): @fresh_db.register_function def a_times_b_plus_c(a, b, c): From 573de14ab6f4cb23528b97d85578f21eb1ae04d0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 Jul 2022 17:28:46 -0700 Subject: [PATCH 0638/1004] Improved docstring comments for Table class and db.table() --- sqlite_utils/db.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0533bd1..1a15d1c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -488,6 +488,8 @@ class Database: """ Return a table object, optionally configured with default options. + See :ref:`reference_db_table` for option details. + :param table_name: Name of the table """ klass = View if table_name in self.view_names() else Table @@ -1242,7 +1244,29 @@ class Queryable: class Table(Queryable): - "Tables should usually be initialized using the ``db.table(table_name)`` or ``db[table_name]`` methods." + """ + Tables should usually be initialized using the ``db.table(table_name)`` or + ``db[table_name]`` methods. + + The following optional parameters can be passed to ``db.table(table_name, ...)``: + + :param db: Provided by ``db.table(table_name)`` + :param name: Provided by ``db.table(table_name)`` + :param pk: Name of the primary key column, or tuple of columns + :param foreign_keys: List of foreign key definitions + :param column_order: List of column names in the order they should be in the table + :param not_null: List of columns that cannot be null + :param defaults: Dictionary of column names and default values + :param batch_size: Integer number of rows to insert at a time + :param hash_id: If True, use a hash of the row values as the primary key + :param hash_id_columns: List of columns to use for the hash_id + :param alter: If True, automatically alter the table if it doesn't match the schema + :param ignore: If True, ignore rows that already exist when inserting + :param replace: If True, replace rows that already exist when inserting + :param extracts: Dictionary or list of column names to extract into a separate table on inserts + :param conversions: Dictionary of column names and conversion functions + :param columns: Dictionary of column names to column types + """ #: The ``rowid`` of the last inserted, updated or selected row. last_rowid: Optional[int] = None #: The primary key of the last inserted, updated or selected row. From b5e902fcb02953f0f1fc4c652a24c262559a37d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 Jul 2022 08:13:47 -0700 Subject: [PATCH 0639/1004] Applied Black --- sqlite_utils/db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1a15d1c..18a442a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1267,6 +1267,7 @@ class Table(Queryable): :param conversions: Dictionary of column names and conversion functions :param columns: Dictionary of column names to column types """ + #: The ``rowid`` of the last inserted, updated or selected row. last_rowid: Optional[int] = None #: The primary key of the last inserted, updated or selected row. From 1acc04c07124b17da0ca0cfbf34f38664d29fb7f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 Jul 2022 12:12:37 -0700 Subject: [PATCH 0640/1004] Link to new tutorial Refs https://github.com/simonw/datasette.io/issues/108 --- docs/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 49bf8b1..430244e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,8 @@ sqlite-utils is not intended to be a full ORM: the focus is utility helpers to m It is designed as a useful complement to `Datasette `_. +`Cleaning data with sqlite-utils and Datasette `_ provides a tutorial introduction (and accompanying ten minute video) about using this tool. + Contents -------- From 1856002e3c0fcc9f09f72ab7d97ad8c75f6de7df Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 2 Aug 2022 09:02:43 -0700 Subject: [PATCH 0641/1004] readthedocs/readthedocs-preview Tip from https://twitter.com/readthedocs/status/1552354156056395778 --- .github/workflows/documentation-links.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/documentation-links.yml diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml new file mode 100644 index 0000000..4010a44 --- /dev/null +++ b/.github/workflows/documentation-links.yml @@ -0,0 +1,16 @@ +name: Read the Docs Pull Request Preview +on: + pull_request_target: + types: + - opened + +permissions: + pull-requests: write + +jobs: + documentation-links: + runs-on: ubuntu-latest + steps: + - uses: readthedocs/readthedocs-preview@main + with: + project-slug: "readthedocs-preview" From 98a28cbfe6cea67f6334b42b74f35b0ddd309561 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 2 Aug 2022 13:35:56 -0700 Subject: [PATCH 0642/1004] Oops, fixed project slug Refs: - https://github.com/readthedocs/readthedocs-preview/issues/10 - https://github.com/simonw/sqlite-utils/pull/460 --- .github/workflows/documentation-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index 4010a44..0069e4c 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -13,4 +13,4 @@ jobs: steps: - uses: readthedocs/readthedocs-preview@main with: - project-slug: "readthedocs-preview" + project-slug: "sqlite-utils" From 271433fdd18e436b0a527ab899cb6f6fa67f23d0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 2 Aug 2022 14:15:52 -0700 Subject: [PATCH 0643/1004] Discord badge (#462) --- README.md | 1 + docs/index.rst | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6f81fc1..49a26ed 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ [![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=stable)](http://sqlite-utils.datasette.io/en/stable/?badge=stable) [![codecov](https://codecov.io/gh/simonw/sqlite-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/simonw/sqlite-utils) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE) +[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://discord.gg/Ass7bCAMDw) Python CLI utility and library for manipulating SQLite databases. diff --git a/docs/index.rst b/docs/index.rst index 430244e..6793255 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ sqlite-utils |version| ======================= -|PyPI| |Changelog| |CI| |License| +|PyPI| |Changelog| |CI| |License| |discord| .. |PyPI| image:: https://img.shields.io/pypi/v/sqlite-utils.svg :target: https://pypi.org/project/sqlite-utils/ @@ -12,6 +12,8 @@ :target: https://github.com/simonw/sqlite-utils/actions .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg :target: https://github.com/simonw/sqlite-utils/blob/main/LICENSE +.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=discord + :target: https://discord.gg/Ass7bCAMDw *CLI tool and Python utility functions for manipulating SQLite databases* From 45e24deffea042b5db7ab84cd1eb63b3ed9bb9da Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Aug 2022 09:24:02 -0700 Subject: [PATCH 0644/1004] Link API docs to GitHub source code, refs #464 --- docs/conf.py | 20 +++++++++++++++++++- setup.py | 8 +++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d9ebc4b..1339477 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from subprocess import Popen, PIPE +from beanbag_docutils.sphinx.ext.github import github_linkcode_resolve # This file is execfile()d with the current directory set to its # containing dir. @@ -30,7 +31,12 @@ from subprocess import Popen, PIPE # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc", "sphinx_copybutton"] +extensions = [ + "sphinx.ext.extlinks", + "sphinx.ext.autodoc", + "sphinx_copybutton", + "sphinx.ext.linkcode", +] autodoc_member_order = "bysource" autodoc_typehints = "description" @@ -38,6 +44,18 @@ extlinks = { "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"), } + +def linkcode_resolve(domain, info): + return github_linkcode_resolve( + domain=domain, + info=info, + allowed_module_names=["sqlite_utils"], + github_org_id="simonw", + github_repo_id="sqlite-utils", + branch="main", + ) + + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/setup.py b/setup.py index 019d44e..8b8434a 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,13 @@ setup( ], extras_require={ "test": ["pytest", "black", "hypothesis", "cogapp"], - "docs": ["furo", "sphinx-autobuild", "codespell", "sphinx-copybutton"], + "docs": [ + "furo", + "sphinx-autobuild", + "codespell", + "sphinx-copybutton", + "beanbag-docutils @ https://github.com/simonw/beanbag-docutils/archive/refs/heads/bytes-in-url.zip", + ], "mypy": [ "mypy", "types-click", From 83e7339255e811c62e6db8498c483c44a84d0f28 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 18 Aug 2022 01:11:15 +0200 Subject: [PATCH 0645/1004] Use Read the Docs action v1 (#463) Read the Docs repository was renamed from `readthedocs/readthedocs-preview` to `readthedocs/actions/`. Now, the `preview` action is under `readthedocs/actions/preview` and is tagged as `v1` --- .github/workflows/documentation-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index 0069e4c..1aab04e 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -11,6 +11,6 @@ jobs: documentation-links: runs-on: ubuntu-latest steps: - - uses: readthedocs/readthedocs-preview@main + - uses: readthedocs/actions/preview@v1 with: project-slug: "sqlite-utils" From f8ffac8787e299a62c99ed1ce914cd5ace84ad94 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Aug 2022 16:38:02 -0700 Subject: [PATCH 0646/1004] beanbag-docutils>=2.0 (#465) * beanbag-docutils>=2.0 Closes #464 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8b8434a..ead463b 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ setup( "sphinx-autobuild", "codespell", "sphinx-copybutton", - "beanbag-docutils @ https://github.com/simonw/beanbag-docutils/archive/refs/heads/bytes-in-url.zip", + "beanbag-docutils>=2.0", ], "mypy": [ "mypy", From f4fb78fa95057fbc86c734020835a3155695297f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Aug 2022 14:58:07 -0700 Subject: [PATCH 0647/1004] Cross-link CLI to Python docs (#460) * Start cross-linking CLI to Python docs, refs #426 * More links to Python from CLI page, refs #426 --- docs/cli-reference.rst | 85 ++++++++++++++++++++++++++++++++++++++++++ docs/cli.rst | 25 +++++++++++++ 2 files changed, 110 insertions(+) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 86225aa..2590595 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -63,6 +63,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. commands.sort(key = lambda command: go_first.index(command) if command in go_first else 999) cog.out("\n") for command in commands: + cog.out(".. _cli_ref_" + command.replace("-", "_") + ":\n\n") cog.out(command + "\n") cog.out(("=" * len(command)) + "\n\n") if command in refs: @@ -81,6 +82,8 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. cog.out("\n\n") .. ]]] +.. _cli_ref_query: + query ===== @@ -120,6 +123,8 @@ See :ref:`cli_query`. -h, --help Show this message and exit. +.. _cli_ref_memory: + memory ====== @@ -184,6 +189,8 @@ See :ref:`cli_memory`. -h, --help Show this message and exit. +.. _cli_ref_insert: + insert ====== @@ -264,6 +271,8 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr -h, --help Show this message and exit. +.. _cli_ref_upsert: + upsert ====== @@ -312,6 +321,8 @@ See :ref:`cli_upsert`. -h, --help Show this message and exit. +.. _cli_ref_bulk: + bulk ==== @@ -352,6 +363,8 @@ See :ref:`cli_bulk`. -h, --help Show this message and exit. +.. _cli_ref_search: + search ====== @@ -390,6 +403,8 @@ See :ref:`cli_search`. -h, --help Show this message and exit. +.. _cli_ref_transform: + transform ========= @@ -424,6 +439,8 @@ See :ref:`cli_transform_table`. -h, --help Show this message and exit. +.. _cli_ref_extract: + extract ======= @@ -447,6 +464,8 @@ See :ref:`cli_extract`. -h, --help Show this message and exit. +.. _cli_ref_schema: + schema ====== @@ -467,6 +486,8 @@ See :ref:`cli_schema`. -h, --help Show this message and exit. +.. _cli_ref_insert_files: + insert-files ============ @@ -503,6 +524,8 @@ See :ref:`cli_insert_files`. -h, --help Show this message and exit. +.. _cli_ref_analyze_tables: + analyze-tables ============== @@ -525,6 +548,8 @@ See :ref:`cli_analyze_tables`. -h, --help Show this message and exit. +.. _cli_ref_convert: + convert ======= @@ -590,6 +615,8 @@ See :ref:`cli_convert`. -h, --help Show this message and exit. +.. _cli_ref_tables: + tables ====== @@ -628,6 +655,8 @@ See :ref:`cli_tables`. -h, --help Show this message and exit. +.. _cli_ref_views: + views ===== @@ -664,6 +693,8 @@ See :ref:`cli_views`. -h, --help Show this message and exit. +.. _cli_ref_rows: + rows ==== @@ -702,6 +733,8 @@ See :ref:`cli_rows`. -h, --help Show this message and exit. +.. _cli_ref_triggers: + triggers ======== @@ -735,6 +768,8 @@ See :ref:`cli_triggers`. -h, --help Show this message and exit. +.. _cli_ref_indexes: + indexes ======= @@ -769,6 +804,8 @@ See :ref:`cli_indexes`. -h, --help Show this message and exit. +.. _cli_ref_create_database: + create-database =============== @@ -791,6 +828,8 @@ See :ref:`cli_create_database`. -h, --help Show this message and exit. +.. _cli_ref_create_table: + create-table ============ @@ -821,6 +860,8 @@ See :ref:`cli_create_table`. -h, --help Show this message and exit. +.. _cli_ref_create_index: + create-index ============ @@ -849,6 +890,8 @@ See :ref:`cli_create_index`. -h, --help Show this message and exit. +.. _cli_ref_enable_fts: + enable-fts ========== @@ -875,6 +918,8 @@ See :ref:`cli_fts`. -h, --help Show this message and exit. +.. _cli_ref_populate_fts: + populate-fts ============ @@ -893,6 +938,8 @@ populate-fts -h, --help Show this message and exit. +.. _cli_ref_rebuild_fts: + rebuild-fts =========== @@ -911,6 +958,8 @@ rebuild-fts -h, --help Show this message and exit. +.. _cli_ref_disable_fts: + disable-fts =========== @@ -929,6 +978,8 @@ disable-fts -h, --help Show this message and exit. +.. _cli_ref_optimize: + optimize ======== @@ -951,6 +1002,8 @@ See :ref:`cli_optimize`. -h, --help Show this message and exit. +.. _cli_ref_analyze: + analyze ======= @@ -971,6 +1024,8 @@ See :ref:`cli_analyze`. -h, --help Show this message and exit. +.. _cli_ref_vacuum: + vacuum ====== @@ -990,6 +1045,8 @@ See :ref:`cli_vacuum`. -h, --help Show this message and exit. +.. _cli_ref_dump: + dump ==== @@ -1010,6 +1067,8 @@ See :ref:`cli_dump`. -h, --help Show this message and exit. +.. _cli_ref_add_column: + add-column ========== @@ -1036,6 +1095,8 @@ See :ref:`cli_add_column`. -h, --help Show this message and exit. +.. _cli_ref_add_foreign_key: + add-foreign-key =============== @@ -1060,6 +1121,8 @@ See :ref:`cli_add_foreign_key`. -h, --help Show this message and exit. +.. _cli_ref_add_foreign_keys: + add-foreign-keys ================ @@ -1082,6 +1145,8 @@ See :ref:`cli_add_foreign_keys`. -h, --help Show this message and exit. +.. _cli_ref_index_foreign_keys: + index-foreign-keys ================== @@ -1102,6 +1167,8 @@ See :ref:`cli_index_foreign_keys`. -h, --help Show this message and exit. +.. _cli_ref_enable_wal: + enable-wal ========== @@ -1122,6 +1189,8 @@ See :ref:`cli_wal`. -h, --help Show this message and exit. +.. _cli_ref_disable_wal: + disable-wal =========== @@ -1140,6 +1209,8 @@ disable-wal -h, --help Show this message and exit. +.. _cli_ref_enable_counts: + enable-counts ============= @@ -1160,6 +1231,8 @@ See :ref:`cli_enable_counts`. -h, --help Show this message and exit. +.. _cli_ref_reset_counts: + reset-counts ============ @@ -1178,6 +1251,8 @@ reset-counts -h, --help Show this message and exit. +.. _cli_ref_duplicate: + duplicate ========= @@ -1193,6 +1268,8 @@ duplicate -h, --help Show this message and exit. +.. _cli_ref_drop_table: + drop-table ========== @@ -1214,6 +1291,8 @@ See :ref:`cli_drop_table`. -h, --help Show this message and exit. +.. _cli_ref_create_view: + create-view =========== @@ -1237,6 +1316,8 @@ See :ref:`cli_create_view`. -h, --help Show this message and exit. +.. _cli_ref_drop_view: + drop-view ========= @@ -1258,6 +1339,8 @@ See :ref:`cli_drop_view`. -h, --help Show this message and exit. +.. _cli_ref_add_geometry_column: + add-geometry-column =================== @@ -1287,6 +1370,8 @@ See :ref:`cli_spatialite`. -h, --help Show this message and exit. +.. _cli_ref_create_spatial_index: + create-spatial-index ==================== diff --git a/docs/cli.rst b/docs/cli.rst index e3b1294..d2d1812 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -21,6 +21,9 @@ The ``sqlite-utils query`` command lets you run queries directly against a SQLit $ sqlite-utils query dogs.db "select * from dogs" $ sqlite-utils dogs.db "select * from dogs" +.. note:: + In Python: :ref:`db.query() ` CLI reference: :ref:`sqlite-utils query ` + .. _cli_query_json: Returning JSON @@ -257,6 +260,7 @@ You can load SQLite extension modules using the ``--load-extension`` option, see $ sqlite-utils dogs.db "select spatialite_version()" --load-extension=spatialite [{"spatialite_version()": "4.3.0a"}] + .. _cli_query_attach: Attaching additional databases @@ -271,6 +275,9 @@ This example attaches the ``books.db`` database under the alias ``books`` and th sqlite-utils dogs.db --attach books books.db \ 'select * from sqlite_master union all select * from books.sqlite_master' +.. note:: + In Python: :ref:`db.attach() ` + .. _cli_memory: Querying data directly using an in-memory database @@ -462,6 +469,9 @@ Or pass named parameters using ``--where`` in combination with ``-p``:: Use ``--limit N`` to only return the first ``N`` rows. Use ``--offset N`` to return rows starting from the specified offset. +.. note:: + In Python: :ref:`table.rows ` CLI reference: :ref:`sqlite-utils rows ` + .. _cli_tables: Listing tables @@ -514,6 +524,9 @@ Use ``--schema`` to include the schema of each table:: The ``--nl``, ``--csv``, ``--tsv``, ``--table`` and ``--fmt`` options are also available. +.. note:: + In Python: :ref:`db.tables or db.table_names() ` CLI reference: :ref:`sqlite-utils tables ` + .. _cli_views: Listing views @@ -537,6 +550,9 @@ It takes the same options as the ``tables`` command: * ``--tsv`` * ``--table`` +.. note:: + In Python: :ref:`db.views or db.view_names() ` CLI reference: :ref:`sqlite-utils views ` + .. _cli_indexes: Listing indexes @@ -564,6 +580,9 @@ The command defaults to only showing the columns that are explicitly part of the The command takes the same format options as the ``tables`` and ``views`` commands. +.. note:: + In Python: :ref:`table.indexes ` CLI reference: :ref:`sqlite-utils indexes ` + .. _cli_triggers: Listing triggers @@ -592,6 +611,9 @@ It defaults to showing triggers for all tables. To see triggers for one or more The command takes the same format options as the ``tables`` and ``views`` commands. +.. note:: + In Python: :ref:`table.triggers or db.triggers ` CLI reference: :ref:`sqlite-utils triggers ` + .. _cli_schema: Showing the schema @@ -610,6 +632,9 @@ This will show the schema for every table and index in the database. To view the $ sqlite-utils schema dogs.db dogs chickens ... +.. note:: + In Python: :ref:`table.schema ` or :ref:`db.schema ` CLI reference: :ref:`sqlite-utils schema ` + .. _cli_analyze_tables: Analyzing tables From 7a9a6363ffc1b4f1a9444a22999addabfa756c54 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 21:10:20 -0700 Subject: [PATCH 0648/1004] sqlite-utils rows --order option, closes #469 --- docs/cli-reference.rst | 1 + docs/cli.rst | 2 ++ sqlite_utils/cli.py | 4 ++++ tests/test_cli.py | 9 +++++++++ 4 files changed, 16 insertions(+) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 2590595..c0f794c 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -713,6 +713,7 @@ See :ref:`cli_rows`. Options: -c, --column TEXT Columns to return --where TEXT Optional where clause + -o, --order TEXT Order by ('column' or 'column desc') -p, --param ... Named :parameters for where clause --limit INTEGER Number of rows to return - defaults to everything --offset INTEGER SQL offset to use diff --git a/docs/cli.rst b/docs/cli.rst index d2d1812..8cb2c56 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -467,6 +467,8 @@ Or pass named parameters using ``--where`` in combination with ``-p``:: $ sqlite-utils rows dogs.db dogs -c name --where 'name = :name' -p name Cleo [{"name": "Cleo"}] +You can define a sort order using ``--order column`` or ``--order 'column desc'``. + Use ``--limit N`` to only return the first ``N`` rows. Use ``--offset N`` to return rows starting from the specified offset. .. note:: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index fe57fbe..43e76fa 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1985,6 +1985,7 @@ def search( @click.argument("dbtable") @click.option("-c", "--column", type=str, multiple=True, help="Columns to return") @click.option("--where", help="Optional where clause") +@click.option("-o", "--order", type=str, help="Order by ('column' or 'column desc')") @click.option( "-p", "--param", @@ -2011,6 +2012,7 @@ def rows( dbtable, column, where, + order, param, limit, offset, @@ -2037,6 +2039,8 @@ def rows( sql = "select {} from [{}]".format(columns, dbtable) if where: sql += " where " + where + if order: + sql += " order by " + order if limit: sql += " limit {}".format(limit) if offset: diff --git a/tests/test_cli.py b/tests/test_cli.py index 061a578..4eadb88 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -881,6 +881,15 @@ def test_query_memory_does_not_create_file(tmpdir): ["-c", "name", "--where", "id = :id", "--param", "id", "1"], '[{"name": "Cleo"}]', ), + # --order + ( + ["-c", "id", "--order", "id desc", "--limit", "1"], + '[{"id": 2}]', + ), + ( + ["-c", "id", "--order", "id", "--limit", "1"], + '[{"id": 1}]', + ), ], ) def test_rows(db_path, args, expected): From 31f062d4a7e6457bfbe94b2e45a7b80028f1e95c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 21:53:55 -0700 Subject: [PATCH 0649/1004] sqlite-utils query --functions option, refs #471 --- docs/cli-reference.rst | 2 ++ docs/cli.rst | 20 +++++++++++++ sqlite_utils/cli.py | 17 +++++++++++ tests/test_cli.py | 68 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 106 insertions(+), 1 deletion(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c0f794c..af059f1 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -119,6 +119,8 @@ See :ref:`cli_query`. escaped strings -r, --raw Raw output, first column of first row -p, --param ... Named :parameters for SQL query + --functions TEXT Python code defining one or more custom SQL + functions --load-extension TEXT SQLite extensions to load -h, --help Show this message and exit. diff --git a/docs/cli.rst b/docs/cli.rst index 8cb2c56..1632e2f 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -250,6 +250,26 @@ If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will re $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" [{"rows_affected": 1}] +.. _cli_query_functions: + +Defining custom SQL functions +----------------------------- + +You can use the ``--functions`` option to pass a block of Python code that defines additional functions which can then be called by your SQL query. + +This example defines a function which extracts the domain from a URL:: + + $ sqlite-utils query dogs.db "select url, domain(url) from urls" --functions ' + from urllib.parse import urlparse + + def domain(url): + return urlparse(url).netloc + ' + +Every callable object defined in the block will be registered as a SQL function with the same name, with the exception of functions with names that begin with an underscore. + +.. _cli_query_extensions: + SQLite extensions ----------------- diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 43e76fa..1ac8d78 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1633,6 +1633,9 @@ def drop_view(path, view, ignore, load_extension): type=(str, str), help="Named :parameters for SQL query", ) +@click.option( + "--functions", help="Python code defining one or more custom SQL functions" +) @load_extension_option def query( path, @@ -1649,6 +1652,7 @@ def query( raw, param, load_extension, + functions, ): """Execute SQL query and return the results as JSON @@ -1665,6 +1669,19 @@ def query( _load_extensions(db, load_extension) db.register_fts4_bm25() + # Register any Python functions as SQL functions: + if functions: + sqlite3.enable_callback_tracebacks(True) + globals = {} + try: + exec(functions, globals) + except SyntaxError as ex: + raise click.ClickException("Error in functions definition: {}".format(ex)) + # Register all callables in the locals dict: + for name, value in globals.items(): + if callable(value) and not name.startswith("_"): + db.register_function(value, name=name) + _execute_query( db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4eadb88..057f129 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -683,6 +683,11 @@ _one_query = "select id, name, age from dogs where id = 1" (_one_query, ["--nl"], '{"id": 1, "name": "Cleo", "age": 4}'), (_one_query, ["--arrays"], '[[1, "Cleo", 4]]'), (_one_query, ["--arrays", "--nl"], '[1, "Cleo", 4]'), + ( + "select id, dog(age) from dogs", + ["--functions", "def dog(i):\n return i * 7"], + '[{"id": 1, "dog(age)": 28},\n {"id": 2, "dog(age)": 14}]', + ), ], ) def test_query_json(db_path, sql, args, expected): @@ -700,11 +705,72 @@ def test_query_json(db_path, sql, args, expected): def test_query_json_empty(db_path): result = CliRunner().invoke( - cli.cli, [db_path, "select * from sqlite_master where 0"] + cli.cli, + [db_path, "select * from sqlite_master where 0"], ) assert result.output.strip() == "[]" +def test_query_invalid_function(db_path): + result = CliRunner().invoke( + cli.cli, [db_path, "select bad()", "--functions", "def invalid_python"] + ) + assert result.exit_code == 1 + assert ( + result.output.strip() + == "Error: Error in functions definition: invalid syntax (, line 1)" + ) + + +TEST_FUNCTIONS = """ +def zero(): + return 0 + +def one(a): + return a + +def _two(a, b): + return a + b + +def two(a, b): + return _two(a, b) +""" + + +def test_query_complex_function(db_path): + result = CliRunner().invoke( + cli.cli, + [ + db_path, + "select zero(), one(1), two(1, 2)", + "--functions", + TEST_FUNCTIONS, + ], + ) + assert result.exit_code == 0 + assert json.loads(result.output.strip()) == [ + {"zero()": 0, "one(1)": 1, "two(1, 2)": 3} + ] + + +def test_hidden_functions_are_hidden(db_path): + result = CliRunner().invoke( + cli.cli, + [ + db_path, + "select name from pragma_function_list()", + "--functions", + TEST_FUNCTIONS, + ], + ) + assert result.exit_code == 0 + functions = {r["name"] for r in json.loads(result.output.strip())} + assert "zero" in functions + assert "one" in functions + assert "two" in functions + assert "_two" not in functions + + LOREM_IPSUM_COMPRESSED = ( b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8e" b"f\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J" From 85e7411bbd2884e42c65c3e93330f0ddd986be38 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 22:01:58 -0700 Subject: [PATCH 0650/1004] Skip test if pragma_function_list not supported, refs #471 --- tests/test_cli.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 057f129..9b43cd5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -12,6 +12,15 @@ import textwrap from .utils import collapse_whitespace +def _supports_pragma_function_list(): + db = Database(memory=True) + try: + db.execute("select * from pragma_function_list()") + except Exception: + return False + return True + + @pytest.mark.parametrize( "options", ( @@ -753,6 +762,10 @@ def test_query_complex_function(db_path): ] +@pytest.mark.skipif( + not _supports_pragma_function_list(), + reason="Needs SQLite version that supports pragma_function_list()", +) def test_hidden_functions_are_hidden(db_path): result = CliRunner().invoke( cli.cli, From 59e2cfbdc12082bac03e8ac6f99c8c41a4bc72ba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 22:03:53 -0700 Subject: [PATCH 0651/1004] sqlite-utils memory --functions, refs #471 --- docs/cli-reference.rst | 2 ++ sqlite_utils/cli.py | 33 ++++++++++++++++++++++----------- tests/test_cli_memory.py | 9 +++++++++ 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index af059f1..9a025ba 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -161,6 +161,8 @@ See :ref:`cli_memory`. sqlite-utils memory animals.csv --schema Options: + --functions TEXT Python code defining one or more custom SQL + functions --attach ... Additional databases to attach - specify alias and filepath --flatten Flatten nested JSON objects, so {"foo": {"bar": diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 1ac8d78..24c8c03 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1669,18 +1669,8 @@ def query( _load_extensions(db, load_extension) db.register_fts4_bm25() - # Register any Python functions as SQL functions: if functions: - sqlite3.enable_callback_tracebacks(True) - globals = {} - try: - exec(functions, globals) - except SyntaxError as ex: - raise click.ClickException("Error in functions definition: {}".format(ex)) - # Register all callables in the locals dict: - for name, value in globals.items(): - if callable(value) and not name.startswith("_"): - db.register_function(value, name=name) + _register_functions(db, functions) _execute_query( db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols @@ -1695,6 +1685,9 @@ def query( nargs=-1, ) @click.argument("sql") +@click.option( + "--functions", help="Python code defining one or more custom SQL functions" +) @click.option( "--attach", type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), @@ -1741,6 +1734,7 @@ def query( def memory( paths, sql, + functions, attach, flatten, nl, @@ -1854,6 +1848,9 @@ def memory( _load_extensions(db, load_extension) db.register_fts4_bm25() + if functions: + _register_functions(db, functions) + _execute_query( db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols ) @@ -2993,3 +2990,17 @@ def _load_extensions(db, load_extension): if ext == "spatialite" and not os.path.exists(ext): ext = find_spatialite() db.conn.load_extension(ext) + + +def _register_functions(db, functions): + # Register any Python functions as SQL functions: + sqlite3.enable_callback_tracebacks(True) + globals = {} + try: + exec(functions, globals) + except SyntaxError as ex: + raise click.ClickException("Error in functions definition: {}".format(ex)) + # Register all callables in the locals dict: + for name, value in globals.items(): + if callable(value) and not name.startswith("_"): + db.register_function(value, name=name) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index aee74e7..bb50d23 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -289,3 +289,12 @@ def test_memory_two_files_with_same_stem(tmpdir): ");\n" "CREATE VIEW t2 AS select * from [data_2];\n" ) + + +def test_memory_functions(): + result = CliRunner().invoke( + cli.cli, + ["memory", "select hello()", "--functions", "hello = lambda: 'Hello'"], + ) + assert result.exit_code == 0 + assert result.output.strip() == '[{"hello()": "Hello"}]' From 23ef1d6c20f6a8ef0db508b9711ae0d8ed6a4156 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 22:10:43 -0700 Subject: [PATCH 0652/1004] bulk --functions, closes #471 --- docs/cli-reference.rst | 1 + sqlite_utils/cli.py | 8 ++++++++ tests/test_cli_bulk.py | 8 +++++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 9a025ba..c194967 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -349,6 +349,7 @@ See :ref:`cli_bulk`. Options: --batch-size INTEGER Commit every X records + --functions TEXT Python code defining one or more custom SQL functions --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 24c8c03..16a99cc 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -926,9 +926,12 @@ def insert_upsert_implementation( load_extension=None, silent=False, bulk_sql=None, + functions=None, ): db = sqlite_utils.Database(path) _load_extensions(db, load_extension) + if functions: + _register_functions(db, functions) if (delimiter or quotechar or sniff or no_headers) and not tsv: csv = True if (nl + csv + tsv) >= 2: @@ -1305,6 +1308,9 @@ def upsert( @click.argument("sql") @click.argument("file", type=click.File("rb"), required=True) @click.option("--batch-size", type=int, default=100, help="Commit every X records") +@click.option( + "--functions", help="Python code defining one or more custom SQL functions" +) @import_options @load_extension_option def bulk( @@ -1312,6 +1318,7 @@ def bulk( sql, file, batch_size, + functions, flatten, nl, csv, @@ -1368,6 +1375,7 @@ def bulk( load_extension=load_extension, silent=False, bulk_sql=sql, + functions=functions, ) except (OperationalError, sqlite3.IntegrityError) as e: raise click.ClickException(str(e)) diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py index 6d01952..909ed09 100644 --- a/tests/test_cli_bulk.py +++ b/tests/test_cli_bulk.py @@ -28,9 +28,11 @@ def test_cli_bulk(test_db_and_path): [ "bulk", db_path, - "insert into example (id, name) values (:id, :name)", + "insert into example (id, name) values (:id, myupper(:name))", "-", "--nl", + "--functions", + "myupper = lambda s: s.upper()", ], input='{"id": 3, "name": "Three"}\n{"id": 4, "name": "Four"}\n', ) @@ -38,8 +40,8 @@ def test_cli_bulk(test_db_and_path): assert [ {"id": 1, "name": "One"}, {"id": 2, "name": "Two"}, - {"id": 3, "name": "Three"}, - {"id": 4, "name": "Four"}, + {"id": 3, "name": "THREE"}, + {"id": 4, "name": "FOUR"}, ] == list(db["example"].rows) From a46a5e3a9e03dcdd8c84a92e4a5dbfa02ba461fa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 22:20:09 -0700 Subject: [PATCH 0653/1004] Improved code compilation pattern, closes #472 --- docs/cli.rst | 3 --- sqlite_utils/utils.py | 9 ++++----- tests/test_cli_convert.py | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 1632e2f..3c5c595 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1359,12 +1359,9 @@ The following example adds a new ``score`` column, then updates it to list a ran random.seed(10) def convert(value): - global random return random.random() ' -Note the ``global random`` line here. Due to the way the tool compiles Python code, this is necessary to ensure the ``random`` module is available within the ``convert()`` function. If you were to omit this you would see a ``NameError: name 'random' is not defined`` error. - .. _cli_convert_recipes: sqlite-utils convert recipes diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 64bba50..c0b7bf1 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -432,12 +432,11 @@ def progressbar(*args, **kwargs): def _compile_code(code, imports, variable="value"): - locals = {} globals = {"r": recipes, "recipes": recipes} # If user defined a convert() function, return that try: - exec(code, globals, locals) - return locals["convert"] + exec(code, globals) + return globals["convert"] except (AttributeError, SyntaxError, NameError, KeyError, TypeError): pass @@ -464,8 +463,8 @@ def _compile_code(code, imports, variable="value"): for import_ in imports: globals[import_.split(".")[0]] = __import__(import_) - exec(code_o, globals, locals) - return locals["fn"] + exec(code_o, globals) + return globals["fn"] def chunks(sequence: Iterable, size: int) -> Iterable[Iterable]: diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 10b4563..d26ab8c 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -606,3 +606,23 @@ def test_convert_hyphen_workaround(fresh_db_and_path): assert list(db["names"].rows) == [ {"id": 1, "name": "-"}, ] + + +def test_convert_initialization_pattern(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "names", + "name", + "-", + ], + input="import random\nrandom.seed(1)\ndef convert(value): return random.randint(0, 100)", + ) + assert 0 == result.exit_code, result.output + assert list(db["names"].rows) == [ + {"id": 1, "name": "17"}, + ] From 19dd077944429c1365b513d80cc71c605ae3bed3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Aug 2022 22:55:47 -0700 Subject: [PATCH 0654/1004] Support entrypoints for `--load-extension` (#473) * Entrypoint support, closes #470 --- .github/workflows/test.yml | 4 ++ .gitignore | 3 ++ docs/cli-reference.rst | 82 ++++++++++++++++++++------------------ sqlite_utils/cli.py | 8 +++- tests/ext.c | 48 ++++++++++++++++++++++ tests/test_cli.py | 41 +++++++++++++++++++ 6 files changed, 145 insertions(+), 41 deletions(-) create mode 100644 tests/ext.c diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 75ef594..9303965 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,10 @@ jobs: - name: Install SpatiaLite if: matrix.os == 'ubuntu-latest' run: sudo apt-get install libsqlite3-mod-spatialite + - name: Build extension for --load-extension test + if: matrix.os == 'ubuntu-latest' + run: |- + (cd tests && gcc ext.c -fPIC -shared -o ext.so && ls -lah) - name: Run tests run: | pytest -v diff --git a/.gitignore b/.gitignore index 32032d5..7947f33 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ venv Pipfile Pipfile.lock pyproject.toml +tests/*.dylib +tests/*.so +tests/*.dll diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c194967..d3951fd 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -121,7 +121,8 @@ See :ref:`cli_query`. -p, --param ... Named :parameters for SQL query --functions TEXT Python code defining one or more custom SQL functions - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint -h, --help Show this message and exit. @@ -189,7 +190,8 @@ See :ref:`cli_memory`. --dump Dump SQL for in-memory database --save FILE Save in-memory database to this file --analyze Analyze resulting tables and output results - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint -h, --help Show this message and exit. @@ -266,7 +268,7 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr --default ... Default value that should be set for a column -d, --detect-types Detect types for columns in CSV/TSV data --analyze Run ANALYZE at the end of this operation - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint --silent Do not show progress bar --ignore Ignore records if pk already exists --replace Replace records if pk already exists @@ -320,7 +322,7 @@ See :ref:`cli_upsert`. --default ... Default value that should be set for a column -d, --detect-types Detect types for columns in CSV/TSV data --analyze Run ANALYZE at the end of this operation - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint --silent Do not show progress bar -h, --help Show this message and exit. @@ -364,7 +366,7 @@ See :ref:`cli_bulk`. --sniff Detect delimiter and quote character --no-headers CSV file has no header row --encoding TEXT Character encoding for input, defaults to utf-8 - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -404,7 +406,7 @@ See :ref:`cli_search`. textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -440,7 +442,7 @@ See :ref:`cli_transform_table`. --default-none TEXT Remove default from this column --drop-foreign-key TEXT Drop foreign key constraint for this column --sql Output SQL without executing it - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -465,7 +467,7 @@ See :ref:`cli_extract`. --table TEXT Name of the other table to extract columns to --fk-column TEXT Name of the foreign key column to add to the table --rename ... Rename this column in extracted table - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -487,7 +489,7 @@ See :ref:`cli_schema`. sqlite-utils schema trees.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -525,7 +527,7 @@ See :ref:`cli_insert_files`. --text Store file content as TEXT, not BLOB --encoding TEXT Character encoding for input, defaults to utf-8 -s, --silent Don't show a progress bar - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -549,7 +551,7 @@ See :ref:`cli_analyze_tables`. Options: -c, --column TEXT Specific columns to analyze --save Save results to _analyze_tables table - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -656,7 +658,7 @@ See :ref:`cli_tables`. strings --columns Include list of columns for each table --schema Include schema for each table - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -694,7 +696,7 @@ See :ref:`cli_views`. strings --columns Include list of columns for each view --schema Include schema for each view - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -735,7 +737,8 @@ See :ref:`cli_rows`. simple, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint -h, --help Show this message and exit. @@ -770,7 +773,7 @@ See :ref:`cli_triggers`. textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -806,7 +809,7 @@ See :ref:`cli_indexes`. textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -830,7 +833,7 @@ See :ref:`cli_create_database`. Options: --enable-wal Enable WAL mode on the created database --init-spatialite Enable SpatiaLite on the created database - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -862,7 +865,7 @@ See :ref:`cli_create_table`. foreign key --ignore If table already exists, do nothing --replace If table already exists, replace it - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -892,7 +895,7 @@ See :ref:`cli_create_index`. --unique Make this a unique index --if-not-exists, --ignore Ignore if index already exists --analyze Run ANALYZE after creating the index - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -920,7 +923,7 @@ See :ref:`cli_fts`. --create-triggers Create triggers to update the FTS tables when the parent table changes. --replace Replace existing FTS configuration if it exists - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -940,7 +943,7 @@ populate-fts sqlite-utils populate-fts chickens.db chickens name Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -960,7 +963,7 @@ rebuild-fts sqlite-utils rebuild-fts chickens.db chickens Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -980,7 +983,7 @@ disable-fts sqlite-utils disable-fts chickens.db chickens Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1004,7 +1007,7 @@ See :ref:`cli_optimize`. Options: --no-vacuum Don't run VACUUM - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1069,7 +1072,7 @@ See :ref:`cli_dump`. sqlite-utils dump chickens.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1097,7 +1100,7 @@ See :ref:`cli_add_column`. omitted will automatically use the primary key --not-null-default TEXT Add NOT NULL DEFAULT 'TEXT' constraint --ignore If column already exists, do nothing - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1123,7 +1126,7 @@ See :ref:`cli_add_foreign_key`. Options: --ignore If foreign key already exists, do nothing - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1147,7 +1150,7 @@ See :ref:`cli_add_foreign_keys`. authors country_id countries id Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1169,7 +1172,7 @@ See :ref:`cli_index_foreign_keys`. sqlite-utils index-foreign-keys chickens.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1191,7 +1194,7 @@ See :ref:`cli_wal`. sqlite-utils enable-wal chickens.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1211,7 +1214,7 @@ disable-wal sqlite-utils disable-wal chickens.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1233,7 +1236,7 @@ See :ref:`cli_enable_counts`. sqlite-utils enable-counts chickens.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1253,7 +1256,7 @@ reset-counts sqlite-utils reset-counts chickens.db Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1270,7 +1273,7 @@ duplicate Options: --ignore If table does not exist, do nothing - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1293,7 +1296,7 @@ See :ref:`cli_drop_table`. Options: --ignore If table does not exist, do nothing - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1318,7 +1321,7 @@ See :ref:`cli_create_view`. Options: --ignore If view already exists, do nothing --replace If view already exists, replace it - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1341,7 +1344,7 @@ See :ref:`cli_drop_view`. Options: --ignore If view does not exist, do nothing - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -1372,7 +1375,8 @@ See :ref:`cli_spatialite`. --dimensions TEXT Coordinate dimensions. Use XYZ for three- dimensional geometries. --not-null Add a NOT NULL constraint. - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional + :entrypoint -h, --help Show this message and exit. @@ -1394,7 +1398,7 @@ See :ref:`cli_spatialite_indexes`. paths. To load it from a specific path, use --load-extension. Options: - --load-extension TEXT SQLite extensions to load + --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 16a99cc..c09273d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -94,7 +94,7 @@ def load_extension_option(fn): return click.option( "--load-extension", multiple=True, - help="SQLite extensions to load", + help="Path to SQLite extension, with optional :entrypoint", )(fn) @@ -2997,7 +2997,11 @@ def _load_extensions(db, load_extension): for ext in load_extension: if ext == "spatialite" and not os.path.exists(ext): ext = find_spatialite() - db.conn.load_extension(ext) + if ":" in ext: + path, _, entrypoint = ext.partition(":") + db.conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) + else: + db.conn.load_extension(ext) def _register_functions(db, functions): diff --git a/tests/ext.c b/tests/ext.c new file mode 100644 index 0000000..f5b3276 --- /dev/null +++ b/tests/ext.c @@ -0,0 +1,48 @@ +/* +** This file implements a SQLite extension with multiple entrypoints. +** +** The default entrypoint, sqlite3_ext_init, has a single function "a". +** The 1st alternate entrypoint, sqlite3_ext_b_init, has a single function "b". +** The 2nd alternate entrypoint, sqlite3_ext_c_init, has a single function "c". +** +** Compiling instructions: +** https://www.sqlite.org/loadext.html#compiling_a_loadable_extension +** +*/ + +#include "sqlite3ext.h" + +SQLITE_EXTENSION_INIT1 + +// SQL function that returns back the value supplied during sqlite3_create_function() +static void func(sqlite3_context *context, int argc, sqlite3_value **argv) { + sqlite3_result_text(context, (char *) sqlite3_user_data(context), -1, SQLITE_STATIC); +} + + +// The default entrypoint, since it matches the "ext.dylib"/"ext.so" name +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "a", 0, 0, "a", func, 0, 0); +} + +// Alternate entrypoint #1 +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_b_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "b", 0, 0, "b", func, 0, 0); +} + +// Alternate entrypoint #2 +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_c_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "c", 0, 0, "c", func, 0, 0); +} diff --git a/tests/test_cli.py b/tests/test_cli.py index 9b43cd5..24f36ed 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ from sqlite_utils import cli, Database from sqlite_utils.db import Index, ForeignKey from click.testing import CliRunner +from pathlib import Path import subprocess import sys from unittest import mock @@ -21,6 +22,17 @@ def _supports_pragma_function_list(): return True +def _has_compiled_ext(): + for ext in ["dylib", "so", "dll"]: + path = Path(__file__).parent / f"ext.{ext}" + if path.is_file(): + return True + return False + + +COMPILED_EXTENSION_PATH = str(Path(__file__).parent / "ext") + + @pytest.mark.parametrize( "options", ( @@ -2284,3 +2296,32 @@ def test_duplicate_table(tmpdir): assert result.exit_code == 0 assert db["one"].columns_dict == db["two"].columns_dict assert list(db["one"].rows) == list(db["two"].rows) + + +@pytest.mark.skipif(not _has_compiled_ext(), reason="Requires compiled ext.c") +@pytest.mark.parametrize( + "entrypoint,should_pass,should_fail", + ( + (None, ("a",), ("b", "c")), + ("sqlite3_ext_b_init", ("b"), ("a", "c")), + ("sqlite3_ext_c_init", ("c"), ("a", "b")), + ), +) +def test_load_extension(entrypoint, should_pass, should_fail): + ext = COMPILED_EXTENSION_PATH + if entrypoint: + ext += ":" + entrypoint + for func in should_pass: + result = CliRunner().invoke( + cli.cli, + ["memory", "select {}()".format(func), "--load-extension", ext], + catch_exceptions=False, + ) + assert result.exit_code == 0 + for func in should_fail: + result = CliRunner().invoke( + cli.cli, + ["memory", "select {}()".format(func), "--load-extension", ext], + catch_exceptions=False, + ) + assert result.exit_code == 1 From c5f8a2eb1a81a18b52825cc649112f71fe419b12 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Sat, 27 Aug 2022 10:45:03 -0400 Subject: [PATCH 0655/1004] in extract code, check equality witH IS instead of = for nulls (#455) sqlite "IS" is equivalent to SQL "IS NOT DISTINCT FROM" close #423 --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 18a442a..25c0a70 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1791,7 +1791,7 @@ class Table(Queryable): magic_lookup_column=magic_lookup_column, lookup_table=table, where=" AND ".join( - "[{table}].[{column}] = [{lookup_table}].[{lookup_column}]".format( + "[{table}].[{column}] IS [{lookup_table}].[{lookup_column}]".format( table=self.name, lookup_table=table, column=column, From 36ffcafb1a0f94c134cdedeb626012bc8e2c1d8a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 27 Aug 2022 15:41:10 -0700 Subject: [PATCH 0656/1004] table.default_values property, closes #475 Refs #468 --- docs/python-api.rst | 10 ++++++++++ sqlite_utils/db.py | 29 +++++++++++++++++++++++++++++ tests/test_introspect.py | 18 ++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 07f2bf6..c68611c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1802,6 +1802,16 @@ The ``.columns_dict`` property returns a dictionary version of the columns with >>> db["PlantType"].columns_dict {'id': , 'value': } +.. _python_api_introspection_default_values: + +.default_values +--------------- + +The ``.default_values`` property returns a dictionary of default values for each column that has a default:: + + >>> db["table_with_defaults"].default_values + {'score': 5} + .. _python_api_introspection_pks: .pks diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 25c0a70..3f9d400 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -9,6 +9,7 @@ from .utils import ( progressbar, find_spatialite, ) +import binascii from collections import namedtuple from collections.abc import Mapping import contextlib @@ -1458,6 +1459,15 @@ class Table(Queryable): "``{trigger_name: sql}`` dictionary of triggers defined on this table." return {trigger.name: trigger.sql for trigger in self.triggers} + @property + def default_values(self) -> Dict[str, Any]: + "``{column_name: default_value}`` dictionary of default values for columns in this table." + return { + column.name: _decode_default_value(column.default_value) + for column in self.columns + if column.default_value is not None + } + @property def strict(self) -> bool: "Is this a STRICT table?" @@ -3527,3 +3537,22 @@ def fix_square_braces(records: Iterable[Dict[str, Any]]): } else: yield record + + +def _decode_default_value(value): + if value.startswith("'") and value.endswith("'"): + # It's a string + return value[1:-1] + if value.isdigit(): + # It's an integer + return int(value) + if value.startswith("X'") and value.endswith("'"): + # It's a binary string, stored as hex + to_decode = value[2:-1] + return binascii.unhexlify(to_decode) + # If it is a string containing a floating point number: + try: + return float(value) + except ValueError: + pass + return value diff --git a/tests/test_introspect.py b/tests/test_introspect.py index fe9542b..c3d1b80 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -299,3 +299,21 @@ def test_table_strict(fresh_db, create_table, expected_strict): fresh_db.execute(create_table) table = fresh_db["t"] assert table.strict == expected_strict + + +@pytest.mark.parametrize( + "value", + ( + 1, + 1.3, + "foo", + True, + b"binary", + ), +) +def test_table_default_values(fresh_db, value): + fresh_db["default_values"].insert( + {"nodefault": 1, "value": value}, defaults={"value": value} + ) + default_values = fresh_db["default_values"].default_values + assert default_values == {"value": value} From 104f37fa4d2e7e5999c1d829267b62c737f74d3e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 27 Aug 2022 16:17:55 -0700 Subject: [PATCH 0657/1004] db[table].create(..., transform=True) and create-table --transform Closes #467 --- docs/cli-reference.rst | 1 + docs/cli.rst | 2 ++ docs/python-api.rst | 19 +++++++++++ sqlite_utils/cli.py | 26 +++++++++++++-- sqlite_utils/db.py | 74 +++++++++++++++++++++++++++++++++++----- tests/test_create.py | 76 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 188 insertions(+), 10 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index d3951fd..d74855e 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -865,6 +865,7 @@ See :ref:`cli_create_table`. foreign key --ignore If table already exists, do nothing --replace If table already exists, replace it + --transform If table already exists, try to transform the schema --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. diff --git a/docs/cli.rst b/docs/cli.rst index 3c5c595..d071158 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1520,6 +1520,8 @@ You can specify foreign key relationships between the tables you are creating us If a table with the same name already exists, you will get an error. You can choose to silently ignore this error with ``--ignore``, or you can replace the existing table with a new, empty table using ``--replace``. +You can also pass ``--transform`` to transform the existing table to match the new schema. See :ref:`python_api_explicit_create` in the Python library documentation for details of how this option works. + .. _cli_duplicate_table: Duplicating tables diff --git a/docs/python-api.rst b/docs/python-api.rst index c68611c..b8794f2 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -554,6 +554,25 @@ To do nothing if the table already exists, add ``if_not_exists=True``: "name": str, }, pk="id", if_not_exists=True) +You can also pass ``transform=True`` to have any existing tables :ref:`transformed ` to match your new table specification. This is a **dangerous operation** as it may drop columns that are no longer listed in your call to ``.create()``, so be careful when running this. + +.. code-block:: python + + db["cats"].create({ + "id": int, + "name": str, + "weight": float, + }, pk="id", transform=True) + +The ``transform=True`` option will update the table schema if any of the following have changed: + +- The specified columns or their types +- The specified primary key +- The order of the columns, defined using ``column_order=`` +- The ``not_null=`` or ``defaults=`` arguments + +Changes to ``foreign_keys=`` are not currently detected and applied by ``transform=True``. + .. _python_api_compound_primary_keys: Compound primary keys diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c09273d..c51b101 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1453,9 +1453,24 @@ def create_database(path, enable_wal, init_spatialite, load_extension): is_flag=True, help="If table already exists, replace it", ) +@click.option( + "--transform", + is_flag=True, + help="If table already exists, try to transform the schema", +) @load_extension_option def create_table( - path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension + path, + table, + columns, + pk, + not_null, + default, + fk, + ignore, + replace, + transform, + load_extension, ): """ Add a table with the specified columns. Columns should be specified using @@ -1490,6 +1505,8 @@ def create_table( return elif replace: db[table].drop() + elif transform: + pass else: raise click.ClickException( 'Table "{}" already exists. Use --replace to delete and replace it.'.format( @@ -1497,7 +1514,12 @@ def create_table( ) ) db[table].create( - coltypes, pk=pk, not_null=not_null, defaults=dict(default), foreign_keys=fk + coltypes, + pk=pk, + not_null=not_null, + defaults=dict(default), + foreign_keys=fk, + transform=transform, ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3f9d400..b653cc4 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -34,7 +34,6 @@ from typing import ( Union, Optional, List, - Set, Tuple, ) import uuid @@ -876,6 +875,7 @@ class Database: hash_id_columns: Optional[Iterable[str]] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, if_not_exists: bool = False, + transform: bool = False, ) -> "Table": """ Create a table with the specified name and the specified ``{column_name: type}`` columns. @@ -893,7 +893,61 @@ class Database: :param hash_id_columns: List of columns to be used when calculating the hash ID for a row :param extracts: List or dictionary of columns to be extracted during inserts, see :ref:`python_api_extracts` :param if_not_exists: Use ``CREATE TABLE IF NOT EXISTS`` + :param transform: If table already exists transform it to fit the specified schema """ + # Transform table to match the new definition if table already exists: + if transform and self[name].exists(): + table = cast(Table, self[name]) + should_transform = False + # First add missing columns and figure out columns to drop + existing_columns = table.columns_dict + missing_columns = dict( + (col_name, col_type) + for col_name, col_type in columns.items() + if col_name not in existing_columns + ) + columns_to_drop = [ + column for column in existing_columns if column not in columns + ] + if missing_columns: + for col_name, col_type in missing_columns.items(): + table.add_column(col_name, col_type) + if missing_columns or columns_to_drop or columns != existing_columns: + should_transform = True + # Do we need to change the column order? + if ( + column_order + and list(existing_columns)[: len(column_order)] != column_order + ): + should_transform = True + # Has the primary key changed? + current_pks = table.pks + desired_pk = None + if isinstance(pk, str): + desired_pk = [pk] + elif pk: + desired_pk = list(pk) + if desired_pk and current_pks != desired_pk: + should_transform = True + # Any not-null changes? + current_not_null = {c.name for c in table.columns if c.notnull} + desired_not_null = set(not_null) if not_null else set() + if current_not_null != desired_not_null: + should_transform = True + # How about defaults? + if defaults and defaults != table.default_values: + should_transform = True + # Only run .transform() if there is something to do + if should_transform: + table.transform( + types=columns, + drop=columns_to_drop, + column_order=column_order, + not_null=not_null, + defaults=defaults, + pk=pk, + ) + return table sql = self.create_table_sql( name=name, columns=columns, @@ -908,7 +962,7 @@ class Database: if_not_exists=if_not_exists, ) self.execute(sql) - table = self.table( + created_table = self.table( name, pk=pk, foreign_keys=foreign_keys, @@ -918,7 +972,7 @@ class Database: hash_id=hash_id, hash_id_columns=hash_id_columns, ) - return cast(Table, table) + return cast(Table, created_table) def create_view( self, name: str, sql: str, ignore: bool = False, replace: bool = False @@ -1487,6 +1541,7 @@ class Table(Queryable): hash_id_columns: Optional[Iterable[str]] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, if_not_exists: bool = False, + transform: bool = False, ) -> "Table": """ Create a table with the specified columns. @@ -1518,6 +1573,7 @@ class Table(Queryable): hash_id_columns=hash_id_columns, extracts=extracts, if_not_exists=if_not_exists, + transform=transform, ) return self @@ -1544,7 +1600,7 @@ class Table(Queryable): rename: Optional[dict] = None, drop: Optional[Iterable] = None, pk: Optional[Any] = DEFAULT, - not_null: Optional[Set[str]] = None, + not_null: Optional[Iterable[str]] = None, defaults: Optional[Dict[str, Any]] = None, drop_foreign_keys: Optional[Iterable] = None, column_order: Optional[List[str]] = None, @@ -1664,10 +1720,12 @@ class Table(Queryable): create_table_not_null.add(key) elif isinstance(not_null, set): create_table_not_null.update((rename.get(k) or k) for k in not_null) - elif not_null is None: + elif not not_null: pass else: - assert False, "not_null must be a dict or a set or None" + assert False, "not_null must be a dict or a set or None, it was {}".format( + repr(not_null) + ) # defaults= create_table_defaults = { (rename.get(c.name) or c.name): c.default_value @@ -2861,7 +2919,7 @@ class Table(Queryable): pk=DEFAULT, foreign_keys=DEFAULT, column_order: Optional[Union[List[str], Default]] = DEFAULT, - not_null: Optional[Union[Set[str], Default]] = DEFAULT, + not_null: Optional[Union[Iterable[str], Default]] = DEFAULT, defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT, hash_id: Optional[Union[str, Default]] = DEFAULT, hash_id_columns: Optional[Union[Iterable[str], Default]] = DEFAULT, @@ -3141,7 +3199,7 @@ class Table(Queryable): pk: Optional[str] = "id", foreign_keys: Optional[ForeignKeysType] = None, column_order: Optional[List[str]] = None, - not_null: Optional[Set[str]] = None, + not_null: Optional[Iterable[str]] = None, defaults: Optional[Dict[str, Any]] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, conversions: Optional[Dict[str, str]] = None, diff --git a/tests/test_create.py b/tests/test_create.py index 60181de..7252e48 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1155,3 +1155,79 @@ def test_create_if_no_columns(fresh_db): with pytest.raises(AssertionError) as error: fresh_db["t"].create({}) assert error.value.args[0] == "Tables must have at least one column" + + +@pytest.mark.parametrize( + "cols,kwargs,expected_schema,should_transform", + ( + # Change nothing + ( + {"id": int, "name": str}, + {"pk": "id"}, + "CREATE TABLE [demo] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)", + False, + ), + # Drop name column, remove primary key + ({"id": int}, {}, 'CREATE TABLE "demo" (\n [id] INTEGER\n)', True), + # Add a new column + ( + {"id": int, "name": str, "age": int}, + {"pk": "id"}, + 'CREATE TABLE "demo" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] INTEGER\n)', + True, + ), + # Change a column type + ( + {"id": int, "name": bytes}, + {"pk": "id"}, + 'CREATE TABLE "demo" (\n [id] INTEGER PRIMARY KEY,\n [name] BLOB\n)', + True, + ), + # Change the primary key + ( + {"id": int, "name": str}, + {"pk": "name"}, + 'CREATE TABLE "demo" (\n [id] INTEGER,\n [name] TEXT PRIMARY KEY\n)', + True, + ), + # Change in column order + ( + {"id": int, "name": str}, + {"pk": "id", "column_order": ["name"]}, + 'CREATE TABLE "demo" (\n [name] TEXT,\n [id] INTEGER PRIMARY KEY\n)', + True, + ), + # Same column order is ignored + ( + {"id": int, "name": str}, + {"pk": "id", "column_order": ["id", "name"]}, + "CREATE TABLE [demo] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)", + False, + ), + # Change not null + ( + {"id": int, "name": str}, + {"pk": "id", "not_null": {"name"}}, + 'CREATE TABLE "demo" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT NOT NULL\n)', + True, + ), + # Change default values + ( + {"id": int, "name": str}, + {"pk": "id", "defaults": {"id": 0, "name": "Bob"}}, + "CREATE TABLE \"demo\" (\n [id] INTEGER PRIMARY KEY DEFAULT 0,\n [name] TEXT DEFAULT 'Bob'\n)", + True, + ), + ), +) +def test_create_transform(fresh_db, cols, kwargs, expected_schema, should_transform): + fresh_db.create_table("demo", {"id": int, "name": str}, pk="id") + fresh_db["demo"].insert({"id": 1, "name": "Cleo"}) + traces = [] + with fresh_db.tracer(lambda sql, parameters: traces.append((sql, parameters))): + fresh_db["demo"].create(cols, **kwargs, transform=True) + at_least_one_create_table = any(sql.startswith("CREATE TABLE") for sql, _ in traces) + assert should_transform == at_least_one_create_table + new_schema = fresh_db["demo"].schema + assert new_schema == expected_schema, repr(new_schema) + assert fresh_db["demo"].count == 1 From 365f62520fa080bc363ab3820b0c800c5096abff Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 27 Aug 2022 16:20:35 -0700 Subject: [PATCH 0658/1004] will, not may - refs #468 --- docs/python-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index b8794f2..206e5e6 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -554,7 +554,7 @@ To do nothing if the table already exists, add ``if_not_exists=True``: "name": str, }, pk="id", if_not_exists=True) -You can also pass ``transform=True`` to have any existing tables :ref:`transformed ` to match your new table specification. This is a **dangerous operation** as it may drop columns that are no longer listed in your call to ``.create()``, so be careful when running this. +You can also pass ``transform=True`` to have any existing tables :ref:`transformed ` to match your new table specification. This is a **dangerous operation** as it will drop columns that are no longer listed in your call to ``.create()``, so be careful when running this. .. code-block:: python From 165bc5fcb0600a1898249e48b03ce798010e07f9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 27 Aug 2022 20:32:01 -0700 Subject: [PATCH 0659/1004] test_extract_works_with_null_values, refs #423, #455 --- tests/test_extract.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_extract.py b/tests/test_extract.py index 10b5b09..70ad0cf 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -186,3 +186,24 @@ def test_extract_error_on_incompatible_existing_lookup_table(fresh_db): fresh_db["species2"].insert({"id": 1, "common_name": 3.5}) with pytest.raises(InvalidColumns): fresh_db["tree"].extract("common_name", table="species2") + + +def test_extract_works_with_null_values(fresh_db): + fresh_db["listens"].insert_all( + [ + {"id": 1, "track_title": "foo", "album_title": "bar"}, + {"id": 2, "track_title": "baz", "album_title": None}, + ], + pk="id", + ) + fresh_db["listens"].extract( + columns=["album_title"], table="albums", fk_column="album_id" + ) + assert list(fresh_db["listens"].rows) == [ + {"id": 1, "track_title": "foo", "album_id": 1}, + {"id": 2, "track_title": "baz", "album_id": 2}, + ] + assert list(fresh_db["albums"].rows) == [ + {"id": 1, "album_title": "bar"}, + {"id": 2, "album_title": None}, + ] From b491f22d817836829965516983a3f4c3c72c05fc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 27 Aug 2022 20:48:36 -0700 Subject: [PATCH 0660/1004] Release 3.29 Refs #423, #458, #467, #469, #470, #471, #472, #475 Closes #487 --- docs/changelog.rst | 17 +++++++++++++++++ setup.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b8f35c5..6ac1c18 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,23 @@ Changelog =========== +.. _v3_29: + +3.29 (2022-08-27) +----------------- + +- The ``sqlite-utils query``, ``memory`` and ``bulk`` commands now all accept a new ``--functions`` option. This can be passed a string of Python code, and any callable objects defined in that code will be made available to SQL queries as custom SQL functions. See :ref:`cli_query_functions` for details. (:issue:`471`) +- ``db[table].create(...)`` method now accepts a new ``transform=True`` parameter. If the table already exists it will be :ref:`transform ` to match the schema configuration options passed to the function. This may result in columns being added or dropped, column types being changed, column order being updated or not null and default values for columns being set. (:issue:`467`) +- Related to the above, the ``sqlite-utils create-table`` command now accepts a ``--transform`` option. +- New introspection property: ``table.default_values`` returns a dictionary mapping each column name with a default value to the configured default value. (:issue:`475`) +- The ``--load-extension`` option can now be provided a path to a compiled SQLite extension module accompanied by the name of an entrypoint, separated by a colon - for example ``--load-extension ./lines0:sqlite3_lines0_noread_init``. This feature is modelled on code first `contributed to Datasette `__ by Alex Garcia. (:issue:`470`) +- Functions registered using the :ref:`db.register_function() ` method can now have a custom name specified using the new ``db.register_function(fn, name=...)`` parameter. (:issue:`458`) +- :ref:`sqlite-utils rows ` has a new ``--order`` option for specifying the sort order for the returned rows. (:issue:`469`) +- All of the CLI options that accept Python code blocks can now all be used to define functions that can access modules imported in that same block of code without needing to use the ``global`` keyword. (:issue:`472`) +- Fixed bug where ``table.extract()`` would not behave correctly for columns containing null values. Thanks, Forest Gregg. (:issue:`423`) +- New tutorial: `Cleaning data with sqlite-utils and Datasette `__ shows how to use ``sqlite-utils`` to import and clean an example CSV file. +- Datasette and ``sqlite-utils`` now have a Discord community. `Join the Discord here `__. + .. _v3_28: 3.28 (2022-07-15) diff --git a/setup.py b/setup.py index ead463b..88b186b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.28" +VERSION = "3.29" def get_long_description(): From 087753cd42c406f1e060c1822dcd9b5fda3d60f4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 27 Aug 2022 21:01:55 -0700 Subject: [PATCH 0661/1004] sites.db is better name than dogs.db in this example --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index d071158..ed089ac 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -259,7 +259,7 @@ You can use the ``--functions`` option to pass a block of Python code that defin This example defines a function which extracts the domain from a URL:: - $ sqlite-utils query dogs.db "select url, domain(url) from urls" --functions ' + $ sqlite-utils query sites.db "select url, domain(url) from urls" --functions ' from urllib.parse import urlparse def domain(url): From ecf1d40112e52a8f4e509c39b98caae996b7bc36 Mon Sep 17 00:00:00 2001 From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com> Date: Tue, 30 Aug 2022 22:40:35 -0500 Subject: [PATCH 0662/1004] table.search_sql(include_rank=True) option (#480) * search_sql add include_rank option * add test * add FTS4 test * Apply Black Thanks, @chapmanjacobd --- sqlite_utils/db.py | 4 ++++ tests/test_fts.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b653cc4..27c46b0 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2376,6 +2376,7 @@ class Table(Queryable): limit: Optional[int] = None, offset: Optional[int] = None, where: Optional[str] = None, + include_rank: bool = False, ) -> str: """ " Return SQL string that can be used to execute searches against this table. @@ -2385,6 +2386,7 @@ class Table(Queryable): :param limit: SQL limit :param offset: SQL offset :param where: Extra SQL fragment for the WHERE clause + :param include_rank: Select the search rank column in the final query """ # Pick names for table and rank column that don't clash original = "original_" if self.name == "original" else "original" @@ -2427,6 +2429,8 @@ class Table(Queryable): rank_implementation = "rank_bm25(matchinfo([{}], 'pcnalx'))".format( fts_table ) + if include_rank: + columns_with_prefix_sql += ",\n " + rank_implementation + " rank" limit_offset = "" if limit is not None: limit_offset += " limit {}".format(limit) diff --git a/tests/test_fts.py b/tests/test_fts.py index 477d599..ecfcff9 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -556,6 +556,50 @@ def test_enable_fts_error_message_on_views(): " rank_bm25(matchinfo([books_fts], 'pcnalx'))" ), ), + ( + {"include_rank": True}, + "FTS5", + ( + "with original as (\n" + " select\n" + " rowid,\n" + " *\n" + " from [books]\n" + ")\n" + "select\n" + " [original].*,\n" + " [books_fts].rank rank\n" + "from\n" + " [original]\n" + " join [books_fts] on [original].rowid = [books_fts].rowid\n" + "where\n" + " [books_fts] match :query\n" + "order by\n" + " [books_fts].rank" + ), + ), + ( + {"include_rank": True}, + "FTS4", + ( + "with original as (\n" + " select\n" + " rowid,\n" + " *\n" + " from [books]\n" + ")\n" + "select\n" + " [original].*,\n" + " rank_bm25(matchinfo([books_fts], 'pcnalx')) rank\n" + "from\n" + " [original]\n" + " join [books_fts] on [original].rowid = [books_fts].rowid\n" + "where\n" + " [books_fts] match :query\n" + "order by\n" + " rank_bm25(matchinfo([books_fts], 'pcnalx'))" + ), + ), ], ) def test_search_sql(kwargs, fts, expected): From 686eed9a49faf87b0f2d3eba5fb12caa0250988f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Sep 2022 18:37:13 -0700 Subject: [PATCH 0663/1004] Typo in release notes --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6ac1c18..1245014 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,7 +8,7 @@ ----------------- - The ``sqlite-utils query``, ``memory`` and ``bulk`` commands now all accept a new ``--functions`` option. This can be passed a string of Python code, and any callable objects defined in that code will be made available to SQL queries as custom SQL functions. See :ref:`cli_query_functions` for details. (:issue:`471`) -- ``db[table].create(...)`` method now accepts a new ``transform=True`` parameter. If the table already exists it will be :ref:`transform ` to match the schema configuration options passed to the function. This may result in columns being added or dropped, column types being changed, column order being updated or not null and default values for columns being set. (:issue:`467`) +- ``db[table].create(...)`` method now accepts a new ``transform=True`` parameter. If the table already exists it will be :ref:`transformed ` to match the schema configuration options passed to the function. This may result in columns being added or dropped, column types being changed, column order being updated or not null and default values for columns being set. (:issue:`467`) - Related to the above, the ``sqlite-utils create-table`` command now accepts a ``--transform`` option. - New introspection property: ``table.default_values`` returns a dictionary mapping each column name with a default value to the configured default value. (:issue:`475`) - The ``--load-extension`` option can now be provided a path to a compiled SQLite extension module accompanied by the name of an entrypoint, separated by a colon - for example ``--load-extension ./lines0:sqlite3_lines0_noread_init``. This feature is modelled on code first `contributed to Datasette `__ by Alex Garcia. (:issue:`470`) From 5b969273f1244b1bcf3e4dc071cdf17dab35d5f8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Sep 2022 18:44:56 -0700 Subject: [PATCH 0664/1004] Markup tweak --- docs/cli.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index ed089ac..3adec26 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2097,14 +2097,14 @@ Use the ``--type`` option to specify a geometry type. By default, ``add-geometry Eight (case-insensitive) types are allowed: - * POINT - * LINESTRING - * POLYGON - * MULTIPOINT - * MULTILINESTRING - * MULTIPOLYGON - * GEOMETRYCOLLECTION - * GEOMETRY +* POINT +* LINESTRING +* POLYGON +* MULTIPOINT +* MULTILINESTRING +* MULTIPOLYGON +* GEOMETRYCOLLECTION +* GEOMETRY .. _cli_spatialite_indexes: From d9b9e075f07a20f1137cd2e34ed5d3f1a3db4ad8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 20:45:36 -0700 Subject: [PATCH 0665/1004] Documented the release process --- docs/changelog.rst | 2 ++ docs/contributing.rst | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1245014..adeb6b1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,3 +1,5 @@ +.. _changelog: + =========== Changelog =========== diff --git a/docs/contributing.rst b/docs/contributing.rst index 7ac1ad8..a1041f5 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -78,3 +78,40 @@ Both commands can then be run in the root of the project like this:: mypy sqlite_utils All three of these tools are run by our CI mechanism against every commit and pull request. + +.. _release_process: + +Release process +=============== + +Releases are performed using tags. When a new release is published on GitHub, a `GitHub Action workflow `__ will perform the following: + +* Run the unit tests against all supported Python versions. If the tests pass... +* Build a wheel bundle of the underlying Python source code +* Push that new wheel up to PyPI: https://pypi.org/project/sqlite-utils/ + +To deploy new releases you will need to have push access to the GitHub repository. + +``sqlite-utils`` follows `Semantic Versioning `__:: + + major.minor.patch + +We increment ``major`` for backwards-incompatible releases. + +We increment ``minor`` for new features. + +We increment ``patch`` for bugfix releass. + +To release a new version, first create a commit that updates the version number in ``setup.py`` and the :ref:`the changelog ` with highlights of the new version. An example `commit can be seen here `__:: + + # Update changelog + git commit -m " Release 3.29 + + Refs #423, #458, #467, #469, #470, #471, #472, #475" -a + git push + +Referencing the issues that are part of the release in the commit message ensures the name of the release shows up on those issue pages, e.g. `here `__. + +You can generate the list of issue references for a specific release by copying and pasting text from the release notes or GitHub changes-since-last-release view into this `Extract issue numbers from pasted text `__ tool. + +To create the tag for the release, create `a new release `__ on GitHub matching the new version number. You can convert the release notes to Markdown by copying and pasting the rendered HTML into this `Paste to Markdown tool `__. From 0b315d3fa83c1584eaeec32f24912898621e437a Mon Sep 17 00:00:00 2001 From: Mischa Untaga <99098079+MischaU8@users.noreply.github.com> Date: Thu, 15 Sep 2022 22:37:51 +0200 Subject: [PATCH 0666/1004] progressbar for inserts/upserts of other file formats * progressbar for inserts/upserts of other file formats, closes #485 * Pin to Python 3.10.6 for the moment as workaround for mypy error Co-authored-by: Simon Willison --- .github/workflows/publish.yml | 4 +-- .github/workflows/test.yml | 4 +-- sqlite_utils/cli.py | 54 ++++++++++++++++++----------------- sqlite_utils/utils.py | 5 ++++ 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 59c8a23..355f271 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,12 +9,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10.6"] os: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9303965..788df25 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,13 +10,13 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10.6"] numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v2 diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c51b101..767b170 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -953,14 +953,16 @@ def insert_upsert_implementation( decoded = io.TextIOWrapper(file, encoding=encoding) tracker = None - if csv or tsv: - if sniff: - # Read first 2048 bytes and use that to detect - first_bytes = sniff_buffer.peek(2048) - dialect = csv_std.Sniffer().sniff(first_bytes.decode(encoding, "ignore")) - else: - dialect = "excel-tab" if tsv else "excel" - with file_progress(decoded, silent=silent) as decoded: + with file_progress(decoded, silent=silent) as decoded: + if csv or tsv: + if sniff: + # Read first 2048 bytes and use that to detect + first_bytes = sniff_buffer.peek(2048) + dialect = csv_std.Sniffer().sniff( + first_bytes.decode(encoding, "ignore") + ) + else: + dialect = "excel-tab" if tsv else "excel" csv_reader_args = {"dialect": dialect} if delimiter: csv_reader_args["delimiter"] = delimiter @@ -977,24 +979,24 @@ def insert_upsert_implementation( if detect_types: tracker = TypeTracker() docs = tracker.wrap(docs) - elif lines: - docs = ({"line": line.strip()} for line in decoded) - elif text: - docs = ({"text": decoded.read()},) - else: - try: - if nl: - docs = (json.loads(line) for line in decoded if line.strip()) - else: - docs = json.load(decoded) - if isinstance(docs, dict): - docs = [docs] - except json.decoder.JSONDecodeError: - raise click.ClickException( - "Invalid JSON - use --csv for CSV or --tsv for TSV files" - ) - if flatten: - docs = (dict(_flatten(doc)) for doc in docs) + elif lines: + docs = ({"line": line.strip()} for line in decoded) + elif text: + docs = ({"text": decoded.read()},) + else: + try: + if nl: + docs = (json.loads(line) for line in decoded if line.strip()) + else: + docs = json.load(decoded) + if isinstance(docs, dict): + docs = [docs] + except json.decoder.JSONDecodeError: + raise click.ClickException( + "Invalid JSON - use --csv for CSV or --tsv for TSV files" + ) + if flatten: + docs = (dict(_flatten(doc)) for doc in docs) if convert: variable = "row" diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index c0b7bf1..8754554 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -155,6 +155,11 @@ class UpdateWrapper: self._update(len(line)) yield line + def read(self, size=-1): + data = self._wrapped.read(size) + self._update(len(data)) + return data + @contextlib.contextmanager def file_progress(file, silent=False, **kwargs): From 85247038f70d7eb2f3e272cfeaa4c44459cafba8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 11:57:11 -0700 Subject: [PATCH 0667/1004] install and uninstall commands, closes #483 --- docs/cli-reference.rst | 38 ++++++++++++++++++++++++++++++++++++++ docs/cli.rst | 26 ++++++++++++++++++++++++++ sqlite_utils/cli.py | 25 +++++++++++++++++++++++++ 3 files changed, 89 insertions(+) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index d74855e..b88e38a 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -59,6 +59,8 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. "convert": "cli_convert", "add-geometry-column": "cli_spatialite", "create-spatial-index": "cli_spatialite_indexes", + "install": "cli_install", + "uninstall": "cli_uninstall", } commands.sort(key = lambda command: go_first.index(command) if command in go_first else 999) cog.out("\n") @@ -1349,6 +1351,42 @@ See :ref:`cli_drop_view`. -h, --help Show this message and exit. +.. _cli_ref_install: + +install +======= + +See :ref:`cli_install`. + +:: + + Usage: sqlite-utils install [OPTIONS] PACKAGES... + + Install packages from PyPI into the same environment as sqlite-utils + + Options: + -U, --upgrade Upgrade packages to latest version + -h, --help Show this message and exit. + + +.. _cli_ref_uninstall: + +uninstall +========= + +See :ref:`cli_uninstall`. + +:: + + Usage: sqlite-utils uninstall [OPTIONS] PACKAGES... + + Uninstall Python packages from the sqlite-utils environment + + Options: + -y, --yes Don't ask for confirmation + -h, --help Show this message and exit. + + .. _cli_ref_add_geometry_column: add-geometry-column diff --git a/docs/cli.rst b/docs/cli.rst index 3adec26..5f85e55 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2116,3 +2116,29 @@ Once you have a geometry column, you can speed up bounding box queries by adding $ sqlite-utils create-spatial-index spatial.db locations geometry See this `SpatiaLite Cookbook recipe `__ for examples of how to use a spatial index. + +.. _cli_install: + +Installing packages +------------------- + +The :ref:`insert -\\-convert ` and :ref:`query -\\-functions ` options can be provided with a Python script that imports additional modules from the ``sqlite-utils`` environment. + +You can install packages from PyPI directly into the correct environment using ``sqlite-utils install ``. This is a wrapper around ``pip install``. + +:: + + $ sqlite-utils install beautifulsoup4 + +Use ``-U`` to upgrade an existing package. + +.. _cli_uninstall: + +Uninstalling packages +--------------------- + +You can uninstall packages that were installed using ``sqlite-utils install`` with ``sqlite-utils uninstall ``:: + + $ sqlite-utils uninstall beautifulsoup4 + +Use ``-y`` to skip the request for confirmation. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 767b170..bf7fae4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -4,6 +4,7 @@ from click_default_group import DefaultGroup # type: ignore from datetime import datetime import hashlib import pathlib +from runpy import run_module import sqlite_utils from sqlite_utils.db import AlterError, BadMultiValues, DescIndex, NoTable from sqlite_utils.utils import maximize_csv_field_size_limit @@ -2657,6 +2658,30 @@ def _analyze(db, tables, columns, save): click.echo(details) +@cli.command() +@click.argument("packages", nargs=-1, required=True) +@click.option( + "-U", "--upgrade", is_flag=True, help="Upgrade packages to latest version" +) +def install(packages, upgrade): + """Install packages from PyPI into the same environment as sqlite-utils""" + args = ["pip", "install"] + if upgrade: + args += ["--upgrade"] + args += list(packages) + sys.argv = args + run_module("pip", run_name="__main__") + + +@cli.command() +@click.argument("packages", nargs=-1, required=True) +@click.option("-y", "--yes", is_flag=True, help="Don't ask for confirmation") +def uninstall(packages, yes): + """Uninstall Python packages from the sqlite-utils environment""" + sys.argv = ["pip", "uninstall"] + list(packages) + (["-y"] if yes else []) + run_module("pip", run_name="__main__") + + def _generate_convert_help(): help = textwrap.dedent( """ From ee74bd5f8149b5e4403a4b56c74e9b94dbda2a32 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 12:03:54 -0700 Subject: [PATCH 0668/1004] Fix heading levels, refs #483 --- docs/cli.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 5f85e55..ecac91b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2120,7 +2120,7 @@ See this `SpatiaLite Cookbook recipe ` and :ref:`query -\\-functions ` options can be provided with a Python script that imports additional modules from the ``sqlite-utils`` environment. @@ -2135,7 +2135,7 @@ Use ``-U`` to upgrade an existing package. .. _cli_uninstall: Uninstalling packages ---------------------- +===================== You can uninstall packages that were installed using ``sqlite-utils install`` with ``sqlite-utils uninstall ``:: From 9a5add659d87738a658d8610ee461b038e28d268 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 12:09:00 -0700 Subject: [PATCH 0669/1004] 'just docs' command for running the livehtml docs server --- Justfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Justfile b/Justfile index 52e6190..4fe3fb9 100644 --- a/Justfile +++ b/Justfile @@ -16,6 +16,9 @@ @cog: cog -r README.md docs/*.rst +@docs: cog + cd docs && pipenv run make livehtml + # Apply Black @black: pipenv run black . From afbd2b2cba45cccb305c3d4638d18db4dd3d4bbd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 12:09:32 -0700 Subject: [PATCH 0670/1004] Link to convert command too, refs #483 --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index ecac91b..8bc4176 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2122,7 +2122,7 @@ See this `SpatiaLite Cookbook recipe ` and :ref:`query -\\-functions ` options can be provided with a Python script that imports additional modules from the ``sqlite-utils`` environment. +The :ref:`convert command ` and the :ref:`insert -\\-convert ` and :ref:`query -\\-functions ` options can be provided with a Python script that imports additional modules from the ``sqlite-utils`` environment. You can install packages from PyPI directly into the correct environment using ``sqlite-utils install ``. This is a wrapper around ``pip install``. From 6b268a1b3664784ed2267482d8c1d021a597d2b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 12:26:04 -0700 Subject: [PATCH 0671/1004] language = "en" to fix Sphinx warning --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 1339477..62d5ab1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -93,7 +93,7 @@ else: # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From c7cad6fc257c178b24b3f574b8c6992002c6f072 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 12:26:31 -0700 Subject: [PATCH 0672/1004] Documentation for Just, closes #494 Also added 'just init' and fixed a couple of missing pipenv run calls. --- Justfile | 8 ++++++-- docs/contributing.rst | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/Justfile b/Justfile index 4fe3fb9..a479717 100644 --- a/Justfile +++ b/Justfile @@ -1,6 +1,10 @@ # Run tests and linters @default: test lint +# Setup project +@init: + pipenv run pip install -e '.[test,docs,mypy,flake8]' + # Run pytest with supplied options @test *options: pipenv run pytest {{options}} @@ -10,11 +14,11 @@ pipenv run black . --check pipenv run flake8 pipenv run mypy sqlite_utils tests - cog --check README.md docs/*.rst + pipenv run cog --check README.md docs/*.rst # Rebuild docs with cog @cog: - cog -r README.md docs/*.rst + pipenv run cog -r README.md docs/*.rst @docs: cog cd docs && pipenv run make livehtml diff --git a/docs/contributing.rst b/docs/contributing.rst index a1041f5..ef5a875 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -79,6 +79,46 @@ Both commands can then be run in the root of the project like this:: All three of these tools are run by our CI mechanism against every commit and pull request. +Using Just and pipenv +===================== + +If you install `Just `__ and `pipenv `__ you can use them to manage your local development environment. + +To create a virtual environment and install all development dependencies, run:: + + cd sqlite-utils + just init + +To run all of the tests and linters:: + + just + +To run tests, or run a specific test module or test by name:: + + just test # All tests + just test tests/test_cli_memory.py # Just this module + just test -k test_memory_no_detect_types # Just this test + +To run just the linters:: + + just lint + +To apply Black to your code:: + + just black + +To update documentation using Cog:: + + just cog + +To run the live documentation server (after building with Cog first):: + + just docs + +And to list all available commands:: + + just -l + .. _release_process: Release process From cf9861216b8f2200535482f37d2f83f25a934493 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 12:36:09 -0700 Subject: [PATCH 0673/1004] Comment for 'just docs' command This shows up in 'just -l' --- Justfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Justfile b/Justfile index a479717..66863f6 100644 --- a/Justfile +++ b/Justfile @@ -20,6 +20,7 @@ @cog: pipenv run cog -r README.md docs/*.rst +# Serve live docs on localhost:8000 @docs: cog cd docs && pipenv run make livehtml From cbed0807822dd3ba0e51b99c6b28125422f690f0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 17:10:59 -0700 Subject: [PATCH 0674/1004] Typo --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index ef5a875..3a68b40 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -124,7 +124,7 @@ And to list all available commands:: Release process =============== -Releases are performed using tags. When a new release is published on GitHub, a `GitHub Action workflow `__ will perform the following: +Releases are performed using tags. When a new release is published on GitHub, a `GitHub Actions workflow `__ will perform the following: * Run the unit tests against all supported Python versions. If the tests pass... * Build a wheel bundle of the underlying Python source code From d792dad1cf5f16525da81b1e162fb71d469995f3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 19:23:17 -0700 Subject: [PATCH 0675/1004] Clarify wording --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 3a68b40..4a13725 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -111,7 +111,7 @@ To update documentation using Cog:: just cog -To run the live documentation server (after building with Cog first):: +To run the live documentation server (this will run Cog first):: just docs From 34e75ed0dd3091a6f94d6bd70150caa70660736d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 18 Oct 2022 11:00:25 -0700 Subject: [PATCH 0676/1004] sqlite_utils.utils.flatten() function, closes #500 --- docs/reference.rst | 7 +++++++ sqlite_utils/cli.py | 14 +++----------- sqlite_utils/utils.py | 18 ++++++++++++++++++ tests/test_cli.py | 12 ------------ tests/test_utils.py | 12 ++++++++++++ 5 files changed, 40 insertions(+), 23 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 66b9d38..5b5fd25 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -101,3 +101,10 @@ sqlite_utils.utils.chunks ------------------------- .. autofunction:: sqlite_utils.utils.chunks + +.. _reference_utils_flatten: + +sqlite_utils.utils.flatten +-------------------------- + +.. autofunction:: sqlite_utils.utils.flatten diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index bf7fae4..5fcc95a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -24,6 +24,7 @@ from .utils import ( chunks, file_progress, find_spatialite, + flatten as _flatten, sqlite3, decode_base64_values, progressbar, @@ -997,7 +998,7 @@ def insert_upsert_implementation( "Invalid JSON - use --csv for CSV or --tsv for TSV files" ) if flatten: - docs = (dict(_flatten(doc)) for doc in docs) + docs = (_flatten(doc) for doc in docs) if convert: variable = "row" @@ -1079,15 +1080,6 @@ def insert_upsert_implementation( db[table].transform(types=tracker.types) -def _flatten(d): - for key, value in d.items(): - if isinstance(value, dict): - for key2, value2 in _flatten(value): - yield key + "_" + key2, value2 - else: - yield key, value - - def _find_variables(tb, vars): to_find = list(vars) found = {} @@ -1845,7 +1837,7 @@ def memory( tracker = TypeTracker() rows = tracker.wrap(rows) if flatten: - rows = (dict(_flatten(row)) for row in rows) + rows = (_flatten(row) for row in rows) db[csv_table].insert_all(rows, alter=True) if tracker is not None: db[csv_table].transform(types=tracker.types) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 8754554..8916c46 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -513,3 +513,21 @@ def hash_record(record: Dict, keys: Optional[Iterable[str]] = None): "utf8" ) ).hexdigest() + + +def _flatten(d): + for key, value in d.items(): + if isinstance(value, dict): + for key2, value2 in _flatten(value): + yield key + "_" + key2, value2 + else: + yield key, value + + +def flatten(row: dict) -> dict: + """ + Turn a nested dict e.g. ``{"a": {"b": 1}}`` into a flat dict: ``{"a_b": 1}`` + + :param row: A Python dictionary, optionally with nested dictionaries + """ + return dict(_flatten(row)) diff --git a/tests/test_cli.py b/tests/test_cli.py index 24f36ed..b8df8d6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2176,18 +2176,6 @@ def test_upsert_detect_types(tmpdir, option): ] -@pytest.mark.parametrize( - "input,expected", - ( - ({"foo": {"bar": 1}}, {"foo_bar": 1}), - ({"foo": {"bar": [1, 2, {"baz": 3}]}}, {"foo_bar": [1, 2, {"baz": 3}]}), - ({"foo": {"bar": 1, "baz": {"three": 3}}}, {"foo_bar": 1, "foo_baz_three": 3}), - ), -) -def test_flatten_helper(input, expected): - assert dict(cli._flatten(input)) == expected - - def test_integer_overflow_error(tmpdir): db_path = str(tmpdir / "test.db") result = CliRunner().invoke( diff --git a/tests/test_utils.py b/tests/test_utils.py index d397176..f728bcd 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -71,3 +71,15 @@ def test_maximize_csv_field_size_limit(): assert len(rows_list2) == 1 assert rows_list2[0]["id"] == "1" assert rows_list2[0]["text"] == long_value + + +@pytest.mark.parametrize( + "input,expected", + ( + ({"foo": {"bar": 1}}, {"foo_bar": 1}), + ({"foo": {"bar": [1, 2, {"baz": 3}]}}, {"foo_bar": [1, 2, {"baz": 3}]}), + ({"foo": {"bar": 1, "baz": {"three": 3}}}, {"foo_bar": 1, "foo_baz_three": 3}), + ), +) +def test_flatten(input, expected): + assert utils.flatten(input) == expected From eb67fc69a227276b8ea635b885e5e4baecc43180 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 18 Oct 2022 11:08:34 -0700 Subject: [PATCH 0677/1004] Run cog -r against latest tabulate, refs #501 --- docs/cli-reference.rst | 107 ++++++++++++++++++++++++++--------------- docs/cli.rst | 12 +++++ 2 files changed, 79 insertions(+), 40 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index b88e38a..82b4b6c 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -112,11 +112,15 @@ See :ref:`cli_query`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of fancy_grid, fancy_outline, - github, grid, html, jira, latex, latex_booktabs, - latex_longtable, latex_raw, mediawiki, moinmoin, - orgtbl, pipe, plain, presto, pretty, psql, rst, - simple, textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, double_grid, + double_outline, fancy_grid, fancy_outline, github, + grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, + orgtbl, outline, pipe, plain, presto, pretty, + psql, rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, + unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings -r, --raw Raw output, first column of first row @@ -176,11 +180,15 @@ See :ref:`cli_memory`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of fancy_grid, fancy_outline, - github, grid, html, jira, latex, latex_booktabs, - latex_longtable, latex_raw, mediawiki, moinmoin, - orgtbl, pipe, plain, presto, pretty, psql, rst, - simple, textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, double_grid, + double_outline, fancy_grid, fancy_outline, github, + grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, + orgtbl, outline, pipe, plain, presto, pretty, + psql, rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, + unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings -r, --raw Raw output, first column of first row @@ -401,11 +409,14 @@ See :ref:`cli_search`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of fancy_grid, fancy_outline, - github, grid, html, jira, latex, latex_booktabs, - latex_longtable, latex_raw, mediawiki, moinmoin, - orgtbl, pipe, plain, presto, pretty, psql, rst, simple, - textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, double_grid, + double_outline, fancy_grid, fancy_outline, github, + grid, heavy_grid, heavy_outline, html, jira, latex, + latex_booktabs, latex_longtable, latex_raw, mediawiki, + mixed_grid, mixed_outline, moinmoin, orgtbl, outline, + pipe, plain, presto, pretty, psql, rounded_grid, + rounded_outline, rst, simple, simple_grid, + simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -651,11 +662,14 @@ See :ref:`cli_tables`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of fancy_grid, fancy_outline, - github, grid, html, jira, latex, latex_booktabs, - latex_longtable, latex_raw, mediawiki, moinmoin, - orgtbl, pipe, plain, presto, pretty, psql, rst, simple, - textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, double_grid, + double_outline, fancy_grid, fancy_outline, github, + grid, heavy_grid, heavy_outline, html, jira, latex, + latex_booktabs, latex_longtable, latex_raw, mediawiki, + mixed_grid, mixed_outline, moinmoin, orgtbl, outline, + pipe, plain, presto, pretty, psql, rounded_grid, + rounded_outline, rst, simple, simple_grid, + simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings --columns Include list of columns for each table @@ -689,11 +703,14 @@ See :ref:`cli_views`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of fancy_grid, fancy_outline, - github, grid, html, jira, latex, latex_booktabs, - latex_longtable, latex_raw, mediawiki, moinmoin, - orgtbl, pipe, plain, presto, pretty, psql, rst, simple, - textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, double_grid, + double_outline, fancy_grid, fancy_outline, github, + grid, heavy_grid, heavy_outline, html, jira, latex, + latex_booktabs, latex_longtable, latex_raw, mediawiki, + mixed_grid, mixed_outline, moinmoin, orgtbl, outline, + pipe, plain, presto, pretty, psql, rounded_grid, + rounded_outline, rst, simple, simple_grid, + simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings --columns Include list of columns for each view @@ -732,11 +749,15 @@ See :ref:`cli_rows`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of fancy_grid, fancy_outline, - github, grid, html, jira, latex, latex_booktabs, - latex_longtable, latex_raw, mediawiki, moinmoin, - orgtbl, pipe, plain, presto, pretty, psql, rst, - simple, textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, double_grid, + double_outline, fancy_grid, fancy_outline, github, + grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, + orgtbl, outline, pipe, plain, presto, pretty, + psql, rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, + unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings --load-extension TEXT Path to SQLite extension, with optional @@ -768,11 +789,14 @@ See :ref:`cli_triggers`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of fancy_grid, fancy_outline, - github, grid, html, jira, latex, latex_booktabs, - latex_longtable, latex_raw, mediawiki, moinmoin, - orgtbl, pipe, plain, presto, pretty, psql, rst, simple, - textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, double_grid, + double_outline, fancy_grid, fancy_outline, github, + grid, heavy_grid, heavy_outline, html, jira, latex, + latex_booktabs, latex_longtable, latex_raw, mediawiki, + mixed_grid, mixed_outline, moinmoin, orgtbl, outline, + pipe, plain, presto, pretty, psql, rounded_grid, + rounded_outline, rst, simple, simple_grid, + simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -804,11 +828,14 @@ See :ref:`cli_indexes`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of fancy_grid, fancy_outline, - github, grid, html, jira, latex, latex_booktabs, - latex_longtable, latex_raw, mediawiki, moinmoin, - orgtbl, pipe, plain, presto, pretty, psql, rst, simple, - textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, double_grid, + double_outline, fancy_grid, fancy_outline, github, + grid, heavy_grid, heavy_outline, html, jira, latex, + latex_booktabs, latex_longtable, latex_raw, mediawiki, + mixed_grid, mixed_outline, moinmoin, orgtbl, outline, + pipe, plain, presto, pretty, psql, rounded_grid, + rounded_outline, rst, simple, simple_grid, + simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings --load-extension TEXT Path to SQLite extension, with optional :entrypoint diff --git a/docs/cli.rst b/docs/cli.rst index 8bc4176..1d67e88 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -187,10 +187,15 @@ Available ``--fmt`` options are: cog.out("\n" + "\n".join('- ``{}``'.format(t) for t in tabulate.tabulate_formats) + "\n\n") .. ]]] +- ``asciidoc`` +- ``double_grid`` +- ``double_outline`` - ``fancy_grid`` - ``fancy_outline`` - ``github`` - ``grid`` +- ``heavy_grid`` +- ``heavy_outline`` - ``html`` - ``jira`` - ``latex`` @@ -198,15 +203,22 @@ Available ``--fmt`` options are: - ``latex_longtable`` - ``latex_raw`` - ``mediawiki`` +- ``mixed_grid`` +- ``mixed_outline`` - ``moinmoin`` - ``orgtbl`` +- ``outline`` - ``pipe`` - ``plain`` - ``presto`` - ``pretty`` - ``psql`` +- ``rounded_grid`` +- ``rounded_outline`` - ``rst`` - ``simple`` +- ``simple_grid`` +- ``simple_outline`` - ``textile`` - ``tsv`` - ``unsafehtml`` From 9cbe19ac0547031f3b626d9d18ef05c0d193bf79 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 18 Oct 2022 11:16:43 -0700 Subject: [PATCH 0678/1004] Skip cog check on Python 3.6, refs #501 --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 788df25..e994c25 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,5 +49,6 @@ jobs: - name: Check formatting run: black . --check - name: Check if cog needs to be run + if: matrix.python-version != '3.6' run: | cog --check README.md docs/*.rst From b8526c434a3d6aafb4102f9d9f5da14dfc4e3002 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 07:17:49 -0700 Subject: [PATCH 0679/1004] Test against Python 3.11 --- .github/workflows/publish.yml | 14 +++++++------- .github/workflows/test.yml | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 355f271..f3627d3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,15 +9,15 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10.6"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] os: [ubuntu-latest, windows-latest, macos-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 + - uses: actions/cache@v3 name: Configure pip caching with: path: ~/.cache/pip @@ -34,12 +34,12 @@ jobs: runs-on: ubuntu-latest needs: [test] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: '3.9' - - uses: actions/cache@v2 + python-version: '3.11' + - uses: actions/cache@v3 name: Configure pip caching with: path: ~/.cache/pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e994c25..98729c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,16 +10,16 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10.6"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v2 + - uses: actions/cache@v3 name: Configure pip caching with: path: ~/.cache/pip From 5133339d00252cb258a4217eda830ac60f43ee1f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 12:08:58 -0700 Subject: [PATCH 0680/1004] Skip macos-latest Python 3.11 for the moment Refs https://github.com/actions/setup-python/issues/531 --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 98729c9..ba88451 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,6 +13,10 @@ jobs: python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest] + exclude: + # https://github.com/actions/setup-python/issues/531 + - os: macos-latest + python-version: "3.11" steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 7b2d1c0ffd0b874e280292b926f328a61cb31e2c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 12:23:20 -0700 Subject: [PATCH 0681/1004] Update tests for Python 3.11, closes #502 --- tests/test_cli.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index b8df8d6..20fe99e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -737,10 +737,7 @@ def test_query_invalid_function(db_path): cli.cli, [db_path, "select bad()", "--functions", "def invalid_python"] ) assert result.exit_code == 1 - assert ( - result.output.strip() - == "Error: Error in functions definition: invalid syntax (, line 1)" - ) + assert result.output.startswith("Error: Error in functions definition:") TEST_FUNCTIONS = """ From 079bf1f4dc8540f834adae68c7feeeffcbc1d4f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 13:06:58 -0700 Subject: [PATCH 0682/1004] Use tmp_path fixture in test_recreate, refs #503 --- tests/test_recreate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_recreate.py b/tests/test_recreate.py index 504a0b8..91f1ba5 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -21,8 +21,8 @@ def test_recreate_not_allowed_for_connection(): @pytest.mark.parametrize( "use_path,file_exists", [(True, True), (True, False), (False, True), (False, False)] ) -def test_recreate(tmpdir, use_path, file_exists): - filepath = str(tmpdir / "data.db") +def test_recreate(tmp_path, use_path, file_exists): + filepath = str(tmp_path / "data.db") if use_path: filepath = pathlib.Path(filepath) if file_exists: From 7ca497a8f5be24c127946813e3052a19b48be1b3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 13:14:41 -0700 Subject: [PATCH 0683/1004] repr improvements, refs #503 --- sqlite_utils/db.py | 3 ++- sqlite_utils/utils.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 27c46b0..0d9eaed 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -308,6 +308,7 @@ class Database: assert (filename_or_conn is not None and (not memory and not memory_name)) or ( filename_or_conn is None and (memory or memory_name) ), "Either specify a filename_or_conn or pass memory=True" + self.conn = None if memory_name: uri = "file:{}?mode=memory&cache=shared".format(memory_name) self.conn = sqlite3.connect( @@ -3534,7 +3535,7 @@ class View(Queryable): def exists(self): return True - def __repr__(self): + def __repr__(self) -> str: return "".format( self.name, ", ".join(c.name for c in self.columns) ) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 8916c46..4e5bbcc 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -392,7 +392,7 @@ class ValueTracker: except (ValueError, TypeError): return False - def __repr__(self): + def __repr__(self) -> str: return self.guessed_type + ": possibilities = " + repr(self.couldbe) @property From c5d7ec1dd71fa1dce829bc8bb82b639018befd63 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 13:21:37 -0700 Subject: [PATCH 0684/1004] Fix for mypy issue, refs #503 --- sqlite_utils/db.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0d9eaed..80ad026 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -308,7 +308,6 @@ class Database: assert (filename_or_conn is not None and (not memory and not memory_name)) or ( filename_or_conn is None and (memory or memory_name) ), "Either specify a filename_or_conn or pass memory=True" - self.conn = None if memory_name: uri = "file:{}?mode=memory&cache=shared".format(memory_name) self.conn = sqlite3.connect( @@ -320,7 +319,13 @@ class Database: self.conn = sqlite3.connect(":memory:") elif isinstance(filename_or_conn, (str, pathlib.Path)): if recreate and os.path.exists(filename_or_conn): - os.remove(filename_or_conn) + try: + os.remove(filename_or_conn) + except OSError: + # Avoid mypy and __repr__ errors, see: + # https://github.com/simonw/sqlite-utils/issues/503 + self.conn = sqlite3.connect(":memory:") + raise self.conn = sqlite3.connect(str(filename_or_conn)) else: assert not recreate, "recreate cannot be used with connections, only paths" From f6b796277f783fcb613136e5a230b8657ef6c090 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 13:27:18 -0700 Subject: [PATCH 0685/1004] Try a 0.1s sleep, refs #503 --- tests/test_recreate.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_recreate.py b/tests/test_recreate.py index 91f1ba5..be161ff 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -1,5 +1,6 @@ from sqlite_utils import Database import sqlite3 +import time import pathlib import pytest @@ -19,14 +20,16 @@ def test_recreate_not_allowed_for_connection(): @pytest.mark.parametrize( - "use_path,file_exists", [(True, True), (True, False), (False, True), (False, False)] + "use_path,create_file_first", + [(True, True), (True, False), (False, True), (False, False)], ) -def test_recreate(tmp_path, use_path, file_exists): +def test_recreate(tmp_path, use_path, create_file_first): filepath = str(tmp_path / "data.db") if use_path: filepath = pathlib.Path(filepath) - if file_exists: + if create_file_first: Database(filepath)["t1"].insert({"foo": "bar"}) assert ["t1"] == Database(filepath).table_names() + time.sleep(0.1) Database(filepath, recreate=True)["t2"].insert({"foo": "bar"}) assert ["t2"] == Database(filepath).table_names() From ba7242b1f231a9707944a4c11a8dfede9ff9cad0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 13:36:17 -0700 Subject: [PATCH 0686/1004] Try closing the database before recreating it, refs #503 --- tests/test_recreate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_recreate.py b/tests/test_recreate.py index be161ff..0b98967 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -1,6 +1,5 @@ from sqlite_utils import Database import sqlite3 -import time import pathlib import pytest @@ -28,8 +27,9 @@ def test_recreate(tmp_path, use_path, create_file_first): if use_path: filepath = pathlib.Path(filepath) if create_file_first: - Database(filepath)["t1"].insert({"foo": "bar"}) - assert ["t1"] == Database(filepath).table_names() - time.sleep(0.1) + db = Database(filepath) + db["t1"].insert({"foo": "bar"}) + assert ["t1"] == db.table_names() + db.conn.close() Database(filepath, recreate=True)["t2"].insert({"foo": "bar"}) assert ["t2"] == Database(filepath).table_names() From 05e2bb85fcd11729db40c6452f2f7287232e2f1a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 13:57:43 -0700 Subject: [PATCH 0687/1004] db.close() method, closes #504 --- sqlite_utils/db.py | 4 ++++ tests/test_constructor.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 80ad026..b8e8d4b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -336,6 +336,10 @@ class Database: self._registered_functions: set = set() self.use_counts_table = use_counts_table + def close(self): + "Close the SQLite connection, and the underlying database file" + self.conn.close() + @contextlib.contextmanager def tracer(self, tracer: Callable = None): """ diff --git a/tests/test_constructor.py b/tests/test_constructor.py index d8c8d72..8743911 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -1,4 +1,6 @@ from sqlite_utils import Database +from sqlite_utils.utils import sqlite3 +import pytest def test_recursive_triggers(): @@ -25,3 +27,15 @@ def test_sqlite_version(): as_string = ".".join(map(str, version)) actual = next(db.query("select sqlite_version() as v"))["v"] assert actual == as_string + + +@pytest.mark.parametrize("memory", [True, False]) +def test_database_close(tmpdir, memory): + if memory: + db = Database(memory=True) + else: + db = Database(str(tmpdir / "test.db")) + assert db.execute("select 1 + 1").fetchone()[0] == 2 + db.close() + with pytest.raises(sqlite3.ProgrammingError): + db.execute("select 1 + 1") From c7e4308e6f49d929704163531632e558f9646e4a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 14:00:53 -0700 Subject: [PATCH 0688/1004] Use new db.close() method, refs #504 --- tests/test_recreate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_recreate.py b/tests/test_recreate.py index 0b98967..49dba2b 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -30,6 +30,6 @@ def test_recreate(tmp_path, use_path, create_file_first): db = Database(filepath) db["t1"].insert({"foo": "bar"}) assert ["t1"] == db.table_names() - db.conn.close() + db.close() Database(filepath, recreate=True)["t2"].insert({"foo": "bar"}) assert ["t2"] == Database(filepath).table_names() From defa2974c6d3abc19be28d6b319649b8028dc966 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 14:23:24 -0700 Subject: [PATCH 0689/1004] jsonify_if_needed output of convert() functions, closes #495 --- sqlite_utils/db.py | 2 +- tests/test_convert.py | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b8e8d4b..a06f4b7 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2662,7 +2662,7 @@ class Table(Queryable): bar.update(1) if not v: return v - return fn(v) + return jsonify_if_needed(fn(v)) self.db.register_function(convert_value) sql = "update [{table}] set {sets}{where};".format( diff --git a/tests/test_convert.py b/tests/test_convert.py index 796a08f..31a9d28 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -15,6 +15,14 @@ import pytest lambda value: value.upper(), {"title": "MIXED CASE", "abstract": "ABSTRACT"}, ), + ( + "title", + lambda value: {"upper": value.upper(), "lower": value.lower()}, + { + "title": '{"upper": "MIXED CASE", "lower": "mixed case"}', + "abstract": "Abstract", + }, + ), ), ) def test_convert(fresh_db, columns, fn, expected): @@ -81,10 +89,24 @@ def test_convert_multi(fresh_db): table = fresh_db["table"] table.insert({"title": "Mixed Case"}) table.convert( - "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True + "title", + lambda v: { + "upper": v.upper(), + "lower": v.lower(), + "both": { + "upper": v.upper(), + "lower": v.lower(), + }, + }, + multi=True, ) assert list(table.rows) == [ - {"title": "Mixed Case", "upper": "MIXED CASE", "lower": "mixed case"} + { + "title": "Mixed Case", + "upper": "MIXED CASE", + "lower": "mixed case", + "both": '{"upper": "MIXED CASE", "lower": "mixed case"}', + } ] From 0d45ee11027700f184383d5c8c25a26770fcf471 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 15:21:34 -0700 Subject: [PATCH 0690/1004] Release 3.30 Refs #480, #483, #485, #495, #500, #502, #504 --- docs/changelog.rst | 15 +++++++++++++++ docs/contributing.rst | 2 ++ setup.py | 2 +- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index adeb6b1..c3c12f2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,21 @@ Changelog =========== +.. _v3_30: + +3.30 (2022-10-25) +----------------- + +- Now tested against Python 3.11. (:issue:`502`) +- New ``table.search_sql(include_rank=True)`` option, which adds a ``rank`` column to the generated SQL. Thanks, Jacob Chapman. (`#480 `__) +- Progress bars now display for newline-delimited JSON files using the ``--nl`` option. Thanks, Mischa Untaga. (:issue:`485`) +- New ``db.close()`` method. (:issue:`504`) +- Conversion functions passed to :ref:`table.convert(...) ` can now return lists or dictionaries, which will be inserted into the database as JSON strings. (:issue:`495`) +- ``sqlite-utils install`` and ``sqlite-utils uninstall`` commands for installing packages into the same virtual environment as ``sqlite-utils``, :ref:`described here `. (:issue:`483`) +- New :ref:`sqlite_utils.utils.flatten() ` utility function. (:issue:`500`) +- Documentation on :ref:`using Just ` to run tests, linters and build documentation. +- Documentation now covers the :ref:`release_process` for this package. + .. _v3_29: 3.29 (2022-08-27) diff --git a/docs/contributing.rst b/docs/contributing.rst index 4a13725..2e883ee 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -79,6 +79,8 @@ Both commands can then be run in the root of the project like this:: All three of these tools are run by our CI mechanism against every commit and pull request. +.. _contributing_just: + Using Just and pipenv ===================== diff --git a/setup.py b/setup.py index 88b186b..2cf7af2 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.29" +VERSION = "3.30" def get_long_description(): From fb8f495582f68d8d49f57b42d12a66802f9ac238 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 25 Oct 2022 15:34:30 -0700 Subject: [PATCH 0691/1004] Skip macOS 3.11 test when publishing Refs #505 --- .github/workflows/publish.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f3627d3..424e7ff 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,6 +11,10 @@ jobs: matrix: python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] os: [ubuntu-latest, windows-latest, macos-latest] + exclude: + # https://github.com/actions/setup-python/issues/531 + - os: macos-latest + python-version: "3.11" steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 529110e7d8c4a6b1bbf5fb61f2e29d72aa95a611 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Oct 2022 12:27:32 -0700 Subject: [PATCH 0692/1004] GitHub Actions has Python 3.11 on macOS now Refs https://github.com/actions/setup-python/issues/531 --- .github/workflows/publish.yml | 4 ---- .github/workflows/test.yml | 4 ---- 2 files changed, 8 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 424e7ff..f3627d3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,10 +11,6 @@ jobs: matrix: python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] os: [ubuntu-latest, windows-latest, macos-latest] - exclude: - # https://github.com/actions/setup-python/issues/531 - - os: macos-latest - python-version: "3.11" steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba88451..98729c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,10 +13,6 @@ jobs: python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest] - exclude: - # https://github.com/actions/setup-python/issues/531 - - os: macos-latest - python-version: "3.11" steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 52ddb0b9ffa5284be668da088b7600b6ff64a2f9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Nov 2022 07:53:38 -0800 Subject: [PATCH 0693/1004] Rename utility functions to library --- docs/conf.py | 2 +- docs/index.rst | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 62d5ab1..8d49b81 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -187,7 +187,7 @@ texinfo_documents = [ "sqlite-utils documentation", author, "sqlite-utils", - "Python utility functions for manipulating SQLite databases", + "Python library for manipulating SQLite databases", "Miscellaneous", ) ] diff --git a/docs/index.rst b/docs/index.rst index 6793255..2bca2c8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,7 +15,7 @@ .. |discord| image:: https://img.shields.io/discord/823971286308356157?label=discord :target: https://discord.gg/Ass7bCAMDw -*CLI tool and Python utility functions for manipulating SQLite databases* +*CLI tool and Python library for manipulating SQLite databases* This library and command-line utility helps create SQLite databases from an existing collection of data. diff --git a/setup.py b/setup.py index 2cf7af2..faa822b 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ def get_long_description(): setup( name="sqlite-utils", - description="CLI tool and Python utility functions for manipulating SQLite databases", + description="CLI tool and Python library for manipulating SQLite databases", long_description=get_long_description(), long_description_content_type="text/markdown", author="Simon Willison", From 965ca0d5f5bffe06cc02cd7741344d1ddddf9d56 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Nov 2022 22:25:26 -0800 Subject: [PATCH 0694/1004] Apply no_implicit_optional codemod, closes #512 --- sqlite_utils/db.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a06f4b7..e819d17 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -297,12 +297,12 @@ class Database: def __init__( self, - filename_or_conn: Union[str, pathlib.Path, sqlite3.Connection] = None, + filename_or_conn: Optional[Union[str, pathlib.Path, sqlite3.Connection]] = None, memory: bool = False, - memory_name: str = None, + memory_name: Optional[str] = None, recreate: bool = False, recursive_triggers: bool = True, - tracer: Callable = None, + tracer: Optional[Callable] = None, use_counts_table: bool = False, ): assert (filename_or_conn is not None and (not memory and not memory_name)) or ( @@ -341,7 +341,7 @@ class Database: self.conn.close() @contextlib.contextmanager - def tracer(self, tracer: Callable = None): + def tracer(self, tracer: Optional[Callable] = None): """ Context manager to temporarily set a tracer function - all executed SQL queries will be passed to this. @@ -378,7 +378,7 @@ class Database: def register_function( self, - fn: Callable = None, + fn: Optional[Callable] = None, deterministic: bool = False, replace: bool = False, name: Optional[str] = None, @@ -879,7 +879,7 @@ class Database: pk: Optional[Any] = None, foreign_keys: Optional[ForeignKeysType] = None, column_order: Optional[List[str]] = None, - not_null: Iterable[str] = None, + not_null: Optional[Iterable[str]] = None, defaults: Optional[Dict[str, Any]] = None, hash_id: Optional[str] = None, hash_id_columns: Optional[Iterable[str]] = None, @@ -1129,7 +1129,7 @@ class Database: sql += " [{}]".format(name) self.execute(sql) - def init_spatialite(self, path: str = None) -> bool: + def init_spatialite(self, path: Optional[str] = None) -> bool: """ The ``init_spatialite`` method will load and initialize the SpatiaLite extension. The ``path`` argument should be an absolute path to the compiled extension, which @@ -1182,7 +1182,7 @@ class Queryable: def count_where( self, - where: str = None, + where: Optional[str] = None, where_args: Optional[Union[Iterable, dict]] = None, ) -> int: """ @@ -1213,12 +1213,12 @@ class Queryable: def rows_where( self, - where: str = None, + where: Optional[str] = None, where_args: Optional[Union[Iterable, dict]] = None, - order_by: str = None, + order_by: Optional[str] = None, select: str = "*", - limit: int = None, - offset: int = None, + limit: Optional[int] = None, + offset: Optional[int] = None, ) -> Generator[dict, None, None]: """ Iterate over every row in this table or view that matches the specified where clause. @@ -1251,11 +1251,11 @@ class Queryable: def pks_and_rows_where( self, - where: str = None, + where: Optional[str] = None, where_args: Optional[Union[Iterable, dict]] = None, - order_by: str = None, - limit: int = None, - offset: int = None, + order_by: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, ) -> Generator[Tuple[Any, Dict], None, None]: """ Like ``.rows_where()`` but returns ``(pk, row)`` pairs - ``pk`` can be a single value or tuple. @@ -1345,7 +1345,7 @@ class Table(Queryable): pk: Optional[Any] = None, foreign_keys: Optional[ForeignKeysType] = None, column_order: Optional[List[str]] = None, - not_null: Iterable[str] = None, + not_null: Optional[Iterable[str]] = None, defaults: Optional[Dict[str, Any]] = None, batch_size: int = 100, hash_id: Optional[str] = None, @@ -1545,7 +1545,7 @@ class Table(Queryable): pk: Optional[Any] = None, foreign_keys: Optional[ForeignKeysType] = None, column_order: Optional[List[str]] = None, - not_null: Iterable[str] = None, + not_null: Optional[Iterable[str]] = None, defaults: Optional[Dict[str, Any]] = None, hash_id: Optional[str] = None, hash_id_columns: Optional[Iterable[str]] = None, @@ -2464,7 +2464,7 @@ class Table(Queryable): columns: Optional[Iterable[str]] = None, limit: Optional[int] = None, offset: Optional[int] = None, - where: str = None, + where: Optional[str] = None, where_args: Optional[Union[Iterable, dict]] = None, quote: bool = False, ) -> Generator[dict, None, None]: @@ -2527,7 +2527,7 @@ class Table(Queryable): def delete_where( self, - where: str = None, + where: Optional[str] = None, where_args: Optional[Union[Iterable, dict]] = None, analyze: bool = False, ) -> "Table": From ebe504ab2103b0426b21162fc30691e42e8abaa0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Nov 2022 09:03:35 -0800 Subject: [PATCH 0695/1004] Clarify column types in create-table help --- docs/cli-reference.rst | 2 ++ sqlite_utils/cli.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 82b4b6c..153e5f9 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -886,6 +886,8 @@ See :ref:`cli_create_table`. height float \ photo blob --pk id + Valid column types are text, integer, float and blob. + Options: --pk TEXT Column to use as primary key --not-null TEXT Columns that should be created as NOT NULL diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 5fcc95a..d25b1df 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1477,6 +1477,8 @@ def create_table( name text \\ height float \\ photo blob --pk id + + Valid column types are text, integer, float and blob. """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) From e660635cea6c32f4022818380b1e1ee88e7c93a6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Dec 2022 17:25:23 -0800 Subject: [PATCH 0696/1004] Drop support for Python 3.6, refs #517 --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 3 +-- setup.py | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f3627d3..a37907a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] os: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 98729c9..2e706df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest] steps: @@ -49,6 +49,5 @@ jobs: - name: Check formatting run: black . --check - name: Check if cog needs to be run - if: matrix.python-version != '3.6' run: | cog --check README.md docs/*.rst diff --git a/setup.py b/setup.py index faa822b..4098a33 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ setup( "Issues": "https://github.com/simonw/sqlite-utils/issues", "CI": "https://github.com/simonw/sqlite-utils/actions", }, - python_requires=">=3.6", + python_requires=">=3.7", classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -68,11 +68,11 @@ setup( "Intended Audience :: End Users/Desktop", "Topic :: Database", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], # Needed to bundle py.typed so mypy can see it: zip_safe=False, From fc221f9b62ed8624b1d2098e564f525c84497969 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Dec 2022 17:30:45 -0800 Subject: [PATCH 0697/1004] Try this fix for flake8 error, refs #518 --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 7a88d6f..6b9c885 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,4 @@ [flake8] max-line-length = 160 -extend-ignore = E203 # for Black +# Black compatibility, E203 whitespace before ':': +extend-ignore = E203 \ No newline at end of file From 6cd0fd2b4c5116889e40245f84a9786fb19f4c40 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Mar 2023 14:25:26 -0700 Subject: [PATCH 0698/1004] Fix for Sphinx bug, closes #533, refs #531 --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 8d49b81..859c6b9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,7 +41,7 @@ autodoc_member_order = "bysource" autodoc_typehints = "description" extlinks = { - "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"), + "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#%s"), } From 92f77c32620d282f8e15de860bead40563b48dcb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Mar 2023 14:28:43 -0700 Subject: [PATCH 0699/1004] Ran against updated Black --- sqlite_utils/db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e819d17..c06e6a0 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2876,7 +2876,6 @@ class Table(Queryable): self.add_missing_columns(chunk) result = self.db.execute(query, params) elif e.args[0] == "too many SQL variables": - first_half = chunk[: len(chunk) // 2] second_half = chunk[len(chunk) // 2 :] From c0251cc9271260de73b4227859a51fab9b4cb745 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 26 Mar 2023 16:42:01 -0700 Subject: [PATCH 0700/1004] Link /latest/ to /stable/ - refs #388 --- docs/_templates/base.html | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/_templates/base.html b/docs/_templates/base.html index a9e670e..43c2003 100644 --- a/docs/_templates/base.html +++ b/docs/_templates/base.html @@ -3,4 +3,35 @@ {% block site_meta %} {{ super() }} -{% endblock %} \ No newline at end of file +{% endblock %} + +{% block scripts %} +{{ super() }} + +{% endblock %} From 8f9a729e8aff972cb18de25b40f4113e26bbc758 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Wed, 12 Apr 2023 21:44:43 -0400 Subject: [PATCH 0701/1004] Add paths for homebrew on Apple silicon (#536) --- sqlite_utils/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 4e5bbcc..3bd5db1 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -24,10 +24,11 @@ except ImportError: OperationalError = sqlite3.OperationalError - SPATIALITE_PATHS = ( "/usr/lib/x86_64-linux-gnu/mod_spatialite.so", "/usr/local/lib/mod_spatialite.dylib", + "/usr/local/lib/mod_spatialite.so", + "/opt/homebrew/lib/mod_spatialite.dylib", ) # Mainly so we can restore it if needed in the tests: From 373b7886d26902f54d72f1a414f988f79f0ffacd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 May 2023 11:26:03 -0700 Subject: [PATCH 0702/1004] --raw-lines option, closes #539 --- docs/cli-reference.rst | 2 ++ docs/cli.rst | 3 +++ sqlite_utils/cli.py | 53 +++++++++++++++++++++++++++++++++++++++--- tests/test_cli.py | 15 ++++++++++++ 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 153e5f9..c830518 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -124,6 +124,7 @@ See :ref:`cli_query`. --json-cols Detect JSON cols and output them as JSON, not escaped strings -r, --raw Raw output, first column of first row + --raw-lines Raw output, first column of each row -p, --param ... Named :parameters for SQL query --functions TEXT Python code defining one or more custom SQL functions @@ -192,6 +193,7 @@ See :ref:`cli_memory`. --json-cols Detect JSON cols and output them as JSON, not escaped strings -r, --raw Raw output, first column of first row + --raw-lines Raw output, first column of each row -p, --param ... Named :parameters for SQL query --encoding TEXT Character encoding for CSV input, defaults to utf-8 diff --git a/docs/cli.rst b/docs/cli.rst index 1d67e88..dc0c072 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -239,6 +239,9 @@ For example, to retrieve a binary image from a ``BLOB`` column and store it in a $ sqlite-utils photos.db "select contents from photos where id=1" --raw > myphoto.jpg +To return the first column of each result as raw data, separated by newlines, use ``--raw-lines``:: + + $ sqlite-utils photos.db "select caption from photos" --raw-lines > captions.txt .. _cli_query_parameters: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index d25b1df..da0e4b6 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1653,6 +1653,7 @@ def drop_view(path, view, ignore, load_extension): ) @output_options @click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") +@click.option("--raw-lines", is_flag=True, help="Raw output, first column of each row") @click.option( "-p", "--param", @@ -1677,6 +1678,7 @@ def query( fmt, json_cols, raw, + raw_lines, param, load_extension, functions, @@ -1700,7 +1702,19 @@ def query( _register_functions(db, functions) _execute_query( - db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + db, + sql, + param, + raw, + raw_lines, + table, + csv, + tsv, + no_headers, + fmt, + nl, + arrays, + json_cols, ) @@ -1728,6 +1742,7 @@ def query( ) @output_options @click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") +@click.option("--raw-lines", is_flag=True, help="Raw output, first column of each row") @click.option( "-p", "--param", @@ -1773,6 +1788,7 @@ def memory( fmt, json_cols, raw, + raw_lines, param, encoding, no_detect_types, @@ -1879,12 +1895,36 @@ def memory( _register_functions(db, functions) _execute_query( - db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + db, + sql, + param, + raw, + raw_lines, + table, + csv, + tsv, + no_headers, + fmt, + nl, + arrays, + json_cols, ) def _execute_query( - db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + db, + sql, + param, + raw, + raw_lines, + table, + csv, + tsv, + no_headers, + fmt, + nl, + arrays, + json_cols, ): with db.conn: try: @@ -1903,6 +1943,13 @@ def _execute_query( sys.stdout.buffer.write(data) else: sys.stdout.write(str(data)) + elif raw_lines: + for row in cursor: + data = row[0] + if isinstance(data, bytes): + sys.stdout.buffer.write(data + b"\n") + else: + sys.stdout.write(str(data) + "\n") elif fmt or table: print( tabulate.tabulate( diff --git a/tests/test_cli.py b/tests/test_cli.py index 20fe99e..c3ba682 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -916,6 +916,21 @@ def test_query_raw(db_path, content, is_binary): assert result.output == str(content) +@pytest.mark.parametrize( + "content,is_binary", + [(b"\x00\x0Fbinary", True), ("this is text", False), (1, False), (1.5, False)], +) +def test_query_raw_lines(db_path, content, is_binary): + Database(db_path)["files"].insert_all({"content": content} for _ in range(3)) + result = CliRunner().invoke( + cli.cli, [db_path, "select content from files", "--raw-lines"] + ) + if is_binary: + assert result.stdout_bytes == b"\n".join(content for _ in range(3)) + b"\n" + else: + assert result.output == "\n".join(str(content) for _ in range(3)) + "\n" + + def test_query_memory_does_not_create_file(tmpdir): owd = os.getcwd() try: From 963518bb16dc933694955309e7c9559e551b6a8e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 May 2023 11:38:54 -0700 Subject: [PATCH 0703/1004] Build with 3.11 on ReadTheDocs Refs #540 --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index ce66cbe..b092ec7 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,7 +4,7 @@ sphinx: configuration: docs/conf.py python: - version: "3.8" + version: "3.11" install: - method: pip path: . From 80763edaa2bdaf1113717378b8d62075c4dcbcfb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 May 2023 11:40:47 -0700 Subject: [PATCH 0704/1004] Different approach for Python 3.11 on ReadTheDocs Refs #540 --- .readthedocs.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index b092ec7..aa15c7c 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,8 +3,12 @@ version: 2 sphinx: configuration: docs/conf.py +build: + os: ubuntu-22.04 + tools: + python: "3.11" + python: - version: "3.11" install: - method: pip path: . From 2376c452a56b0c3e75e7ca698273434e32945304 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 12:24:10 -0700 Subject: [PATCH 0705/1004] upsert_all() now works with not_null - refs #538 --- sqlite_utils/db.py | 22 +++++++++++++++++----- tests/test_upsert.py | 10 ++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c06e6a0..44d59da 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2741,6 +2741,7 @@ class Table(Queryable): hash_id_columns, upsert, pk, + not_null, conversions, num_records_processed, replace, @@ -2778,14 +2779,20 @@ class Table(Queryable): pks = pk self.last_pk = None for record_values in values: - # TODO: make more efficient: record = dict(zip(all_columns, record_values)) - sql = "INSERT OR IGNORE INTO [{table}]({pks}) VALUES({pk_placeholders});".format( + placeholders = list(pks) + # Need to populate not-null columns too, or INSERT OR IGNORE ignores + # them since it ignores the resulting integrity errors + if not_null: + placeholders.extend(not_null) + sql = "INSERT OR IGNORE INTO [{table}]({cols}) VALUES({placeholders});".format( table=self.name, - pks=", ".join(["[{}]".format(p) for p in pks]), - pk_placeholders=", ".join(["?" for p in pks]), + cols=", ".join(["[{}]".format(p) for p in placeholders]), + placeholders=", ".join(["?" for p in placeholders]), + ) + queries_and_params.append( + (sql, [record[col] for col in pks] + ["" for _ in (not_null or [])]) ) - queries_and_params.append((sql, [record[col] for col in pks])) # UPDATE [book] SET [name] = 'Programming' WHERE [id] = 1001; set_cols = [col for col in all_columns if col not in pks] if set_cols: @@ -2846,6 +2853,7 @@ class Table(Queryable): hash_id_columns, upsert, pk, + not_null, conversions, num_records_processed, replace, @@ -2859,6 +2867,7 @@ class Table(Queryable): hash_id_columns, upsert, pk, + not_null, conversions, num_records_processed, replace, @@ -2888,6 +2897,7 @@ class Table(Queryable): hash_id_columns, upsert, pk, + not_null, conversions, num_records_processed, replace, @@ -2903,6 +2913,7 @@ class Table(Queryable): hash_id_columns, upsert, pk, + not_null, conversions, num_records_processed, replace, @@ -3112,6 +3123,7 @@ class Table(Queryable): hash_id_columns, upsert, pk, + not_null, conversions, num_records_processed, replace, diff --git a/tests/test_upsert.py b/tests/test_upsert.py index cef8af0..b7267ea 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -28,6 +28,16 @@ def test_upsert_all_single_column(fresh_db): assert table.pks == ["name"] +def test_upsert_all_not_null(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/538 + fresh_db["comments"].upsert_all( + [{"id": 1, "name": "Cleo"}], + pk="id", + not_null=["name"], + ) + assert list(fresh_db["comments"].rows) == [{"id": 1, "name": "Cleo"}] + + def test_upsert_error_if_no_pk(fresh_db): table = fresh_db["table"] with pytest.raises(PrimaryKeyRequired): From 4fc2f12c88054a4bcc29004e8e9cad39e5b66664 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 12:39:06 -0700 Subject: [PATCH 0706/1004] Fix ResourceWarning in sqlite-utils insert, refs #534 --- sqlite_utils/cli.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index da0e4b6..9f708f0 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -948,14 +948,15 @@ def insert_upsert_implementation( # The --sniff option needs us to buffer the file to peek ahead sniff_buffer = None + decoded_buffer = None if sniff: sniff_buffer = io.BufferedReader(file, buffer_size=4096) - decoded = io.TextIOWrapper(sniff_buffer, encoding=encoding) + decoded_buffer = io.TextIOWrapper(sniff_buffer, encoding=encoding) else: - decoded = io.TextIOWrapper(file, encoding=encoding) + decoded_buffer = io.TextIOWrapper(file, encoding=encoding) tracker = None - with file_progress(decoded, silent=silent) as decoded: + with file_progress(decoded_buffer, silent=silent) as decoded: if csv or tsv: if sniff: # Read first 2048 bytes and use that to detect @@ -1079,6 +1080,12 @@ def insert_upsert_implementation( if tracker is not None: db[table].transform(types=tracker.types) + # Clean up open file-like objects + if sniff_buffer: + sniff_buffer.close() + if decoded_buffer: + decoded_buffer.close() + def _find_variables(tb, vars): to_find = list(vars) From a256d7de9887d8476400bbe3753439f2e406134b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 12:57:43 -0700 Subject: [PATCH 0707/1004] Fix a bunch of warnings in the tests, refs #541 --- sqlite_utils/cli.py | 25 ++++++++++++++----------- tests/test_cli.py | 18 ++++++++++++------ tests/test_cli_insert.py | 27 ++++++++++++++++++--------- tests/test_cli_memory.py | 12 ++++++++---- 4 files changed, 52 insertions(+), 30 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 9f708f0..ba0a416 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1839,40 +1839,43 @@ def memory( stem_counts = {} for i, path in enumerate(paths): # Path may have a :format suffix + fp = None if ":" in path and path.rsplit(":", 1)[-1].upper() in Format.__members__: path, suffix = path.rsplit(":", 1) format = Format[suffix.upper()] else: format = None if path in ("-", "stdin"): - csv_fp = sys.stdin.buffer - csv_table = "stdin" + fp = sys.stdin.buffer + file_table = "stdin" else: - csv_path = pathlib.Path(path) - stem = csv_path.stem + file_path = pathlib.Path(path) + stem = file_path.stem if stem_counts.get(stem): - csv_table = "{}_{}".format(stem, stem_counts[stem]) + file_table = "{}_{}".format(stem, stem_counts[stem]) else: - csv_table = stem + file_table = stem stem_counts[stem] = stem_counts.get(stem, 1) + 1 - csv_fp = csv_path.open("rb") - rows, format_used = rows_from_file(csv_fp, format=format, encoding=encoding) + fp = file_path.open("rb") + rows, format_used = rows_from_file(fp, format=format, encoding=encoding) tracker = None if format_used in (Format.CSV, Format.TSV) and not no_detect_types: tracker = TypeTracker() rows = tracker.wrap(rows) if flatten: rows = (_flatten(row) for row in rows) - db[csv_table].insert_all(rows, alter=True) + db[file_table].insert_all(rows, alter=True) if tracker is not None: - db[csv_table].transform(types=tracker.types) + db[file_table].transform(types=tracker.types) # Add convenient t / t1 / t2 views view_names = ["t{}".format(i + 1)] if i == 0: view_names.append("t") for view_name in view_names: if not db[view_name].exists(): - db.create_view(view_name, "select * from [{}]".format(csv_table)) + db.create_view(view_name, "select * from [{}]".format(file_table)) + if fp: + fp.close() if analyze: _analyze(db, tables=None, columns=None, save=False) diff --git a/tests/test_cli.py b/tests/test_cli.py index c3ba682..5360e56 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -13,6 +13,11 @@ import textwrap from .utils import collapse_whitespace +def write_json(file_path, data): + with open(file_path, "w") as fp: + json.dump(data, fp) + + def _supports_pragma_function_list(): db = Database(memory=True) try: @@ -1016,7 +1021,7 @@ def test_upsert(db_path, tmpdir): {"id": 1, "name": "Cleo", "age": 4}, {"id": 2, "name": "Nixie", "age": 4}, ] - open(json_path, "w").write(json.dumps(insert_dogs)) + write_json(json_path, insert_dogs) result = CliRunner().invoke( cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"], @@ -1029,7 +1034,7 @@ def test_upsert(db_path, tmpdir): {"id": 1, "age": 5}, {"id": 2, "age": 5}, ] - open(json_path, "w").write(json.dumps(upsert_dogs)) + write_json(json_path, upsert_dogs) result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"], @@ -1048,7 +1053,7 @@ def test_upsert_pk_required(db_path, tmpdir): {"id": 1, "name": "Cleo", "age": 4}, {"id": 2, "name": "Nixie", "age": 4}, ] - open(json_path, "w").write(json.dumps(insert_dogs)) + write_json(json_path, insert_dogs) result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path], @@ -1091,14 +1096,14 @@ def test_upsert_alter(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") db = Database(db_path) insert_dogs = [{"id": 1, "name": "Cleo"}] - open(json_path, "w").write(json.dumps(insert_dogs)) + write_json(json_path, insert_dogs) result = CliRunner().invoke( cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] ) assert 0 == result.exit_code, result.output # Should fail with error code if no --alter upsert_dogs = [{"id": 1, "age": 5}] - open(json_path, "w").write(json.dumps(upsert_dogs)) + write_json(json_path, upsert_dogs) result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"] ) @@ -1767,7 +1772,8 @@ def test_insert_encoding(tmpdir): ) assert latin1_csv.decode("latin-1").split("\n")[2].split(",")[1] == "São Paulo" csv_path = str(tmpdir / "test.csv") - open(csv_path, "wb").write(latin1_csv) + with open(csv_path, "wb") as fp: + fp.write(latin1_csv) # First attempt should error: bad_result = CliRunner().invoke( cli.cli, diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 45ad362..3c3b88d 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -10,7 +10,8 @@ import time def test_insert_simple(tmpdir): json_path = str(tmpdir / "dog.json") db_path = str(tmpdir / "dogs.db") - open(json_path, "w").write(json.dumps({"name": "Cleo", "age": 4})) + with open(json_path, "w") as fp: + fp.write(json.dumps({"name": "Cleo", "age": 4})) result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path]) assert 0 == result.exit_code assert [{"age": 4, "name": "Cleo"}] == list( @@ -78,7 +79,8 @@ def test_insert_json_flatten_nl(tmpdir): def test_insert_with_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dog.json") - open(json_path, "w").write(json.dumps({"id": 1, "name": "Cleo", "age": 4})) + with open(json_path, "w") as fp: + fp.write(json.dumps({"id": 1, "name": "Cleo", "age": 4})) result = CliRunner().invoke( cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] ) @@ -93,7 +95,8 @@ def test_insert_with_primary_key(db_path, tmpdir): def test_insert_multiple_with_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") dogs = [{"id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21)] - open(json_path, "w").write(json.dumps(dogs)) + with open(json_path, "w") as fp: + fp.write(json.dumps(dogs)) result = CliRunner().invoke( cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] ) @@ -109,7 +112,8 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): {"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21) ] - open(json_path, "w").write(json.dumps(dogs)) + with open(json_path, "w") as fp: + fp.write(json.dumps(dogs)) result = CliRunner().invoke( cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--pk", "breed"] ) @@ -134,7 +138,8 @@ def test_insert_not_null_default(db_path, tmpdir): {"id": i, "name": "Cleo {}".format(i), "age": i + 3, "score": 10} for i in range(1, 21) ] - open(json_path, "w").write(json.dumps(dogs)) + with open(json_path, "w") as fp: + fp.write(json.dumps(dogs)) result = CliRunner().invoke( cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] @@ -182,7 +187,8 @@ def test_insert_ignore(db_path, tmpdir): db = Database(db_path) db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") json_path = str(tmpdir / "dogs.json") - open(json_path, "w").write(json.dumps([{"id": 1, "name": "Bailey"}])) + with open(json_path, "w") as fp: + fp.write(json.dumps([{"id": 1, "name": "Bailey"}])) # Should raise error without --ignore result = CliRunner().invoke( cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] @@ -212,7 +218,8 @@ def test_insert_ignore(db_path, tmpdir): def test_insert_csv_tsv(content, options, db_path, tmpdir): db = Database(db_path) file_path = str(tmpdir / "insert.csv-tsv") - open(file_path, "w").write(content) + with open(file_path, "w") as fp: + fp.write(content) result = CliRunner().invoke( cli.cli, ["insert", db_path, "data", file_path] + options, @@ -233,7 +240,8 @@ def test_insert_csv_tsv(content, options, db_path, tmpdir): ) def test_only_allow_one_of_nl_tsv_csv(options, db_path, tmpdir): file_path = str(tmpdir / "insert.csv-tsv") - open(file_path, "w").write("foo") + with open(file_path, "w") as fp: + fp.write("foo") result = CliRunner().invoke( cli.cli, ["insert", db_path, "data", file_path] + options ) @@ -251,7 +259,8 @@ def test_insert_replace(db_path, tmpdir): {"id": 2, "name": "Insert replaced 2", "age": 4}, {"id": 21, "name": "Fresh insert 21", "age": 6}, ] - open(json_path, "w").write(json.dumps(insert_replace_dogs)) + with open(json_path, "w") as fp: + fp.write(json.dumps(insert_replace_dogs)) result = CliRunner().invoke( cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--replace"] ) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index bb50d23..41bb00a 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -24,7 +24,8 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): sql_from = "stdin" else: csv_path = str(tmpdir / "test.csv") - open(csv_path, "w").write(content) + with open(csv_path, "w") as fp: + fp.write(content) result = CliRunner().invoke( cli.cli, ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], @@ -46,7 +47,8 @@ def test_memory_tsv(tmpdir, use_stdin): else: input = None path = str(tmpdir / "chickens.tsv") - open(path, "w").write(data) + with open(path, "w") as fp: + fp.write(data) path = path + ":tsv" sql_from = "chickens" result = CliRunner().invoke( @@ -71,7 +73,8 @@ def test_memory_json(tmpdir, use_stdin): else: input = None path = str(tmpdir / "chickens.json") - open(path, "w").write(data) + with open(path, "w") as fp: + fp.write(data) path = path + ":json" sql_from = "chickens" result = CliRunner().invoke( @@ -96,7 +99,8 @@ def test_memory_json_nl(tmpdir, use_stdin): else: input = None path = str(tmpdir / "chickens.json") - open(path, "w").write(data) + with open(path, "w") as fp: + fp.write(data) path = path + ":nl" sql_from = "chickens" result = CliRunner().invoke( From e4ed37251746b25ca69b5ace0c8c7992024556df Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 13:31:56 -0700 Subject: [PATCH 0708/1004] Show more detailed error on invalid JSON, closes #532 --- sqlite_utils/cli.py | 6 ++++-- tests/test_cli_insert.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index ba0a416..c3c55e2 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -994,9 +994,11 @@ def insert_upsert_implementation( docs = json.load(decoded) if isinstance(docs, dict): docs = [docs] - except json.decoder.JSONDecodeError: + except json.decoder.JSONDecodeError as ex: raise click.ClickException( - "Invalid JSON - use --csv for CSV or --tsv for TSV files" + "Invalid JSON - use --csv for CSV or --tsv for TSV files\n\nJSON error: {}".format( + ex + ) ) if flatten: docs = (_flatten(doc) for doc in docs) diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 3c3b88d..f403328 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -43,9 +43,9 @@ def test_insert_invalid_json_error(tmpdir): input="name,age\nCleo,4", ) assert result.exit_code == 1 - assert ( - result.output - == "Error: Invalid JSON - use --csv for CSV or --tsv for TSV files\n" + assert result.output == ( + "Error: Invalid JSON - use --csv for CSV or --tsv for TSV files\n\n" + "JSON error: Expecting value: line 1 column 1 (char 0)\n" ) From 455c35b512895c19bf922c2b804d750d27cb8dbd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 13:52:21 -0700 Subject: [PATCH 0709/1004] .convert(skip_false) option, refs #527 --- sqlite_utils/db.py | 3 ++- tests/test_convert.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 44d59da..3c5d91a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2618,6 +2618,7 @@ class Table(Queryable): where: Optional[str] = None, where_args: Optional[Union[Iterable, dict]] = None, show_progress: bool = False, + skip_false: bool = True, ): """ Apply conversion function ``fn`` to every value in the specified columns. @@ -2660,7 +2661,7 @@ class Table(Queryable): def convert_value(v): bar.update(1) - if not v: + if skip_false and not v: return v return jsonify_if_needed(fn(v)) diff --git a/tests/test_convert.py b/tests/test_convert.py index 31a9d28..9fd29c3 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -50,6 +50,16 @@ def test_convert_where(fresh_db, where, where_args): assert list(table.rows) == [{"id": 1, "title": "One"}, {"id": 2, "title": "TWO"}] +def test_convert_skip_false(fresh_db): + table = fresh_db["table"] + table.insert_all([{"x": 0}, {"x": 1}]) + assert table.get(1)["x"] == 0 + assert table.get(2)["x"] == 1 + table.convert("x", lambda x: x + 1, skip_false=False) + assert table.get(1)["x"] == 1 + assert table.get(2)["x"] == 2 + + @pytest.mark.parametrize( "drop,expected", ( From e0ec4c345129996011951e400388fd74865f65a2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 14:03:20 -0700 Subject: [PATCH 0710/1004] --no-skip-false option, plus docs - closes #527 --- docs/cli.rst | 2 ++ docs/python-api.rst | 2 ++ sqlite_utils/cli.py | 3 +++ tests/test_cli_convert.py | 27 +++++++++++++++++++++++++++ 4 files changed, 34 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index dc0c072..572b0e5 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1342,6 +1342,8 @@ You can include named parameters in your where clause and populate them using on The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. +By default any rows with a falsey value for the column - such as ``0`` or ``null`` - will be skipped. Use the ``--no-skip-false`` option to disable this behaviour. + .. _cli_convert_import: Importing additional modules diff --git a/docs/python-api.rst b/docs/python-api.rst index 206e5e6..4d883db 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -916,6 +916,8 @@ This will add the new column, if it does not already exist. You can pass ``outpu If you want to drop the original column after saving the results in a separate output column, pass ``drop=True``. +By default any rows with a falsey value for the column - such as ``0`` or ``None`` - will be skipped. Pass ``skip_false=False`` to disable this behaviour. + You can create multiple new columns from a single input column by passing ``multi=True`` and a conversion function that returns a Python dictionary. This example creates new ``upper`` and ``lower`` columns populated from the single ``title`` column: .. code-block:: python diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c3c55e2..ce354e0 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2811,6 +2811,7 @@ def _generate_convert_help(): type=click.Choice(["integer", "float", "blob", "text"]), ) @click.option("--drop", is_flag=True, help="Drop original column afterwards") +@click.option("--no-skip-false", is_flag=True, help="Don't skip falsey values") @click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") def convert( db_path, @@ -2825,6 +2826,7 @@ def convert( output, output_type, drop, + no_skip_false, silent, ): sqlite3.enable_callback_tracebacks(True) @@ -2882,6 +2884,7 @@ def convert( output=output, output_type=output_type, drop=drop, + skip_false=not no_skip_false, multi=multi, show_progress=not silent, ) diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index d26ab8c..97988b9 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -626,3 +626,30 @@ def test_convert_initialization_pattern(fresh_db_and_path): assert list(db["names"].rows) == [ {"id": 1, "name": "17"}, ] + + +@pytest.mark.parametrize( + "no_skip_false,expected", + ( + (True, 1), + (False, 0), + ), +) +def test_convert_no_skip_false(fresh_db_and_path, no_skip_false, expected): + db, db_path = fresh_db_and_path + args = [ + "convert", + db_path, + "t", + "x", + "-", + ] + if no_skip_false: + args.append("--no-skip-false") + db["t"].insert_all([{"x": 0}, {"x": 1}]) + assert db["t"].get(1)["x"] == 0 + assert db["t"].get(2)["x"] == 1 + result = CliRunner().invoke(cli.cli, args, input="value + 1") + assert 0 == result.exit_code, result.output + assert db["t"].get(1)["x"] == expected + assert db["t"].get(2)["x"] == 2 From 9662d4ce267accdc8f5301b20a4c7cd82b5ccf34 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 14:05:30 -0700 Subject: [PATCH 0711/1004] Updated cog, refs #527 --- docs/cli-reference.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c830518..8993213 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -633,6 +633,7 @@ See :ref:`cli_convert`. --output-type [integer|float|blob|text] Column type to use for the output column --drop Drop original column afterwards + --no-skip-false Don't skip falsey values -s, --silent Don't show a progress bar -h, --help Show this message and exit. From 39ef137e6760d385dc48d03eccf9b89943636fc7 Mon Sep 17 00:00:00 2001 From: Scott Perry Date: Mon, 8 May 2023 14:10:00 -0700 Subject: [PATCH 0712/1004] Support self-referencing FKs in `create` (#537) --- sqlite_utils/db.py | 2 ++ tests/test_create.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3c5d91a..19896f8 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -813,6 +813,8 @@ class Database: pk = hash_id # Soundness check foreign_keys point to existing tables for fk in foreign_keys: + if fk.other_table == name and columns.get(fk.other_column): + continue if not any( c for c in self[fk.other_table].columns if c.name == fk.other_column ): diff --git a/tests/test_create.py b/tests/test_create.py index 7252e48..54d125c 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -288,6 +288,25 @@ def test_create_table_works_for_m2m_with_only_foreign_keys( ) +def test_self_referential_foreign_key(fresh_db): + assert [] == fresh_db.table_names() + table = fresh_db.create_table( + "test_table", + columns={ + "id": int, + "ref": int, + }, + pk="id", + foreign_keys=(("ref", "test_table", "id"),), + ) + assert ( + "CREATE TABLE [test_table] (\n" + " [id] INTEGER PRIMARY KEY,\n" + " [ref] INTEGER REFERENCES [test_table]([id])\n" + ")" + ) == table.schema + + def test_create_error_if_invalid_foreign_keys(fresh_db): with pytest.raises(AlterError): fresh_db["one"].insert( @@ -297,6 +316,15 @@ def test_create_error_if_invalid_foreign_keys(fresh_db): ) +def test_create_error_if_invalid_self_referential_foreign_keys(fresh_db): + with pytest.raises(AlterError): + fresh_db["one"].insert( + {"id": 1, "ref_id": 3}, + pk="id", + foreign_keys=(("ref_id", "one", "bad_column"),), + ) + + @pytest.mark.parametrize( "col_name,col_type,not_null_default,expected_schema", ( From 923768db2ee15f521fe49ce75002cdd02c82e2bc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 14:11:48 -0700 Subject: [PATCH 0713/1004] Assert on exact error message, refs #537 --- tests/test_create.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_create.py b/tests/test_create.py index 54d125c..6931f0e 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -317,12 +317,13 @@ def test_create_error_if_invalid_foreign_keys(fresh_db): def test_create_error_if_invalid_self_referential_foreign_keys(fresh_db): - with pytest.raises(AlterError): + with pytest.raises(AlterError) as ex: fresh_db["one"].insert( {"id": 1, "ref_id": 3}, pk="id", foreign_keys=(("ref_id", "one", "bad_column"),), ) + assert ex.value.args == ("No such column: one.bad_column",) @pytest.mark.parametrize( From 6500fed8b2085869b9714ce3a08c30f61dc829ad Mon Sep 17 00:00:00 2001 From: rhoboro Date: Tue, 9 May 2023 06:13:36 +0900 Subject: [PATCH 0714/1004] Transform no longer breaks non-string default values Closes #509 --- sqlite_utils/db.py | 24 ++++++++++++++++++++++-- tests/test_default_value.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 tests/test_default_value.py diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 19896f8..dcd3b5f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -542,6 +542,24 @@ class Database: '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits ) + def quote_default_value(self, value: str) -> str: + if any( + [ + str(value).startswith("'") and str(value).endswith("'"), + str(value).startswith('"') and str(value).endswith('"'), + ] + ): + return value + + if str(value).upper() in ("CURRENT_TIME", "CURRENT_DATE", "CURRENT_TIMESTAMP"): + return value + + if str(value).endswith(")"): + # Expr + return "({})".format(value) + + return self.quote(value) + def table_names(self, fts4: bool = False, fts5: bool = False) -> List[str]: """ List of string table names in this database. @@ -839,7 +857,7 @@ class Database: column_extras.append("NOT NULL") if column_name in defaults and defaults[column_name] is not None: column_extras.append( - "DEFAULT {}".format(self.quote(defaults[column_name])) + "DEFAULT {}".format(self.quote_default_value(defaults[column_name])) ) if column_name in foreign_keys_by_column: column_extras.append( @@ -2020,7 +2038,9 @@ class Table(Queryable): col_type = str not_null_sql = None if not_null_default is not None: - not_null_sql = "NOT NULL DEFAULT {}".format(self.db.quote(not_null_default)) + not_null_sql = "NOT NULL DEFAULT {}".format( + self.db.quote_default_value(not_null_default) + ) sql = "ALTER TABLE [{table}] ADD COLUMN [{col_name}] {col_type}{not_null_default};".format( table=self.name, col_name=col_name, diff --git a/tests/test_default_value.py b/tests/test_default_value.py new file mode 100644 index 0000000..9ffdb14 --- /dev/null +++ b/tests/test_default_value.py @@ -0,0 +1,34 @@ +import pytest + + +EXAMPLES = [ + ("TEXT DEFAULT 'foo'", "'foo'", "'foo'"), + ("TEXT DEFAULT 'foo)'", "'foo)'", "'foo)'"), + ("INTEGER DEFAULT '1'", "'1'", "'1'"), + ("INTEGER DEFAULT 1", "1", "'1'"), + ("INTEGER DEFAULT (1)", "1", "'1'"), + # Expressions + ( + "TEXT DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW'))", + "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')", + "(STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW'))", + ), + # Special values + ("TEXT DEFAULT CURRENT_TIME", "CURRENT_TIME", "CURRENT_TIME"), + ("TEXT DEFAULT CURRENT_DATE", "CURRENT_DATE", "CURRENT_DATE"), + ("TEXT DEFAULT CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP"), + ("TEXT DEFAULT current_timestamp", "current_timestamp", "current_timestamp"), + ("TEXT DEFAULT (CURRENT_TIMESTAMP)", "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP"), + # Strings + ("TEXT DEFAULT 'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'"), + ('TEXT DEFAULT "CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"'), +] + + +@pytest.mark.parametrize("column_def,initial_value,expected_value", EXAMPLES) +def test_quote_default_value(fresh_db, column_def, initial_value, expected_value): + fresh_db.execute("create table foo (col {})".format(column_def)) + assert initial_value == fresh_db["foo"].columns[0].default_value + assert expected_value == fresh_db.quote_default_value( + fresh_db["foo"].columns[0].default_value + ) From 02f5c4d69d7b4baebde015c56e5bc62923f33314 Mon Sep 17 00:00:00 2001 From: Martin Carpenter Date: Mon, 8 May 2023 23:53:58 +0200 Subject: [PATCH 0715/1004] Support repeated calls to Table.convert() * Test repeated calls to Table.convert() * Register Table.convert() functions under their own `lambda_hash` name * Raise exception on registering identical function names Refs #525 --- sqlite_utils/db.py | 18 +++++++++++++----- tests/test_convert.py | 9 +++++++++ tests/test_register_function.py | 17 +++++++++++++---- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index dcd3b5f..5972032 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -219,6 +219,11 @@ class AlterError(Exception): pass +class FunctionAlreadyRegistered(Exception): + "A function with this name and arity was already registered" + pass + + class NoObviousTable(Exception): "Could not tell which table this operation refers to" pass @@ -409,7 +414,7 @@ class Database: fn_name = name or fn.__name__ arity = len(inspect.signature(fn).parameters) if not replace and (fn_name, arity) in self._registered_functions: - return fn + raise FunctionAlreadyRegistered(f'Already registered function with name "{fn_name}" and identical arity') kwargs = {} registered = False if deterministic: @@ -434,7 +439,7 @@ class Database: def register_fts4_bm25(self): "Register the ``rank_bm25(match_info)`` function used for calculating relevance with SQLite FTS4." - self.register_function(rank_bm25, deterministic=True) + self.register_function(rank_bm25, deterministic=True, replace=True) def attach(self, alias: str, filepath: Union[str, pathlib.Path]): """ @@ -2687,13 +2692,16 @@ class Table(Queryable): return v return jsonify_if_needed(fn(v)) - self.db.register_function(convert_value) + fn_name = fn.__name__ + if fn_name == '': + fn_name = f'lambda_{hash(fn)}' + self.db.register_function(convert_value, name=fn_name) sql = "update [{table}] set {sets}{where};".format( table=self.name, sets=", ".join( [ - "[{output_column}] = convert_value([{column}])".format( - output_column=output or column, column=column + "[{output_column}] = {fn_name}([{column}])".format( + output_column=output or column, column=column, fn_name=fn_name ) for column in columns ] diff --git a/tests/test_convert.py b/tests/test_convert.py index 9fd29c3..8fffb4a 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -147,3 +147,12 @@ def test_convert_multi_exception(fresh_db): table.insert({"title": "Mixed Case"}) with pytest.raises(BadMultiValues): table.convert("title", lambda v: v.upper(), multi=True) + + +def test_convert_repeated(fresh_db): + table = fresh_db["table"] + col = "num" + table.insert({col: 1}) + table.convert(col, lambda x: x*2) + table.convert(col, lambda _x: 0) + assert table.get(1) == {col: 0} diff --git a/tests/test_register_function.py b/tests/test_register_function.py index 5169a67..05721ea 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -3,7 +3,7 @@ import pytest import sys from unittest.mock import MagicMock, call from sqlite_utils.utils import sqlite3 - +from sqlite_utils.db import FunctionAlreadyRegistered def test_register_function(fresh_db): @fresh_db.register_function @@ -85,9 +85,10 @@ def test_register_function_replace(fresh_db): assert "one" == fresh_db.execute("select one()").fetchone()[0] # This will fail to replace the function: - @fresh_db.register_function() - def one(): # noqa - return "two" + with pytest.raises(FunctionAlreadyRegistered): + @fresh_db.register_function() + def one(): # noqa + return "two" assert "one" == fresh_db.execute("select one()").fetchone()[0] @@ -97,3 +98,11 @@ def test_register_function_replace(fresh_db): return "two" assert "two" == fresh_db.execute("select one()").fetchone()[0] + + +def test_register_function_duplicate(fresh_db): + def to_lower(s): + return s.lower() + fresh_db.register_function(to_lower) + with pytest.raises(FunctionAlreadyRegistered): + fresh_db.register_function(to_lower) From fca3ef8cf2a68b7a5fa1d740c4439adc7f83e431 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 14:54:24 -0700 Subject: [PATCH 0716/1004] Applied Black, refs #526, #525 --- sqlite_utils/db.py | 12 ++++++++---- tests/test_convert.py | 2 +- tests/test_register_function.py | 3 +++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 5972032..7cc1c99 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -414,7 +414,9 @@ class Database: fn_name = name or fn.__name__ arity = len(inspect.signature(fn).parameters) if not replace and (fn_name, arity) in self._registered_functions: - raise FunctionAlreadyRegistered(f'Already registered function with name "{fn_name}" and identical arity') + raise FunctionAlreadyRegistered( + f'Already registered function with name "{fn_name}" and identical arity' + ) kwargs = {} registered = False if deterministic: @@ -2693,15 +2695,17 @@ class Table(Queryable): return jsonify_if_needed(fn(v)) fn_name = fn.__name__ - if fn_name == '': - fn_name = f'lambda_{hash(fn)}' + if fn_name == "": + fn_name = f"lambda_{hash(fn)}" self.db.register_function(convert_value, name=fn_name) sql = "update [{table}] set {sets}{where};".format( table=self.name, sets=", ".join( [ "[{output_column}] = {fn_name}([{column}])".format( - output_column=output or column, column=column, fn_name=fn_name + output_column=output or column, + column=column, + fn_name=fn_name, ) for column in columns ] diff --git a/tests/test_convert.py b/tests/test_convert.py index 8fffb4a..5279bc5 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -153,6 +153,6 @@ def test_convert_repeated(fresh_db): table = fresh_db["table"] col = "num" table.insert({col: 1}) - table.convert(col, lambda x: x*2) + table.convert(col, lambda x: x * 2) table.convert(col, lambda _x: 0) assert table.get(1) == {col: 0} diff --git a/tests/test_register_function.py b/tests/test_register_function.py index 05721ea..5b92631 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock, call from sqlite_utils.utils import sqlite3 from sqlite_utils.db import FunctionAlreadyRegistered + def test_register_function(fresh_db): @fresh_db.register_function def reverse_string(s): @@ -86,6 +87,7 @@ def test_register_function_replace(fresh_db): # This will fail to replace the function: with pytest.raises(FunctionAlreadyRegistered): + @fresh_db.register_function() def one(): # noqa return "two" @@ -103,6 +105,7 @@ def test_register_function_replace(fresh_db): def test_register_function_duplicate(fresh_db): def to_lower(s): return s.lower() + fresh_db.register_function(to_lower) with pytest.raises(FunctionAlreadyRegistered): fresh_db.register_function(to_lower) From eebd1a26ae626cdaba6e568bf11f32c76b60ad09 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 14:58:28 -0700 Subject: [PATCH 0717/1004] Removed FunctionAlreadyRegistered error, refs #526, #525 --- sqlite_utils/db.py | 9 +-------- tests/test_register_function.py | 20 ++++---------------- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 7cc1c99..2b2091a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -219,11 +219,6 @@ class AlterError(Exception): pass -class FunctionAlreadyRegistered(Exception): - "A function with this name and arity was already registered" - pass - - class NoObviousTable(Exception): "Could not tell which table this operation refers to" pass @@ -414,9 +409,7 @@ class Database: fn_name = name or fn.__name__ arity = len(inspect.signature(fn).parameters) if not replace and (fn_name, arity) in self._registered_functions: - raise FunctionAlreadyRegistered( - f'Already registered function with name "{fn_name}" and identical arity' - ) + return fn kwargs = {} registered = False if deterministic: diff --git a/tests/test_register_function.py b/tests/test_register_function.py index 5b92631..e2591f4 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -3,7 +3,6 @@ import pytest import sys from unittest.mock import MagicMock, call from sqlite_utils.utils import sqlite3 -from sqlite_utils.db import FunctionAlreadyRegistered def test_register_function(fresh_db): @@ -85,12 +84,10 @@ def test_register_function_replace(fresh_db): assert "one" == fresh_db.execute("select one()").fetchone()[0] - # This will fail to replace the function: - with pytest.raises(FunctionAlreadyRegistered): - - @fresh_db.register_function() - def one(): # noqa - return "two" + # This will silently fail to replaec the function + @fresh_db.register_function() + def one(): # noqa + return "two" assert "one" == fresh_db.execute("select one()").fetchone()[0] @@ -100,12 +97,3 @@ def test_register_function_replace(fresh_db): return "two" assert "two" == fresh_db.execute("select one()").fetchone()[0] - - -def test_register_function_duplicate(fresh_db): - def to_lower(s): - return s.lower() - - fresh_db.register_function(to_lower) - with pytest.raises(FunctionAlreadyRegistered): - fresh_db.register_function(to_lower) From dab23884ae49f1497acd70d855105bf9701f4e36 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 15:08:02 -0700 Subject: [PATCH 0718/1004] Better error message if rows_from_file called with StringIO, closes #520 Refs #448 --- sqlite_utils/utils.py | 8 +++++++- tests/test_rows_from_file.py | 10 +++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 3bd5db1..06c1a4c 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -303,7 +303,13 @@ def rows_from_file( elif format is None: # Detect the format, then call this recursively buffered = io.BufferedReader(cast(io.RawIOBase, fp), buffer_size=4096) - first_bytes = buffered.peek(2048).strip() + try: + first_bytes = buffered.peek(2048).strip() + except AttributeError: + # Likely the user passed a TextIO when this needs a BytesIO + raise TypeError( + "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO" + ) if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"): # TODO: Detect newline-JSON return rows_from_file(buffered, format=Format.JSON) diff --git a/tests/test_rows_from_file.py b/tests/test_rows_from_file.py index fdc7a03..5316b86 100644 --- a/tests/test_rows_from_file.py +++ b/tests/test_rows_from_file.py @@ -1,5 +1,5 @@ from sqlite_utils.utils import rows_from_file, Format, RowError -from io import BytesIO +from io import BytesIO, StringIO import pytest @@ -44,3 +44,11 @@ def test_rows_from_file_extra_fields_strategies(ignore_extras, extras_key, expec # We did not expect an error raise assert list_rows == expected + + +def test_rows_from_file_error_on_string_io(): + with pytest.raises(TypeError) as ex: + rows_from_file(StringIO("id,name\r\n1,Cleo")) + assert ex.value.args == ( + "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO", + ) From c764a9ee8fdb2c55785cf1f538aa5a462cbb292b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 15:12:39 -0700 Subject: [PATCH 0719/1004] Avoid negative hashes in lambda names, refs #543 --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 2b2091a..ec4bbfc 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2689,7 +2689,7 @@ class Table(Queryable): fn_name = fn.__name__ if fn_name == "": - fn_name = f"lambda_{hash(fn)}" + fn_name = f"lambda_{abs(hash(fn))}" self.db.register_function(convert_value, name=fn_name) sql = "update [{table}] set {sets}{where};".format( table=self.name, From b3b100d7f5b2a76ccd4bfe8b0301a29e321d0375 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 15:33:57 -0700 Subject: [PATCH 0720/1004] Release 3.31 Refs #509, #517, #520, #525, #527, #532, #534, #536, #537, #538, #539 --- docs/changelog.rst | 17 +++++++++++++++++ setup.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c3c12f2..9c104f1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,23 @@ Changelog =========== +.. _v3_31: + +3.31 (2023-05-08) +----------------- + +- Dropped support for Python 3.6. Tests now ensure compatibility with Python 3.11. (:issue:`517`) +- Automatically locates the SpatiaLite extension on Apple Silicon. Thanks, Chris Amico. (`#536 `__) +- New ``--raw-lines`` option for the ``sqlite-utils query`` and ``sqlite-utils memory`` commands, which outputs just the raw value of the first column of every row. (:issue:`539`) +- Fixed a bug where ``table.upsert_all()`` failed if the ``not_null=`` option was passed. (:issue:`538`) +- Fixed a ``ResourceWarning`` when using ``sqlite-utils insert``. (:issue:`534`) +- Now shows a more detailed error message when ``sqlite-utils insert`` is called with invalid JSON. (:issue:`532`) +- ``table.convert(..., skip_false=False)`` and ``sqlite-utils convert --no-skip-false`` options, for avoiding a misfeature where the :ref:`convert() ` mechanism skips rows in the database with a falsey value for the specified column. Fixing this by default would be a backwards-compatible change and is under consideration for a 4.0 release in the future. (:issue:`527`) +- Tables can now be created with self-referential foreign keys. Thanks, Scott Perry. (`#537 `__) +- ``sqlite-utils transform`` no longer breaks if a table defines default values for columns. Thanks, Kenny Song. (:issue:`509`) +- Fixed a bug where repeated calls to ``table.transform()`` did not work correctly. Thanks, Martin Carpenter. (:issue:`525`) +- Improved error message if ``rows_from_file()`` is passed a non-binary-mode file-like object. (:issue:`520`) + .. _v3_30: 3.30 (2022-10-25) diff --git a/setup.py b/setup.py index 4098a33..3964b3a 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.30" +VERSION = "3.31" def get_long_description(): From e047cc32e9d5de7025d4d3c16554d4290f4bd3d1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 May 2023 14:08:31 -0700 Subject: [PATCH 0721/1004] backwards-incompatible, not compatible --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9c104f1..347601a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,7 +15,7 @@ - Fixed a bug where ``table.upsert_all()`` failed if the ``not_null=`` option was passed. (:issue:`538`) - Fixed a ``ResourceWarning`` when using ``sqlite-utils insert``. (:issue:`534`) - Now shows a more detailed error message when ``sqlite-utils insert`` is called with invalid JSON. (:issue:`532`) -- ``table.convert(..., skip_false=False)`` and ``sqlite-utils convert --no-skip-false`` options, for avoiding a misfeature where the :ref:`convert() ` mechanism skips rows in the database with a falsey value for the specified column. Fixing this by default would be a backwards-compatible change and is under consideration for a 4.0 release in the future. (:issue:`527`) +- ``table.convert(..., skip_false=False)`` and ``sqlite-utils convert --no-skip-false`` options, for avoiding a misfeature where the :ref:`convert() ` mechanism skips rows in the database with a falsey value for the specified column. Fixing this by default would be a backwards-incompatible change and is under consideration for a 4.0 release in the future. (:issue:`527`) - Tables can now be created with self-referential foreign keys. Thanks, Scott Perry. (`#537 `__) - ``sqlite-utils transform`` no longer breaks if a table defines default values for columns. Thanks, Kenny Song. (:issue:`509`) - Fixed a bug where repeated calls to ``table.transform()`` did not work correctly. Thanks, Martin Carpenter. (:issue:`525`) From d2a7b15b2b930fe384e1f1715fc4af23386f4935 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 May 2023 09:19:30 -0700 Subject: [PATCH 0722/1004] Analyze tables options: --common-limit, --no-most, --no-least Closes #544 --- docs/cli-reference.rst | 11 ++-- docs/cli.rst | 10 ++-- docs/python-api.rst | 25 +++++++-- sqlite_utils/cli.py | 19 +++++-- sqlite_utils/db.py | 54 ++++++++++++-------- tests/test_analyze_tables.py | 99 ++++++++++++++++++++++++++++++++++-- 6 files changed, 178 insertions(+), 40 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 8993213..c108138 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -564,10 +564,13 @@ See :ref:`cli_analyze_tables`. sqlite-utils analyze-tables data.db trees Options: - -c, --column TEXT Specific columns to analyze - --save Save results to _analyze_tables table - --load-extension TEXT Path to SQLite extension, with optional :entrypoint - -h, --help Show this message and exit. + -c, --column TEXT Specific columns to analyze + --save Save results to _analyze_tables table + --common-limit INTEGER How many common values + --no-most Skip most common values + --no-least Skip least common values + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. .. _cli_ref_convert: diff --git a/docs/cli.rst b/docs/cli.rst index 572b0e5..d567cba 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -730,11 +730,15 @@ For each column this tool displays the number of null rows, the number of blank If you do not specify any tables every table in the database will be analyzed:: - $ sqlite-utils analyze-tables github.db + sqlite-utils analyze-tables github.db If you wish to analyze one or more specific columns, use the ``-c`` option:: - $ sqlite-utils analyze-tables github.db tags -c sha + sqlite-utils analyze-tables github.db tags -c sha + +To show more than 10 common values, use ``--common-limit 20``. To skip the most common or least common value analysis, use ``--no-most`` or ``--no-least``:: + + sqlite-utils analyze-tables github.db tags --common-limit 20 --no-least .. _cli_analyze_tables_save: @@ -743,7 +747,7 @@ Saving the analyzed table details ``analyze-tables`` can take quite a while to run for large database files. You can save the results of the analysis to a database table called ``_analyze_tables_`` using the ``--save`` option:: - $ sqlite-utils analyze-tables github.db --save + sqlite-utils analyze-tables github.db --save The ``_analyze_tables_`` table has the following schema:: diff --git a/docs/python-api.rst b/docs/python-api.rst index 4d883db..4b143c3 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1115,7 +1115,26 @@ You can inspect the database to see the results like this:: Analyzing a column ================== -The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` method is used by the :ref:`analyze-tables ` CLI command. It returns a ``ColumnDetails`` named tuple with the following fields: +The ``table.analyze_column(column)`` method is used by the :ref:`analyze-tables ` CLI command. + +It takes the following arguments and options: + +``column`` - required + The name of the column to analyze + +``common_limit`` + The number of most common values to return. Defaults to 10. + +``value_truncate`` + If set to an integer, values longer than this will be truncated to this length. Defaults to None. + +``most_common`` + If set to False, the ``most_common`` field of the returned ``ColumnDetails`` will be set to None. Defaults to True. + +``least_common`` + If set to False, the ``least_common`` field of the returned ``ColumnDetails`` will be set to None. Defaults to True. + +And returns a ``ColumnDetails`` named tuple with the following fields: ``table`` The name of the table @@ -1141,10 +1160,6 @@ The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` metho ``least_common`` The ``N`` least common values as a list of ``(value, count)`` tuples`, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) -``N`` defaults to 10, or you can pass a custom ``N`` using the ``common_limit`` parameter. - -You can use the ``value_truncate`` parameter to truncate values in the ``most_common`` and ``least_common`` lists to a specified number of characters. - .. _python_api_add_column: Adding columns diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index ce354e0..0c91a8f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2639,12 +2639,20 @@ def insert_files( help="Specific columns to analyze", ) @click.option("--save", is_flag=True, help="Save results to _analyze_tables table") +@click.option("--common-limit", type=int, default=10, help="How many common values") +@click.option("--no-most", is_flag=True, default=False, help="Skip most common values") +@click.option( + "--no-least", is_flag=True, default=False, help="Skip least common values" +) @load_extension_option def analyze_tables( path, tables, columns, save, + common_limit, + no_most, + no_least, load_extension, ): """Analyze the columns in one or more tables @@ -2656,10 +2664,10 @@ def analyze_tables( """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - _analyze(db, tables, columns, save) + _analyze(db, tables, columns, save, common_limit, no_most, no_least) -def _analyze(db, tables, columns, save): +def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least=False): if not tables: tables = db.table_names() todo = [] @@ -2672,7 +2680,12 @@ def _analyze(db, tables, columns, save): # Now we now how many we need to do for i, (table, column) in enumerate(todo): column_details = db[table].analyze_column( - column, total_rows=table_counts[table], value_truncate=80 + column, + common_limit=common_limit, + total_rows=table_counts[table], + value_truncate=80, + most_common=not no_most, + least_common=not no_least, ) if save: db["_analyze_tables_"].insert( diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ec4bbfc..850a3ae 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3419,7 +3419,13 @@ class Table(Queryable): self.db.analyze(self.name) def analyze_column( - self, column: str, common_limit: int = 10, value_truncate=None, total_rows=None + self, + column: str, + common_limit: int = 10, + value_truncate=None, + total_rows=None, + most_common: bool = True, + least_common: bool = True, ) -> "ColumnDetails": """ Return statistics about the specified column. @@ -3430,6 +3436,8 @@ class Table(Queryable): :param common_limit: Show this many column values :param value_truncate: Truncate display of common values to this many characters :param total_rows: Optimization - pass the total number of rows in the table to save running a fresh ``count(*)`` query + :param most_common: If ``True``, calculate the most common values + :param least_common: If ``True``, calculate the least common values """ db = self.db table = self.name @@ -3453,36 +3461,38 @@ class Table(Queryable): num_distinct = db.execute( "select count(distinct [{}]) from [{}]".format(column, table) ).fetchone()[0] - most_common = None - least_common = None + most_common_results = None + least_common_results = None if num_distinct == 1: value = db.execute( "select [{}] from [{}] limit 1".format(column, table) ).fetchone()[0] - most_common = [(truncate(value), total_rows)] + most_common_results = [(truncate(value), total_rows)] elif num_distinct != total_rows: - most_common = [ - (truncate(r[0]), r[1]) - for r in db.execute( - "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format( - column, table, column, column, common_limit - ) - ).fetchall() - ] - most_common.sort(key=lambda p: (p[1], p[0]), reverse=True) - if num_distinct <= common_limit: - # No need to run the query if it will just return the results in revers order - least_common = None - else: - least_common = [ + if most_common: + most_common_results = [ (truncate(r[0]), r[1]) for r in db.execute( - "select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format( + "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format( column, table, column, column, common_limit ) ).fetchall() ] - least_common.sort(key=lambda p: (p[1], p[0])) + most_common_results.sort(key=lambda p: (p[1], p[0]), reverse=True) + if least_common: + if num_distinct <= common_limit: + # No need to run the query if it will just return the results in revers order + least_common_results = None + else: + least_common_results = [ + (truncate(r[0]), r[1]) + for r in db.execute( + "select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format( + column, table, column, column, common_limit + ) + ).fetchall() + ] + least_common_results.sort(key=lambda p: (p[1], p[0])) return ColumnDetails( self.name, column, @@ -3490,8 +3500,8 @@ class Table(Queryable): num_null, num_blank, num_distinct, - most_common, - least_common, + most_common_results, + least_common_results, ) def add_geometry_column( diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index 5795a7a..a3e4e2f 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -24,11 +24,34 @@ def db_to_analyze(fresh_db): return fresh_db +@pytest.fixture +def big_db_to_analyze_path(tmpdir): + path = str(tmpdir / "test.db") + db = Database(path) + categories = { + "A": 40, + "B": 30, + "C": 20, + "D": 10, + } + to_insert = [] + for category, count in categories.items(): + for _ in range(count): + to_insert.append( + { + "category": category, + } + ) + db["stuff"].insert_all(to_insert) + return path + + @pytest.mark.parametrize( - "column,expected", + "column,extra_kwargs,expected", [ ( "id", + {}, ColumnDetails( table="stuff", column="id", @@ -42,6 +65,7 @@ def db_to_analyze(fresh_db): ), ( "owner", + {}, ColumnDetails( table="stuff", column="owner", @@ -55,6 +79,7 @@ def db_to_analyze(fresh_db): ), ( "size", + {}, ColumnDetails( table="stuff", column="size", @@ -66,11 +91,41 @@ def db_to_analyze(fresh_db): least_common=None, ), ), + ( + "owner", + {"most_common": False}, + ColumnDetails( + table="stuff", + column="owner", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=4, + most_common=None, + least_common=[("Anne", 1), ("Terry...", 2)], + ), + ), + ( + "owner", + {"least_common": False}, + ColumnDetails( + table="stuff", + column="owner", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=4, + most_common=[("Joan", 3), ("Kumar", 2)], + least_common=None, + ), + ), ], ) -def test_analyze_column(db_to_analyze, column, expected): +def test_analyze_column(db_to_analyze, column, extra_kwargs, expected): assert ( - db_to_analyze["stuff"].analyze_column(column, common_limit=2, value_truncate=5) + db_to_analyze["stuff"].analyze_column( + column, common_limit=2, value_truncate=5, **extra_kwargs + ) == expected ) @@ -164,3 +219,41 @@ def test_analyze_table_save(db_to_analyze_path): "least_common": None, }, ] + + +@pytest.mark.parametrize( + "no_most,no_least", + ( + (False, False), + (True, False), + (False, True), + (True, True), + ), +) +def test_analyze_table_save_no_most_no_least_options( + no_most, no_least, big_db_to_analyze_path +): + args = ["analyze-tables", big_db_to_analyze_path, "--save", "--common-limit", "2"] + if no_most: + args.append("--no-most") + if no_least: + args.append("--no-least") + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 0 + rows = list(Database(big_db_to_analyze_path)["_analyze_tables_"].rows) + expected = { + "table": "stuff", + "column": "category", + "total_rows": 100, + "num_null": 0, + "num_blank": 0, + "num_distinct": 4, + "most_common": None, + "least_common": None, + } + if not no_most: + expected["most_common"] = '[["A", 40], ["B", 30]]' + if not no_least: + expected["least_common"] = '[["D", 10], ["C", 20]]' + + assert rows == [expected] From 6027f3ea6939a399aeef2578fca17efec0e539df Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 May 2023 10:19:16 -0700 Subject: [PATCH 0723/1004] No need to show common values if everything is null Closes #547 --- sqlite_utils/cli.py | 8 +++++--- sqlite_utils/db.py | 24 ++++++++++++++---------- tests/test_analyze_tables.py | 26 +++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0c91a8f..46af7ae 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2691,9 +2691,11 @@ def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least db["_analyze_tables_"].insert( column_details._asdict(), pk=("table", "column"), replace=True ) - most_common_rendered = _render_common( - "\n\n Most common:", column_details.most_common - ) + most_common_rendered = "" + if column_details.num_null != column_details.total_rows: + most_common_rendered = _render_common( + "\n\n Most common:", column_details.most_common + ) least_common_rendered = _render_common( "\n\n Least common:", column_details.least_common ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 850a3ae..997c2a5 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3470,18 +3470,22 @@ class Table(Queryable): most_common_results = [(truncate(value), total_rows)] elif num_distinct != total_rows: if most_common: - most_common_results = [ - (truncate(r[0]), r[1]) - for r in db.execute( - "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format( - column, table, column, column, common_limit - ) - ).fetchall() - ] - most_common_results.sort(key=lambda p: (p[1], p[0]), reverse=True) + # Optimization - if all rows are null, don't run this query + if num_null == total_rows: + most_common_results = [(None, total_rows)] + else: + most_common_results = [ + (truncate(r[0]), r[1]) + for r in db.execute( + "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format( + column, table, column, column, common_limit + ) + ).fetchall() + ] + most_common_results.sort(key=lambda p: (p[1], p[0]), reverse=True) if least_common: if num_distinct <= common_limit: - # No need to run the query if it will just return the results in revers order + # No need to run the query if it will just return the results in reverse order least_common_results = None else: least_common_results = [ diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index a3e4e2f..f4559ff 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -40,6 +40,7 @@ def big_db_to_analyze_path(tmpdir): to_insert.append( { "category": category, + "all_null": None, } ) db["stuff"].insert_all(to_insert) @@ -233,7 +234,15 @@ def test_analyze_table_save(db_to_analyze_path): def test_analyze_table_save_no_most_no_least_options( no_most, no_least, big_db_to_analyze_path ): - args = ["analyze-tables", big_db_to_analyze_path, "--save", "--common-limit", "2"] + args = [ + "analyze-tables", + big_db_to_analyze_path, + "--save", + "--common-limit", + "2", + "--column", + "category", + ] if no_most: args.append("--no-most") if no_least: @@ -257,3 +266,18 @@ def test_analyze_table_save_no_most_no_least_options( expected["least_common"] = '[["D", 10], ["C", 20]]' assert rows == [expected] + + +def test_analyze_table_column_all_nulls(big_db_to_analyze_path): + result = CliRunner().invoke( + cli.cli, + ["analyze-tables", big_db_to_analyze_path, "stuff", "--column", "all_null"], + ) + assert result.exit_code == 0 + assert result.output == ( + "stuff.all_null: (1/1)\n\n Total rows: 100\n" + " Null rows: 100\n" + " Blank rows: 0\n" + "\n" + " Distinct values: 0\n\n" + ) From e8c5b042e49c627aefd620c8d4b1c84eb8677f73 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 May 2023 10:35:48 -0700 Subject: [PATCH 0724/1004] Validate column names in analyze-columns, closes #548 --- sqlite_utils/cli.py | 9 +++++++++ tests/test_analyze_tables.py | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 46af7ae..5a88bd5 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2672,11 +2672,20 @@ def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least tables = db.table_names() todo = [] table_counts = {} + seen_columns = set() for table in tables: table_counts[table] = db[table].count for column in db[table].columns: if not columns or column.name in columns: todo.append((table, column.name)) + seen_columns.add(column.name) + # Check the user didn't specify a column that doesn't exist + if columns and (set(columns) - seen_columns): + raise click.ClickException( + "These columns were not found: {}".format( + ", ".join(sorted(set(columns) - seen_columns)) + ) + ) # Now we now how many we need to do for i, (table, column) in enumerate(todo): column_details = db[table].analyze_column( diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index f4559ff..dc90325 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -281,3 +281,40 @@ def test_analyze_table_column_all_nulls(big_db_to_analyze_path): "\n" " Distinct values: 0\n\n" ) + + +@pytest.mark.parametrize( + "args,expected_error", + ( + (["-c", "bad_column"], "These columns were not found: bad_column\n"), + (["one", "-c", "age"], "These columns were not found: age\n"), + (["two", "-c", "age"], None), + ( + ["one", "-c", "age", "--column", "bad"], + "These columns were not found: age, bad\n", + ), + ), +) +def test_analyze_table_validate_columns(tmpdir, args, expected_error): + path = str(tmpdir / "test_validate_columns.db") + db = Database(path) + db["one"].insert( + { + "id": 1, + "name": "one", + } + ) + db["two"].insert( + { + "id": 1, + "age": 5, + } + ) + result = CliRunner().invoke( + cli.cli, + ["analyze-tables", path] + args, + catch_exceptions=False, + ) + assert result.exit_code == (1 if expected_error else 0) + if expected_error: + assert expected_error in result.output From 718b0cba9b32d97a41bcf9757c97fe1d058da81c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 May 2023 11:41:56 -0700 Subject: [PATCH 0725/1004] Experimental TUI powered by Trogon * sqlite-utils tui command if Trogon is installed, closes #545 * Documentation for trogon TUI * Screenshot of TUI * Ignore trogon mypy error * only run flake8 on Python 3.8 or higher, closes #550 --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 5 +++-- docs/_static/img/tui.png | Bin 0 -> 238641 bytes docs/cli-reference.rst | 18 ++++++++++++++++++ docs/cli.rst | 22 ++++++++++++++++++++++ setup.py | 1 + sqlite_utils/cli.py | 9 +++++++++ tests/test_docs.py | 2 +- 8 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 docs/_static/img/tui.png diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a37907a..d5a0b69 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -26,7 +26,7 @@ jobs: ${{ runner.os }}-pip- - name: Install dependencies run: | - pip install -e '.[test]' + pip install -e '.[test,tui]' - name: Run tests run: | pytest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e706df..ab30b2d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: ${{ runner.os }}-pip- - name: Install dependencies run: | - pip install -e '.[test,mypy,flake8]' + pip install -e '.[test,mypy,flake8,tui]' - name: Optionally install numpy if: matrix.numpy == 1 run: pip install numpy @@ -44,7 +44,8 @@ jobs: pytest -v - name: run mypy run: mypy sqlite_utils tests - - name: run flake8 + - name: run flake8 if Python 3.8 or higher + if: matrix.python-version >= 3.8 run: flake8 - name: Check formatting run: black . --check diff --git a/docs/_static/img/tui.png b/docs/_static/img/tui.png new file mode 100644 index 0000000000000000000000000000000000000000..04cbeb1ed36b4e4155708dd04194f62a1f52c795 GIT binary patch literal 238641 zcmV(*K;FNJP)bQc_21sz+1dWs-r(2R*wfV0&(P7#&CbZl%BR%ly~FOr#KpnE!P~WXxVX6UvoNr* zu#mOTr>Cc2vHhEzoPV0HrIKoqk&#}W)A(Fdh=_+I$2x9}k$ruAMufhYZ#i>vasr6i zR&RG_W@Qn1yk}cW1$4MrT3doo9U@|z9bteTW?ukgvQ1A<0bhp(SAz>%Tt7re1W}g? zRCXapVMIPN1yfTzIypi(EI=|PI5H~|KS`1a2nIZFFfubGGeJ2mBSj|=F)S!6EG;r9 zA}A>;Dkvu_B_SpzCNLrzCLh4+sVc2@C}U1OWg5Uu}lC00W}HNkllm?uD{R=XASl>?z!9*|?G~_*)p<8;M0U`(}m@ zzmn|j^>E*i6!joEq&T03)DO%58|KUTe3@!J)E;V}kZ7e;WF-pQfyL`?q;?Q7w2GrJ zjN&+sOo)Q>(_j#V1M^rAyv=Vi!C2y0@(>*8WE%JSeHC3&nYbIA22s2p8`sfp)CpdB zoSvQroyZ~{3NQ?dF32r39wKF{a-u~-jzp3e~%`Rp!BbWbnyES=@?a5bIYW$Q@O zWTZz!t)ehE-9!*{qtCswW~>zn5GQG3pcCEK2oTke`@OiQrmNL7r8h%Q#xX7#w0ga+ z_QdG)#``kQ+R|R$l<#(fe44`P!Fsq5{(G)q!0!hXz!9j95G;L#0ily zlz@_r)9xUQ6jc`uqM-X#cp%M^Y)nat17{#5X1z_dW5rB<+h!NMlYm7W-w4VV{rhNnp!XUiElVNRTYD}pc1Xoh;on(rX! zD&1_+;P!d!be1smJ26Si2}B8_Vma}QK-SQ1xA8Id=#lXEKW0y11X7qxg%tAQO=uhi zFMYHmW^detcwxMRk!fEnh$>|#;cc~I1Rx$@f37Vqw6c8c>MsV0Ag-D_!g~0%v;Ek6!Ay6SP3)Q7M!C*GdtXsfSEKA3D zk(5J%EL|G*dd5+RY4L!X!rfJaf*_dIKsTTSVujPrh9x&cAT_Y?7@ng;MNYh zFJEw(!3u8vYL*(`8K%_jcEgY@p9ma;)21&DU|B(FCWZ0M7a@hb_&vl`+(L|hmd`9% zi*19AOw1etSO)^nLx>TgRYB&uLkSvz=|fpcfSC#R1X8{ohh{+tGk9Z#XFc3443Aqr8@Ito#nIYDj@$WHz_;dR3pk-76}TZ4$ig9b6@U@ugva9jrS%jPJpxYPEUBU-fuk@iteXJwZh{koJAlaqkZ*d_r%ee>E+h>a!&Io8QShJLj&N2D{= zqw0tZtaIRLVS^C707x&8+SuH36JA(5JVL&=pD<@JVL25>7(nbnX=VfjP=cp}Fp4le zgVzCj4v`RoLxjLkzkC=fW3{xVMln<+tPphnxCI@gW2S)=sT06tMo9`sL7QDhT0Dr| z6MRO+t`RzRU~$9+Z@dJ!6lQO+Sk9=!LIy?fjIb9q!w7ZfC)9}zRJtq#{H3s8)CKZ@}iY zGMF6&zo(#wFbV#ss|aKQuU!dbJAv$eOi@Sxfq|)MGl;R_1YSJwhJuH65bS}#D~G?m zB1*_}d1njsu()=lpb!U<>~$!&(MUTHX(1fo6Uz*q6d=S3n0<*>v7QMh%&pwlrr=Lgsp#ZbU)}Prt#rqPAl}E)) zy_X=AP%Bp&x|q$Db1A#yB{a?_LJ5A|@Qo^>Yj~fg@3B185 zNH@H$KIBR(<8~>-3A)!LMv(aIMnR)VzCmWlfd1uR^6y}^%!!?Kn+ah9EgK=5l)%7o zFEsfd7@fVGFvmLe}!mYm|$rx*0p0((j z(sW@o9H;Scf~&k!K4Rb~Tjs;|6MlkHmd!MO5gGoQCP+pK2;1Fg`1RYT-+yzs`E4wj z-)}zf+Zcwgy{AC!lCO}C(SeVHZb-K8zq020>%4y1-#04wzl!7}EN*`r1JXAD&jDH> zH3+(G>m+~@YL!wwEaqcOEi+u;c_|^!x9aD`?B?_2BR~(KJE?6VS!+OAU^PQe2$m0| z#c(1D0#w+rfuL_@^i^yi88CuH!{us{+6bZ`Iv8e;#p8TB`FJ~-eE$6T%aiy#nM~%n zYBk+{!ub|6>})28*Z}e970V|%VLB`R|9FhLouC7gEVdAJfOhn7|Mlx1pZ@st0etoG z!>13QKVi3jaQ#pK_x}XmN8KR0fBSH6a2?%Sf@OIINq=#D{qXSqeX7IY&HG<4X?_{~ z`s=SL;wS}vy8*^ym`Ixbdf%a6kC0*q_m`KQ%WM3HctChFc>DJH{d=p=0~#0%hQ)sZ zEWsDmu?JH3Kl)d_W~+D+tECwQgZ{g| z;Hn>^3WJu~E$D~Sj%S97P!k2Yu%P^Zoi5h)KJ&@>#j@B_UdYJLa0_NmC|w9kB6jN&sQpY2bGh(V7+}y5RK@L7MwRJISkxosu(_R;8I6z~f97LNWTpcYg=Feey1SMm_S=h2^MR6qO?g2O)W zJrt&q@~@wimzHr$dVqm+Q^Gc7v^#Je-+*fZ54rOrxx(!V(5~-@B(9){d(`6x1RO|X zU)b00o;tSkX9J{>7icH)Ec|Foil%mllE>0!e|SMq!|{IwXe>1$aFkRI&jM}Nh!wh{ zVkMNY#R;o_6~kJoT=h)zKSUL5NwN>614emYxnsU{TrSabBqc;-3CGCrWn>SIIre?^faO97XaU&+cE*mY&%zkGF@~~Oe+9=Bl`F z(-meaTL}9(Ab*4@FI_R@Sot^UR?{0djzrh%G{0s=a}+^9Z#pZWu{!ighd%XX(7?dR zz(5Ar!46_1F#?DG|3Bu1(Vr%q$sv5epsz~d{wklRe%pxB}R5xs%iyj&5sY4 zdCEp2z^aZ&h|`SfMnD5%or}W2l*L5FHo1W!lf*=8x~Z*LIDncU zu}a}2x&yFfdy9-zKg_Z-oH0kt^A$=Qf9Dg|a9vlgIM)TucV};Ic9M|)1k~=P{r9`s zb$EWk2~Y7uaAM3JNlU^aWXogWZ4WWrEu6XR3DV$Mb{ z-?VkY3Tqgt7$d!{hoPk(*;Jfhe6t0J8lIBOl9HU2ZMNY`MzCziZoY)GHo0Ddg+M72 zM3@_bwxvF+DM8z&WCSaOLv09Pg`~ky<2&)7xoaMp6^l^?52|^UXG2lO=JACRZdH~# zVsDg6MnGb(W--tkz%-1yFiH< zMz?w!W+~JrC(IwS^H2+#iWIbn*j%fZvx2Y|vD-N9R+J#?YU6IVC?O@}@z6DoVW{e>)gLlNIRrN1vsbSmhn_p3B^dI)Ies)LS%M3VFUoDSA5Z zP)LL{U`;AQ0^5wh3L(n`;RGIWixrG(%$gIJoH)q6tZWq%&-TTxiY*C2s#!K@Q40Dc zCy;|N8(2DN=L$c;)}~G>0Rz`&UB#O6^my48&jFFeg%M=&MzdNaFeUhHH6c%cq#w;# zlQzI`C^a3~UokXP9p;c{SzyO6$8KhWUBw44T!uzYQvMkjZ(&0neaL05x56g0PObDq7+R8?b8ePznAu;A+9}f(@Ll6tI=`n%{l8!D`f~DiC|H zjaS>z1Z1&sgDlXN40GmcS2)>L;}Eb~(8`MUK)rL{WOBp_3KtZmQYI8m8Z!b~(|gpE z`*ku$LjFc~mD`{yLr1PCVfHfQu9ZrVd5GR54p4MLR^S6_TOkfI#U#U| z0D{l~va-SgoVRsCm9}+)v>~B@1fYDy=ReR1LJF>FKM^AUUusV1orwST?ye28v{Mdu zcfnT{?n2l1U2x5&|KPX65E5c!lOz6$SuRuVyKmfHZ-(7tBO?is&`t|{R{-`f5dtR)kZ*Eef**6y}xkDY7# zun|w7Sfm#BVc!xZAOp4QkkSmt62bQYqpjZ*eTcNho2bg%NfViE;JXU#kRNy@hMnL`<3Hn~L3>6tlwO zc59Jo5@1A@ZmReqS}+N)N}vyTPL3_PV1&P*6Plu;NEmi^K`QHDqZK|_VeuV=B@3oew*By z<2Zy{J5NZtE*Ra$BXL5GCz$;7#C0&ru*r7g7__nb>3nPx4-Th8ArkyBoyX1?KLV$N zmQaFt4bHC!K}3SY67>A$h>P(}k&wth6BnRFQaF$+$Ds}{*83uiAE+C2n5IdYi6Mc4 z2sQpMogi1s+g&LPq*_DKQR@VqdN{?;98Op`;cGgcS~puSC_zYpu4GCOQb6JMZ)fzP z6R6Dd7XGN=oi*B&uxn1Rd@Dhm2X{N`1o7#)T50>N6GTT;BB64nb4@_KY;!{HRkkJ= z%@!PK1U^_N{7|Bq$6+79a_-^*eBewU4~DNSl;9kD`)xe=L=1L_JqdYJ>I8!{+bV|= zf=`2xZ!Zy5qeYh!B{*Ex$+8bOVBPfhMPv;7*s0^AS0)2@h0mAM6l1Gc6f~-hgVL$6 zd9UJ@muVdI>q$@`RoxJ;PeJQ_JRLv0Yi)OV{(NM4K#FLLsP;Mi+t!c+G7?+V zD>D~E&*Y<&x~XEna_812q)Kf>Bxov$hBh4s2``G$6t6P}7je8|UO=H1LJ+I|YdWD~ z58isnnF}rjs;cFkR*OE|F@6$GkPzcTn60;cv!Kn(eE4xe$adYnT|g3JP1(rB4`c{!rFw2T9nZb@LUyz~}3AWS)xgzVv(#WW7osjDiWo-P0N>ExD z0j2$+f~RdczaNi%<6M6ny2IeW38o)HcermI)?Z>gVrp3=jbiW_c&kxSg_~YK*gm`Fir#1dC4y{(; zgq!12v*SmcV0CyLlTI1+{@6Ny82TgT>c9!zaR|<#<+0N$q%qOIZA8x-3uufneh52v zIeR(Y%H$i7f7FjZlXY^7lssa$~dQ$<0N$LZXWr zInx21*-g=z7K9S2=pxEmCG4`Zx&vg(;T0@XJE4OL&2}j+^JWXFOK?Je-*$u09G?tD zBM$MAU576fCtT-s-n}W17ymoqgu85?{}N94R~1F5_Ag4`p$;*A17c=bcjW#^Ca7Bi z^Kt%0CC^utY&(=X!B8PwB()Mc!|E-}W7a~!@Tw(nEO{kq?J7!m9jL(=$0-h`kB|8O zst4aEBZ=Mq;c}UL``i5~PTsnNaJ>e5i2HDVK404&ZIVtv^AnoLUpCD#3cHWjclU8> zLOh}K%e!klIkcFjxYvEWUaxU916kgP#<+AEqf`FBCrsvGAX*OTzY3f7_wxl~G?1a^OWdR13yt9#YczrUIn2^`n02^mV?fi9Ee<|E&o9-%2BuAiqy%z>~0zOJ3I#oFnRzK z{{fW1I^iplZ~`0Q8{ns(fBq?*^YZ^=fgE{!TYwr?dSDG-TsMHOgL{Ho2{PuVAkhTH zq+s*PRSE=Mb3(OkxRMFTi$?f|5lHBg4z)!I1lf)hrQnc-5$Hfx6(_t#PDM!fwb!W) zecz1F2Y+}t6DJ&F2a`UyW_;=_oCw>tZ5P6H4(YN9@5kV9IhnM-PQmw`1A_eT4(4F{ zU(qqx#s4;%{jqz=m(Kd+` zv;)oD_VL)b@c!hTMb)}?x*vZ(dFzb-&(nv!)?l^sJFpADe_BH?aP2f~`3)NPGbI32 zxV7J4GKdheNKkqEF^3k}a4e5j1VJsMRtX$R*Z~|zz&Se&$Q!5$K?dij) z2LfTm3MwVzOHTN*sS`3M0O_20qUHp?T6029fM2-_T*%2So~+PpaP_v13jBDUdO!; z6GT9`TrwvZGXCi@O%v?+6d&TjpP3W1|2TTLXHIZOH1C@f+neI!XTql#I9O@W%iez; z4UAT)KJNR^={{zu*QwMLF3@eNJuAJ}KS#$Yfs1o|OQ z{g8*F;U4l5W*L}+U9wErNf@U6{lCMNs#GbJB~S0n4tIMc+fvJtEIY2Bs!Ani?_szP z!OhDP_3UXGEsAExKf0SC$_d@m8z#8^{CH|za5jwd>3%T7akTJUn!O=Ju$dQgIo%+x zlH|M?WNf0s8}kh&?baqhpkopg1QK%Z$51K`j0=VkAu}TgFBkl;NMs5ArB8FvO$m#H zZKoxpA_N0I#Ak|_@cRW};4{k!7s3Q=`nsIJc?uJ>e}W=DLs56Nb1;f0aPd{)Ho2D$u%q_VO4S!h|rtdw3Xe z`qLNy4Ab>6S_8Ty@m-q{n3Szg+!FSM+CWrHIK&!^0~BD3yYP*yaVW$D=d_gI7!hKO zW&{!QOXp=Bh!kQJ=UI_vQci)SUR`lWkO&VRTV+g$Uj(kp3I8bV35~WVU|vx61m){o zx;U5i1lAt>nm==KTsLb85h967FA?opP!J_RX@8!#ubLA!7cV(|61zl8g^SWOWlw~1 z0xAy}cW9_R0E?dQyW8c^AO0ONVfMFQA|~8+0mOX=1?YtYp?~a-FVnvNcp6Y|RVj6V^KhA+L;oO@3-QD}a#aC^bJCF~0!i1M@g1DWh z0305Uzab`!#{vD5475nzN3MH3JFepU3 zA|BY9fH6dgb9_@SaR@bA0#qiZB(;GA8^!8+IEMb*K)iyqN7VAxZH+8LF@O95^PrF z3}t#GAwghZn{s18mE0=L{Y|DsA;g2wFmEP$({lh|X zLpUv6Je<3xMJeE2lsA&KuMeQod_>rp0{o;Ncv{}jMr$ScSRNl9`w;qvWof`qC>SGU$O%!KSOEwS z4`}Nsdpyho+~Y7$!|B|0(-4NK3!xv!HuUoW(&^Q1$9Wt^_}NcW*JHWfL*_iB7hY!L zP#&NAd1`Gl4i?hgG)DX00HwX{rr!Bs49@hBjTDCYFt$Ek9@=ev>{cHFN_V(0Ca~fzevC4$sQnlzZxolMmDkdm zxFNv=HVl^LKFBg}py0^yHRd2FSjzU^vDo%0q#b(sws7AXN}ev zl=jp8tf)^q7!XE2S`2~@4SU2uCY|KZ68K7=k=NSeDgTMd> zLIhi#-{5;vI>??#w1z@VPmt=1tmU2JC$DlAP{=veDo`L^+d*KzW`3u)JdPO=6i#UAz-u7L9es^(yQxSD z+xAD~j36v64~7#ft+FJpturaWW^J*cM+>!Sv4VW0HM?7RpHoic9-Jl-_iMUOq%?oa zs35_`Q|##MFwYuV2Tr$-C1{?nQOr(^xNEC2Jv$a!MueTqZdGceI)#v6_u`gN`Y@=% z?DU?nz#5r?#5j3GxQg*YLyMpS5zjtu8V(gI)ILvEYT9~OEs0I8bxbfiX=Ka9giKEO z^b-mR%ASA+HQcMdtFGgLYL#+9o-US121bOnNx|agtv5TS3=KH)k@7{&l%mZSmIW3g z#G{#^ph}AA4F)EI+p2x1Y>FZxsE5{KBvhJY>jDA>em9+!;%W>8Qr35211Gdz4jJ3< zcmp`V=bYygP#BA+f`%Y+(@2CpLlB6dz?hK!wxQ7+gsa91_78rU=@m- zj-3zXH_0;BD}n{ek|HrPk$qbTNgiPdG?HT2NWYLEf2RSa_&OsF+dI)3SZh)YzN+Q+ z$#>CYwN!;-(Gw8K{Kkl*k}QsfgXjCL0)mj;v+JT7dJBJKH43yVAow2Av$C6Vg5i_~ z2P5+$2@^iYf4vJnONE@!tfXD_vjTN>^{35GOP-KdmO=k@n}StBf)&J9prEac9ycxK zd(j)Vsjw{y801M;K!ToUkhL0l(AHyN?E0G#(*CA6gtpX`wKY6_xugmSK`oI+!ydOT zBpg5qXZD5*$^}JGus-(98~#kI!UW%}?F42Q+WK^&lh5lGlaF;NL7NiDx>$vT(&x~S zCj;zo$Y&uTPHaOLt}cWWX#?xAR{|n}HXP)GYb%KGm+=7m#wmUkT-~ZD$Y3~=-IW?8 zC!je2a$1#(D4^44gC9WuX zgDGYk3Sck@357%?;HcgXAf+lWrxz4BM7BdnXg3XvnVk^%RHWrnbs$_MK&X&cumF!; z?k+T9XLLj{)3fdn6ea|g6GU-va_ij3Dk3nIlaE3oz7r()4Q0bu{F$wRipi4l=Bi{I zbCM0KrIZQ~1QDPGb{{mpDONEkov}!LFB)X}BuCqF0tQMbS;1v1tyxZZ`3!Qx<+5BN zp`1|nuhdCxP1RJ-tRL6|_!uY&m&hecUaJiGnrRWwuC+ zpE*fyIX@MWDcT^BH8YAViXJI#e<>@%5pL`GZ}Wc zD#;oA|4+FOf!)zVj7S|b=rQpZZ26gSH_|iT-I$QGrSvEg{R=Lqu&ge_aRp{aT5Y5t zuwF&#Y|{+{$8B71aq0pxIDimIpLhqzm~rxrT=ZYr0D;ge)t_EdXs4!sm}ao&S0M4{c}HH+%Rgiln5m>fx#C zru<>V8C2{yn@wFetFziwPk1iZ@bk^!0lb2ZPuiCN)rsv5OzNc=g*(NppgtBKX`<3q%+^* zdqc(LbNeTYvd4JdH|Q(IN1b!q6X`qqFvj7-@0zggJ%9kmT)mHS!TNnfuUrz52L6uEN(ZEHT0-2n%Go!8mE8 zUGi{{0>kt5@H~AvY<4?TGj4YcAQXQ+YWy24O% zqQVT{h8oA&VU0{RB(va@Od^`klnOKQQfBr@o)kKkb<^U+suiZDZ7|B)+(7G9kW(oE zab%cHrql5>$P2IV219YyDxOhZsl+OF4-Z?~MFZgD82f4Bz6sQf4+`6SZ$Rz4hNw;V8<>hl{2l)iW|%K})@Hwo*_ zx;5c#U@|o#Sz@i$_f}!~e8q!!7A#~1s)GK3OtZ<~*Np%2YneAk*n}@F?}rgz7dz!9yJ@ghqq{hKvdO%W=&!LVKCJzzxg~ zyWQ~{NKl5f7_gc>!Fh)X6Chy{_%EM}UyxP;CLq$Vn)$A#C;Qj$5TPIuAUgn)J!_g# zF}UQS($u~YTdA2cqSXD%(AY^E#}ipb=@+J%2^Gv-$+-mzs;VqZMi~*-%Ag#P?H-oX z@wiPnMrn+VN7IHxfE)*PSjc`0ge7Jo56)W&O7XTqkia9eKMamDCBdwN%}fLXPUWx9 zQ=EB}fI%2Cx*bhss}2xSYu&ig@P3xU;1eeQuzW|50d6EKHrc8!{A~*pzLE)#(h_-O z?jYlhjL|1hd!fQ}GynxCeJFeC{`(D(uwnFX6b18{_1Oh=V0#^HKG9;ut;KhFyND~X+^qXH>Y*FsG zL*uc?i6Kgrp~Ms#;Ysm@GzfR4Ca{+A+wP#BGs2`WV6^SweMKJIM=3XYbX z9L35j4mQFENgp;6=}rVZpE5xfvL<^ExC8(IBROD(fv>aSM(F9xVGs!^6xQ?!9VBq! zY8wP#Ef!(GAEY=)ztxPFuv+(t;NZv&ve^U$iZj8H5X$Hflt|U(OOk#d!$L49#P~f! zVQNWm2<8)F-KX%>fg%#-Z5=L|gz*SOST{{(5`Y7aXOj@&c8eK(J6;%Th~tYU!2{ZY z8KW&OCFX7t0$*K+MM?z9iLf#zxKhIZ{{vxW=;FKL$Q8~^Bfx@);IdfxyLKjgai1Uq zH1^!VsqnmVLO)fo85g>`UshGMTyBpz z3a^p;4QLXw>-4M>lC%iiFoc8-cV=!9`X0gkwB!Lz=M;;OWGp-h54R#}P_NYvIE4Q! z;xVOsP9T{oBP?RxoPy48jRKT=g+&|XOjsdzFrT0%fwJ7e^$f3IvlI!5Qve03a(egi z4mJJxeEaaQi_yB4u;sL@=JVy27GN)g*dG!db*@ktFbZ1|fitW+b7UFa!+^NUlKbL; zZ`QZDtVHc5N}wRQL)A8U8w-iRq@iuU4TSvrtf4XsD;9tfCan7G@Tlol>v@suUESh- zWh2q2hnvb&K~+^vlR&|9|7m1G$e2(>CQP!_;bvSlq37!LY~$$?f5P;m#X5pa7MI1M zC>FPjO7#dN0v0?pC#5UR93m2!0C46e!H~eS&?4kGa+A!pzQfU zm*AiqzyR`Qp9KS^Z!F~AW+&;ddme1j5G^e5DRDX zuu8sK%7hh}u!Gbmd>Hfz%pjhy2Qry0@FaqdDcf>9>{`IU$8tffF$fvJh89Tp1S;Gk zZ@9mo(M6a-?Kj6?^89)NG#31t1VI&ej1Pt?~Y%nUYm~g3%fUxcY zAyM4aH}11|J+GrdNSV<0qgh_=RKSx-v790iBn5EVn^KMljR5S~O0#4j1393=3 zx=bhwo}*JvP`kpd;}6qiFI)>~I#H9*3!|I}5o}giIqifhmJ!o zAI-u?d)!T!cElGSt6bbLR3ZeoZKd=oECfsTl=qyT}A?_DUy;2{+56 zzUX;_nuJyI!8CCo5wJ8MSkf?YTNJ#lg379kjAxUl47$>3rmItvKx~EFqL>j|DH2+@ z2M&bhpLPZ{r$Hh(739z3^#iPNX3jxZMMX>HOpwF@Lf>XL3ge4D0UELjzR=;&h~sSb zo=51@58@G^gD}CVFs&ERBPLz$nmLdFB0z8nck`_P;kY|`{vfMIk-*0q2lhKw0QqQ( zhhqRd6>v>L4pMwWl$(OMqyfM3X=JRqEk+2 z&hG`3!+-(m$>R?V`M?O3p zZP=0oZ8cYw5Ej*v0H8-O5f2!0r?BwEL3l!Cm4_BYfCLN6y_4c$0|qoneX$83f|~^Y z&SD-)1{rhorotl$yHK--BNVNHE30DzD*V(k!Gzpd7igw6Do94Z@R}r6Xq<3Ah=ii7OG5Mub!aZ33s*`lxg` z!(qf}=13U!3DixjC;XW*K_*?00>*+~((tHjn}U@pVxig#3bXJWmOT^IKkcRFnD z;1G0N9?l0HzO0o7uDXPwU0()*1SFy!L&Mg(vaE~-nO4xVkT5CinrBETFzJYxmDiW# zBq$HCfiIm1p}}D7Q6W^ygf>h(#%L7eY}^#iKA{wGJed;wD!$AoXuC{+O?YID@qH{P`27y12m_N-fJ>SLd#FPL zAQaOQgQ(J`LPj$en6}X;7!~H)bC6eSSOu0CKXTw6hX*QlRw5AgJ4eFt*e;huLMv|+ zdsLAL<(x^tKDp{!7GYX%7CRLmEHMpmZmkJx-N2It&f;X zge*veG;5jV9U%26x82Wn2p-7`opdE`@+B7`*~MLiEc~dLZTbO)xE#k6ObmZcBVQ`cPQuf?O zCLoDUAerCh~nQc4M} zz`SME4vBuDp7o$m>zxp_Uw6k-G9gAy00{V6WCBq%31kA1Fl~Fagb9(*_A3c)CNv^c zM8LWj*|nZ#Xzeacbq>7T^*b4KG)a=@OsVsvB|Vle+iP5|a}D8uQEm?y?rseHVH|w) zClZF_M6e!9oQQ>7o;h?sh%QF@wJPb4^_p%m zvQCv61%<+rZ87YJjgkp{f?|Otp$L8rsZ~r9LZcQTu-4KfJXmEgNP8{XiL$gITyhCS zLRw%9Are7)O1J#5gIe3YPu8U1P=MN*8wKrxMC{Nf@KuR~pwZY=dFF672>@dwJtew* z5@cfDfSIc(By`uc1ak~dJcb6Vi+L^K)+-4%VHR+TUMH!L5+N-vvXfG`CFyT=JPt;< zIMX@AITwlr-NT|@A>wXxVofac|?PtyrZ5^nD3c4$V_jquFLkI zEjpq@{fCml$O9*wo1G-wAZ~35O%0FyCsalN`gxrcW>^ z1Sy2~9Bos}RW#-dl2V29i&`^9s)p0R{WBpeqd5_%W>qIh2v{;}m`?PDYn<64-(FAD zhXO|4C~Z&(b=+?v47kF4WNuT88E(4?qLL>W> z@sFJReLVbzT|&}pQO>$bdPxo5e@PNSq1Q5enFsZNMX<=nWb(&c`UW!&j)+JT8#pt0 zn&`;_;oupA62ZGB$mAcznE;m{#eTgnGCBqK346{mW$lkP#42N$mO3!v1q$;+LWSep zii{kf`4&uIVt<4x?VjLJIEYQa$e!43D9dIqK(OIpNx~Tm`xZc&mj(rBe+B}E?Us!x zBn?i0>8$vXcP=e3kY#|va(Ta(-B=$vAah8WAV^p{5h%d`h|oqNlw4=S_OKe);8m#a zZ6~VWNGN0yBm^xZ@NLLj)v9^jQ3GY}L1=|IRNxpCnwA702qbyvF`?v2D5$C;5;Sx0 zQbH+tLscSi$V&-cWA9x3LPG+?%R2#^WmUm+$t83LNM|V#Y9&G#&Klwkkb_7Fmr@4L zMRfF;D|=`=6L2;Ehj8kW6bIa-t559v^%8D4%vfU zL_%58D3A&*;!Xq2Iw<(?2?&_qYw{rRd5;Yd30Uyi7I7$Vw&nFsNW2j!xKF6y1JX{& zQ;1Ml>Yb788r>R!0qZV7**pKH_C}fmF=9j@6}W??wFt&y)~tEq^-G@;VGSUV2n|-B zg25mW!VnQWM_#r5a|NZ~pqL<3+0!^ky23U^>oSFBIF>=wyXUEm_E@PPfC-JL4OY0D zA+q%bY4HS+2_#Tr6M%%VB0+YbUzrAR39Ez%)=i0{-v4O8Img94S;^9{Kos^z7Xsfg3b5GU~=yxop|DG0{5>)s(0-tooWc*7yA&>~R z9N9+xYJ!Yz6PyW;@Ck71tT9>#g&Xq;j)W|GI9T5B=uohkj4%oskp_h^@(1w|bPH^F z_?Qfcd{E(jxsJh0!+_P*s?3rUT9#y z@E}TV)f^U*p;nmyXJNnGAb`ReZ32CQh%TXNJ68}1GlXQzq+vdvPPzKT4=C{Q(^*5_ z;6UN(?|UFT!Gak(5a|tkO)c&4!EAn4Glea&V1l=Y0^^8AK`$vDZN>zgDFm%BM=GWNm&62)hhjPMg^~=iHdNJ+!^moZoNe(uP)}z1%7jb4Kx~Zg7l@m z@PDBa0R_9;-BI*34>%;z{tPtu-yzKxE*ph{iuXTpj98E@5Egu3otlT3aFluie8T$? zoI>{d36$dCkk=WrHc8H8e$dPT5s8KebqeEY36c;C!i9MX1xCJzb3A~&qQI>2{RvJ+0azj!bWJO!+-+E)G+v#J^;-Q7!N2uG!>p8 zUSxtaI(WN-jQ#r}bDIFMvZm8RBbOHC!C$+k;gwS_(g@@^TN?od%^%dQOU2-0<`h1T zf0sPsl*z_BUW!Uwgb*b^Y(Zn-=#x}Ps{0Tqn( zjuIMhR4D9e6X+A%C74NwkvHKl2Z>1-kEUE@Pr!CHYz-j2E)7)?G9q}Iyw-@1PnUIx z*OHj!wL#*N5niGa<+5JM2ym<`6&nAUNFEVnKPc$CwQ$2CRB|Z!qc#Ckz+9NX)=zDQ zBp)I9N5LT#{xGWq2pFj(Fcm_p`1$K*ZA5S+q%I*d36Prvj!YU>2@ojr2w83MHmNP< z#D4N-MOR@b=A6Or_bUE zPNMs}`5pCt7+`dxO{jH68b*jD8IeAGZ6}k zg#+6m&o#!HjgX!-kO)Pk<{NCpr>e<>TGm$Iat6&w^bm4~kjOnkRNY1-xJf{`BYiI$ z!lDM8N`=Ri9N-VC*liDD8n9i>LM@^1ArM$oV2nbmPRIr8pHK!OVOrvKG0`Sq1*aTI zA<~F2E-N4*-aq9?;I-Oo34x)Z_l9z6AuQ8$D`5%`?74*ew-;wUAQ+4k2pJK)OTv&x z$Ysu+i$$LY7#$0#xTQyhJlSyf;qr4Ow!kZmT*FN1_H8DhprAFz(RuhVqE8?TEOo$i z5eh&;@ml`lh{*{7v_JiNb@lGVwn_5jybf{X$MP+{{$WfcYNr`f~je_vg_{bRbB z$oa0{9~ZSKAOoR$xHFY=cM7iEL7`9s0cI)Nay(X_a1jaF#(|yW9;eo7j7%J4lEv;J zVM1AyZBhd-#3UfH*h6Xi=IWPIz>f71YhOdnU0q3p@Ne+$VOHmMG^%TfElFl60pjyw z60j|V=~%u7k~Fjm1U%DZy)f9XLUu2v7~&n(epL*m{UBp)OmLr2@?i)ub8wqbl~PO? zjhB_1gsKiiLfZ|iI*BKl+poza>V1h#IWY6A!Ey$uSA(#0<>Jw|g|;>j<^z4lh6;-= zhe5^yB0a+b#(@$;t|GSy7$MHIMbaq&D>=|_UYSrou$rJgfxVllo9gSAKfgpIWJEv_ z-=HA9hfeRlm;bWu?dbq>aJqW&;`&r0iD=$j|N41+M=A&o)G08P+%t?0+8B9*{RoAz z9D8+|Jihz>1DX|@p6i#miku+Zf~hYD4wp(?bodRc)1NLH;0P<_Hy!~gk4f$Ilj z0(Cee0V`iROsFMuuc6$L`G8tS^(1m1p8L?|Hooe4OjI)xq+rpyo2m9T!O zzt_w&RV7r6N0lJKOhQPJfQLHQ3%#yHBD6i_M4*C<-bR<*vJ?m2l-BF?h{-x6T)by$ zV%;UFC$oms9B@&1X~K2Ok<#erHs8V{2568ugp_~^040MiQsuWp#?R$UkXsSy6QK7p z`|ZPTzb*g71yGvsEp>1;xX>R%5Gd4Pw_k})h4%WV?_Pog=mQwnT-dg$Ck0p5m_m5>_WHZ;$OJKaoENf&*^Fykw=b>#uBcxw zYxY>Iq{J_*Xd3U+ob*cK+_YKOvojT#KT!7L4uk@>)=ikWPY{0~M@g#!3&A01|7Hcj zcr>rhCgF^PrW?@ObFmwqH4x?R@6yd5t9~gVy<30(9}SEg11!~6V`$R?6EsQM1tA`X9BFUn}jmJ8I&j06ebAz z=>+nY*_}?I*qlz-VU4fh>5VdBhTZWE!u$bK@t5p?ZO~|UA`x^7ts#N& ziIWNdf_QZyLRCf}tX8=}NZrD!#GVLghe8eMH&&B}y2FH$DT5#&f)M+GgOWBy5RA%7 zS3m;wF$ofqv;p(fBIGs~L+>|gG0|dJM5mWcofHT;5ok>Y(gwwCph?hIeZnG8j+XqE zOyWR_ru}^W{0|{yNX$YpoH*njgh=qN%vpKSZs^RAkTT)Xm;k*45Jdko|Ksx(MZSAr zL7>n<0*t`VKmP00n}^N%^zW-zFRzdz7ZSs7Px~)_y}p9poi_M>zx@H%|M%n1<4>Pn zzx?Ut^@sbh5n+0Bef8$^{QWhK-TnH*i&v-jv&RLGrBAp$U4N%c`1|zld=VyKn|%HVh1l@$ z;7kykU|)1u4b!rz;$P^^pQF>S*RNjw0C!NtHRKNAjD*wM>z6zkZn5Z-A}3M8K&_aqg9P^)nGF z>A7u@4`C*<4IL`f1)va}LhYP`)F_Bgurk?9nV=n(g4zUmAXy7k2$aku(AJgzOW67K zCXTCNd{Ly9BEeRuss;%O9i-~0d&316Rk~bcc1RO4Nhq15VG`IW`1^l{^SrO^I9bEl zw_pq;Fl6wL=lx@6(;(sU3Suk~x@OgKShY@3aP16HEDlcv~L+UAnP$@0<#xp#p=&@*ymZ&?b*rV$Grg+t!}@ly;d=vrORn1Zaf+ zd|@K|@kjjm`FWs9q3;9*Lxoge2s^hoy+M&-;P#vw2P4{(I-xv>k%PzPA`tVtTX|0N zsrc*O>0z@G^z-V~eZ}SN<)(LkF34+m?#u1^Q|v1}X!r}738&{i-tD$`=hYGlWtr3T zA>%)wp^l6p6m?l;)PMho*Q8SEsb~{#;x-a&u}~LrTF`>=@3gvnMiG;|R9L7fo=`9o zDux1i4?4tnY%Hg^e4;XttenFv)JJF}#N-CO8>ih<5n?~`#ata!u^8|UgT+BH!Slsk zZxAHQe(!$P(`Va%x<{ApO@&7m%0PMq!VZkw_o~Ba`cTp`>=)9bd*vQ!_(P zrYu4Rmtlh6VnWa`A<(V^kf8VgKPp;tFu{u>NrX+DG)RcLGXu6kBfL}N)EK*V&IFK9 zmRV>jSn)gl=Te=KRZh?$f~^vi`pR=Pey{StxGi>2(cmReC;Tx~v(0c}7j>|}jcxuV zonR^vOpoS4*()_`YbxU+&4h9*bCK@i>B|=;!pon3rQ-AaiIosRg&#j!N+A^R^rymo zdQQd6afZ)}zVo1ULUlaxXr2p%6-I*_=XN+4&cwuV#(zIJ&P1$*w9U(%GZytT_I)1R z>Ft(h+Sf4Q9PsUg-fgRmYm$^&*w7y?bY+Xn8ccw6_{cMCvQfBr-&{yLXn}xBnPvr+ zi;r~Jo!M=l8;R%0TnQ+)eO}afIl>jO4(-9gxE$*NW<~PpW+#7%n zPpWq>A|WlRmUm8n<}D^OF?Mbq#p4Y)>o!anI-?+Q2fgvkx#{=z7W0*7|))ESSWkZtdWr%>>s>eKK5- z9p4sShOn9mh1Lm+IgkJ%{Q3Rop~~o-+ZYd^0!anK1RDLL`B8{ejOe#~NgjeHnNaSt zkeOh72VaQw0NZ^&-GTy($s^e`a{Cw)zi11!op%089il`wt?*JGc;Tzy&!s0!>*E6KYOG zmNiU5K`$$ZKdx5NNbsX|lqR7V(dTjR@LBI^e5b?&({rctXt+epM-kLQfUrYRp9W5U zk!U8IhrOYnESU+;!@YAm=@S!hI19YW31d&cubNKGGceu$I)k4PzSUa`#ts*_es2-WGkLt+9tFtA6;1kfZQ&44kBht`eT5E2qL>7{8x%4ztg zKCmK-xSC9}`(n%m!MYV(Je5)r!461nyFg%`8lLFbqAD%>mpB3%iwJ?@L27vL@8^3k ziXqDq6K(t#BG@v43lgZYat<~UY$m9&_S;;lgm2#|Kfp}j`G!A8DKtzd4jw!KX8@0P zLEdq`>`Nv%U;_OrNG9Zm9iGQCVR2sG;x`S3kuh_6WH(E{kA1gHgPga^4)sYf0g1{~ zzFt`!f-%3_@*FS~xREmtcJ%8*10GZ`mX%2;nedK;2O^D<3bh?az(Z)DE81cvkdKOfVlt+*8z$ZgHEc?#r>@SH4cg~Qq4cr@` zClx~vMoCwQs5wTzOLBq~!Wj3@9VP^>WCEOqdngSsq5qt?!zQEIgb{WeNJwG~M&c6G zi!PeVaL@9$%~#4$L7^cJ{g_^>340hO_-!V5q2vq)5wQpE00rbZtYr)V4cdf}pd5z_ z#zbh-z@8M+G9Eo^uBLf?dDT={`9Y@^0k3E;=Zr@CD&5r!Iz-S3NNduG4TO%hvyCu$ zI?iipOlq#m1yCUg+EveX4#Kygpv3{Mx^1Ju&&`C~>rB`diV0tS2NJ%0dtr;upI`!y znjhZ@3YrS}QOw5V@w%KZ5NNed_=H$jOi1fO{{FAlNB7q0KakCSzRag&Po)Q)@C$LGS;RoHuB>mrFFd+j9Xk3$6 zc(>dJooUD^fPjftq=Ygi1B}n64Mc3hxIh=1feCr8Sbz&V5c^86VowI9ry2<|*cgZ> zqGtAtHUbxpk&D*D^+3Im@Z`~$do~kr98QV}-U&?b$_0)Xd9jmYU_l6m=NJ^6WpRgC3FZ0QH&0H!~pEN zUbzwpVN=Bjl@MP7LP$ht=g2O44LdVHcge~H4R9eWD0A0pMC%-jwHCC|38n1Dh%KGVR9FM2UsSklPS}0b%7Fl(kNAXHcLY@^{y&l`b>k_b+f9 zzPsAf(Rxgkc zS)_~MVbmUs$(EQPK}O>BBx4UJfUfDsIB6JD$_`iBB3iL>i4*)$aN^Qyx7;Z znf-)4%yKf=Fz{8Wk{8C*O(S#$h>$)w#LgjlkKJCAckt^4+^a|Tc{1e~QTH+5h2q=XtLcc447_kQ7YiB5(=6L zk!C{5OdwNkOUQ{Hj#!4|9*|4`613kCaGugaf(rYqR0BH`acZ_O_QqXj{jN&`O~p0y?I(Wdaew zs$SGwi1cB$1B_iJ{8Bmrv4q!eAOSO>p!TqrO5tZC6v|xOBOUJl}S=$l%fZ`+>h*EXK4orxT_Y$8xZ&HJ6n+ zTQ)W&*CB#=e&&ay61*^IKIK5nRBzboHQ4vW>=vJsbCE;NMGhAQz)Q#A_PN>Pe%k9j zje4VJWNhzy{bjzp>kZE|igT%mG!cA`?RjCQH9}Vr7&;i~&=L<>lYq!$kE4th6O>Mn zAx88934%BYnINlGn)n`&u!@!6zzQR>kpSZ*@7wyowhJfS-g!JP=asH69dkKSX?gJ7 zLV|87LAQ#aTeo;M{iw`78;tr7;f&+eN$!jTeWNhdO3iklU@HY*UM}eb8wBQ_Zz>b6 z%o?fMI^pZ~H^_w7zwwx1!tnran&kA4H!T#3(_->4LRHN|q+)uHGMheU7v?CZd6*!A zBzscUh~hZm|)*I*DN7G@h}=aJd2#_q93tuwHI^hO$xEZ zyAA2xYLoUWcfx_2;A(4(5A0S;zV zyN~11_|ZLhUj0Nz@^R_c)B7Rs@r--ZhpGGc@I1q7`T<_~oMZrkPVC|RX%MeD3&|28 z4%wtm$^#RNRXNC*=8UQ!4l;v_724&=A283}k)iG+ybjW!Z! z(W&ceRa=ooX}btWu!z9+%0WHbi$`PTI$6skbV?Fb9Kk|@#TC1xL7n*Pau6c0&AiIC zEf+2*1rtl~yECdTW*uBBz6eX&$JwH$#e^+0;X58OzJtK{!$7EDSl>iN6bKe4dN$o> zLKMP7)PO_*8%E_?zJgztRV>8YPw|Pcj}XBxY`ony8U2G$Z&jU+mF(B2TCmke-8&tq zrKUQzteG=G?O(ae*}gkPzce#{$)uTQs`cSlX=58vC&i2@tfHK${X1(C}^q z0`w6~;7o%;!fHj1gPQOJrd&n>@iV(j1VaOx2rLtfPH;P1aO2#3$_sI1d4+m{#zMYW*HPrOFa>5x?DRHR7@bWVHpk{w2Wx;vb!q&{{HlD{A>)b(yF z2YpHUNh&(c?4wk*nQoHnp@>QKOPi*5G7+xv$ zqawgFv{gtO;Zd}Cp5w7*=D#KaouN78bzPz`?40Gz!h@wCbjQRL{3ztSg3Sc}cdvth z@(UyrSP4*rp&tPWxLFet0@zg~gy}W6!9Ejz%hE_@At4oT`wt71ay4-$$p>3n=mR5h@)KijPTg_U{7%TL9jU)=>Pv8 z_I<0nX`2S^YIUXZ26Re`Jsl<2? zCUB#r+R#-W&dUZ;X(m7zZp@WDxJs=2S;UR7X?OpyeC#-S69S484B!Kd{5;8;reF|a zEh68XW0zq^pg_qlQ{i7_R(>gRME^n)pqkL)M%k7XzK~%Ul>k{7$}quJS?Ze(n>;a= zLz0pUM1m~*=4QQovFC(WE*!$}H+R`F>p$})PUPEu=eE-M&1PDjKv>wyDFvmUZRZ+4 zmzglfy zQcp5%IzX^Fm6tp}btRApZr!EJDG5CycucsYV%Vuq^P@OUYbf|c@M#=%x#0U1vZ(7E z_Q2pYo&Fr;oDJKN6&L-BPMg{|CXfnbf^R$Q+FbkyZ)5^#z`K=d`QxqHDGBA-2otao zJ>dt216oE&NDFA%zd`o9kA`6i46!JhvOsfNu8Vq`#;dHZnfuJEYz`s_5HP5>j7x&x z85{_VS)@5pLd2L5sW)&pR@JfOY=hTpdLj)GxUE|!gh-RwXr%<#N6`|XCWLrACznfz z3J`4m!8$vp>u4(=PBQ8CCbrDExzI$Qwg4aq53H*hjV$tF7dsvO>rP@yeWzg-feAf5 zL1LTr%5Q=MNsCUklC~CTz$A>rfCTV5=8Z`SBf4&~tR#C4|OY!NT*0)mX$aV zDm0X`A}ijuB!ZT$Wr8IFm$`RBqomEOnn25eNT8ZDIi-Z`1KL!lr zGz=m+<%xtj+>JRk1E(XbmdIj{F4+sx6z5uMME_wr?gXbNc#`L*7kXVH!9#*G*=$O* zOz%t_2)@kb0VqUHVjL_!vb z1t8(4fkGO?cxdPbz`0$%u5yroK%k%Cm4y}%xC|OV-Y8NzhLq!gXD>L>DaXh+$}8t$ zlXKI2_GUYmD~jlnGHHj>S;>FhX6CHKx40DHTFn5%?%d7zayrNV3j* zpv9FPGNm$y18kzG?^#?Y*uz{W736SKrQ?e{pk%qf=7TBAcFB`Kgy#Nw5l&7lz>d-2 zQqSGaot?V7Q$!0qmcpdxM}FWA6%4%G+@$p=L?=}6#lry~c@HyvLOA|<0a>2ypywU_ z_5C;(S$3EJaZI3KMiQ-Zx0ubo{q_E;$4bIT-RMu4(uupfYLG8H>>gm`iE>U;x0g#J;Hpc`xsU`DK2RRZ6 zu_>nVQq3Nwtwut+n;^&Kv6(NQ9;Qk>D=XY2DPjpEq=(6P({UiI)00xxX`&|MWG!cF zJ;67B+9!?)xLv@)eAQusSwtr4TMk}C0*UFW4V~2y{7fkqog_NG+dDRUF0?-2T$oZE zqU_~<3K=p@1tVT`V+uLic#aCP@l6N$=djRQR{q3qb6Q5uG2u_#LeUfc{kNwlY&<4V z0EJvlhmg_ieJR6b1rEep(^p~9*QvBr^ShDo!FvrD6iovzkPStd#o5LLF66J4R-(&#KmainztuRXj%pka`f;Ds+mA7diQic znAJ86wYa%gf&7lrI?)kml#>Wu6~PjLCtgwnh?RC_TaA-}9t_SqqDc+G*bEK=)5m?* zd7knRXFyD+1wUDRxt^5{7L>Hhgt#~DU@Qk3GAz7ikieRoYQhYRn7NZJLR)XHMrtm7 z%)eL(5n>V{e$66vf75ndisj?9NqZ|?sKIqmOR&)nS?74K&@qAHwms>d?)@Ul{#=iV zNH|v|!6$-#)rp; z(c|NA7#x1Xgo7`PM=Wai7YjHUOGvU}q!9}y<*y(@j?s)kUoBE3(>mchw{6PwU=EN_ikEZEWM~d*B)N_Dw74Cx?OGIe zNGt+>x~zLL3ie5al)JHKHXj)#1gsT+SLwu%{pKXg3Mzb*cO7jy?wJuB1owl20CJd?2H zO_py(KX`+5Gb0;)si)~s7JvdC!313U3(*z$qA&qFSa-q}y-ui|fC*6`CNm9fULF1Ta+FY@CKx?o_c)E3ve?h3 zGnIo1@1x-#e37z@A!%BoZ+@5!Cx`tEISUxA|Nr238*b%(SsG?jv#df{7MvU`o4nov zX{*A?C4sb1&&5JJRcVHYo~7uqK07AlrXlK#PjH((^Q$@RG>gED`*2D%?0RaQBelc@v-~ zL^btkW@Ftrryo?45wFEI(+o8o);L*_&YV3DnI#?!z3}Ge}s75n45tsGh*Q==t zSt8-;hJ{&~XD+RMaP%zlqRJAj;Dq%QQ29n_+QFy0%?`PecXuq?8fqrsD0O?g0wS*O z(D4NsllLr)JGg#eMXNCeot8vBeHmQe3K2GPe8O01E2y_Ud-<>GNFSJ+6i-hypbp++N=g z?#IgZEf&WkaO~vz>iXu>U22(->}L3%Ui1H=a4`QuSDzbb67p$)L1dW_GNGPT|Gs}a zFZ06M0j{9ZCEwy1BJ^uScBIFV9(2OcR}*MHF-W8f1Ybw!n-jrH81;&`XdaGL0x&R( z@fV?i8wOGy9i*>Gz>X)}pQmS-@ZmLE-apJ;9$Gu#`zzVICr;e659;whX&-VDl8OQh z1~Q?_n-&2S;#|}wgEJir78uVZ}0uohKRp^I=gbotg z>Qf(hILiI;kb6aTQr{*e?4)6n&_DS5e}}u)*sckoJp)OMlQ^N&D{IE%nJJ=4REw({ ziwQAKO&)reGp!f12#t)j9${LE#e@KuFc@r8+6owCVRp?BOjGJ9cg-BQ<4g*vez?84 zyt=);K?8T_nR<*?5=%qF`d;?l+>9I1aiaMpSi|gM7-+fa)L_CBx-dTK=pl?X!{9`d zi(z_@`72M%L!mK#K-y}g;}DA=+t085x7=7XLxIPb&NPk6R03gi)ncgd>F~B+EzW4hy!tO7xs{YQ-~N7vZ4a z=sWgruz)&G?ZiBOq-IqDy z36aC10BVqcEI_M(lJPQyE>SgjG%LVuRAnm)n);Fxs{_1Z&Ggagg4)amugq5yrkAs<*IC|e7j?UD4_|6>24vw zf&qnF7VV(q7GQt{=m5qb2vK3*RxqLNVghP*d4(3EFpZ4fKu}=zyO@3HPNT=pHF_j| zw46xL>)VpU_GKAxWKxB!izy68KQVIWzd8w>!QY)1%da|hB7 zGzd~)7;FXhxPE{;o$wN2=r94k%z?hWyx%cSY$SDvw)1}N&Hyqkb|Eq0$QvLk2Lt|1 z(rVBPRdg8k_xwg5yf4m!Lw_Rvx;OB5k~cZ;c7rc}NgOg{8$4fC)c; zkSLHT02Lfh5JaHR*Ux~$WE9@hs8D9`Mtfb1rn)jr98Kr^1?t6U0yEKgG}qhlDACRN z5rc*mZH1sYD)}m=cBCrNxO&+kg^x2_p#d5&X;WEcSySib?rDvlyTNcNA)#rh4DMw_ zP=V=jsUgri(O^Oe#iMc&T7e0tPKYAmu zAP%n#CZO{#BYl5)ql5j!rNxAZEQ7t5J?$714wfh6#Drbc1eequfC)o`3B6#4Jq{BJ zKNgv`F~K>&xUtAk#v-%u08BtsywLO?3M7W(Ck_h{?;@~mCSra~a0y%lYh1M%)z8!j@JALk~=L2jj?8`;aU|jhu}Fk4vw{2MYWy%< zLh$(imMZ-9_vE@M|?+z8?NN0j1njhXvq2P0Q#$)H9D=ej%9gQ+=kY&|2G1eTkWJAWpxRMuX0y_ww&6~q^AWVB4Gi>v| zL@7e)`Yj|Rg#1bJ!NnxH#a>{7q*vUeaf1-fWB35gs9%rPYuXr-m^@({s2ESnS&lOd zCR|@zOjwi=c|x4#6f^X|s|WMg+ZbVVN}PKF_Re9)6rNC!C)BhZ+$%L3<)q35Q@C@K zb3vv-Oz17bS?^-@!QixKmMdMymy5wQ?--Ko4*J8AgwPKPA}q=GA64 zieM)gHKPlU36w~zD&Z`|-f@k+!-98}&- z+*TMHwP{7hB4;$t&2g#g%S93CtG-UsEVq)d?IFe!n#J{>y}pb^!pvZZLL8dC1BG+R z6MDTvsM%R@xi^V^%sY`t{evMrGBs@iGC1gt8|=dOS&w(x3q*H7Eii$)&V9q#7b~gj zCt2jq*y!0Ma$epIE_W?Vm|o&u`k{$QI~rarV9_*Ji?Nrj0KmjGwBNtzm$b?ygazUP z{*pyuaEENk)@9v2HtgGt>ClXbzqyKq9>V_&Znx!RjYa0Zd`_`up{_1q7ovI7}#; za5fhU!+bIxPw!uG!1_Iy%wk-lK;2||Uq6?%4xskUwz}yKHaP2{hlICwpXW!`v*g9Y zg~J6`@EBpQR+nT7Wub~Xhrd$2eF7%zgdKRN1Vay;&$2N!aKb~)y^OKKrNn393BU(# zaP+A;ftXOrIHcnV=Fw4Oh-Mo{@baGSo9XTCRFB4W_<_<4rA+|E0gwQR4-?kG)k{e@ zB32qtpxe7?7CZ*y+uL!dRh|Hk5w=1%eT*Y~);F{h2cPe+S{K{S@XST|ziDO`+am{X_tzT4CbhZ%7%(|v6APVN?1 z%N}0$Z zNlU{>e1bizowV-iAF{$3OnCk}-iCj*y%>EqY46xx}1(*54fm=K@Z6Nn4-9|4F0 zWR)!crsGo6G`}e9G#(Kswbnlg3h%GKK_&YFX3wu6KWnLjm@{6IEa4WP z(w3OrX|C+vU93f7rbSYF7HgH+!}hNCL45kP)6_7zK?=Oh16HY2_H%E?l+f{lwpv4m zvn(WZLfLruaIOGOyd|-V2knVR6n;h)n-!bw!h(E`+OM{RZIPhzs;#?pAmP)kOWq8K z*0wRmn!3XxpfEo`u0P)Q;>TkV!GprKN;i)`VR*CxP+*r(ikPSrmNHfBsk;Vm&k;TW zfF&;N=l)x#_kSgFUM=TiRXeWWK|xBO38cQeysWcr@dyK>w^0pb_J3vrf0pKsa?$E^rKkJDryXhvF_r?T4{|-y&;DL=v$Aaex;>B3$KVZgX z)^3q7JA8b6)B*@z{8Z_ep8^IB6kb486=?#b$n%DQmd{Dc38AG8oRzmuLBv4;RyCHk zfx$^riSvru=0%+q0I^?|B~aF24M40HtgO_2L$S64Zb%e_b6~wi8RG4f+(As(W-`-3 zWXb=3+ihAQdF%@n(wU;ClYpNsmOxpBs1PgJvGaby79|BdCm+YQWue7}wfbLQafOe@ ztnH<60a5UB61$g|O_rD|;>*#T#?geEBt?@OENQ^aG1sj__vJ-$-qA23a*87g|n{e72Lb z6j_a5=_gbNE6r}rQAu)y^KFalKmo*fP!$#v$Q}$PxZ^%Cp(!&JTb{s)#Bt&v!6Sl$ z1$o5(%Va)i|K#jLE|j0t;Gnhhq&+6A63&2q%;AAEGb`&D=Y;@~8}Z(?cHqI|0$(Vd zh$2mh@h0Bk3~o?sA%XOOeO(ZNkg-C*JBtXiT_v7i^K_n-o)d)w(@uB(Uj8XvRQotq zC=t2Rmx!R+h#-7BJ;sCRjZi-o_aIy9Ooz5(07shY9b*gf1j-;aJ_HrwPcvoG5|rq5vvX z(owy%{12%)I9%xVeE+%b~uFTe}$6alN_Qn^_5h#{E zw60liIq*ojAQPy!#Dtw-f&l}H$xL0)oIrl>X#)CyhlDmy5RyR3-G&6z6DE!cY>1~V zf7ccuFsD25z_saDF6jtl(-JX#`|5&=ilS2n8h2=d%3RPO_UTy20yPFyDW-S9+uVo* z3m_yYA_BI_QV;>v6bU8~b`e4Pnl%x2Txq@&|NK8g5^U)N6MT0}zf9%g#Klz0{>Rz5 zE-8*PY51P@PcJ=u5p)MMLTelBv9QCYVm4;({~k8qJdlzPRnwb3yIf^cpoce8DXBIX zU}Mm*3l*zNBG|KhIU@DaBFO|WVFd(zS3Voq!8%zhk+7^^8Z)1UYh#2Q5Cu`CPcUwKW#S001gj2TxSx3D1xD@pK?eWV4#%2;l>ULP{euVd4{$Tx1j(Q=7nqq1IpmKKx{t zg-XCg#{0*^_hVQi;ZPvqB7TF{fL)ac1_bYvNA6^~tX<9ok1shYj0xt~C@U z5(*|nw}nLL_&?7m0SuH0upAv4kO@9YzJjX31pAT)Zc+Yi``!K*T`?A1IOXPjP@5?# z>%zRoL2!`QMgAZ|e;=w7NG32B>a|W7+(w$F$N3lW05aiLMPb4{iA-3y0DeG$zxFN? zig}*}OI5%)$i#wjKm!Ve^MJM6*wk~K#;Q6$W;GyTdQ2G*=FBEfQk*!JFtYQ2(R0O{ z5Iuo4&&Gr)USNnYUZspU5}XT)1S&bJ#25_Ao|S#^8=9un2#C`e4WgQhPoYliS_8t` zoC!+`Z0Lk0@{0(N3W1TJ{WM2ZYhVwq5`aN?Kp;W91|%h@L;wVd8fzj%G1myc2L+E3 z7!)?5{C+9!w`M{tEF(hJUy6;K%UdBr$huJqe0edHl|RF7fAE_|K>f}i3t{}dd}mxH zY$=6PVbwU~b6<0dhuit-NO(OTlkvcju%-h03IjV2 zfdSqkL@`5-wXRfl?Pq0hBH2JeJ_G1@N>s95^es`?2$_Bll<<8G$OrZW${{F{w0s;#A@q|(=r144d zdn#`?jwXN!Y~Gp~W>Exj8oo655~TDC4zizt1lYJxFwuaG>^T!&V?xI&%`H5oS>qTU zAJ4}u*+!^n9Wc$9uqvrQm!r)?>PF~<83dq9CQnaLCcNl}K~#b?b+{N3912)zg~REx zNR@NP;U4+@=ltf(2@WC>g7?^m73LKd0EMn?K7oQUp-=^Sm>Yz>A^|H_C^o#6!hrF2 zzJ@47s8CTNL^#1X`WV*=K_isnHPpXt{BpePW4%A%Y}3Tjwz^;yO{c>-Edz2n6Wn-M zos09qg}qY+`^wVGuy;pfAg+ChDX0oVDnfeC#*VYvD^z;QO2 zfPZnx0~o7>15Pa+2g(#{{9~g?C`TygGI-Ua`>tAG4{jN5Ex}cqs2{AS?#zG?`#I#-*7+$c0@d6-t$We|gvK<}+(;gDMh>CKwfx zFyVCnN0=Z$*sY6nAh?aMS4Bu5B7jYakdWFTBZ9YZ&Q-$4W*vi5VWkaVRDwYR!2#AN z{;Ccb2$(s<=up7UFc1^Q5QGU+^?W{X-2jn**E9{-$gsGy|K@@Z=Q0m%QYIv2f`Mla4wiFLJ9EV zyM2*hwGA4eMQ%7kz58{j#7rW3>_GA6{_-5wfFs1Zy^$^>udgxNNw&Af63 zK$t#oM%)R{V8YQ{r`ed09?u7^90qrUFq|40q3zKnR4lM5 zp`Ii0pan3P31hAk7#VFlzourK?EN{%fTFPw39?XFUP9He?5zVT!TQtzInECuf?Xz& z1U@oAP&s7#jv012+h>MHnrJo#TyyI^D>ql9g*j_9oPeE+*H`dKJPJ~&IUge%?aWfE zmp}*%g!8S+orVfOrq$PmK(_e_W!Pb>CGxUUVzDI>t}>bwu8+o65cN=^_ z{Oa>eC@dRxJ>zZM2|Bo6ciGYjMR;)w2_@mI5)#bk6*f!-*9jM@b8wyT4bxuEl67lNx+Td60Zs5f>Va{bJt3^g@lp)c3e@<~C7-Gl5%B z*8Th%yJ;1GXD7{igOco`P16U=ewvQRUki=N=@BEdol0XR??eE)v%nLK*Q zL#mt0so5EUQLHnW;9or(7UYMl2dqZ^9QbmLEdov_O;hlOKo^7?s{D&Oz@kbDsbY<0 zHLw1SK@@xXO=E_3?8UX$b$nz`j2@2rn>UpuJHl zC&2}^`8Bk2B|2exJf4h%K`d1PBPWfFrSaKS6fgx95^k+eGA4fE19Fy-y(Ba+CZsyO zUa6m-Cr)8RFAC^Cs}KpW!3@vNYbg5{1womDuT&Rt_4CP#u3LF54@Ua5Vf`5VHQ z+{~dqT~}p9PZ$%*&zR1zJzw@BL`ftAB*KaQPevJ9ZM80K~sl!b(}_ZOcim z6bWRY+YS>V&F(TV6>vA5O((o+;J|ePVPQ9CX&OmfLmUi(hSax@2V_Fc9$2eP81lfP ztp3_dLMU1os*7Ia5E(jT_ysAtlX(u~q{p=PG0p;sL4Yt%Mkca$Y(kQ7LPh9EMuaYg zTL*f$s2DNAY~Lp$kv!f!5<&W5R3;R0NSr~nL!YKjBaAUXN4se#0mb-me4ordC5i+y zYX#+>i~!U9ixR;mVBug?z{1#J_n}^T-FCaW>*{MF;Ip2K;ooC|)VenXjExW9UAJmXfwk+9m+e^e zk5Cd}9|+e?BEWzIl?nG%VzUXb9N=CZd2%a|u!5)e&}`KWSIl?+q}`1Py)%Ju04(gq z6Q*%~QuE?R^)NDwL-nYrRMBH}1rtgnl!?oZ0$<52W#4e%$pbHs_l+;0V3;MSAW)bF zkl=-YLZY7zNopWQ27)*bsq2^tuc>;x=Hw#bhN~7fcU3nIfP&^I2ouI0yYmb($>ZUP zMy__~hOrCM)FC1g_{M?R1?Bi0FJj_)Z>a>zXAT`BVasn|BB($B4i2&sSa;QVybyHH}5w9gQnG5VV(v zFESELNq9z3y=j)fQC6gnf;mIM@RMHh5R9 z-Fz%~;O~(vdW>l)6Rt5wLc{YqyqF~~DtvfzS6_vFhl~k8!T}D%UVg$f(eMal53-JX zLG8L5F`+?b0B+0Oibv>1B{6UKB4sm!&sqp3?HsKC!PPAL%+;6lKgnaxJ&x~=CBLh6Kx1Vj?tk)(Fvd4~MFWUJY?PZzBOFM6k(l;4%S7z?ma|W1OPvgr!oz!}{lS zMt%?>C(%pz@!6~g-C#ZW!7wW zpdv89$|jejtP}ppnQ-6^6ljKR<}sq-6L-ACP#F3yRS=48Pu3a}&?NG+e8qV^sdE4p zzyt$oIG1Qr&tVvFHLtLYa)deuk_%%(G@+mQ-UD~x@PXAlcA*g_jB_GGZ9}N&g3^o& zGecqEFvAlnL6|V)Oc-Xa2x%q6mn4h1~@x6|dzgodx$0aASNcbT$%B`w>|ga0YL%33Bg^n?*6nOSur$uuRy` zW1Jq+ZvvQrI)?6czBMhFka}anIC+Heubt$i5((hJ(@ZtNhZ`N?f|R)b6BP&&e9(|W z!V$#7FwifK0);tgy(GCn^AIZh-KhZ+V2|*~MS@Lsf&-h0{gda|2@?ni&V*^idK}sW z*JSFapwWbA#odu0M1T{A_N>K-h2N2w@Gg!gMS@$nI1ykRNhBZ~xPFLdYoqSK1AIu2$D{C#SUP3Rq zOmJq~xiu2p!!jpo-3$_Q%}POVfY~;%UdEJn-+^^P8BbVJ;Ym##(8%GL($FSm0%U?P zL7*^C-c@2@vwI9h3?btI9XklrJp7lsB)-FB(FFA##)16`aDY~?)N}FXeLM_Aeta_l zPl;}pJ~qt3An6M;4+En*fl9wKVVoEY=(=_8(v)U?n=9n;2@xC!$FqdEu3*C92(LSa zt2&amGMG>xfyKKtnR6aUw**xT;RrA=1bF2WL!qX5!y1k&NH>CIeWV_knL5Zmgh3}J z!i5d`V2cO_gp&dxsNvEZ10k;i@G_;F90=R(|I5IcOpS-yw=TA!8(X!ega3fv=z6P- zeBXvyP|5`KZ&}jx_Ae3 zm{)cgz=T{U018;P@)KnDIm!sre?HQXI)oE$l`ur~h5 zH1=KM*d7n<%0(7CAj3SvahTePuIm#Eg?>_@K#$bK5e({viJvk@!pkQ;fdv_k2| zv#zVo=(d}1Mvca-;F0ON(S6HnPbc8pXj$(tq1$uV z|A;{*bi#yg!2}fwJX>wU>X2b&~ z+#M&l{(*4OMF<#ULRT@hJ`)Nru5X=~38_ybGhqVpcuwLva3@R+Vh!ji0Qr!{1DFH_ zqe}r|!r^qF^Z34x5i-Hw?Bi*LgloX&%p&1%3?1CH%=*3w=L2D(fCAx5}~G7Q`vL*Jyfd)&-*Na@+NwB|7uHOg3~ zD;8Cc1rT0_gKcL+?TrPY0&f0yl&jVh4ouX~xsW@ndJK zT0aXYpzeWS047W+u{@Yy^DsCQre>rok=u3)v5*K{M$b-kdi%kN15Ka;$%HzZ?+MQ# zE&~5jLeC%qDT;mp1YV_7U5-u#?C^5DMy;=o>*ht?$5LiE)UBC!0+{Ipb0D0sl-o|YgUFdfk9=0 ziv;`i-yp&Xlka0vA_PT36=BvIsYHUygk^-$LkVo2-@V)MqSZmtY6b=Z2B>GZAiuISi%6<56(H&*U!mz| z)(0=~C|W=vHN4_fhJKc=A?I<76;cyZnozW2rU!O(0+Jp$qA+zSb;g7=i6v&jz`n!K z;D;m=ZraX4fB+%@30Z96Ksg1_37`U}8HB7qUP3}xA^{r~3Fj*HEDU7b!St|CeE<|P z?*a1#&VxYVj>$zs6EB4hut^SN&g&UgI1ocRMu_ha0!|^(nW)DCp{vwe01aY@|H^`- z>vvvwEdz5iju5fY>;o?F4h1Z3a@<-U2TlYsR7=K%dI<&uY&dVxLlIr0Rw7}~Y0&A1 zFM8@4BWMqmP#3*3mdzYi^^k@N6$u#iX5~A82qqLHo*?!{ z0`-4BsRPjCztnfPhZ12JHKRbP3H|#wH_twLip}ORc)6d2R zj1AvZ>QO>zIr8yArf*;ZfN;6>-sSGzT?39N-=?O5gcgpas`2)K+xnQt%t;VX_}}RRQ~2=8_Vv% z2(pfB*>C;JCY7{O?sUjqEQf^+@6Og33HbhV8L|SxHEaRFkYGE0C_QIFC1`w68$=c& z0h3tS*;1ahp4?{SPof-tl+9?+cfa^Yhi;-G?V0iK45=7o}dRAH?d4?LmJ$YSZ|h+ zoaCqIns(qONcBJ{NK@V7sE^(Jn(BIbO>Ld%vO2wHjMJ1-49!UQuTvXiptFmirF~a7 zWC7Nqq(LAT{DoIh zhu96UVOr_wf?)jG9O` zjt@y+Z+lp>CY0qH{itqiJXkBPNT>(m7nNC?*_~M~+=i!#tW=^7Jwn8r87}RZYI?AU zyvpVhhA9PF4fUlS$2tyk7sn~p^u$QX4mJLr$y5lYd7Np*8?0$Xh9FA1(mnej5Yj`_ z#R#XPVGE&adjzfHrD-m)LV0;4&FDn|`AP5g&p&?RA4dmdXUw#?(a}w_4z6wW>yfe- z-}DWv!~g!*a~7MnO`WRlNcIONGP?CK^mXmna2tDRIJeaolSy?y<;TUBB%CP%Rq9Lz zR0)!9eB%ABFyh__vv*(L-R@neo$}*Ew)6VRs_+35*@)Q<&D#rkN%PXC(%xrgx%Z?E zcoIRbANYf*#Vd8qFsu1pe4Iotg!(_t3U<8TuRPzQ@5^eKhU)g~*ZJr9f%^`N;@k@I z?ld>|R?a8`aSi|hDmn$*K;}7qVgt=;xMBmC(|`UexBq7Q&p&^!>KzW?g82^r+cre` z8+|MvdIyY<-z6&b*Q+q$5)=t2d=sZNRt5!rbL_b=rS1&`9=lnA(Bic@**d3zE^V#H zmT6k|c*)H*(3YUkOnom17!sf`_5y`#OL%DKDYi4|3FN7#F-Bw|=Y-P_4^0Xi0 zwTa|w#Yhhs#hv^==FaxlZ6iD51qy810ttd3y9V+h@Q3t6U;mim%r7dd1K2OR%pIbqd0RID-Z!7)9_P10kg`icV1|u=*fH zIkhiSj`D|? z8s<%si&O@-f&?%&KswMNFFf<4D?Xp4?kGrT*8rkf48hos1H^8>D{V7e8zTYM+K|9s z#*nxrg1!SQ^|S&Lj0o45fL@U_TVLrebU5>4Z^DEIOz807x)ZOYI1(GNB>@ZE8HU8g zoMTEkh2RK+Bxca(X&&O526xokW{2<90ShSfDe#T2L-6w)>VQu<)Gp0q@Py+bOk)#o zXCIfb3$;&a?ixf9w;rVRjFr8bUyFZx7kne|(3mV3SpSuQLHBNB0+nPkBJj%g9fznx_Fi!bdR8x8g74WNfS zKTI8%Q1}F2MZy_UP_+*YN7RG_lyMffR66XK1TqU5Y}&C!;8R7#ZG)+0sM!P=sPI(n zD3zec;dSN10x1zxcZ*TL(hh!rUoy#iui^&t&m~l@j2?&urDis*{ReW`nnY0DAwpW2 z1W_+dkJKROC`OqGVSo`yQBny6{CQIP!G3;{+%86Cz-OPZ;w*Pk*KqRVmG?DUCUj(A=)6 z>#!P3p&yjIv{>^pU;#NsH>4@K+E0sj#Di%Woj}1kjw$F)N}(Q>iSEa-X+Y`5sZYQH z&C%^ew8v?kyPHwTMaCnvE-(|KM^|8^7=;PG2BlyVIE$Q30^0<({_%7hp?LBfHMuq^ih0f=h56Dh{|L@;`;(Zg$ELP!}2IiVxjgm4Z6uGa(TTKT{? zBVgZx?o&h(h24RiY&m%V@~4b~c*a4zrZU5jiNMm^t&(LD(m3&fR%BWXD|X^n5a_0i z%S*E)5?0vI_aG!}6A9*b40D!PWLZ0ave_`O5b$xC2tYzrAYsG=$n;xx#AO4U?xyLfudye2R<{wovTOh zrggWO83YV+__XW}uBitmyFPnaF;UtsBV(aUY4C^~l1HrILIM<$a|7On>m)|s5FtW9U!_>!#}lG9@C6sKg6}$+PY9AsIP8u~fdnnoNyPt5q<~HBsls~; z7AQM8pfP_cqlfNeFV-r*0Tg8r&KW-+37ZltSn9yep%GK0FH6X}BE*u0hUsrnDo%a# z#q{K4I4JI0;Kg?rCPMjoEYLW_*?3?;U>Oj!ZIu#L3<4&B0;}gK4Cn|nBdd074i#KN z#Qw}9NMY5$fr!Az{|K$K2|%$T!H8hU6Qx4NgVgH^KEa4k`h;;CJ6M~gNK6P2X99@{ zk&k>CfsYot_12DRG0j`a$)f{<$F_}Efq>cr9hbTBDJAFIX^Ciq0~};1_#G)%7gjaNC9C5LF2oe;He@CY za>}fI8wS{frVdTxIgL;f;dtUx=(;^-xv^lGWkg6Y;p`Lq*ojLx-j?+?g~Y-iAz#i^;# z*kv9F5#Tsjhf1LE7vwAs!~`LN$%l_H2^L4RktD4VLdQIqCHzaAXS|8=#3T^WCSINE zAQshKD?VX=I*Pv9jH@cs6DfxcGq1KXmmq7BZY{L6=zWu(WYah#{7xpb<=Yxa8{W+K zSd0Z&unxRCZcrD2LMB3W?wm%}AK!)?n>~XJd;=F{q*(a@d@Mp^}w-u~HKdqTC zj? zeGTfBZG8xpdY53Js58;A>#0%6YNPfCPp zhg5xqELmp-d>4&r*%jZUreS1ekFkY1=GomK3a#cVNT1t&!AD;+YFbD2zxl zrn(--FebN0$bg4a>Nuz1nlwi=UUET+T_;L?LqEuiY{dE&yRnsOkj}kf&lbJ|M6o zjr5cx6n1P`sxBaOnEIrO68d2J7;#Dfx4e(oLL35!V&oAsMO&vhU02FgsfoF8WGaA71UI6#5O;9XjRiwQ@O;OK6rzGU1vkuM^Mgaj zqZ&z~=zSVGFv0H|PO~ud4-5q1!E(98FZVhHJeJ}(j2$ui9e`B|HFF5Ws|H-vE*5`j;I7fPmLu(NZ{tYno#mTo^*z?shjDB#f*s2Mw?_ ze$w49t9{QEkicRltfh?(KAH!nmISOzf7SeZ>k+EDZvU@{yV_4DP~m`D8s4AD?4}Vx zn<>d`R4_70Lw{AKg=}V(DK8>^+IWPG=JDP-Z*$ghBlXEAXKgt)oa`66WhsgDjwdo`}bO7;*F07Yc;JC;UQ8SU}&Nw61s9 z?YX~0Kj3T@t+U8!V9Q#yeRE&9@(>DO!aV9>@gQuwFo=KJ9 z&Y<81DQC_hwO%7uM1O}6obN)l_t_?7i(m!)Iu-8MJVx(;glczM?jDKaScM2Uvg#;# zuyV(^-wG4hCk*Tput1$J#v-_v2M)8UypmFzlMEWzLRxxC6%UyCPUBWnrosVUfutPC zUSYJ_N6jE?;|8mDEV(j6vQa*!O0gTaQG=-X5p-0QpaKp2x#CeH2J;AN5?+f*m&J`qB8UYJXfQjE?NYi_z!IlZrC`fR89VpdaVh5;W zBjJF8P=g9w|Ii_F01Segrlv`Yue)U;Cbam`BiVyJNB}jB)JYx?Jvcw198a#fImLkB z(4(>+DC+#a@?MZ|IJ~{lVtKz7%-4jxba0^yEr2zbZWT(A&NV`Wsuhb{us~A#^XaD2 z-dDtgw9?>z+a-8RET8asG|P1sPn6pI>|2<8&BpA#GDHKDn3c148UaW<1+7O5|i2~q{W z212JMp~x;W6R`Y74P?pRfyn@2bg&Osnc!2;at`$lm`FQi!VpP!%N5z499@Jb~IYGi3Iwvq^@Z3jKokej&6P;%9p3KQ0V#k{V%Fdl3x4g@2mlnC3X z0oz1`XhbN}lVy7Zt!prhW*~Ux!LowC03o(Y1kV4YB2A?6zE(qO0+8o+HMUd5m&dqZ2_$ueTF%!bKG zum!@zf{QgG=$^O3`smO!>a={X01W{RsO&e+Kp4?xQHZdEs?&wKq*06%7gp=^Gn7PV zTPcTNX)+{Wy|rnhWf!qytjhryh+em(tFgncKYJ!ZU)MGA2}*>_1n^ti6%0Amn6rzH znc#g@`BsC6==tM7xPjVMk#M)|V&H|-iW?s2h62HEyf&&VetT(?mb?qzA>-%<)KI46 zd4(KXP=Od zA*_)c6)0Q^8ZcYXa2||sHP)}n`lML^fehhzCtQf>CdHd{UE@lM68^3utO8TvWKdA( z&6i~TB{#a5MVJ~3MJ3ex{u08p#$LN(b z131J4x37-BMTBLg>8#?hjZzwetI_SEP7B{9tqwuFC;BpAq!?4$sBK~zh9>vLszQH` zBOMX#vO%EPBuKE8EIQUNZdoN1B+w^WN}nmgL>M>1V?~r3b*NNTB&)Ck7LL+Q2pzBT z_(vN}zo83H;{m0FScGj$ih^2YErar~mu9_7!GDVw00i#VU>;#Mk5FLIc%WrM{Q@kB zbSp`Cz~TncZ?t4q7(hMy5Az_yls~cg1Vcjh38+-9`tT$cbQ+@`9!>6)9H&Yy0MG~c`>E68Ko zA1`MX7ffrtKg))4dVfjp-o1N6VauC$dLPqSZ(hIQl|K7{mMgoFUA_D1-A_N!-{7uH zZ{J0>y|h5rA_p4>P*>i|7Kn^#r4GU#W5foIUx;pK1Ih!)DhaN6g))i|D8MR!3f%;O z3XN~28$!`0a=JtaZBmW9_Lz1bT%eiDE~*T#ID|6{!dZj%555*N%4l@)WEMe?(3}?S zyRr@ZRc8_iCj~5BROf9T;W7{t7Lz3utfeqbK}-UFf7C@5Jix3>s9#~)|I_NqaJzHt zkWPmKE(Hf61<(Kq#snYdxvfBj(1tGgb~BhJ)~DkZ37jJAL59mRqu_fX0>FGlW~Z^m zgRRu4rbLruHzXe?L=S@)1q2VfrkjWTvkGTbVS@SuqQar^6g!L!H8}I0#;*JkGwVBA z{`lfWx|A+l3h`O<#g9K?rm5$zkBIQaOY9A6Uiew)J6hHNNYw6?A ztM)2)tF&!(l_cCE#$7nh* zk=PSquzaPmfR(3N%e(QE$7tzwRMRAZDwMzk;ek!c1pL&10{H|>C#X+I>Ft5eP(p=~ zTUI>=kN(=gyciQce;s<|BwIp?mJp&O*2F| z?f-g>2y-^k30j^$q3P)pTE9B!g%qFu@C16w3+V^a<42DkJ%0N1>0>+sJ%0R{p5Z4S zi|A?jyL`)!Jt;(c=Jz`PiM$i<<;f4Pj`c)TYeE4s^W z^MY#A9@oyMGF5w4Q&p%qripF{;6Arhxgaafe$#hDYOt-rg|ck%U7S^_K5a`DI>k(SAe| z7y6ek^$)PYFHj(4*$!VqPfI<{H(9_D?*H)k$>XPBf)zPcRb0VL3q371W0EJaY~`S4 z9W+Lj4V~bRasp-TcMyuTs4+R3u|WCq2e<$N4<8PvpeJ9N@-JF?(}gy@|RO0;B#|p`O2U8V;L|7Md4aG*6V6YpE6If zo`!8`yE|&H1A;waNaw+xJ7f~psgv}cn2>G4)+fBGfrLGLf@&{JKo2xrUH&z!044EgR}&}yVgPWp-@kgN@~CQK|(-!>z zTS%x-yAk~(B>|cOjDlZ6Jzxr{l3bvYO-T=fne=4^7+dF!EV+#spd`5=tf% z@q+fzfie?#^$P960xc1Dg$pRXWD)X2+Yi3{no2K3Tf~be^fXI>V0!Z4%RhYihi|aO z@K7dT1EBd1e}*S;m%YL(XD4_o`TuPRK@2Fa=P#%lbR&d(lTE{(#Or6g1xo44q&Pp#H~U8qe|#T+17146s&ELHwV6pxEFNNoI2MNhw~$;!Mgd%Cb`3p3CiH}QMTk)FoR}aT7@7(; zA$ny3W@ZA2P%vSgKgfgs$b&O!?EBRB%Zu#yVeC<|O-~30HvQW_f0iNe=|6u)bjV0} zx)J$@fBW4hpZxCYm-dh#fp4?$sJH~|%R-E&|I1rXc?;XZB-~tj-Q#=r$6UPhEI&aj zOf=uOm#3E=(@!cl_Bz|zexvK}Asizq$r%EpO{=+!V{fA#aj30horh%F8-ro$7rXxb6OdVe{lMO6e@|Y*Uf>1% zLSz-hDEx3w+5Fh8>m~R87wL$QITPY4WN)xGkM328S8qgs%Gf>{SHon2TeufKvE>MH$*gK zX(VXCYy(v>6BGzC^lX|hjO8h}#DvvZdaajVfBE_6UwrZ4hetoq)@T1pOgL|?UBb(6 z{_y#ypMUu^IRR|^0hZw912BPe4-e?>gD<}RhEfcleg5?qpM6R99z6Ks)6XBk#L|(0 z3A*Pu*sQFr~lX){Uqq)3=V0-*>MUH>4j5S$L-Q}9L3UEcN; z&NZ%$6XO`{IKPUL_WOT_n`g%B1q`*gw6A)Pe+^#SEQo!Wncdkb)vBsRTINsw53jEM zuf+~_h-6G9N+2Y#9l!&!VO6$+D_R1rN>BKxeT+#Wq_FtRf1nGbP=q4Dc>iMNf>%fo z1f!+v&tgXSblM}F44xC`WH2}(=_$FOJ*`0!^afc#1;babHyBc352OHA;FLKU*JwN^w{2XwAEMqNI^;Z(rt{^@M5-#H^@k#ZxL+X;1Com$Vpm=?O|n zC{MB=`659Jdd)cjbO1Hs>@V7K_~;bl7^P#4vdvEycW~=P!U-d21Mlhh@Eb`lkd~0n z-!nJ7CoRIAr-@L)S#NNWIKAN$?RvcnVu7c4&^vofh9{@b9ykG|uPzmy#>1206RZd6 z=FBR&pop|r$TT!7B=A?JK@tY{sYBE-n^VU_6v28BfD&>>aIXB@+gVNtkx&A@J&g>U zpu%kDxs!F1hY5-kR;+m_^*NyZH@f{J@*Q;R#!&~hgO<1<(zR&SN5)Z?RDrI2Y}DV` z#3aYIS=bIOqe-05tQ~ia`q5Fn-Z(P4$HW}%?xE4LO}Gq1?X8eup==uj)JY zv;@jEt+x8#{;zMn;VFNjK4GU%B}W3ldV;0|f#?aZU*Jw4Mz|18*civSHli|3kt{`pTn1Brim16b&IY}U*Tm}$pQ-7S7z~E57rg<6X%ipB z;VLJbeo0-N3uuXw`)uz?drF~kxwZgbBX>q37lfBx1u66dMG9(KQzUBp=zdGfpl_W!=GSb1tu+nyoBUCXxSun#Gjz0QK#Le9>OuJp-DG3jibI@a6+>NPH2D=jCRX7>@X)7#}lhrYgwj6O~JxpOS`tJn4sL2 zD$c!GPpGyXDr;)7WWIq3e$^KKKj;Z%2bVzG4p^~3P@DioVMR}HuqZtNNOR2Rcz7gE zM7nD^Vqx)~4&2x9K+-vL!nY{PXyPDAESkC0>cbeEz)c^qL{DcY!{;DO20bS(IKhJy z(mn{Wq;MmYxnLit=`+}`F!N_=koo~)AACRagIJRS)C5ryT-WC*;)D{w32UwdLG+lZ z6gFT4?N`Y3Q2CHh=m}bLmtHFXExcvh;DkxTXqe`RJc}ceMk7e$u2BPwuH80hXVTKG zQ&TsgEp%<0Z3mO|gcVNcAJG}4!*m}yNU$by+(rw`3W=YUMzcXnb67!lCAbNmtH7PoT0RwjByis5MN>f(Ahc zx<)(SD%%5_EF$gV32R^N!1{iRv`VE_Hi1>`PuRPD`?ptHOK*AJȌoRICv8*~Rf z0Z`PSIf3|q)(t(Oh=HP|K%A~{g6A#Gb=!fZMD&F0>x;PEh!!3{_Q<>V3_pQz0(uw= z+aV;|;W3E^;$wUYUQg*dFAS4yZi6j6Adn+o^RBEeFe#)#GEY7K-VbNO3DkUr6)fj(1nebV0oKEWIblK#zGaiGU{Bg4 zjY&@c34j^K0&+W&C16{-Y=;)Le}2k3C-iOM1hF32cCae`rL8};UvK|jNvqoLpt*x3 z>j&!u)?2NEH|rvMvV)!N&=QdIMHvm67!)Uf4}fhkP?8IND<^~@G=$NDbp-8gOqnAt zN9HnNW1e=mz1~?k?DYyxASBAi_!N@;;0=1BCnOu07?SV=0<=Q-YQD?W1SNc%w1fj1eL)K`LAc% z#|Y#t63By@8f5&pjYDFFCV3IwL9$80xq4FuGhrQfkI125_Tgi!6+TA03vIzPZE_)W z@su?^!DvuJX&SH{q$kq0bgRb^tCnm4ctD50k)#x`G*3Wc!P2EHS^77K!i|<|DM|lU zaVTBK;%2F(&dE`o(@$B~4)Sx>x&*QOd9@_9D%G#tzVtV={?BcfKJU(R`hE51)iuCT zOO}?lJg;;ePV9J!=7e;E5(qknEUnsh_#%1&mh-LtggBW?#3}F4%5zS@bYu@jU*u9r zaT+au6Y(X=15*>}iX4Iy(B;USK*mCk`y#{a^aTA4y#e<-P*>zx7KHJ42my~(i$1JP zCPBLhB9>JH;%OetGT)yE^Q&9p1R^<1ZH;T=*-SG+94kr)Zaxzwln~v;Nw%%0A&&F7 zyBQ}0ld^}Eag1>IsSqgiCK|QFI`uJjA=Qn8rjlxn@yP6t%&rMi=o}hE6Vex1JFMk> zkp~8y?N9PM&=*o;EW$w+ba6-trE%L;FH+h6?B=CHiiggPm{L{Nr<3KhU zfg{M{EFb!;Ibl*vX#jtMax~KTqT}vg+Z}VlQm5Uq>~4#sckRyD9Q}<5;qOEYJdUYB zr!h8(F6@{#0YjA9??}WQxR*}!DR|0kxABTN0neiYX$pvuJTCxo7fx41b5$zC)1atCdAfo)7>>{7&KCN8D} zn-1g4hijyiyT*ErB~?DE&b>7^;7|mSzx6_;o*9Lhb?(w zNxz`d6<|^+4Jp0G`{QBQ@OadgfQm}ZK3vmMX}{P~e5XG-ZTMJ_u5s7e!x#Dvb-<;(lbZyX z0gQIj)pygJutiVMw!;=q03m2S!Qq|agapBj42g`#yr(XehJKI)e&9O3>vBv}6!H8e zfET5-v6vF>$)9+7Iyf1)?0s9G8k7(PbmYm>dxO-&saUB9-r<+|2yxZ5T&chRMTFB*`%Jf=speHTw5kC_%<2WCE2Smjk?zyAp&Tc$A7f z6DM4z61D0&)SQ*0F~_+Qzz7`Y49IIlLUapqpxbahw?FtQUWG#PO~JBw2TFkXaOt~% zgEe2sm0Sd;`)FDJs>1L23@q#L@`DILB(ZENc^AOpuz}~HAfggFQZLWy#7_Za=Yr-P zP|sPbu0#3z2vn+Ku0JvnRO7Q@DvvB`mxqKCJo!P>76BZ`qb9gxxE=%%bKn*rpM(ilW6>8v$RXdlhU0qDv~c6? zIwvHni~v1B+YV(;pjGJ!4i;dZa{@3=3Q~yE&<%VyjQwDaq)0B$p-^Oe#=ovk@suzE zf5`p>C&AA|mRyL`>wv$|!7yc_NVcX2z}W~}i4026YErCSj-o0&rorvhzadVT#YvXM zK!Puhufu|CseRx6m$$R+Z5!9t_7uTMX9~0^f*?SEK?c~!eCWmrf~Unyu zH#ABFI`X*{;S(061ZCz_Mgp=SbZMOBFA`Pg4`RUc?M=RH=|C`nZT1Xx>g_~&3bmvbTznNExzwMgeH$|VbrCSA*D zGh_M+W0R43WWeNoVAADS3X`Gx(9bb}wrjcdGPgIFC)89Vz36cg;xfoTj6cH$wez)< zEcrV_s-@J+NFFG8cbpXGRpC9e0ZS0N#}x9m?Hp1T@_4YkL`ty`5la1>IsLacJCnlt zd8I`$6M%%jX(1*|bVJeN^!@H`0o~Cedpx-K^8NA7 zB=d*6FE}gci|ppuVGqsa_HO8&Cp2os=2RM8hN(14j{-3cMrK8+(4`RhelLFM8v9}-4t zwKy?C0D_9Ff~?>Io!~RJ;pI~7=cxf;<%ptdQiG9B*6)Yzg5 z5E+=F3taH4;vjgm?u0ayiL+mA(1fs0F#$Yi`Gih9L7`CjYjr+O3X@r4g2O~hIsS^` z>i9Hkt!YXWxUZoM7a1X6!K|qi3xQ{XZRH2MWGpTsdy6+kSn4h zPjc6*HYgKxDuekvr`4vmH_!Z9?Z6C7fSPt1)vLA$AiEyBIQwtP<#Niaya=1F{zfE6 zI4{z*Q8$h?p(0rqB`vIKN+k@7>fAepHY6sINr-%cEtm#c;Vg2d0<*Ub1th2MGQrp} zeTAM;2$hFR?^M(%3^3umL$fvwu4(1N)Jzeg7#lx-Mt}moEhi5vpq5`~4SThiPIF_eFhQUpZWo#R&?Lzo!Q@S2XUkP& z5~P0u6ebj%&gav7Zkzx?^;O3}!Y7C*QZ!D!jHh3}Oc7nW+b;JM6yo^O0_%bwTYn{e zx@3K#g+pwMFZhbU)@Qr`FQW%3Y{x0zKT-7U_H$p;&wbaADWxhnR8f_OG8X%U78%Bo zONgmZ_#&oUKq_{}dH0GOuG-m-QOJ5m0SOsiA{AXw-)i8%SZs%ohW;(y_Rw22v3tu+Lb5P5`|Wg_tTvM3Q`Ct--9O$5lz z>bTY*lg}N|g2|5y*odULyZ;5z7!m&~T{|s#?%%^i}Y$HqvDuD%D^7E-7 zRR}6zuy)TRC=);g=CU;y)?spw1XU+TYpK7M8Gfdidka2Qg~m_xx31*NZu9b6OuXTL z;A|Zw?v+iDFx`^q0oo_yG@~*#PA*63lbWE2CF9GaDq5P~_K5%z$Ob7BR2h~?u563z z>qG@(3rL_K7fTq0Ts?GVNH7YC>83NDgn$B|Ffz{i46dXI2SYK&(Y4;TjY8G^u25Z} zo=^mZ=kx@BR#&@`(3^V8ilWkOGB*B%ga{%|5i+AyLX0X6?YamFw&r9%J&z!ICY}Hw z{C!SLP$b;kBEj|$TO$sPT zk<3zJ31h(;B&2r`eqgy7CwWdtn9`-zJOu;-32hX1uhb?$FbYD2xQKng8v6t+5f@CC z35739*5n*GtER^0Y!&jPYZPD)41Ig8Jg_FgDB!z+7+^vQCj4ALVba$1)+ksp3PstK zh8ta>bL7M)4NGYN4VVN4f>s-LL=tSvCWP0_GHjxBYZCfIfRKl_J=+x4GRQr_1PL&n zi%$SlfrQI{PanR2`#PQL8&+R#(YjkS6IZ9Esx#y2(q+y~jMYo16#vtgFAlDl33T-K z0-zWW&fWX3z5hC)le}66q3;uxtR0D2!wkV8*OESbr&K6|Q&`m|Hx}s)lnMTXh6Nz- zd^Gfife4BO`Yww=gldTi-gA?~B}A(6pEA3bc85^LrIce*%Pe(er7GRP)u)+^f=@8W zg&B}w!Vf1iKmu{$a!hF@DGv9HPe@R6m{#I!HOh$c6}3CaE@+}e*dn1To4Kf9;1&^S zkug3^xkeccTtZ;N$FT+(qksY`4;D`+yWz%wM&Cx$0${Wu@!wS6xuUv7bM!B{l=aj&d4q;n#KEFA?9i<_pFeBxhkKP>|_*Lyj2l6Fd_dT1wK^ zusd4l{J;d@poIkZ1SUe+g#6IW>VB8_E6~JzKAxZ|C#2G)wy0FDWET;uw(toolZIlL zB)%H}EhbMG31dP+a``cNa$2VZE`e(fB}tJ`Oh3ip1i96_N$RD+gh-Z45G_^b1M(!7 z4+>210R>|-?ARt`mWw(LL0JRyw@R*|44VdqC>b7cZl%0ikw?-M`-jxh>JlS^Vk{_uVM zbvpIL$6Q)yD4Ztw%9NA%V+0gX43JGii#)jt=ifuolpafuxAa_)wMC%Q^s31}C!l@V!qF@wqG(D(K@bbZT?xK)r zkg*j|7)cL>k!b}P|95?sR4FPCx1sVd)>Xcd!W0GXZMUC#V|q0fA8{t4Q$`s7^<|yY9HkCh}`=-cni;#}dX=aY#&} z*YbgI0+0}{j~&hD$ul-VaU(?7twLh(TBan-R!mLqnPA=4C*)E{42es68iI^zuiMhL z@-WIA;|D=UEIXSa_3uz1bPoa(a{gx+rh;{8ls?7POpfif+9j#2$Tw>>jfqx{y8S7PpCW90s2`wOkv-l!GJ1M9+(NEypXH2v^UU(0yQ3U zc+nV>P$@C#Q%VR4Hsc|PvO71b_yCEn0$bTI})s1JpP`obBRu`LG^00(0= zo{)!DNYnQT8cyi>gcOthb1kETg!6Et+XwFxVyA$@D3u4JNZ_qhY5Csm7mBXp^Pva` zA|(Qwg#K&A0*bi8fPKURt~LyQR+cjpN|jn+H)}KDFWI4h`nm&%K&#K?ex16<&pn@D z&F$0=NXZLS9*ik4+`*NNP4mU%;%JY??^5jGa7F`mU*qUNkH|Z)E)a6ue|2(pdUW#s z&ECsnkmT?}nef}*+lvdl%kdr!4c^~>^>QDHiuaF>i7Xqt2#^L8?{_NL1v+WT zPzZrUTP*9PGL#4!IfboH5FqdYOXq=lmI=^T^$DsHzx6sxi%K;fSoLr#& zobZB#>6Bao7cy=d7*;|qfeOOqZRmfD&!3ZIKUF9w7gVEQ6BGxrY^lLiImk_)2`(@} z=Zp+jPSgncK}Lx&;=2I~DfnAEy(Px}N^9pDq#P&Mnv&1UlnO!)eJ>k|?% zVG1Tx>5^6i6c873O_iCdIs0w@*V8L%K{&j6z5kMqhy;8z<9Nd95$gn{fBy*6&(OIj z|7XAMzdGB&gd;lb_1>%F{rwY47;tz+&m3%W6l^7eB0+RNC_nK#TLo8oqhKBFTVchp zwqvtA8+p`wN^-gXBol_>300H&D5Jk$K7|BVO~ks7sPf>l(Dsm7j|59dK>t2z-5;(v zB*7(&=aqK`7%IA(_cw{ZG?2NURl*Lw#i`{Wb$UY%VC6MQ42 z@4F<5A_;?Bz_~;LZxog#Gr>n5d96|3lFZLsoq(xeCUkpGLH{eC(CqXHwVPyIc~Fn= zMC~+BM}GM4D0eQMWGv~Yz%BtMJWIJDFG+_*L|~gRMVHNddYf#=5GSmgBzdBojg1Bq zXc>^Jf|LoVV-!@L34(-Ns}45v;RI=79NR2zcpxD9x`Xvh(4m}h1tKtQvQyWo_X#(` zgkd5?RQOrhrz;JqalWk~?zbp`#n5w6dnG6mGO0EU;zz{sgi5b*MX(7x2JuVVC!B!| z$^;?8hx1=HE&Zq-zkUDqdAh(TVqpSlo*P4R*c;nqrYejj($!>wHzz3J7boNsP7zbQ zz+R88&Q6X`u)hB(CcFd_em#Bt;_w7MVeifRzy#mhK#{PGI6lJz5=toMa4XOviBLkZ zWG0kO;K`O@+b622*(cbVnGmP|Bn&d)j`WOAs5_>5LPfjI(s~#Ptxj-;D-UCeDx{^e zaK2u%RFHr+`{YTRASPjqqyyusnos#&l5i)?qGE9zc|1ggQPLXZfK3$*cfh7iCnhOe zk()w>k6M#=s}5PK@{r);6B~_pz~JZ(6Y`!5ot;{Nq<+8982M>k@zjk5*Ki=m-Jd|!kCV@-E$T~nF^IezHvB^=kGME5h z1DBxT=pyovd?cFyBKRMs@2amypn@zdU5-$5=!6${%a&1q1l^SoMW~9hRdtvewF%ac zq>`;8BeU+f$cZ4~qr?-eO@_xxD;g#P5}$xvE#s=(9WLwh{*6|6dnGyl({HkjbuoIt<&1TE>H%ivJ?lPf96#}IRfjFDs zCzRMm%D^&pE-f0)kWtiiTAxxvK$?$?$nc(~Bs)H!Ny=JB1c5UWw?yvKgNQL+yxH5| z#u%xpaDwMw@BhZ{a(FCEINg8QVZz<2KM2aJHN$gE1UyFD>Q10gZ<&C<46O^?Z-GJ+ zl{Ks>DW6x8T(JQWymTI@J`*~#R1WwAE^n$Di9t%67O_t#SY;atmdiVk@NrT&2`O+t z=Sh;wWZ!AZV+xZeOaci=_;@>oYoEt3s=Uy8ffYNya0B7fHo@}FDfhO2jj^xvZcoJX4rT5cd+=Ns zK|3hLND!cCngm+0C=mM9hUl}x$?6l>1xv47x(QSW@r1L#*dWMG=8)ZdD}T`M8x8fd zZr!25o`3N4oFOKdV#x&nJM><(jWrlSRyFyAuJj~sV?4sq{{D-%GdbFO@s^k%1xxA- za^C50+!pzk+ah1kd9T0*v^sKIBpM+PC=PM*`o(MV6VyX`lWVnMN4JE#pa=hc?#}ia_K|f4ltBa8kN4PSZ za5Mp67$IL(I+6?aBr(RoX^XcWKtkf1Ab|v2g%g7|acOze!r+H9BNTochyV#7#zudd zKVuV0MS?ZhB$&3f2}W%KJOVb7G%&JFh}s5dOhj}EEN(Xr$hJ!u_rane3Z zuii?ZC3;%=ES((6FsO^8H?R2>r*!WaH;1Rk^qv?=_4{U_^#=6`T_%VrP*Jl)xZf-> zxgA46LI)+MVPc<9lLM(tVmLz~>-L7Qkdsv?Y-AGHCn$47`0Oy}6x6Rf=y(Se1sQFb z+TJj_2nlj2O*p_v=>=~tN49WwGU7`{BIAlZ5-xAeZYE6FBydv0k2g~gA^sew@Iw88 zq+A#Q2|B`IShj*XaI@&kjK>)Xa*s!FO)U#hvu!4fEe?0)PvB<*6E2@~38P7H1ti(= zD&9lu4v_D2IS^koPM#?Tsqm~r(3Xasy&t4kYG)AjtXFGz#w0`y6h`KnZ`T?0pu>a< zFPUrJ&Sv<+6npvQ$B)OyA71O48Xq3jkcT5lTCy?)Di78;HVWn0`>TsE=ou@H-s6Z4 z=qQXTZ1-)(OK=+oRO1c!wWJGvfVa?JdUiEiY09-238)>&*vOViXqNKT`4|C8fkNZ9 zOmI+5UdXaLxDnmT9<46*7y8av)g%O;z#Sfn5Rxt%gfP{;4E0Y?)mb*l>NKmnOo)*n zIsPs=j(+)xpfEH4q+n}h5wFVNB!Cp$z;D91E2N6i_3FSGUpi-9rA`IFD8#*lM z-{MD&=KLuB9I*)|1Q>;Q5JwZnm)ft{H$k7_cMaE1>!GzsNa6ujkX{b(L0t)nW zN>G@jQf=@^xEW8fxY`gb!WcJ6ka071ebjD)GJ z0KOg)rc#A~EFeI)>9}PR;&6g=+u{)83{2=FK##QMpf(|FSg?vu(6&hFgB{d&06HE) zsQ?YxgSan3Q7QdnkI?FpFv}PVP@KNC7;NbDH?dF9`U39(1T?y@brvJ-qsvSnA`lQr z!~{-h2qdikQ6@xCK=G@yl})Rq7=^k-$@PO7F9HD|DbJnZ&%+Aj5XJMcEj$sl;6-B` zI!tJ6t#_8>6RL*VE%5g&g$Z@TiG^1{!fnz5!hf*)EKsO5mT;xc@>C2%5$kWga%1#f={4F#U|KH)M<4Pn;=7= zW|{ND9aJ$OLi_+qdke~!>GV%i{xYwfdy%xg&DLPg* z$_^%BkO&>&6=1^ulUK}IyRzm4<}_S46g4fOer`A+0{S*DXS6UBlnLU;`?|lwmUA3vE;69kU6@! zn-dV|I?6P_id$?n`vFlwzl3ULCF8bC1Q!_Y|7(E-?-Z^?|9@vZp$1mUEYqJ#Y+MBi z(%VSqEFdVrc#kGR1i6M{P-J3y{T8K10vB$=f+E58Wk|NG=}m-$=VWau@_#{Y4QdUw z&quWtVhP*$qDB}KZxf8h)LI>uG@M{F=Zqp@C=E>Np^(arE?B{YTV;ZJ1yK}XEL;5+ zgxJF#!SA)k(2cpqafCP*+pJe@fP@I~C&oYs7@;(rpn$grj&TrDkdaT&WM-g3e_Z7G z`T4tFX8+K_v&65zDHCF$AQtJ$rNy$sUz9rz0`S(+hpPcyV2zh2<#Vs~+8h)Lnr+Qz zMYFIl5>Ftl>gD|kFK3k0Bv;pVW=lfC4Uixt2)e~FZJ%#D;l(Qf)OD7z4>l$VnvA(? zxN|d5L2S&L^0Q(g#qk6%!GZ~8HCS=*M3}HX2o$)F0=GKKH85O3={rOwLAou%SX?p^ zF7qPq|E!FcKWH)G_LFFw{45b{(XqwWm?Dgiv@C-iw}j>)yhw{Ek($HhR2w)nRorN6 z6H;K|ez_&1j;A9n?kIHkU6${Q z>0-V9?zLXr-SvVA63A=p>R!D7eL-=kw98qpX#A4^0ym+3D>=6UJ_q?2B}r=(Pk>=4GK;#gM&x27 zgdS@sYi2UySo%k-42BWpyB*Wl=v5j*V*_FWa0)1H0SQKwn2xdz77Q5$wit=|86aeA z86mCmax~N5Q6xycAs|7-y_%T`iS7TgHEA>L$0&)`nP)?c56KT654Vq)2NH6iq0& z>X71GRn)ina*{$_8py``5Kj2SRfpIo^x_E-5>)Yg54rW7#p@mr;y}d=D@hH4?EyhN zLgt{sz(s5;L>YcHp(d+5kkrNZ+#sk@CX8sT!@Ivv|9*G+OGyQOO8+Zq=9Ym83z1u{2ii4xzPzjQIV~76(@+U( z(yBu3p|%PIQ#FgCr?`ZZa4HbQr>_Oz>gEY;*YH zXp`YD=^$a7HDL)7WSgTGe{M=8f=>mN6BKB%Q=9XG_EJfLvaIla__H`^N;X?f0}^g- z7t#qjgeeeb_>V9_O~QZkR1}ljM1U{_5fF-G1uBfj8E0!;&1Mb$TXv3%1QXWhhld}& zIrRx0B-paQJ^riKR(&q8KtFyYr~vlf33tHbcg`ejr31Y zDHE#+51&4M{kFE5t%V6baalf{Pn%UWUr@~fWv;p>Ig2o3-Ef@IWlVHi zHD)p?KI{_{>e@_l9GME!Gd7??N~QSFOwK2bv-q!Z&?I#!sURjGe=jDLPS@-cI8^8G zr&{7f)>u-@Cz#DK(WP3Z6GKEulVoD1Iqh-+f#I8}m5m^Qzyd>I#T5_EN|)7hwA6iq zOSoMz5LEXIF67efLWDzF&L%e?0t0~>C+IY)Mvz)wA$2ZujU<5izAJwMcIoDiH*fy6 zv1M7LS%I>})vOdk^D@rLOye>x2Zr1Jt(6*tq*s^3iEj@#az%uYJWq%KCSd20AUtrn ziq`HV251V7M@GGy(UA6d)kCdW1D> z+UlSh=&~$vQm;NP1{BvB;@qn<&7x$7;t1(%APIbwV?_lXKwtvqL?iWppz}(rr0>AP zdi~=A@qkv+FJGF~!^eMr|M6{Uont1*`!x+lNlX911(Y`&v@>~QO)gN#HY|dI#VwfN zOgJKYH8n^9ohO^O`=ibLTUBP?@T|*9*U$})5y-*qx?DCJsN_nE+px-TniJ^Vt>6=m z$tPs=%r|TK(-y|i9io9*Z#E`p$AGXF8|E6($W?==0#zR16Ve*3_dvqUXCR@gdzgs- zC==ikq<_`uCYgc=0t6m%oYF!9B$x^l>oSM`g@N%7!i1wYZ^#anCH>)+|G2_sKQ@qY z^f%Kv`~>|m_mtaWSyhgsF%)h5{N>{<`4jdEU;>HIxOupHcua%{Y!eg-h$RduKp`_- zC2SL(m$6UFmjWR|982JN>c(hajgg?i1mtfhEtRvyt-}7%HjupeTJ>1~&&2R?*bTP#H1djK@Q9JOQ)l zrP-{xU;pyBzFYj~^TWfsDN9#@2`kcKo-Y>jiakBH#j2|AS%s;Z+#tt*wVQt(lK$9C z^39)gKsAP}2}h|h#FROj2sfM8ujoge=h^1`=&e)Idm+oQ>2 zxu%!{d;);P??S^J?vIb@>4QHuZiz61_yjQFZgO<^?mmCOYdjb?bGU7fW?F6gNKPrn2niBE1lH&?TLjwQFc0uwGl7T- zz6&JJg*co)0sFEU9lU*eFj?8!QrP8>qoY6Wt=*jee*8bQoeOK@NVdl(1d>j`!U$n7 z7|b=8Cfot84Ye{gHcjsVmcTMZ$%J}M~ex3;C?=T3V{jnI(8P}Y&5V2 zqj{|kC~vWL90-j;Hx$B!%R-@LK+NDjWr89BKb(3FG|8Ka%f~d--gX^O_Z$PMLC#*d zNBK&f0Y+E)9P*(BhVRs1negpFrYC0IOh}4c1$^{if~>!vothRP(((~dA*f=D8VxCs z@QKgBJW%Q&&()}DvTib=wUF#dEqBBL+aJ*F50#Mi%h%_hU@z+m6MAHf5rc#4GB0a` z{<6xUyRc9QXoZKe$X+d)?q~`}46t?HY()jHS(ImD(7U)W3}Y(9^vhZA>a6c%ue0-h z|Ln_IUw8-e?5cNWl+~L7Wtk=*3AQQU1>W6#xw6h+R^>#bC5In25i!B*pP7qrZJvFB zIh2kV{I=iGxtn!KRkJN>+VT_+|EUK7ojh+^_X3vBtU+;+nfCL5ubf+M0 zM+)?Q8=#YLsuUg;51%6e?)LXBXrv`@W)*xzJ-~)4LCOS)gv9^2>KY?~&1tn9)$~J_ z33;ZMSJeZIU>|!9r#Lu`52dY~#buN*5opsEa%s(f)RGY*4IOAvbmW0deM?9ubP(Z$ z3CFxVVgmI2w>M_O|9riF+3(_@!U|=?1h^Hhn4Hss>NOMcoPdB>D3J$gkl^oY}}A$yzHC(sn{FmV?lw$a`xr?auQwi1>D6H5O5fY z$bm=!FkzdB$vNBwqY@c#Vj#wFiT5hV_T)S447f=sMrY;$K06e`fEYvIeCdfGAR>f# z4j2nTTrm_d%D60icR?8m-<}W&WuC`mDS)A3D6E(Xzw1oEg{Omf)1I202M$1i?9p8W zt+~?0smuhV^=6aaP8WS@P?glfcODQU>p|ZaKHRUcCuCj%!CBB5kwg6a|tAk1P|)Rg7{54de5G1){>EWCA0B zZ2>Da!)4`>1!_9hCU$EofKq5NL32C{-!%PM-^`N&m@pA<)&Q{90~QKmg1JjduWt#& zhGood;gW$d-Pks~2Gwp%C$4n@NePIZ{a^RBf^;K1Fuq7t- z{SM$*pu@6opX@L4v+f=-VqD&CgIc6A3}S*p7|`!_8eb(QaDWk>#UT;FaTjo=vSb8> zHB*3Pf;9K>VSz;8(oH7?UbeA>!b7R~4n%UMADM75qf4Bm%y#zW*C9eSghf* zL;CF61)*Reo*bE?OGE9n*GYO6_T=3rTl$M{#Dw*B9S3z9l7l-yA&Ljf}jF}H8~*5jN2aQ}4((TM3iiUdRgl}w9|$(<^H;AG^H0}8GdP;i6K ziNN=&)$r$FAVQPTgbYs+p)C{S;_{Tly#OSn5(yni%`><#kcD5`=I}|G|G+7XS^7?9 zNNax52lWopIL4h)+T}ugeXzIN@uBQ(h1Tiolyzf zT}vnY!6%&wSgX`?ytE3O=ulvd)>Vi|keS65AYrq{+#`4~XXq4|s7n3+O&Dj$gg&&) zGm}~eYI!?BwhHnRe1|=VqS|+`FRy)O0>%vYy$eU6r5he)+#nMSx*sv&P!bb@y&x+c zxawwfb~O_Y_zt%MEr73leSuQJ4MN-q2xN*3<&N^3lK^A_x(w zSn*jx!FixswVMJb$*~w~MkaR~cX7gmrOt$8Z}lxO!+On-36$sLnC555H<4iTqVt-`geUJqQvt?f@Gwd6+#O%k!VUZu zT4!0^w%X82D8Y^vX-)$sBxGre2)aH<6@rxM-=1|swbAAe)0vn6N6VI{Ojs@x-|_R< zynb@+Sl=ymZ87Uw|C`eBvKz^j9X{BNv_Vv5!?ltV7wN@gs7x7R--x+ zIXLSrUO;#lw{bWc3fB>%bMzr<48X>FNGgF(IYOaIJY>SR`=@OfAPJ5#fsCQB|5xog zNW+%5$C+0Jm)YTo97-(kAm-vToe+tcF%UbkH~YXe6Lmrod2g06E{BF;?J;(cK&!=s zsbS8_ays6M0ghp=kqPsK+5h@uHl6<%kLL5|DKG(vK!>{mgriDmU;)6;u7k)zfs+|M zjR~h%2&6>7Rsby(v;0eO1UmDqLHtVZIn5p}g|DV^xLgCC5)+)x1V*e~Elte>g0p79 z(R1L+tVz}-_qD=_no8(&8?=HczDh?}whAP84I)4zZ4*Jwta6-VHj)kl-O`le30=z^ zD4n20!h1RazpDpgdw%=&5Wc;<{G^=4BND*Em`aRo*^#EqzuXN1K?^*N&J7`k&^jNU z_n~zTHh`ALsh>ETq7+wG`!dRWadUNc1(jDl7>mRqOp1l>&9n3COL*#I)-n!2I`v&H zO|k!SEr1DdKc8fJ8O07)Gl_ein32)H@%Gpv==GYN!J~6B-x_n+%!o@2b}=>#(=g{hhS!E^ z8ph;)&Pb@|0jt_c+8XAKsgPmqJNTbR7~yjD4haq1!Mf!_b0Czb4az{B7FPG%HW6az zc}gX4DQP`h3GS@L>S*Jg4i!vb&*2zNNZ#+uT3Mtnm7t)Zo0g8i7QLHx~2>e=#= z2bwz2>S@aQ+my$+>om!P562zKrO!zCx)M(@tONA|!jqv;LI+!|aQQ?zbvg$^XL;*Z2TnaHht^@QNY{o)UDTHypZ870@JZNpWGLcB&hgA9yRqeQi zi^YdbNDk9cBEam?REY6J)OIr*-iW>httqtdjoVQ!CKFMNZG3w)#OuX$3@3Q`=kxP3 zUOD{w^XHEVq&tmfFVEiy2@6JotZ$-(tlnAc7$bI~u7SuvHE@GnJtJ#I6S%3KsVD-E zRmA*FS~8)b5$q-r@;sA+kSdm-xeZ#)frfi7j|LL%Ty_hX2wH?u<|uZngccJ%jVw|u zp}FZ92}}eEBq(6BK!BqzYNu6Vz!=l!h5A}czzEqw85wL^N16>7txQD)TALng#8RMp{ zN+Oxy5)%TiQzj?|{1pp2tfQ)F{Ta8Bj>JN(kRl=Rodv}bVAF{Z=(XW$4)}ahi(Ilw zh}E)e8Db&RVgT7kBM$l;r;nD9U8-yZtvowz8WD&a4S#`npiHp&>QiW^KKKCTP6b7YE7H~D8zo`d9ml`NrGI< zaL$su2>d8xzJO3y0_>OLW^K1lx`j-S<*c=(T3qIZ&G?sh6}E({3GpIzIB z!uKmMKiXpTgaSLaixqo-2KEAbx*H^Zl)aANazhk#k+v+`!Z}qQgt8cDw!k6@PcD#v zHa;B>3^cA-Cn&!GZ5gBfR@QJK3Ppv`Ezp+F|NO^qIujz%zq)?MdT2)P`fph8^y%ao zLJ7G<0`WlqLDiYydamlrUP~qfbVBQgVz?*0BIN=78J!a+`PKl0)C_8hKDTUBE2${A8gsu@jmI!KEL!=jD)`y+MOLvT^f_>KYK`E+#}pKD7np#nBH1Jo z?3xSea9v=6hJ@dpRnpSP4Hd`nMy~M|${MY}gr`#`B-MEIo>_fJOC0?T8}i0&-oJjE z0}`|#6oR2M^0(mBCTRQmpaWAu^ChIuz#lr9%%D$$>a>6qf+WHnD+I9Pw@)d$h$_J* zAhe0VJx(`?1=#VRbKqiGtuSvEKEh2OaA13!s41Qkd?kAZ=Z zYQ+~w8FYd`XToorZHKjEjIy;nVZt5Q3>=5oKmYu5Nlb__KVcOm6;4gY49QivW|h#; z30sCjT9#>)q8!LEv=;#zj@9V6dWS?p222P637e9|LDsagQmd>%7vG8P?X4r4NWgQ7 z$@my1a0v;}zh#xVojtRqe*w2Ci!FbyiUSv+ps0W!3b49bNNxijcZ)lw0vrBthQHTL zz(=`wjki)nXfc7QAU%lfKiPi$^|$XU&3Ev{yO!Cvw)i{tT$>Otb?fKdJW#8;#LaL* zq#H4SWJ2J2)YWy*fnPi7g?7rlZVDQN3fMm|$pexH!~=?9v9_1>;87$jPTdC3hh0Eu z%X8&OxGt)MpvYRD1DF;<>|BPS7Ux=b?TRdN!lG7PSqT^~YG{O@_88g%AyE+njRoDd z`s3e!bm|1f1Vsj2C(Ho}^Ov_h5MlYf1STBP0y=h(R%odGpTZR&0Ui(ts~QR9o!|__ zd{aeU%#pq0*mWok0KtMGLy-`!vC|frW?VA?4mejp7{3zYHA)&)*U%zHf=ftP1p$f# z+NurwouId?Lc}=&jnqd&~uqG*dc*jc3 zq!Kv$Pcb1o>CPuAphb(W@=5yW<5ETvbRM)MLYm8@Mc;4op#6I$FS=-uzHjgbOlsVA z35DuI-{BQDWC9G-ulCgv?ECV2hVg_L>ni0{NvX^$i3HRLk02H5YU&l72ns>8*~DH@ z;yO`LPNn~F)H?=S<+;qTOwmO_a$q0MbSt^uTke$ zzYa(U@zoN608%AT^>R~q=EJib?+EV}=xz`Zq{RhHS&U&wm3H*uIBR)|Ls8e=PIqMbr;?jvhIp&EQdyMUE56D&yJ@LX25U z3JTQ0VyU}HpMAE6WIKQt3nl!+=kGC}SFg@A(Qe{7Wph>oxE3TOV@+Rg67w z`La7qX(le%00cKm8aY7GhdM!e?g16;XCyEa2ndqgD5UHV0?SP6=5QZ;XBq21Z;{5RDkQrwe36H zrB09m1>%B^1gz1DsOeyW3M;IkIBP8wzK4EVH2uKD4^+W(2?XsXIT>)&iPA*=|2I?X z(lmAfW)cXDgrHy~$Xmu>bn41B1_mtdQ76d$-6<11Wsck&esmqzqwQ#c+c3%x#Sl(Urty(E=rLHAS8mvM+8Es8%zs{k3fGP^C&!2C2qq>3HHaoO(#sYxv=T&+o zq}yz))B-F*8r(5=C2U#30Xx>|qalyhLrsKeV%PSd7F`>lKjEUc2@^;g>obS!?)>#) zD>uvpjWJ0}EeRrs5rN|lx}^exlM+xeg~I0>3=t?)KrrA%J_HJ%yxrO#OaxY5$p8a5 zrA+XACnm5KRjDQwMpXtR7)QG>Ut`^;0tQ#g4x0SCS_K7&g(m{daHwLGU{H^@7aBoK z$b>Sn+IC_KSlH-F6Khr4@ zN?JXl!XrWGVL%}!4s}voM9s`_csOyG%xnk;dQ(X^mG4nDS}7AE1Z9vf;S+YrtV`UC z2zXeAR zgfS9`5L8bvsqD0>&|IU+&7aJK+6}TkOi;cvCh*Rio%Scz$;G`;hDfl&@Obh z_;lOH_478^ZnZzJ~9O_oL>gHl&s*F-9n9V%iWP6eR;;epG>f;fV&@I5VT* zy|y`_?P(@ldnTYTPRJJ}vf;MRG6TUoK%qk201G}>pW%cdLj?;v3$?~Lo+FGM3t9ry zxO1ENh+jX#Mjazb!qfjN}6SICJJ$adNgkcr37Ybh!bLJkQ}BjabB#a_oqx?6QyaTBlSxujrKnj zfIvK$b#+W+tTDKHP&dQ`T!N+G2oJNl12e(L3CkQVzI{9FKj@Zwo;NF9fj-merlJ{< z>kS*PIb)iRh_G1!5_*!_%G^7%3N2w{4_ien%#`HOTK8SeR-}l8uJ0Xgo5rvMsp+~N z)pQLa;l4-fRtlQ(D5{J-T6cZF7K)x@r8KY%OPi~{7h>C^sLr4*j)q8!k%Gb-25E3x zz0uin8YX0vOu+aT)e;y9e5QKBe3)QujQk0P%~y;QvO=0s!%`tRca21t072qv3Wc&b zfdo>P@!_C|3Bo51nkE_zt|@VtAk7?yN&V=K^>)C}MCiI!G-GP`uK7;GE!!~BO?mET+ zO%t}pwuA)W#_DDrJX&Q(TXa;1;L(~7mFeQh9<5Fm?9tlY%H816T5idsb$L*??9o~` zZDUi9R(3)XLNgO?3-D;YYwzB?fX8m`-kzO*72*y(T4D79UPN|n>y(rVs30_@53Avj z2Bjh>T&)B_VZ3~x<^Oz?fJ-=5ZlTqDbP%`e(Ax~-4p}iOD44P|xRX$2k$^D%){T7w z0SYKnl=d_fBo-MLjD*Z_Op8F^eaT`v&-KNG3XFsbhhfpRJs<&pGBHPdNaZXR@TurN zqC$WIJ8-OclTo0;Py|m@7sN8HQ&Rv|Qcs{j0Z*x<&l{uzO=|`Ani`w^h{AnKg6VKW zynnu!fM>7d7%riT0Ma_93a#Qz(tu9w{(G2^T|%Jn?O0u|@||Zw zGVMCZ5fZShuW>Y@Pd!?TxTQ;)45!uI8GE$8I|h$d;qET}7Cc(_CwEegD(np1=6y8s0e z?j#f{RKA?$Cqi+QkX3(pM-?UlU;ec+56R*wGJ)fSJQftJEGS4IA{YWf+>-(p=vaix z3{Z%H!hNSk@`tE$F8v~YEKX4B(IDZz-+>`95$;D2^>`!#AK9E31e}@O zz&uDNIx!lxQsuaDIhI)KM0Un4@=0;RNj=1b8VubJbb8jWm&@JZc=$d69qxB_vwB8LKty}VFU5z|iU-zZ=Xl=?<9<9r& zCkbT2`&;yA)%e&i^w=Z#soaU>PruyI)qZ&)Djt#OwA8Cg=nkRaq~PGCR0^R`UE#!u zX_x>5VB!Xlz&xOZRjh`Xzz+!q7Q~B$2}Nm#%t0ANK|(kvWbGO+LPY6{3CcquJgUAA z1p|vxF6TJm@-lC&B_t^4Y$d#|2n%@OcXt&fwYVfh*Q}DLEU7OhG?-PHU?EV!?+=p^ zRvI6|19W-}xF&Qf&3`G*tlJz96@aMXhfBf7P|@AtUn7Vk)M=e$Sjk$je9U;OyovPY{1k5)H$v=)lJ z*PIC+tuIF&t=Zx&>fKTVk5-kf2anb}z{Zh0TG40aT>2!wC2de*kxu~8QnZC)I2nF%wp z)G-W1sBS=2Px~=4ft^{mKmu}2EB0fL)Y^7?2%H(6ui44>Ba*P$O2735XpZ`1=u7g$vb?SnJ+-2!X5{09C;|jtfSQ@y;Q&lPAnb2J&vx*L!|K;d zTa?G!cEW@-bHE@0dKs~QL6b*o&K|A7nU%JdN8FO$qg8jVK|jW}_8zV37(80#cE33c z9<8@c!Eu5ed9?1I|GlsGNQVrb@BIAzxsc@1`ZIX6?(*Q#y2{|W&k#=lvV2P6Wl5DL|es31xhRT5%7fg5W`ra`9$N1T`> z5hEsiT>64St)#+>Sk4Ab!O%34_31kooGRp-W&MEs;-CT(^A|c{{RXBTu`H*vvrYcjR`b` z?Tovkz?D?An@BYp+rVK+lF)>NkWIy*GfAf?!KZmTlqaZQq$u#2Igf~~dMu`bP9pfH zCl4U+`x0XWg+xeqIit*o2zI1MgvQl#F+!3!BuG#pPNMrcLZFl(>d{fhr{Ew}SrHSO z_O!lW?(hdMQzqEkz1OoHK%xKj*DIWCd?I()vNqdRyY=d-0wPp6H4+}Q`0>Z{cVCggk7nR$NI4IN*!FoC!e!rw+gbSg4fkGt~(beq$z-SbLBX+h;>#oL3qu)P9HN z?7>DRq)1SlJ1j6xsOv&jJjbylC`i6giZ(D*06YxMOXyO9%CxR9wvZ2YDCOftQ~ThAL|L*sj&UQ z>9rP+HsEAwf`y&)XyvWaIUJ1!X?^$e&ugcx?9bVwRkb~Ow3gdV+jjlnvzkBT(VA6M zGjwf19<4|8XuY``9;_f^g@FhI z6%@BNMko*!YFazD(o$re3J>iQtofk_wszuH2PQ%~$x)pO5$Z87mYOacX5GZXf(S2Gj#-*(@`ujiM2e+W$2 zPQ<BlcqO5fY|I0-lKKD>Vq?@!44zZQb;Aob1V0+U$3>&T9QZWo5QWyW6zSs z+@amGM{B-mmG@{h-Nd7{*{WA>?nFiNnA*mmM=KEO6}+OpV~2L7&rC69&70NLbjE&;rf);GARL4Cp)x&W@3=!C1mo#nt z@l}NR4#8OV{qC=DWk&7H#=kkfP2wNPOn{I+bDF%QI{9`<3QwsSB}f7j#_*ZaVg0Xx z2XTX#VD>u}<_`OV`1M`Y?+?d4F=5OD6#1Syy}CjqtYc2mZGtoF%HbBQ+FgrV1@x;FbB#Ym_JC^#t_XI%kkP=)sbtWW_88uxOV58i57`2c}Wc#d&`^-e&5 z-p*==4h<&$JKGTxekUdXf=WPwjmE$8YMLel3gLSVm~isjMCQv&P9DHr1p?YciWDsK;r~{NCO*UN487?DVG4@iH*yT z_HoAwj$)1b6F?299&2DiM1ntgCTw@CXAmoV)4!gJ!w%*=@UiCOr0aNeT3uDpl(5Ej zrlHQzAt2jAu2x%Pl-rgTZYBpdprmbx2kcWQ%Z{I_#`i&9jfH|6ng(yB57*QkK%pq| zvL1P~y0U~vHF+jvg%LkKA7Gd|5(~+Gt$Cq#b9cqI)%0wA=K7VR8Gp)+7_VD zNT~@5N>wPD)C)3gw?+pU4X|Je;CdP-*cwGqLDO_vUP%`QA8?@+e;{A}_1kaEgtAn1 z?c3%+apjWhTxTppm9efvoG|KbGc2eu;zCdWAr$D1%h)OdYZ)s4LTY?s$wnoB1W8Cx za;8s-Q--|2vnK&dgi{C}rb?l}`2#!=Y=V+02X1IsV6xh$g$l77qtcG$$sK`UlL*0Y zMPQ(2ngwV*)6!nE=>FKcX+03)`ZN#f?*>9tfg0+VX$w^FZDwtF2aYKbA6FWdAy63N zgk87oX!m}0JnnZ#Fl1D_{bA>sK>%OMgfa&c!U;Op_rwUkc8wnw;SgPtbJI);_g zZgQAgIHPzYyhzuWj$gK^H_@ZDrF$Em3oS*3PbaZ7Ay!zp)|ITZF^WA}Z`BZ9+mQhY zLqTF2qXHq!ISz@LR5$otnpKv>!x)=T#2gX!3P5;I@e!W zz=Xem34)ID6GtGUPG4`zBE$)nm>^9Z)9_dj6PIL5Mj6BnNXy+IDnYq}E{t!LgBAJ5j89)D$PcLfhU1qN}%k=RI2a_$dyVppXIZFfEUY1{abGQ<%Ix)oNdjVf$w3Pn)hxoQj3XqVrXgi|ohk}^P;%Z|n!tnzIg<2Y0!CKRFJKuYRZq?sL;5Ge>ig3xMmY^Mnkc$dc&k$_hjoWfPUWFCl_l7g2p z2nC-#B)!4JZ%+lidP3Kd8X^RFB4;o$`x!qPN#!#A}6u_-lt47TkY5CPqk zPR0q>F8o7lC1u?r6FyxDf$z347Uk!lkbeKiZ^(qKD1F}WovJxdP^BRz7|EYKX<&lR z5C&v35Pcajf5?kccnc3^g2WXvLCK8m%m@hrkpPl0N{~fV&RF0$A!A2 z=|stxKByuulR!bCILUJ>pu#8mP#^D6h!ZLWJi<;J7U9gWnR74X2!e+m6CN-(!WBA$ z94;WdN~lPph7cGMQs^gdF?Rq*VC;_s#X6(==kNdg{U5MEoWM*-kT9P_G_tHib1+Y+Ri(AQ ze8vCF2BZZd0f<0~Hs=dAd-erGMNB9QNld7`O*Q;DY-7I?N_r@a6^0f#XUITkix>n{ z$(G9)FFZ6SN{wp@d6GT&I|J}mSXlTZt_TbBE(N-ZHN{GE2+uHpQ{3NcmztK1fMJru zI>pU+b8 zhzMu}FW+9%WjNLYGQcwd*wB=wMKKe|Jb}vzb)!_Hm4oYQC4|-(&(mVLq{yw9#)k;_ zyq*deI~T1pXg%L(sVw_%g|8i$6tLN7j|3glhN#j7BFO3rs}Bw44xeO4V#`mC6A%c1 z1Rp10^6;pXYAA8&`$iQLCMbGyd$7Rij5Y0{!_0v#WtYr_v8y{$Aul`=@IuP+nwUU& z1G$=s3R)PQQ7D;~7xSh+U~v(v1zF~g-%2N5KwrL~2|HIBaY+M(w9qJN>_ML@-E(FH zY;d}|5_oKGto+`?f-)KexgGt4Kp>?3%lxFAGQjmTZ3sZ{AW1QhUXQtZ8q@JAm6d{6 z>P2l64<`+C0m3jjBYEhIR0BI_D&8C8yhwxrE5r#+-CQ?_1&9+63X6+&-|gBOm~a?& z-zY36Y**FV@e!2$i$eCjilM{afHP|g+q~r>p_&jVAP}P9SBL;^C{W0Q`_LPFW?bhFi5`CZOnvOq=5|krq2e*JHFzP|#9##(XBnJ+Kk`eMlf5?n^%tvN_=xS-U>|9mW^pfUovR9^uSuAIFeCdhz< z?)}-jT^@+QA{NjJFFw9`gIBX`R8+`kDhf;mgHT{Mk#sNc^?Pmy)gB81xiA*Z{6gp%xSKM1D#Jg1wi%$LycJbZ1cTwCkye5Dhk6fQOaWzy(nyyYxH}3I+mJ4-$i? zhlL)k-JGWW37kJboNz=Y)YsSbHDcjekGfsHUmgc0fSRc^GGT+3?rjhfL5Hg276{a% zGpmBD9c{^WJy2oo-0+MvPKvM0Zxwr*thNrZfPn(XJ-g!F`=f2b#0@0`X1)Yqi#pT98^gc*~d@vEs?^X~j$oKrBhGjEv*ioygV zEnKwcFiZUH#U9??oAXqNC82^Hqbk=@H4PlfpJOKx+(OhX~>LP!IOt9$vS{n-wXv$pq;V_*SZCU2Ir9c6p0H;8P zru9sij}z*Kq~w*Xafp;{TUuDGX*t2G`Dj+y#~7F}R#OxXN39%+5B=1bZT{omvEd4s z@N=&6JZC0=lybG%FW#OPCNG?_5GQn|b~Z0B-Y)jkO>c^vsBjsvu)x=~*mwK!-z?w( z@6Q=bfWp68?q@XTD7cz)B>~SD7$@X_1bzh3fD|r2{_?YbwhtkL7*nAvPSqJriGji_ z6PVsZOh@4YV1O1>qya;9l;!*9vIGhnhaQGer=e`|U&R4?(QB};ib#Nks1Udy^b8Xs zp`Vm6D1TB>o(Uja^F@cN8UKV3C*W8Y1sR?R$A0^M(E}4^;sgf(*b)?~8x#{^)AXcv z*BZPeCOE$(+rX#|+)Fa8t?_cJnnoRP3%(4#_MiLy>aOp#wu4+BDQwiM0;6A01@rMT*wS%96={ zkh-oL<)tA?NK|0mlgc3QU`%9yE0t@!<-Ac%V?lntLqo@q6|m1p<)Wx!Q1Y+bIN6ho zX^{!JGQfnNmOZHSOt8`-6W+gF?oHR2{Sn>@v7+gb2~{c7haq=pg8Zh_SF0^(eSUVZ-h_Uz*9<%f4KUVqN?9o>6tZXx?P>p^Cp!^2*`c=4ux3zn=(_3!a% zjHM|IVkLVtWIcO#DEbAuzp8_(b7q36yH~G|s>@%!fnRfR_Hudt;+5Ni=j&Vae3h-n zio!L83XFsi6AU#cByqwu9=(D7B-B%Blx+to)wE4yLQ6eM0u9O4ICHSaSNt0wuYwb8 z-S2*UK`4MY;qaEMYssd|VVF?EU-w;kyKPK|RfJ_nZ4!HC0%*4YUy0li6)w5Bus|q~ zkIMBYLINfMuU>zFzvQcrz~Q$qU%*V2LKXe{S1-@bUYED8UlpKNuQ!`FXJ;5E2nc}v zBR)X3fR8x;1R(g&`Ps{}AD{b(feUC!nK20zPzI}vO3g5Vm6RXjK_Svz(||%g5byPk z1NVy+@dz1-LO&Q?K0)@rC`dpe{Ab7XR@u2K0`;BR;k`TPsw>$ZKaHR!polaLLW3Yz2H_m!Ub%>x z8G<`8IcY1NWG5%v$7!wa{~cyMq*6(zwfBBCE3I8d6Jy%^daYHf{sn^0Xgtr-?wN_7#x}8S5*X_Q_)2z{DE!S7L zc`m82x~^|KzU8NZ95$cKE{@Ec7_`5|F>rGxB*$h)y60J=CtsfMJHGs3fC;;jpTPD9TET<_b1-bE;T8x!^?09Q7b!F~6Ua}XnU=P(HsC-DRA(D_ z1aP2qUb=GJ>o!~6Zf78Y33Pm~(kA_uAdJJ~bUYf4$Kz>sfV9GJN+b~HoC)D^G8~PE zBNqa+1=cu|ssb_rOF9Db6Br6DqyN-un0xC2!%yJ^PcN2yZ9cVHPeve(7GBdWL&|=m ziPAUt?QXSk3-AfvUK+4P>mTd+9<~5wD|24hat_?X=)wI83MEG&_<{)m-8f(PnI@6K zfrzxYGTen_7zkDUSv8&44nMPB=7BbJ5~@UiI}L<6-&>zh(I+6}B~)tf_hN-FtgN4q za6?kSOnTGKdDOh^?I$W%h%NLa{+QVaQ7(P*U{0JmlsMcgFW zA&|p{IlkgLt;f3Q!q}g7-J(_uBYUCI{wJ_h$ujR~XyrcKBW9s!LjA2p2eaR#4g((g6f z@Oj}QpdLj!YIKPSvB1Go2VlYom@qpCKbs*FLd^tmgj_=afyCr2lF*(J5V#w{*<$~r ziXE)E*=PuHn?pA+%|vz#WP;I=_97iN2EYXPR(EI%WBE2}bf&OH zf4>|zo3HaFtec~$Zc2xTMcfl((~d&Ugdq453c*fI^<2*v78M4nAc(iqG|(tuJ|lFNt2GA>cijYBEF5oF^AJiERQGuB$h6E8`3V(Izs!V#_7mt%Ga+y( zr-V#MFwIXRr{#Q}5fj)dOh~q>vYBt-Fr4E;qxA zvmJ7R2Z$c|2}vZR2?uk%wN0DOOBTRU3Mp0W(1TBzT@5&Lp*kYN z3}OdyJ*6TP=h+88L7c|JS>}18F`^=I7s^w>^dC9*NCixL0TTxD((Ap;{?qV0TNvi$ zy$>XHrD>+j1RI%P%h6N8gu(m1*In-6b*t&rFjs)_ft>F-pP;Y+=6oOmE+zX}CPZ2S zjpb)*L{J3?TwqW|0kfdU&hd}3dlik3ALo|06&EV67i^3Kaue!EAaW{z0<1*|2uy@X zRXhHouFM_%gcz2jVkx@}hqBHRv@{cvtFSrKj2f+3j)k+erGAH+QAS}K6BKeV98%V3 zy(x`WkCa)tWowPr_Ds%Dag}-{W=fA0|4Ydw3ac((8qkyL7}f5|?qaz$XXu97EvKqOoH3GluLWCFa0GxvovrlG$&vpCsigmPFPl76w)w`u`A8? z!fth>8@X+~{ugkp*b^XotyyLtj1Dp(3`9)KI*vzfgx5`Q`pASZ8x4WHaK|;c0YQP7 zKturh_OEZ0{00atH$EV=!{^y_-p$s0V*wK~U;_3cY9>&BnqlteXm9xz#&m7A{;`~Y zBPNuoz=_8DenL(K$EC(k`3d#03ZdW&A6ZZ_ogOS*v~`Q<_G|WWf2vta0B#-1E`Gb? zBvgK!Z7F78ReueHpNkjni|(U-f(T)WN>WH5IJ3Z}V+UWWCNLA21g1m{dYg#p9;q)<1lSk35Z)PHgM%p<3Fr%q z+*IITFJ=PnN97ZM2~)IL$U${tQ@`X)IrbRAa%kJ`NGDEH512sbc_?hd{BBv_4fYdk zAd_aoYDrAMJs=ZaUI&B0>wIn)*!5tP{RDOt9M^f|C$PCLH@{>;efUx_UMf`aL8VI| z{O_2?5_UMOF>X*)&@d=D3A|XPeg+yZ`yi)ymj-%hO4S@-LLe#@vE@vtimH@#o3FP6 z1`xlc0E8X83R#xm4D{KjOh|B&;8c%{{HW1-V~tjB!8-pe$zt*y_*r2#=g^e$FyYvt zX-drmiO-^0&O1rHW9-08a7b#^Y@(~+i44_R<;}53b_J`M!CUFd;1=lEAraB3)urRB zFrj0*B6fiZ=z3VW`4LrX6n=v6|M+Ok1UkFBG|HLahmmj`Yt31Mljyc4{}k1&~_nUDnAFs9hyihCj^T+v?`4;S)umQr0JF+zc- zl4PS!F}U$gg$r*vmS`d}+dgJ+IzIYOCwQW(nBc6yhzVHA4AF+%?7f*zg1NGPcsFa-eTz7H)uSV^0X59(G?)~<^0S3&`O_w{4m zUfGts1o){`CaBnIt_wk_;pC+V9?5fRru`xhbSH3$yMJ*$fp;qktLP{2Qrs~UP{{py zjcgnJ1o&Ytq_KQ||C9-^P@hnes__)bZPsdz7hTk7)yk}P(el%n6B0+)Xbe4CM>9vQ zt=ED48`N4aZ+*-$Hm8?1Frhi3XE*ED!NvM}(KM!VHa=Z76Rcc?RR}?{L6+moNwe|l zdXo|77_T%{J!!ZG3lYdpb8Pn-B|o8wOz?u#MkbK0;7^HAz;uGnG5Xd=B>ax~1Y`m` z3a|vu#v!vD)(K9MJ4{I8YV2UT0mM}knVP6X%b!Uat%+QW$xldt30>(65tf+{qjwM@ z6Al5kkO@D6ID{PI$WGJQG>zEe6rg~?17E8O3G0>7rRq+aWowcqqn8NI+RFhtg*qGX zxP}pQBQX;igY6tQGdsWp?1{u|=yE=MneSnXh?mr(Z;9B$T){r`dq z)FY<`py$X?g$Sye`9U^-ul!nsQGQT74u8t(w5jgiw>XyoOK!Q7My9Bs9R&}KW&UeO zc&xNkat`psDlLcH;siBSO6tm_KN9D=uIVT69Wg%@FApkW^ZSHiXpV-d<%dSB{`oXS=g0*sL~+Pw z&~E=oJZyKsZ$M@H)H}CK=>>T(5QUgWbh8MzzG8lLoMCGUt&#mh94GJ_IxCN1F=UUz z2jwWN*h^TywM>?+^(0`xgudV#Q1Yqq^lX}M@1q9TbIXu624e6$Y_)0`=qDf)U<)|C z=JIVwTNo`BI}~Zgbv4cCed;LqcbQPRn(1qPU*KHY&Yu_N1x0 zaIDJ|)LI8`8>m^D@1d+{{5*ZjpPjFvFzTI(WNw*_a<75o_|C#P7-kZL*n4Wg}b z0=wawCEQSg&e%f2^{TUfMX|KUiUaig(gIx2*ML3KLIY} zKkCM@gP`unBtdz03L7wl5U}>HpRoPNgx2%(%Txw&h_y@L9lh9x5QS&qa~w{uFRt*? zmx1&PB!t^^Itfx_LUh0^%{0Zf;QH7rau{4nv_}0^Y1Yf51)y_QUaw`-4U3 z$`MvindaVlA7bleaCP0|t!7?me&t6jf4pcRDkelZmF=NP+T3z#oD@Id+V&NH*7!vc9H9qysY-}4N^1`?acxGWQUp&YPpJR z3rL_{yh=Bu@rQteZ5SnyrvzEOBfW7DiF{omM8Ms9B=j0cjMq=_2!nrl!^>o4IU$h1 z!8M_P){Z1r9L)qSQ(#lEL$sq2os?Q!Q^x}QxQrUs>^_)f}D1-}Q0wV!_uh!c*q*8A( zK10VqxYRJsi)CTMmJ&xR{LORqE}^iQulT&S`Le=3i2Z&Kj|w8_?#S|xhr51)|0NT& zI2H#HG@Bm*;ch}P`c)j%qK1b+`C49iCv1emH|lp_3@>>E*+MZvfq`s+bJEn1@K9(; z@c^BJM+I*I-L9`n6g^V}OdMFW~!^Cc0%axjhH zT3|F%RCpXYtkxO{x=;cAgdIa65k8UwN9sAyQigQ!`RwiX$b_|JZQ_av**4F%ivdLv zqW}_)iAq?8(ROio7AhyWZW!x|y*LSlE^q*=wqgoxqq4;TK_N%Ne6>Mm1&4&?M!6}x zO!1p04ddOh@DBrzaG;nqUs!Gk*JANyeq^fCbFC+fs;3w|W68P7Y*)7h4mEV-lZR;K3ITp=Jq4CNna7~>R89HPh}VV2xtOK@aVC~nGjH( zpuv>EH#NwSA58_$K8$~r6;c_-lKCEmiP)dlxJyF|Wo)3afAg7pPjNou_ zxR}(=$%J8DaHX%kPLc$4R!_r)^P#&y0`r(LB?!_&#H8Ov+#jQbx`d07VnO4Oa%D|g za4<&EwXngLFhdDZB1A$s(Yr6=&;Rr1H88Ie#;g!@0>f$Kgc_tk10v{z9oaxvQ?yo`!E;8Y)G+62=yme$b_I1KnBzaf8h4rc2gqT(Rq7fotHDP9O%HKpucY z?4GN&Eh3HxC!p~(@y2w0O5E{49*W{pQ#S2|l& zWrr8rS|kIwYA}sQY9J}w#uoX3P5>oDAhBp@IbhJ@%PNl%jR4~TG6Cbc78T$)-i4_4 z?gXzEfJMRBgf)9cu$Y9EjDY=>F=Pwjgg<`A@|REIvuF-5^Ytp7c2Lw>|DoN$?P;`Z zdYS`|0)>Kb!osytCQK5G;}on3)W5+mGsoMI74~7YAJ77y9fiUasuNCg0!4fts>e9)CmK-q2+agL0!ch+H1PDYheV=%6@%4;skG3Z)KPRM9KE94V# zj+TR*An1fYgb}bJC**KB>_q~)zSAY?87?se2uV2Mufhq^ruFx~N}MIS6Iy?r>_!M6( zMwY{o=4*bE`gZbtIk|s%bA1P$@aLqb3Mm28E8Md&?rB}!fA{qc6bqqy0(NIuL4n)R zW^=tjuhhkLS=F+8i4qBKJ18e`bS9vL%ZWo|D-X!6)2w&;c-Z)G?}UN{;cSVbx%^Nk zI7c_4oLp8wvI?ZYfI@;Lr4zxxmyo3fKE)5*+fW013_5{)KuXwif}^znrpXJVI%I{H z!?P00FQolu*uxzalNnazdLT#aTFE8BB-cRyIi?wVAbu z08J=?XpFD)H8D#6HeLQx2A?ssy=A3L_(}E7(QzF@U}#e^L?pEa(Jr#QIQj(N&qEH zaFtUi=y!$6h&n+!b3!8=|}(q zIoCkgpuL2AcwyYh#W9w#6M(3k${STnXtpnm`MPNT{xqm6xVJ=A2IbLLKD<|7*B=@GRXi59c>hd{@_!KF zC9&dP72y8$IIwRj&k4r?oiLUNT=fzJa1;qx$-L6?4aWGcMVZyg2_B8dkQR3221OHk zueQGPsgtvCgDpt=UxgFu>yBMqaq~Ip6<;w_zkU1GoBVhnHBVM~q`|sHBBQHfQJccq zHL{!X(xWJ-{Q<*;Rs^$?=uJo-mmyvPYK{0xv1?x zp&&cd37yYl{QZx2tKyIon(4GcAqXkh>vlH>pAk4Ud#lMLLFryp_T-STqVC7yPFRj76v#v7{;Zw_z1- z!>(>G=xWwakKk>DS`o7*#4M&ASQzXU#IlF@iRt7TiSsoRN8*pE0Nz$CKSmWmP8frn zFpDqIpAc6Y^i;`|5XcDt`X*r}_z@>Gx(4c))4UTXEi}8(qqUNnf|tnMcU6bPeH?Qs zIiYQO>B8D~9lEQ2{`D6?Ha@J*UP|}R1=!*UwwMN8QIQj@HO?Cj!#);h>bvIze5I5R zIUe$Pj^CQ8t*|fCzXu0~4~HM|lvjP-ld(F2m#0gK0hUB)Ayr z1fSas+hJWdhn&#jf@?snQHsO2_cCo6ashU*6kTSP! zt@qdEafS&4Ikj*atno_|`GXw~dG6u^5Zz@%gp{R#>^SBkkxFgqS^O}ZURt;z5CUi1 zBK`RMz+_;g%f?A8AWC6N5zen8*bQ;Ut)RjjHv>09m|-IB6P&r*I~n6g7AN%7ABpHr z7!iVo8N&awCC2*=5dR+hQx9mO1SdeM&gz5-*k|c`x@oy=>U+p=mJ5BAzL$GD2EV9H zJ#;eYoc8-3t?9$Z&!rae!Ry|}W0`V-_6l%L3Lsy7esmx0deBd0^^E;kZ*DHHkDx=^ zv7*99Ul?nhFMk@L$W5?h3r&5}D+GWM;0GK?TnSSFerW#Kr{DxqI$1y#VU`Wk9C5Mq zDo%9*Jqd-JmJSQ-u|}5Jj6enjv5Pw4Eck+>@pp(ZC2+<|tL?mr`XJ;462XuDek@G@ z$3Qs0Ll2?vSoUs#TBxoeMxLy=q+kPCvP?N)TLDpogix3rw+%X|{)KyTkgkN)7<{dz zouLE@k(@?fV1!5rFz!eQp^lZ}WDyo70dP~+vNiqlusWm#myOhdXN42a#*@%V;k++4 zl7e!U{418W4U{m_gQ17J++*@z{IYja?Aw}oPPp9l=+Ss1-NnA4LEyKS_aD7keSIMfKUvgN zULV0_%AfW>t!UnLCL#Jbdx5>hoDO1))9qq-B|V+=bh z_!6d^z+$P&Bu4WuVJsNRjiPHSi+jg9;nR70pc5vkM)f}8P=|Pfb#SZ(a{%jO5r!~o z$LEV(%MUoAE{B8=z|l)e&|u`peCz+wqyIdclzY6NEWT8loM1!HVG6lx)^Mm0Z1_M% z@Nvd!tKo-ro-5yiguPH86>w5*Fv16atK=|Fr5e)$$%Cwb3zqX^s%~wnIh4eh}y6gg9&LWdz(@^PE8A0oQx#J5*XwPPh!D zFotTS_y=COM-Sd=kJja5X+_+l<6`QMMNZgBWv-IN%YJc|pqb)p=?f?wlNO6Nem1!& zY~%!dy$e)UFARd($&I*h{Hzhr0GV-xuL^5$|I~erL;=;(J2^q`Y!x#YB zKu)lA3l=b71UeNBq=w} zpyg<)KEWa_+pd{Vf)|OA5O_|Lfs4y^+tc%guY(qFi*chDPw z`V)kh9G)Lynl9rLY+JMY!qPaG3_0O~Kj@<|APg5JR}~D00()8oiCZAH6Ce)df}XiN zR1k6c)ONWLPFTQ)AtO3&oy@&(knVzUMu6XltYDmA=un8HFo(Meh0?J~ zV(ZeSPW+85k<JACsezgV|P_)i)NC~Zy%$LLL!{t`?2qfNVuI`Hh^C|9901mQr7avV|(eVm&7S zA&*gWn~}~ac)zygrc5bg)xnr%$~QSYGjee${*Hm|a8Z_OUpqo}u=WF-l)%xbj6h1L z_gSyiuxAGlo;Nliv`{Ai&kSE2PA;2h5>z!+2{xtKkP)(JArmkR>9@tC&f%L^ticH5 z6rAzK$F-LU*{n>k)*XfBD6Q7TmMKeu9E?ChASI}`j$}|&v|Eq$6A6sDPp{IOumB@g zu$Gjdmtoryx@rCi`=QLrV0$&Suiaq0Uoi|e`AHq7G|!&oWUF?d>2m8 zrh?E3=fK6eOzw1beHr};BeYXAjy+m$*`qam^z@s(S;&6u+e%J&+h5AtZ_~#QAr-Rq z>ZwqJ{d~4M3;kHzYhYd{a3pw6*weug>%sbh<($720*k|xA!Wlmx6Bx8cKF(Vwv1Wr}Ylg-HlcML)T$_NdO26Zf$yZ|E@5W=1gyi`37mGt10d=Y$% zaT(}@$OAdQXt~WQ8)wC&`-RCy50&37d!C9POA9bJ^!G?VGZb>w2r3W~*=Os3 zQ@(h9ZkbI6tq%{q31e1BY-EKIDU{VeV?R{rygEfB?UGrJULdq59E#D7oSFMUz?%qi zLPX3nj5=YK6IAcslL?3amn<{R}_d7s%s2_(?w2j zkO>QcBT8(86==80oqI}%oHACztM6LO{%VkR&-OriCRVJtf^1zACMO#?3`f(VcfcwPe@AgB`J{0{1{)N?4M;3T$W7?YeZ zvK{PFRKnuZfD_(vpe|EKgEIB51VC=QBbuW>z(&&nByva!o3T)clO*!BmW!;d+iJiF zS!O8`tj$0MAy#152t*ccpp;cGY;&`UlpvU)1bQZTJ<198I3v(br~o<|RU)i31>uN3 zs1Nlt*D?KPX%eA=8S4QEx$D!g8JKiS+jb9V?H2)x6T2 zFA__*VEK1)g6qHtz`bwZzKb^a@&%XcRvZMkR|E574s5E@_5$1Y*r?lP*Y~9!%W$kM zd!TE4ch!tdU5T&BI!Q2XZU6nK<#0s<0Pjzb7cy>`se>uz0V^aiK!ZNjg0-m7Z2_wT zoFGabq_9kKL=9moaQp&j>sF^sQz%>_gi+$;CdIFotC-;#IU!I2IblP!;e(f%lL6@% zMc2&frgV0!6S5^`!ut=7D5cY(BGtq-t0F<6ERkj72qV>I$f$8?8m2??E(LUX8S4b^ zQgDQmu|YJj*OIiaqO_cv%y z^glRwm$QU0k2r^{4pdwM;(U>D(fJ*|d;urWq2M<6zIfco0#2Y#2VAQ$mio6Z3XLQxq)53`n6^ zuM3h?BO1Zg&Y054mZ@Y2BP*Io$ZPruOrj~wpo^40@aj*XHpo&d2P*bqZT@r53A*;2 zz`sDfi>WI&0yzP6g2nmk(#h>w{}G?T38_-To)gS_0X2Q-sU;Zrp)>yKPwV4(H~k5s zY?TxAets^8oB-5X?U1%jTe<*_PcnjR#nc4ZU%_&40po&-3amhx5LXIruZ<-g@LV&L zU?+@Vl@YdRAX9mx!6$=yhhp>|9%9^tEr$A+#y}-Z!VWRw5cheVk^-5HLSYVLP9Ub7 zKvI~&JUz_3)w}UQ#wBn9(D|(6xE1G%)Fs?_PN+BDcDNBT2)H`ngj2poFx(;Jjf}GS z+Z*Yw`pfc5iF>Hv?N$bsmLnV~i($s^T-H|`W4qo2xd(W!YKc$ zxMn5ODkFtxN_GbV9bd$7+Xb;>PDOPWgZ&gLV7&qmw$E{)=mCz$Vm)zLml%SzjJ; zFyPIlMiAuCSsx!{MXfi5*BXgUu5;ZNiA#z?@u5rSVKxE=4%%Da?lDB8)CHv z8+dLx;a`l5U~QhWe{R(j2yWmDAz0@H*-s~Po+~~)^LbrXUF|Dn3WQ{=9o>9+(Z|hF!z*}t9|32n~u}<)mz@!G& zTe27VMkygV6P`wo*1!JcukSn~B$|`UIGLNE6fsBnSrAN zDQNZLaeD!eKSYQ*o*;xw7X>#W?8Vp5v7GV{&AM1)MChb+aEZ|&sF4td@bx}cj z$}_ER+K;A8a+=Gn7D}_qAus}uM!8=XCV@XW1fG@=t?BTjt`hTYP64rHQn`#dFkW3|F z;Dk2I>N-u^VNF$~VAL}Q8<|`cf~8?EAQw<4P&JqjW0f#379bVC1z|;%044y-9ETOS z5#&ITa~>C;z-SV>8aYNvKyU62pQpX#J4wh3WP`{Aycz2Ry#R@uI2p6$yim|k?L1so zMK92NLtUZ*0vkk1n7;IkJiuxSG+_1>^!rs^>n3ypK|=6X;RMe4vbP;%-RcgW$y({? zR6OcPs0E~g4KCaqW7MOS59eA@UjdOM-t3AnH3a?Hj1K62Q9ATu4V*xM=p5BNlNMwHTi^skXSj7`?KweKfbK1~ zj+_9ct6iLw6R0jUPASKYZ~}TZ|Kxib+-)~=yr#d0J5&-mQ?cq;QWS9pkf}9X7g$W(-v;@h3#30L* z7V3SERz8R%vZ3`{(dGs`DGNgn8wx(H!aqIsqzyoIpn4_zWJp7%F?WR6_8x8(qKr?YkR(G5UU#FThe< zE&aDT#qyWme!IE(?YC>3Wt^X|e?=l8DH!XKat#oUO=KnYFqL9-L|I>}HN_m$OcBAIv;7$uI?HB(Gt9|WI} zl`&w{s#BqwwtjB+Ypwe7IL!yy9RRo448RFvkRtlX2^lzHfRcTdloO_?_t;>JZ*&8j z$EqRT_#UlzfPjxAClrRHfQx4Z{%&ly#dh)A>vLKbDidngWHJqX>a1Al|AzCu`m8Nd z_~exe1uGkuIiuU!BGEboS()xk-f#x}Jc5C+L7j_Rf(Qa9IJF*1S^K661yhw2ta!l9 za!p6VN?R%^Y-M=f+K~~!C!S+3|3f)}!Pli#FKNP@K!8HQfe-@9{^uvX)jqS@mgv>% zD|yzK6zE`v&!lc@%stu5Y>&59HI}1|Atl68#>6^L35+<97+6WJ<Q>HP>B;99w? z9w#H{fUgH=4!{OE2kbM~ci$bbFODA|msn&yhl-pKxFF60b)XXj4hwEPCmg{bZA|t? zJz6zCArsK)pwris>DwiGw8G$BmJkYVnaL)w_#mXPg6j>Gz#&GQq0*Z3CKQEPqM>z{ zJ$%+LmzZq!VK};P_@W z+Jpv7)*7rao_z$tH(WJQC;U5sjpYiZldPiz8EAPlgB7F%rZ-B~RC9Ty%`21>NC}B= zC2B&TOudZ8g~47~d9zYon&KxyMj+~N7(^~6tN;xABf#RY@V~}) zHfu7RG;7{LkJWKkA17yhEz&7m&R zq;gvC7fZl8WtwWR!f6o&?(;oW+ON;QNME7Dtp5u6bu0S}>rWD&_sCC7|IThFZXu$SepJg#WCa;eoJlKXyA< zc}f`S1g;1>*vzcA83G~LoNyz|4EH-S0!}U}p`kMY*OBRf>!jJzGzMV~Z3nqH9qsc) zl2cANkO|Q|C>Sm|suxiJzC$~x=ebszUsB!K{k^7GkUbXKf@3dtOaNN zTj7MS{go}M-k&8EPI&q3Ic$ax;Dm2*9U^ zR?b=6sG|U+0J2E4jFZ9`Yt;||7nr&>6fM~CoDeC2I>DfQn;|7=?}cosoZyrZ3ivrg zy@PTBDIxg~bDM*r=i2zhSRv9kzdrxv!P7o-{cF7QmqM~P!U=-1&cZP#kLNJ8&gwJB z3H@(UIntNvVx(~vatMYFx5O$;zBbxIbOlKkG0_F1{cx-@!EnfI1c`yj2JBF{AQ7w} z7uX-c1r+;6=&>35JS9Lu+2n&Rgwn!X(ezj+V70UvF|-}B!Dz2$Ie{nxqBmiTssJk} zrecBM6pH*5`IaL>)X|;fgf!}evi_k?7^#FmhMd6BaSxP`Zgk0}oWK)-5T^NtyL~@a z_wX@lARoN?vb+e#lH^u6;hQn+(VC{{(c15Ow5Au&*`qaC`JUVIxK~e~-2eIM(}{kWmG6&3{yT}{1cMkT!J&vG=Y!N7Dn5+wXKEf7;r)Y zN?3f@bAr$V(-VkfValpxXuW^=;3x5yh%V?~@SQ*H-#mNz;GujPFZ!iFCMqIuf=k0Q zz6&S(a$oRUpV4w?KnYleq!5*YS<_~KUqc-U7RarK3%X#qWLCWx1a!hGw^L|Wc-{~; zb9#OsfGn`ixda$%CKxcpbS?+3$H+`#gc1N%3b+u_pKutKoRmPM*$5;Ba>6{0#tLH4 zp`92I1s^!2hKBGV!_sDS+RjLXSxz96rC6tXPN)v~0Lw%tpyDCSD-ER32`M^~Z9b70 z$}uM}Vyc8NMH&0CrqZMJSN3R?-m6dEd`6$8Bzrp?`V*@!Ph4 zp@r2iL^1S)`S$ABv-^*pKYR9D-@lZze19YT7#2TE_t($Al2ynFrm-NI&v@=Po}gd6 z`LKdas0+ps859IWP!uH!5LjSSSfeG3o)XM9PmDDyfEGg`oPhJt`WPiP``e|Ei01^P zU|RHJv6WS}K>-owmo-H@-~^MV%f)+g0)d(+l?hx73yfVVt!N&;YO|k37reUv==JqN zw%$B{_TcB|&!4^Iu~+RoIhJakoWnC7JbcyEVr}&J?7yQr-JxA(<(57qGVP$awybdk zMEyrcf^`F0I3^UVjjWKb)N+Vmr6oh0dZvtsGf?52e66A^^gv;UFj!Jqj=}jKdD2cw{0QeOy#gm+n zV!?&)LOB7ZipHSyTFVi+f)gexA#y^{37OErC)wCwMV$~C0cTS}nBwwF^XS1l?Z=v; zN9!e&|KgGKQ#u!(h&mymP!JN3-~VgJ9<6mJKl zzQ0pWc+r1;|KRCWMNasdxLW!^zy9$4&4XvR=*;pekupnDvs(hK1WyVW=6O{r9U$(Q zSzp(jQt4NnN%%XZ!3yC7$Bs!YlMjYm5Vlj{gjCLg7s)jV(%3T7a_g#2v%0lH31#Gj zPc|bZypNoK3b6N~jwnepL?dng{3p@=@F&Q7d)0TPjQ_Il)pkhxXL8Kh0Vmj$&v5i7 zJdxH_H~r&B68A+b@ifKQ6By46G5DFVk_ij5JjN0=j?l4AV3RUs3ZPg+U<5Yc)eUcW zo>6(Uoiu6oXau8qAe|+QBK50)qugLp$ClEAuFlCpHsf&dm$-X0Zz;`U;9)ifS{})= zbguo;%+t;mAw4qE!bCD)w9U68$GY;gcfLcvZlLKMwp`0eyrNJ`O)KSnS&E>EV+5R*9l(@;?Wr5Pkvrr!fr;7 z*5%E+eUH}uw+Fz(w_lcIm1T-^O0zlQPo+_Qg;CAo<(6y*tR`RY$KoYS^IyPzjv?eF8 zgqf__k$kdH)yz>y0AImpAXo-zV4n!sw9I4!%VXE}lE0p!WUad;CqNiOamZQdPe7MvSsV2+ z+VW}{`nztE)#q2wAN=&_4JOLQ@Fx<7fr@r(IH;4pfD=NUGZK#S&SB4B_9&lNz)DG@ z8498krpe8piUb}7gOq3(B2XFN1OXx?a=5i+nbm<4{`&i;WZ~uOz8~Xeu}HxQSU@F+ zVBf&5N_VBK-#ic=xP2$<^ontZQWz4)Ivy)4ZJxGU{4Ctj?0(i*q;<`s$8GD%SV+Z* zg4)W)&>R3pKogVVn0MPwqypDA8V{5StM%B2KmrJiz=v}ub3ej)(7M#cn%AA}y<5r0B8&j^YN#V6$BhzT_e zTCbF9M}Qaz&j$pF3%p3fAwvBLP)|gkiDF8JZ z=F!A{tl)&tYjQ$7W*vL9ULZpydBh&A>GcjhT6g_*dGqAKq6b`k*%NK$zQ2Vs_}k5fs&AACCfBr^-9| zD#EK6Ibi{h!uurgaR=4USE=M4-O9Us+P@V|@I~CQlBsn`weUr3C^>ec$4YnG)?YH) z*hi19`3%M#s{S{4T!aM1T}wIBrXLXi)0ium7GQ)4IIKg)V=S)*_Lc)?{FYr@xQg&c zT&A%HcM_vfQF}~gH;8@Oi`-hH1*U{G>X_hk5R?j33cSF4IF+anHUZ;;LeLxUPpC-= zGnnWEHEaA^eTxgowxq$HBP6m=Z=hA)^Q36Ksc1d>Kf- z>Q=g|6l|Hjqt!5vT`#}&uhEZ{obVN#z#gssm~+qx)$(eaC)%Uc*js6?vUvOU?ZS$C z;n};ce2Ms)BkEcdz=KEc`nRw9h41wGT(+L&A70-7`Q1%IPH>AlrEhb!dig?}6ffSZ zyJBJMT8BTnDEUs!8$scFyn_^s@o5C|;Ch(=&|VPB__z%21Z>eFlOxImZ##741ZM;c zaH4#Nm3_`A3VhK`;Do;-KOxCd@gFRL?XZB;@dJu=w5qVysWP7a^x{UM2T%Lmhsm{g-F%m0j>l5_6#Cm`q_<~bxDYeP6s3Nm}#(9bxfgQ-;D<*>I( z8hnf%CnFC7)zh3tRanx3`@tC}G+8r3Ma@8@3de(Ti7+Eh*f_5f#`y{ATqNXXU^`@j zd#Xq_I$1{X?aQ}stdcxM2~w{V`>~pbPrj}`zn5RKc&yIYqxDJkrOVIXqep9U^={vz zHN6f!TGM(DbbWpBVIBu@GG@%@LdKeLm8 z)okO|vA42}UvR=Y)J-S0b<_dcmd&Mju$H^jp{GhhBciPjIe|F}%X8s`bhF{Kv4AnY zn+m?cD0~377_%F2!V+Jv=ugO&pHh?}XpVDY9F?W*f4eXE_2FZQxM1tW&p&rvhV3WF zK=8*Nx@IW^2w$x{Lq5u%e*O88Z~{J|2QPa&vSsT%T0?;llu0pLjc;A3k1}HMMOLuE z-B{$L1dT4ZDByL2i?=wi|My`!HbOA188buMh^=73-^h?Rqybe5r?e6 z%kkpK)UQ%|iW1x~!MUvFIH8>$nXtlXULgDwf0h)mBy4$9C%AGS#ue~eK#&tcrjjzk zfjGdVlQ8b*(*Gch_aQ zH9tT8yk4ItzseE0NG$ylNlw4~mNp{6VgOaLZ=U_~`b+sm&hlE$(v`m=$?5Gk+vLet zWJtY8*1o^%%Z14oFMpAwi zPj_Oy0t&pwri1P**WdvJT#iB7ux{sN!So1W)t!vLxWKtfN* zpG>MB=>(^;A1z{XyS9YeBUhC`xWEDe-6BFr9jku~M+Znvj7C*%3^Uz~K=VPjs=mg4 z#%babR@0K{r8~%4N?oG^ex4Iby??|Bf*%n=J7R^LtT2ybv)jj}wCVz$+72efB0Upy z9)S}gBLJ+B+}jQt8V&cHRQLaz;Qhz5tLw&d!f8rieRD5Z_Px#fesy+s*7uzrtNQ*5 zkqO}X%J*nZ&erF>hil4`8<%4ybb2rDWF~a0=QvN+q8gL0*j@qOuytWpXFc#G(b^}G zNI+(2Ai&g5%d&uz(P*P38VroXcId-KB$p(u008J5VB;och=5zv`h0Od;);?n)X4~h6z65um69iE(1f39Fj5zSmvqPYU z$PU_m0hyRC))unx{&*3$E~YNNg?7laFiR z1XJm;_S!mOjO!Y*^Xkh>^l1Giq<*$emPCqd3YtNx5+ZOgt8r(VmCLkkWdo4_lQ(t4 z(3-D-lIIp{Y#5uCbQ)+B5;lPP3D`@GVm>b_=WtJ(dQM;pv?8(K#}da-AB9@a7QNRn zAKVyAJX%wn6voidaF%l{c6C~^!jcTJ37n8+W{|U#f)UimD8@{TJt6vK@INFcvg!(YSSx0Jf z60K*o9JHWGno;cYawp8s7dh#u+5+?^KqdTVR0&uDC#2K~-~_23_;&l{T28Rh-fNK( zPD_PMY??ejgG@NT(qkiBUiR$K`pvA_5Tz>XSax0qRI$9A_Y~^!tyqKUTdm;%TU0&* zrk1;NZm#7~xi4Na7In)$g*i*O5xQ!+P21v7hB@)rsfp*+%tkVT2n1JSgZ_kA@!*#< zcdV9*i&;7v9Nf-Zq)Nsft+X9dK_Vnj5>>XOMi9W5(U$G)H0v$y^ZrWU$^vBoF!YN3 zht{%xEiee9JIgjqBQ)S8B57tAfzT`@Mp}`B>;iJDqYn?^GRFv6YzJHV{a8lz&tXup zybob+q02C}yXME+B!ncf8pkAON}?vVL+UTF>Cs%)($Jw$$4ALt|a6f{s5kyW( z&j=BI|L^7m%=z+rw>#Ec8Rq%+&uBOBcAwjH*2LA`sJk6Dfz^y=I?*OxB7-mXjXyWYl)M4xc}R zB@|#$C}_jhaG7^3s7@I6ZgzNq!&u|Na6D|^9RIXgbuirqD{!uC2W;F7e#{B}eg_oN zBY4UOp&f96`5i_wA?SoG>Vz*hSCAO3wl$VJZ$Sxn<%C*fYmuL=g%d6Y;sE-9Z3M6d zo4C8a-okj0fe6g=0j>@ObY-oN!#olY2k2N}sTT5;&fZYgo)hv4^NDS-$84UiG6A@Bf*l#5!7b3W9eXb1D`|PK9dQCT$KdS10PgXm zP{6%q5PQH-W?e&$A)K!~DVQ7|wpnYZM7b~*fk?{`C`hWdEtg5El;DF8);Ks9xq}rf z4d?{*ERJX5Uw5|gG!)7L8p&*Gq9c$R$A9=j(%B#qhTHyE>#pCe%K$u;azvP0?!B#lMOd%1u3-f2AgL4)a=C z{fuP-m!a(-v~X{S&>Z2|Q>E|kEE4)2ro)gEE^inIK;xBGZoA6>-nm|Z6Uq@093*@5 zW_&+LfsI>DzD0k%`Dh@7xCu5B^Cw^iXAHYWsbmd5yM=%U3?v5^um%As+sEx`zK zEjTL-Z3hCSV^Jr7JLiRJ@VQ=u?*Hww~&xWg!>P*yWKAupEu&kV}E;`ad9Z>qo1>4Ij&k3XHBKJB08bOxjJq!+d>y;C3@5%{RH_gje z{SusTwwC2mdZy4@A)?P6J(hHI%kY(PYSEGVHZc) z@}xl90kJ6Igqou@Z#wn{OY*I=CLs*7mdFVm%K@yhc(|fPzeZ!6>ICLDEc z6D_orl0wNP3zkdBFgOki;>zBwq4xB)l(DrY& zGi$v=PU7pn+$p?P5Vym5QH*?yRagw3qruMhi%|NDB89lCuskKTvL?fu%Iq6 z9tTuI5Z~#*!DA9*xwa*tgjyf?|%0PZl|kEh?nhw^~3-s0)}5 zv;7$3U#khx zt|lW~qD?5cU`8i=(&e7hhKzbqPEefY1oSspERAqNvbqqJkd@cvTlY)g1i7u+H0$=f zaJ5|v=w&1kOab#pWluapbRY$?0xkr`e_~oem|32-gcd$mEi^#43)4QJad^k}^K#t? zBL-S!fN^(Lp9(0x3%*nHjyd2n|rB@OQcu0_xHP3h<&Q*vs<%AEA2~L3` zO3g4XwY&V(EzlnB*MSl&D4{<2c&UrS&D=}iO}q>uf1=NI)RAOJa|sX5Onw?=|lIF41DEudv~2c zPJsKNH^^YzUhls2eSLkcT;Mq&6Q9HPHfn_&$vrU8#*u~yEtMS|@o3=RCod>8fym;~ z$_cW-@y9z10*akTh_&qJLf9?*r~=Z$CR;AEnmQrPI5E93lnE}ufL~#L2Ot^`+*=kX zMdqJTz-qv?6eemwMZzXN8!FUADkm&NCBQy)9`rDmPzh-^>nd!=rbDpw%UA$VpAe!0 zQYdRqfmxY}PMD(waNH=)@Q=}hbs&YxWr!iEO0bR?PT+KW&ZxN|5Sd}j3V{@~UDJns z(;B_J9~VVUTfHZK(#~KxRDU-oOg#!HFCGv9Z%4rX3@xlJ1Rfi(cIZ@ZC!*?66>QGTxQandtEMp&69 zBysV_U2e)HySM_Gp9aFWVYW;+NNlZTPlBKlWGn6D$r^dIwP3@pu$U=*B}yB$4=t2~ z)xOPv6BbK+*T!*X2V2dR_1&f$c2*VrBt$dlOESw2Wy!x&ZdeY0L`FM>8v(793cl!R zMu-pDV@wLJDZvFYMye`ycK5(O;h&!pnLRET9Ir>(n(s+LM~|Hfh1iNM+;SCk!m`W= zWd*edPB|H4RG7jyBp(OVzKeS#g?frW@guasj`044WwJ!~6tZ4>=)&F9FLvN`_6!<;&eTfwZ&R_fLJ@zejCv{}xq_st?LeyC*K6{`9=o zDdsRzuD0NWW441o((FZortdXFA49V?qW_y_o9AK(G;G_BfI|T-fT1pMIya?rRk_y; zEx@03o(-8x#=u(WDD^Fa1}Qh4%P178e;*kSNOVTrp>C8Ci~%EHDeJ(YI1hrcl=j<200k?67RKStjY_~;Tc-auTLz=bRDffUBU}*d3IYNPk z_OAgfp0AA-4J5rOg%H38rK8;2w2p0@GtEaZoT@%au}RQ2n9TXYJjTkd2yd&Dby<)k z@{+lciA9ox*$p`%g*!J%oJXdtAh&pS7zudW$5A(>)!ae-%x1Id<_y#pG|7}KRn+fo zh1g+JR2a(yB7_|rL_(m16OqVhI}n}`${~N$`z~_0^E8^^WSA6DvBXa2fAAI3 z35j7L%Q7o1HgUlMB*F>p?otOU;Y9)vbwZR0(VwtPNelNV88&jscKw%={sd15ffHW# z*KZ$4$3*FdWT%|)3SE_4-39C}yY-5kFr--fc+h`8IdB8v;= z9c&qFaDfr(+L<+ehc)uB7QS#b0tMX1x?zg5XoiLaBIHmB#a71H7L>s1Xp2-Q$G!(3 z*}5QgfD6(rQF;JYo(4u$4@<6dKbg@jI3}hO9B2&xzvez-Kx+~-KLoz(kU<2q)hr* zQzo1&Y2YI~VG(v*Hj$|sz}(m%1Rs&q!Jrx2Wq5FgyDG9GIW9CaM~#PO+X!YD!CTH0 z7&+(NSOzy^v>$+RwnD6dpy^O%`;@5>CDcFUgnC{l#Q#L4tPlucgnEKKeKbb_UWkwb z{3{wkJUHNlOOM?i-8DHOEJK~(9gLqA!U_7JrpsO5pR^sWB77{ed?UVu_reM6)*3j$ zwYdG|!K1f**0l-<1>uBkp&$07$|1NB8n#ASZ)F4V03R(tPyx0>n+pYCTDY#wG7f#E z1RUwMYvwQ&t-zR~xuHqI#L2Pkkgyq;aU)W28@EVHFv5G>yI5!$ggs;h;lY%YAee?By?3$psPLdk zolOu5&c`B?rieo~jaq@*{E1yrCY&%C1Y-u^B_tfhJIc}u*_D(DMX;3$*Zf_3gFB}5rE1&h%}I)O_SWx|vc zswsxGfK5fxJU}xKazZmk4Nkbc0+bGhjL@~GOf7SB##ZQkHeHJ(Ykl%4T;)IyKq-Z#>ja&%jV#m+iDYT-Tz-i%4ngA=0B+1ALcAX>^oDfN2 zZ$>11A)yzr3pqg>J5bF~C;<6(zzN1zMcT3HP-aH*1eOc_n1K_Lj~XX5#W)OEdUZVv zQ4Af}qhnRY2To9KXsW1AFiq(QP{Ys_U{ovEHK%?%QOTeipnzHILk}=~LaA#Let4ah zFi|fQvK_6Nea3QCsdWWjaNg4}%Lz&fx$&f6EQA7e0^ar^I~=$Z5{Gy^D50E=+`Dl? zWCWrf_g@4KnP3c_rr>8Xiy2O+$D+}5!bBio5l*-oZ~`x~j5uK-IzcwkOE~+||Il`r z)IoP@JG{96=>6xj#}D7IGpnDiDOkb)DU>pP;EF?%JYpX#BM+rCRy{2DBW1~(0DU@l-sJT)C zIDubZWrgkjCtheUc;hAwlu7^_V;ceoC-wT8GHKLwpp#FLJEtN+`5={bYGIVGb9Vw% z1SjD;0Du)l^!W*)Ler6|3ZsX#P{bIe1FXL9!m*P%jzLN|dx8{_Jt<7R3CBw%Q2+&J zXrP3caqz?b369lh;qf7ZJUqMsVzF95RL4d&wGqE>d~Pse*BsU$20ssUjUX>a0g@Qay4ZpfAQ&)YOb&`z=NAoHO*erP z+HTX98SDkwXmhhNTjn)d>W*Aat+s!n`G;xg%rbMbmpeJ=EI1*8Lc!~VZHJtOCI<_E z1j@FyzT84yGPU^M4CaR8Ff?+~fleq(lfVZ?Mi9_;h*Q2MdE-$tXPC{%3}-DaVN`OS7$_4g!$DaVwW~q}z)Zr@xM)4d z1m%Qk#Nd0WustZA9K*Q@Tbu?tVgEp${#-fXC?!Ol;6;L7BPT3AE#8SvSb`E93L0nQ z4p&F8yUDbFD)JaVzmQI;UnFPAIql3^;`ZJ3mHSnCt-8HV*s@F6z-p>=-DpwR3X7To zVMZrrC?Wd*nL!FUvL*Yj&3fAvE7{dTuq{1+=WCk$HNJm!3pav+15!3>YaN0WzP&1X zb}=j2ZG;n?1ECmB5@$Ro?0t+uC#V$yfnyF3}FPCF*|4r_5&ZK1t>OiqnC0nBzrK3(L82(rQ zwm?b00nD64ijthPX3@0OrcLFD6A}mMF^EMXPR5LUQi77gVm~Dd*SU7~#b~M9Js-d^ z`iU2XCYZ=UAqM1xM^fNkyXS;j9zbt`z*Ys2+oBxL31?q>>hFK__U4Jc0`XNrdKx`<5_vFoP_}!)n*bEIp!9(nty@NLQiY>&;^6G?5%8 zDfmF7E90+}YWW`}C^%qb%i+QWj}Nw=qKWpyloiOo^rzs7^N97AqkRqXl3T|87`wm- z2VjrZXgLtzgzsNc+6~GHp!Lm74}3pjJIpWwBA=N6{4Jb-Hpy-3I{5ih0Sx+7G^Z82B50nnV zZu9|%#DWuA%qR~Az^W-YT?<}N9=&hCkfi&0q-<-G)zD^${scrhou70zubCMsC#u@I z)LILj*P3mdYvlyvYyvn1oSV?o=Z-gDoF4Lho4HKU3MY^X#yWwV@M%Gn z@DBcjEOYH`4@`2KuW&sI7C+0SO;Zb@kn2lEjFc7U9cU3#INvI(MOL7i7c1amn-{Py zgcFdH#2;~p14;*Cy9GyVo4Q>mo&rqKAvg(~fg>)r21*1W2NpGJ1;R>)v8*t|ygUx` zIN&JoH{ZRv$#4EQ}D6S*grr?@Z@E{^t9oQyCRHf??}{r-h_>R5e8p=tmpokNHT=e zPq`kYeR7XC%rN37Ub_FcG40!|JzA$|sS|=qi13^MnE-IkmyJ_1Ao>$<0nlP~juLzx zb>=x?i`Q&DntECU)_P9ZdW{tC%FSAGncyt!I{$kwXS>X?K}JxhV94df?bcfqMV_OJ zLKxcTVn%@!a>#=Yldv!^A?NQuyET#;jFGEzh8&C$R&6 zhR=11f6u^9p*$eW2njEGlSEL2+#RGIqWnitV8LoAjx5g`sBJ8o!X?NAf8jCRkHFrv zM=Q~`ytcL44!Cj&obcuQmn9fMG8xm&J%q07?;}nS1WuU4C0QW|g)<=qa6&+%b;%P_ z=yGl6u7QVe2S)KZM|OSzBH-_Rr(uNkSs(;1@F(VShK`R$4MkIc6fmqo2{^K4=&_}# zZ{;@V1R(hcM$k#2oyF6z2yCf&jzTk}iXaoh%{#d+&Km0}0k9M}>js8{pD%*4zY(7H zot8dv#sP3bhN^z;JDwPw?sN|+r`DMUA`*n`O6_ZbTB#pnlZj&?<)eJ#>#`l7S!<}1 z*8rnVMAX|8YtEgK9}^sL;um{3r{WZuF5EZ(OlGJ>&D}35*&sq-vht1uW;Fud!CHZ? zOr8gefT4-0LqUO_({M8i*6K>YWd>{~QSzJG>|6g;Tr@;Z#+o`59MEdM*kSOYs#{>6 zHWI>yL_ut;Z3QR+!<8D9kPdp?`1>&E(fW0w53nZGcHouZ1~~!lgmkl!5p>UON|+8) zoxmH#yiV{!0b+m52@{CoZy85Kt#k}T;)62u15h4ZZYd5Lhykw|U`yor`Su)6L@koG zI1yN=W4J+|e0F}ZLrukH4o*NRgE2;b!>JQsUBCgw*@=}CA~?}zv88nas7@H;!&z$* zo+!a9n!2?sF;lc=+A(NCPrx64up{H|zyI+GK0A;?ViM*lIII&Q$b$v$LF8eCb%g^(9w@UB*+b46vrm;l@GFHkrYy{bWvpEb)2F( z!Xb|2bC|(2WQBrZDA_7}esigf7D8B$9{>wm@*LyGDJjgm84+-xR$$3=!V6d7L*SKw zgHb=Vh|@*V>Yypf2-CoW&mK6Asu-ZE7;m$Aal%B~3#qsQ;{K#Z>xd89h>=c+ASWP( zwUfRm-R2&$;dlKZClK>|K%qdHu)C_$EYB$vM%`kkXxbd*4n?6^@0ru|J8}X>HRF`I zthSd0`k!`NjK8(eK}Y}T4)ssq1!D{JX${H*LIHF_8wzEU3^K6d!EnIR2r*b8o)Kn+ z&KwRArYHf+yCAavMotI;Yc9b38)he!B}K@ zb8`bm*lg~>hs~(NG!{8W3Aidi3Okp1q0lOn5vGu_3wMHYLM1O%l2I{u=_uG$GICDvC22NO7 zO`BXGQweM!+e?ESg-8mBpcMrs9=>z?#f?OgToAt)N{7`ExsLmYkGTa5yd2!0{kfv& z0P74Jgdg)Wyx=Vdu5gXOp!aXfcLv}#=zx6SZ+UKYsK%sV^GZq4Q!mf_&q78ueS5e7Q=KrT z1=c|IU#c4+1TN^&s_ujscet9R16;`pYBpT4DRY};&Pe?4YgzXC)6qQWP!sSP2D1L82&{ZrgNPW2LT#tpW1f80#A% zV2b!i3D5%gCBctn$=NMRL3fG%&|7$C|I_oq@Fwv*VB1|VYw{iHIB{{T!>pZayA5&e zbC$x_&@y`;lwg+&Ey>1O_SEU6q#%oOh$tp>PHygqT1X+?b3$1jHgRI6=$IA~^*Va( zu-+DK7FMTyzzWboKLg$yRuQqMXvNf2&sc?n9MGOZK98x4v=CWAjyGG*!4*hhboO-Y zQApIGV8I0BW3KcvHlRkeHq`{>EGHBPHV0eW)gkG$z9TCP{y!Rg4LxT5ab!9ACv@#Z zCdBz7NBB>A%MG&xJNIuppTCF5#t z>FT<~7o^;xgR&`7*{R*_Tj~0|)Q=DW#tV+=X1(H=6n5}lg1p09(OlAPj;0D^7oUk> ziyHBT?PZg1cQTZ7t(-=dBED%I^Ai0;v6dqm&OOitejaln0i-9xCBVt%yUWJf4m{)M zci>IHYQ<;gpe$^!f=sAfS=$6rQVY)$Kcij9V|*3BPoI#MIG{-P#H-Tp$3Gt*_0GyCvt*ib*q&H0s*w3Yh4nq3ENEiQJl6MbnqS< z`DGp4KNBLDnh#L~)8cU-;on`-~(9) zyTSEDTQ5K3Hm|z6?|IObeZMQ~)_uoz$t4|!GyURwQB8l*1+hBG{#t(h9tAj!)bC%lH~ z9qJCO!(M2evn5g+Fo!JKt`Yv$_fN<{U4{`%coUewm>0qc!dKQd9qJuAt%3UE(;8}k zT9urTUUckn@y8Z+8eSzr zWz2euQ8)|Bd}bRd;Q&iqMwnp){g#uRN|s5|^~90ysz3{LB&cfGvx4*4jN3p8Zb}M! zz(3;C(ntkZkq&})0r;qtKtxWsV{=w#-NDGGTBJ-k&IlvwiIi~Yc%WdIWRJVMkOLkS zb;20&HA-@Vh9V=B6L#%Rgn|IJ!_5IF{4m%MaaHzjetG`z4C)(JC{Nz5v9-=Sx*WGH(5d@Uknn^+bG0Lyw7I$@ zCzKFcOiymxjA6VDb%G~SB~-a)=U^0h;UeTMZ9xf@L7>DYc>$HsRW{T`O5CA9^gs1x z0$qt<1&pDEk>jZx1uck7SjyrJ2dtpwg%wURLfm%CPu_T_4vhy5MWFxlQC2W{b5bN^ zUe=BVpILI@wrm^_!!+&?l|ro%2N(LtaOhi&NmL}EB9a4cc|Rp2`^GrWHF4h5Y(kxvA zxGBg9wRiT%i~x!razZ47IA0_=VWYSwX3&>@>`$0;B}7WtVO91oAKZT|U0D16_0K;& z`T3`3$yN98@pGw*eDF%jB46}rDut1+`ttctkDmPW^lbI{?N861{Ui_UUp;+t|L3Pq zpZwNmO`B8j(?4L=ff8i&$!lEXCRf?B<(TH9&L9&yL80yga>7N9yU*~g!jX=gkS969 z8j8PBClotHXIo*1+R=CrAP=CcopZJ71W*E&sIfK|#%Kvva8$NTU?h7P<1$AYik$Gf za00nIwJI0r^+tRWR)p1xR4h~qJ~ZH65bFl{Sa(Oek#m&;PN;@KF&x28Au2r^-unLsr#l?gmLDP1_(xwVQ;hENNB9B#15 z9qNSIb3$MPqQ6z=e=sNMOR(KAc}%bWKC4AV3@vq=1|S^mhhlAV!%dX4}@X zOQ_7Sov#)v8_x;s7}?fk-R?@+t<|3(Ac=B=%f?zxDFtCFe=KAIDM4_S8|y&HLP+^o zCuBgHLm_b7&MPtSSt@koR!?`3% z1hG!af>H2!P$3Xm=pnfz5$KehNZ`aWr3G9ja9JP)dJLkVDGIIW@kLN%1x;fZgzU;9 zkb>2LlFk_q3@mv@Nc1Kwg2XhjKSJdRVYn&Nv49p8V3c{UX{9G#<_`3T|6aX7j0fAy>xC#5jxa(b1&j`M7is{=l>#eQ1Sz5v=DHxj) z$085H-4F_Z4@G)~kc15w4)w5QN@%1KQQ|s*!{TMaBtL=5Y}Rs^Qo=k&Pr`WvvkQ*f zOR*vqI-v|xsL^xJF&jD@{22*>H&G{eMi5TexU2qdIsw-x6LwAiOuDSqyEg4#ib6>3 z#nT6Gzdm|=<*wd=6JATB#^ukC+naX}?>`WT#c-p+25YPAnwy!Umx+B8;bzpyPX)ohM!=6t-z6)49y=x2RC9$F>8}=zhh=d8y!>(P+b($OXq~!CEyPC=sXhE7=G?%%|zP`!Kj^Go2> z)i<_wmLKZE_8NC^+F2nFF2I!zV;Sn&PdULDx1@FQZBXN&}^V1qnf}T!R(@D<~cs*Wh`fl z2BW|7tf1aTkU~+UDWt>HrS!mf03{%q9r)lcffn*+F!INo5N^tuP$Z~{pXDTp zamooi(IIMJjuNT~C(L8!KK5CX!5U+QU;~1j5a>X`4dn!-gp{_!ji-d$Ii0{fW_g(r zc8+eu-Ho>$oLCHx&(9w{{cJAYkrRIV>G{)#5`4IR{NQ~LC=sMhRW4=Pi<~%$b`IcCI{Eh2WgTV3;`4( zKMY6|{sYCv$;EBBQR0>e0o z@LB!9swl%7bv-5pl8dWenHloha5p%elry$8V-vlg9zWiJ-AZ!16QF(*_Lkw52OEP_m^M>t;SP)(skFo)Rmy`JHOQ`8Xq zHb*junCb+qA}4T8IME3q>@ej7a6u?-IK-|L>)^p}P5b6`-#-<%!FN!EiJVXrkF{&7 z_!ytPU$0-j6x-o}9KcuDT?vD{6aVCw5@T1}CgeFx1@A7{3i2E`=Q%h5&6Maj6rBK0 zxD<^*U2%SONl#=4$6?z*-k&#CJ1cFK6goN;u|k4l!+Zq7ZT9u2$_V~u`|n4PHD%TC zjEp{L+7fdn#hPbhIkT-?Q4Bf3BmlmUU<8Z^{fQ$Pd@!lAwNwSd=!LYh=M^CkVFrwvg3Yz+A^&bAGlK(_t+{uw&abH5-H=tdIi3 zlaLPTc@(bUZP9iCZNYi1Biu$8Qh`(9K%Fqfrv~B595DwW$*mU*-EO@qy0#NtljI$K z>;fnJft;l^Oj?~c!or&fQczBiQK<$mghTEVc?;(`VcXtOA$VlVG%eeyKLOjC!#L;% zjV|dHKFz`rkr>De$_fI^Q030hU;vzS~H zEQkgZN%_1c+4LXOItT@u?t4K_IRPIp@j;mZdMuxa1YjH)nCsW<%EDS7?wQyQBc_;~ z52vaoR50Avfj{quptEtxCkOAug+5DTU~mLZh-9GKVP5G?29>~^rSD1!Q76o~7k7RY zcB+uG~2L#L{2~+BkcqVg_W_G#ssQ8 zF)m4+Gq{f~$8}*%YJDz%Ss`U94tiak*soK%6Cx*+)xaI)>fz(*z)n+htQ3;8fZXpG zZU`s@xgjeMi^1H?3ojLv97qb33Rvt|CvX)IT%WWOxTi%#V1=S#Vgg7ZWGxkyORbht zv(v80W`u&ZD1d|P(U^m*C==K#KaSKXC+JN*liCb9i77aa*baf-W@SP-p#)-%6Cx|% znC?gXD0^bukC7V@-4%5LF9$jSDgi*;p}qwp0LM5Xco#h)HQ=V~iSD}6Td~yp9xAl& zF_h2yezj&NwH~-?n#;a-9drT=#U7|ypu;aZu~z#5?t@(yCgW#15)UFYd4}CBEE>)U zg$JStOxBPNx!4o}--wAu$zCV4Oo<$71ggPcJ#27{N}-z732l&NmpPn`a!PQ*`8lLO z-c@B$RB?WX#Ti<|YuEwR*(n?1%8px_(it2@U2ks9(zu}4je*5`N*#>RmLfsiP$qC*&f9f>A@I%Xi=RYO- z$NLBO6Xh&oqpGn^;B5!&4(E3`7I(vyztzg}j7Z?M1gjw>tUxevyp|)>14@|Xguzz2z~_vgk#TXZ4%fW$fIAt29%81SNvD+I2F2?1TuP<4Q- zgk$*$p~G54R0_nXpMpnQ2qa!`4u{~mQbMaWaHIv@E4=9lbl!TP$H3&sT@v^G#xp{$ z1_(`yize5o1lzZ7ulS-}#eiDb33B=(WEEP#v-}!R2hfOHr%nZcI)NUAKMwo}kO`d@ zmJBr}=XvJt1~`Jd9D_0RwK^Z&rd2u?^+PS*l8yHMM~5WHVGK@4p(!wIk;=u~io9Y9H8G{G9EfERzu1*BjYkThN? zxIH6Gg+f5Bf36+`Dexs3q0~AkI2T9?>QUGm59EePFf!yPz(Uh;JXhd!(sICuVoV(U zIETWwZ9IzdUxTOy%18uZi`R(QqxBdk%+4X3Z2}@EOm%_|_es=YydV21c(Kw1LO={r zR*@2v4g`%C3j8?;VCj=Q<4n^L@N3_-2uPlxA>+2u@WWQ>8l*Z(aDjrztDT|&CsZhK zq>s`EA3QCvSLA_8Ag#(;wnokEHr%iOJ>rj{PUsNF3^kYAreKuBqhJR1`b?WFNjt{h zKb@nyM`Ac9TuM%m4fQ+v9&A1tD3OZ~>I)j;?oxp=p>7U2!T*@lW6PlqoIvBDaC0bx zoRtr#)dfy3<%kr5P#_zS9v0YFrJ&XWo*avy#;gEqVf=QWYXU1&u252_8fOd3O=ln! zOon)5qC(+VX{2JMgn-IbAkG3kK@~CYOfbO+&=mzgj~rSP|9Vy!x7|M=f{lyFVvp9D z`5m}~OY|ooO_;3&!3m5yj5uNHU!+%Y!U>_D0*ONzAt(e6ZTJK)nr5qn&>G6*BcV9y*Zl19d%&riA-a6%`K1t;usv%M&^#j~(f;d{Tv$WbH! zw+VhlUDLBr0`vI^HJPOa8e{a_q0PK$WbmOgYK`XMugg@O%2Au5FtClL6o^O&4b zpWp=G2b5r^l%RQuvoVK%RvXMCiy|T8E-zTx%N+#s6tH@!6Cy|oq0f?d0^!o%-E%ut z2|^XO$_bNfrJXK2Y>pU$_s0V2mW-f3?dK?ILi1GoD|U!EHl3#fS%Oe#ypwU~UcTzf zwv*f=h8xeX`fm(1VBUFt+FdJ;s5_4R9DNwc3Aq|9Iyo)a1d}|_T?tMAbC4E_DpCUB zqAVabymCbx^)YUm+M^YysPP@>FFS+EY4*-O3!Ly5a00qK=f-4Nwjw9sTl(!Ar9tMP z8PFtT`|>xZ=0o!G#fukjzn9G%uCMCm0435Mc(lI09DNV@qaInL{oCLE7AV1GYCM?Z z%@Z=3@1M~TCv%4o;%U~tMbpcd14kn+q4q&Z0YX6*n`)!iu(&yo0?|4f=0;!xQ=3~` z)s>IcmX&%LjVnUr;Yjml%eaG$i~!774#X)+D6O%2k}>}0IKkZwkrDg`8mNuX5>zkj zsX}ou#GhL06mT!w$$_a(xQreJWd*@Oc0#z*ue+islo9fCWF&42vG}(rXuf1WR&O*k zKnOkf?dr=?iYT)dLzpRuXGWfp=a@NiO_@;d>Ict|+mPpE1$z9!2^~+UeH5q;go<>} zXn>%_HJX4r?t?Oz2nKS+oK9$25F7moD2hnh(5BFd?c#!9-%r*PkC>nWT_MEgs!}AbH|hkh%w1N^Sjav8&^Lv>do4_jY4O zC~;Itp=M{B2aSJ^x%Yxz>&j$pb`Ks5u3?o zb5tj^Q6^{;Cwf1wp`db;v+HfQE7s)7ox~y~XX%4MUs~7*mOp8U48q03r~M*HAroZe zpq=~8qbE0uWC^Fj+r+ElUmv80Z9I zAh_CDnLudSQ8|agm}y!<1XLCpFl?zFkgeDgX0`4KkOwzEOV|RSuD^(?2^jy2#>T=X zTsYC{yiOomA3tSZSayTDYLZpq!@;@tV|4da8lnM3R?{thR9s%-4 z98WbB1cx$V#0iu0oir!zWGuuejx1v~Bqy{;?tze>%n?7&3Ee;^Y+yCq5@Sxdbl3hi*KdoBs8vQYN@Z3 zhp`&UKjZ7Ve=qwYUxWbU*z;e0dHF37PLQ$*x*hbG3?2Ld(j2xp&(FW%%oH~vW#pH z0T!G3O+LN(Mlfvp`O;Q%*Rr9cz?EyRraFO~KzyI-1X>D#4X^;%1!2S51K^z7+R zsE{=0&)ARk%dYwM%l-QgAILHGV}17Y>2JNg)^4p`xAz!rr9cvo6AnlYpYh~npZ))w zUE7N4%+{q5v<=vZECiuJ2*|=KFNEJ;I~Q&{(HN7gj^{(nR_p!WVUIa0Wf`O0>-%Oj zR;n!1X|0iYs!>AK^-6DGe|;hDHts>ZP5U;uVXLKbe>dLMUS2CN71$j!DdQ9q=C@>- zJm?t%vN)iaxUq$JeWE9y^DkfyM$6ENXqO??~umZi6sa`zmlkmG*XYBkMPtJR5vLmQ>0Idq&> zvwb|ao9%NyZEbHe?tJ+(^;b7~LaAJPDHV+MfX`8Q?VN`+#}$8z5ecK@VAPh%Le${f zsEYxdqx;v_U+7UjIQZWesfTc=M-e{&o#8xCXpmDBcq!`VdIB3kSw4u6!jF)-Ruq`g z*~KGb8v?=(1@{3axC#@LpW-J>@Pcp1l{^UvdD9_}3AfqI&j8?Y{S!{i{OY|f$~Rcx z$)ztya{LJxrFXJPzVRnqF3NWJbI$1rIuNe`}%y51|;Uw3tU0N3Gt zlK54KVP`r?H*ZG8jMkt(o>5k2-NWH4bxPY7vgHemwsKj9Ua zz_R1H(iYE*W=wEB6rH$TtrlN#ZH-;hER@Y@7nTGuP1L82!ke&q7hpnS+5M#X@~xx~z!Gs^h6Wq zIJAq!rn5AR4Xgpe1fwFHVQdx}z=V&{6g?VfIfz!F(npV)h|%MH@L_0wLQfXdtCBGj zITA21f!iZJ0pCn6?&|s4q3|UYg($@YB4YFC6N!u~g9*+ZCTOlIInCfFHQ`wl(z5dz z%_;*yR?YKvLm2$MdlfdSG1cvQsJj4n%qiYBPVgyK zpOA&tkA{%DbH2K@9C)3coDfpT2DIz-Ii6zzTzOhnB$8Yj@y}13S?dWB6I>6GNWuh* z><4_OQ!#_>kcgcn-U_DhhlB|g+>iaZT<(tsg7gH8siazH&b$5BH!;!q1?N4igZO+1 zj;&^QI~khhCOGuzA30eD(Q(7J!_o`Fd;l)^Gg|@?R^uR?7zJQ9?18S+>`2e! zo&UgQn9!D)-!+6V;^EKLc%Nm4X4_$t-qQJnH!5)H_+QBRzh~?(azjZ7g$X32Cy2*- z0%O9}pTKgH(yTaIX}#A|hd^LzyO)%YwNRB@&PVnx#=LjEWB=MJnnjXQC+QU5b|14| zk$O~5gB^1;Nd{MUocu0S(VyxD7(4FRDj6pl`PPiE!y45JFGjtSjICGo$;_h+j%K-l z<-PG2=G8c*0%A{Pe(OVjg0tfL!4(hW#?GAe+9zhnPgWH)A0sef8v3ryOe&sZF%jFS zgDXM2VbLvB)_t#!qrFi^DCJ$A@HcJ=QcdjUOSo{G zW;gb5X%n-xpJbDObc(m|gbB%j zQbG+wpw^j)H`PN_N}Ug2g4Po<`hu1bCr7h9_9rkVgjqg_tnm=0#)G16j_{;U^+DFI z)`QZbF(ISy&%c9&tLYFvf&@$H8_~U*;Cq^c`Nx@+-1gUiz-9tz1-uc#w~dPQ+s&Ra zqo1&zpkY9XI=+ZU!kBPYnDA#33%sHt>rY^qNoPZ9M9{KXNsiW)Ymb?Gm6D1N-D1ke zDoj-MMT^it#v%h3m{5|f@n+a=x1r|_62gu_(!pppQR*|6SUU!pv6bk$0g%8sc8hPK zMQvPXZ((K>U&`&1^;&B1C!c0CbD3xx)LN6-$!l^KG3N*s@u~nmh#<218$E%84jLAe zmO!&Fq|7w;8exJ@KTpC0&Qm0&MR-O|Joe2*|8W;XvjOi+stA#K8^Uoj~G6K_hGQ0H@a zxbts@lelcGfC-3g_DjG7M`6M=CQQH`(GhQ|hi8z6@L9%QOHUV2)D8t5j?DQJCaLj| zOWAh)Yl;GJbW*_j0B@ux9P)YsLjvuzyJB$f8h`^BG%M~esL3CnDyyXjLqxX2`EV9u9vtCJfJ@=v!X}li^QV2N- zFUIYpX~>p_BC;kge z@TZm{xXJ4YzU`qxVS?7*pF%>eB@YMmZvq-{koa5T|^sKDBly z!}rETrpyrOvR5uNKWc?SP4!?sVNfi?tZ40es2g(Kpj3_B_d=b#f^7s%sw*8 zxa-VD?+rou(0iT|IUD)8i7E6VaaBbbwvZ-)2YG>1p8 z?=8ECIc!$l&OGin&2VTo`*9E+OtX*H?C`o}=}A|Jl9(_Wz{gl_t>Y->cNqwIB-T!TGW!@8d!G9=hVd|u9)Uu@CC!qX?zGJRKc^-z zCIASs{)83t!>SeQW=uV!FoAiNEH}hJVD5271W=dlaK#mPQBHj2jn`LuQKoAvA_-$O z0B#~e`Fl3lVPG1sFK>zEUft*6p#wX?G$+oo-6SV%D<(Kt?Fk~bK`dLhQQn^bxw^QQ z2s92|7fFn}I-jF*6lU?I@bb$r_QK@v18=b{;1b%uJTJzf6EG;q9>yu{Q7&m7Hh$_N z%LieCmdAQR5I&sr5UMgT!DdYGrOgoBU_#Dzz|-?dJL8+U5>C_j=a67aNXUc2KZb;S z!_RQwhmrpN^78mnzEVthEE|mpmwQY&YCYl4Md}F}6XXVY8i-_TB}#La4qwE*YSZfI z?)sAQv4XC(hTN8V1I%bE(X7#dcM!65Lp(*SYli_z&B=)M1<58fdFN$0J-56H>4*fPn?m+L)P<(GE~60%T`#{@+?fj#FZ zEVZamq)Zu|j^wJ=Uw*K<1K!uL8?YFy!09euP?xQH~w$ZwY?jYeVrI&J+9G7Xy zTB&R1ROwi2I@|CF9QbBS27!yW0OdwQ1tOsI+|HyR6=MMP(So;!ksfxgzujS8( z!14nkC_KPU3a_yZ$-zP1TM^zROKLsg9u2oxo2UL7 zyD!KbQ=SwbD;HcL<`6GUOO8S=9F|O$Ksf}E^nJ3FjOe4#DoCOg|3&D&{^E*wvV>5{ z7UF9yFEF9GQd2JO@S$LGzL^jf{Tm78I*~a&A%_VpywokIA~Ds1C|Wfp{AV2z_b}%u z+@z&Cf&>U5Abl!v#RBdIY&$CRfK5(`xXe&sMg`8l3MR|_Dn8JU;BbtO$5pr&afpVa zI7Df}bBuzLD2oYxI<5um)(;Z+Z0OsX$dqNmgbA0hv+%dfs4RPUDV2~bNM_tMNd|a4 z6|E;Q8pxMtdO|qQ*$%!cYD|#hM-L=mf;Y*DkA^$bA$*DnIQ_%vbhQq$)_WQQo?^{-n zAb|vTSy`0#Sthe4g-898)P2oo&s-!e@l^Q^13s3r%;*X77el>yg438h8#qVLwt3PM z1;2$+jwMkib1uRoib}Q_9ESrJQwpZCB7j!xdmlX%zrh2E2~ttO?vb8=jql0#wu9@2 zs^}}*K~lOCDFo{YQ_4@sV*(IFdlpt{{O}1L zFmTf22g%wFRLCd|qyPxs&LS;bBqk6ZoB;`p37d>R;lh}pA%QU==VfHsPH*K4n-5rc zCYiCc9*sE3=W{00|1g-bp+GVv!JU`L*XS6nl3H=GVN8&e5yZ*(MfkDWZWqkqj@}e* z*lrah$XL|B5Jm!RK%{y?77=Ju+^_iJx6tRev=H_#VIfTTiuisa6$PMy^T%jcgJe`CcKF5P}PqEf~q!E(yMtlzp3vje5 z+kuDdRg^y=FBirHttViq2OW2~)e};j$zXyKsU{Gc!3-PiA*3*@=TAkAB*>E3jwKvO zNWhNa;giNc&$z~O0k^m@NMi~BaBwQU4Nm$mrocWF%$@SzW(@srub-f0%qU`k!ATT)&m~a7a3ULl{NHBeS>5L4J2KHx<8T7-aW~-vgWOSFJGqhh1`136#Pknv z!ljs>u5g~2`N9g~J5Z06lzVuwh&<66%H}or5LgeEJUs>YR;UrU@-@mrT~}5XSG6$Ls_TRBgEb=r6REt(AF~ zVLRxwmMi514KdmUhh{=99Oh%c(&-#VGNBzyql&d<4{fghDZoS zPDaq~g{GQZ%D5zESDr^a2X<<#_tJ}v1swi8r34)VwSMS1;iCi;G(*8SfvrAbZ%Bd( zz|MJ7O6sF+tqL;>Ewlk)2!E}o>UlK@VFRhP?e0*|xP{H#PRM0PU1kUC8b(09)A!s# zLJ0& zG5_O(mbK0lbJ_|l3zfTBhI2Fe4_X&pD#u|W3x>v|}10+&fHzpb%S3-ph z9Gu|savSc@bAq?40L6e4vPN&WUmm9fRj`~=f^cP%2r@T4!8mX<_XZ3ZA^k<4@lJ_h|o9fD@|Qo15F-PFP-dX*>ZqOyddINaG2O z8+d`LC_ww5-eMZIotBk6Dj)v!XwS6_Rod-v`u5P=cb`~5mSq_C!Ef2Q9GZ;W$GvWz z%WCVzf)(k*+HF8fv<$!|luCs^f@z~6MnfuvaUP9+37eW+keUf8Cu}2Bq;|(FDSW8> zpCARH1lERg&UyaaIx)ZjEU^;{^(TUlP9~5NvsR4<)zYzG}HjD?#}{mQp6Nq4grfK;YjgRsl}9s_);tcz0LtYW}`!iPl{$v4PibKDkH< zbFQ~#;as_HK`U-2whE(@JEzW33T=e8^EGB!GdED#6B|6v*$UG{JHgBZOeZ-( z-6zyWGB=sO(J8j%svOfXK+ zuVgPr)|j0ztmSfmMr&6EGh2wkd&j`xMq&W7P6jL#JJ;+rEigzwZ~jCc<7}r6k>fZc z8F^J=#aB|>y;cWC;tsSc)J)KZxsRM+)>9;Xu5d#f1>U{RvaPGJDh3{wv$Pu$0vAC* zfu-m3OW`DN!pDy-{-jqNE6oPa9|Z;Y%xuUq1$Ry1IR96J1SLL0o?d*#>3hJhCS`Lm z6@@JFf@=y_{(p$#D@Rx5eTgN%u;w=ymSj)N;QutJLqI;KH52RnoI44e+E;gaBzUS2)r1h#xN7<5JZD1$d{ z$BjJzql0KGL_0x^5Gf-LitIVwVN+yfWyG$bg!2DA!f)h%87%F=NJahh z|5gp>e=r7KO0TmNDIl&Kj1!LIwndEb1~ndp6Qr8qKD869{$n79W1Ijqe$f%WIdomy z0jsvEn5Zb(MrB8=R;tmuYQ&6SGGdD>^ewm()W`(%U031l??tqz3c9VcjBEMKVTp!! z$13)OuUSI&0NC2qMr{QZV>G1UmbF<1Yh{Tw-389NjU$XFj9CGkVB$(#L|pkfGb8Z6 zy?#PVPN>T)TUoM^a)R@-OdZNnGUl^kX=JY85vrTH%y?6(Fr@X$E9_xky>#RG9+kp+ z8^)dU-rF@{)MLuL_MNMGG7!7(fT$o)^3n;cE}GMlj59ts>4v^R?)am7%5@-C9Z;E%^x*#L!;3fFOG%S2RE728%c|O%Mr&&ttxbEQN4r*nay1J^T_V?C;t4L_y@C+v z^Iv~^$elCY*37B)>IOqAUhVR`_b*;|o~R`qfyJhFp;Hp8hYeb#Am7ovPYJ<5FJZhV z!8THX%@$+aDw2RgKG&2h?LD#F=KuYVf4fDI#ik&2pBT=^PA6=@PKQO0i;PP~6pBJB z%`(V{Mb0!vz)EpzghY9v^s8!!zT8$j83=`VdIHbW82?pTSPzB-M~Ij+{lxfFEh)E7cG|vJ$01&8)W-wRO|yXQw1nJ=jH)3Uq!Ag9 zxRAZ74B}wxUUib3U|f*$!FfBO?~wRG%mh>@>7t!rk}7BN7`IZ5q|1iXF`)#zWX^e4fFKrnpIwp;ls;Ap6^ z%;q7u$Qi{VhusyG@ym2rudgHoR1nUk$*Of`5yhSnbdCYWGNpuNnZUZEB0MFQLJ(Ct zYzh{euc+g`F4+ATIwG-ZD2P75sO^(__FPX9*l27d%;lCFV$z>mJIOJtTitc0Xnh)2 zEcFlPN+vBE($}Z(_IU<&0cLNF>i0>}xZgvB-;1BF(9&Q2gi zKj9Z@v>rNYv@TMO*4Mp8>#tLd)+Oa$_+A$vfH(JZwCAT7qkAW+q}?0nwnBxCx~)Nq z_2I)WkKVilFBCLm7+gd^FJwN;1Zd|27$KB^YB8Te)1HBnfw2K+`VL32-)n@Ec~i}9 zpr7ERibLj$tPB7Nge;(3NnwVCLDnwlImjtRnX~6Xg5ZN5fWQk{liQLN?K@rxF(^g_ zkYwSr83_Tm47Tqb|&z0wfX%Gak1a8!4%|#%~R09ov5{TqT9NYiVon3 zx3fQZz3z?ZE#6%^*Z@1$o_4s4jpZ^@zC9Nkf!z~=mbCDNdr(|kbS4l1J+oK6vElHG(0je;A%%k03stW^8qW4zjiWl0Muzp zs1hr|+p0SF8JLItkW2BML9x7QY9?4yqUYevn%7I6%uGnfMQiqE-563v6(b?@wtaqd zv@t?yj9{a4zuZD}bFM$(2 ziJjn=MJNPCXdP+n$R#5`!xjw%mF8ZsdW#Z1azEkbB!}j)^Mk_eMG+Mu0U%VsAih6Btx*6gLPoCY)$qDc9xPYJ= zIKf`AY4%W6!gXf(hPYHNxN=yUdOfq3G|6em3V)juFc~$3euvZ8OCg+EbX8J2!3|?; zMV=R>*TkhcorkgyGG>%|&%$sc?=mzvQbK+rb6JNzC$ipjBD|rR$^*$51%)eD;G;G@ zMcWeK>EMtSLT969G;&rryOP-jc>d!*e`3bOcNm4p$|;JfDihD4irhnUC(m5X2v5od8dv-rPTVdS44CP`9-c-PZfd zr$N4LOL_{q%kx^;fYH>e9RQo*l6Ol^Fnm8J;3d&K;L4eJLYmMhVoL)} z74g2EFZy_bcNytGbBJ{a%q*kJ!0Iq~kP$+-i_I>ZFhbl$t6|E%v4Z^uMp78ZwR%!; zIIXxS+k_Hg=)7S_ZJv`ALd&Vrcc&-vA4~v^1599qu=vZx#wttQNPzf2BWO2fod9p&G5_ySLX5$_edOHCp#os?mBIG+OsOdcDQbYQJua#cI888*sv- zH+u+<-adKu$+MQr0=li4=(gTTr1HaI|N7M@)Qf_Q`%jQJo$IKlO-)h|KGq^EE9&nLu>?& ziBdvvlIA&s-=F~n-<$(^z*r%Y!pcZtYPTvW%$%H7SXGoHjbY3Ryx=j7182cRqXRs1 zL-;KIgfCCn2|@`jUbSuF`bsGw_jOfacd1Uf7q&Fd!J|RM_i!-5uos`pJckmL0F>p~ z_@eh>TVN{cqr*q40s{QAoCm)iOE43m0DPr{pqMg(5rV^{(3FFu^Z;-+ZT$!*^t#P_ zu(jGVIU!*Jn^TQeA%t8kr<+7S;i#d&9iimZb(4)WTAOxXHHUK=t;MZrv@UTiaK0oA zUC2>fr0jlqa+mG?cqF|IrrSDu2(JJE{rr9hQhEGg1Cts~f;jM5jVOZ4Wg8O}7##Z? zA|e}1X(6>CxDjTufX@1a>ErIak+7|Oy-PWvUY2x4^B7}nD#j8d`+!kKdCcY#v&#Sp z=X)nwMsppq?9S>X@Ud1_&~o0PloPDJ17LjxvLxs`7#rARc0wZ>OT-UF7F0ayE^4`_BOZ?ECx8>`Jnv8!Ipi!M zT?w4ftggGUM(fo;qxD9QE^*0&&%75hEpU~WNJXk{U%h&0uCkY}?{hi-&E3n_cll0V zfnEnGaXjC=!exX6G_2^{hE?Fe1$^X;GNg7wJM4bYwc)lM-ww9hUE~5zTs_SRd#X2U zesjL*;|c%k-y1egwDms5=vShsf_9d%u346=%fR8%u5L&O$OW7`H>XC?nQ_o@o8iTk zt%qO2W9~ym)XY89Al9Nt42ALVe=iaVLJF1vk%}BRvlDFkOM{C>2}^Sy&QbyqN-A+R zFoG9KzyWDtSW(Aev829|lR!bV5*!#ohc4-7@OTj^q3~5Xs?$nJFt0&O1a0(Fc;RNS zc4IiNs-%n{y*7~$#*}O!NP&-ZFeuz&Xr^f6)yA9vIx(d69h4Iy_BYA|2`|Xz?V37E zpKm|k{y;tu1&#GhH_Bt24tR)m!U3Fcnca5+&!7TsSH?Y>3Tx8_?Nu2Cwc^e-bDM!1 zI3b4uoX|+Xz6l3goQwTkhsGV$T5G(EzQT3geWo(5Tckklb+8vx7-jGgXH4XUoyHRk zMhSe1AqL%P$4A>1T~t;u-{AxyXfY_fF)3k~pRkcHKvh|v8KQbvTNW7yBhOW4sbCDj z(OAwiu~g^$$rDDX&+8?`ZIXhT3T!Nq6^@P1uQNCg^FRt^n{Yx3UppZMA4wtcAL_Xp z3PuL|mS$G34s7APTf zLy||ZUKiyspb*A-7;S(Nm=WfV80BPynJ_{%Tlr41U}%NUg6FbbEL*`+Y7lOLLI%gO#$uM$cowXs!XH}Jaw*p@W|!1$ zonK}ix8kTq1pb*hBp&SKiM924gFBiABGeS*RY(e(iTeQHDe0RW1s~1NQ1o_!5yF%c z_MB}{s^>6Ip{R|X)v~ViM&f8pu?;BrrOYTILoR@Ir9IzfA!NDqvkEgf-+VLLb!CY~ zn(z2gNuf-#Wn!5)A%$EAEiHU|iW5i*IZicFs8(JR6%09L7%+1|@kl4MXHybc98&_0 z_})m5lmKoNn@RW#C<_`1RS_uxAsU?4=lcyPp|+NXD+9PJIAxqHtY_$-dp%qi!(q@ zxVgoI`3F%qx?*!!-xwzd_9TTdCn#_vqzj4cns;vFHnPS)N1}R0Xk)aNWbBe6~AUGZ2mgyyOaB zTTa5T3$iHWKt;%rU?h02c|bx5I4liJA)y2hN|+lZRDlxk?4y}r#*;SAJgDMkZsPbi zZB^nZj^|#*nnoE%Tq)-j|BFhs(WaUz|6}+&*%0^n@5Fxkkyf+$%R>Db2U{?a+Eh2{n zh$8F{l;Bk*Ib*9^kP_yOAJ7l{hFba#oKb?t z12g{tH0MYI!)7=#ndT}VtGj&@MF z*$pF||5TYJ(P*tT+$G_Hf)ePVqd`QL*z;dio_&yW9I}oFYvA8%Aw)z{s3IviKaQ8P zrHe0JnDwHK;zWHk?pCD2GH zXe4+%l|IkdwZM<3JX<*CB@ZdVS8{$WpG!C|x((iEuJ%DWf0nyRd(+6GxDC0lalvNP zbWJ$bNMX?P7+B3IhDY#8N#_OBU*Vms9 zffEAOP0tBN3#TFq2Talg$O_vYWPu5`1H{V|-rl;Y!3d-RrXw-JaVZc>#O1P2_`xq* zHbC)L2qy!Ioa31~@e(Lelfd zXX97n1E3&r2D!Xc0+cVBfQOFY>*@=rIq)5vK#&t|t^rw?ogmn+*CRWDl#teqh&Z^M zt+sCZMn7~J;;Lx0rgj1^(-h5>GQuvf%Z74EN&#jph4_Qs7-|fPei1oS=4s6CH#Z)0yY|Ih72i z3`_4tHjss7Ge=G^^(V?dscqR;IIyP*pFp=;mh`HU<7rHRzog4xC>#ln`HV7;MhcAM zo#F(a;|BE?*Qh{IPCyw};Ix9uibg`k(7_;#pjkeK$^wu=1=Nca!z$!8Vp~uuBh(Bf z_==F%p~03)3E-6ha1Eq=%!8xGKAo|R;Ckn6^n-p>F40mECXL$c15Z9MLyCNWc|b!6 zktBmu&(gu_EFfd{bx!CeoUjr)_`L5qL9uVw-GgyL)M?$10t)ir@jCo%dmXa6X^!Rz zqN}o+#1r<1#&0Rd3P#A45%zK=cmf+5;!d#Xy8VIbD|gUnyK{^psLOD(c>U_t>-QuB zm14~QgEcv?o7xHcEkkjO6OKSFc~ae^@msT4@C9ND73L4!>z$ zZ84_|QnVB*9JY)n>UIEL9zZse?gZTvS)m-EFk(3AG~i+@WrRuti!Dura_BRJZ8=8c z0H^ZV<-thMdn(BYnuh4u4ryD6JI3~3&~A$Ge)h*-b0U)zCG+?PaRd$Qh>Z}gw>IUe zF(Y`3Dy~e^FChf-!H^K@iAj?93z!Y3{e*9%gc75izuQhQPS|7KCw4;H7$=YwI0=%F zg@Kxid0qEL0!(@lVxDLeM2%K*Li8Sl6xjUdd9V@0&kqp+*@N)Jp}=b!2fHDomg}sp zEsU`3e*fjsPZ(8gQOTSue|0Z$-g3+d?ez-dESKAFLrhEHk`rT$`zZoP?rQN{+&ufZ z20rejFLDPmaQNb222>oiz%B+8g(q?sw3{V2ex(JDL@R(n!Z&ZfB#pl89zW?M3|{RQ zPagdQC7NaYVIi~?nAWmc3eHbCfri4|GLW(1n#y8~_D~K8-b3bPyHd8Wd+~^8h1GtoWk!a8- zaE=r9hdmf!FLnYOs&N7ilM{>AwQba4uAg}9NGxneFRhA z0as>gk0t_}P>dqPZ5;EwLL)s9nzE5)v%!L+f`<^4gYFhEdWYP@Dq$sMLxiHI(iHP3*p;48qczUWo1sW5l%RX z_U7SBm@p@t06tKpoUj&7h?IawLJC+Wb?AIYAR^dxjgEJ0Djn{K=5|`6wb2VAbb!br z+Y{d(l8>I-+gyS}3(j=#D$q*9QlCyCbrCgYi>n{}$k&6T(9kuHWLlc?0-( zyccjnsnzaps}=?kbYyRi#~Ai$wvw)VJn5!cf=JCSkHL6{&8n=tYOfhq8i`}devvV_ zu=vt#V4mTQynL#{OYQzeLr!3|oU$X9-~@(H%BJ0SzbPlQYaV?lw`t!)6-mKqtFinT zi;p>QLNpY-^o{sc1ZlzG^NfU{DL>&AQ^7#2O=Ah%;ssgegoHpR(<}nP+DrzsB6u!# zFy~t>1x4ovp@*WhGqKFI6>|h3lC^5 zU9Imp%`oIC4gennLXf?qj*Fw6$j$c!56ZJ};DlhUALoRKNDQjwzE_On2{>s#A%&@U zGxx0^ocjFvi#MMx|1gc#@+N4sR?QKO)~45JVY6(0u?W+}+>h8)~_B+bfsnc4eP+FX1~h2i_hS29d^lo+mB6gXZgRa#O2qrJAqs z9S*#M4-caG+N`_ZU%cu`!BP$|T^+R(_(z)}cFt^TxIJ#Dy;hbCBINydvqZ|K3`EU} zSuVeTvm(2|&Aos5{QbLMU*5fW@wS`Iu77{=8pH}t0IFuiyT>kYLij|@qI<2Jux_yH z`HS~msgk1764t@&mGyCy5wo@POwtOh^;S;$K?4<-Yd}`;GT~?y7zu%0D$fDcLrU;ka?1!IA}|6g zbcSmcuGQNqea{IrYpm2hQ1CkIF(EkRgPidS=K~=e&q+!rX|&#ddS@D~%j<8| zXsx7EnUwI$k58Zf^mulcJ$m*lMo2=%tsKAh>Fe&3YPnW5f{l62vC0 zbkBKSxox}0k45wK(VI?&WB&-vS8zh7ny-u7T37o)M^BycfpVIOeWoYzjDOyT0YP!G@(rYq={LzokpZxgf?b{!pb;S%jUcZ0# z^rxSnJ$w2}?w+6bnQ+33M*s$#EzM4lL8m`Id-VAC2f7cfIS+g4aHMcL%j4 z>bpMv^*LT!qv@b|dpG0J?mdnc1z!F3UIf(dr830*9TK8>9ytLl^78+Zb}g!nD@pVY zh9nbk2tybM1OkC@2`q#m>;`AN5InLhDUw!}pRp6KC;IZ()KrS$5m_4)dxb4botY|e%A^${cu=fH(T0ntMvwi?H{r6=r#gc&X^7?fGo zF@g0Rs72QQnv&Wlm=H(IjT}5y7a*%{e>%NvT>zvm*IlsCO>e%O9vq&Yo?dA0mz^VE zg0R@WG5@^pFd?e0KAnitaE1eq38p7-JbFxUCmY6t)ALi|eo;MPS3wuS1deKI2g)sm zo&6q!47a<}E+0!#z~MorZvCN#Ch&pGc!JOo{1;;C5SS@vqhY`d$SfHLD2cDe8Frh9y8dPWa zRtE9vEqc2Zj@CXnTC?K=$HMw}>^WMU?U28=?P_{=d3gzr)DC# z97_v4rCQ0T532{9EdQ;nBz}`}$GrEn@o(|sbE5PB-;mBtPB(rVrU3< zHA=8~onZ*$y5PqWzdaMM2esNan_r2(Uv06PT`_a``>O zYhRo~q@oDHwL|WbDo$4(*gGCOKphT~_Yruyb@yI-iq{os6S>&V1P6*g0*jsU41$DbbMR&FxLBP;bOc3o^%vd)-v|L z%UWbt`JF>N6X$s~pzXl;g!}71@w$nZGv$082qv_bPVurhJL1HwK%;T?XLk|Y|(SXzmLx#wsdMOq6XG=D-bzzGqFW)@5chzTuV!iiwQ zdVSS86{;!lecb~S@YNM|O7#Bjfc^xaL-zT2HdajN53}m{fIrbCzS8vw1^cEo^#s*X zo0#Cne3}b|;l9*njWZ;9ivj{a_QoMV99XET;9Q5;B^HRJYF2oUBZYWCuYgU?xYY)r z0fG*l-s2rwwhWP6qtHo<_>fRY9#Skr=-&*2%c;8}CQQfWY_Wr*wQn4)R7|L#65QZwg{wFp;P#seKg(!Ytyd5fh>LjNL@?nP zhQ45ejHfqrmz5*#bWwAFreE4pp<-4`XWG;g@VQ4%sHsmyQ83Pe#IfCww@t{OAm4k= zcE03`OkVgpN<5>Q4*d=>;aD-@0lw6emgqyVU_yWT{Bj39;T63D7k7QuKRCU}1WB(t^19cjKo1!u;Cj3*HtTDvg8sx{{PBsmaeFEG4w1USkU=Tt;{ ziiZQ7G4-HUS0;?*aLhAu%He@&;1Z5yh#-nVTIPxB(kxXagVjYVdZWRxCRAl#?GLmg z>zlb1(^z9i+*mg_;9G^g(MQNF=%<^%8|=)r!khlgj!5Z`fB`{ttD(ciY(rE}@Trj8 z+u~0!sbK=R+}E$KE-!N8Uf!QTnHe6K)V1aDBqK-ba10;gGclo<&Xl8dQSRetwX5sy zh;AYjB_o8)lySjI6LGt+0Km2@)w(A0z(7GITo<^OcgR4hg?aI09 zv{ROoDaIuq`NaFCk8{xz7BlJz6CRVRL+LZu9bP(lBsH^c7PG5!3v~9j1C4)TLM=0LHzshYk)Qy5VRxNKn1}7A!CQ=KuB;ZOG&&a z9oLW{(o^tmUHvya;A6s2$Ri@^26{nOFuCYJdPb2RmpmX&x5CDfEEH6F5aL#@nd2e` z1tz;D_J*yH|H4vmssSIj)lwVo#srOH+?M(N*k;Gk`uY>Hep~I1#)_W4NJjKUVg>VI(W6K_gWrhS_G?8Tsv+ZU9axN*8g;fS`V)6LA%P( zwuQEcHi{gVYrv~5xf<%7$DL(~_@}9|E{Q(T!hI2ZaW9i)(mFBDS7-~O+_a95a2arL z`x6iW+k7~K?9>zdM?+6&*l7#s2qsg}7Uqrk1jYy&DA02r3qNZtkuNYEl0ooDGW*ZR zSU~_Le1AUqbiP7lejvSWAGAl>`}3ni#5RAzZwOw80pk-|E%;BQ6ps8vCpalGObipW zqO!RYR2u$-OcovhP+Y>#RZ4(@MjJOp)qbN6;e3xr*F6U^G}7D)p~GFzhIA$H8Kp%d zTI;m*)`QYkYu=3Vs|1vYtOGVTvNIOXkmsT#2rqonPfD2?d^H9O9pr*04FNLWb{%8Q zI7x^*cWwqRE-$oX0&Sj)hp zhkNt{ymsmdbG!^D89iyDCF*Kxkxx;Af;ve}QAkp#6?h9lYfRH2h@P*{ZqeEGpJ&%U z?jKeshxbopF_V?pv#U7J-k&4ZOX9JuEhVkWwnvqV#<*6`Lu z{58WLW54f6ra0YS<}S7wf75Y2Gm>8DH883Tv2SVO5{A}yaxbLyf73MNK!4QY*7r-%2qEm z*w|$wQSt97AXq*8B33-0$zXs$3yU9`n%YG{U5R{-3!g5D_QV=>C7J}i(BnZ~)=)OZ zOT>;dA2C{o1^uk~UAp~#jD!oVjF4l*I3O@FT$wUQ zs}TN#$4LH4qbF)-NGvW;ygiN5sAP>+)>H#AqiEyEzd~m6qUb_Mx1h$zh)Go#Y;(zh zVjP9+3U#>dO%k%i&htKwaJ4_2?eZry#o`zZ9Zg42d6F{oVJ!c(Y+M?qVL~$SC6%Kf zmUV2cq#|T1Nzje)Q+1NE-!Ie0=Zmjrt>Xu&_Yz`*uX85~0#K=da=qUl;vXb&Wx-@0 zK%o38Bv^s?Wrr-|XmSyLyVyk&#ROKODU-;)uERXSUnx)hbZ;I3H!NjOOHlbcyhXen z#bjT7&$@`lzO@l9MGeu+L?WAli{0#3FLSqq5aqH@?t^z)1brkIAB@#bubRkEkA%6| z>;+mA#47V+gc^!u75^O#g2OdT{aXtqu2o>@STK+aB5EX1eQVwK*EC6Sn({!u^7j!4>=O_Dp)0Qgk~c|1xz_h^BF`100LetEj{0lm9xJR zjal;I^#s$H?7g6(z)VgheH)FQR}{u9NdP(|F(JWd@4HMkk&ymGk5MGPg+#snmADfB z{PEz5K*a@|k-&uV|$z8?do2zPTsO6v^H4V4}wq=VyN}RhfFrg{RfvU@rok)x`qMIFCK+8H1Z7T+Hfg z5f7-Kuz?V@>v@8YV-yx3F|Q`^fK-#dw7$9h6sJ-MHbh^BS=PY8eQdx+YEt*K@+OLJ zy;)=Sqe4vhW@8VV-@^pTAF}+gTu_!X#RNQY8x|n)xmPA-Ar)9)4G zN7;x0`=F%4(K=~LZ64f*3C7#Jgr2ZgdBu^(gxMAax}^>75wVb@Ee26~cS5GkVsl-t~z|kDT4$bI25aiHVdWttr|A3kh5)=MDD4^+< z?cgy1|Ga4Uw-&98(JAyQo`QAl+dsa2`$pR#2!tVwV=AIuM5eHikkBQ2Ygee2`kDUS z=heFVz%nYFqKX)IMZcjZFo?ya0no&c<2c<}t_u zk~XNPlnaQBKoc9#jips5#>&Cr2@cR6iP>O%8Kyp^)mYk(9~#qqHUpW~6Ufnu$G`SB zPi{Zn-re4*?ALO__S!W65!_4*yGgbxr~pghq^7X(mTtP)?}H5AR@m2MUO$&HO=J-B ze`zy*@>K{Nv|2=3BB@$}0A2+HQ4|H)Lc<4iP;mS<%HsinGKmR&VV;=EX6MBkYMsCN z?|+#5k65gD7J8c%!-392diLBb^HdY^Ob+1Pm~7>97HWQ{m;mUg){kRurWp>gaJxah z$Pp$0mAoEO6w4keWJHBbj?z25?BO@WY_*IG!oy z^k|_ejbzdTi!!pf7Z&kk9*8cmQ*M$IkED-@w}Ge3q1}UGq$g!%t9>2#m`7I zfJ95-C>AtMMsgu*)BGuFJH(@D6mU)LrsHH9qCp+ii$WTu66O&SFtOk~t`Yo>bSoY};6jYG4xD1xwGk6SOGt1|T7a0a z#a2kP_K%S?*ar?qE+QsyVU_se;#%?@v=~);128b&=2m~DS&4x|UJ%D1`#F{n33{Z% zpt{U>SC&WNSVx6=Lhj@qeKsI8eaiuo)Pz%ckyVD5AlG6Odkqu}6=>tpk|~o64BY5O zv&eyTNX4aj1FjS^M9kixlFjXEaGR9l6$WUfFwsex=2oaIa)F!C3Bar-bxs#nb}}p0 z7OO0sP|MJU;B0KWg8vQ%9n4f2tvWQ2XEO9n3FsvCr9s}u0Mic~R=$r0Rw9F9D#cjh zmq=6{7YWpbIY!g=_GV8p4n+=RJFVW|f10oz5^u8vGw$`|z4;guyGViswkN!F+}=d= zY~m#2CgBQ3HumZyud6ql>cAp+?pz!`{5B6Q$=qGDzuxStXU{*%z<2Fiu(1C4=1%G{ z`W?gw2P$CLoZ{x#V2qd`8u`LXZN4@SUPsQ(*lxB03$A_l(~E@zB#4KgYPK+WGGsom+S=W# zFP}aolM=wA4G>N_qDwS~7V$#TW}z2A0z|O@JFabXPwNCdg2}%7a5vBq#+Z9>-W1~fBqT%gi6BWI{pOVWOa@u9%NE-p2Tcbf zD@+p{Ktd{FFd}3WOvoK;G@i}eL2dF?gEjn3f*DXmqp)5rVLc$o}sPM z7#xV{KqDd^!o6KwBLfgvONUkU@Fyoq+H=86#577qB|uz-qghqQ%nBRWX4<3I^S4`K zX;SsX4W)qSF>IHP&C=qp5_k|CIM8i+vQP1 zfP}e|pT>oam5j9{e-kx=>O}M~Vqwdv?Nx#Vk-Djp<#5uWP|92IIlLB4ZQZM~=&1=g z!6`4N#FU0E->M&5XE96TDClHb+Z~jHM1^QjXf(B$&x$C55(yAAAjxGKuE8LBaj!y8AY1F9_8BoD zhXg(M4(OoV?JdP>L_$MiUHVN!@}txZWxs4(z8=(&08#BPYb3d008p|~e? zL8BBxv+1h=aW%1RGKC4dq$H>>Q;>ia>13UYo5QZl|nNT{j-B#fPmZpO?Dm*_;bQPAJra88tvFjNYr)C8N7 zw36=H!A%DBBq*tRk_IChbr&_=fq_m*Djr1cH4)xevRtHNOe{Ku-YW|D$fM>XCrjRW z3HWq6`PjR`3obdQ=Ep4Y!zN78>sMZ+=W-n0)QK;{0-JOo{(0SCE z4&kO~x0r39o*W< zbqNh2mlynPuCI&UUVVqE^868(QSwaHTbV@X-L0-JW|_!k)Q=@qCzm9MT$fZ^QKMbnK|(Pq^2BjJ6f+-=m>37u6;onllB=$T zq2Dzw^~phQ&N^Xo<|t)7R!R|D$JV*86+P@6j$k=3<9AM^br7w{;`#64cO4A?1rqf; zs9%CMhIAf#8kOCx#3Mml)sr$#JjgNvVVZI1%b(pQCJg#lXW0&8Sh&sJ{g$@?h(LG0 zI|*HU@}7!9pFiO&5sp^LV;DXS}b!K46Cxd7imP)q;>2)k1=yi zcGH+Z*{)-BLhCp+N-cpB(0iO0=^%N+gksJgFyrtuy>ZE%O()T0_e>B0Z8Yg9e^Ba` zW;#V09xjwRG@gEv|1YWu827nPkfB$o$s}+}Nw~tJtL<_c7EEI>W1<-FIor@MN&ZDH zTq%E5PjvWYQE2ijee;pZ5|boI$4m&(m}edDdF8PtB2b)X?6Wi1;F0TqPn=S)$hVJ@ z5;l`~&uzX)by{pcwVyb#I62H8zipT>?0=-~U~*gK@7}!mZ4r>~KrI1(403vmo=^)P zSr$_7q2p97c09QRuweh?Ety%rjBKrocE$=0 zU{nB3#B<9KF~Rx3SqF1A%Hu$|j^q6S7(#q6kAl26lOPO!**-W~shkqK3o6)eA#+bc~UDWwz0uxM4;EPrL12~o> zcbJGEkTA9s1RID8y`9meh_0)oBp9K)GBTyi=%H};NspsJK}?2>ll;r8(mbX7^quL` zNH<5TSYh_`db133B;nFK=XuK24)TmuBw7~P^7g?4met0 zN(Z~~(@z|&pLQ27DMhFVC|8^Eh(s zrd*wm36L-YDm+n0(CTt(JA8H?GRYM=TGyC{EEN-Qk?qabPiKjv^+29grEmpt$fo0H zEvj&!AaaRSVlUBtsb{E<@sN>RWIm_u035jg=IL9WQV9xZ9yKvxw?{*90e>d4IY+0J z#~zDZXc?aj64I9=VbDUbhfb_ni4Xa*qB@RWO-Q$yt zD}zup8?Ua@nD&GO{rykt2|DS(#7pU~^<xif$y_Cp(Wr!IJn|NabhI6z5;{h zI-@l(9#kqWxEhQBo2j=jz7$ko-n;~gB6@^p+$M&-j$k*JFS6>$UFmyu*oS8&d+MTW zyfiY-@2G~MP6b5}$Whh!*u!^b^F1W7jv)za5i8W00F6*KId|}Q9U+~+T1{cXj&Q)| zBQW8fR7P*8ClC?v@a9iAM^z}v(ORxvzPM3Lz*z-ZSZ{)Iw7yV|)>-@J>DO-$uV3ug z>r3=j8^zHr*xH1_6}_m!VBD+kzWnyCAY1EVE_#A{8ywkHcUzp>Yy*TIM&6hii5dDc z4_xJ~;lU0oh%9Ye#rh*6T%+S*J%NgXBV7X(76k>W34%h`ln?4-)TBF?>N^kr4gb34W0tr6y2ni~lsm1RiAZQNJ2o7xg z0}=4lKgh{4P+g`o(10x?6`HmfPQ?w@69I&|Gr=Kw9-Ce|d?TSBWMb2iUSxSzt-G51 zYB>}*oP>^?V!)H#1&$-@Bs$aqsSspPo7*d+PdhpkI93|4i(Z}1wHPC_rl%U#4{VQR zrGM&AF!p#fOi#c=1DaE`(!-cQx&QcNrJ9+iXbsux2}l^EIFubn>+LHnE;ruit{ri- z-epcXT6L!K3-%T|7(?(?*V{3(-o3+h$<~@JX4s1v&n)C0?A%Pqmqa@t1TZ=&U}0ej zCVv1DD5fP~#ABgP6Wi|hPw5G}C}{XRS}BgRq6};PgKEO|Xf85nnX$}uqrm0D@x|vn z_c2(%fN`Hess|<*>Y=QfB)&>kLz=BVZ93F0Gf41M4}R<8fpg`QT#1@s659y~xJvN~ z#?xkz3#KzL|ItJZa+~m{u)%f`Zs#hQt%0KQMaB-p1kGDemy9R`wasI)S!K?h~2Txe`ueT`oX#n7kmkkZU{eC$Pqb2dDemI3o#+d zUY~{%XJEqN3E#T+&il6Y$NUM`LsFHIhmIVrF=h$j1&l$uiyW=nj!Cr**E{8Cy)VMO z_Hy`z)!K6U+r%T6M>7x_yp)He8d#aT zwEoQgbM%B=NJfflz1R=T>I|W2%yuwLC}yrmdV*>Sc+Sm_SQVTD$BYB-Z+}XX()&P_ zQiX&{GOak0IKj}_3-hc4grc0%6MzJ0bkA;;P=PiBl>}|20xy-ZC&+=L1VPI~HYurM z^)nANtiYQe(N$`B*-RsvPAS&(CJ#-3kphHX!hD!u{cNEUFqU!x6ZX54@}QnTc>?I8cnq|uxyb9Nd7D^% z(eglP+8`XQ&Id8H^%c+^c~)`grX~P6@0-}W10cu=G~uhr^tzQ5uGucQ<3ohC{yNgIb>b4g1jS6qfvF&rW%3%V+G9o43w((wkQ$b zEcL-0BEx)ZohnJP=J^9Fg4-ucO4XB?u-#3_etQlkIHxyPA7Ba-6GoX%xKtk!D9X{g zEHhzTn`JGXex44rLJF=mLW&uRQVtD;q%R4r0b~9pe@P((ox%H}SQgEWoz1Q&cP(rA zX-@QRRTSAwnp<9^4Z^g<5$^>aqVO}AlL3Mzg=5t@*^Dj&Jh0aGaNEw=|NeQ~0eBmZ z9T%QE`C{Jdw9HO10mI$)4L65Qip(F;S zNm}Ki*NOJOR|fSW0r(LUCQ;!WOjt0Th$%zHy*5CE{a(k@fV5bXi-HFtIJqm!IB~7W zMn@mI0$Q9i6n!q}CAgJ4?F$|fy8AO$6mOYLEia@^oTh7?P2i2$Mn5iQ5Redo$B+ir z1-raF%zYpW6X{6XpVt$%LGZ857bXdTuA8v^sG>f`43OYvb1fJyRA~HhmGac32{!+K z$rAG3e&&25{|hBT$~m9)vj_d}{Pags_k@`6H+sfS%2|4X;DJrT4RfG~2*%e!G4>+E z1n-C`iU^*NfU$2zcOMZb+>*_xBFNF^AwAM?+Rhm0?)(Bh%szAroMSmqX*}{(^~9if zV1R&kh!cjR-*$18y18-lIZW`RlOOqA4R472yy}hh(55uB_q2Jxnv=jjM)UuhjDRsY}3){ARY>h=sYpY!1{-2N9bgMN# zg6`m5mPb|WHe#qgy|G=W`x3C=eiGG{b)oFiJd5^6Jkl;=X%M2{bB`?K2zx`goSt2y zGtpLek=Q(5CmXpBjgFt0$775AjK$D3-vUog{Z088vr2r2xfFWIGG%P$lTBA(7c+V4hDF)dTBO9$Z7?nB zaax}toremKSfB)i2`Y4vsT_~P2k)ZhLpUYc(S>NSyH5V z9I-Pw?d9{}QI(u5(W!~(Uvv%jXLJdm9?ZDBMhkZ^_!si)NbZUsRB8Lv^2Zi3LhmSHzxeQ2k z(V_*7PL&Zc&`884)1qOFSNb?D*xy%dNmiiVc*26T%C7W$$3~PY$Z!vV&*+?)?r2kg zYY8IYF!KyE*5pRUD@r_qq&er_ru%rHuOhpTBMUPp8ROJ0BNURWQg+!EB(^wBz#ZQ9 z>0$kt1JW;8?i3+!;38&|DR4?EtGIw>5)03)Pm7NGNs0a7K0Oh@Z|<~nFo9u!a$ko; zJt0{Q2@?j+#SRiQQXmi!Wb}eME#O>J-XXGY*>qMzKnnqrTD~E8k=6$u%93zS&_HJt z@g!YI*mYzHoB|25ADuy(fueg=c>CtVxY$k@SK~bo>j?r>F3^@~^?X<+q?Lt+wbzU& zh#>;~3FHNvy_2Yff}o($_oQ%k0uHRp#u!^sa?K_FaQ^(tIQu@wvw|afpa8sK<`ZKh zD0NkvXZtuXu;hdK{{STl!G)`7De;$PA6PpIYatUx1KW{sp7+~SW!{kCxX`rSV9A$2 zWMFZ}!e&?atg-YQ^I7E>!7OO&Ick#t7FZch`X$f7Ab|=)LIJ*mF(%-3Y-*DGcvxDe zZypB_7fj7b8bj~@M_kYkXsr9(Pf>lYvky(uxPMSKO3b=FA)nV1?vM2Zdfzd83=@b5 zCY@$&sAG`eZNiU;5L(e0SSyOUf{tR<5%!pAM87Bre3fP`H*YUGoWOzKLRuHx9g3-b z%yFiLD$6L$Nxn+)VPm5o{d6@DFaq(%2$s?;A@ATKILn~z&*%xiM?Ha5kby=+%S*O+ zB^ol`t47nh9!G1N36)V<3!{=wKIUT@=R;a zV40@Vt2*7HD7_L5V?BY8Few_z)UP8|dTE!xM9q%g#g z2CJ2=9~8)#k{_g%N?sBcNb1gaVb4`^MLb$S2db)kq7+I3J>mXgOo+UNkYKjM^|lWQ ziW+uR(-Pc5n3D+!iU>hrW8;-axz)NXfP~BEFYk1&K?_b|Cyfd!3+qOd6d=TQcK9AC z4q0%51o4I+999YmiC+nA2sVSc7Ia_GjpY4nRjAGkz4TrVh?~>y^)X_iD@SH8krHCK!V@|lX5a*g3}UaXjSLr#8`cE+aAfW zXKc5>UB&kXn8DpyVCH;Q`QM=&*RIuOflP#gCEIJEsKBakLi^63T*>L2VeY#xkJ+w|V`93hWZD z1_K4%pI{-yIU0kU0)yd#eZb1fStu;TUSrC6tXN&?=_&>j3c))m>s-`TR)i4bhhCLS0T(dL|*9j2z+v4(i z2b5T6Za3S%`2OWbFRDR1%Up)xR#c0{45Vh6>%x~?g$H6nD^1`2{uHRui2~(6OxSl3FU;IFy&OR zFa#O4Wx@4Q);20E_JAHlqgd*6h*G1SP1BBptb!fEj0Lo?IQ~8=&5WK4XM%o8@0zi? zps0~Xag3vqwvBhTblIdfN535^b_jQkK-plSig>0>Sf&XjDPmC;5)({@m|%phPb?Pz+CgZjCt#$W!i1d>8;i&!qzlUfC4_LeZ>6=~DM#zh%T&Gg zHoydOw2Ill-oThgjbW(pZklE*$D#?zgA*yfdqV^t&es!uAI?Pv)}Ua0OT~gzddid1DcJX8`3_mJTD@OhVHjZK7vfTY zeeU#Un@L$VEFGSrgcenU9n=(xY+6GY+XqC2Q9s+}y!KU*Q=Cg`g7urGU}TL+QNZ6o zs3e?&3G%S}{_l{G!8w zRkXw!+kAg*jMd>Ma7$o}%;AmmG37DG^{rX@hRK-ZzKJUSE-cHXgN8Ip?~B?#a8IhZiY zf%^O38M72}1BG21Az@BC#RrXR7WYbr2|xc&Xvz|b0)G7kEeF;p!?Rd}iR4rtR{khX-F4`9UqpyCY_s@mYlEQ!v+&^ID*5PSQuurgRUM-&s} z0Xe7EJOKij@aguG$`c+Lf?DaEqwFEVFg&5N9NGpPt$!g$Yx|KLt>uWL^%Z$i`W&tI z>@5_Wgsbt(ul@G$T3A%x9De-b`5|w!#K-#m)9~XLPhZqu{&bkDvtN0Yx?HDr7n}=t zik0uMf1%z#?9l$8M~O)DOicJap@D&d=n0Ia;#KiF`dY0tVr^+h z3ULlfF&8rI@^brQX<6(MFV}L(9N!{u&JzO#9oomD=D3Tj<&KYhK@PGpWs@&zY{iXK z%GAh?9IGi&ND*f=)1L`mN_qlA{$sL#MJ2tkLKL-Pmz9m@P6dfN4K#uUUaBHU^b)|> z^3=VQmVIa>Xefr~R<7ufPSz?8-p?{`4j6MUSkq=Bxk`XhLwc^oY6`(CLxsxsCzMx9 z#Rc;_*2hK!I{D?;+oGG8D#pX2{)DbV)5Dl>h_&H9-HIqC*c~_A-EO8ZVcP=*8G@l8 zM{D8l?p)$%T^UC!w!pYbtHB!!YubXUUq1ce)zcUEb2#@HkpKzZ)T2&?;wdB2wXz!vh@u>Jc9;-&R}cJ5Bi90V znuOh`fGj}ay;x9-oYytyBIiC_%UUl!>>15@T;Bo1zYtNL;Hc`?BAMvx01m8h zg+16&i#DAV;!s$hOUe}uht(3Rf#8)e`$jFn;ThG0rmrSYin8H4QJ?VkW`Yy@GW!h= z0z)x{3HC5eLDK!%Sf{WhtQEZ=mBmjjjizlCgN%7!r>C*5c*aO7gT6kotz#(0-022E zVRV{uv-ZBy5YEIiZV&e9W7B^IMd zy5{zDt@-BD&Fu|g;mNfj!p+^`Oib8%xh*&3V{~oqLgZ*I%Y(77emCN1#n0!%o5a!j zlXA4azB|0Y=Ik$DFMP-Z6ArHZ__jOlr!Ri`>W6Rs^V!o^A32{$1%MzXj{i8=TQ4IV zO+4F0K(_-LI64DUPE;%&7wi+YdC-=nQ>V6TI}hV8|MmA@A}Cz#x9HOt1wM1VLJ+N` zrAvW@uB{H#&p*GFA0#!~=-$g+#`2Lf-~#s51!p3h$1}?_^D6)lSnTCYF>eART!xsT zRHbG+n6_*uCX$sHH&wcr;OljNWz*zULWSSt5#=3lVM<#lO|->db+RCIY*&f}-PB`p zGn$R9=@8--@%*ZQbA#Ksespp%4}Aojho}@3d^LxRPOTz)QspP1*9N?;YGZ$6*V=YY z2m>k2gcw7*ni19!Q)3MlfD34?no(8ob1;FTWGLx5jR`AMlU7uC5{>eYcd~yy^e608 zQZ^yI56-!;-3TV!NvN()HfCYh+zmNeUnGv!{D`A9v~OQha#!FoFMooWkhvK#p$h4^ zyO&SD{qp4xPrv{bVQBPD7Hr-KebS;GVsYK^Lf`mnf z5InT=F8(4Kg8CCKf*z=p4O*~LE587yym%C_*oGrL;bI}FMjY@3%f_NrIRS*S_O5V^ z0|w-?yb@@T@u~;`Ovp38K9|3fu4z-oM=?wciT<=Pj5rgys&p=b+vaBMrW?{0&Y&jEeNKob1Qmwg2T$ioZ1FkJfiO;(8C{&;2Q>{!_7@{^U1%m~hk5yP$$<|7~99}tLLPCXU zOjvJD$-|gHxqU(n8%jdLgu%2!-^tji3EPq!tqyaLPBDRQ_phH`CXUuS;b=8X_;|R! zH;&dk+;V;AJUo^E@!fwuee)sDMNfEfXr23Le5_&imoL8BKNr2D+6J+QX%lcwlGKGm zgQZq>&I@Qr)UmkwS`A{{P)}$%mSrT*bG04zakojEr>*S*m|*hjFQO~(dD+sDnA1f8 zK){Q{bWg>fpuo@^*$yp!hu&q>P*p{6v>6($b?AW)3{jZnUKWyAEq=$$qje@Am}8on z0x@COUq$cYBqUG~(4r=T==xgmK$lR&6%e?PzgY(h*-22K#1$0-POUXHwpi@YPyt@Y zI+`w}jflipgud0%9Q7sRzbz(tJ))cHDGS!`NMOanl8G3})`kksCOgM6824N)Yl4LO zJ=y;J|EsI_P>=e}cv{wGQt)F(SfGKWc&gm}ojeW`D4)TE`gVV#@`(fV9DTC*E)w7vlg>;CWzy{$uU9lYpzoHDri zMvf{LZ6Q9!#oI+jKGrtixI3Y7zW7S|2?=U6U=^b0AZ1j0jish>kEir9$wJd8Nw6#l zORd^NEmNEm3_vW!JKz<5w-WQe{6ZyxwgUjjH#D{AR`g8qqTdtkEf(A0@ru2gFMt7G z(0hO}N#h}771>O~8S{)0IrK8lSQ?6+2e7Q4j|dz+`~|7`FzT-gwH)YeGtr!=gb4sE zqvh!3EF0@lNl+N63Gy&hC{Ae#h6+p0H`p90y^S=eRTJ;=0CWm8{gTC%w3jwy8u8cq zT5?G#qjGEt3q2FU=SE;vW6fytp?Xi)uC!+?mIPs2{b|aWK8DMDR_e;+)LRGaSzxEa zq<(J88l$Gff7w(_nMp-Vn3DC){Yb2aY^(`!^8|=+%i^+!F(&jNVP{3#eZ+*U<7mBk zg~b(mz2A>GS`S5TEUe{S$I+^e0WnDgN2Ju#qRr$uOl_8-JC3@@;X61podzT@LK-L2 z@1GK%m?B~0Q?nq_>_jUy+^`Ii#i*Fjq8UAp!3>KKF#$TmAHV<=H6FMw7sN2U@|>28 z7dHl9H24r2s!1Nv+dVFK2TX_{AjAnY9jF}$FRSy|A!QzXt|cPpEGo_VOwD72RLqT; z52Nm?AR^GLKH*GIOek=+NirGYnq`;pj#W*FkYKQ&sBlVEP(8uCjLTvf0Ss|$D}#gu&(6t3DoiwXN{Mk`+LpX_h%?hF$~wnMs+ znnX2W17ry9@PZtz_cs>^Ol5L{8Vw%|X!L1dC} zC}p_b58s2VJ=}c6ao;_g<%oNF6=d--SFT0X5Ll9pNl1zbv?Ai5mI3PG%mfd{{uaYH z5E)=Q7<1FSm8O?tE%brXnx0^oKp*3;|CO~am+QJ%EjCmV#_SoS9efq|%D&kcCj9Lm z|KorOIh6tNDM-t3L2pbf`GjQZLUN?$S+3zsq*=G0k}W>J9RuK`LGxkIA0v3CAOTVV z6Y4Q05EN4V)68@PE@q}A;5lJHBt1b!dQ}=ItgtlCfx?+|YN$nJc@AjAV=y9&qQzpc zEOB}KeTXto&qANVTAyxCDF&w|cjHM_p+2D~#E6Lp6**f_`~)f(EGY8|uS!%P8wFKB zY%v7#1Ov-s^@LA&oW_J}$o~2X5dnXPF(&LNL;eKsTET?M2bUcvo9zukn5j$b*Pgp> zt{KLBha&<8$%;G0Two(O<6R0QVmTaW6|~xmJuZ)YtPQAI+cg$*#RMz<6ml2XY`LWw zm^;ISohB}FNy9?a1oTK+k^?P4HJI<<(FTeD8QfaOQxy_o;boY z3^Q=4d0<{t6rpxG!h}}yqUj_cCRuH76pMMrEmVH|rYk7JQg!JqRZA6m8u3(L=g!dZI4C;W{uA<7e|2)B2)x5t>U>zNM2ga=@P zN7$p2!O;}U^0G$9)pEy9FbNb8;b;wFGPgy+m%%SXa(_Z#Wne=DlCW%B+$k5zdJKBq zLMUGsEsX6VYj^E@=5_(Y1cTN}IFM9<`H=a7&)ru3FxU+gL=W&=wHpW}d2Uk=LG+_Q zNKo6sEhs_hm<$ude-QPA*YGp4aTy3glDEO^21PUirQ~q?5M;0S{OZZkN*|-EvRq>r z8O2;RG$ZB^j4VL_kDVE0j&FwkMyd(Z{0ZZ6V2T1qmX=REIsqy_N(NWM3Nkhe8OMF$ z8jDy`sq~pJp(q_d!SsX#37-drtB493i-6@S)-wFuv&*Ie8GKzE93e7JkG;(Trvs>meqbT{HRtU4frip)kP$S=)e? z6R8i6(-VfMKujn#kU=rw77v&X^e0TrMN*RK0R7lk!L=IP-w`C*`vE05#K)El1up-2g;bLL{H;<%pLnkRj0pV@}4F$9cgl1ig4pJ~t*l z80|s6pzT0(z>9cX*6;S(OFaR9bTYmMB#2xHY_PagBNjvu6!e~yQ5(Plz#<4P_#=*1 z2^;h|XBaB20X1`8c$}q;TlXBow!ndGVle|&%wX?6PEuA~ou(v>TY$)82S(P8R5?;5 zb4}6`=vf?du@#j!yCFeB&w!ZceN-?+Fin9f0zrXmOVlQd1MlygVo*$rtO#+CR6KQc zY(W%)v?qrkoesF5D~j;}8Z2U)qtmr`1J!_0f0vFy6NN1Uj5B5ZkY9nYu(T{No`J+# zi0TOSN%rT#yu;tl(GzrKkNewF%E7(IV@*t;n(%<_U}e()1;7N!IB+3iLK18yOz?z+ zz;^2xUZ{Y;3(?Y-OQ0`sxN0$0@+XNyAa6NafQ>F}5qqWC8bkyQE7&TVfvGj5d}36R zQ|IF%`V{h5N^XaBOqN7Drxg*@Dljtm-whLfmyjhO1U4@g62UB(PzAqp0HaOVd7Mg+ z6Sz|>`~6q4w1R@|0>hm*D^5G+(v<1O+AI5#b2D_)WtS5Z(9q$Q`^VN8HB3OOhY5AX z%85XsD&a>r=RiF`1quxJ)X9s`qgX2VXi(+0uO}omfifQkJeOsPQ!r3itERBx8k?pn zu#rG)1ZJ~-{uTGfe#a6!b&mW{S!fi4ih$0n4pYqW;zHa zV1Vz43C7VFf`u=HAbW!M%D3)0qOo?WY8xWtY7U@@ww7o{OPN>cFbMSJv7aJ{z(@n` zq#q&I!7pce>i)`r<_tPyuCtB#33Hpa3^84j5zTB9g=kusKOqJxVZq5rSzaz$Pu$o# zXcXSzqz2yyUVwV#mJnM#xoNXSiUaK?AJ?6Ijkcnhz-pmS<%I&#p;(3<03$BemYWtK8Jz{ zU~bUwa*=913A3bVb#BPXvg%_(GsJ||qftR!3$@vLqimH+b1e`%^xX-D38C3+fDA5h|8-dl-&!fZ<4rUZ6dsxoJV#;1 zIAn+2C1ex*grsH+K9+UTeB)g!A@t-~GDr~cuyUTez+niITzgtzQUzB*?=b&JAS6! z_QO9YKa+cHCjNt2)(FE=7I|o~r*P`*Vr{lrju7E@v!s&Gs`E7jvuFp30z?{hIs>og zwbd&eX_!zr&sYQ#Lj=Hle8wveGQ9-~$<5WC+tD;Ix9=PjC1lYBezQkGE%1 zA&}n`3l@cQAQPy}iML2vMm{VW#8P=QwH(OS}{`ba0gpa2%wG@@2}pMzy!oO!*m$y2|JTsnReGICgiPb zZT1Cn8>&)52oB8_{B4E_Xp@xD2mkE(z26Fr^u@CupFMl~!7VIaDQL(KR{_K0L;eiB zDzEPSf&|V27IYuth^W_V-UGJrR71d#9$sS#u|FOeENzkVAuYEEFJ_5g;PWlepP*iZ zTr7%~r-~Z^fB>T*q5|YfFu`-)VdouvnS2ry@L{QrKsqk>pWiMdyaqU+swi3vU#d09 z&pVxY!1CFTKmPdFKX~sQ84eGH%+~8P^RV5iqCij>1Uh>_q7DUgsSpzIT(T6kGn3_=cyQm#%=7$ZlKGm3QL`<4IMMCJcu-sTmyPs-8mDZ|1m zDu~#T{r7M44*0+T%+6?HYf7O1pM(Rm9TF5kI#9U1djB5(r1!w`CpQq74r4uG*Fy!Z zkU$sXrV$cmM@H5)@}LowSdd*6fl5K9<^>aWzyy-Fu5T}q-?+S%V_$vw^ouWE+%eRE z7h7~g+|ju>`%e^^|Ki0VpMljiZ>2w%yRb*64t%GX1zUp&&WTdl;EtA-gXsoz8CC@j zW~JAbGz$?GWD-*(&PoQCs9+Zwp8zzVVB+Cd`WVG!Ay(6ln81ihHCJR;fVhQs&b_Bm z79`&xti+%1>1A|TRd7*>$9#=7Nyss{=5N0G;^`MZeU!WG%=N+!nvtv$_2SB?ehd@J z*Ke<-f7MkeJ+5o+Ucb(jkw`_sKvBSz)%$<`^Plgl_X8E7NMZx4h?wvZ5$GS^{AQ4l z6opJs0mok)zW&SRl%imCFLkFWG&CO~TF84ebw-zFBh;ocE9C;#lCp^|KZT&aELC3^ z_ZxeoLfyE3?J|v3SWja@B^%RNKuIdZG%7eXAWZrW$GQY6C_!%g)VguSBEBc{A}3{9 zPXHX?arvH@0OO%oA8MHEK(RVg0V9=|;APkHa{C$-Naf}23(*sV#eG%CjfI#X#?WQ? z=@Lg=mSv47$5+|+2xG43umsxA@>ZO5gar(G$$|PlpS{W!AIRXyX@|V1>mnln$oSLQ z4junF)daG^Yt5Jr61?XDqb66SZ&t-$SqW*+m)Xk_7OW3>$ zRbW~JqaU}j2zVtlTJGCo9fHe`I9l6642LSi6R@CMOGQ&&X0wZ;zO3rYLdZ(qz5DB% zAO8cGFv}7mtgkMcD~Pml2IOi}UM{b$5D{`E9{a1y^0HJl!7!nC{^FD76zClpjgrv{ zOYB`&&z~=x2QV2bn2~@txBTOuDu29p7Z-fF5^{NgMz{KFeeQ;?P(P4fXFJDjIG1X7zcUdZYaD2T2= za6q%lbgFeM%II1X6h=`*=5EXqCJg#{@7E zs{;jNzKPBK6*!bc7_s;RFrksqrfb<1{MIufPz0Fp9r7W=q0JQ&+}wpmRhUR_6bnD8 zqft!$ym(T$Xub88jx-;>yQ;$es~5_nam%Vv-oaH z=`P`1LPMm$e~5pF-D<^sx0m&CfWp=D7q?4}q~#7yqD{R5L3e}Y{U=zzf(dgCnev1L zlRhTkRWA9$Wx@3=K!I-+(k$_8vj4-{we=>eHR0Wu)I*|an#RPWi3y1pUU}g~W9tRG zTNEhpQUr>k#+Lv8AHHX1))Mfr)17=%1>~sK^)Sb0-j0X2w*y`u#mRU$;ITOnBy0|w zVl$_-ixk^??Jqz=T~S~-2ox5OPNMUHd=)Oh9!|uV5L6|kjBq;8KcNCI3Me?YJ^!9V z6Z--XbRmcv-4s$#Bww( z)R_w6Vc>{o8mc(uI;1J@%Z+3CK_&xa=@72b7*?TB5k=RAk_aq8I7yPwpk!T>WQ9LC z7?R?du!C{S*^JWe1yelN8WiTjs6r}J32*kLSW#HUR-#;#asqNm#MJP#MWqyep0t$5rG%e$=Awh5d_iP1{ z8Y;wbB|5ocWkH;)1PBs^s3{98(Gy~pM1#XQCTKXQ+YYRaUdIHI?KM4t0|mf@IHHWA zpy(3|D`A4LtGbC_jGK+_!+j^ngA+&V?Ec~L4i%K6qhl*QAzg2T%at6J92@tt@YwI& z-}ibwqr&Q1j9Yg&Uwgjc>}KsdkSs+kaXJEtjPy{Vk-R6CMHmt!;}Mc^VgW}tAyP%b zy&BNuje{I~0+6Y2tN|j%17QMXaVGGUP`y)_s9XrDf1+ckC3ZBpwaNyWMy={x(Ngfj!1~z^Z~*it8*ZdcwNd ze!Oco+FKL#ky^hbCWrN z01FmHR*ql-KTZvKJUAEl2GXM~#Yw@H|g|6JM*HB8Wrr}>_{3<(p245c$>+BKVg zKBK}&%K4=!<4yzQx^17O?>8JZJfbtW)y4HD9jz(gZkD0V_By6j+d@faxC&rx5g#vXH2z*&P22IZc8AQ**f!_nNu(mXwGWQVJt;w1(b(+PXK6 z#ndnl4^#(E-^)s4Lg~b^3*w&G8^lv*G}@zl*6J0{je7$;*+T$@ao00(&=U^Qm>{Db zn6S;J!_n$z0yu~O3X{ojwEr?SVx*MCB<7523Q1~0#dhd6?gt>u-r~C588O}z>G=MB z=&*Z@AC2ZK}XEfZac2gGD_^sbA_xu zDE*8o$w)0kA-*)tg<31rn$PjJdAZJNjxFy53MY{eq3)*ba7yECdNfM9xWRTOQIUW^ z;X-Ka{&$J4i#8xm?c?S z9b3%8>P-z{YlWRqbUOvaI~FF6gUI=G#}{@5rdw%W&2fVeBbWv?36ma?8qXz>j)nyG zCx&#$7LsHVyTECLdKs8Nc%Vur9KnPv1q=WODxh$}1Tq{V3aDfPk;3aP$xN z?0vyMVk&50w$?B-@rn9{ZSRZq680sY#n}0XRWRrB%MKa5p0Luckb^-!)apKhOu+EGyZauApXt`BD(#Mw7!$7;JSQ^ z@|Qjk!FzXIV8Q^`d%^c59 zEuSYn;TumDS>EFurujD>vpOckB_^Cn2H`2*brQw+`7{5zy<<$U7!y3W^UTM_UAI3g zn4{HX0wzKOJ2RWTJZC-O@V?xXEUbkgY^`udILB@W#%`K=`w7l!x^cF%alTG}T0~o+ zZ050#DY2+QFc4YRSSu++vI^`0*u!pA&+XSy{nbZnD5S(t$}D-2?{UKNk{ z=1k>JDBCx!iihTOZ|&P%u-$hH%Xb+QMg*H9U_wW}zi#%aa~Cm6Ech6Sc_U$%#l(-? z@5yjD-TN`TlE!E^FKec4z!X6OrA^@JLsj^PcnZQ!r!Z}Ez9_nKjm`qNHOnlFaKePe z`|Dc=nDDYWVM3?9p~%DQrYA?VUk=^o5Q1qynu@K1#Af(f_yDMV9Mn@*>M3H##A$9M}AAXI}GAzyL*E~B1p zhm~LgrI%-g@pymNnJ`DI$sDbO8gQ%uLA(WH!g5_;&W=)kEjiCiM`{;v zlvWP3FeYpX63A`HZ7EfyG@Bt@5v97~SGm{HAY}AkIOI+bZQHX26Ku~EOt31LU`3_a z(9+E>w*V%LfvM(his>_U?REtd?g!71VGy6x1lO}E?ttaY7>*~9sUP_HcsLww@gs?S zY+WFIzzLOOYw_-o>yxQOB(DN5-4K_gq)WntK3~JL+ns#n7MKDPUf$kcnuG~jPj~}F zY+f*X0Ved3mNpp9-#cxzfjD&7cBqNZ9)(g-h(EEQ6$RfdbFmo|&M|@Y0U9;lSq~&1 zpMqQpS}NK?0AWzTWp42sco)o8_3d;l#diOr2p(KG4sS~!ZiVtASB|X1S=EZL-AY?N zPcT8Wm|E?&!h{kGs^GAb=xx%w;5e~Ss*j3&fpM+waDhY8)~p zf&?KQa_y`3P?*s`CrF@sy70SkkO6C1&507)QB-|G29MTa>jyb2$P1YRXhLwgOZ8Jt z{xF(^d_m8Nwy_;%QZkh?YjjT622NCa!>JNQ+l%}lq}*VOaxjX-Lp*WX>=apEu^p1Q z?D+~zXw7oQgbDP79iXI!2@hB#31U7d@IAW!8syYr+hJDFzxC7I!+^{*2-;Eg!6yW~HY^~`UEUe2kj%bk40LKr0O*i`tc~0#qSX~c? z!SiCv=d*zb8WMy_jgG)NdgfwSV5P(3Gn0gAT1Db;HU&OF5_}0Ju(g3*JxPN~IlSpO ztZU%7wAunnFaKfQl$6~Wq4$-=30*p+4)l`5lyN~<6%*D$#6s2ok!{mbhMrl2#e8De z7;}9ZX;0!~9OknC+Jwqqhd-e`+q&dW*mfGv0lsI22|dOH^5$)`;Q*mZh)Q9Ot*Ccxk!-{yFm z7<~`W-rpn0)CEp7h93wh2_`i9?-M}62@~LBY>nPthy6Jn@j*ub5-t@5&T0O{0;d&= zq9A&DMZb4BTc`pFp7t{mD4f*-l0CwOC4wOrm>{6wyPp?h{(Rx)_Ny_;+mKMXdUdxO zB@lJW73P&MKlp_)RKo?Luv}RSOUsh|34#D}!RpaL{!>Q-ExVQfg}Vr^DQ1?E-;Nd2DNS+ zhjqetKuALJ6-3(;Ch$ncdWI0*PR0O)R@N~=Ui{%L9LWf?D2o`DxOi(xKqLuN>srpz zQaZ~MCQx5k$}6-e7J1*i>mGNL?p+(9*6FuTn9xAf?>jG)d==e$!UX6NfC9*%)ntDH zto;#uwiE~Mw4ghz6qhfxY|kgK6AUBqvoW>V8jf-kaF8)XWS-h8^_b-5&?M20!(bq| z;2;)RfLakap{~(A!sm0>xO?ih*GHp)0O?b!MYh8b@3{>u_@~ukOyF2#10SjLagZZA z&5%K%1`B~N@yCT#U=t<;{B_m7h&m>S z>}AG0OEK_3ak*ps{}eZUjP+`1et1hV9x`BqWFkEcshzy>)W<@wn+-tQ+Mg8O_NX{o zcQ&WHOra5^0q-}6^JFJ%t#jr#-6~(faWL3nu+!Cw;S)~2`FP>;!3V)_R4wpxg#&d=SbM6yHaRW05jPM3 zijV@J1tzZMz<$oAZN)= z+&vPAT*_TCkS+8zMv~HCP)z$HS^4-pi*-c||A=bcI(7grGFZ2Tl&vDsXuyF&H>P=#*_=YE+_3CM2 z5+S>0+rjtLrS$=2cZltGMVNI>1pOj+EO~wf6K<#oIv$DGpA%=)PBknb5gywurjQt8 zStQ7kgyt@`%DMw*1IvCKPUNMXWGAnMQ5`4DXd%x-w}7BpY9j{|`GMpUVJ^K_ga?7B zl<1E}CVLY|O*nOcpaZa4#(I{h`}~>!hEEn{beQI<|JsH?5Ej9KO5}n1MWzma11X0( zaSFMNm>Vf83YgS=j8R16&LK>&DG84Z*ZGFq{s)O2vPR%0oXHINER2UomWL5uCQ*d- zzHGGktKPf#PIp+T@|}0SC<%(bFWCJc7D*xh9J$T=#7~Nx?#TNh$7b64@&2apzzYKi z^FKkPzb%Q4E70H!1J9EGmK)fZaio}+4zI_5*bS z^R1yOY;V8uABf;$w>65(YoA|_>(|5j+ZU)gJKvcTF;z8CF%xOjMCl3mBPlAP`#=$w zdv;4tz~hRdpp6Ibi{@+3wS&y^Xl-QKwD1*C36vEO-6Iv(e%W^NWz#P$%J>!=SXj-z zKU(Xu5uTB<&n|!4#sV&mMVOomVn(u=eoN{dn0lEYL7W92Wi=$`&M6MbLb`-XbR-B# zRAs`J86}0ZEel2G=e5}9i%T7IoL%b)nrw-me|BDky+V4!OX*e%+2+hB9}@UTEq6i^ zZzPW~vgIxuMbszK0)xn`c%BZt6d$c*B#Z!t zn3l=cz@@0lvG4=(KE4T=iWENa920rG@H(K*9D>Q>O64BjA2|>h8q~Y?YKlUvbC_uD zWtS6D6yQ?0^e8w01@8<4{4;2f>6~BnBqziaI8!VuvfBAQNZFdH$40m_bU-J55UYN>%gw zM1poo%aCAzqH7hmJ(kz39nd;j04A_LpK)!2W=c%y({To}q$;XAy&8v<=`2fGO1rAt z_Bs?FA|w<@NJvFsND&o+u!D*;lRRTzL@Ff_^`6oqs~t9Hp`rQk&PI1Os zzEy50j}iU^$Kf$&xqqxV3u1u~AOqF&SPMI-IkNnf5+Ccz@lwlqt!MnITW_RA*3*u> zBrI&DEZ!Is%5!obNPz?30nosPI3%9KcV#?%68JR!57H76NQC7)WM@S5Kla{5$8BWU618E#FTjRj zGz=IpV8936vyI*7diM2nH_ENXRLgphRFbABk)ow4@2b-O|HECeB16fCSS9ytoI3~D zOA?uUO7+O4h{(tldCnP)$_Oi^ggh3{LHjn2N@K^^AHb{uIYZ=CR1;z56%#4IBuo%% zmNO}V`9n-3s&sTv*hp(pGoe6kQB+ijmWvsX)Ups8x852j$ig^5OG?WVnC@`DQWK$+ z66%@*n&e0E9#puE3ax)Yqm?m6N+w5N!4jW|g)4H_e0S7RjFNRR3?05_za)Z-?4h1? z)-E#b%$Fo56itYnq6m*EGC}UfpHzY=v%2_kf(N@C-_+7=Q?DmVp;qc97$I;O7N4Ye z4S;3ePGChSrYRXGpyhxNWi(_F_O64WV46Z14TWw*0ml$gP(xwZRPa4G zN3&ZwVc9kk?(iS{dN~|!WR$u~3q}nA_Azp31aAkix$F_b-fF{b>N3HU_~hPp)3+07 zDfGb6JfM!_Rb1367qBT0sG6k3dxs8?iynzNw!HC)TMq8?k! zsH~_8(abzL)PwU(V8tL;E8Rn*gQ&J77~jcb<{}G4_>AVoLUuC%LAF7&GEOM*@&#Ya zM&Np039<1`7E7Q!BS66@yr>%e%g;YUq!mJ~KmYkPb1KgI^FRLa^WT34WM4d`U0HNb zK)MYpKm|syXRs4k&T2wVU_7DE2`QK$H8C^3rB!^4f}sF{a^wU<0?<&H9WfMi=wU{~ zVh|*u;je6!kiV@+bVSwrb_jv5b|m3s6h)YwaSSYw86pck^s+JV{i9Ynm%0vf6E1_l z5I|1YEa%Y3SYBR}6T0~cW^44p+rm&-IiEmOmywf75$UwR%CIp*0n||84DdoxNo#?e3)^!{c3WI%x#YI zBA&R#-+ubrPe1YX>(>-RJZ0^F&F6WnYF=3E=Q%6YVOtL55=8`z5a>RbeuCUP$IIVE zrƎzZqFdsrzY{H&ZnnWgBfuz`@m-+ub_TebN8^8!x9R{`9huXm#AZdQXQ;)C%h zUMns;fcGLtaFznnbHwjsf9FbW5+1gYLyh~5f7h$<>_711HL&XAD?cWdd;-l+q%CVN zWTUG5hX^;2uNK`jEaWg%v-o?!6AQUiHdbhGA69}&UiKtH!#>m?@Gg3(F05&dmF=?E zv1julE^KKFLuKy?aJyQy(esqb-L@XY=6J37>zj-DdUtubfr!HSy-nF0cQ8?v9yZ<# zaWoVdzN1EK=BUw{HA9TF6Y8?^8m$cLol&FpYK4y6q=b4!Gl88EZ!HI;XUkmXPf#z# z74$f8qUyW*sA8-Wr&azyib45)EsL6+oV^u}!-`-)fk)L$nBgNa*r9wOg)7k5*0RwU zZctMJt2slf==$qi@2@_|%7mNgJz)1Qzx?lC0QwHB8a1 zkJc+R?Frbrdzqbd^o0ISB(SzNk>rlopJzCL&6Dx*R{Y5u@^K3O=(Vjm48F<0kvre7 z_olslME~!>*8qmDwv9VJi#2OK7I$+3GaC1o#r1Ceg;OKR34J@E zFUJ@(8L@z4pR?P>->A_#rAF&CgcZa}0E7`PCdfxj@;r~J(JE!-SG7j;gxqUyCNTMs zW&&Mys;iH`s;VnAoEKXF=c}`zC1PiR5^9~qU`6JIF2pX8xk>c@E6Xe(pU}<_1A!fz znbD*VkOK1{g%biPKs|w~Dsz0!-3}Kq!`hdV&Piah2Fb97W?g>yg7O(!FS3YX~n(Ei-d_FbEUs5k8=XhlX53Q zj>1Yt6S7JCk#nUlel$BFnSw zbd}3n%T75@0|Pk+6;^5;gaz{xXqE^k;9T$m3tXx86Se`?w2kwi(RwCDilkiKa*rOSk$nh$1z}0%~732f;iYlH_D2>Xh z(wgzA7|?is&0@25E4e2*!N>nrIp;&k0W%ZOqw$CjMPq396dd}%NlF8(^%CYv!y2)v zkqk|ur;`c9Y{0FUGJ(BibG}35BB>*v2a%U5sqC0eKNDPo}#@(ZzA zU_;)8fr^G6hb-b3r-_kW6EeAD;jy7}mWrw-!l(Fd=wr>*Q6 zn!Rm%gr~iT-UqXv{hQ|mP=R0E#tE?E$qD^%f^b5zal#0rtNp!lU0(Yk%SIj}UbjJ~ zOdm8_ooTc>(`Ze$M>JaZUbopHx_|ZZ{$psgNVHW(Pm(D|=2E5<=vb08 zGPW|?)`cSJ0i*u{9d|sOBRBkltMClS;Ej2VCVuEK42fktp)(-q_>grJMQ8$2lEO+$ zTZ%%!Zz%mR2?IcTpKGzSK01-X`x01OwWIKUqw5-zb)9SU+CrOgTHZhfl zjz;41O~M5!p7zik9$(TsuuLYAoMwNR46DxXGwmYL3fBPonV|$dQP~56`0L93L34Gk!r66 zjn*P)v_eVY!Gkw9qD4B=Z~ZuTqS1QAlcTFW;r2f~yn6lM;g6F?Z*lFj_czp9O7zYw z;sS(XP~Zera%M^~QybAKRl`8jpl})o2sNP~26QxIQ8B_ubDyCMCK`aJLuv?LY21i_j3rfo2w8!$xL)^Q4otM9Pv1{*D*);qbEA z&K-S)pzV5qFAdj)s-RikU}gY!-#*153o!mqrLB8#dH6hcOpve&xzPfgFwJ#h?3a`3 zK+#Sx!Da$m(z+ad%Fz|kxV)EFzJlyc%Xb*!gxZ#s`zji(SLci%dQOO_sS0LEy+-Q? z(`dc=?cwupd8C|BRie>4IT!V|;!MQ@&%h8LUp{*H>H`c4x%Nh)mlNd#@!Yqp?8M1Z zRvelV?f~{xGnH|+Le;9N44^@CLYDZM)Kf71bqXt}24j`4MSg)(E^NMo`3;IMmBSS( z_*R*vz%fX`dXLm;Tr9d0_K=Fr9JP zy`GH$_%j*_%}vtEIyi+Ei0msk;T+7%iBi2fPyxDa@q{4OYL*hR~WzP%*~+v{IxdMdST*?^0mr4z2tUW0W~iT7j$?Pxwsrvn(ynCz&hKbX}Z? zEX!IlzxiSQD-Nk7W5{$~AE5-KPa41Y|7lrb6E{H%j{_;7@1$xiSaP$=o!JP;OB&sLcys*5B7MK(LRpJS8q*K@ci!rW`GlKdK z*Ihoq#$Gj5uqAfFB{`uRPw0_Bj};2``VoDHB=VfFDYi|G*0X~~>!+`x(VBTXAqFRS zjn z@HL{8Pc(IBlLEc+ZVSPX0a~!2aS^si2Sip3Tu(|E#od4AWnm37H2CERLZGF9wNnI3 z;WK%GPgp!n*TJSgDI#Tr`2Z(~etYVcE?ozQ2+eiRP9P@~9p3=}CoCFv!W@rj!b?{E-1_X*oG3k&H<~8Yg_Iid>j5@(3}; zvH@EOzA6fTP!+4E2Ef!2ZQbHY(wl_u7-6Rt#~)i?qDsyL$2`Wx6FrbcU| zoPg20gGQ@ct@xp?s>|tP6d!trm?f{h;D#e5}ZYoV1-&;%f@l+#2J*$3y! zCMT>WtY(I%k1_u3Z@+#;$8%AUj1Ud$&dI2PKvUIRW02nMq5&iNx@=_!U~_;{sV&v@hl2v!co^DEmADV2}wp0FirsaoWLL> zyLF2bCL>5hxj+bW&j|=9j7RblSP;Qh+&&ubBhWe`xE{G-PFSAMb!E5SU>Iv5#qM|p zuDhEQ$O-Lw*KS4aH4oz#U5NyKt=@vsL7Nl63K!%A;HEzoIT&QLbs98UCljyHI=WI# z)yMC@bv0TazuQmZk!rL~E;a{^))B&iltOZ>9fbXF_TkzBca|%r*Be!O+G*~x7KY5h z5j8$2t4Yg};XYD=BsvNWpnsci3m3Uhb7b5>w$RNr;K(wtajrB%DhSR48|PpP&dsiQ zUCwZ4U5I&6ZUL9ql&{zjgCdW}T16WsYIJ9WQTBF;Iw(3eSfMYF(2ZaY zazfkRq0SQfN?CUP;@rB}8yoaEfs8;-*ip^B#R>Et?m}H3G+N`>YqZAIN7ZP(!gRma zs?mCtP^0y?{baGy^Ip5Myxdcxbwbr-hVKg;`&e-@Rej_2U3WX|m8;_&NWjMiG)o8> z3(5pK(Gj;1ZUL}Or3KEI!ilQbhqlLCw|ElwJ6Xd_Lv0J`GT>qQ`8e1sC5 zzEVZzj0BK(ncH3?1^>!oAq6@RJxhV{#aIb}U`IWIEc2Y;O>;5dEu5ew;(}-*g;LtS zL&P>YA!^wPk@^mis5ca5C+NbEbu|;)I3j1}qF$AlCBcSu-9b*!)35}(axUFa5=V_% z$a)yy1pPRH6Lt-3iJjkfxj%**YP3?3@t-^8go_(;!b$oGq0O`1elG_7UQq7pJ#i&H zQ{HNZ{N)|r^y`FrY2Rd{E~~=v#IKrRBs$DHks!^nxT))W4~!@cS@Jw20T45GdVU5K zMvXf8<8U`AP~xSDX6=@y($SES(CPopdxryo0@E&rSt0%UlW~GU zIYCK)U?X(s`Zi7=BZzMEgYC2^7q|Fk$eX`WVc1|uggSo-tr0%fuR*% zRfJ=sn3>6{#kCPJg-OH;ik0fGVTE0BjjPICl}9d_#Ze+3t%z{_Odxc?=ss{rcJ@h$ ztog14;0c2FBw#;aQy}81FcUP25a(<2q$dnieXHSWPk)% zC$grj>^62gk%R)gMk_es%Pd+KgeKc*JRx-N#OX zPEgFvPH?mn#yMSrB*M9FkP_JAhB={V84AZM$7ktnoWPYBgjw!@JU?nkS!JK&zIsOB0t!4m>rcY(1rCIm8|?@<)xrlO(|^o+4y z&3GLA9apOUto_sS3|Ey{;Tks-0Yxq-S_H_#uD~HB1hhyAa*Zmutw=3_ALac$ftjXz z%r=Ej>X?I@I;YzEE+vj?awE3QW%y4s7ks@)@C;9m7kI_KdSzMec2!hii@wb_g1OD{ zRITBqPqqK`Z-4vaEC2rAPY48sjEhW`%M;E=U{q7v4C}h z7d$1%Bet}JK~8X)Zf!WbihjLHeoXe2xl zdVGfx`qGX4Uhl(<>j7!SKnndKNM=Rm91Ah9s)o`)|9-xBQ`)D6OkTaQPq7pJqsa+{ zX|(qG?g^Z5DK5kIpJFE{mt1%C6Yek+Sdmp>@qZ5R16Xg$b)lJtTG_esVv42%Ra3i! zl%NvIXtSszZTm_V=+eO*;>x%1WOWI;2Q2@>X$Mw{iJcsvaE5`pwWs4=B)igIH4e_ zUw;0FJpRtH$nR$aLndqhwyT64A^ zOh^@{L=-YQ>&s)19z4JaK(2HkGS&^uW}pS0-lv#5jD@U?)z5$bd#bkz)n)$p-_U41 zl@o5-TJHbc0Qa=B)fYYegh5WIb=?HhgjL9-hT%-{Mb%AIWz-dI1Z)^`f$5;&IxoRo z$IMI0Yt?3F5>VX|lJ}IbosCysz*#5(Sm`CJSgntm!9qsDf3)NVn_F6S{Wd5)J&)*ER6aFG7 z;!Ho*Irf-aC?t0?VT5M2f}?zp#0eJ79e5zh$TX$nASr+m)Jl*SDIuES6>IKdbeEZs zWpo`Fe$_Hi2G@2SoVX5dDV$(htN_*{L=AM8*$I-z7$<5c#2INsC<}EaloW0aFB+CT zu+Zmn2*FV37n9CO2*Lt<222K6fjrOG5G%BYDVMNE(QcQ55slUX)VFH6Bbc!H=VnJv z0P5XMUq4}p6u6QH8_9APSYcHuH1zLTK*kI?@z78KPWr0!#)+%6wp#3xXrg8UXDZQ2 z*nkF<4P-;w`fjtanh8QfUcb3gg9A5d_0Y;rVBEp0qi^?1XhQ!RXX&iod%|Ng<<^ ziI&tAPS=^BK?t!h!k7~6h9)0mdEtap+<_)oTiPii?>txrqcsh=1{91I@J2=lF`fx2 z;SMtaOVAI^gpk*~Rx=?4_${2^vcL(6dJz&%m}H4ze&B@II3dH0C%=!#3KQA(KE5%+ z-G)M8hJy30TS=j7CKN6fMv%u5%v_rk`u9)Cpo7Q+^ZA!%o(f;>CX zkQ8S1D})Zvuqh9;NQ@8U#ifipZ0X}@=_)kiypBbjslc^6VHHZou8b3Y7CT`Q_w0nf z?mMit7n2$?=VOM&9=K)!X6@PR6SU^lYcTks1~e302RMKg;*wYzG`QUm;;|VDOjM9S z0;^5Nme1Vx9c;2P>%D0GSYZM&7ue`U6U=ppC9QJVK7W7NPX96dLzz47xH^ATCB+hZoFeQ)@Z)b%xEx%To!Psu2#Rk0$u}{nd#_p7^7+Vxd*rC0n*m^aN$qDw=y+U(^E%E|Gl$v)CYnRusqNy$j z6=~Fw4ghy!NoZxsZG#dlcM0PT5_-s44#HJ974L;N7e5Fe4O@sjO4h@#zYyev=)ehl zn4J*7sM4w1S!M>pvT&Xh66oH({&17(2`-K(&mc1(^X8R6X!hXbB{Qg51}Y{TG3Y&j z5ey3$2>=ZR*XDtKOT3i~05PPJg+*T?3m=s25MdP8K_aWJk)6QiXvPqu$DD}CYfVm= zjShL&!V2D|H#RsiqL8`Bg|OftDfB6Uq!9Svvw!_=tZ>YKDEhQ;ix)wS*7itc294HG z^4W}f!)=_fL5xuCgbm0+wk{g^2@EIjH9Qo#kwv)|jn<^LgB&6d7#p-H;bQs@0x5ar z19hIB>x1#`IWYTKpA0-_BEl#S(UPDda$}LG{)&Ke6TotC$jvH9D$tg6v^Y zNUJT&F$ft-oUL|1HRB$T4M{?fQ%|j|lm}XJjLc8)^RXFH*n(wtmAD(i5H*6H6;+%d z(F%>$>JQ@tU_Jd)5dU&ZJfS(|Z2;s1=Uq-6fhfCG<7lV*T1X^+;&!fgrD za$6&j(MU=@xwf5vZN?KUk&!Ag2Czbu7s?3%r$-cSb07MgKuUmz(6JDV8^{WyS(_9F zM>Y~g+C{>7kwtrGVkH`_TQd$$({d*#>^6$>9!}eA`M5MY!C;ocAnI(ol^M;+W}eka znq)Y=jIu;HqMoN)>{@Dd_L_@=dI}Tf6hf5@KH=m_I04}zgba%h6VYQ;GhyP=XtVQ- z!j%wCxxtHL1hX0XCVhvMdL9doF>Wg%h@82RIrjL7C0n%*UFO>y7i&n7dDXSbwCrT1 zQyi(~PLF{NfSM|P`b6%$VL|!=DPRXvtEuV`n?`H!9o9=`E;v62s_zucr-k^#+?Y=j z#0Rizjk2+FsI>F5EJGPtkrY~k4W?+M*!kh?wQ_=-4os){__t4!xr2&RR?er(h<7H` zr4Dd>V4Z4^JaT4jv{)g*qm3Ab0zpO~A7J64j){=={0FB2Ml_@}9LF}BV-9LEtPhkx zGa(}X%kL#+~?wCj{ zg=}?xGzS%!zeH^%u@N-&X@nF)asxa}!n=5jgXFc2AK-b|RyKE>^k$Y~@e;nM`) zAq6Kim0w1%6@*`IM=0HFnMXyG6wBL?HAq2!eY3y_0iWe*IiMtknCAZ7w7|sY@T*nK zJtwrV@>=fJ>)2##bCR7(@6o+vTgbIv*mdjkGYy;UhQ!>CJgAGap`xg6jyJfT<7f_Qh(XoFKyqUct5SZV^@# zN2=4skPZSUy)RR{S-QoiBTkjT*Vs2iV#kqW_vC{2vcY;cUaO9 zgPjoYpSGHiU+5gkn^B%Aj_Jqn!tIW7PPK6Y=SsVEYraD_o37k%H+q4{}2J zMIQHX-cFu07P`K}@tX5~wN#B(^z!Qu_g}oY{}{6z_dhA7s>wo!E>+7nw2gk z_V1oQd+_-A^Jj1O(T%CF{uUHmRjo0{xBgdEa4mNHX-EmIutGBdi%kXho%sn&aAt%N zwg>iAl?Dx!6F5$w)1}8WEYM`y5yFL&nNY+6mnkRI`PNsP#vA_8AO-UsCNKg_vw>lZ z8LlKLFewt?Z>Gr!i%9|W?1Xij6r%mJM?XA3Fy-xDisw9fe*ekG{fmdwvxoR~JbAvK zu9XssNPfLfoI+SU^j7pmnC>zZa8ffR zD((zHe%2*f01G~~HJVxAlZ!jc%?5J;4dsOFxSc@5fNL&^TRCA%;gv6I;e@Sc1&l-# zKE@a@Jp7OZjn>N#j~+b!F+pql9RJuX3U&5a{u5t8Nnw7yC)WEPU%q(og|Mhw10z5T#K(rKb%rgV!385$sKl-GQxm0NP+6^WPnP`!^KqB^d-Cwz z)&AqV>HZ~t#UgaOpHDp}B>QJici;q&$lC`G-#x$o;w>(hJJ&xxc+Q*j^aCAVE;Z2et_ptb?>?H_#99vm(*#SP5z58dHY@B~WI`JO^jdgjNz` zVw{jjxN|bIit`+d3<#{?(qSAELR&JTa2r~jFbmOyzVqN&!RR3_+BV4`iZ(Aq(R{u- z!U;oK?swLK+N@zC$nzcvf!G=+?6w_7z^jLPvloq4RBsSYc!=-$>9beVOPgcg)0^{0 z_>ZI(>74LdNiJpxgw&P%%zwFXuaEh{P^e@c;Uxa48cDwQK_?JQYre9zVm5AMnTd@#Dicyz|BW>9ai(bUS_n zmuY|#(0$;RA_gstXM~R`0!buYI&w*$49rC6a07_LQ$k8gh|tVijzf}KGf+a#@E*35 z63AbSCM-si!gcHfoJBK%bD#nzjGL-2o)jkT_Tk9B3Yz6D4I@maXay5K2ZzBP+%W%P zlJ_VGxQ!bG^dra#Q`ZhYuKqeFh>-x$caRq!#s0>|A`L?`AIkklL;~iQXXFHS-*n1< z+UXh=>&FsoFyEd(d=3v;RaignqibrmPGah|O{v!Y8_r9%^GDBiJ73G82A`OMt20VK zzf&`TJ75q|fKl^|m}<-d^yvID;gNDGGcJCXorMsiIcKLOxCgWTLoy|B0(-i{Uv(5UZlHZFj{c!EU^ll_ZFAFt-u;DpUn z{mDg>^3Ag!UOajG;KzMb=p3OrOeeS(%XEsP`2E4rfOl|CPM9huz{-6BJ`tzk>CKPA z3BnpUvX7k0fvku07%8+7D;t=J;3Y;Zu|)?GWQMbBWjqT~06rfob3lQs~Pyy5tT{D3+EyoyV4#&@&Q( zfe<(%E>03|?lXhVm7XSZ6S*6<69R@gq5MKD!3lz$*$Hxi>L=XgKde=mS$zj|br8Ts zrkqd+C)DWg(;Mmg>+|Myn3P{~Id91x~mi z0db}@HC}Z<18XfR!~X;jLx?AwEv*0FGVXv*vVZ>MyhyIV3EQU+U%vs~?#>@SfBg2% zgBMTEc9GBhos$EmS#pdMmcj{}IV_0h`!^tx=g;=!gvV#wNI1c38oNAL3vD~$kjt#3 zU~L7x1`>i32H+yG6_K{7K%e0lzi9}98BjD6bQ}^JG!(c**Hlzn z({08c%Yu}C8&^Igy6}0=A+SId6s9fJ^gwW9*)6iGMU{gGET#FdOsgW#b<#II2Sy|% zir`B=kSp*;jrtCi#tzm)b^^PAaKhvBI=Q0nApRH-!GC)6;r&BI0%S~({~UA1S?YSP zzQY!!vmSl8QQslO(Wg&#^7jxk(=crgR3HVI4ATnWy{NpPR!GfEN`@A&OAveqasn}wpRlHe2;vD(@KfJ_ z6C`f?VV%Eyy_3#;vwe0S_CREP?E2&52d}5e2a7SjI>Z=vXUQCV@^1IT1F*>3mYr}B zNj$-i{JU5cu+4E8Accv_4&Z4EfkY7iH4Ypf!5RpWgf2mFL^B~JL`n&47Y$z_W0MPZ z?TB1}wpj@-Qt2cd9F5>POgd&lG>vj`9bye9#6I6R#1lS+CmzJaxghu6pfc>}5CSsSdvo$7bp$E}em-P!x~2&q8@L)vQvP7Uz{P{BdAr=k9)TJA;! zTr@cWbnxZRFMl>UVYl_1Al^FAQFPN)K+tFfC&)i1_GSC`Pwqo{^+h4$4yw$WvKQCH z?CKRXT)_mY!ulHjzvrgln(ZRhW1U}=SEk?^DJAH9=bHD|>KiPzWEGyQh#u2TB zAL0ppPB^BYz_G}4q!;14e=1crpFYNSJ%?;Kg0tRxTu%4z9^HTV_`%c7^~dM;A3wSO z?88+eL7WG$DK1b_19*kIAf5oDVgDnj`T0Y!6COX=N37wXw4kiuT4qA(0`z;@2*Cly z3Vz@bjW(=6V_335I6}3?lT7R)8*eZK)hM4W6=xi15lToIBuFNctRN+bzEaF_g?7oz zbDW-dGl85yxC!;3I`IT!1sS>^A9RsseYmFcV0OKKwKyS*n^U7dhulRKsx#4RKDan= zf?o{y5lh98_L{6Rizn3W%CYTwk=Gp)fz3;?5dNG3#tAzvj1!a;0wsjkagEm985T>q zB-?*{B`GqO7w3UkdQvqS8>tT{EF8iDJOgN1J5gD{yEig)`FQ^F)?>E`@3a>LHAqIH5ZU;Xgv<2I5@-|aPL2?pkb<87(2p%jR)attmxL1`6L$6nGHYnpqme*9u<&A( zV4sBlooAL;6h)8!F&g!17igG3l>Z0 zKDzPFJ|);{*+yTFF-up<2}8Igo`B2OUzY2#X(xD2(EHWf&b$W=J+k3ZGItqXe|?Ds zcl+Gta(O*{_!5nqvQzM4C9zgBl8^sC;WjXeTPM? zgpiYkNC{hk?9*4()@^1vj&&Qt?Gn>hpm(KaTj1ZF8?!S)kx5(sP$#U7O29ZltKm042E_1_Ah$3zg z(hA3Xo^gv5Qod|qD+VEG5-=t}W=JIaq05R^`Yxy2pr;8}C?iDiB+7AI1`#NtPZsp#ds!na5)Yk(kQF?!WiL?{|s}c=d1Y9OR7Biwn3jb*10THz)(9B0D~N)57|G)O@f25SZ{0`P?3 zJ#%h((y(!kLX3Ksu|e&G?sQNqhM9Xt;B*Mu1saVYC4^uCJO@S-g2WPBi$XsWM;*W! zX*b5$cOG>3!6nDg6+h%wl6LF_V}*_{n8oxREJrDfHS|pcIe6fN?5IRSztpWPiS&p@ zYvy&EYv4cxr*MLUZ>|wLa)JWi)4@(KQ^64U06;@gZ+p;m%JO3V{MA0))P@b3J`T5{ z9+Q~$RBTN{37Z(kg01 zf@eggNIG@`*LGGD?Jeprn1zsNOKyA1EN}}XWO|zUDPX50_mVL)P(rL_qm!0m^Lau7 z@uLnP!WJit`y&s043k3IbmR^ASY6!|S@-oY@|KOzU;uR=^wG2vvUCVM(&YsUJE}%& zz6|JSv>Gp*#0eLyp(dQbD1;gcZB8&+u<20NXbMF4Py^;I;91!gm()2iR>&6;kgIn+ zp!+Un9QW6Y@>)a_H-)+pcx>3J4OK~Zp)$GxM_`8)msX`GwG9|?TrI|Nt+N9#T z(457p)0)RZ34dVm$A5~Y@CV6Zx{&yV3dZoBk_QoZ3HrV5uoh zHRnp3IIE>D%^6Q%k^>1&evrIrN`YUTbLjuS5I&yJ&|doqubzIwQnp=zr@(p-5T+t| zMmWd^W0f>HVXcKzS~hHk4u)>e!uCfPAq}M9t%O87eM2!pIDyc$UpQTaJuWk9(3iE@ zZ+(B3X4cb=Vz3;$Esg*WFAE1gdz7qSs`*s z4@bY8k8uL_953uY_6W8?%VVIftzXYvdK&s;60k>$PM|^dZHYcJ++imeCzK&Sp>HP; zIJF;C2=E%MbqBCM&~dP?4fMh}Fu0KK_1UKxg;;x#oWxnII*#H#&{Ciuk5`FKYp4nR z$~T>N9bce(z+?s0R?_iCmYY&BMP(A&2v{Rgp~^v0D=^B$;RcK+kQM&3G48;6GdWcz z#etwKY!w2rfPCbyFc1i{6YkP&rbq3B49t6Q2@!L=orq~whYPZ#Fiuz>B#zrR22avZ z7-QXos`9v*kwRkjK+s8`_mG7CZ;K@O5C$IUA)z-z-KNBK3*ni0VEsMT3Rj7bFh(jF zn>#_y=py`viOjtKCA0&KNi=clIVAB67&XEP#tFf3uvLgDjN)UyL#OvKb`Urr_!Rvj z@@6R*4Vc%^dH5%DmeTJ*Q+p98=NMW{wwnjkBCKKG~ooE8u}`QEJuuDka#} z)k;W(l((Qwf+d-VqRq8TdN6GimZ>Kv$XktlrMDQLiiFTiW<;fjFFUC$&yTZWN`#iE z>3;kRaDqJXFgu~I+iX~~Z>XsN!azO0U&z{;nTE^+R84KyAPq{Uk+YpCHP8d%i5Lnq zF%&w{vM>%Vfkxm!1s(}uU-k|7IHEB?;DxXq_R+*q-=QRUdbqN0X2)?xKIR zEd+aFI}AJ62*`JM5v#9NOS73l#*5vpoX~^!9WHk$`XXKXU?<$l2}TV4cp^1g3mh}B zNlMVXfEWzGfp6#-G)18}A&IVnyciH< zLJ^mEKQIU^YA?m8rD~X3FxDz_Ip&2LGC49iQcrXj*$ZG1iaKvSZf(5 z1QICH+S5;1o1s7_I@EO`GDY8Cm2j6fXeaO{X*ZxiVPF~xg&7AxD|D5!kb?#n*_Y7_ zaC{QY$xLc1^z8)OHUvIs{0OZwXpTVywIS#+HOvZt5e@Y$;5NvR&Lql62?6sd#~jFL zHWE22j1zKl!ly78$wkk9h}nPCZ$ z&#VJ#vJKopxxW!9{XHUI*xC zg1HV8rs$2}+XL-U&lo3gjuI(B`z&n(U2yzi!$6}_LSCEsBG(J^EGTp`v%BE1u3|

BjD?^CR48;}tKkK_*Az!n){64OYaJ=CFukwq|~SUJHJsfQbXYWWU^ zKnuiZm=l7bZUF^s^(n!ZneN6Ev@pLBfh}y47_%Ua;$&hlUZ+ND-g0v|7FlnHFw6<- zJs5##*a;_b0%1u=n{-->Mr%~gT^XfGiv7hrjiM5rFmiR2VAlcYfB4IatuV<~qZRV> zt`;ogfFDm16cQvlMCopYyoHt5T4Pxlv^VNJ5V?&;Vri5>E}>IjQ%v5^e?XTfBdo;C z=e(s_g;*`}C<~RIAJ1wkC@Fa?NC>9o?i#;B{%@}yT;>NG z0=@N1&+rg;=tl0uO_H6O&B&wbHN=c0I8#?KOs7dSa#=7F#xn{Mq#kZW8v&+5Yb?^1 zY%$C@%n4)!gXe_4m0-0cTUf^JR0LZyykH>)dUng6y!=VDVA^Xfzi>rPs89DD%ud)} zd)(UuY%RukH)ti4scN)x=Ho`%#a4DAmdr)J!n`C z>-Cz)+vi$<)}=VtY4-w zQ{j2QZ~54R_ZcFzNC?qHb)K-m9y1bjh1@O|OhHG`(D~I)=y3v^iPKP; zM(etz(fZ=$>&vUk1&vmZ>f4(ie)}57SMPE5Yh3w1zOvSf*JHhS^=4nbdh_+`>o@zj zVkJge38iK-hL}Quw(`W5qNxEi6?7^>C9j2_)Eu)K^de^RfsjH)tA&bf9Ep_elDW_I ziqooe7*bSK@bKQlR7Ii}2w(yw&aKEM@lj4tJK=8qgun@z)__!5mn%O1El z{1F&D@Ua845{w#VaodGFP{V09Lvlp7d8vAg0yS7qwG-BRoep&)c0zf&?+^exAvaEg9H7?jgaq1{1;7VXQ_rsMKfLJ3Pg8CNi*T^ey? z@g7**QKF6MkfER+ZSq|pBlv)02r94%sPhw4;syHY2_|8gyW#{q90*~}_CX=9{0+!XD6IyC^R%$FAf^5)h!yWoBfAJ zPZ#mG4^UQ^oq;*Ll zDQs$%qHckluz&-g`9~_Ipla_D8l-G1Cr-wA2lyA`g$lNoc`iz9xt6K~DM;obh8;NF z!I&WtHprWtP~lEBUR_QwJ;qb?6KtS9cn@qjI0#D|F?e!W>j=J;C1-u2Mm%BN<%C)6 z)Oi@LkjRL_sZ`Kx9nValb`w{Jm?^0M>wu+283OHPj)aiuZplXgCSt_ovJ2fY0waJu z&|qwll#r;Ez~M_X6P9T+oFK7;N#KMQ9oYI5P9U0`U{l#zL=aj5l&%XN1EKYO*X;o) zF52G6Ff$~F`VM*0N@_AYAwUV@1WxeHdw~%ebD&P3g|ehZ>&;nHqm`Oq5ww$@UImTT zOVeo0Ko($(S6GaM3*KG*K3;r-9_vK(Sh4@Z%SR7ieGpmq3Wzm1p)5%X6;TH(p{5yN z2-X5U2j&=RHF=7uR-88}P4rq}h8hGwr=nhnLcGuqyEw0}I9Hk>1;9omr*a1X8PF!_ z#v=9cYS2zNNk4%V+hiFk$devYmSX2Ig2poy2<+c3X8DM~2|YVuA>KobZzMNd(jcG7 z|5;S%))^-bV|gnz z;Cb9m7^nlKH6%B8*$L!?zMXIq3e{-6IB2w<7m?R!EpDL4xE77pT}z|&ij?u`@zcdZ zYB_)tphokX>9Il`=d17}YJn%4->LNF77jG#fz$l1hBjW0&5u8&gFff5q94j>7*4&xABY_LH~ zv^74-PLQdQzGaa_3^g(HA3BVkDgKZ9Mz>o9VHB`yUTqJbM4b!xzuKO5n~H znyvDZsHkV8*iecMy_n?JHx$4DIvS}WOjt^z!cITavjQjJxRjgR>FF>bIA>D>kE`uJ z|9Nq7u>&ro@y~x=;<2gZ79b;@!+Vnx2JD2>^b@$SaFzVqG`t6mvinRUCN-u-ixMKe z-Z&w%`vu=&0aD=XNX^RoSKA2=i#{tbz8KOFEOChf2nL?e(Qfxyn~vvuJ`lxn@_;cy zMvjP_H z_37J3Erw*66G9>p*X_E}!{ZJ_d)#4pIwvf<8m-Z`E~wEONqvX?w6D=xJA@!!K7aVz zw^j8?e1|={)mPDDy~5$er^gSTe0%#49OIlA_ptEAQYcEB3~8*uoMvocbtF@ZY6hMY zSjJ_y@>$K4WERwwoF=(P;GA4jsWt5Y%pj2heBwTESw&sv6*bm4DMO1?4~#nu#}oR# z1JCkf56(zI2j2{-xDtd>0?!K}Ms5b=2qWaM6yQ8i-s?D=8~;^10nfgzkTse*;K-;* zAukInq`|o$vRsZT)(TcOX?9N`2eg;VWEv&T#vsWuG)2TTr zc1n%b3*`jF;jf+}Sa8+WXe~GT4=){g{8Y}5su-N`;N9Z4#SGl={I}KO&FgOqsZVnD zM2=G$J*FN?S?CA!q+p57IF8j(QrNNbymA8KiS;fgC$J8F?YpgPR4aX#Ek_s8ra~LP z`U59~U*7djj>fT=C?d>DIoPTo>kM!L^%Dm36Nq)-gueHnU5fDiBvb>E?@8`&h7-7L z$S2^0TufETj7g7!AxQU^7=f!~xB#o1;dI;kHrQAb&EzE-Kna`1kQsOLYFx@&(P>x; zBTOSGvcc#&r?av+^G+F~9 zT=Ve#jYO11j}@ZClf_rlW93jK3KzZEM>0Jjrrcsxmqk^Uffaztr#;t1-+9>9K3FJW zKy+MOf(v!_rWp!2FUNxE&P-!i`C$y91aLwhQul&$A!wv0C5cA}0+PgZqfI3t4U7OfbfM02q*emZ^c-+L#9Y-((AOs{fRS*5#Toe= z~bU4A`74#V8wHc^2Pwg(OTAXkhIDxXRjX6-K z3_FlW+8V7+COEjBMr*chexh$5e|Yl^e?25KiXQ75(8CIPthi({#;m9h7qa*^Uc@=P zd#CmyiNgRpbs#cBp;QvG!XK4Zy4?j!EP*C6QeXav0_Wtsg5+ zi1Dups|0M@y3PH1kzbZgr1H`UcUX#bidMqyLw~~|1JBHoXv$LplBByMiX{k_MHBQG z{L5R9;^4}AhWAb&Mo zaUH;YhOEEOTFZX{j1#^zZKJMkGtuV+zV_xt^f`fOjYXaaSQUU1jaK9&OMx`_=_%i$ zEDjp2+q#Krwcn4&qNhCG3uFp&&eqo$>zh1A3&|X!mYOr3AQu&tuoQy|1VelcO_d|G zP~{vH(E8H8*0h{#(kred zAPm#Ps=MR!=g$jSvN>?0&p_+i)#`*6o1yVvg9RR3_1|~#8CZ;M({>Ky_x`T-Px;>k z)M&Ly7@ZhnK+|^^)KppCTwY$|aVebeWm9@d){V}U#wMS)LOQ6JZ8Fyht*jy^)b+Zf z(b_>Z`h?*Cp@kUVP>wkR@nke{=!|Xxrve+XCJg7X^jwu>to$dc3?vKV1g<)PMqG<0 z?k-|TDyYPh8l|mRRl24COD*!x;2!pOa001RpSHO6*Kyu8{TDIPW`(7Ayuude5I7D2;FehBRqKtn zFcS=0z5*WvS|FKlLc`uMnmw=o!*jAuZX5{T_Lg7Jiv z*$H1|4>el<{bXsQSm>|L5O-{<~Synp5QU^Ny*%!w+Vbc(s*RyGYbg;dIM)NPK)B5Ma6-4!q|Mfrr=l{avAO9%7 z{ny~@XI*~=5W)xt7)2nye*5nX{q#N}(*0Sm$3q*#_;3HaSa;{rf4lm2_-Fr~nlIO& zgbP49!Tbl$3Epy$Mc*T-N;(UoF|^%?eGoZy9O{mlP;UB^V5Bfs*8${!He3l7R;t*X znKiz}ZW;>fQbL0zoPa{qo4^?C#XU2GbO2|dhdw6|IMtV?$5NnOoaY|R^a{1MP41c7 zeLY6Q*<#`f;@bh)mnP=AoHw@S09v{eEvN*bbsMHf&X}K{^I}xkVOV{1J~d>Ubf3YqqX)p z(M}*CkQ3HM3V{=(WltjrX;d3_Mtb=&LV|UV+%)4Lf# z&Ix1%+6j}WAigAD@RY}L-5!VpvQ*bB@`~duooaFlOju+};}B$`U^mUSh$nF&E6l&+ zKpcC7tgyi;IF=9LQwzRu=uuzI2Nz)&UTc%tkSm}neO}>b09lBrJ zK)uT}Au&M2+~q;oPsf}n=Tu?A#bZt{lrdLL+sgEJ3WY|QX*8NmW@D2s!Geu3XGC!Z zL;%wm`Li1MV|0cgxz-`$FH6h7;GxDza#SA~7B~+;doa(j;3w8gY-Kkgbgw>5Ow$f8 z<<3PHbwi`ICcc*w5F$U2g5@@I1DoMa;xPcU2h2(!3tU9dXf44B>O>5n-G3SD9|H@? zCk7SFWJ`FWDP&f$LQ^eL$rMqM@I@)P$mFD|pOCRtGvQ=T(DY{3X&}r#0qUE*JS;A& zxHy3l#P0|it>=q);zCtt7zVFRor%6DE@4}`uKml~jAGJ(%E0>s z1uF|@SvGx7<>U3_4gNB6hsh0|5u&`6-zSs+Mi|eqvU$nY&>51*dG2U4&N@G7 zuS!m;LW~M)X8;;}IxGXSP&}@w?z~>Eh4eY84QuXS4&(T-tbbn7%vKXIP+)7&Zh03c zP`BCagmwAFOoje@2eTGx0hypGDzg%>D6go|+Ei%fXlQ5;+6ttF0Escwy9P>`wSf0? zLJFLPNKNIOGaUd{nmXM~r~`I{g$whA+|HtfffEd!afe}8B^>-sAkxZukuyh9Al7^} zp<0D3>t=dOlt5tFW$bgZTMs#t7HHpsb0z0Ve+I>5;z*1 zE-K%*ZJeQEgGR%)m=@G%jUt(vJx_P(e4Qj^D$=a;WRoP)IL-?dYeIRfSZ%G{>EI-& zs)5Lx=DEHtU$LL`@I7XNau~5`zK!X!)?04jc?&1dc^JmwtqvNkvnZzq3ef+Gw|PIB z2^=_qSA zA8Z_fXM!Mh46-pe-PJ7iawriSgj%zPWSfa2JDwA?n1~F700<*Q4MtGL!N;3W)Rk@n zUo)iyKq&zfKruE0+Os5T848&|J%9$9j9{$*+tb@y48ofU`VwUJQa`fw=-fT&o5U!}4*JzCpQ=nQy!oy*wFivSzsqpIj%4!mlLa*6e z-yGuvg2d4!gB>44Q$Jxb$%`nWm{<>)S_fL`WNCH+8~ie)1K|}K37XB;ht|AllyI2vYhPEoOAEvaoUjEa*f%vnPN+)FK=5^71z-e@Gn77* zP}dS)$W5d59P+B!wM=?=`|9k6m)W})ulETxT3@P0YoLX&vAYA+q!MNJl}|rXGLl>Z z1{g>S?fW(<0b6|a21JQ|m-g@%8z*q7^T@~v9oRL7J}0b3v9Fv?tiytoz@|fADFP>~ z&3P#B3f=>;Od=rB-iP)aMnbqLm&9a}TO}xiSqii*Qeg$ac!5EHSb9F5fdjZ<)&jY~ zj&>NqYRxwhdjFCy^H$_NsCj_HrUi4D7D@!2caNaW#&HA@TybcEAB`RDYS~Rj@CBVc zB?J$G;nwl2XC}09a2T?HV6>xc>u^w4w$YV(=OQ4u0ze2pgqt*phfA{vY;SXCK;Q)8 z7%OZphkXb8X1@1zyl=EZlXE4N5h}~$Ht_pi(bxy5v7kv$JyKxPV3c5X z!ai`qP2hxP=}g-Ws2lSlR~iz}q}y9+NvAY?#;U1HHpvMG?F8r6<~+D(qgC|R=4kzZ3D8fV-2B7q2k(o6Mr+dPyVzj#{|?l%xF#vY!bjdx zsYwYe7fJu2Zh%tk=FuA;gl!!f31xA+*mgOAyl}NACjk3hD}mY;dg%ZsRM8~n=z{l$ z*8(y^Y8^p37R~^cO-^u)%24t$P=n2j3k)dw!DY~f%iI5jc(a)I0@qL-nQtDd#j~%| zHbJYBgIYKuc)J$jpkuX^bHwfuZxy@Q8#9B5b-m z_v4jLbQtW?zzBfLVI&Zr8XV<=<6X2L=i#Wka0cCNtG#R%@Lz@*Sh?icIUHSw}HW(9V)__<_XGLa?Zh zQ^Lgm{{bs)fqCNm}rL~FA(%8%#$z&qUSyIv>7u%*K8?x;?nz0*mMUZ zzDnfMMK}dbL9V_;p5?NkPL9by2PmOX<2f*b#18-;R8$Q%brjTdzzH64BxnRwTY-d@ z-vC&1Ah6&!jvEI`DLmXBZtRUzg!`XN4jyUMuLITN1Y+2CxYz|w*eNG;5*JAXq=cNk zPAS1rqcy6856$N62lDPs}n05f&{D{toC(_om%dMZ|TauRRbO9RH!N8!&Krqp+G5#0WeLE}^Koa47_%p=AoI3eJe zoj^_ihB=}3F-2eDng8-SkXQtbR&au}eiTl4Xq<3u8m)eIc(hrWt6kAmC=NWs4F>ozCcsOvz;vAg&H&sIF4fHiA_1UC8}KuU1M zGH`;kBQE1L`qJGe(Sn;8B>;9Xq`%}v+#rR-h0o#44UR+M1EB@|4pnLhqmg(9m~UWi z1DZmA%0z*gEBJ1|P<&;Zv24;Uku&(NxxYnO*+5u%I0q41_3A`S=}Cps0&Tk1s` zCj_)P;ljrg(7=NT@dN{m1X4n|7D^}&lmM)QMk_c0t&8Vq1v}yTpwSu~%Vyl&i8?AR zn4sYVhbBiLv|OaGUmeg;R5$`MT3Kx^@uIe@#m*qE90VuObD)_(*Fo)sTS{}xXlR(D>tM|yf`j-qKv2Gki#LAZJc1hjy#G&7FEn9NBE2t z;Cr;x3zZe-WQGuV?7^9-Aar21f~^hwKxQwg>TH)MP3xIHLmcDAY7XdGjqaIxwYCYv z3%M}DOcLg`K;tZi7omhe!ysfdV$pIOl1MQ3c0z=8X6V`iX?9#`EmN5FV+fqg&aBds z220>{UauBwAZ=O+X+Yp!oYI2JItaeQG3@#^UjipsPq>E_H-s0uoM6GlvS%frZ(UBH z(K@F_YxeZf`?D7hT6V(aL8Ep3-L3<%jp7=X0$4#oMgXdnS_raiNKxLNT+I&+AEu~l z&Elq|U=c#L0ws_WNC`9($O+ioR{GJ0QP&+#h}TXyAq|1`M7?$$U00JGI8Z{!%Ly4q zSnv+Lk*)rAOiG}Wpa^5&h9VcqV}nbq5zUkp;Cux6(l#q-1Wq4l^u^W=aYVWdvT~N& zm?(nE`%;TZYFcxv=5%Z$k=Sl4zpL~s!Jb6jMk54ZA=TYKOHQmB324oNN-MW@9Y!3y z83fE_iC1bydN-0lN-%_W7-?;#o*N-FDZq+l>iNzo6$GBh>@tIHn-P+EesGosMz|s0 z;qA$u=Vhjj78o*lO(1$ z_kS#dM(bbXgj`~bRUQL$8U$V)b)&3MV_6$+r35g^uFmnX*ERW|1n55a67?Yzz(ecH z110DyaKcdvHm#>FC(J0GT!b$WP=O&fM({bg`uFu7G&#XYf#Vi;i++<5VlxXY@x?Gi z3WuhWR956_Cj_bUPs$2&zATZcA5T!LS6%jG25YCplwVe!MjCVCwO*J1PFfkfk_8SR9N=^ zC}9|O*`x-m*qdg7A#eTjt{c1fZZ7;LLm0y&_zrxXRxeWLI|weo360))VdD;g5`+d= zd#R|sQW57tp+8kc&}c38uV4P~{)RbA(G8N7-oN@7bv0Tyr;a)p$~+d~L&~WdBjmZI zCu{&&21*zP1Eq|V;**P^Py%R~@(K}*1pEyyjS|dE04G#c7~nJ<(}`$uf&(WwNfWYs zc1F)_!XJ8zDK`_r90;5sgrKjxkYmIFcefEQ^MFY?9^3#=bxA=Bm_>lg$qAilgWgmy z_n|u`nF8Jfa@@!SY%G-&mKI{&Pvx z#|fv~2?RJnOoJ^sfuI?EQ3pFA_zj>0dhI-*O_jT9OBf-jz@la>@zIVZ41=N4y2$a#5mE5qJo7dq5cnfpGnimzf(9Eaz(zIz zt_*{mfFPrEXRQ)#knctQn;mvoH#%618vf&XO#aQ>UEzd$21+1#`2XeLKD5n+Ors74 z2`~7p1)2)^xGQu{QZVb94|JV0LW@AC{hqZ8u5^YBiw=A$Lzr#W9A&YL zt-G+a^&nL&z0uL|B~o%i{r#L!Twee?0GvS3iVmDmdB33&$Dw2?cC!+&*l@k6<$t4! zQrHq`WjY6HQ_s_FF}*a6*6&YZG;E5xa77&$K}*77Nid?&==Ak4WTHgHdz3x z2z*!ejF7GCtD8$w!WCYOCZNH|pLav=^5yFv{{SZ>#bm}rfJT+3r6T+zY3E@Vusubi z016f-7zz+<|2=0TYm0W`;=|NABZb&VfjJAVFfi2zZ&|BK9B^yEl+IENEhn1iQoFUgM^CB0xTu#jKaA+&Ui6D#*KnNX1u;B+X zf?>|5(n4pV0@ftB_9%lb0(-PT}^eeoWO&_rb0ZC)EuXzpd=8Y;oNK>%j3z+{`uCf zO$sq?T(gTxm8+L6IIDVT3_@IC$kek!chi z8QRHB&k1s#RD>}zZfSJS(2@>NOe}MA^a@~X%v8Qg3}&B zgIdk&Y}j{jQS2~?h5NhKO61%(xpw|wc#AO^AY%x5dhMzG_W(q(m-xrl(t1kWZw3XCGm08a>?$pDL) zgc<`w@cKSD<`BFFJo{FHg=q#%1fHM`j;VyQ-$N@$coW&Mmi6$)K1<1hj0YGyB{4hcj2sXkH5+zQe z@dsKOdJ+bm($7W`e$XWlg3AWP6Z)m8ZTjP_^RP|csRwy$$jxpxKHOzXtQEJ{nr8&< z@?OXTG)^d-4=(1g5yS;2YLimY@jnuUbl5s^$G@RrjwQZtZZiZHHsR=roWSmkPP~J@ za*jp?(*b+J)X0YG(6>xBfTmW|a%tu2ZC>oaD5~hDA`l>jh5*vKl%FV$;!4?oiz}H3 zg%vmRs@1O;YqVsJ0^pvESDMCxL2SqzVdppoAJ9lJ=K+`zY+HOGc1hF&c1^iH zFanl+Mj#pq$h~mi(t<<&!>Y?L@KN;K^B$BL zPT~X|V56NNtb&JXxnJeEk0ekyy$CVBiYZIgS|WV~&Zix*lW~W>K0MwAk$H|VMtw8Y=mtaP8BZ)kTKPqx7ZpfIG2|i5mYO6 zjbEasL(Fho;|N9xAcb#NtTpdN-wgE;wfpTcP7u+>C2Pcmv93^8aJdJz$$A|;=s3!Y zC|P@-n}!1ayP8HdmYgNY?|dR3krc9OSA>6GfTNz-=4RU<1sD~Fbh{`DIfS$mLU_?$ zVMvnWc1HsWDxs!=5m=H76Io?~BywsH+4!8N1)Jt!Q4k9(Xv-)8Ta4D3n1WgeEsPJ) zVTeVn`z(bqCL{DQ)pST0OMN4OW4WD5(WJJfbc23r2LmAtvZmRvzhHErV>U^28RxcL&=D<5zY4|F+!MVB@M0fhp@15POAE!mTiKk3UwRs`Z#r05 zcUvKl;6+;hTtvCd)>{V=6;1a|TUgHVyL$cMx)Dwgw+T#|t?4|dx{7I_n0ajzm?rV; z)5KFmF5{6|{qch_`f!G=0`mk!GX{S=tvVd4X^qnZP%#_XD~3LKd`BUbIvXol#Bi!TS*leT4A`@_`(6 zy^bf_(LoVk`yXW5L--0mC=`(rqId5}3breN`21)e zQjOM~)R0}o1vw#zttl&{O)DX(&z?MZc>nEwZgT)7#26LnyC1JpY&)=lk1_^vGAsq@ zWI{R8QI6|OZ!Nq0@!k2l%?ZvWmml8Xh6kh7)Ebo_A|Wi}giwmTJ0uz6jCkEb3Mm~S zua_V+fF{u{_OWPuJ5C6R zZkdNqgudUVgfwdt0yenG^G#%yKOWkBSl0Q$GH9hUuH^x6P3T(RAdrK%uG;gWdOAyi z7D5fQQ|$zD0x4iM%S42QQv;ZoP)I>l{dx?PQ1MMPijhq>)M%Y2VlzyUsel%hS-07T z4H#I|xB^LGwbkqh zt#7^QL6j{K30?7*zxhT^aDfwo0^?ljo6P0ETS}!Q&P5;2U*CVXpND@me4_gN(ewQj z?qPJ!!NTNx;-=H}bV@@%hXE2z)3wyyl{Ef%?MobhBOEwk;u4?pU^zvRQ#*lPC|P1W zigRxW<0wCzaYEZjuoa#|-a~`p)Kc)IAYFjra3uvY8at0=^abfv13QP21&_AOW9=RK zj=F2@{rRk!(sY1jKSYGCxH%ONxXC6m5jjYGFDT`Rf9dxWg zz!)R2s5sw38Pg1fiY8SlOX1`lgrGtmW8ehE(uw~N>20UUR1-m`F5B$ifD@u1@gB#? znw*eiDb9(qwX660$sW=*z(gEiezc?%6_MDDBC-9m^5*hlwTQq8(=CLv#2k^7)yrj) zCfiF9*kTQdw8TL4)y3@hnY*~YyxG78iL%V8z0Vk zaO-lN$EcFADz(-;J%_cYgsql_Y}AkSW1gFJQ=>JdM(gy|Tb%`wZKlczkTiOYvoxNN z?%)6LQVy?o`*+xjD|YZcejrb5u5sy$SLge(G4e25i3X!ocg;x#`qmteL={&hgi2t{ zRm!q*U=6vD@dAcI#1E3+jgZ2M+;2o8FW-uIJ{m9ZTTODr+BF=x&Mv0!fBfUz zeS{JG$_T;cLy}2!D}WzYStgJeJ9m!;PTwXkcy{w{L>Q|KoDfizr8gDs;Dl@13FIm# z(?;Se>?bGS7VuANGD7V`c>(606kenChlWP$Sx{!3lM|Mk&E@k45AJh*WQs}D51u~1 z|6-r)e~@c&#k=Rv9z1^W{Q28`3N3~w&+b3Ryo;RyCF7{3B;n32BMGua#~$4eOTzRN zDnHUd2Z2EYm2yU-UMnX^V1%YZB^B>if(0r;#ds}ZP!Lvf(<>l|w4#meYUW9yT$MBx zN(DDcDQhVt*5(8$$RwO#SmM|6?$P}x&mTOyzWMO<{?n)TpMAVZnw*fPQqpAq{>Af$ zkDfn&_Im&B{_}k_#oW`^`$tcHcmRNV5u$nY{Qi@VR|v4>UbSU+_U`SwAK$*4R6o9Z z|MuP6w;zyXj&1Bc&$ej1eSayX#_T*d8^8bmcst+L#I@~@&tag`DRd}h7$_73!3Tu{ zK9Ix3A6|amIck$M2{9%ylW4K#{lAC%UEj60+cefQoq@a5xbx4db$znd+H0@P#&k_4 zq-26`?Sx}6*t;_nPTL8wYHO@eu${16aMeJ8oM@#v`}Xpw;=ZtUw;P2OF5Rh49=l?9 zSr|x~gqD4siI}mV4hgpE1Dc)6ti*)T|CWK^20V3y+l{**Y~=Z;R6Tp|x_S~v_JWBn zoUWfxk_u!3NBMSSn}P{)+wDIrvn=sIgvn~RtF@Lpf91)#I)Q{jkx*568n_y*uJ+pa zP?^;;On`MTMQZ>iESram%ZH{J4r=JY^{^S@yut0w$E(}hn_h!$w++s{Ya(YTfC)mN z#7_89Wq3cT<(vja?%MGnSYQKQ5HyqZp~wVXxzbg~01l!T&MZalA{YN11PChT9DMw6@ez{PX zaC>uciSOmU!EpkCtIw0>8e`LoyN_QQ0fn6OwVBNPl(Ni&es5IP^Uvw71$s=SiwwCn}IV@2^qF~Tbh*(`VQFduhbV|)^(cW zF7`o@r&(51ii9>3GV}wq<|PqD{?>KWejZ14AN>%) z223c!IO1_}oY&E2&>I$y0gj99EK#9v${G^`F51Wy8VbjyTTjIkAi)6&k%NLMsix^j znwKe}8l_kWM7Z6B94!@ebGHPleAMFSInGRCsa$fgciA z&Ux@TBNddn#zS~y%!I-(%7h*STF~#Rg2tlp1clY0LvAXy>7oo1Iv+rXS_umuZ5jqo zIMX~44B!J|j9WY)L;|27Ou(rL8AB=%Fge(O2mV$KQVBNNOjt}LWyy)xe1-PUW4H@_ z`3~mzP6o}?GC`5x`$-l>5@Vb`2AL-SXl_0}>~~j}VLsbkeH!kr^t-y&%p+yOLvOT+ z@MD~<&=*-oWxu~00}eq@V^8f>U_#-=(T2z2ho4mNAt1dz=b>ZOR$+qaZor*jz=97n zMUFC@Pcnhoi?%;8P;Fll3XD7kX+|i-XtoAAkOO8EHFFXYr-(|0I6OObf-%y`D&BqH z(}yQAv%>mT>exFVXb^#S1;cCSh3e-b2o4C#ww5{`nCdZ|;6cXwS*sv>=Q{IG6$??v zPH6W*FrLum0+_IWf$UzugehcJfC$@7Q5<{+>(R5UfO8!3N`-ZsPt|B0yUMKe9av=v zE&Sqxb&4PMu!-Py@YredfaRR8;N|gt`svfXNWUo^6QITjNt7#^2}D97j{<9DA`tN9 zF7d)dq5{|=<1A|BmUSL@Qks@R7GYfC0|FAgG`4~Nlt2s+7BDqRS7r%fIv-X%!7sFX zHpLlfnihCybrwMa%#5T}w|P1H?zfCe!-UA2k_l7C1R+JFNbtnAuGsGofeASB1|b{q zAFiIg(7U`^{J6TfyNC3Uq1<6Y2qw((W!3L*9TO@tq4dJ=zyyggJPaYDNMR(!&^d%q zMa_c)A#8Ku%mgRxQlf$Fa|a!)rNDVv)pWp`raRh-WVXJT8xH{q51kGg*gFdZH-(Jr z{)hN{*okb+i(0Lxi{R=zIxF8{#!&7+2hFxRBO0MJd09VMS@-vaucTNwUsJ{87t?>} zGJ&oGm@u16{SD-KQ#dA62P7PDATc#jHcD0`;p-Z$yDw;s50zOTK>`q<7ADY6DA%G7 zcg=c(wTSl zkvE-cPN!be;cD=#JQXvc=IHK%S+QaxsI-WX-~dfl(xNI5HdZM1o`}m5-NT!nRqa0a*Iw zV_%uzH$6lkuw~GQd|#wS)F;TZvJRgH{mD9poltoF{u;i6y58_)A_++d%j>pI#F zdjHyOaOGGE6OiiQr=hqG?nVdV@T8oWowXCF3t?b&N3C;zptCrzQ-bZ-SqaQj zbOqOVB*pXsXz-ykTKLpuR&~<}VGvkq9p_T>Ku?6J$RZ>_%V4lUymNZ2G;%X(|L;Yd z2w}qd?K(>v;=lwN_iZMqq44ry>g7_L@wqB~Jg>-vby3wu6bY`|O_4zRJK4@JKlbN0 z?Ha9PTbcDGqn!{<%QczMUr(#J8C+cS;`CtvKjET3BZH!88Pm7Gv3zs_f1wg zCg4x4#Zgc*L6;IHtT;{*&nC^lOh&stbUv=}qyfg7HU=FKMo5Gu)PHh;LVIChiLfrV zrkKhxP?-}^u!^}OF3#@T!nt-yp7 zcETVjT#PXgCggbiM2?G6#2D*QuRkvG(NN5J$#qT;6aC(?6YP_Q7HlDfc1iara7?ff z1?ov)i_*(_pioeq=enz2>aMabmgmMP8Xf!R5+ly5&)n?xPWTsEv{1e=vdB~18CJ7x zIN_BWa9p+B)kJK&5?uh2KyAM?oM*-Fp+rKCo`23xXvY}Q>@Z>fvilHMK*IB;dKTT% z7-Q9S914~?wy!G#kZq0DyH<_X(V;RcGn(P>26aBi5B~x?gP^%bKjkI*BC!Q$+#`aX zE@kf*i7Y}%K3;v21FLo}BOb(pdJTpSRfs1|W2ztqSaff2m}o1HHqOIJYziJ)(oet! zi7gTYk_|67coHD+Gp{ya!fx00A8Nb;sG#*-j>l{;^>O8k?^3*;S9;>N|KS!7e))(uM++UufX= z5&-rz#euHdZFb!O!;IVfr%=HuS-!FVmtK!`ma_6(-R3>SBB8?s7z!Wis(|NEJo8dZ zCLI0NU2rik@)Bl)F%z_S>!C(#yq6lS8??rzGHWEg4gIg*zTTx}9y}t;|^} z<=hA4;B4x$ZhBv3CEYgn*x3u(qa@Wvs1v^`2?qfZ_1-U8VI{%Q(9kkhHmGd?1HY6N zg#tc`T;a_e)dd7rf?uexJAp|{1rr|(5<9A?B9I2b_y zKjZ7yZx77``YXvxxA)@$@!`bUuJx3;NO?GX0E_bBu=Eq=EY0&E3bHA(mWB~jxzyvb zWxB%N!Em0*v<^BGh5y}o3ziPfWZ;VDorx%@f`m4L2Mn0vPgxi&X&JEE+fri+c38L3 zz0i0OY;qAA(drWaSdz@O->!%3#uQj;9Mgb+aHpS@SY`Q3mv|DbCB`%QJ+py%jA@d; z%>)-t;KDIs`=aup5Hp+x_%rPUnN$6>nZUGN(I@I19Ca$5)@Y64|F3gtVRfUGS$9y? zH2@$YDFfO{naD^1TjU9A@~W9`tMz4?m#R~430Mdd6JRW5jruN+I;B5Q)_?*yWh@Cn zfEo%MWQ})C)h}rB7oydR1sI{TWmmXiE1ISs0?(TrYcbjh9^4$Vqyhv_oTkVW_jn=_ zNaxZqVG7+%UyIFcj#H0cVD7Z=!nc;fdcWQ7H>tOHp7>e(vc=!@Img|3~zjt7-DgN)1gP#b|k1XEHn(p5Zw2Z(%wktf=33xYTajo7h*eEMf5 zbfD0wY{i9Opq04jY=aIBy6Z=UP1063Ygsn0G z$054-yrQP?NBY`x$VD{IS=*sEDnuJ%AGBULxQQ%N;_fUXXogi=)6ZF0psk^ zqly=qB)#+zos}0i!yOnBH8(fU5)0(iXPgU7@Bk=~3Qn*?n6RX-FG5pRg&dH-GXoU# znrr8!Clg>NU{~lSk#+e>M1qy?w%ehD1ZO9(K&z@^rV#~Mi&%orBNWhFDu~*plHZ(3 zndmU`zH962t=cpZ$TOaKd$vbeI&&f8?N`Es%U_hAwH6B0XsviH&04lX*z{v)pS&>ZM2%jp#x5E72Spn`Y!T_lzXI_s04=xl^Gv=;i7330lZSeAs~i_aM%{B|jK z7hL3>F-G!$Pp6mw0TW!0oEi$TZLv-?XT6Vg?h`XLjrk6i-KmZE-b0tPv^?iUxE3|o zE_ii!YEA^56h;f}VQD*z*(S;IWK{5-On8eC@u|SV?kQL!z|pQ$` zA<6_^^gsmc+dAKY(Q^(2ZmX~w@TDe7Vqwa#nF&=G<*VpM6kf=!z=K>)^45$m@03c) zaAKL%)Jnw>b)OjrzW5cg>L-T`rE*o&8b}a^u$RE{g-HpL1uaR;RY9TcV4(nY-!)sq z1UxmSZZpxx5n^}%CMx8kN=aV)rShlbCq(gY_BqoGkHST0ORdovf!1q!7W zE-CB^WA27Nso?Ac2Zhu>>M9kVBuSMCD&0BaxLCA`lbp;?Sdt0K!Tqj?R}>s@NSRcj297irE5LB1tRpD zIn7~Yuge5S1z8@LfV=n(w3dsF?6Y=)BEgLLd#%BRm|Oj}wl)TNMl<0@{^ArT4(S`4z)jkMIEkY*$# zgoCJr2)Hl)#xJBFS`<@5wU! zFMDU(+cvVK?F|AX69X}VAUJ>l0|NvI7Wot`2G1O1;=u$SStjWwPm3Zc4ki1@;{X2- zPt{%BEVlUKmzmk!4_lJRZoVkCqrOyC-F-J)1t<}_Ar@>)d4LIVj&vLveoVQ7nBxbB z1QUS?nMJJN%`7M*Y7j-JQ}Dan_cu#jcYZXVpus~9^9$R75l-1T$_hXMoU^sdr(DUIK_( znGiz3Ghrq91bY1kw+tbI_Y)9wZ%UD5I}ZLEr|3@mQgQ{|YRj9Pw%&H;{G(_!--=r6 zZK;xt+b3IXY)x$8nMbafvxm~;IW_zx6yKLIjx!6Wz!;#tT{lknxm#Vz#`U%W6E5B4 zXxfnp66kUH`*K}j1ZW4*J-{IUbnYed)zn8o-t}iV32grY6Q1AQtZTVNI(pSI8RFbi zfyZLv0UqFa!)^itfu3*_tXVExCQ~eyPN*hqgOCO^{sohDk)9ibPh9{~e7hNc7?aKV%5 zk@h$aDul*j9MS^~*Hg>6VXB`n0)+!Jfp7pS5EBqJ95JDRl)D{Sh=H1+v-h z?;r;yU?qAbQ+k8KtF>me6K)>y6PhM04A0iKT73MxkD&&M@k^Dl0D!DMT@e!~im)!j zXfe+L0K*0PW~||683dP?K!S$?LI-(VlHbG_@EQtkpSAwiyRI7OcPmF6e*P%BSjV`O8ZIQf$nN6Tm#Lk~IPS1cnv=|B=C|7ZW3EDf7V$;w*sX3$sCQ{P6x=IdL zJVP=`vbtG)Nk53<615!GxhS`x6;YNYIiaXb(llXWBq?p0D5Fx+q-jcp{J;t<7ZtlG ziv`^f#oX?}0ru%R^^eqJZW}D{0yaQ!$@ZC0V^ZWLk_nPS5G+vPQjoBu>fL@XDAJqUDkdPz&uq&HQ+pf46Q2;>&wkb${m+mu_Xv7wLKQxm;axhR#ejYIT^YCeXSsP@4F{&qIk z=O^#e4@vkO_Avc*vi^Y0G3Os=A7Lr3#U3|F-9M*I&v#5vm0D)P@T_<#Dy*=~ z>$WECPc#wW?6$49^nS(tchY;^ojFy~^d9rDi3WAN|4bsM@6YTuXC|yY6E+*>!Iq1n z&9-JKS9{D%TUGSY7aEYUa{wA#QX&ES<(jgV&ONyMLl&66rMirOUc(rPuGd$9fO!z| zrzt%XG@qccgI^z2UR*{otfSV6d|{QQ?z~;Zd0`VO>#A@TV8DYQh#R{Ymp0bPx>$=& z2{ioJSbH1+#}~4$;*9Ak?PEv?8iOdc=JHGoZ%yVij#*AL;2J|A2jt}U@O^|COh+8! z_YKrh)ldtvjedFDAkSEY`9uAi`@Jp-?f&>&DE!?hVBa6en7Ly`|Ghv4{ZVu#!$h|O z*y0f+_u@5cxQ3vBvN!K14BD=bAJk6>p)kq>?RRJ@x_hbvfKJKC;ku}+chFS5-_@^A zNSXB|DYK?tdzEKRhia`Er7dqog>|>BYgvM~*|Kiy3+T4CpWnQA`{u>F>$Z4#%7ma} z+aFGNb|r}id^&ym>Ia$}blz5_u45K&9waZ; zOK$ZJ3CJm4f(QZy+ChNrZr>tu@Ov4@$QW=~5XA+lpyXhA69<|qxtMP|PtGv9W#W3a zT%vbRKC>1GP_tCcJPSHAa#cp>$uf&AETDoRHvrFREc^n^vXND#)%0uK=L#{FkxRgY ztn6Tj2H-#pPXWpUe1QqqQwC-X8jQ43%{_)hPqc=ze5!J$(zV|Sn*}!?;;8; zYU;22Wd_CTV9d8NAvEYQHZx3E-A~632MCi)7``3R+RCUs6HV3gc0t;qXHX2K&FPo+ zGn5%st<^Ch(L0hu3)>Z)ZtHz}N)ltTWYlX`c4lR^{FGElSw#d|u5aGId-tQrxV9Hx z@*U57C~U)B32VD8sOINVV=_Z*t7-^0dvQt3;I)T|2m%Ch7SI^DjErM*+sF^-k_z}k zf`pCe(LqK5`31I0fFN3K>rLP%6oH@c_~|U#4;h)@e7v;G1oyRNjJs5{iX2CdzGiI9 zjj^kBa}H?}teL+21|qn}s4bJA%V97&K^=Ib%d0CB=x6lv93bHAmdUrSqawHT@~tP! z!w_iPJUYESVZihF-`{`D%S8-QDe8SaBsfhNam#{t$L0}1#Ok=ddKMF4%7l?g0&xGw znSj-#^EoEe&9s}q=did!S=1G2ek{JTV$#J4HC#jsS8qsr)hjrkx6o%S9TUpU{o7Zc zuW(2C^&>1vfrxa|wnP#pLHhcn-oEloC{9^-B&AZ*UelIrL1a1on71vDp>EhgU?yeN~zsI@kZ3B|orVP$9GBkk$t zpIYd)<_&@4NAfAIMYEBZkb()@lM`yjpv^7k7{8}Q6({dszBnPVfzmO7CqY*+rcPoZ zIE-h+EXXmli@M>CNMV9U!VR?oE>nrp2^ZDAhtT~k_B?Uabw|3l);~RmQ!yq-R@G)E zpYUiR=!#HL9-3LkV1kQM85z3n0StR>Ou~c=P@t`DO#kD$@{R&Ou=|}h!?6i-NPy`A zsi{s2$eoUg&7|o>OJhA0Y_tRvKn1!6zt2>|nLljiW>n`-H%&uwr3mvGfn$K*%SRD~ zDmXHxH8HC87sn22!J03}Qw^(pdd-*UkG(MA_DM{rA)mkrwvw6f>IczOy}Nls?H@V7 zk3@qhwbtVB)vK@`cHck4vJDuJ)x|LruE3Yw3HcfK+=p=uMqHOgV=0gM{?&^&Us@AG zWrdkQFYa)#m5r-v8M98~cGMuHN*RTwS`q@72E4&jLI-}QxZ1U9dl}2PmMK*}TJY`x zLWXrrwyvv!3}Fqk??=8>RpIA8*T2S$?%7>ukc%asMyz%DlM&ie{FAu)r=i7{j-6jpy76p$Bm z!Md#U(x*2L!N63Z2vC5+3I(!W_!FQ(>(8}xzyzn;>8eQdU>OLKWUTqzh@X&kOjtRans1MdS{&K#pfLdx5DFmQ z;703m9oaaw?t@70V@;ZSAo%u^!g}IiOOFKg7hr+~LIP|ibkN>G&x9gi0zDBFAz)ao z+^|L&do9Bs1qyUdICl(zT*_+U`LypTo zckgjii+sJll;bd|K9&h9^%IUvyC1fbYs3AqBq;1OcF?tDTV_Jlj3L4IIY6%!ImUNi zcO_}6#>9jStH1n20-~K->mg#b)htl=Evs~^kq^QQx$wu?W?BCBrpeHH+WhGg2%=BsNZ2V%7n+M>zO#@gJS}c z4Bk<&bl5kH6_B8&3%f3jvH%;yNKkV*1v5m#?4HlDoZwn?hw zS^M?Hd(~bgCcI#^R_dN?BB-^VBu%j&s<38Ww-tSnCsasD=#k%r)n9i%oG=rpl;rK% z%U5ry)Y3ou_U+5ppIgg50dpa45HHZvU~a>#<%_DWmJ7HDc!{5|q~cA*vmzI9yxzBJ zZf4|Gq+TUpz&Iu5BU{tJe-d4Wdd^Q6osX{;)-w|mCiz21K>F?%V{MTc@eISZ~kH+k6*&_;DZao%ZT| z^p6ivYt5QEb84+kzLPty`|Y;yx~)n3l?vzFq0pZ30XJr;-sL=osqtE_+pp&|$mt(n z+MEzjSxa1CaYddb`6$5_o&m9lYm#JK0|-bd@D^`!v*cB!G@qcV*V`?fk1Z3^Cl)I% zR%F(xv|55`m`}hbNty8I@RnBs1QJ3fz)!GdDft9A(*}TWwUq*)Zzf(c6AXBOvqQt( zR?>`k0uIh!c&wKo{(&nq6Lh+?WIr-vR%CD#qzN+=sK((#CJE6vHVg$x(W!-l4?7N^ zE>EUfm}q=R{asgPd8F@#fIxdE3%yl-UcC+|!(QJ_vG8V4T7lJ8Ii> zV!5E~Az9SAhXh?8O1%2%`%=wf*F2xd9Oczp-e#Vz(wY=qf=re zdT@1F9k0JeQ$E1L|4B>3a?)++0g>)vL zFuz`;XM##G4wxtyBo6jq)IR{5~|$hC1x~5QC>GuLeQuq z{;FKWQC3G0dEhL@Q`E4OG+HAxl_W~8lOvs$B1zMT=Rj=|vSup^uDP27#nmXKv!dl% zZ27V*O-lGf&#`*3sH({DAO&vGP->;t&RWj#WlXeKUsMxJ;LHV0PPPzdS~3|#n$ayG ziJ!fv(OS80HDyBIW)}TGqD>ERgl(4wr#KAYv5cC?B1&*AOiQJq@m(fGYcauj69OU= z5hpyD3aG$In6haXSee#i0{;k|qmawznxPOa;3&YDy6}BeMIo1M$OJX*GVGu*DPW?1 zWu=0C-#-ouc)$J)T9go{M};v3hg8Ba^LQML!y%pY6Y6Hnz=TmKK(~3nUoP)%`%L)u zpS5QKL!pE+g6JqV-b?UqLQbb~O0*MdPy=(1q0efee(bH}o78H4!}+<=T9~}<`PP`q zn2I=XiEhh^Wu_L@wxMssCcdOf6IUaSf=s<;p{AQ^%b`MCbNnDo&>yNHfmVJ>KEavB z4n600@KNilmp-RI8fX!x$4(X{xatN5Cxr2YqgrcfbDpE8A}pJFM6=ne3B+*bK)g88LhO zy%ZZXd@{=tYfF>&Bzcf_4Ens&Ar#^Tv|0yHVAB$e-rvUzWE|)xPxBHop+v7KWx@oP zWQ1|JVDijEL81re_UASdC}=;a|I3Am)L{0IYm!U?jA?v=2J%=gVYGXAaBJWxXkSCn zx1@{LyVII+ymr&qpKsZbnO}eV*$FER?Exm-v!`&=W5VqpzgWJylLlxi6ch<$+_aHl ztjbE2lUS}8SMo3cw|2@JRw+y79EJa8MAVBl4`tkPHgSo;%uQvee96g5>W6O*?Kl&3 z**s!`1ry+A^gDY5R1r6o~pX?sjyKYVFyj|n4Y%+fQ# z^sX>!bWW@BQr66qW;(CNU`CjLZ9n*lB-h$w0vrWl0#XWgK~P9c3MfEEdQ59ce5}E- zPO+_X#7y>hD1@DgeoADa=OV}a(2?fd>*zMMmYp|eH!tz_(R8*K^f?9*sCtbH>aANn{ z|05TH41U5e`gQ$=QhtOLm_zKmX+vnZp`s?>@Lo1r_LVLfMARHo(N0{e7cuf;S`{ zRa_)6O(>X-0dfuM0SiD0DJ1+zlUV~+Cau5TdSYCA6()$qmmC8*CM20%I4GpR96}3c z*}Q4&t|mE!Nei00mbEgYw>oLVh(ZD#t)&ttgg-NW{p(TIqV%*Fg4>X^+fmasUxrK& zIY!MV+*SuZPF4@CQY0`FEV~Jo9fj_KQe<4PVJ1{LYqWCp_UrlCnP{|v2eb==hqB$C z)6%G4+Rcu^@Hd+at7soH1lDm}*#*g)BCSv@zAbRU^O;bKpYWUd2^q`-L9Ne(qHVKc z6bLcL_wAhsEDI7YCBvvpkRd>Jzy#APa&M$D{j{PHa|Ny8HZBnq=Io*IHu`1hy(lai!!e^n;vIL3epxrf54NI!nMPrLav>L57#_-UVsv@*@c=4BX; z9A~s3tDTS|W`qgO3s4qdD}_{v;J&2z@H7q*Ts>u3Pu3_)T;T1Ppuq#u^4?Kc!cjR^yq|(v&p(n2-Fh&gpfhhmtrhPYLCy`waVWteoKMQ;;63PltAd84@C0!Cl6{8H|PN7EY zs;w zl?(6}R)U3Nd5Sc-b{5SrK`n!R!e(r_f#KS4$_CY5>-z~+wZ~lP{l0=Z9C#r3-b4_g zv{-H?FGMPdhaDO&W?Qe(Dyz=pSlzHjE12LlTJ!!=vT6JN)z`MB9|}ykDmOgd8TL2% z30koii}ljdKt{`bZo~u}onXSQdGOs~z4;CUCIlVsevV+Cl-%mSC&uO`!ao(oRon7( zOWK<9`wzr~WGOu)4hf+by~?G{XW6iYYLZcti)4N$FAfw04GE60xvfSJxF*rO<)cFf zBR!GoCul-}_uf%3U_y-OLHr1Y!tq)sPw?h5$hJLw?)2lbzk$+gs0hhLr3{zRUb{Qp zIH|0QBm^a>hcZ@mdvGRno?`+GOb{nQyCmyTCLDaJ9o1etE(caB2cwx=i1hds1kUw)9$&Fz;Dtit+N_fa-t>gI1E z8(&yTEL0eov#@;X*-U8guEE&m?3f9C%Z&`yOY645?C{pamLzH4efY9(zI^z{#d`hE z4`)@?-V!lrvNj>%k}S?gG`cKQTJ2=yV88_R^%JE67mR--M=^x)c`k#kSmG}aZ2}1z zI$+Eah7Q0iIvjoHY2Q(BnI|a-_Ugy>TwHNyjFJ7O% zc=OZUnnfrM?`h_^o?T$7mSvE1Oz0X|Y_a0o$QL)-xO6-Oo`T7|lb}5Cu7X_Sy@8}d zL4uQdcFIv$^#=SDUFfjNJriJH!m9HVk_=1`D7fH4p)kdT`Fd)ypdQ+^enpy1>M2Y^ z0ozb0{z!~TatJOa;p-BvL+m-FvyVtMZzkl}$ zDy%0|`cW=vufCLjlb_IgTetdbT~(+)l?j?!RO{;}$VX*@2?lCO88PAgyH`KZZ+_mA zEX3&r{*pJBZ(p2q;ZDxJdw0I4afRs84@5*#&je#7jTo45&D%4fgG3T082Kq1kU=8{ z|B||AgW4<-bZNMi7kgl81xZHlCXgAkSVUp;a3TGbtH;~(!-pa$KQ4j@hFVWe)mja> z;BzCm%}9mT4Ltov#gm#!t71+Opu!!yi|-40mrzX;IdECOjyywG;J%f!xbsdeO&+a<@M>^ zFL$4)Gv@6Di8{1*uTSaZ9pW3C_j3t7Rn*v^`!0Y2RWzN*#0YSqRB%L4j{#+_pq3gX zjG|%y0G?@2 zgdv^&dszno{0$6GfHIlCzwbHt>_Ru|@oM`;kZ|5A64WOAgtFd@Kmo7Y!MbqDgk4{I zjhV3C-tLNS46pV&&|@R11n(x`nL(qK>;*Jh+k8Z$^%@$jwbyMvIE_~7rBsd9G9xCG zS+d#Y_nc$=o|er{R9L@03ks9%K1jCAzd=3eCx8psNLA{v(b!?Nnq-2FV=E)z6Z4gG zj8{>%JCVQ8+ndXOwr8}i#o-rf6nvJ}c>)sfVH;}j7)YxahC-hT+Ui3l7;zQ6rGA&V zm)_dD!OZ&xYC1a@1tYVA+`Pzi$OLs1VsR7_bri%k`V&;hOga-@%nw@3%>4Ob{4kbT zfIr|-1EvIV(r@`Bs?_+MJ&dLSdE(`MnT<=Sm;sVQu4?z@{CM z8wngLbUBo{txI?b*^x$THm%VrenMKRM(h37X|(3Mx*;aqZZ|x3iJ1_U#kSs3AVDj^ ze0c~8&G?~tHvx_MIZRL(;C`vYrecSOnXuxLsnOe*Txa`}S7+yBKic@>?)}S`@9!3I zTwPGV!!MYzY^Ruzfe9E9Y|)Uzz(ARxrWd7ww}5RT%)k|>t9$S;7{Zp8r8>0FMA?uD zxZm`c5M#Qf!QTkW*r})R^t=LkB5K>fB zbIjlxdk%_)ACZ5TW81h#+1< zy~$yS9io=autsZs`1*2IqjlHQXkFDi-_9B^VOKK~etPxcNQE^S1wrGsD5(FgCr7OB zB+OHxsb`oVAn3&*BTRsKKVh=p!HPzJFyV@suxlfV9X_>e*%gadp1z>8AG3|>CzxRg&oHf~~9g}*;!&t~deFEnuIISpm;jy1!LO(&I1zdmR z1r(XaEPt%0kj-G@0lfrc`t?!SmG1o})TD9F>o>3Dden{Mg71F-7r3_a^B}yV@P*uj z==z~?rHT3puo@QNVU7u8ogU+X%*Jb0GCDXPxvszjAp(FTC68uoeFExhzjde`^L^yCQpU>altr_ z=P^ML5O83|Pnc(diM55B_6^BPe*F}+AIUqaHb0*22(Kr9{O9YJANRj}`{T{4Pjt4( zlPG{fFrVJ|xzv}j30t``Vc12<+u&mL3<6}}vd4uESs*J^q(+AcBb~rD6+4)PPbef# z6p9^%i=}yHmNCbMkvZd0%rCit)#tR1I|e-w%Ae4*?y(>Pi&XQOWfZ^#U_oUkq?&No z!xwTB+C*nX%$WBRHk}_hLG_e>CjE^?T#14 zRwU#GOh^MJEF-o=GYT}4j8Y0n3Ow7gc{PP%TrWY3#$?ASgt<)P2pQBT5iIBts{79d zJOqs*q?Zzo!k7ER_fc{tV$MP)z=Q_Mgdm@QoiO3ww`|IUhDFhH@e@RUlp}}IS;a)i zS-H7pAn@O+@`|fb!y2u{?us;3^WB-#Xub9tt#z3?jn=Jdw3Zv44Aq{U-Q(XJs<4(* z`2v?0@FaLloe0-d4>JKgoM3|5BmIPVCiL2OLVu!nA3pAroR`slfBv=oMB~09V#Nod zs^PQm`D`9QA()|Ti3uJ2OZs)_$9wDl4?6LCNz=A=I16z4>v!;~`IU>{U|=0q*ieO4P@$%Mf136ud$Jga020%#kV8l_d80Db$q>Q6TR%`vS!xXfhBKecU z#%EDY1Q5fdqw9K*feJbxvm4fIG>q`wlxdmu`y*z-uvR77x{kmA zEK^?c)t%M7$xf`oTB-_b$OQk@uhSb*eOz67RZ*x1OsK;HRrM#BP|KfeH^~HK1dVeO zIxJkVj4glc78l95D8hoHb^HdO~8bt^AjRrLdsWt<4GQLXY#!G;rSD%!yt&WImh## zrsD@?cEE*xz_J1sG>E{DnM7p1_cn7}D1uSY%7tk^fd!Ao$N(m&1+l}<=NP+_HeUCb zuw^Ka<*=gkjd5G%VwSy!4Xo502KTS|@>=8=i5_5-j)nvguyOUK-$18xg$+M79(ckY zC=evTTR6%d;PqEAVRI)o!vvEdy3fS8%3IWP%iV7=huYx>kf#F}W7!wS$`6^EET~|k zh4rezGKDfGb4~(U)mZc&)?tsQf|)^rCKJ@+I5R3coC_1O9us2CCy+TN#NRIYPr^_z z&yF5I-zP&MG!u0Z#OR$XwfVK(`cZ^&-a!8l!%+t>xb%ZdFl2!e{5TW-%Q1mB^Gs+% zChPzOWx_7DR+!L@Z>Y$ioYLr2;L`GB2YQrc<+RkwTCK`?3kI+lBbTkMYSd|``Hl^W z3WX^_BoxWQ`t}L+2IgjjAg!PP;KOn*0F&K6n+bLE>#sla^S3?|daa1c9to;V=nK{u zW4Zkf#s9KI)>nm>z)(oT1zxgvVR1v}Di9U`h9wmR3G;scpK=w9PVbpQf@T;sbifhh zGhFNtDHIIeHH#^D5HBZ7p9F>AS}^x{Oi)8#(KHoSz2ir{LqjMi1NfI`E;w^TC=KF> zR1yq6!;lwb0tzR(GC+g*_2-{mw*b%(T6xR_RFw(hC+z)s3noBQ#WzA7z1w;1rS~1I z$51&oY6?uiezPVA#MeOr-f;ov;eo+2{EQ zTheExCz~>1)n@~e397UVG+<{9R2ShWh^^G%9*|j+mqlH~W6`*1j8hvAxKR#M+>smp6A85|qbs=K?A4Y32Zcq$V{A_vC=GM7)- z<9XOA6Y8cjv@F@8mfOOX^k4^~>S6&$1u4odT2wX6QjC~v>h(TehU005qb*sAdYdo`a|9ZZ%WVL>cXOjN7y1Kd=)26|$WRoG!^1wbdE5kQv zvhxBa0tsQ4fq3PgVj0RzhCbAP?A4?~zFaf=jS@L8k zJVWaNwm<=JvnK8yS_;sJxnT8n$A#SLt$5Fg3nb~PENw6%)gtmqn_gbzlMbu|ykkrI z5fi>SB(%x|0s`y=)zLBG&_Hqe5#I2^7DCDD{0U$Ic2Mw`Xxl=*fF(yOrjY7939=JI zp~Hd%Io{DBU(HEexTK6DDn&xgo`iZEfsx* z-bUz@uEf}13ll~xVVkEU2L-era{qK@Lh6}7D^wH)Hx!pQ227}&*LnHY?i>;zBb^S) z1Oqd{+X)=R2qutH&jc|P)_9V#DwsRvkNFL zc@8xG1?qjJxtIq!Z7E0Vo=#Qg&%aU#|EF)xt)QKdx`X|$J~r--`^Gy8olaWjt(H|< zRyB<_5L@==TnF{M7%sCnx zlxf1ZyM8{SP$@nD4cGFs(j>sPxn*6eg+QYpV^$jPf?j+8b~|8U#}L(IY7r z@RJ=GHdh;=26E3iIB0hS7dap3m;eW6f+B%dE+`YuH^Kxg;${q(fI1R-{2?%bErq)D z=1HFjP$l!j)8{CCj@Gha`3MTU^`Rj3xsp5HwI#~F?E($UQ2+cW5@0ACn_caZ&}EWR zC_rEW{w9|&0nhj1Jxn0e%f!m36wX{TJIpv;!*vih9bzV|i3#4F05O_86EI~Zu4O1pQz1SzpP%vR z<^mHH8vO9P^mKguA;i zKdUkU0uulPY|eXO!a-6s!x`u0zXsu66g#0td_qOnZ?Z#1I1>z&Kdnm0%LA>$ma zTcCiBZ1#oGRNw)AnDj`FGC`1#4u&J7WF<(p_GOAUDoy@>XDX!0DlvsXLFZKh z6Jj$#qnMwD1sys$v=xlC!~6n@gnr9`;naP!LfEKGFxkrRlyGP}=y*TpWbBw=EZGuM zHCu4VD++_FB(h_k2$w+tOz?>&6~d#YLfrolW?5DZW}~sWz)?cO^)j7<7FVFH*( zF9aqq0l|cWGNFS)#Ds=4`~kGMwgVU9I7V7b#{>iSBC3Lkv|b7Xzyy{|6|Lfe{(ol@ zqzMUDtg{)*WF=o1+a43($9pOjdVJ_KIk_E(P;`vwkYH@6QrKt%CZq&8{7wo5gOkbx zXDD!Uz6VM>tw$LO<;C`bNsZy;R?I~p2(>=EuX~<)8mDcA{?Fg<;a&h2;KEg3rL5jHqO4%aghUYoju(v5f}&Bb~xWa%!F@@1iV_w$9QZo7wmd>47&n$sb@l2!%(Q~ z%^*i>ULE5ct*_s;_HZRkVBdn=0Y`(l8TnRA>pcyb$o;+BjWA&>Ux9WjbhMZz0whSl zJKG7xj!sCHNuB5M=%_%~;00fTpfX~DRQ~w!{K7GTLlcj8h`quSNZ`O@^feI^))-BQ zPK24!c1Q(p5|%N!pOm#2RFFJZC_GWt*bWO+5=}<~Bg}GOC9D+*LCe9&vkfVOC9zI` zx;QZaF~Lu6oqH%0b0ioZ*T^NQPZ#MSzXB6Jbw&}M4muC8pimh1%M9riba5&eB)@CW zy-C-zy+Bv_V<1FIOxUW~R_#UZLl>tu@-0p@7Izsv8v9k2t=4<0xSif!7*XSNLcS8XTlCl@NI`na^Q6HHI3v3>%1G*FkzRT3HsHtv|3`%X;hmfkhWgXiIb`ZNYoBE;HDoVHc5 zi323)w6^4(X zMP7I)95_d-WL*6`mZSA2pQCku<(M#%qxCZp;^XUo{P@cUGALf~y&uPzpizpny&MQw z6$#kmy1%;y6x2%KZ5=S-z)Zl)nF{ynPmony2V>>FKY?Vm?GWcV!S@U)MPSilP2!M4 z7xm{2MSa6MDaJ_r^~pc_8D#T56Q+P*M~Gn3zLmgji8N$_mqbefSfFP(BFb!eiw=7k8;b^@?KTi6NT&~RY8;MgGnn2-o%K|-9PHK!b{+Z$#A2%$MzkI2#L zm@t;3^#f%MdHd5Z|9biBkDoYq-~r{dAZRJ}Fy_yrk?bJ- zt_H&@Pyu2hB;a1(nXm#9lnK%mOPhohCmJ+?KOyrW9xS7uYFc_e7z_1mP=E^l^hlEH z637gx;G<2#5GL8qMl%hCzqbI`xfxbYi3t7#3qWTq@Sy3>`Jy!FG2sL|0Yt#-_PV4x za4TP;%jM2{_J9P(ghR&!&C%L6lzT0|AIZ^rj~uO==^U*aYAd|@$NOJ?eDyE7i+BR6 zY~&96bRuY8nJFghietf027>i`=JfTOZlyqRR`9-V}fOCV4Ml)fCLi|t0Z5C zg>Q_bhCd|N9+mVXxB#I*UcaN)>F_)ngyY5r0!s%0C=o0VVHhBR>%L}`VSfkbO)%Va zU}u5}m|$j@;Qf}-6E7z)!MhZe={FkWdkw9ZJMZ1E)0hbY zg}Qn1W7e|vCt||(jwmpiqje)WT2F^&n4|S@{`}Jq|2+TpW0>F7GvVgxM9^VbeJ0f2 zPN<_P-8>U&zST29O$A{Bpn!Eov7LbSLC1uA2nDGfWy*%apPa7UmZE*SDyBg&GJ0W* z3C|}_gsADD{U@j!3I!B-%x}R+D5sFHg~!84PeN%lU`gcizNE|q_8?>)3h7#8VmMy% z&1817!=$-iSlT?@+DK+_ti3wl>$@fWMUg`eCBkCAGD+yvI z*t2sd5EE4Qw`Io!vZHI)bl;&&D_5*akWfT9TK6wrw)Sq2qxF8nIa=?0j@CnlJ9azd zXq7*9Ro%bhPKtlh9dz$+pXpv_C%oH!Zk_-HiH{T=PzDl|3H7z8);}BemnBrA!batjCx@M>%Y9u|WHQV*3B&No)ZKk)?Vv!-Sz+^rK#8pO*-b%zA5u24eX{ z{qdoVkRc41P7IJhcUd*0rR7i%7?4c`j)TsfpiB@XbWET_D>Ki&VIv~ydYB-?6C;_( z)}gKZzOD)3u`tmydGY6l#y)SBWg)rvC&0&en*6$CJ3(;}GeOLRfC+c#clai42gEQ6 zWp-;Sg9!d7)8adcE5~@Onr#kWtw(P?Q+mASsbUf;vY_MS_*1XdIvoMccqe2$(>; zl%@m8Sq^JxxmuY=eCul0!{sp0`u4C2>4QLP%F#22`b`Kcy7UjOK$~QO3QU00Th(w` z^EQ*lnLs)*6K=KfaDT&bjNhb>;Rv}<1?Q-*!32nqP@cmat)=8>J-ZyOXP=`rZw@X; z>-Ok#w60FxF^Y2^Z)o$Ot>`4~Z5nq+e&Ra)6d*u{Arm;F*&~jzr}I6+Oz1H|#eWy> zn0O=;u9K8y0w%yzz%ZtCWy|%#;2VZuP^1gO0wDyg2Opy_PlUK=FyJ9}HC(3b0d4>V zp`xgzLDezb!NTt;z@VRr3dxFzka1dL|_8>A0*^gRHx2L_zET* z0y}}nKeenI4p64jv2Cp!(j2XKIakYVKS!(99zJ(h(dNE7_20vn9pk}97X3Q zwpRTOe5_|DG^!{^>(EXB5yVb7-^NU6!^*Z^UPDrg>O&<=DCxE55ltIfDvW`mYH*d& z`6S!qhpU8+U2Qg-^1y2HD@s9%UG|YCE`+#o=l0mpewHQnut{j|XgOsmJcaTEXbJ}h zdIcT?eoZrBCoV?XhU}vvA5ZJUm_GqnJ=V3u!YmhDTH7Nr;q%UL&%6nkkmAh;G9f$- z%Baj?%Cyp9CzH6{(L~;^V{zl1wg7XdP3+ z0Z!2eC9h)TgK$W4Yy0$}Ip}r2}(>NSPh14ag!@T4Yta_*?9Ksnn={Dg{ zjG#bh>llaM2{j#M-pGwrj|sdXCN#Jhl84qY4RE>$bF^CTjB~tt0Vh<0nK{QMF=Xc%B1eGjPK22p!km$(H>J2kES{z3-O@_NscJBty?3)>E7~umY=?EmGMZ`*V z1OzT-(NQ{+tP_LZ%kfqBRnKWCShqz0gqHn6;u1_eBY9@+$dANju`;aLV?Utn$^4)H z{@;!2@ok5r*a`IFRhSTGB*AwmtZaH98^r+=4zZXBx<4Ux{)y(GAeRqd5ny2s_WfiyBC#SZXdUc!(@+R zvb7l=%C*FYL%Jji#!r2l3W~Ju1E^t7BtEO$wO;aq4Q(IH8tMD_#SdzolavZ5msnN) z=-6EJ_^s$&_MG@Q%6dS$n+@%zmdjhs-|YUD`|gL$hB0~8KzSUfttD0d_4anm*?tKV z;94Ulz)<)Gr-J$uLe*$?GZKUeqO$5$Xg>%`6Fp5ZpEj> z8d?b*7I2BS9U@ENSPKv6nW1)m*`Zi{4Rfz~Adv8F=hFGf|A+M=B_$vwC@h^1 zkzi=?ymEsb+-WLn7$fAM@Srs{;$?jH5a(EX$%n=B=BC!CND_6Q11!4C**G*Y6ce$w z`>J}Xod5|Fu#(hH7^-nh@Es2WCiuM3+4eBge9$g|ymVInjQVG1nV`~Z0JPYLQD1v~ z^-TyAkTOD<&=;71IW!N_7O98A@hC^Dt0kNgslX6lOq#eV_^X5`Ju_q!u!_|pO*1;T zNa=jQgaTdgm(^{vdK432@AHZ}m|@2gLHiz^!@MiI-UXGTh1W>lC6j^(rGz+zNm^IP zUnWSvcpj|5+;pLtU>p*x@jSrdpzwgbA2Gpo70z}x#2|Pa6!!jCN2haPiS|%T1w1Sa zdd|JfBl*Cqb~II|V_X?7V?r(shtq+4jJIbd0xe(y`_e|$A)y@F3F=HhkK`UDl&7jX zW|yqypb(?rVU0ejISLMF<3J<9~b0Fcml0Pw5l!>kW=$b=Mf=5&$5 zRHou;m89zhb-Sm;1c-J_V$pqH8l9~jCnOL1yiAs3_=LR>3dYErZ-HkxW@q?`YqS2Yt-by^I}1P6r- zzYEqGL%}Cuvojcg<}q=Rz(a3Y$=b6Ota70n5OxU`>|~j9v^GO19I(zZ;dmk-9F7g_ z1TcX%jtR|CRR>mrw-cZWxzj?yIyhbZ_P#AW5%4zZU%+*H|Dm5l&43*m)+}3hz5xLu zpMq$)Ydqe*jWWT*fGmZIH?b=J{$>BAk{qqqt942cScy`Y3X8Nz(#0Q(KbJHLGF>c7 z5ex;qn?mCQ_tL`1FOlKAZ4naYQDV5wAUMfrKl4PeIyo5%0Gg5f&)CHo*M=^6VK(?) z|59bdt!U{&A%a#)7zzB-JQ5JCU; zo`i<-Qdz@P5K~GfiS+W08mMbTJi+0qdw#66AV8s@j=4D|9FGiz1K9~Tx6Fj|`P&E+ zic|_73EFlj=%-X!mTfpU^A&Hm6rW&`JK+j=xXP+cmSo(d$S79fD(7d|4c(GArCjHF z37*ML7+lE77aLYp3nU>i6oy*U#opo>w-Yd4g4IpBU1BY!SJg6Jz&h$zt^;m}Mnp{b>Zf$M+48&y{d za;gv$cwmzYhR5KBVzDTag)s{n@3j15xm+M>am)4qoh>b2K=g=O53^7(v<{eXasQ=w z6EbHdz>nbPHMTAWyO4eL9ipU?`sFN#foH*5!cN$1*utU)5wIt&gwzD;DuqWvA~uBY zb%3f7mV$CYnebFSOa#3OIF)H|Z0A>?Z8*9~ZiFS%gZJZ92ovHihwi(z znzIu~2NK065ENh~__l*iVp61B0m6)8Eazi9_>8MJ?Yp1f+-L2FpMPzS?MGq)$(}z^ zvejQ+zkb_h2cLU2&)WAt(}0I}?a5zP`R2~^N*#<4+4(^ehf08e7Ygg0vxV@dyB}i$ zKIjfSC>aZ-Hx2krdnXT&V}cX^`|r65$0oh3IM^b*(!?a)pTLxIjC6+AD{E>buylD%>qV^9(gxH2u5DY_Pxw35NF z{DWE)TJ?EcV`AA>uu+0w3R#%$Y|ESsnDB^!4$jOLA{ZI9WHV>xQkjleQW2O4upB%d z&`Hl5$QrLqwj!N9&Z0Ic5lF@%A){qD7o9DApF?UC3EsmPdlR4qLqV0tZiR5P!2dTp zVdhG)!%WHRzUORIJ`ILQ8LZfQl$}7Y2n=4+OgPcCVkg|*&~hfJk5Tf5Al>)Km5jHp6}nie)YqP*RNl_ZO?SLLp2L7z_fJ z>L|~Wb?!s-L$20r3tul*g`r(|75`io1OlhUlHieZvGYE6?L~u4j*nCJs zB47z+1DO~KPq7#tMuLs6G~n&t3U$XPe*qPs=wC6rln9ZJQFRd$4o=@O5>CKE#{@Bv z{Wu}odMZ?`rZfx%+Z;7VD`i#WEQ?!v-+q!dLB3mum0cwlI`1e+g-;& z3aDVxKzTkjY9^?m0A#x614cpu3`ijr#ZOE#0dB@a#UGjXfy|mk1>$I$*4?| zh?xKum?X0xf@1d*HRRT9|=BZ#tdPU06vtR>=U*cTQd7#EsUBra1)Nr^F zJK^M*z`jL|`-O-Cu5a;F8HR!hbF_Y@<^fgw?2z5DJ@7f(mT)3|t~Q$%)VSkf`x6^`;RMM4$7pP_R)? zX-qiECDe96;{kR;zyuvShy_$AIcoBBs$eG=&Cz;qo9gBFl+JzQ`yy?=m7T-M26-rt6V*=_G^jt;)YAx?+FwsF(L5G^fMRyj>uZNjHaO9_>aGes*nx?4f z`0mK2Lf7x``+sViQcIyEU*f7*3lk)%QX$p>o&IB~CIOLxBbGSS(SQi56i|@XB^H?e zq0(i?1T)h^f%nd_NIUYLv4McxNFaf=zSRICxIGPFw7kc=>TfLOLJ~q@zyv$Vgr{kS26ob|kQlYIKm{97LHj0^3PD?uh5c;K zpYS*BgxjyTH(z5rL7=efpdjKnB~C(eFe%#!hojHYnsWlRpI`j&w%s@s&@tJm_F0g7 z6?xX^Q?*s;udAd2g9y(E1(_Z;96G5E1#c;+4p{t1%y}jN3PK!cm!>;8cH99EJMD}7 zbd6|b{ty1gI;HjkoN`EIUM$&Auw+6k3Qgj}1Ys8xTAnI7MNyAxDP(<_wQD+{g*vZ} z2x4j&E;9|-Oi&~wDQtx3>m6ar)^j4o#-xG<&kElwN%C(N4FnGd+&SO*8vY;C@ixq`{LI88JRFoh3O~Q_)_VD@PNfQEU72t~j8Xv#PBK3aiw0Sr@cR#|Q?Nf|t@#Mdm}1 zS#(@VkH2o#1ckLBsUIN}50naR z83{&GrA6L=f=6qt-O+tCy20B7LCr7dRcM^k5C~SKMEFzM4LSt`4D7HUQWud@VX@9c z0)=d1B$Q`@4wRnznjfmf`ba4iWW*!{r|$4i;i=%*5jz&NXGMg4x3l|uO!!ihaLU zEN!cUT-TPJ>rW*(cuG^E0TBcVnq2|!^?{a%H60QlO~ev1p&eyHU2r-T53(XHU^~c| z%P@{{M40Wp|{jmwHOLZt|?hxNh)T73Cv6p zzmIl02oLbi59A2DBr6mN5?hpD07OIts;e`;gMbWve7IMFhK5s6NwL!>0*r(LRYNZ^ z>vS;s=~{7s*Pn~ojt5`Cx)BdTgBd0~)i#)SKp2EDbyUbR!IEQ}Egbxsx3xO> z*ye-3Zr^CeBy6Cr{5UR1q=T9X{n?b*3C07Et8^g#YDklmGmDHCJD5QKw8{acc2Nay0kl`&eShw5} zSxU>nFbWt9_idrYv(1X7Ts{fRvfPfQBGFPOJBG!-DrYoM~D0!z+^9_GB4i_Y5#)f1Rd;R0sD z4U3_0J2D1}aexF3v4E!w#Mp;5j)#O(>JLxu8B|kFRWE<;Jw#s11O~QFwH(ba! zSw>g0GQO?}t0haVeTfTt1QT$Fue7)`PT5eaKOuB3u#`v<6WSgVR6G4q0tMPM(l=SL zC}!!uhzYh>bH{_tGQ(XEWj8sHEioM&Y<0FHROt#cOeZEtVH86F-=Pr^*03Ntv)pp6 zSLJ;fpR5gABoze{OqnB7v$P@nJdZ{?AiQruKTc7YKxiclRRs2ezhan+SS*mGCxTca zIOPQ-OoTATR9GajouF6-HdZjlT+$rZ2N5oRT@P^rS$XYh1SC@gqWq;8bHBv zO6G#QAhcV+a4^*VYP;H?VE)mRzJ zq?}D&3loqD^5F&p?GXq9CXA$VtzB^nD~-I=pU@8b9TWpQey&2HsrT5s<`k_yM=Km| z9JjitKr4_HC|HYfPf_Jztx+Z-@J-`Xn9S*l)kr`7mYWS* z_yICo%7`eJHV$S3{0Pc}H7V|TD6G+u)e3}ANXLSFGN}q7iGPMo1}D^@010X)=qTxs z2iQa&1;RpXCqzt8Bq$S zlu)k08zhZ=h(MrB7+2#Kw{fw}{A#D{1Yie0R4s*y4^WQQ-~avl?~-RJHK=3@7zoJ0 zO79v4Z!l>|fmz_f@E)02)}kyMl>rqx@vgYq2nzH{J54eHAukLB9aU^(LVR>J@S90e z&9BJrN3+K2B2yb`1pv{+G4!D=Z_EC%BTLL_)7Tn|hHbu&hdhoPb1 zdLq?M7{(=!f5lsTa3D;TcD7j&Q^Bg{eT_!sO$7@oOi3R-(t=P?j@BR*qFb?jfC>BR z3+Ds+^4(W56vR@vQ6J-&G~qD}1yhvO(Qfuln2gca{7oLTAYwZ~%li!_TG`K~QbS{) zOsb1G1beUp4*nLvT6+wMfk776F{7jC>;C+q>DtH<7N2W#|&rJ9?tzZIr{MjH_ zaox0Lm6Vv!EBDEUmJ2uLnO5!Ip&@MA@gau+zT`)Vf! zJnu^B0VWUa$7I00LfWG?6Y4~uP>?ymoqR9|HuQ@hHWuPBQK<5)Js@{ebB$R>#qc(!VN{*2-7saTi&7-s(CyPSCNYx8m zjAAQ{5dnq*h%nysAT@r332OASqBesA0_XRWs^g z^fet!&~ls}9xH^hX98)831EyC4ju}}{{|HP7j3@OHcO0jyRQTaWDvBU-oO9UX4bN= zpH3`NvJ-A^laiI+eR%W!K1VWFJE&eLTU5TH5OYBd22_W?XycUXY%H-dqk|$D4Bkut z4P;|S5yAKfgoFfVV;tYSFP}qPcl`HpCqE)G>WWk-Aea&Q(o4D2k8R$L zwH>U;7eEAO2~5e^5V(ZybPMT%P=&qC=r&`WP_{D?Lt&-~LErQhOTiF0`N4!cebIs) zwG)^LU+eP?V}Wuq&a@qN*G@akd6X8UD;mGR1D_zR_T~L2j!)E&AzN=H$a{J7`6W$8 z|HHdBpY2W9P`o1zw`LHa4tn`(By|esa-b*SgtO|SMg|JFeWZiq5FF(D)lT?^#zU)2 z=&7oE@zX^?P@onW{b6g#p8(qs`N~(AakyCByro<+uYPNjoRpZ-ura=P_kIT`B<2OxW5S^EaFUjTIQ^&tqA3hv;q&>OvlBL) zjy02bg#4=d_#>E*WZ8z!ZhZz@=6q~v*NODZ(wm1&wX$?}Bfu8r&jU*I*KX-b(eVJ8Sc zstUJ5crTeoY}81O)}6JbLDdpd!IWA1?&YhugoXDl{nq*Ct1O`t%TKRA8J;!i4r0l>diHQJ9VL*hq=OOPS0a7NELnf$rzZY?_e~4fO zT(ESD+WB+R=ZIG!fC80g70@g1WHgR(3JW)YfNuO7m{185&R^>jm_SfCP1p&idJQW< zR5u*pTolC{O1%1k>S%NNDCboV*;AaoeQHr z_wCF$fjjV3AN`+k59hHpJ!T+54gZG8cDR5ERV(8`aN{zf*xuD#X*g&E4@803to@I1 zcc5P9YS4LBK&dWd1k~&~4pp{(2a9kz_`I++PI?{J?YNVJ>N$h1L+52v9Owg1PBhj{ zI$t^Ba1(0#7q;yP)uvsml7x?N1)o%6+i$StjhV} z$OFN`oS}pVTOgv@Y>FKz8d`j9QTV03kW3Kpgfvh=^Ms#QQXU%Sgv81xM2P>;0y5`G z?<`S>Z$JgyuU?ih_zo5Lsttpq$#=l>qV8afi-sZShF}7OF`?KN$8GUJ@d+Ej1d^ue zK+3~!hK79(S+A!6fH7f01aufE55_lMRp4=kqdxF7(-_xY-S%ZzA#fXgP9 zEifdA6H%Bko>5bbBT}{u2^7R^FhNv@Fc3(<23w!g9pop_1}aEIG8<0QR4f~iN>=n* zf6p6Wa2XRWCK!<+Cgm8)22HTa!w<_63{jYCuY)cR@dYL{vZQ68m1kJ}AeG%)+ky(X z$Sny~Wlyr;Sh-3S5*x9wN$lYg6ZSUKp@9h^)|ybWCtxWt0Yu@w!~}XYF~JHjfy|CS z4JO2G3{pkjhP;a-rx`|W)U4o|948}|Wu7S2=_sX6>KU&r)#9atTtq;O9gTez;2xXlh-DD~Iff1nClQ?&PG4{y@X4jsfGbCjR!VJv2^JJC zDgfFSVKCw2GbY?N7nRM6^eL1hj14(oVptD9e4jDF1w0gm>&|jjI6d6xeg1Q<#vNcn zoKGEBWt$JuG5P?8Lb*MK*WB?!saWh%K@?Xs2s3XRB>budTUcg8O13m30@DPI3H(i@ zWi91^Ov3cqe9;9F);dN)5aO%MCo>~jHO;89Y3w$Ix~HN&SgF=dXxbU&F%QeIrKab> zhBh}b!N?_!kp7K!2(%wsmu3xlU2y~iZPmSbyFGpYk6^mUY#jNb> zGmLZ}r>87*@9*0E39OfU3ktRdqum%6!evajAsY))soVS(b4Z{VszQkHgx9%Xf}6z4 zlj}~iUS)RALDmoJq2rqcS zx~$-OigpKP`^zk!J+nmnGy_k_cN^beLKm1YA$4Gv?~c>QN8L9QOF@tbVBo5bI<`Bf zO|igUq^}ncPv|{7J*#Cgf6{%Ad~GWG=x2M_Cm-%w&tM9N22;?E#hN>kbp~=g$j*(J zAi+xp6Hue|mrr#Fg1G$s_Ia$ozP|V2jC6a&GG4guF3K1a$hbFWOz`WN;Ldi@i7+8S zY?jk;o&^*n1zUTwI8s1Rp~@uV&&EY8yfF{9Cg2&eKT`AuUc@?DMTAfg0Vu(UfLAnV zPyCxP*J>s<{2IP$ZFGR&dP<#e%oTn6OV{)*S-T9Vo0}k)?ze z@KcJrQ7~bb;Ki;|d}UxlkufGLbH#F?v!1=X)Tx1TI*!Vx(y+z%dKsiV9`+VI!}Onj$xFQ@B};ii+s+S z)S?*iG|r9+B3SuHz!1v6KS{m^2CF~#HBh)t6|C>Fi3ub%vLKH}e8S1ZHmAv7&>oV~ z#+bvNaI`T?pI{;1W4;Gb9DWO4-_zAx(^8x&ZFTh76ws` zmX}C4p08aHfuw0l5FcbEVkwBUHNh+r*er|C1cHRMwHpZR5DaL}P_MBlu;iO$!7KHP zZfi4|iiAyj0t3U3DlXWEV8UMp7iv zzlRUv-eF(leObYEj~Ck0BFRcotyTd{h{#Xaf&h#n)rNx5oVdE3}U=!dY?aB6B_RV%l2lT{`aJh$0=l*f5HWg#ZzTr~TFIi0ZsAGbq z6W1ZZVgj^sE*P-Z5v!YMGjZW#Y`gz`{pohQPy|M&*=7ZQNnnD9!wF4E65^l zNg)a1cF^^f)+?P6Wz~i?9}%~M&at+zaB1krm)fzkt!yfWJ~OdH)1c4{TKb<)sORNeti5Iv9z;sN zL&7mj#RpLZ7!(d)*cbr;NboUXj2y?fUtwM~HR|+)a_PIEu*fm);Q@CjPHCKhN@Rlp;tb-xr{bL! zI_u@M!#&35tr~LQgTIIs%#nGP9SH-eBNOlqxWK zjl_XYLJF~Oc)~@sL9+yQJwUJ~n!qI;phBc^NSGk_AhTY zmo77{H`mGiASlim&r&4c*w6Su!b zszHrW&D7}If9iE)ND|=*cw8h4k^^{cP7|I8QLAp_p2zz@SP+Xv)f?Z0jB(suIA8hx*NkLMHZH|TEq>1!}Wh7R5 z>iRJ@U>kiiQw)byi||?JG|jMHZ@I_$nDXhK^6`hz13S$RB`D-1XlUUBRZM6q4~N~Y zVtIm|mVmf|q!@08p2ts!;|_<|(NgXYA+;U5#!8MMYH4FcGzW3~iCm%yfviGW%s`QN zr6giZ*rbF7)n&sGOTNG6QnEg?9m6%61hb_g}h;F`Af6aOws=`Na9r(8=Z3tDk$^Ss-R6wDM36cO2UB?6w;Rz<5U|&k3 zM(e3uYP8Vg(e192XPZw)l zEYn>it>vN$322+l$zq+#a{d7xfiHY&*?o{T619abKFb)#e)5*{;iXvr!ICpOSvTQ+p6C;81hVO{+q1_TSm&?^PW8-FS2-ll!yFk?sHpIAQRpkGxOm?bB<>UONnu?G%z%%!i66=3ljDWjR{E^D7musMI%2N zcwim^3dYhHRm9MQrXArr9Q?V^6kzfHxjX+Kw~b?gJ1}798L(jlX@C@N-5Sjgb_yJV zpw4rj{o?v$ov4#oibqm3Dan-f|NkG}eA(sfa#tizCkJ^QNhEjqO?Sur+|2Cktp3Ty z*$?=QNHwrCFmYLwg`fZvek$cb@9GL2ChR*oRR9W1VYYpn+X>797iL*MSSyNGPAQ9cbFgp zj0ZfdW)4 z?N}FDc}NH*YBnpjB4XAI$jwOZ#mwr0pgrVgq>x4g11$jw62FYZAT=Suof%fjxM?d{ zs|g11!LSnHvdx4TPq73Tz$Isv=~kFlD?(5zL>Q|YxD9x$B@vnKhj?_b zvtfc55NtoR5{CtHv{s%_n?A+_lV>T{xz`GePiTM$GFDm?1)255mVyii_9tAk@qku) zzSeKf(R#92OmSi}0byzGSPO3QdQlTtvNV$k`9Hf7nA5dJ<)IO!xU$(6P*KTCOqD_{-!Ai6sjbJ6FLzXVF#zZc0 zxB`|E_ASgJEYUb=6V6C>1E~pV%9~DKB%uMyy4vY#~AGS5#C+IsNj_t6H?9v z5IE&RpphaIS|k|SAt&q~j`tTR6*0kwLcJa(AttzNCoGu31g6J-K|7ZLcQQUHR!X3R zsUYER;<-DnCoH&^f!V-ZFh&Av1ngpzo=9c_?*d69>TckENvP%6W4~7dtS5BaT;!!)<8hI$?I6P>K?|F8=gf;qM2#6OgR{c**b@nMKk&~t zHmyQD=f^^zK&k@N#XSZ@QImBX)D@TsAu8ZJQ0qKH!I&jXsDeRD!UPa&2B;1bZlW7) zJ2Wr8z1ZPY>!c`XD1hld!yK)rQ$W@-gN!jyj+RAD5GmkYz&j~y{~-ep11^XQ@*68q zVDGx>Frm?U!l=LmhY9m#Kr|NS(y6B<{8#pr%1-!gR~uitJ&{NONvhPwD+vrV*i=#f|t62=H%=;{j6 zEeTj?=gnL4c(fCMfD{;%v_}$&X&mcRNajp4Nx0UrA3|h+QA|gmKuj>DP8BsbPXx4;~V##CQ`vP}+kJ*Edy3kp@+nVf&OEt#_2J!*cDFLFDL=nox*0 z0fdQQz)&P8JSQa^4S7KuB{IQR6snQ4-}8vm`kgQ#hy`q%Xm*>JKR2dTmZ67*4n4|* z*dYRc^b{{P66Ty<2`|S>G9BQ)}Bfz1zgVaoa!h}0S<%S<3Vsp`I`zLj(Z=&w9XS0~8ad!O@Db$Hk8A5DdNHYGDG4HMEH0mV^o- zmq3N^iy;ArL^2Jhjg)|6WG$_`74EYEN#+!-#K%0wX4q(qv7o%507s$ON6El~!rK+{5nm zYDIN1MkABtz2fQuM1t;ka5*pus0j#((pD2}F1XMW^`V4vmk9@Ef6f$&|$YKuBH~M*@`K44lz{89;-%$$KdQjPaZx9>vUqk|YEQ%sL%SB05okm5C8yjfoN_=uMx4d|Ny^q$*4_5H6?i(W*l4 z%d94$d=FvaJDxUp>lhbq0|k<->uH1!Pc}>9hbt@#92Y4*!7vjmDjL~T)(VStKLfn%0@MWL3-AciG3dVUFrEsYrl5@~ zgu=qgj}g!lD*6mELA6ZSnkI5@2SflI02V;5(V=$j`rc|+Dw#I&3FT;=M#bjgqi?+& zt#vm?D>6aX#p>ll$eQ}#(Z@PH!2{`T`3^>b1I-;pToWjWmJmcj2}k=(I3N>9ZZ+OP z>+B|pNl=&z1O_9ahO+X+!U8U>L@J9;V1jWc=+2D=<@9op5W_Z606rYOBuv2j({Lzz zK$MyG$Ou{s$rSd+<|jySX*V=+joPuy;$85P2f-O3CNP9PxWa9|0|VLS-BB_+dU+TG z!U7Hs@s2-cJ`ARK*Rb@WmyE2)+~ZmSLx&0aRSq$MJ&w$T4YiVchy;MbHBPs*t>jW(H^qji6A-_6nbHRou3FLk_lf3aNW=aED^6a0RMgmi=&gnfM1eq{_MnfPtI5>&cL)*5GA1x3#(fSTX(qO4+0NjhoEo^MwB;M3M#^m+rT!%WmVin0tGM7591P8rp$x_nIJh@U#k6^V2;*hws}l- zOpGTlR!YVBc?73=?~a1Vi=6r)+aH1S0BHE+T{K;iR3tv2kcL$EG2xcB144mcxS+yO zEa>6IM!cpH=)+3!1wui2e3CnJtiCgRsH>9>B_d!e&oHN>aV$tDB)by~6XjZ3<3NF`fxBu?nHUEEY$#1qTbkgq*$@E~J)oj^h^> zT1|jrtl5G0A;031Gks|!U>yf3V9{bigtw&lgcZ*a1uIO=xQNAjs3(#&Ib1og8J8Xu zuq;#%r-J-=W`ox@6TEx|Xu%k7f&oDX2$%eU*EAXoR}97#GO481A4NiNe+WX0$!wf9 zl6vNY9l=6F)Px7V?SKn!jC0w~gdGlsy!YzWP7^_xP*)6vIwVzTpzy&kVFS6>qNXoL z>sO#ieB4X!)p)I5eDmO|M;GVO6q0Emf|1pP()sGZ049P_7|e=xvE?94a9s*$Qw%E# zxyuA2;_MqTB6#8yp+Z1bp>emVZ<$MH1iQ;Hb364?=roS;9H)k45C{@Da)A~1G?{BA z#KeRY3OE!pirQ?9LqTGAc zv`A2UW&#L60d?*G7bLxk&f8qCkqL)|X2Mg-qzd^&136l^j~=||x*V53L%sT#-%-SJ z@P|;d|sJtlJzi{J;5;HCuV{fm#kxkXx&SNm2E_fnhHpSo+8;x z8bqcQ1hF2#7S)MeY0bt2)e$O|6!%O_fa(}&5Y19p&gR&TqZvgr+kS;CbMgu$kv%*z z_S>RRfu4OmR6|%GXj?>a*kdPtg=j)P1SZ0zqa-jDf@XuT09T%f%Hj-Xv?H=5zDf@D z#E)WPx6FX1Aiyy!5rfe3G|nP-20+`NVC7GM*BB-=4N#%kHTQN(WS_T8C{R(T`xB4M zFh}e9eZAQ{dZL&KZy-nOOmeiAsgt9X>oTa@e*f$$SI^GFgks?bHM2v3Ee8?iCNn_u zKwc=7Fu{XDQ4DAbobSP9g2a5SO_Q7p-<-t0&$;TK2Bda#WH>m*5PM_ zfh-pD0o4TTk4TvbDP>NMQyei(OjvMCBaV&E3=%xY`rAm4(-H^haiv}B2-3DhxDdfT z!|nuES$O+DodMIlfYoH0gzn#$+N6Wq0TOdrEFMn+b7e2maZIqyht}jY_u;9dT=PNV z7)`H(u_IXen6L*b!0TRfuz-_k{axOC%@;`tfFSQzSj@L*#@ z!Rk*k(B%0buxxe-6DqAC7$^+t3911opxl$EU;GK&zi>0egjDBfO^ffJHAyjn&FE;3 z)-C2}U31;Xtor$T%9?rxALDv4O-grw1HK3eN#!$v2_RS~EU9{!pm0WQw5NI%1w#j6 zLZ28HWEcxU8^R(%t8?Sp-5;g{r)~$~!4~53i%nF5Rufj>dcYmb1bCt&b}`a-nk?FJ z2a)yvms6@h+r&6#yc)=yGoy>KaC9z}a1hff);VDf6Z&Ec;O^{QU><<#fT^E4C%DBx zS4TLAYP=j_gWO-j6yiQ6NXga|T5&v$MiIof*bdSY2^(DMV?st*%83ceX97x9(>xCS zk}DKY(UnTzaHXP~1Zx&bt*R{ZaJl+f(h?Re z9Z1SvCKL%kgP_poU(9dA1WBsSWLWqkrNrUr>k0Ds^*zAW)pX|rC1;E+ByfxZG+$UG z)a|&8nF$MG0?jpzlh6wj5D7+0pvh#ld;lf@9Vj!YGlr9H7;MnAgV(m!&fivNgygEg z`LYd?u5^(2j6y82Za8YuacVjeG-vwDu1%K)nZQt(rttt0bg|cbKq#Q9V5lHWz&8uA zfUA0$Ktwn&6MTOHdlaw*^hs5xSF@|QN^T~z8xUH~F!=T35>qP7uV9qZGMnqsPajvf zyqeLFsMXcS`PIkxu0+W4#L^P-oLGR1H6o!1I!GA@_{N3E4V3zrP$vmdqppr8XT>E+ zR&&3@ND;qda593P?#UH>c-tj`{-CrfXvXFPM;J05wDrJ8td{VM;RTTi>57=JOo<6A zV1nog7&3%*I&A?7CX*?Op6vn5<>}k0zM+iO3Q~b3i47ZYqs5n`fz>T+DBoE|Zx03o z=LCHQ@!e#=rx11gjC?;23=R`w9LDBB0X>bD3L&;b8>1)sLLU<_${CSx5GIH}frY({ zBcw=fP-w}5ji=3-wu-a-3rUtcNHDn^3*3iDsOm5jSQrd~gtBNgt}@7kf|!7p=+Dv0 zB4eQ7BR!C~E#=yjCIS@T%~}FaEKSG6F(y&N zI_)#S`#~{64$PsjVkU6jAiU&a0z&_kMWR`GDr6{m9--k-5Di+RfrFKN zBuF%4;R(%LA@NILCR7B4+J-{Nn0d|ekDq?}S8_EUBeOuApD}D6fzxrmAerm+vR7Q~ zce&&{Elq?Z;(^Em*D)bNn01kdLh!KdIqhSkbvCDOG9h&VKtOm2*d;OXM>trHXSeClH-t;knsGtH&kbw;@6OI6)5)=p- zq9iX(t3fEF=fpt^1wcYl6m2j9ym$yDti#?AuIfVJrhqSv8)ezggbK0YKw;y{_T8`l z4X<~cA=Q<)4wBHwonBYu~8s_x|Kbwu5OlMOiRm&wRuG+vxS#k}&QFO7W zV0Xz?DQ?$AQR6bYDK;#%Uh^%%L%pGp559ISx24V7N?j3+VPpI$;mU#g*I!UN->MSZ zc39*W!P41VS`qvWhaQDxv+rSoYE*MjK|Mh~iOKb#KOvi~g$W`NX$lqF4=CwbRE5mi zXtjjGnKif2g`(F%T9q~s$kwT<4ikJTXa}Xf1m9>M8cy;bE(cjQjXENYKI%XIw{aI~ z&iPl7tvzm!7hwWI0p1}WBe$jzwgVo%MkZV%28>+e`#M?_mE=>+&^*GXJYU|`shE+# zb^=H!On4G03b=-4fk-Hi&-8>Zwinvz?x=>TK%Wp3HgLfrFDn}g`-3lgBcJz&-;pR0 zyH0lK8vgLA-JmAkuxg3hT+`r!Pi;hY4TjB=X^IEP&Hb;S#tU6;^clkk4&4552f@4Z zYT%zlqkp5gAW{6{Dq0DGd%l?tw3Q++)OZm+q3ciBXSeb(W;k=J%6aNY^I|n20!kFn zZX{<~MiI1_(fLjaLYC2zIvflV5C;SXDCM+b%c1Yvw>eoW5Jz_)Dm2P}WJsX~D74B3 z+|z3Hd;NB0&o#8%n;JQqUfz3D>vk#*Y7hp44N~glQG^-+hu+uilz|!+cKudqBb0}bO0paqNm^CdVmSloN~0zk_eH& zAKb*W&H#m|q;@6^8$2ZW`xh@qLLuvxNemdWl#sxEkF1UuX9AP~3Bv}rB$5hv9z)Mw zt-Bm5fkGbC6J#YdXO`~c0b93wq6uOkTcj^x zIF$)3>U1s;RnH3r!T1S!Lc>t#@h99|55+899Z*kLC(-p}ax;M*YSa^;vl`yl*HKb| zOLUe16ta)=3?1HGyGcc1fL+ z3dgS|`_4=SpTc4%oD8LAtIiGuslPQ7kP1C8mY9qNQAkQQ9mY#QQ}Ca1 z@w2Q23RSQX2T^c-)uGeOLR=WweK2bu31T{Uw!^_?!fxYUG%)+_xuxkdioUD~Atc~+ zfDBUT72)RTP*4FVaJ@Q-3G7g~*97&1sw$>814yVTN2^-H$Ox08bviw8?p1Ix?l~W8 zv)H_Wfe_z)^UX80RzHKDpwxbuiXzbCNR(|9-lV~?pd>I4a6wjsNLdH;gr0e%d4(HH zg#=%X@2(feL8M1jI=iwU9FPs6LJQg7+uJ%; zgtFdnj@DIKT|a#E4drOv?S6Q8A9JsMh}L_~$66)o;;W}mzxo;nOh0*i|Ldnu9zRzf zm+>mI0u4l{;nOO&bOU`96Jdf%0uliig^5VOG9iJP;C3{WrcA(d3hkEY3S~I9*_>9& z)#Sklr~>ryun;OsQ8vdL6Y7`@pd)Bbh`nHKI{pN6qA?x(3+wKOgSf+CaW*Q5!;#4# zfVd-MMjhB`kwMSGZT?T+O;He73lvVW9yosjyirg1q)(QXyDW^(WZX(Ii zin&)0|Gb#aBp+*i@%8wn&PSq7c|_kUj?#*6Pd1u^n(4mzo;A zy#x^q$hzW1mrEh2Kb$zgH+!hSNf#7L&$buEE;PpGm zKR9`$Q8i%U!^Oa0@u_eU7e-mkCn2I4K?V7}9SKUHwLtGnSr76x=;J~`teo31f#lF& zKE_skksPg4K*Assu9yjwqxIAD(=X8er~jnftG_5KAL|Aki*J)zRLr;JW1Ph_3}R)* z>=74GY=?ps#GLY!1TzpILg92Za9@KU0i|;F1bbR(u4O0)2B?h7Atp$NgTB1^^?Qw- z7gUrLEzOpFkiVPH;q6h+$0(C}*cl-5xfC&Hvx{NTPxc)F5)0sE3aip397kw2@^bA?&u!Ogu~7B13MFjnIJh@ z8_k5qyz(R;1VcBL-a|ySVCf_duOkk8aP;igiyKX@$3nOe) z^yvws%ms07hyPog9J1>pK}kWF;Gltv+k@}qAl(6$%CkWF@HO1&_T{N?k`AU^8-G~j zS9ixK1>UyyG-3eW-k;DY^$DneSh&25noplTP0<=3W`g8sU2P#pYj*erdK*7c?$xip zzJRjH(OO)69wC7vGYY0CFxSR8H?h=s1Z-(u*EFL8u}BVtzy6NWehfyTyRU8YAiJ z3GR?6sTC15L4H5Keg&-AtB!tUS80Aqot4l$kKvgF& zL7yUVjJ4fAM<{6D^=Of_&et{)U>)T(K>-K=EjwvxkRcIh6DqOIvo#^Q0&nprDVQ`x zcuGriGmx_~p|+_|?p0VUr-Co8KjHi`?Dodp1KxcNDNz%|#n@p&zdHG2R+%6dWqFT?_hCn#@2F@*Q|Q;H`pz0I#E( zLPAhDO;0HMnc$3-?ltxTW0ypQ_`C+uy2R)8=g+4$`~Bz7pA}x?-|*Ux1w~!!?+9B` z9nN)zf32+&Cb($^-cn z+%=qct{87nQ6I9XBfO}~Se*k&vfP$++b5n%#DL3{~X zaMR1tIt+yp8?aFn)b2W3ROSL{4zI-_d4}0pni*X01Q@`s28KgH zRKN(PA~H$cLQsDqshNttT+TKc)>op(_!qK~054^{N z!e~7K-}fXlg)qUfoGY6Np&YHfJ`fbxZU zuW+EmB-+8?>iaN~Vo^{ZM4BhK0g@&O5P_qYnFs`hP?k8hPH=xQ)?OH9f=C||4r=-G zZ-4&N4=>jpucIq!{Rfo40}1xE`tc8c_@i2*^6;aE198L9L9a#j>hJov&vY(s{t>VD zqgtb~?n21(?;SSk(Id6NGH-cAV4-HbS5NLg`0BZ;q~$wI5aYpRLZu#m4cxIF&L4j8 zo>`|@@+_fD%;d&#pgV7V_%L>OTU}ZL6uI5W>lTb6qy~*3@ z^<+AkUQcNZ)O!gs$`?s8fe7huXYViG7n1_@1Zc+7i?`q2e~C;;iWFQ66YOIsqDd6Z z=DN9SO@XzKu%aMDJCi@59LEGz0ui47>Cb<J~~z5}rQ5AR<8MKj^^Uh4^a z^>2a$F`)je_Uq4fxHO!_%l)C|W3#i1v zm7n~Vf`@Da8^h|r)blpOIP zx><40S&3OKGnvegx53zJy8(Co|Nn5Gs_sskLlUwxMaJprt``HYhw3U*xnT+FHb^B| zez3Gwd8*1Z*X3G}OvgDDqF-yHFOJs$9WNzJQi#G z6whNa(E{d%CFe*zR-oUr$C!oC_O9MOxL2iKz&lSEHMH~V|WApzMK|z6ntTOt^ahD9CK_6_DUkWA4vhyLHsz+VY7km^I#e z*7tG0*={#m4MY%bzst;Bd`&zO|D?i>6nV0IrE_2585>Mc@}te9=PyK{l3Ccx6nqx5 zg#(I=EhU8#oyS+U%yk=(YUHBt@QHg;Mo?woq`az+F=6#AFm;%ad;MO8uxS0fTJphb z_nemYMn2=!nP7sfX{*ao;Wc|VjoItVLV0UeH)PFf0k_k?7$|sbH~|W$bBufQAa~DF z_6EVrj(fD`%Jpb{ifP*Kb$dgV#U)Qg3tH<%f3)Qut=n(JaY8jQncaOQ>a^JE_YC?t zP5}K!FkuZu$fVQjeE;JsneML=sT^(}zP;2=_7y-#)71W!A4_(c4=F$1hY2{97cpV_ z0a3z7s?v`BIENFTpYKTa{}^4+y}<;R3AmOx`xDsco^b-RXJ_Yscu)Wzq%^TC-CMqA z+5Pr$3w;;c4IZCh4Z`>2FA_!52ruxER6#S0>TMof*^&QJV@?aaS$&3pav<+uL5al# zMT3aXV=$mu|7y^i&3n;`MiEW=y`eyX^M9kc(IRcE^#>Mxj2qdxLuZ;t{Rk59bk^rn z5#YM&Uk!WHML$xr{+Pikj)sCcMxV1$0uw}5fWTpb;;H_WifY~=;wyCkVYiB>@kGO@ zphKO1!?dA~o0W@R*&)h&Lv?#^^-&ZQW@ zcTyTeOU(B|lal>;Pbb#{Xv)MXupSb*enBe?Am64%rgOSYGV|D@)PShtf{mxs5F{vs}z}1C5 zad04X_KXQTI+siU!-cTPe=sR@Pvy=Mv6SB7c<=bmoov3{uJ;BL{@yO_G62QhK$Fuj z2p9{R<8#gil0YH7fg8xzG<`8>;^3&ktD7O+V1xy$OpdscA!B1=5 zu-56q+LxfIDZ}|h1byLwz6McOWB~?w;V^-pd5DZi1r!qM2T>>qho|S&D40>ZHir9p zw*s`aYxQneUl3oT!qaaJCgitJ+SAZ7AahvG);>}( z#YV=2m+quGkxEQKzU^A-=)dCx{T3$VFJgkGe|uFL9pR`FV-}ZajT#)p|sf{nuYbT zrdG?3ytgQdTL5}mOk&9(^BX`}W-%d?PiDmu;~*S&YuyQ4w-*FGnucx4m3CUL>fyMg zc9XDBpGbr-@Av9;y!(U7obq<(^>Cy_e>VvF(BE=bww!!0?$tWj$IB&k3mjV9dVRAn z98*O$Y!MkuNJ(L}HHU_tf0ijlLD0RsYBb5McJXgCXJ790DEH-=F3eFjS<-+n?=etL zF(z=1F{WO)poBqW8U~|&tu+ZF(Y%~D8${2TaM$lOLY!$tG!Gjyh<7RHg!&Zf=q_@r z`sa5Kq&?XJ@cTi*V*(x=CzP6h_(AW&1n+-O6-wnsP z`s#YLu4z6Nx>X5LyN#(Wd>hh?37gDfLY7fGnEW-LTFv8_C_ZeHz>sjTb;wST-7{=( zbW+c#V2|kwn4l||kiLlt5-A|dIKCo%fC*<8J06`&5O9bIJBtYrNTi6s{*N`glejo% zy14h4KqNvu6@~~ z>t>TEr%pJdxQYq7gajsfP&vs%U;>85!(Hw?6VFYU8>?f=d37&5pcr!_p7U{nm*CHO zoB>xK=2TYV1W{BU&{f!in~R4-zvwg1Lu7j*+YPTMvvEQyny;$W44LcNh&}?Btj=n~ zaoA{63;s^dFkiq_uw*B^r90r?gdPMF9`+A{2`Out1syqwM&Eq?V?|MUw0DSUAlAAe zEadGVtWPNyZePIpbxZ)g(kABg-eJ#i!d^(r;h$lG@6lQ`TeL3c@Z85`FQ~N#bKj%2 zN97d`%k<`E^Ic2bVHMv2%9-j{6yI&ytP5qKdj0yBbbp0_ zcmGtPgg$3mlF1#F|7?v^HSzghY4DfU=Ji#1{1=iL;^EX9ur2?Yao^wFpv%9!pqjIKTdN6 zYlTj%?A*#G784Rc7hh^YV_Crju4~=EQx&g*g2tEIdV~+_a1su&ZYZY_bi}&MU>ca0 z&=AJ&pgE!+TVMjL#Rd}w)DN?~(U((OVkN626z;%NaKV9k#u{JvGz}qjE309ml4AeQ zFyR$YFoPHRLHvU_!IH*Lu_ma@($knw?a`VQ3K8ieF(%v;G_gHeL))XZlt2D-lc^$) zCCB&$1u?al3OyAmUoYQ~8?bQ#eT9b_Ot=Fk)aynIZe`L&vj+vZF{C)Ia3A(3%8%mD z0ts2>`Gx#;;vk=(5Az9jT?FOf){jdMV?+uE;+Nv5*E@*=N*rIp1Wk$f`c5$6oC(v{ z7fja%Nl>j64$G%4CajjO;VAdrw%Iyn0XW0P9a7Kslr!6@)~l~>D0$S8aq|Mrm%m+#NnoB1D|S1^Id`ZbSa zPatw(pbSp~*24$qv7u3MRNX;geD`-AGdovsE*f*hr>p7OaLn08MNiYF+gQmlTyb;NjyYiSIl-a#QE|?;e zus)8xCZ?apgwwYv2q>JcMcO#w7!%&6S9-Lr1rzeYaK$}ZBio}j8x9HAO`mRA zAkhm>=v++z3b7FTBkyhFkvW|&;%xgf(i91Cd85qp~)A{q>8mAfDxzAfr2!#t2ea&M`)m5!vq<>)D7Yk zCNL?+1R08ms$fED4#)ILkJcxeV@NObXjL;DzOY=jCflR+wi2{ctNG)bV1m`R%5HcR zw3dor%s{kg*4z&8?yjo^hRs9j#%}?Fb#@me*yIBbj@ZEW7A7R=iCSby3e3^6jUZbmMau#@)f-qB@M){kVcMP1j-UGKfMovls*P6N=ORfmIxwV)Ou z3UH$ad3V8bqZT%#7KsXStJb>PPBnc64)o;x~daDV~C0(gSa)jRrsgPAf2V7g}l7vZ`yyaP|7NBmn;Pho3SN2rU z#%QKl9T#k0YLNrLB^6;GCpRUaRFG6K0q26cY@@!^Gyh{9c{m+5swulJ&d99ZQ#{YH)qdbniwNRN{ekFkp^FP;EFUU+~fe78*$OppA*)u zaLtUcvxl|SExZpBrYykv-18Vkb1#dZs|e?OMWv|j4R8Wp;TX1au~wMXIqys-I6%>? zYp64#HKhaC7(oQOd$3SI^PDx$D0An`qA*YU!I;M2$?%`D;-!41Q4g#o24W2o15!}H zstcOQs?tA;3CB(0NQS5T&Q>u&*}8+;%1G$lm~h;qHGQo|Ynf$=Xzg5hZE(7(B_%?$ z1I`sXeNHnlBr&56Zd!7hU!u1VL5h!j^yx*AKOBR?AtpFrkR^l(C+iNN|9qSsHx|0lToLVpkWEiUVF@j>|l~KZ?epCrO zb9cqLtiK=b%^<@hFNjj%*W!i#dr*ci<%Po9@OMNjdj`I;Q&uHPXVz0`%-EGl(5I}z zqsi&>rz}EFcYf>vPE7k%&CDV)>lx~fs3_cGshn!z6wL+s6Wb7NxBSb!Ou9mb(^>P+ zxbVN;-)u>1H#nO2fN)%wFraW^@1V@y0Y^*fG$!P4^k}_#sYh#uf$)ILCFEroWe7Oa?(efQV9XD3Ry|=#zCUI8G_yc9xxZg;85ABtYa|)hj~V{6 z`CBl-LK09qrzCbG!R#VQb4oF)Qki7Apu6Ta`lU##r>E~vdx@5etop7fOGw9xQUQ8s zsWwq4;y4F`EWkz{YxiZOf?Xtip3`|=l37cQM81w=sI8J-my}}(MeHaF8~)CP82X63 zzN&1=1&%8D}xC>Zd{u`moCcLc=&Ctbi-F;2n!*Ip>VB80V)bd)RLu zHv7%@0>^#O^IoXj9e6eJSO3SbkYFq!N3ZyjDqC>;GAn8P*G9p?0RptefwO{n1G zgfx4t@{pgV3fBSuiOCJ-Uss5`ob&%D@9K73w~g?5hg<~6Re=JF0w-AH0BwO@o%X6o zkfNItTh`i^mKwkRJ7hGR$#6XU@oFuvPx6tXhN4LQk)MV?qDZh1E;tkmIP-xx9l8w& zCqyXwMoz#b?p4&I6jen>cBdJNC2RMO~?EtD*YUKL>?eJl0=q#sj@?o zRWM8Na|eas|5VCgi8nhbpkSg7d_oJbr(uFP+IrPF96_?eU_zS!{o@T4zY21R2DtPv zDwJlA)wPE5RMc%fffyl>PdGt72Fl>D4%)N7_S;I-L%Dp*g4k2d;cx*b6ux(0PUw{r zaKD9k`OnL%yPN=F17J2Yga@p7K8{)gQb)EhYMBnB{#3Ojrtl+sI@3)`tF&{XzK}a@X|P(w7~TFN^a;8y_3c!Q zWV2TGtTO^RY!TFY^n677tZcx{U^`W5<(n%gr4l4FyLpD&oM5tU!bRe_v!6%8)gNB~ zA%-y0=$0kgqhlglgYaKS5=xvfsh)sq+ZWk$nKHz1SPO4TyEY!-4BawL@RXo< zpIi;6gc4XMl*&RBq(Nb@?TKJ>N%N;O{_W0IZRys0?8o$b>8| zXC$z)zk=$V-JX8Do-htWJX_Q6z-&P1(X7k~%V?w2jG%L*kkB`(4!ttzYsyRYElgnWnGz{P?Y#!3g$4r>fnnuxB!tf6_&%D=?QdNYa!7IMq|fZy#XqZ zA5qy45-9(I>X%Fkc}|#c;eLnNpiMbJ8DV`&*f(gQryoPo2hL4aNc97js#OQ+TU?+g z6gdHIp(NP6F^GCFq@?g1m_r;X7ywL&K->_(X}6geA$Zwq4Fl*#|K9=Hc4t4I2UzKe zsYe)>EoAG5jkKV=WA^X&=FSr!#|aa+F^=etA>e9E>JFlRYro`SH8dz;-#}E@j=LGeF3lRxy8h6;FfG70+9U4&M6|Zkuct(&#(($C=+Zc%zrc_VRDg$F(N~i~S zPoyElQrjW+B`u*xI9=hgUI&gqYnDAHYfY*=(J{>Je^?-&@LA%nQ3_(xmL2!Y@@;S?q`Axx|%3`S2t zq-AAgN~r$AzDGt#55MobFMa+ShghP?vfQpZ_|8km2{kpL13MpP`m+qgwUkapekck# z&pB3*z!>$M6_)DPVF-={OzHP>UMmRq*2=glPl1$NeYh&S&SHN)+P?@c%!GTGH%Z8m z!mx?kry%1KYpeLa5HHUq>>R@)3=)OH+m)8@()ztzZ$hp${ z#rkZr63R`CYGTy^WFN?o0xYEjM*@lfUKY`HXx^WiuqG*JMGjcB&zHj33xz?8fJj3d zvn{En;79?WtS4|MNvbEXy#*y8%A*uJq=daEiZ5Mhm{Jt?t^j*Z=-JU2hCni$&}BKn zQ38N@K5u6OIDg`W&Ty|LJnM_dkitS%ka$A?+tE{;z(f$D<~z9RObXAmMQ-b?z2gOF7pP70#x?1 zO$Sz6dUI8N}2!dIdxm{Pjnkx{u-KY5K`FCEf)L4JP*3$91XG$W@l zK0O8mhUEkz1)7hM=5-%Y@bf1e)`e&yvqR=WH39kJiyOI!u0hx01glw;>JGIg#tu#l zr343nwcb}g1i~|6Ja9b*lvE%`3M5ahN(3NSNpJ_Oj#y491g6q(i_^+bZd`Ed;nlrU8+1gYK`4z-yMv4GL#`VEhbIdr(dw$|pU6)dVw+5$!D~fewNzrUSAJ zlvGXd*mpnoZk`0+(U9SUUKc|nO-|VDGJ8RyCa~-gZuxO9AjK0O1gnjDQm`K&2d#;g z7$Ku1P(imHY2{9dDIVXazA_c0k~ZCFABJ@Pt8{F%YQdpyFW?d{`3<{XrtNSg3j+2m(Blv z*=(YR(!W+bSCKiD~e@26hkSECg>pQ zi3y@Xln*Asd=vPNktG*EAatzQOi+}++>@DPr7&`AL*d>+8)}cq*QhJ zwdr1n^0l?`Si7&4v;70$LGTGuP6!{UIU(Ru&=Wp>`^~q1!Igj|3Zs>QhMK#MmKJp9 zAD`mse1pJsibu21eJ4qRAHMqzeg2j6`1l(&*YhGfa)TR_oBLLW0iOyU2tz|L5Bj!; zh8@V|T>>Eq3H!qC%m4pBb`Ye_0kpq+X&n3!2jkmrS zY)+p}p^&Mj29X5woJx6aTF00m4}#X`da-m=WXj&2R4AK#6ca`*E#)JIxXkmsv^~}C zTted5Fgp{8o%=v{?TI(||AiF^6aK=59`zGI_Uz#su;O12m{9)EsW2hEocsQK#_V~) z5i5O;`cXPsP{ZGR0?n=VWZ4>Xdpbv&{+8)ol*M!6v4?X9-2MAyzSx)X{UArr=w_I3|#Fc#I0!zqSjzkyqI>u5mp> z`b-xni?yEk39U?!ZQ}88WsV~(Y13(YRI+GJvnz0*?NA{ZB{BX5gWtylnWv?RwE0xm z_h?x(jcK5|$5{TKU_p#tf2N<{Pcrp?ssIwe6Dt%GKJ!NtKbL~(aXxp&V*kPfTfG&` zh+9V83rn%PL@#O@ehsGrXm3(wCMM)y(xTpG!%Y@3otF7jTGV9TJWl`@n@`-ckL-$G zG9*x`IedF*Bi{>FG+~oLw#Vy7>cf4w#4UsL zY0|XjNhz!NXdM%>Ik#0Cv4N-{;*vYN%>%<$A2#3bh6Q;i%l(9(zQG^xx0z1c*W!oR zVS5|6nDa%WdO<<<6KbicbkKUEu1hnNG)w?ffw6@(fVLCawiaZjNV@_~CN@EqgE)*z z+R{5kGg+}m%Yw#WBm^HMx}>>+1TI2gLYBzu4bhMoCbX*1PdTKyNb{ocK1YQt$`;or zw7~|2ZJ`0wtJu~%CsYimus^UAm3b#(f@_I4S3%M%wxShLZ<;GjvUIM28}(xNPSFh{ zFNz7(F(K=PO0hcIm@3?a*<@RV*RnxBwM6pL+(Ad(npIUR3u&vagRF}Wo8?RQRxNFd5~KcupcsX z`FN@L7A#3pwwIWn08G$#lL!VUY$GOo4j6#?227x7nHhQcnS7M)tta5-A>;xR=&$u< z0a=4%j6p7_&5RuLTEjF+(~cOqR#fQmgf3PDC}d0si2@TYw{wae+#$L|z0u#ye!?8@4X>1cPAdiN+bxkj>&&tu5CCSaAuiAs2*Rn#oq0c;1@GF#+nBSt<>wj%syo z*KXpHWlXr096R)E@|IP$CbkJ348Cz%lWl3sT_Wco$X$*$hir@HrMXT_Koa#@nHa1700#kV2fp_0&eZa zy@#~6X_$Z>X>Ta)m4!VA$LTd-rJ+A1HcSY;1SVVz6R0Z31gE$#G++>+XlBsZRH6xP z9k`-gyURh@1cPMUQD}XM+)wa&^n0NFsjSV<@nk)Lt72UE#<=&_6U0K1-m#y9AZ{(e{A}y)h1~EqWm?LZXyMu&yYO;eNhv?DBq0A>S8J-2H^FfBdI`LdJw& zY>m+=VM29G@a5*wPe9_O#DmdKnA(G%K<8VU=!BgXX-#PBBGngh{Z`ti&orZ40uw+p zn1l(~Ny3Ed5hj4Z1}0D_mJWWx5IdacaDtg^;+o;LMobXtQoC$x5D_x;!nT5@S9DQN z%bhE{=dNOY0xGHn6>__Nf&-M+TWg)RT)vlvh?mh%0D%om=v8jihPzajD7B0Uym`r{Y> zs5AQse+fU~b6C*eC+Ip&U_z|T*Rz*qt+%j(gb5yaQdjh=7AQ-HQcP58w4Hc~IJ|_6QVJZ1lg!4)aOP;JJa5>?Q2|=&Tl_BPe5l!848I#h1 z2u$EUy|9`61d0@+Din5M1*y@W%HfTSM$aIpK{Dh^s*%~}ZUy}WV#3?s2?9gxko|TR^ir%aubXCXfrfvv{2-O07|pT|dDQ zBj(UCCPeI4pFMY=pE}Q&VX<~l>oZL7FiA!~!NDxrHb*&;T}&8EJV>7{F0aL(6#5OWT_neTICtET`6iW`+q?T_pJlA<^h3TsjD-pAZspwL0o){?2m;i5Vak zJJ>Tbj)g*Z*+e;`I~=x%9o9WeVAKG0sh>c@CqyzY@L=>4Ov~2>3T`!$gq6Hzd2!*^ z^#T*}hY#27dV(TG+%-t6Z0)PzkjsBUKfLZ@g3CcWwEO=APA}`08avv-gu~dOZvDKN zvf(onFQ zDj%rDmo$?tz?XEVMKx`UVwl{B2_!RSCN5F-6GEahB|2VDkmM&Ywkj)fiT-cRztzTah=$4%lt|goT=13a9u&F6bIW`Kg!jxx@KZ zN4*eTxeLum2uFNw>mpqL1J+TfHWLXb8@Xn31SU*ufd&~9gytmSCrp6}aSjN4;3o{P zs)l`tuTd19W8u{))-&^URPF^W$$m2?IOa=l8xqyX%a|~d7!Z77`m8(`%2;)*6RA@8 z32$%z+`)vu5fgZS=N=l&F{MZ3ynUb$L>puF4M~)a?+rh*%EKPh*A_-4<| z%q=HTrL0L%us(sj1~laa>OIpeJa&*gKj=G#M7~^toCk?Yeio_KgIuBR=8&%{M^!0R zI5IQC)R1kB#?;0X&ms*IuJjX(E81PbRI~Mj-eO_TZUxPECv$2!DzZE=>jk&>y+ifY z1Akz<@dRF~$A))5Ay)3{XkP@Q39`{`f!OvevS8JenBd%O*9~%opJ1{*zS~TqU%x!~ z37d1sv-cR?Nid4owk3GCV0N;TNjy7TeN?Q<%$h#aZBz0ScH5$b=W*=T=vn9>2a&-3 zvHZ&X@Dtv?d>b+0AH(+!6>+1!)_W<1nk=Ps?c%5wrAl=_mfmqCJX>&e<_8M)u1b-3 zXGvClucNYd(NdCv!qQvTWiBnmDeRyiuQyoMxx{-yn^%nwB*JJfI%ed6Jex_>3)-M^ zo!xP1QdA(0mBSYwRWsT0ml>kkC}6B@TOfWZ$gkD8?jH4)T~=(An(AZTn#hDlYFe_f zl#0w|9BHr{NZBF5FJkDWKG6#soJ-yG&>m%*O1$;6szgW@lTGG@gDMcP_6?O?wvuF1 zDRiEAJ=cpk^j%Gp#S zQD7kx&caA?7(7VM&ILkDa6jRPzYm!3?Wg?Q0d!*ssqD>?nx}JUhK2(pMCRr3(iDw`rM)oJ5a5YB=sDlN~#R=)eKMB*xN4Pu3VoG&lBGd zO9=xa?9o^lhlkjatJ?bsKfZy!_!co4E&4ozf9LVM*B(InDqinR$BwBD3WyCQZNsN1# zi7~2_IZSI!H?{25#xgVPji$CwrftnE{S=vrlY^)rD#Y=7r0+JrfqvjJW5$GEzS(*L zeqq$)zKhtC-5TtVMDB#clq-{|)3ZCj9k{^yAk@O!)Q-G2yqR z-*rxi|Gh^o+4eEnE6tjEPGicw7_GYd=m>+|WFCq;#+aTQKXS@GSd7}2)hN$*Qs||1 zEj4UDtoY-}2brB73-aI}Ya|Jd$YeVfujyG2Y0ZO{uy67Qjn`W1yJz{d>hBXa{}bcW z7=C+NKlSGhU+5>i{qV&wA?WM>&i`orPw4gjWAa-|6esv^(=~pcj`W)NS7WYLbbV!G zaKD3I&@KDd9gp+SnY`k!C!RaVa`mxWe^E*au;a#rKd^T6iXV3WZtvQ*9~9KCIyhKmYsu zHS4^ToL?`>xgn*nxe@l)RY9T`4qZkq$YK?`P}Qw7gk1CpIInt5%yN}B(k>{{O3c#1 zPRnR*LB{8x%o7=*iu{k9u%yF(5|*V{kyxvGVvBh(lycB*oYFK^+C-1S5s{us=@A?B&Tc zTbpP|O}`T{LO9H4(_Rki7^gJP&}@fd>%Z6E35h>zTzF6ZnF!?klVGpNNA#krv+-BL zZpdeUzsvr=*pm_zyG%~-q%a@;6RsR-S;AbxOWq$ZoIRBIi z;VtK27iT>AMX^EwO8J5(Glfc~l?UW-ipFm1~b)DI%Z9+CA9@hE{u)hO`s?vOB z@El`$Cj4?`IPsh%+Vi1wbvK_bTm=h6vZn)3vK;j)Tg;N8POHOXl*h)pPk?`97$2!^ zX@0F-;+P)s_Z5*JAOWbS>yR(% z|8TE(K(2}fNGfOni>}1}#EodJ-cSJdHQ!!d{Bg;;;OjRVaL2Ay-=Ql+ZU6f1D-P!S z-Nohgtqf$FmUMm!1((JfqmtC_^7ft*`R4Za9%&tR^Ptaz{aM)5F}2k1<^G}Md2Pqv zf4$c_DxY_sZ*c-GSpFSumKg^$`qi#?U(m45S8?~{{7UdPNdgkU$#K% z*xD{PhU{SNoR1wizMcL4IK>ntc90zfaKi7y;_U zR&7|k`}UCFGEP%E;WFxTO2hT8=cvslZJj^w@}aHPH;Zdg`mQYJa9EM^IOZ}xg(NP( zIm{dAl^lxV^cBmTj#`!{z!9mKv+7`rwGP`0whyQ%o|p^dwxb(Th3_x*;*fxdGzuRO zzmpTZ5udiTE&M%6Mq)t`BMzhtyS^%M#WaR^(xHz)iXjXE$1o1VkgqT_$2`S=Bj$ip z5gKI$N6H$AMPYwOJZ}(l%7p_fmS+**TADr5yd7VLbTnY-E@4vzXW|g;vq+)LP#z{! z_TF6Lw~t=}t}p@d7*j>S5r^x`>u75dlic$F=6+YqVkrbMv^l$>$cmV#3BQuD?}v2e zNA?Q1SvC}rteX~3Tq*qoMo=XY=NA&0{7Gc$Q()6sq%&tZkd6LwRv*nzcx#+6w*)en zkzoHWsanuQ2q7orn#!7p%p>N~90fj*j9ZVfeeq%N7I6vzq%jSZ7U*J(rH#u|6@8AG z^*S@JL6HkGu^<^IP?Vfw%_F85BLo@Y$on|F4D&zYaEcQShOxUGb6@KDKCbrHvRc9+ zeUGdd>f+N6(867aNr*t$0H_8%bi3Vk+^&ez6M5IM)sVnDd$h+;CyS}?%WRuOdl|0a zX8^gRaU2KXo6|5;I*%mEFf+;hs;h*{NgoeC(WXy9!U~(aSqSPh+i<_g7s0+HuE(+^8U@l^QZ{Hphe`=L>w>AKFi1syWb6 zAcGfX1sVeYG1fG`Z*BtZ7|0>_V?v#PN{|>Y7d29FD`vnz%E(a2vTCKT@=tQYDK}$r zoDllUtI~~(DTcIl6li#d^bl#24zQ01TKM?!+kFrZAx=;!4xc~lcDp{g^(3Tfv})Q!t&F&p=fq#WrSw` zMf?Q*Sg0v;6b{7K{k!$+KE$CKekzO#H;TwGl4x~Pj;%;xtv?905{Sy;hC+Huh9K*GZGWd{ecFhjd_ zV1Z*l4k0PfGa~Kdd3SnJIZ5(Jr;K>@@k*qdsowp$tE4UHEtEEIsj+;rAGb&3@eB{Z z%$Hh?o#$03j1pApIlC1p>dA)uR{Vsecq9oQs$yPdTG(o2Y|v3OMJAF1nSn?1?5l{7 zv~4m1w*~^NevN&Lj~s?DPTWu!K?%Siilcym&;x@9!ibuFf@cLJY9QyL1V#m|sg;*k zaRPCU6PD;^?Yb)o9#TK4MXft>Y6);6{OeS+NsOH14$x14w~2 zx(dul<|pv*ASG~px?&Uh66B@ece&Z&;Ym*D@t&#l<5G)*1+C3?%~3dn@3d;0Ut9%g z2gu4&)D8{F@_zjkcU@V0TmcR2k*6>bvFppMi=nCoP6Ocr2yQBf9Wd|fPi^n*6bV_= zrZzpX(`cly6!XZVhs{Z(Cqd_d^2NLmE#^0%@)RsQS*xqVp-!&ki8qw{x3R|v6mLuE z94pL{SWD(6_l6bF)mvBE+r z&O34wX6J^)tVWGO+{cu6)6$+ZOn3E(&398K8PGjB@#Iq3zA%&MA`=OMC&YbmR^PqG0M*M^| zdPo#hjNk>Ad=l&W~Xcs&#O4WhYnjTe%jJ?7vtEK1;t*R%6&4J zA>4mQL$zL{b10iKd%F*RL~d?Eu|6#>V6VM!0&(3J<)|4#vU(lISjtf22Z@OVQkcA( zprnB8llC>Z%|;hLEZT!vVV7XWv2Nx)3&nJyVA{f4^o!}h!=_LwHM+?|8`Qmt-%n_q zkeGAwzJg-5QtO$_u7nFB1dH;jLGE0(jwO1?k>P6`La3P^V;?zy3kDp(*h%C7`m9HY zL!Rg=3_K^uNR1FEaDCla$l@iW#2O?H`vgdg^Eysgj&>NI-~`)HxXRtQprsEQiJP-v zq)_he(yZxO@V^wj7pvxjADo-OPHOLvG5nZ_*k4CmjAGB^*0IdGsjz;)_t7BsPh~|( zfYiOakeJjzQI@k`F-iRKYfnR4d4QJ59gG=YhA4{ZbjSj@dtixKTd5V<4x4r@8=VHI zJiL|I;ZVea1ljM=OClrKNi(o4#2|za)laa{p%(M|ckkdRaGNny!1kR>M@r!70bSFA=W#QXM z@tVy(%~Lpl6!>i|m+&YxC8{||hM*)HX!wh?pK`bJy9s;$yy2EjKnPRU7f%Q*N(kZB zQI;k5-8i_AdQW?RG+SYsM0zgJlEG-#hW>1%fsIUePjOQ+Oy5 z*eax8bw#kAL^iiW)?#0qQ~SajyEhU$92PTtW@kl?9(<>aTx=6T{e%`Sv=S3D-$y3$ zdk}9R&W@sShNpoZ-O<0g2_c7w90y2lD-1(M4Ml?U>wba|RaTRyp8%PZzL&e)!ThU^Fo)nfZ4{7!D&$tW8a`O#j8-a4q&A&Ls{&K9` z#QvlVWD4P;@B^F>D*DcOEe3XHC}UWPl7c!4)0!=4pwtSXZ7!ZrBmwWY$nJ^64=I|t z3;C?$LroL()>p701`7wai7h4_1>{Mc6UTRff`_yqvxt{l{(ikaV(T;KkmJY-cDqu<8yAj|j_OGK2#RiCQ%si7kD23`S689H zJ7qH(EF zvdDmeg{sw65Jv&g104nM!7xBLM|KeCYe;bursfoc<}!rL9R)V1qk!RUC53v8X3R^- zZRHix56A052SVroGM=WCWtn}$?gDQfI=*@6dou!M1r|UFjTF9oqe)6UV0>UqKp}he z-sx}VUO#BvWnOG->jweE)-8Y&a6CRfhW@k0w|huJHLtO6Qy3}qBn7{fU_xO*5 zCdvkRuRhTm&TnLo5s)4EqQQ7oK3vE(7s#5Z5>ylyL2^TuyH0lN3HneSg)vVdGI(fM zqq*skoWO>DY>`8094gL24pbp>M`4o3Da1p0w5cG8_oAzYW}PCl7qGmF6Ovd-N@xGW zIX`jt$9Oe%UBT($I-BF205wAmqz|Q%!pt+wOm3@CBZj@cEHDRv+lwu7IFv>G1P+nz z?2^#u$72Y{;wbE8Fa$}%MK==5D3*+y8A1?S*-sECycAIsVHdPVaTGt~!Ta;HM49&>NgD0c`uVyNIkM^*1Z)!fu!Ky z-DhNKIfNawoBqaPREL1vpdJI%>e`zgD=Cz53PVf4!=t(htC#RdN~p*fH?{qQA%_sF zs%)Bvfh|Z9ps0%?s;#OfWw=^YHfUOpa(E47%dVWFqXTv4hl|}(<_6M%?^Z>P9ez9#p}+M}b%J1)=tg}KA3uZ; zy%ifu;XrurRHyhoMK{zUEdD|*JhS4Npj)FYP zLfbL|lF)%MgW!<>EAWNNW=9iQ2O5i6B{a*oy#vlXfE1vSwqUnF5d%OCfde3$aQ>JE p-I~ZghLB&incBA>I_a-1{XfK40~>}lP$U2V002ovPDHLkV1hgfFr)wg literal 0 HcmV?d00001 diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c108138..e38d08d 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -61,6 +61,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. "create-spatial-index": "cli_spatialite_indexes", "install": "cli_install", "uninstall": "cli_uninstall", + "tui": "cli_tui", } commands.sort(key = lambda command: go_first.index(command) if command in go_first else 999) cog.out("\n") @@ -1025,6 +1026,23 @@ disable-fts -h, --help Show this message and exit. +.. _cli_ref_tui: + +tui +=== + +See :ref:`cli_tui`. + +:: + + Usage: sqlite-utils tui [OPTIONS] + + Open Textual TUI. + + Options: + -h, --help Show this message and exit. + + .. _cli_ref_optimize: optimize diff --git a/docs/cli.rst b/docs/cli.rst index d567cba..3b9ac31 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2163,3 +2163,25 @@ You can uninstall packages that were installed using ``sqlite-utils install`` wi $ sqlite-utils uninstall beautifulsoup4 Use ``-y`` to skip the request for confirmation. + +.. _cli_tui: + +Experimental TUI +================ + +A TUI is a "text user interface" (or "terminal user interface") - a keyboard and mouse driven graphical interface running in your terminal. + +``sqlite-utils`` has experimental support for a TUI for building command-line invocations, built on top of the `Trogon `__ TUI library. + +To enable this feature you will need to install the ``trogon`` dependency. You can do that like so:: + + sqite-utils install trogon + +Once installed, running the ``sqlite-utils tui`` command will launch the TUI interface:: + + sqlite-utils tui + +You can then construct a command by selecting options from the menus, and execute it using ``Ctrl+R``. + +.. image:: _static/img/tui.png + :alt: A TUI interface for sqlite-utils - the left column shows a list of commands, while the right panel has a form for constructing arguments to the add-column command. diff --git a/setup.py b/setup.py index 3964b3a..5898ffa 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ setup( "data-science-types", ], "flake8": ["flake8"], + "tui": ["trogon"], }, entry_points=""" [console_scripts] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 5a88bd5..ab9c524 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -33,6 +33,11 @@ from .utils import ( TypeTracker, ) +try: + import trogon # type: ignore +except ImportError: + trogon = None + CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) @@ -112,6 +117,10 @@ def cli(): pass +if trogon is not None: + cli = trogon.tui()(cli) + + @cli.command() @click.argument( "path", diff --git a/tests/test_docs.py b/tests/test_docs.py index 7f2e3ed..c305c7a 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -5,7 +5,7 @@ import pytest import re docs_path = Path(__file__).parent.parent / "docs" -commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+) ") +commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+)") recipes_re = re.compile(r"r\.(\w+)\(") From e240133b11588d31dc22c632f7a7ca636c72978d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 May 2023 11:53:33 -0700 Subject: [PATCH 0726/1004] Release 3.32 Refs #544, #545, #547, #548 --- docs/changelog.rst | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 347601a..ee8713f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,18 @@ Changelog =========== +.. _v3_32: + +3.32 (2023-05-21) +----------------- + +- New experimental ``sqlite-utils tui`` interface for interactively building command-line invocations, powered by `Trogon `__. This requires an optional dependency, installed using ``sqlite-utils install trogon``. There is a screenshot :ref:`in the documentation `. (:issue:`545`) +- ``sqlite-utils analyze-tables`` command (:ref:`documentation `) now has a ``--common-limit 20`` option for changing the number of common/least-common values shown for each column. (:issue:`544`) +- ``sqlite-utils analyze-tables --no-most`` and ``--no-least`` options for disabling calculation of most-common and least-common values. +- If a column contains only ``null`` values, ``analyze-tables`` will no longer attempt to calculate the most common and least common values for that column. (:issue:`547`) +- Calling ``sqlite-utils analyze-tables`` with non-existent columns in the ``-c/--column`` option now results in an error message. (:issue:`548`) +- The ``table.analyze_column()`` method (:ref:`documented here `) now accepts ``most_common=False`` and ``least_common=False`` options for disabling calculation of those values. + .. _v3_31: 3.31 (2023-05-08) diff --git a/setup.py b/setup.py index 5898ffa..0d05c5c 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.31" +VERSION = "3.32" def get_long_description(): From d8fe1b0d899faaaa3d4714a39328f4c24932278f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 May 2023 13:57:22 -0700 Subject: [PATCH 0727/1004] Reformatted CLI examples in docs Closes #551 --- docs/_templates/base.html | 5 + docs/cli.rst | 1201 ++++++++++++++++++++++++++----------- 2 files changed, 863 insertions(+), 343 deletions(-) diff --git a/docs/_templates/base.html b/docs/_templates/base.html index 43c2003..a253a46 100644 --- a/docs/_templates/base.html +++ b/docs/_templates/base.html @@ -7,6 +7,11 @@ {% block scripts %} {{ super() }} +