From 6f4f9a3effeb16de0348d3cf136664f7531f498d Mon Sep 17 00:00:00 2001 From: Dylan Wu <6586811+dylan-wu@users.noreply.github.com> Date: Tue, 18 May 2021 21:47:44 -0500 Subject: [PATCH 001/102] Fixing insert from JSON containing strings with non-ascii characters are escaped aps unicode for lists, tuples, dicts (#258) --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d9a6b26..0e37b12 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2222,7 +2222,7 @@ def jsonify_if_needed(value): if isinstance(value, decimal.Decimal): return float(value) if isinstance(value, (dict, list, tuple)): - return json.dumps(value, default=repr) + return json.dumps(value, default=repr, ensure_ascii=False) elif isinstance(value, (datetime.time, datetime.date, datetime.datetime)): return value.isoformat() elif isinstance(value, uuid.UUID): From a95954c481012cc46fff2df5aaa4ee24e43dddf3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 18 May 2021 19:56:53 -0700 Subject: [PATCH 002/102] Tests for unicode characters in nested JSON, refs #257 --- tests/test_create.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_create.py b/tests/test_create.py index c1daf6b..e08eb25 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -748,7 +748,7 @@ def test_create_index_if_not_exists(fresh_db): {"dictionary": {"nested": "complex"}}, collections.OrderedDict( [ - ("key1", {"nested": "complex"}), + ("key1", {"nested": ["cømplex"]}), ("key2", "foo"), ] ), @@ -762,6 +762,14 @@ def test_insert_dictionaries_and_lists_as_json(fresh_db, data_structure): assert data_structure == json.loads(row[1]) +def test_insert_list_nested_unicode(fresh_db): + fresh_db["test"].insert( + {"id": 1, "data": {"key1": {"nested": ["cømplex"]}}}, pk="id" + ) + row = fresh_db.execute("select id, data from test").fetchone() + assert row[1] == '{"key1": {"nested": ["cømplex"]}}' + + def test_insert_uuid(fresh_db): uuid4 = uuid.uuid4() fresh_db["test"].insert({"uuid": uuid4}) From e7b2626291040b78b9a2dbc2982ba72691fb1a0f Mon Sep 17 00:00:00 2001 From: Rob Wells Date: Wed, 19 May 2021 03:57:26 +0100 Subject: [PATCH 003/102] Fix incorrect create-table cli description (#254) The description for `create-table` was duplicated from `create-index`. --- sqlite_utils/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index e42be83..9d899d5 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -905,7 +905,7 @@ def upsert( def create_table( path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension ): - "Add an index to the specified table covering the specified columns" + "Add a table with the specified columns" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) if len(columns) % 2 == 1: From 3e62ab62a88992d4bbb0fe83debec3bacd93ebf3 Mon Sep 17 00:00:00 2001 From: Damien Ready Date: Tue, 18 May 2021 21:58:04 -0500 Subject: [PATCH 004/102] Correct some typos (#245) --- docs/changelog.rst | 8 ++++---- docs/cli.rst | 6 +++--- docs/python-api.rst | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9d4770f..7e1e6a9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -201,7 +201,7 @@ Other changes 2.17 (2020-09-07) ----------------- -This release handles a bug where replacing rows in FTS tables could result in growing numbers of unneccessary rows in the associated ``*_fts_docsize`` table. (`#149 `__) +This release handles a bug where replacing rows in FTS tables could result in growing numbers of unnecessary rows in the associated ``*_fts_docsize`` table. (`#149 `__) - ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (`#152 `__) - ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (`#153 `__) @@ -272,7 +272,7 @@ The theme of this release is better tools for working with binary data. The new - ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (`#122 `__) - ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (`#123 `__) -- JSON output now encodes BLOB values as special base64 obects - see :ref:`cli_query_json`. (`#125 `__) +- JSON output now encodes BLOB values as special base64 objects - see :ref:`cli_query_json`. (`#125 `__) - The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (`#126 `__) - The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (`#124 `__) @@ -594,7 +594,7 @@ Support for lookup tables. 1.2 (2019-06-12) ---------------- -- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by instrospecting the database. See :ref:`python_api_add_foreign_key` for details. (`#25 `__) +- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by introspecting the database. See :ref:`python_api_add_foreign_key` for details. (`#25 `__) - Ability to set ``NOT NULL`` constraints and ``DEFAULT`` values when creating tables (`#24 `__). Documentation: :ref:`Setting defaults and not null constraints (Python API) `, :ref:`Setting defaults and not null constraints (CLI) ` - Support for ``not_null_default=X`` / ``--not-null-default`` for setting a ``NOT NULL DEFAULT 'x'`` when adding a new column. Documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` @@ -603,7 +603,7 @@ Support for lookup tables. 1.1 (2019-05-28) ---------------- -- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key alread exists (`#21 `__) - documentation: :ref:`Inserting data (Python API) `, :ref:`Inserting data (CLI) ` +- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key already exists (`#21 `__) - documentation: :ref:`Inserting data (Python API) `, :ref:`Inserting data (CLI) ` - Ability to add a column that is a foreign key reference using ``fk=...`` / ``--fk`` (`#16 `__) - documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` .. _v1_0_1: diff --git a/docs/cli.rst b/docs/cli.rst index 8a049a6..b314b2d 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -34,7 +34,7 @@ Use ``--nl`` to get back newline-delimited JSON objects:: {"id": 1, "age": 4, "name": "Cleo"} {"id": 2, "age": 2, "name": "Pancakes"} -You can use ``--arrays`` to request ararys instead of objects:: +You can use ``--arrays`` to request arrays instead of objects:: $ sqlite-utils dogs.db "select * from dogs" --arrays [[1, 4, "Cleo"], @@ -75,7 +75,7 @@ Binary strings are not valid JSON, so BLOB columns containing binary data will b } ] -If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the comand will return the number of affected rows:: +If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows:: $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" [{"rows_affected": 1}] @@ -1186,7 +1186,7 @@ Both of these commands accept one or more database files as arguments. Loading SQLite extensions ========================= -Many of these commands have the ablity to load additional SQLite extensions using the ``--load-extension=/path/to/extension`` option - use ``--help`` to check for support, e.g. ``sqlite-utils rows --help``. +Many of these commands have the ability to load additional SQLite extensions using the ``--load-extension=/path/to/extension`` option - use ``--help`` to check for support, e.g. ``sqlite-utils rows --help``. This option can be applied multiple times to load multiple extensions. diff --git a/docs/python-api.rst b/docs/python-api.rst index e5ffd44..11b1f0c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -603,7 +603,7 @@ The first argument to ``update()`` is the primary key. This can be a single valu >>> db["compound_dogs"].update((5, 3), {"name": "Updated"}) -The second argument is a dictonary of columns that should be updated, along with their new values. +The second argument is a dictionary of columns that should be updated, along with their new values. You can cause any missing columns to be added automatically using ``alter=True``:: From 328211eaca1247cd6b33a2c0a54642f87866d85b Mon Sep 17 00:00:00 2001 From: "Juan E. D" Date: Tue, 18 May 2021 23:58:21 -0300 Subject: [PATCH 005/102] Typo in upsert example (#244) Remove extra `[` --- docs/python-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 11b1f0c..ad7b394 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -647,7 +647,7 @@ For example, given the dogs database you could upsert the record for Cleo like s .. code-block:: python - db["dogs"].upsert([{ + db["dogs"].upsert({ "id": 1, "name": "Cleo", "twitter": "cleopaws", From 2f3371ecb1ad075672d3f815993193732ed00be5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 18 May 2021 20:26:13 -0700 Subject: [PATCH 006/102] Suggest --alter if column is missing, closes #259, refs #256 --- sqlite_utils/cli.py | 13 ++++++++++--- tests/test_cli.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 9d899d5..011935a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -732,9 +732,16 @@ def insert_upsert_implementation( extra_kwargs["upsert"] = upsert # Apply {"$base64": true, ...} decoding, if needed docs = (decode_base64_values(doc) for doc in docs) - db[table].insert_all( - docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs - ) + try: + db[table].insert_all( + docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs + ) + except sqlite3.OperationalError as e: + if e.args and "has no column named" in e.args[0]: + raise click.ClickException( + "{}\n\nTry using --alter to add additional columns".format(e.args[0]) + ) + raise @cli.command() diff --git a/tests/test_cli.py b/tests/test_cli.py index 8775396..2e982dd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -354,6 +354,21 @@ def test_add_column_foreign_key(db_path): assert "table 'bobcats' does not exist" in str(result.exception) +def test_suggest_alter_if_column_missing(db_path): + db = Database(db_path) + db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "authors", "-"], + input='{"id": 2, "name": "Barry", "age": 43}', + ) + assert result.exit_code != 0 + assert result.output.strip() == ( + "Error: table authors has no column named age\n\n" + "Try using --alter to add additional columns" + ) + + def test_index_foreign_keys(db_path): test_add_column_foreign_key(db_path) db = Database(db_path) From b2302875c97f723e02cc39136d0b20fd706369aa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 May 2021 20:55:46 -0700 Subject: [PATCH 007/102] Document --type option better, closes #255 --- sqlite_utils/cli.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 011935a..960d34c 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1276,9 +1276,12 @@ def triggers( @click.argument("table") @click.option( "--type", - type=(str, str), + type=( + str, + click.Choice(["INTEGER", "TEXT", "FLOAT", "BLOB"], case_sensitive=False), + ), multiple=True, - help="Change column type to X", + help="Change column type to INTEGER, TEXT, FLOAT or BLOB", ) @click.option("--drop", type=str, multiple=True, help="Drop this column") @click.option( From 51d01da30d45c1fbc1e587e6046a933529cf915e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 May 2021 22:01:38 -0700 Subject: [PATCH 008/102] Ability to add descending order indexes (#262) * DescIndex(column) for descending index columns, refs #260 * Ability to add desc indexes using CLI, closes #260 --- docs/cli.rst | 8 ++++++++ docs/python-api.rst | 11 +++++++++++ sqlite_utils/cli.py | 16 +++++++++++++--- sqlite_utils/db.py | 13 ++++++++++++- tests/test_cli.py | 11 +++++++++++ tests/test_create.py | 14 ++++++++++++++ 6 files changed, 69 insertions(+), 4 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index b314b2d..e5ffcfc 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1033,6 +1033,14 @@ Use the ``--unique`` option to create a unique index. Use ``--if-not-exists`` to avoid attempting to create the index if one with that name already exists. +To add an index on a column in descending order, prefix the column with a hyphen. Since this can be confused for a command-line option you need to construct that like this:: + + $ sqlite-utils create-index mydb.db mytable -- col1 -col2 col3 + +This will create an index on that table on ``(col1, col2 desc, col3)``. + +If your column names are already prefixed with a hyphen you'll need to manually execute a ``CREATE INDEX`` SQL statement to add indexes to them rather than using this tool. + .. _cli_fts: Configuring full-text search diff --git a/docs/python-api.rst b/docs/python-api.rst index ad7b394..418c58d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1866,6 +1866,17 @@ By default the index will be named ``idx_{table-name}_{columns}`` - if you want index_name="good_dogs_by_age" ) +To create an index in descending order for a column, wrap the column name in ``db.DescIndex()`` like this: + +.. code-block:: python + + from sqlite_utils.db import DescIndex + + db["dogs"].create_index( + ["is_good_dog", DescIndex("age")], + index_name="good_dogs_by_age" + ) + You can create a unique index by passing ``unique=True``: .. code-block:: python diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 960d34c..52de64a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -5,7 +5,7 @@ from datetime import datetime import hashlib import pathlib import sqlite_utils -from sqlite_utils.db import AlterError +from sqlite_utils.db import AlterError, DescIndex import textwrap import io import itertools @@ -450,11 +450,21 @@ def index_foreign_keys(path, load_extension): ) @load_extension_option def create_index(path, table, column, name, unique, if_not_exists, load_extension): - "Add an index to the specified table covering the specified columns" + """ + Add an index to the specified table covering the specified columns. + Use "sqlite-utils create-index mydb -- -column" to specify descending + order for a column. + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) + # Treat -prefix as descending for columns + columns = [] + for col in column: + if col.startswith("-"): + col = DescIndex(col[1:]) + columns.append(col) db[table].create_index( - column, index_name=name, unique=unique, if_not_exists=if_not_exists + columns, index_name=name, unique=unique, if_not_exists=if_not_exists ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0e37b12..fdfe360 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -144,6 +144,10 @@ class InvalidColumns(Exception): pass +class DescIndex(str): + pass + + _COUNTS_TABLE_CREATE_SQL = """ CREATE TABLE IF NOT EXISTS [{}]( [table] TEXT PRIMARY KEY, @@ -1156,6 +1160,13 @@ class Table(Queryable): index_name = "idx_{}_{}".format( self.name.replace(" ", "_"), "_".join(columns) ) + columns_sql = [] + for column in columns: + if isinstance(column, DescIndex): + fmt = "[{}] desc" + else: + fmt = "[{}]" + columns_sql.append(fmt.format(column)) sql = ( textwrap.dedent( """ @@ -1167,7 +1178,7 @@ class Table(Queryable): .format( index_name=index_name, table_name=self.name, - columns=", ".join("[{}]".format(c) for c in columns), + columns=", ".join(columns_sql), unique="UNIQUE " if unique else "", if_not_exists="IF NOT EXISTS " if if_not_exists else "", ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2e982dd..17ce27d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -208,6 +208,17 @@ def test_create_index(db_path): ) +def test_create_index_desc(db_path): + db = Database(db_path) + assert [] == db["Gosh"].indexes + result = CliRunner().invoke(cli.cli, ["create-index", db_path, "Gosh", "--", "-c1"]) + assert result.exit_code == 0 + assert ( + db.execute("select sql from sqlite_master where type='index'").fetchone()[0] + == "CREATE INDEX [idx_Gosh_c1]\n ON [Gosh] ([c1] desc)" + ) + + @pytest.mark.parametrize( "col_name,col_type,expected_schema", ( diff --git a/tests/test_create.py b/tests/test_create.py index e08eb25..6e925dc 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1,6 +1,7 @@ from sqlite_utils.db import ( Index, Database, + DescIndex, ForeignKey, AlterError, NoObviousTable, @@ -739,6 +740,19 @@ def test_create_index_if_not_exists(fresh_db): dogs.create_index(["name"], if_not_exists=True) +def test_create_index_desc(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True}) + assert [] == dogs.indexes + dogs.create_index([DescIndex("age"), "name"]) + sql = fresh_db.execute( + "select sql from sqlite_master where name='idx_dogs_age_name'" + ).fetchone()[0] + assert sql == ( + "CREATE INDEX [idx_dogs_age_name]\n" " ON [dogs] ([age] desc, [name])" + ) + + @pytest.mark.parametrize( "data_structure", ( From 668e8c9fd1b0eab4000902e4226aeaae3860e802 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 May 2021 22:00:11 -0700 Subject: [PATCH 009/102] Better help for sqlite-utils create-table --- docs/dogs.db | 0 sqlite_utils/cli.py | 12 +++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) delete mode 100644 docs/dogs.db diff --git a/docs/dogs.db b/docs/dogs.db deleted file mode 100644 index e69de29..0000000 diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 52de64a..68bd986 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -922,7 +922,17 @@ def upsert( def create_table( path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension ): - "Add a table with the specified columns" + """ + Add a table with the specified columns. Columns should be specified using + name, type pairs, for example: + + \b + sqlite-utils create-table my.db people \\ + id integer \\ + name text \\ + height float \\ + photo blob --pk id + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) if len(columns) % 2 == 1: From 8de5595c21b9be40f120eab20192baa465bd7628 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 May 2021 22:34:17 -0700 Subject: [PATCH 010/102] Handle BOM in CSV files, closes #250 --- sqlite_utils/cli.py | 2 +- tests/test_cli.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 68bd986..cbc0555 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -694,7 +694,7 @@ def insert_upsert_implementation( raise click.ClickException("Use just one of --nl, --csv or --tsv") if encoding and not (csv or tsv): raise click.ClickException("--encoding must be used with --csv or --tsv") - encoding = encoding or "utf-8" + encoding = encoding or "utf-8-sig" buffered = io.BufferedReader(json_file, buffer_size=4096) decoded = io.TextIOWrapper(buffered, encoding=encoding) if pk and len(pk) == 1: diff --git a/tests/test_cli.py b/tests/test_cli.py index 17ce27d..3807eca 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1915,3 +1915,28 @@ def test_attach(tmpdir): {"id": 1, "text": "foo"}, {"id": 1, "text": "bar"}, ] + + +def test_csv_insert_bom(tmpdir): + db_path = str(tmpdir / "test.db") + bom_csv_path = str(tmpdir / "bom.csv") + with open(bom_csv_path, "wb") as fp: + fp.write(b"\xef\xbb\xbfname,age\nCleo,5") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "broken", bom_csv_path, "--encoding", "utf-8", "--csv"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + result2 = CliRunner().invoke( + cli.cli, + ["insert", db_path, "fixed", bom_csv_path, "--csv"], + catch_exceptions=False, + ) + assert result2.exit_code == 0 + db = Database(db_path) + tables = db.execute("select name, sql from sqlite_master").fetchall() + assert tables == [ + ("broken", "CREATE TABLE [broken] (\n [\ufeffname] TEXT,\n [age] TEXT\n)"), + ("fixed", "CREATE TABLE [fixed] (\n [name] TEXT,\n [age] TEXT\n)"), + ] From 8c542d20ca95069476c8c84d4078a99d07561c6e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 May 2021 22:47:59 -0700 Subject: [PATCH 011/102] Release 3.7 Refs #237, #238, #240, #250, #257, #259, #260 --- docs/changelog.rst | 14 ++++++++++++++ docs/python-api.rst | 2 ++ setup.py | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7e1e6a9..81e7729 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,20 @@ Changelog =========== +.. _v3_7: + +3.7 (2021-05-28) +---------------- + +- New ``table.pks_and_rows_where()`` method returning ``(primary_key, row_dictionary)`` tuples - see :ref:`python_api_pks_and_rows_where`. (`#240 `__) +- Fixed bug with `table.add_foreign_key()` against columns containing spaces. (`#238 `__) +- ``table_or_view.drop(ignore=True)`` option for avoiding errors if the table or view does not exist. (`#237 `__) +- ``sqlite-utils drop-view --ignore`` and ``sqlite-utils drop-table --ignore`` options. (`#237 `__) +- Fixed a bug with inserts of nested JSON containing non-ascii strings - thanks, Dylan Wu. (`#257 `__) +- Suggest ``--alter`` if an error occurs caused by a missing column. (`#259 `__) +- Support creating indexes with columns in descending order, see :ref:`API documentation ` and :ref:`CLI documentation `. (`#260 `__) +- Correctly handle CSV files that start with a UTF-8 BOM. (`#250 `__) + .. _v3_6: 3.6 (2021-02-18) diff --git a/docs/python-api.rst b/docs/python-api.rst index 418c58d..78b069c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1848,6 +1848,8 @@ If the ``_counts`` table ever becomes out-of-sync with the actual table counts y db.reset_counts() +.. _python_api_create_index: + Creating indexes ================ diff --git a/setup.py b/setup.py index 6e7f978..53d71ef 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.6" +VERSION = "3.7" def get_long_description(): From 670f92285fc931f706b155ca20ac2e6fb3ca45b4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 May 2021 23:31:04 -0700 Subject: [PATCH 012/102] Fixed RST --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 81e7729..19dd349 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,7 +8,7 @@ ---------------- - New ``table.pks_and_rows_where()`` method returning ``(primary_key, row_dictionary)`` tuples - see :ref:`python_api_pks_and_rows_where`. (`#240 `__) -- Fixed bug with `table.add_foreign_key()` against columns containing spaces. (`#238 `__) +- Fixed bug with ``table.add_foreign_key()`` against columns containing spaces. (`#238 `__) - ``table_or_view.drop(ignore=True)`` option for avoiding errors if the table or view does not exist. (`#237 `__) - ``sqlite-utils drop-view --ignore`` and ``sqlite-utils drop-table --ignore`` options. (`#237 `__) - Fixed a bug with inserts of nested JSON containing non-ascii strings - thanks, Dylan Wu. (`#257 `__) From 2dad4f583cf3f9be40a4388093ed74c9043a6989 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Jun 2021 11:57:05 -0700 Subject: [PATCH 013/102] Improved .rows_where() documentation, added test for :named parameters --- docs/python-api.rst | 8 +++++++- tests/test_rows.py | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 78b069c..3745ffa 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -202,7 +202,13 @@ You can filter rows by a WHERE clause using ``.rows_where(where, where_args)``:: ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} -To return custom columns (instead of using ``select *``) pass ``select=``:: +The first argument is a fragment of SQL. The second, optional argument is values to be passed to that fragment - you can use ``?`` placeholders and pass an array, or you can use ``:named`` parameters and pass a dictionary, like this:: + + >>> for row in db["dogs"].rows_where("age > :age", {"age": 3}): + ... print(row) + {'id': 1, 'age': 4, 'name': 'Cleo'} + +To return custom columns (instead of the default that uses ``select *``) pass ``select="column1, column2"``:: >>> db = sqlite_utils.Database("dogs.db") >>> for row in db["dogs"].rows_where(select='name, age'): diff --git a/tests/test_rows.py b/tests/test_rows.py index 73bd94f..3ac52f9 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -13,6 +13,7 @@ def test_rows(existing_db): [ ("name = ?", ["Pancakes"], {2}), ("age > ?", [3], {1}), + ("age > :age", {"age": 3}, {1}), ("name is not null", [], {1, 2}), ("is_good = ?", [True], {1, 2}), ], From 9c67cb925253cd5ef54a1fe0496e0ff9caeacfd6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Jun 2021 20:51:27 -0700 Subject: [PATCH 014/102] table.xindexes property plus improved introspection documentation, closes #261 --- docs/python-api.rst | 103 +++++++++++++++++++++++++++++++++++++-- sqlite_utils/db.py | 22 +++++++++ tests/test_introspect.py | 29 ++++++++++- 3 files changed, 148 insertions(+), 6 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 3745ffa..de26d0a 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1449,6 +1449,11 @@ If you have loaded an existing table or view, you can use introspection to find >>> db["PlantType"] +.. _python_api_introspection_exists: + +.exists() +--------- + The ``.exists()`` method can be used to find out if a table exists or not:: >>> db["PlantType"].exists() @@ -1456,6 +1461,11 @@ The ``.exists()`` method can be used to find out if a table exists or not:: >>> db["PlantType2"].exists() False +.. _python_api_introspection_count: + +.count +------ + The ``.count`` property shows the current number of rows (``select count(*) from table``):: >>> db["PlantType"].count @@ -1465,23 +1475,45 @@ The ``.count`` property shows the current number of rows (``select count(*) from This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.execute_count()`` instead of accessing the property. -The ``.columns`` property shows the columns in the table or view:: +.. _python_api_introspection_columns: + +.columns +-------- + +The ``.columns`` property shows the columns in the table or view. It returns a list of ``Column(cid, name, type, notnull, default_value, is_pk)`` named tuples. + +:: >>> db["PlantType"].columns [Column(cid=0, name='id', type='INTEGER', notnull=0, default_value=None, is_pk=1), Column(cid=1, name='value', type='TEXT', notnull=0, default_value=None, is_pk=0)] -The ``.columns_dict`` property returns a dictionary version of this with just the names and types:: +.. _python_api_introspection_columns_dict: + +.columns_dict +------------- + +The ``.columns_dict`` property returns a dictionary version of the columns with just the names and Python types:: >>> db["PlantType"].columns_dict {'id': , 'value': } +.. _python_api_introspection_pks: + +.pks +---- + The ``.pks`` property returns a list of strings naming the primary key columns for the table:: >>> db["PlantType"].pks ['id'] -The ``.foreign_keys`` property shows if the table has any foreign key relationships. It is not available on views. +.. _python_api_introspection_foreign_keys: + +.foreign_keys +------------- + +The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey(table, column, other_table, other_column)`` named tuples. It is not available on views. :: @@ -1493,6 +1525,11 @@ The ``.foreign_keys`` property shows if the table has any foreign key relationsh ForeignKey(table='Street_Tree_List', column='qCaretaker', other_table='qCaretaker', other_column='id'), ForeignKey(table='Street_Tree_List', column='PlantType', other_table='PlantType', other_column='id')] +.. _python_api_introspection_schema: + +.schema +------- + The ``.schema`` property outputs the table's schema as a SQL string:: >>> print(db["Street_Tree_List"].schema) @@ -1523,7 +1560,12 @@ The ``.schema`` property outputs the table's schema as a SQL string:: FOREIGN KEY ("qCareAssistant") REFERENCES [qCareAssistant](id), FOREIGN KEY ("qLegalStatus") REFERENCES [qLegalStatus](id)) -The ``.indexes`` property shows you all indexes created for a table. It is not available on views. +.. _python_api_introspection_indexes: + +.indexes +-------- + +The ``.indexes`` property returns all indexes created for a table, as a list of ``Index(seq, name, unique, origin, partial, columns)`` named tuples. It is not available on views. :: @@ -1535,7 +1577,38 @@ The ``.indexes`` property shows you all indexes created for a table. It is not a Index(seq=4, name='"Street_Tree_List_qCaretaker"', unique=0, origin='c', partial=0, columns=['qCaretaker']), Index(seq=5, name='"Street_Tree_List_PlantType"', unique=0, origin='c', partial=0, columns=['PlantType'])] -The ``.triggers`` property lists database triggers. It can be used on both database and table objects. +.. _python_api_introspection_xindexes: + +.xindexes +--------- + +The ``.xindexes`` property returns more detailed information about the indexes on the table, using the SQLite `PRAGMA index_xinfo() `__ mechanism. It returns a list of ``XIndex(name, columns)`` named tuples, where ``columns`` is a list of ``XIndexColumn(seqno, cid, name, desc, coll, key)`` named tuples. + +:: + >>> db["ny_times_us_counties"].xindexes + [ + XIndex( + name='idx_ny_times_us_counties_date', + columns=[ + XIndexColumn(seqno=0, cid=0, name='date', desc=1, coll='BINARY', key=1), + XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll='BINARY', key=0) + ] + ), + XIndex( + name='idx_ny_times_us_counties_fips', + columns=[ + XIndexColumn(seqno=0, cid=3, name='fips', desc=0, coll='BINARY', key=1), + XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll='BINARY', key=0) + ] + ) + ] + +.. _python_api_introspection_triggers: + +.triggers +--------- + +The ``.triggers`` property lists database triggers. It can be used on both database and table objects. It returns a list of ``Trigger(name, table, sql)`` named tuples. :: @@ -1546,6 +1619,11 @@ The ``.triggers`` property lists database triggers. It can be used on both datab >>> db.triggers ... similar output to db["authors"].triggers +.. _python_api_introspection_triggers_dict: + +.triggers_dict +-------------- + The ``.triggers_dict`` property returns the triggers for that table as a dictionary mapping their names to their SQL definitions. :: @@ -1564,6 +1642,11 @@ The same property exists on the database, and will return all triggers across al 'authors_ad': 'CREATE TRIGGER [authors_ad] AFTER DELETE...', 'authors_au': 'CREATE TRIGGER [authors_au] AFTER UPDATE'} +.. _python_api_introspection_detect_fts: + +.detect_fts() +------------- + The ``detect_fts()`` method returns the associated SQLite FTS table name, if one exists for this table. If the table has not been configured for full-text search it returns ``None``. :: @@ -1571,12 +1654,22 @@ The ``detect_fts()`` method returns the associated SQLite FTS table name, if one >>> db["authors"].detect_fts() "authors_fts" +.. _python_api_introspection_virtual_table_using: + +.virtual_table_using +-------------------- + The ``.virtual_table_using`` property reveals if a table is a virtual table. It returns ``None`` for regular tables and the upper case version of the type of virtual table otherwise. For example:: >>> db["authors"].enable_fts(["name"]) >>> db["authors_fts"].virtual_table_using "FTS5" +.. _python_api_introspection_has_counts_triggers: + +.has_counts_triggers +-------------------- + The ``.has_counts_triggers`` property shows if a table has been configured with triggers for updating a ``_counts`` table, as described in :ref:`python_api_cached_table_counts`. :: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index fdfe360..6781735 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -68,6 +68,10 @@ ForeignKey = namedtuple( "ForeignKey", ("table", "column", "other_table", "other_column") ) Index = namedtuple("Index", ("seq", "name", "unique", "origin", "partial", "columns")) +XIndex = namedtuple("XIndex", ("name", "columns")) +XIndexColumn = namedtuple( + "XIndexColumn", ("seqno", "cid", "name", "desc", "coll", "key") +) Trigger = namedtuple("Trigger", ("name", "table", "sql")) @@ -863,6 +867,24 @@ class Table(Queryable): indexes.append(Index(**row)) return indexes + @property + def xindexes(self): + sql = 'PRAGMA index_list("{}")'.format(self.name) + indexes = [] + for row in self.db.execute_returning_dicts(sql): + index_name = row["name"] + index_name_quoted = ( + '"{}"'.format(index_name) + if not index_name.startswith('"') + else index_name + ) + column_sql = "PRAGMA index_xinfo({})".format(index_name_quoted) + index_columns = [] + for info in self.db.execute(column_sql).fetchall(): + index_columns.append(XIndexColumn(*info)) + indexes.append(XIndex(index_name, index_columns)) + return indexes + @property def triggers(self): return [ diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 73102f8..66dbcff 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import Index, View, Database +from sqlite_utils.db import Index, View, Database, XIndex, XIndexColumn import pytest @@ -93,6 +93,33 @@ def test_indexes(fresh_db): ] == fresh_db["Gosh"].indexes +def test_xindexes(fresh_db): + fresh_db.executescript( + """ + create table Gosh (c1 text, c2 text, c3 text); + create index Gosh_c1 on Gosh(c1); + create index Gosh_c2c3 on Gosh(c2, c3 desc); + """ + ) + assert fresh_db["Gosh"].xindexes == [ + XIndex( + name="Gosh_c2c3", + columns=[ + XIndexColumn(seqno=0, cid=1, name="c2", desc=0, coll="BINARY", key=1), + XIndexColumn(seqno=1, cid=2, name="c3", desc=1, coll="BINARY", key=1), + XIndexColumn(seqno=2, cid=-1, name=None, desc=0, coll="BINARY", key=0), + ], + ), + XIndex( + name="Gosh_c1", + columns=[ + XIndexColumn(seqno=0, cid=0, name="c1", desc=0, coll="BINARY", key=1), + XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll="BINARY", key=0), + ], + ), + ] + + @pytest.mark.parametrize( "column,expected_table_guess", ( From 28dc5aac347ffdecb2dff154d23a73883a2ffabf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Jun 2021 21:26:46 -0700 Subject: [PATCH 015/102] sqlite-utils indexes command, refs #263 --- docs/cli.rst | 27 ++++++++++++++++ sqlite_utils/cli.py | 61 +++++++++++++++++++++++++++++++++++ tests/test_cli.py | 77 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index e5ffcfc..41d9a0c 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -293,6 +293,33 @@ It takes the same options as the ``tables`` command: * ``--tsv`` * ``--table`` +.. _cli_indexes: + +Listing indexes +=============== + +The ``indexes`` command lists any indexes configured for the database:: + + $ sqlite-utils indexes covid.db --table + table index_name seqno cid name desc coll key + -------------------------------- ------------------------------------------------------ ------- ----- ----------------- ------ ------ ----- + johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_combined_key 0 12 combined_key 0 BINARY 1 + johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_country_or_region 0 1 country_or_region 0 BINARY 1 + johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_province_or_state 0 2 province_or_state 0 BINARY 1 + johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_day 0 0 day 0 BINARY 1 + ny_times_us_counties idx_ny_times_us_counties_date 0 0 date 1 BINARY 1 + ny_times_us_counties idx_ny_times_us_counties_fips 0 3 fips 0 BINARY 1 + ny_times_us_counties idx_ny_times_us_counties_county 0 1 county 0 BINARY 1 + ny_times_us_counties idx_ny_times_us_counties_state 0 2 state 0 BINARY 1 + +It shows indexes across all tables. To see indexes for specific tables, list those after the database:: + + $ sqlite-utils indexes covid.db johns_hopkins_csse_daily_reports --table + +The command defaults to only showing the columns that are explicitly part of the index. To also include auxiliary columns use the ``--aux`` option - these columns will be listed with a ``key`` of ``0``. + +The command takes the same format options as the ``tables`` and ``views`` commands. + .. _cli_triggers: Listing triggers diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cbc0555..2e3d08e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1287,6 +1287,67 @@ def triggers( ) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("tables", nargs=-1) +@click.option("--aux", is_flag=True, help="Include auxiliary columns") +@output_options +@load_extension_option +@click.pass_context +def indexes( + ctx, + path, + tables, + aux, + nl, + arrays, + csv, + tsv, + no_headers, + table, + fmt, + json_cols, + load_extension, +): + "Show indexes for this database" + sql = """ + select + sqlite_master.name as "table", + indexes.name as index_name, + xinfo.* + from sqlite_master + join pragma_index_list(sqlite_master.name) indexes + join pragma_index_xinfo(index_name) xinfo + where + sqlite_master.type = 'table' + """ + if tables: + quote = sqlite_utils.Database(memory=True).quote + sql += " and sqlite_master.name in ({})".format( + ", ".join(quote(table) for table in tables) + ) + if not aux: + sql += " and xinfo.key = 1" + ctx.invoke( + query, + path=path, + sql=sql, + nl=nl, + arrays=arrays, + csv=csv, + tsv=tsv, + no_headers=no_headers, + table=table, + fmt=fmt, + json_cols=json_cols, + load_extension=load_extension, + ) + + @cli.command() @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index 3807eca..56f1953 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1802,6 +1802,83 @@ def test_search(tmpdir, fts, extra_arg, expected): assert result.output.replace("\r", "") == expected +def test_indexes(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db.conn.executescript( + """ + create table Gosh (c1 text, c2 text, c3 text); + create index Gosh_idx on Gosh(c2, c3 desc); + """ + ) + result = CliRunner().invoke( + cli.cli, + ["indexes", str(db_path)], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert json.loads(result.output) == [ + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 0, + "cid": 1, + "name": "c2", + "desc": 0, + "coll": "BINARY", + "key": 1, + }, + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 1, + "cid": 2, + "name": "c3", + "desc": 1, + "coll": "BINARY", + "key": 1, + }, + ] + result2 = CliRunner().invoke( + cli.cli, + ["indexes", str(db_path), "--aux"], + catch_exceptions=False, + ) + assert result2.exit_code == 0 + assert json.loads(result2.output) == [ + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 0, + "cid": 1, + "name": "c2", + "desc": 0, + "coll": "BINARY", + "key": 1, + }, + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 1, + "cid": 2, + "name": "c3", + "desc": 1, + "coll": "BINARY", + "key": 1, + }, + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 2, + "cid": -1, + "name": None, + "desc": 0, + "coll": "BINARY", + "key": 0, + }, + ] + + _TRIGGERS_EXPECTED = '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah AFTER INSERT ON articles\\nBEGIN\\n UPDATE counter SET count = count + 1;\\nEND"}]\n' From d1a372b3006e6cf7d2017b3ddc484bf5c033e45d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Jun 2021 22:16:33 -0700 Subject: [PATCH 016/102] Release 3.8 Refs #261, #263 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 19dd349..f1d2261 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v3_8: + +3.8 (2021-06-02) +---------------- + +- New ``sqlite-utils indexes`` command to list indexes in a database, see :ref:`cli_indexes`. (`#263 `__) +- ``table.indexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (`#261 `__) + .. _v3_7: 3.7 (2021-05-28) diff --git a/setup.py b/setup.py index 53d71ef..194baba 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.7" +VERSION = "3.8" def get_long_description(): From 9dff7a38831d471b1dff16d40d89eb5c3b4e84d6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 6 Jun 2021 23:02:18 -0700 Subject: [PATCH 017/102] Fixed markup --- docs/python-api.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index de26d0a..f629c1d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1585,6 +1585,7 @@ The ``.indexes`` property returns all indexes created for a table, as a list of The ``.xindexes`` property returns more detailed information about the indexes on the table, using the SQLite `PRAGMA index_xinfo() `__ mechanism. It returns a list of ``XIndex(name, columns)`` named tuples, where ``columns`` is a list of ``XIndexColumn(seqno, cid, name, desc, coll, key)`` named tuples. :: + >>> db["ny_times_us_counties"].xindexes [ XIndex( From 9696abfabf883d1c877ee71425b382587c288981 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Jun 2021 13:03:12 -0700 Subject: [PATCH 018/102] Rearranged "Inserting JSON data" section --- docs/cli.rst | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 41d9a0c..3bb2a83 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -472,6 +472,23 @@ If you feed it a JSON list it will insert multiple records. For example, if ``do } ] +You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so:: + + $ sqlite-utils insert dogs.db dogs dogs.json --pk=id + +You can skip inserting any records that have a primary key that already exists using ``--ignore``:: + + $ sqlite-utils insert dogs.db dogs dogs.json --ignore + +You can delete all the existing rows in the table before inserting the new records using ``--truncate``:: + + $ sqlite-utils insert dogs.db dogs dogs.json --truncate + +.. _cli_inserting_data_binary: + +Inserting binary data +--------------------- + You can insert binary data into a BLOB column by first encoding it using base64 and then structuring it like this:: [ @@ -484,17 +501,10 @@ You can insert binary data into a BLOB column by first encoding it using base64 } ] -You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so:: +.. _cli_inserting_data_nl_json: - $ sqlite-utils insert dogs.db dogs dogs.json --pk=id - -You can skip inserting any records that have a primary key that already exists using ``--ignore``:: - - $ sqlite-utils insert dogs.db dogs dogs.json --ignore - -You can delete all the existing rows in the table before inserting the new records using ``--truncate``:: - - $ sqlite-utils insert dogs.db dogs dogs.json --truncate +Inserting newline-delimited JSON +-------------------------------- You can also import newline-delimited JSON using the ``--nl`` option. Since `Datasette `__ can export newline-delimited JSON, you can combine the two tools like so:: From 0d2e4f49f324de01bcd8257d0adeea3ebf391791 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Jun 2021 13:51:49 -0700 Subject: [PATCH 019/102] db.schema and 'sqlite-utils schema' command, closes #268 --- docs/cli.rst | 13 +++++++++++++ docs/python-api.rst | 15 +++++++++++++++ sqlite_utils/cli.py | 17 +++++++++++++++++ sqlite_utils/db.py | 12 ++++++++++++ tests/test_cli.py | 27 +++++++++++++++++++++++++++ tests/test_introspect.py | 8 ++++++-- 6 files changed, 90 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 3bb2a83..d39d367 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -348,6 +348,19 @@ It defaults to showing triggers for all tables. To see triggers for one or more The command takes the same format options as the ``tables`` and ``views`` commands. +.. _cli_schema: + +Showing the schema +================== + +The ``sqlite-utils schema`` command shows the full SQL schema for the database:: + + $ sqlite-utils schema dogs.db + CREATE TABLE "dogs" ( + [id] INTEGER PRIMARY KEY, + [name] TEXT + ); + .. _cli_analyze_tables: Analyzing tables diff --git a/docs/python-api.rst b/docs/python-api.rst index f629c1d..725445c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -297,6 +297,21 @@ If the record does not exist a ``NotFoundError`` will be raised: except NotFoundError: print("Dog not found") +.. _python_api_schema: + +Showing the schema +================== + +The ``db.schema`` property returns the full SQL schema for the database as a string:: + + >>> db = sqlite_utils.Database("dogs.db") + >>> print(db.schema) + >>> print(db.schema) + CREATE TABLE "dogs" ( + [id] INTEGER PRIMARY KEY, + [name] TEXT + ); + .. _python_api_creating_tables: Creating tables diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 2e3d08e..6c40454 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1348,6 +1348,23 @@ def indexes( ) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@load_extension_option +def schema( + path, + load_extension, +): + "Show full schema for this database" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + click.echo(db.schema) + + @cli.command() @click.argument( "path", diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6781735..53fae10 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -301,6 +301,18 @@ class Database: "Returns {trigger_name: sql} dictionary" return {trigger.name: trigger.sql for trigger in self.triggers} + @property + def schema(self): + sqls = [] + for row in self.execute( + "select sql from sqlite_master where sql is not null" + ).fetchall(): + sql = row[0] + if not sql.strip().endswith(";"): + sql += ";" + sqls.append(sql) + return "\n".join(sqls) + @property def journal_mode(self): return self.execute("PRAGMA journal_mode;").fetchone()[0] diff --git a/tests/test_cli.py b/tests/test_cli.py index 56f1953..a73cdff 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1916,6 +1916,33 @@ def test_triggers(tmpdir, extra_args, expected): assert result.output == expected +def test_schema(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["dogs"].create({"id": int, "name": str}) + db["chickens"].create({"id": int, "name": str, "breed": str}) + db["chickens"].create_index(["breed"]) + result = CliRunner().invoke( + cli.cli, + ["schema", db_path], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert result.output == ( + "CREATE TABLE [dogs] (\n" + " [id] INTEGER,\n" + " [name] TEXT\n" + ");\n" + "CREATE TABLE [chickens] (\n" + " [id] INTEGER,\n" + " [name] TEXT,\n" + " [breed] TEXT\n" + ");\n" + "CREATE INDEX [idx_chickens_breed]\n" + " ON [chickens] ([breed]);\n" + ) + + def test_long_csv_column_value(tmpdir): db_path = str(tmpdir / "test.db") csv_path = str(tmpdir / "test.csv") diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 66dbcff..d54ca88 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -62,8 +62,12 @@ def test_columns(existing_db): ] -def test_schema(existing_db): - assert "CREATE TABLE foo (text TEXT)" == existing_db["foo"].schema +def test_table_schema(existing_db): + assert existing_db["foo"].schema == "CREATE TABLE foo (text TEXT)" + + +def test_database_schema(existing_db): + assert existing_db.schema == "CREATE TABLE foo (text TEXT);" def test_table_repr(fresh_db): From baafcec4a5e653d0c242f79fa5437591604d5292 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Jun 2021 19:05:12 -0700 Subject: [PATCH 020/102] Release 3.9 Refs #268 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f1d2261..c52449c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v3_9: + +3.9 (2021-06-11) +---------------- + +- New ``sqlite-utils schema`` command showing the full SQL schema for a database, see :ref:`Showing the schema (CLI)`. (`#268 `__) +- ``db.schema`` introspection property exposing the same feature to the Python library, see :ref:`Showing the schema (Python library) `. + .. _v3_8: 3.8 (2021-06-02) diff --git a/setup.py b/setup.py index 194baba..fd2e98a 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.8" +VERSION = "3.9" def get_long_description(): From b0f9d1e494c9891ce407e27b0f5c6deeea361d30 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Jun 2021 19:07:18 -0700 Subject: [PATCH 021/102] Fixed typo in 3.8 release notes --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c52449c..7be1f9d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,7 +16,7 @@ ---------------- - New ``sqlite-utils indexes`` command to list indexes in a database, see :ref:`cli_indexes`. (`#263 `__) -- ``table.indexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (`#261 `__) +- ``table.xindexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (`#261 `__) .. _v3_7: From b9629099ab21554a00eb11506201e6972600b93c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Jun 2021 19:57:21 -0700 Subject: [PATCH 022/102] Fix bug with upsert_all() and single column tables, closes #271 --- sqlite_utils/db.py | 28 +++++++++++++++------------- tests/test_create.py | 7 +++++++ tests/test_upsert.py | 7 +++++++ 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 53fae10..31d4cee 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1732,20 +1732,22 @@ class Table(Queryable): queries_and_params.append((sql, [record[col] for col in pks])) # UPDATE [book] SET [name] = 'Programming' WHERE [id] = 1001; set_cols = [col for col in all_columns if col not in pks] - sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format( - table=self.name, - pairs=", ".join( - "[{}] = {}".format(col, conversions.get(col, "?")) - for col in set_cols - ), - wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks), - ) - queries_and_params.append( - ( - sql2, - [record[col] for col in set_cols] + [record[pk] for pk in pks], + if set_cols: + sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format( + table=self.name, + pairs=", ".join( + "[{}] = {}".format(col, conversions.get(col, "?")) + for col in set_cols + ), + wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks), + ) + queries_and_params.append( + ( + sql2, + [record[col] for col in set_cols] + + [record[pk] for pk in pks], + ) ) - ) # We can populate .last_pk right here if num_records_processed == 1: self.last_pk = tuple(record[pk] for pk in pks) diff --git a/tests/test_create.py b/tests/test_create.py index 6e925dc..926a7d1 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -991,6 +991,13 @@ def test_insert_all_empty_list(fresh_db): assert 1 == fresh_db["t"].count +def test_insert_all_single_column(fresh_db): + table = fresh_db["table"] + table.insert_all([{"name": "Cleo"}], pk="name") + assert [{"name": "Cleo"}] == list(table.rows) + assert table.pks == ["name"] + + def test_create_with_a_null_column(fresh_db): record = {"name": "Name", "description": None} fresh_db["t"].insert(record) diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 100c6c1..9b1990e 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -21,6 +21,13 @@ def test_upsert_all(fresh_db): assert table.last_pk is None +def test_upsert_all_single_column(fresh_db): + table = fresh_db["table"] + table.upsert_all([{"name": "Cleo"}], pk="name") + assert [{"name": "Cleo"}] == list(table.rows) + assert table.pks == ["name"] + + def test_upsert_error_if_no_pk(fresh_db): table = fresh_db["table"] with pytest.raises(PrimaryKeyRequired): From 10f4913c144c4680c0feec576c2d4080f5005b33 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Jun 2021 19:59:08 -0700 Subject: [PATCH 023/102] Release 3.9.1 Refs #271 --- docs/changelog.rst | 7 +++++++ setup.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7be1f9d..3339f0d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog =========== +.. _3.9.1: + +3.9.1 (2021-06-12) +------------------ + +- Fixed bug when using ``table.upsert_all()`` to create a table with only a single column that is treated as the primary key. (`#271 `__) + .. _v3_9: 3.9 (2021-06-11) diff --git a/setup.py b/setup.py index fd2e98a..9c73ff4 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.9" +VERSION = "3.9.1" def get_long_description(): From a81c05d2350de2fb6931ee40fc540580db366bf9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Jun 2021 20:47:34 -0700 Subject: [PATCH 024/102] Clarify types that can be passed to .transform() --- docs/python-api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 725445c..45dac23 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1078,6 +1078,8 @@ To alter the type of a column, use the ``types=`` argument: # Convert the 'age' column to an integer, and 'weight' to a float table.transform(types={"age": int, "weight": float}) +See :ref:`python_api_add_column` for a list of available types. + The ``rename=`` parameter can rename columns: .. code-block:: python From a54b6788b0aa915c3e85a00220c6a841f094a1a3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Jun 2021 15:34:29 -0700 Subject: [PATCH 025/102] Sub-headings for .transform() --- docs/python-api.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 45dac23..b25eb21 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1071,6 +1071,9 @@ The ``table.transform()`` method can do all of these things, by implementing a m The ``.transform()`` method takes a number of parameters, all of which are optional. +Altering column types +--------------------- + To alter the type of a column, use the ``types=`` argument: .. code-block:: python @@ -1080,6 +1083,9 @@ To alter the type of a column, use the ``types=`` argument: See :ref:`python_api_add_column` for a list of available types. +Renaming columns +---------------- + The ``rename=`` parameter can rename columns: .. code-block:: python @@ -1087,6 +1093,9 @@ The ``rename=`` parameter can rename columns: # Rename 'age' to 'initial_age': table.transform(rename={"age": "initial_age"}) +Dropping columns +---------------- + To drop columns, pass them in the ``drop=`` set: .. code-block:: python @@ -1094,6 +1103,9 @@ To drop columns, pass them in the ``drop=`` set: # Drop the 'age' column: table.transform(drop={"age"}) +Changing primary keys +--------------------- + To change the primary key for a table, use ``pk=``. This can be passed a single column for a regular primary key, or a tuple of columns to create a compound primary key. Passing ``pk=None`` will remove the primary key and convert the table into a ``rowid`` table. .. code-block:: python @@ -1101,6 +1113,9 @@ To change the primary key for a table, use ``pk=``. This can be passed a single # Make `user_id` the new primary key table.transform(pk="user_id") +Changing not null status +------------------------ + You can change the ``NOT NULL`` status of columns by using ``not_null=``. You can pass this a set of columns to make those columns ``NOT NULL``: .. code-block:: python @@ -1118,6 +1133,9 @@ If you want to take existing ``NOT NULL`` columns and change them to allow null # Make age allow NULL and switch weight to being NOT NULL: table.transform(not_null={"age": False, "weight": True}) +Altering column defaults +------------------------ + The ``defaults=`` parameter can be used to set or change the defaults for different columns: .. code-block:: python @@ -1128,6 +1146,9 @@ The ``defaults=`` parameter can be used to set or change the defaults for differ # Now remove the default from that column: table.transform(defaults={"age": None}) +Changing column order +--------------------- + The ``column_order=`` parameter can be used to change the order of the columns. If you pass the names of a subset of the columns those will go first and columns you omitted will appear in their existing order after them. .. code-block:: python @@ -1135,6 +1156,9 @@ The ``column_order=`` parameter can be used to change the order of the columns. # Change column order table.transform(column_order=("name", "age", "id") +Dropping foreign key constraints +-------------------------------- + You can use ``.transform()`` to remove foreign key constraints from a table. This example drops two foreign keys - the one from ``places.country`` to ``country.id`` and the one from ``places.continent`` to ``continent.id``: From 78aebb6479420217454747870737bc593a259abc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Jun 2021 19:36:16 -0700 Subject: [PATCH 026/102] Link to --load-extension docs --- docs/cli.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index d39d367..c4dfc0d 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -85,9 +85,11 @@ You can run queries against a temporary in-memory database by passing ``:memory: $ sqlite-utils :memory: "select sqlite_version()" [{"sqlite_version()": "3.29.0"}] -You can load SQLite extension modules using the `--load-extension` option:: +You can load SQLite extension modules using the ``--load-extension`` option, see :ref:`cli_load_extension`. - $ sqlite-utils :memory: "select spatialite_version()" --load-extension=/usr/local/lib/mod_spatialite.dylib +:: + + $ sqlite-utils :memory: "select spatialite_version()" --load-extension=spatialite [{"spatialite_version()": "4.3.0a"}] .. _cli_json_values: From 287cdcae8908916687f2ecccc87c38549d004ac6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Jun 2021 21:40:28 -0700 Subject: [PATCH 027/102] Turn SQL errors into click errors --- sqlite_utils/cli.py | 5 ++++- tests/test_cli.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 6c40454..4208830 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1088,7 +1088,10 @@ def query( _load_extensions(db, load_extension) db.register_fts4_bm25() with db.conn: - cursor = db.execute(sql, dict(param)) + try: + cursor = db.execute(sql, dict(param)) + except sqlite3.OperationalError as e: + raise click.ClickException(str(e)) if cursor.description is None: # This was an update/insert headers = ["rows_affected"] diff --git a/tests/test_cli.py b/tests/test_cli.py index a73cdff..5cb4af0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1049,7 +1049,7 @@ def test_query_load_extension(use_spatialite_shortcut): # Without --load-extension: result = CliRunner().invoke(cli.cli, [":memory:", "select spatialite_version()"]) assert result.exit_code == 1 - assert "no such function: spatialite_version" in repr(result) + assert "no such function: spatialite_version" in result.output # With --load-extension: if use_spatialite_shortcut: load_extension = "spatialite" From fe1562e8a69872b27c1043c4b117d07623f16274 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 16 Jun 2021 09:36:32 -0700 Subject: [PATCH 028/102] Structure of most_common and least_common columns --- docs/cli.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index c4dfc0d..c953a0f 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -450,6 +450,21 @@ The ``_analyze_tables_`` table has the following schema:: PRIMARY KEY ([table], [column]) ); +The ``most_common`` and ``least_common`` columns will contain nested JSON arrays of the most commond and least common values that look like this:: + + [ + ["Del Libertador, Av", 5068], + ["Alberdi Juan Bautista Av.", 4612], + ["Directorio Av.", 4552], + ["Rivadavia, Av", 4532], + ["Yerbal", 4512], + ["Cosquín", 4472], + ["Estado Plurinacional de Bolivia", 4440], + ["Gordillo Timoteo", 4424], + ["Montiel", 4360], + ["Condarco", 4288] + ] + .. _cli_inserting_data: Inserting JSON data From ca2b26130f6c5fd030973ce593b02f08d19c9d84 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 16 Jun 2021 16:51:48 -0700 Subject: [PATCH 029/102] sqlite-utils dump command, closes #274 --- docs/cli.rst | 13 +++++++++++++ sqlite_utils/cli.py | 15 +++++++++++++++ tests/test_cli.py | 7 +++++++ 3 files changed, 35 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index c953a0f..a45f3b8 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1256,6 +1256,19 @@ You can disable WAL mode using ``disable-wal``:: Both of these commands accept one or more database files as arguments. +.. _cli_dump: + +Dumping the database to SQL +=========================== + +The ``dump`` command outputs a SQL dump of the schema and full contents of the specified database file:: + + $ sqlite-utils dump mydb.db + BEGIN TRANSACTION; + CREATE TABLE ... + ... + COMMIT; + .. _cli_load_extension: Loading SQLite extensions diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4208830..1e43a8b 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -308,6 +308,21 @@ def vacuum(path): sqlite_utils.Database(path).vacuum() +@cli.command() +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@load_extension_option +def dump(path, load_extension): + """Output a SQL dump of the schema and full contents of the database""" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + for line in db.conn.iterdump(): + click.echo(line) + + @cli.command(name="add-column") @click.argument( "path", diff --git a/tests/test_cli.py b/tests/test_cli.py index 5cb4af0..c68c7db 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -502,6 +502,13 @@ def test_vacuum(db_path): assert 0 == result.exit_code +def test_dump(db_path): + result = CliRunner().invoke(cli.cli, ["dump", db_path]) + assert result.exit_code == 0 + assert result.output.startswith("BEGIN TRANSACTION;") + assert result.output.strip().endswith("COMMIT;") + + @pytest.mark.parametrize("tables", ([], ["Gosh"], ["Gosh2"])) def test_optimize(db_path, tables): db = Database(db_path) From 276fc4297caf69da2deb4c1334dd49375a248dbb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 16 Jun 2021 16:54:16 -0700 Subject: [PATCH 030/102] Enable codecov.io, refs #275 --- .github/workflows/test-coverage.yml | 42 +++++++++++++++++++++++++++++ codecov.yml | 8 ++++++ 2 files changed, 50 insertions(+) create mode 100644 .github/workflows/test-coverage.yml create mode 100644 codecov.yml diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml new file mode 100644 index 0000000..31a37fa --- /dev/null +++ b/.github/workflows/test-coverage.yml @@ -0,0 +1,42 @@ +name: Calculate test coverage + +on: + push: + branches: + - main + pull_request: + branches: + - main +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e .[test] + python -m pip install pytest-cov + - name: Run tests + run: |- + ls -lah + cat .coveragerc + pytest --cov=sqlite_utils --cov-report xml:coverage.xml --cov-report term + ls -lah + - name: Upload coverage report + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.xml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..bfdc987 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true From f003d051e7e1738143a07312bc96e6c6cbc0db4c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 16 Jun 2021 16:55:53 -0700 Subject: [PATCH 031/102] Not using .coveragerc, refs #275 --- .github/workflows/test-coverage.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 31a37fa..4b99cc5 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -32,7 +32,6 @@ jobs: - name: Run tests run: |- ls -lah - cat .coveragerc pytest --cov=sqlite_utils --cov-report xml:coverage.xml --cov-report term ls -lah - name: Upload coverage report From a19ce1a4d0048d389411cfe11a5dbe4c503720e1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 16 Jun 2021 17:12:11 -0700 Subject: [PATCH 032/102] codecov badge, closes #275 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e8b8245..8c1b4ab 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/) [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest) [![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.datasette.io/en/latest/?badge=latest) +[![codecov](https://codecov.io/gh/simonw/sqlite-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/simonw/sqlite-utils) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE) Python CLI utility and library for manipulating SQLite databases. From aa652b6afe43d2b40fabc7a513c3e68866e030a5 Mon Sep 17 00:00:00 2001 From: Loren McIntyre Date: Fri, 18 Jun 2021 07:56:59 -0700 Subject: [PATCH 033/102] add -h support Closes #276 --- sqlite_utils/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 1e43a8b..390f43e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -16,6 +16,8 @@ import csv as csv_std import tabulate from .utils import file_progress, find_spatialite, sqlite3, decode_base64_values +CONTEXT_SETTINGS = dict(help_option_names=['-h','--help']) + VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB") UNICODE_ERROR = """ @@ -89,7 +91,7 @@ def load_extension_option(fn): )(fn) -@click.group(cls=DefaultGroup, default="query", default_if_no_args=True) +@click.group(cls=DefaultGroup, default="query", default_if_no_args=True, context_settings=CONTEXT_SETTINGS) @click.version_option() def cli(): "Commands for interacting with a SQLite database" From eea3851d40ea7e49cf27905cca19d200cf4cdbe4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 07:55:26 -0700 Subject: [PATCH 034/102] Added test, formatted with Black - refs #276, #277 --- sqlite_utils/cli.py | 9 +++++++-- tests/test_cli.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 390f43e..69e04e3 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -16,7 +16,7 @@ import csv as csv_std import tabulate from .utils import file_progress, find_spatialite, sqlite3, decode_base64_values -CONTEXT_SETTINGS = dict(help_option_names=['-h','--help']) +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB") @@ -91,7 +91,12 @@ def load_extension_option(fn): )(fn) -@click.group(cls=DefaultGroup, default="query", default_if_no_args=True, context_settings=CONTEXT_SETTINGS) +@click.group( + cls=DefaultGroup, + default="query", + default_if_no_args=True, + context_settings=CONTEXT_SETTINGS, +) @click.version_option() def cli(): "Commands for interacting with a SQLite database" diff --git a/tests/test_cli.py b/tests/test_cli.py index c68c7db..b085ea4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -24,6 +24,22 @@ def db_path(tmpdir): return path +@pytest.mark.parametrize( + "options", + ( + ["-h"], + ["--help"], + ["insert", "-h"], + ["insert", "--help"], + ), +) +def test_help(options): + result = CliRunner().invoke(cli.cli, options) + assert result.exit_code == 0 + assert result.output.startswith("Usage: ") + assert "-h, --help" in result.output + + def test_tables(db_path): result = CliRunner().invoke(cli.cli, ["tables", db_path]) assert '[{"table": "Gosh"},\n {"table": "Gosh2"}]' == result.output.strip() From 42ec59d8ee3fcfb8ac1affb772aed43b6e2a7381 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 08:00:52 -0700 Subject: [PATCH 035/102] sqlite-utils memory command for directly querying CSV/JSON data * Turn SQL errors into click errors * Initial CSV-only prototype of sqlite-utils memory, refs #272 * Implement --save plus tests for --save and --dump, refs #272 * Re-arranged CLI query documentation, refs #272 * Re-organized CLI query docs, refs #272 * Docs for --save and --dump plus made SQL optional for those, refs #273 * Replaced one last :memory: example * Documented --attach option for memory command, refs #272 * Improved arrangement of CLI query documentation --- docs/changelog.rst | 2 +- docs/cli.rst | 225 ++++++++++++++++++++++++++++++--------- sqlite_utils/cli.py | 105 +++++++++++++++++- tests/test_cli_memory.py | 72 +++++++++++++ 4 files changed, 351 insertions(+), 53 deletions(-) create mode 100644 tests/test_cli_memory.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 3339f0d..879d19b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -47,7 +47,7 @@ This release adds the ability to execute queries joining data from more than one database file - similar to the cross database querying feature introduced in `Datasette 0.55 `__. - The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation `. (`#113 `__) -- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (`#236 `__) +- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (`#236 `__) .. _v3_5: diff --git a/docs/cli.rst b/docs/cli.rst index a45f3b8..b7f0313 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -8,25 +8,31 @@ The ``sqlite-utils`` command-line tool can be used to manipulate SQLite database .. contents:: :local: -.. _cli_query_json: +.. _cli_query: -Running queries and returning JSON -================================== +Running SQL queries +=================== -You can execute a SQL query against a database and get the results back as JSON like this:: +The ``sqlite-utils query`` command lets you run queries directly against a SQLite database file. This is the default subcommand, so the following two examples work the same way:: $ sqlite-utils query dogs.db "select * from dogs" + $ sqlite-utils dogs.db "select * from dogs" + +.. _cli_query_json: + +Returning JSON +-------------- + +The default format returned for queries is JSON:: + + $ sqlite-utils dogs.db "select * from dogs" [{"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}] -This is the default command for ``sqlite-utils``, so you can instead use this:: +.. _cli_query_nl: - $ sqlite-utils dogs.db "select * from dogs" - -You can pass named parameters to the query using ``-p``:: - - $ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6 - [{":num * :num2": 30}] +Newline-delimited JSON +~~~~~~~~~~~~~~~~~~~~~~ Use ``--nl`` to get back newline-delimited JSON objects:: @@ -34,6 +40,11 @@ Use ``--nl`` to get back newline-delimited JSON objects:: {"id": 1, "age": 4, "name": "Cleo"} {"id": 2, "age": 2, "name": "Pancakes"} +.. _cli_query_arrays: + +JSON arrays +~~~~~~~~~~~ + You can use ``--arrays`` to request arrays instead of objects:: $ sqlite-utils dogs.db "select * from dogs" --arrays @@ -62,6 +73,11 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] +.. _cli_query_binary_json: + +Binary data in JSON +~~~~~~~~~~~~~~~~~~~ + Binary strings are not valid JSON, so BLOB columns containing binary data will be returned as a JSON object containing base64 encoded data, that looks like this:: $ sqlite-utils dogs.db "select name, content from images" | python -mjson.tool @@ -75,27 +91,11 @@ Binary strings are not valid JSON, so BLOB columns containing binary data will b } ] -If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows:: - - $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" - [{"rows_affected": 1}] - -You can run queries against a temporary in-memory database by passing ``:memory:`` as the filename:: - - $ sqlite-utils :memory: "select sqlite_version()" - [{"sqlite_version()": "3.29.0"}] - -You can load SQLite extension modules using the ``--load-extension`` option, see :ref:`cli_load_extension`. - -:: - - $ sqlite-utils :memory: "select spatialite_version()" --load-extension=spatialite - [{"spatialite_version()": "4.3.0a"}] .. _cli_json_values: Nested JSON values ------------------- +~~~~~~~~~~~~~~~~~~ If one of your columns contains JSON, by default it will be returned as an escaped string:: @@ -126,24 +126,10 @@ You can use the ``--json-cols`` option to automatically detect these JSON column } ] -.. _cli_attach: - -Attaching additional databases ------------------------------- - -SQLite supports cross-database SQL queries, which can join data from tables in more than one database file. - -You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk. - -This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database:: - - sqlite-utils dogs.db --attach books books.db \ - 'select * from sqlite_master union all select * from books.sqlite_master' - .. _cli_query_csv: -Running queries and returning CSV -================================= +Returning CSV or TSV +-------------------- You can use the ``--csv`` option to return results as CSV:: @@ -167,8 +153,8 @@ Use ``--tsv`` instead of ``--csv`` to get back tab-separated values:: .. _cli_query_table: -Running queries and outputting a table -====================================== +Table-formatted output +---------------------- You can use the ``--table`` option (or ``-t`` shortcut) to output query results as a table:: @@ -192,8 +178,8 @@ For a full list of table format options, run ``sqlite-utils query --help``. .. _cli_query_raw: -Returning raw data from a query, such as binary content -======================================================= +Returning raw data, such as binary content +------------------------------------------ If your table contains binary data in a ``BLOB`` you can use the ``--raw`` option to output specific columns directly to standard out. @@ -201,6 +187,145 @@ For example, to retrieve a binary image from a ``BLOB`` column and store it in a $ sqlite-utils photos.db "select contents from photos where id=1" --raw > myphoto.jpg + +.. _cli_query_parameters: + +Using named parameters +---------------------- + +You can pass named parameters to the query using ``-p``:: + + $ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6 + [{":num * :num2": 30}] + +These will be correctly quoted and escaped in the SQL query, providing a safe way to combine other values with SQL. + +.. _cli_query_update_insert_delete: + +UPDATE, INSERT and DELETE +------------------------- + +If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows:: + + $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" + [{"rows_affected": 1}] + +SQLite extensions +----------------- + +You can load SQLite extension modules using the ``--load-extension`` option, see :ref:`cli_load_extension`. + +:: + + $ sqlite-utils dogs.db "select spatialite_version()" --load-extension=spatialite + [{"spatialite_version()": "4.3.0a"}] + +.. _cli_query_attach: + +Attaching additional databases +------------------------------ + +SQLite supports cross-database SQL queries, which can join data from tables in more than one database file. + +You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk. + +This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database:: + + sqlite-utils dogs.db --attach books books.db \ + 'select * from sqlite_master union all select * from books.sqlite_master' + +.. _cli_query_memory: + +Querying CSV data directly using an in-memory database +====================================================== + +The ``sqlite-utils memory`` command works similar to ``sqlite-utils query``, but allows you to execute queries against an in-memory database. + +You can also pass this command CSV files which will be loaded into a temporary in-memory table, allowing you to execute SQL against that data without a separate step to first convert it to SQLite. + +Without any extra arguments, this command executes SQL against the in-memory database directly:: + + $ sqlite-utils memory 'select sqlite_version()' + [{"sqlite_version()": "3.35.5"}] + +It takes all of the same formatting options as :ref:`sqlite-utils query `: ``--csv`` and ``--csv`` and ``--table`` and ``--nl``:: + + $ sqlite-utils memory 'select sqlite_version()' --csv + sqlite_version() + 3.35.5 + $ sqlite-utils memory 'select sqlite_version()' --table --fmt grid + +--------------------+ + | sqlite_version() | + +====================+ + | 3.35.5 | + +--------------------+ + +.. _cli_query_memory_csv: + +Running queries directly against CSV +------------------------------------ + +If you have data in CSV format you can load it into an in-memory SQLite database and run queries against it directly in a single command using ``sqlite-utils memory`` like this:: + + $ sqlite-utils memory data.csv "select * from data" + +You can pass multiple files to the command if you want to run joins between different CSV files:: + + $ sqlite-utils memory one.csv two.csv "select * from one join two on one.id = two.other_id" + +The in-memory tables will be named after the CSV files without their ``.csv`` extension. The tool also sets up aliases for those tables (using SQL views) as ``t1``, ``t2`` and so on, or you can use the alias ``t`` to refer to the first table:: + + $ sqlite-utils memory example.csv "select * from t" + +To read from standard input, use ``-`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name:: + + $ cat example.csv | sqlite-utils memory - "select * from stdin" + +.. _cli_query_memory_attach: + +Joining in-memory data against existing databases using \-\-attach +------------------------------------------------------------------ + +The :ref:`attach option ` can be used to attach database files to the in-memory connection, enabling joins between in-memory data loaded from a file and tables in existing SQLite database files. An example:: + + $ echo "id\n1\n3\n5" | sqlite-utils memory - --attach trees trees.db \ + "select * from trees.trees where rowid in (select id from stdin)" + +Here the ``--attach trees trees.db`` option makes the ``trees.db`` database available with an alias of ``trees``. + +``select * from trees.trees where ...`` can then query the ``trees`` table in that database. + +The CSV data that was piped into the script is available in the ``stdin`` table, so ``... where rowid in (select id from stdin)`` can be used to return rows from the ``trees`` table that match IDs that were piped in as CSV content. + +.. _cli_query_memory_dump_save: + +\-\-dump and \-\-save +--------------------- + +You can dump out the SQL used for the temporary in-memory database, complete with all imported data, using the ``--dump`` option:: + + % sqlite-utils memory dogs.csv --dump + BEGIN TRANSACTION; + CREATE TABLE [dogs] ( + [rowid] TEXT, + [id] TEXT, + [dog_age] TEXT, + [name] TEXT + ); + INSERT INTO "dogs" VALUES('1','1','4','Cleo'); + INSERT INTO "dogs" VALUES('2','2','2','Pancakes'); + INSERT INTO "dogs" VALUES('3','2','3','Pancakes'); + CREATE VIEW t1 AS select * from [dogs]; + CREATE VIEW t AS select * from [dogs]; + COMMIT; + +Passing ``--save other.db`` will instead use that SQL to populate a new database file:: + + % sqlite-utils memory dogs.csv --save dogs.db + +These features are mainly intented as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`. + + .. _cli_rows: Returning all rows in a table @@ -270,7 +395,7 @@ Use ``--schema`` to include the schema of each table:: [age] INTEGER, [name] TEXT) -The ``--nl``, ``--csv``, ``--tsv`` and ``--table`` options are all available. +The ``--nl``, ``--csv``, ``--tsv``, ``--table`` and ``--fmt`` options are also available. .. _cli_views: @@ -1280,5 +1405,5 @@ This option can be applied multiple times to load multiple extensions. Since `SpatiaLite `__ is commonly used with SQLite, the value ``spatialite`` is special: it will search for SpatiaLite in the most common installation locations, saving you from needing to remember exactly where that module is located:: - $ sqlite-utils :memory: "select spatialite_version()" --load-extension=spatialite + $ sqlite-utils memory "select spatialite_version()" --load-extension=spatialite [{"spatialite_version()": "4.3.0a"}] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 69e04e3..8fe37c8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -716,11 +716,11 @@ def insert_upsert_implementation( raise click.ClickException("Use just one of --nl, --csv or --tsv") if encoding and not (csv or tsv): raise click.ClickException("--encoding must be used with --csv or --tsv") + if pk and len(pk) == 1: + pk = pk[0] encoding = encoding or "utf-8-sig" buffered = io.BufferedReader(json_file, buffer_size=4096) decoded = io.TextIOWrapper(buffered, encoding=encoding) - if pk and len(pk) == 1: - pk = pk[0] if csv or tsv: if sniff: # Read first 2048 bytes and use that to detect @@ -1109,6 +1109,107 @@ def query( db.attach(alias, attach_path) _load_extensions(db, load_extension) db.register_fts4_bm25() + + _execute_query( + db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + ) + + +@cli.command() +@click.argument( + "paths", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=True), + required=False, + nargs=-1, +) +@click.argument("sql") +@click.option( + "--attach", + type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), + multiple=True, + help="Additional databases to attach - specify alias and filepath", +) +@output_options +@click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") +@click.option( + "-p", + "--param", + multiple=True, + type=(str, str), + help="Named :parameters for SQL query", +) +@click.option("--dump", is_flag=True, help="Dump SQL for in-memory database") +@click.option( + "--save", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + help="Save in-memory database to this file", +) +@load_extension_option +def memory( + paths, + sql, + attach, + nl, + arrays, + csv, + tsv, + no_headers, + table, + fmt, + json_cols, + raw, + param, + dump, + save, + load_extension, +): + "Execute SQL query against an in-memory database, optionally populated by imported data" + db = sqlite_utils.Database(memory=True) + # If --dump or --save used but no paths detected, assume SQL query is a path: + if (dump or save) and not paths: + paths = [sql] + sql = None + for i, path in enumerate(paths): + if path == "-": + csv_fp = sys.stdin + csv_table = "stdin" + else: + csv_path = pathlib.Path(path) + csv_table = csv_path.stem + csv_fp = csv_path.open() + db[csv_table].insert_all(csv_std.DictReader(csv_fp)) + # Add convenient t / t1 / t2 views + view_names = ["t{}".format(i + 1)] + if i == 0: + view_names.append("t") + for view_name in view_names: + if not db[view_name].exists(): + db.create_view(view_name, "select * from [{}]".format(csv_table)) + + if dump: + for line in db.conn.iterdump(): + click.echo(line) + return + + if save: + db2 = sqlite_utils.Database(save) + for line in db.conn.iterdump(): + db2.execute(line) + return + + for alias, attach_path in attach: + db.attach(alias, attach_path) + _load_extensions(db, load_extension) + db.register_fts4_bm25() + + _execute_query( + db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + ) + + +def _execute_query( + db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols +): with db.conn: try: cursor = db.execute(sql, dict(param)) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py new file mode 100644 index 0000000..f018837 --- /dev/null +++ b/tests/test_cli_memory.py @@ -0,0 +1,72 @@ +from sqlite_utils import cli, Database +from click.testing import CliRunner +import pytest + + +def test_memory_basic(): + result = CliRunner().invoke(cli.cli, ["memory", "select 1 + 1"]) + assert result.exit_code == 0 + assert result.output.strip() == '[{"1 + 1": 2}]' + + +@pytest.mark.parametrize("sql_from", ("test", "t", "t1")) +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_csv(tmpdir, sql_from, use_stdin): + content = "id,name\n1,Cleo\n2,Bants" + input = None + if use_stdin: + input = content + csv_path = "-" + if sql_from == "test": + sql_from = "stdin" + else: + csv_path = str(tmpdir / "test.csv") + open(csv_path, "w").write(content) + result = CliRunner().invoke( + cli.cli, + ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], + input=input, + ) + assert result.exit_code == 0 + assert ( + result.output.strip() + == '{"id": "1", "name": "Cleo"}\n{"id": "2", "name": "Bants"}' + ) + + +@pytest.mark.parametrize("extra_args", ([], ["select 1"])) +def test_memory_dump(extra_args): + result = CliRunner().invoke( + cli.cli, + ["memory", "-"] + extra_args + ["--dump"], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + assert result.output.strip() == ( + "BEGIN TRANSACTION;\n" + "CREATE TABLE [stdin] (\n" + " [id] TEXT,\n" + " [name] TEXT\n" + ");\n" + "INSERT INTO \"stdin\" VALUES('1','Cleo');\n" + "INSERT INTO \"stdin\" VALUES('2','Bants');\n" + "CREATE VIEW t1 AS select * from [stdin];\n" + "CREATE VIEW t AS select * from [stdin];\n" + "COMMIT;" + ) + + +@pytest.mark.parametrize("extra_args", ([], ["select 1"])) +def test_memory_save(tmpdir, extra_args): + save_to = str(tmpdir / "save.db") + result = CliRunner().invoke( + cli.cli, + ["memory", "-"] + extra_args + ["--save", save_to], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + db = Database(save_to) + assert list(db["stdin"].rows) == [ + {"id": "1", "name": "Cleo"}, + {"id": "2", "name": "Bants"}, + ] From 7684bbf0976431371541bc91136779b4948dbabf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 08:29:41 -0700 Subject: [PATCH 036/102] --encoding option for sqlite-utils memory, closes #280 Refs #272 --- docs/cli.rst | 6 ++++++ sqlite_utils/cli.py | 13 +++++++++++-- tests/test_cli_memory.py | 42 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index b7f0313..69d00a0 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -281,6 +281,12 @@ To read from standard input, use ``-`` as the filename - then use ``stdin`` or ` $ cat example.csv | sqlite-utils memory - "select * from stdin" +Incoming CSV data will be assumed to use ``utf-8``. If your data uses a different character encoding you can specify that with ``--encoding``:: + + $ cat example.csv | sqlite-utils memory - "select * from stdin" --encoding=latin-1 + +If you are joining across multiple CSV files they must all use the same encoding. + .. _cli_query_memory_attach: Joining in-memory data against existing databases using \-\-attach diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8fe37c8..eac76d4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1138,6 +1138,10 @@ def query( type=(str, str), help="Named :parameters for SQL query", ) +@click.option( + "--encoding", + help="Character encoding for CSV input, defaults to utf-8", +) @click.option("--dump", is_flag=True, help="Dump SQL for in-memory database") @click.option( "--save", @@ -1159,6 +1163,7 @@ def memory( json_cols, raw, param, + encoding, dump, save, load_extension, @@ -1171,13 +1176,17 @@ def memory( sql = None for i, path in enumerate(paths): if path == "-": - csv_fp = sys.stdin + csv_fp = sys.stdin.buffer csv_table = "stdin" else: csv_path = pathlib.Path(path) csv_table = csv_path.stem csv_fp = csv_path.open() - db[csv_table].insert_all(csv_std.DictReader(csv_fp)) + + encoding = encoding or "utf-8-sig" + decoded_fp = io.TextIOWrapper(csv_fp, encoding=encoding) + + db[csv_table].insert_all(csv_std.DictReader(decoded_fp)) # Add convenient t / t1 / t2 views view_names = ["t{}".format(i + 1)] if i == 0: diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index f018837..f45abdf 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -1,6 +1,7 @@ from sqlite_utils import cli, Database from click.testing import CliRunner import pytest +import json def test_memory_basic(): @@ -34,6 +35,47 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): ) +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_csv_encoding(tmpdir, use_stdin): + latin1_csv = ( + b"date,name,latitude,longitude\n" b"2020-03-04,S\xe3o Paulo,-23.561,-46.645\n" + ) + input = None + if use_stdin: + input = latin1_csv + csv_path = "-" + sql_from = "stdin" + else: + csv_path = str(tmpdir / "test.csv") + with open(csv_path, "wb") as fp: + fp.write(latin1_csv) + sql_from = "test" + # Without --encoding should error: + assert ( + CliRunner() + .invoke( + cli.cli, + ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], + input=input, + ) + .exit_code + == 1 + ) + # With --encoding should work: + result = CliRunner().invoke( + cli.cli, + ["memory", "-", "select * from stdin", "--encoding", "latin-1", "--nl"], + input=latin1_csv, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == { + "date": "2020-03-04", + "name": "S\u00e3o Paulo", + "latitude": "-23.561", + "longitude": "-46.645", + } + + @pytest.mark.parametrize("extra_args", ([], ["select 1"])) def test_memory_dump(extra_args): result = CliRunner().invoke( From 93594ce15b01f5ceba3bde65abe57ed28dfde9b4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 08:36:09 -0700 Subject: [PATCH 037/102] Open CSV in binary mode, refs #280 --- sqlite_utils/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index eac76d4..c5176f8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1181,7 +1181,7 @@ def memory( else: csv_path = pathlib.Path(path) csv_table = csv_path.stem - csv_fp = csv_path.open() + csv_fp = csv_path.open("rb") encoding = encoding or "utf-8-sig" decoded_fp = io.TextIOWrapper(csv_fp, encoding=encoding) From 00e4bd5ff18ef4c3db6c1d67e2b974131c80d65c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 20:11:54 -0700 Subject: [PATCH 038/102] TSV and JSON support for sqlite-utils memory Closes #281, closes #279, refs #272 --- docs/cli.rst | 51 +++++++++++++++++-------- sqlite_utils/cli.py | 24 ++++++++---- sqlite_utils/utils.py | 62 ++++++++++++++++++++++++++++++- tests/test_cli_memory.py | 80 ++++++++++++++++++++++++++++++++++++++-- 4 files changed, 191 insertions(+), 26 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 69d00a0..c6067ac 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -234,21 +234,21 @@ This example attaches the ``books.db`` database under the alias ``books`` and th sqlite-utils dogs.db --attach books books.db \ 'select * from sqlite_master union all select * from books.sqlite_master' -.. _cli_query_memory: +.. _cli_memory: -Querying CSV data directly using an in-memory database -====================================================== +Querying data directly using an in-memory database +================================================== The ``sqlite-utils memory`` command works similar to ``sqlite-utils query``, but allows you to execute queries against an in-memory database. -You can also pass this command CSV files which will be loaded into a temporary in-memory table, allowing you to execute SQL against that data without a separate step to first convert it to SQLite. +You can also pass this command CSV or JSON files which will be loaded into a temporary in-memory table, allowing you to execute SQL against that data without a separate step to first convert it to SQLite. Without any extra arguments, this command executes SQL against the in-memory database directly:: $ sqlite-utils memory 'select sqlite_version()' [{"sqlite_version()": "3.35.5"}] -It takes all of the same formatting options as :ref:`sqlite-utils query `: ``--csv`` and ``--csv`` and ``--table`` and ``--nl``:: +It takes all of the same output formatting options as :ref:`sqlite-utils query `: ``--csv`` and ``--csv`` and ``--table`` and ``--nl``:: $ sqlite-utils memory 'select sqlite_version()' --csv sqlite_version() @@ -260,24 +260,28 @@ It takes all of the same formatting options as :ref:`sqlite-utils query ` - so either a single JSON object (treated as a single row) or a list of JSON objects. + +CSV data can be comma- or tab- delimited. + +The in-memory tables will be named after the files without their extensions. The tool also sets up aliases for those tables (using SQL views) as ``t1``, ``t2`` and so on, or you can use the alias ``t`` to refer to the first table:: $ sqlite-utils memory example.csv "select * from t" -To read from standard input, use ``-`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name:: +To read from standard input, use either ``-`` or ``stdin`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name:: $ cat example.csv | sqlite-utils memory - "select * from stdin" @@ -287,7 +291,24 @@ Incoming CSV data will be assumed to use ``utf-8``. If your data uses a differen If you are joining across multiple CSV files they must all use the same encoding. -.. _cli_query_memory_attach: +.. _cli_memory_explicit: + +Explicitly specifying the format +-------------------------------- + +By default, ``sqlite-utils memory`` will attempt to detect the incoming data format (JSON, TSV or CSV) automatically. + +You can instead specify an explicit format by adding a ``:csv``, ``:tsv``, ``:json`` or ``:nl`` (for newline-delimited JSON) suffix to the filename. For example:: + + $ sqlite-utils memory one.dat:csv two.dat:nl "select * from one union select * from two" + +Here the contents of ``one.dat`` will be treated as CSV and the contents of ``two.dat`` will be treated as newline-delimited JSON. + +To explicitly specify the format for data piped into the tool on standard input, use ``stdin:format`` - for example:: + + $ cat one.dat | sqlite-utils memory stdin:csv "select * from stdin" + +.. _cli_memory_attach: Joining in-memory data against existing databases using \-\-attach ------------------------------------------------------------------ @@ -303,7 +324,7 @@ Here the ``--attach trees trees.db`` option makes the ``trees.db`` database avai The CSV data that was piped into the script is available in the ``stdin`` table, so ``... where rowid in (select id from stdin)`` can be used to return rows from the ``trees`` table that match IDs that were piped in as CSV content. -.. _cli_query_memory_dump_save: +.. _cli_memory_dump_save: \-\-dump and \-\-save --------------------- diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c5176f8..a4c1789 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -14,7 +14,14 @@ import os import sys import csv as csv_std import tabulate -from .utils import file_progress, find_spatialite, sqlite3, decode_base64_values +from .utils import ( + file_progress, + find_spatialite, + sqlite3, + decode_base64_values, + rows_from_file, + Format, +) CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) @@ -1175,18 +1182,21 @@ def memory( paths = [sql] sql = None for i, path in enumerate(paths): - if path == "-": + # Path may have a :format suffix + if ":" in path and path.rsplit(":", 1)[-1].upper() in Format.__members__: + path, suffix = path.rsplit(":", 1) + format = Format[suffix.upper()] + else: + format = None + if path in ("-", "stdin"): csv_fp = sys.stdin.buffer csv_table = "stdin" else: csv_path = pathlib.Path(path) csv_table = csv_path.stem csv_fp = csv_path.open("rb") - - encoding = encoding or "utf-8-sig" - decoded_fp = io.TextIOWrapper(csv_fp, encoding=encoding) - - db[csv_table].insert_all(csv_std.DictReader(decoded_fp)) + rows = rows_from_file(csv_fp, format=format, encoding=encoding) + db[csv_table].insert_all(rows, alter=True) # Add convenient t / t1 / t2 views view_names = ["t{}".format(i + 1)] if i == 0: diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 9d5dac6..8a0b9ab 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -1,8 +1,13 @@ import base64 -import click import contextlib +import csv +import enum import io +import json import os +from typing import Generator + +import click try: import pysqlite3 as sqlite3 @@ -111,3 +116,58 @@ def file_progress(file, silent=False, **kwargs): file_length = os.path.getsize(file.name) with click.progressbar(length=file_length, **kwargs) as bar: yield UpdateWrapper(file, bar.update) + + +class Format(enum.Enum): + CSV = 1 + TSV = 2 + JSON = 3 + NL = 4 + + +class RowsFromFileError(Exception): + pass + + +class RowsFromFileBadJSON(RowsFromFileError): + pass + + +def rows_from_file( + fp, + format=None, + dialect=None, + encoding=None, +) -> Generator[dict, None, None]: + if format == Format.JSON: + decoded = json.load(fp) + if isinstance(decoded, dict): + decoded = [decoded] + if not isinstance(decoded, list): + raise RowsFromFileBadJSON("JSON must be a list or a dictionary") + yield from decoded + elif format == Format.NL: + yield from (json.loads(line) for line in fp if line.strip()) + elif format == Format.CSV: + decoded_fp = io.TextIOWrapper(fp, encoding=encoding or "utf-8-sig") + yield from csv.DictReader(decoded_fp, dialect=dialect) + elif format == Format.TSV: + yield from rows_from_file( + fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding + ) + elif format is None: + # Detect the format, then call this recursively + buffered = io.BufferedReader(fp, buffer_size=4096) + first_bytes = buffered.peek(2048).strip() + if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"): + # TODO: Detect newline-JSON + yield from rows_from_file(buffered, format=Format.JSON) + else: + dialect = csv.Sniffer().sniff( + first_bytes.decode(encoding or "utf-8-sig", "ignore") + ) + yield from rows_from_file( + buffered, format=Format.CSV, dialect=dialect, encoding=encoding + ) + else: + raise RowsFromFileError("Bad format") diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index f45abdf..c91beaf 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -1,8 +1,10 @@ -from sqlite_utils import cli, Database -from click.testing import CliRunner -import pytest import json +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli + def test_memory_basic(): result = CliRunner().invoke(cli.cli, ["memory", "select 1 + 1"]) @@ -35,6 +37,78 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): ) +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_tsv(tmpdir, use_stdin): + data = "id\tname\n1\tCleo\n2\tBants" + if use_stdin: + input = data + path = "stdin:tsv" + sql_from = "stdin" + else: + input = None + path = str(tmpdir / "chickens.tsv") + open(path, "w").write(data) + path = path + ":tsv" + sql_from = "chickens" + result = CliRunner().invoke( + cli.cli, + ["memory", path, "select * from {}".format(sql_from)], + input=data, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + {"id": "1", "name": "Cleo"}, + {"id": "2", "name": "Bants"}, + ] + + +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_json(tmpdir, use_stdin): + data = '[{"name": "Bants"}, {"name": "Dori", "age": 1}]' + if use_stdin: + input = data + path = "stdin:json" + sql_from = "stdin" + else: + input = None + path = str(tmpdir / "chickens.json") + open(path, "w").write(data) + path = path + ":json" + sql_from = "chickens" + result = CliRunner().invoke( + cli.cli, + ["memory", path, "select * from {}".format(sql_from)], + input=input, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + {"name": "Bants", "age": None}, + {"name": "Dori", "age": 1}, + ] + + +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_json_nl(tmpdir, use_stdin): + data = '{"name": "Bants"}\n\n{"name": "Dori"}' + if use_stdin: + input = data + path = "stdin:nl" + sql_from = "stdin" + else: + input = None + path = str(tmpdir / "chickens.json") + open(path, "w").write(data) + path = path + ":nl" + sql_from = "chickens" + result = CliRunner().invoke( + cli.cli, + ["memory", path, "select * from {}".format(sql_from)], + input=data, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [{"name": "Bants"}, {"name": "Dori"}] + + @pytest.mark.parametrize("use_stdin", (True, False)) def test_memory_csv_encoding(tmpdir, use_stdin): latin1_csv = ( From 1091a9cbd804504efa8e1126226759e736e3ccdf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 20:14:12 -0700 Subject: [PATCH 039/102] Add sqlite-utils memory to the README, refs #272 --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8c1b4ab..4c738ab 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,11 @@ You can import JSON data into a new database table like this: Or for data in a CSV file: - $ sqlite-utils insert dogs.db dogs docs.csv --csv + $ sqlite-utils insert dogs.db dogs dogs.csv --csv + +`sqlite-utils memory` lets you import CSV or JSON data into an in-memory database and run SQL queries against it in a single command: + + $ cat dogs.csv | sqlite-utils memory - "select name, age from dogs" See the [full CLI documentation](https://sqlite-utils.datasette.io/en/stable/cli.html) for comprehensive coverage of many more commands. From 59992d2feedf964d9f7c72110755f36de49e1c8b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 20:20:56 -0700 Subject: [PATCH 040/102] Better help text for 'sqlite-utils memory', refs #272 --- sqlite_utils/cli.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index a4c1789..c5f50a1 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1175,7 +1175,28 @@ def memory( save, load_extension, ): - "Execute SQL query against an in-memory database, optionally populated by imported data" + """Execute SQL query against an in-memory database, optionally populated by imported data + + To import data from CSV, TSV or JSON files pass them on the command-line: + + \b + sqlite-utils memory one.csv two.json \\ + "select * from one join two on one.two_id = two.id" + + For data piped into the tool from standard input, use "-" or "stdin": + + \b + cat animals.csv | sqlite-utils memory - \\ + "select * from stdin where species = 'dog'" + + The format of the data will be automatically detected. You can specify the format + explicitly using :json, :csv, :tsv or :nl (for newline-delimited JSON) - for example: + + \b + cat animals.csv | sqlite-utils memory stdin:csv places.dat:nl \\ + "select * from stdin where place_id in (select id from places)" + + """ db = sqlite_utils.Database(memory=True) # If --dump or --save used but no paths detected, assume SQL query is a path: if (dump or save) and not paths: From fd9867d145c11a6be6c4049e0383832d0e856f4b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 21:18:58 -0700 Subject: [PATCH 041/102] sqlite-utils insert --detect-types option, refs #282 --- docs/cli.rst | 26 +++++++++++++++ sqlite_utils/cli.py | 18 ++++++++++ sqlite_utils/utils.py | 76 ++++++++++++++++++++++++++++++++++++++++++- tests/test_cli.py | 31 ++++++++++++++++++ 4 files changed, 150 insertions(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index c6067ac..bad2d51 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -726,6 +726,32 @@ Data is expected to be encoded as Unicode UTF-8. If your data is an another char A progress bar is displayed when inserting data from a file. You can hide the progress bar using the ``--silent`` option. +By default every column inserted from a CSV or TSV file will be of type ``TEXT``. To automatically detect column types - resulting in a mix of ``TEXT``, ``INTEGER`` and ``FLOAT`` columns, use the ``--detect-types`` option (or its shortcut ``-d``). + +For example, given a ``creatures.csv`` file containing this:: + + name,age,weight + Cleo,6,45.5 + Dori,1,3.5 + +The following command:: + + $ sqlite-utils insert creatures.db creatures creatures.tsv --csv --detect-types + +Will produce this schema:: + + $ sqlite-utils schema creatures.db + CREATE TABLE "creatures" ( + [rowid] INTEGER PRIMARY KEY, + [name] TEXT, + [age] INTEGER, + [weight] FLOAT + ); + +You can set the ``SQLITE_UTILS_DETECT_TYPES`` environment variable if you want ``--detect-types`` to be the default behavior:: + + $ export SQLITE_UTILS_DETECT_TYPES=1 + .. _cli_insert_csv_tsv_delimiter: Alternative delimiters and quote characters diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c5f50a1..96adaee 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -21,6 +21,7 @@ from .utils import ( decode_base64_values, rows_from_file, Format, + TypeTracker, ) CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) @@ -683,6 +684,13 @@ def insert_upsert_options(fn): "--encoding", help="Character encoding for input, defaults to utf-8", ), + click.option( + "-d", + "--detect-types", + is_flag=True, + envvar="SQLITE_UTILS_DETECT_TYPES", + help="Detect types for columns in CSV/TSV data", + ), load_extension_option, click.option("--silent", is_flag=True, help="Do not show progress bar"), ) @@ -712,6 +720,7 @@ def insert_upsert_implementation( not_null=None, default=None, encoding=None, + detect_types=None, load_extension=None, silent=False, ): @@ -728,6 +737,7 @@ def insert_upsert_implementation( encoding = encoding or "utf-8-sig" buffered = io.BufferedReader(json_file, buffer_size=4096) decoded = io.TextIOWrapper(buffered, encoding=encoding) + tracker = None if csv or tsv: if sniff: # Read first 2048 bytes and use that to detect @@ -749,6 +759,9 @@ def insert_upsert_implementation( else: headers = first_row docs = (dict(zip(headers, row)) for row in reader) + if detect_types: + tracker = TypeTracker() + docs = tracker.wrap(docs) else: try: if nl: @@ -781,6 +794,8 @@ def insert_upsert_implementation( "{}\n\nTry using --alter to add additional columns".format(e.args[0]) ) raise + if tracker is not None: + db[table].transform(types=tracker.types) @cli.command() @@ -815,6 +830,7 @@ def insert( batch_size, alter, encoding, + detect_types, load_extension, silent, ignore, @@ -849,6 +865,7 @@ def insert( replace=replace, truncate=truncate, encoding=encoding, + detect_types=detect_types, load_extension=load_extension, silent=silent, not_null=not_null, @@ -877,6 +894,7 @@ def upsert( not_null, default, encoding, + detect_types, load_extension, silent, ): diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 8a0b9ab..ffc727e 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -110,7 +110,16 @@ class UpdateWrapper: @contextlib.contextmanager def file_progress(file, silent=False, **kwargs): - if silent or file.fileno() == 0: # 0 = stdin + if silent: + yield file + return + # file.fileno() throws an exception in our test suite + try: + fileno = file.fileno() + except io.UnsupportedOperation: + yield file + return + if fileno == 0: # 0 means stdin yield file else: file_length = os.path.getsize(file.name) @@ -171,3 +180,68 @@ def rows_from_file( ) else: raise RowsFromFileError("Bad format") + + +class TypeTracker: + def __init__(self): + self.trackers = {} + + def wrap(self, iterator): + for row in iterator: + for key, value in row.items(): + tracker = self.trackers.setdefault(key, ValueTracker()) + tracker.evaluate(value) + yield row + + @property + def types(self): + return {key: tracker.guessed_type for key, tracker in self.trackers.items()} + + +class ValueTracker: + def __init__(self): + self.couldbe = {key: getattr(self, "test_" + key) for key in self.get_tests()} + + @classmethod + def get_tests(cls): + return [ + key.split("test_")[-1] + for key in cls.__dict__.keys() + if key.startswith("test_") + ] + + def test_integer(self, value): + try: + int(value) + return True + except ValueError: + return False + + def test_float(self, value): + try: + float(value) + return True + except ValueError: + return False + + def __repr__(self): + return self.guessed_type + ": possibilities = " + repr(self.couldbe) + + @property + def guessed_type(self): + options = set(self.couldbe.keys()) + # Return based on precedence + for key in self.get_tests(): + if key in options: + return key + return "text" + + def evaluate(self, value): + if not value or not self.couldbe: + return + not_these = [] + for name, test in self.couldbe.items(): + if not test(value): + not_these.append(name) + for key in not_these: + del self.couldbe[key] diff --git a/tests/test_cli.py b/tests/test_cli.py index b085ea4..8ddf71b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ from sqlite_utils import cli, Database from sqlite_utils.db import Index, ForeignKey from click.testing import CliRunner +from unittest import mock import json import os import pytest @@ -2067,3 +2068,33 @@ def test_csv_insert_bom(tmpdir): ("broken", "CREATE TABLE [broken] (\n [\ufeffname] TEXT,\n [age] TEXT\n)"), ("fixed", "CREATE TABLE [fixed] (\n [name] TEXT,\n [age] TEXT\n)"), ] + + +@pytest.mark.parametrize("option_or_env_var", (None, "-d", "--detect-types")) +def test_insert_detect_types(tmpdir, option_or_env_var): + db_path = str(tmpdir / "test.db") + data = "name,age,weight\nCleo,6,45.5\nDori,1,3.5" + extra = [] + if option_or_env_var: + extra = [option_or_env_var] + + def _test(): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "-", "--csv"] + extra, + catch_exceptions=False, + input=data, + ) + assert result.exit_code == 0 + db = Database(db_path) + assert list(db["creatures"].rows) == [ + {"rowid": 1, "name": "Cleo", "age": 6, "weight": 45.5}, + {"rowid": 2, "name": "Dori", "age": 1, "weight": 3.5}, + ] + + if option_or_env_var is None: + # Use environemnt variable instead of option + with mock.patch.dict(os.environ, {"SQLITE_UTILS_DETECT_TYPES": "1"}): + _test() + else: + _test() From ec5174ed40fa283cb06f25ee0c0136297ec313ae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Jun 2021 21:37:56 -0700 Subject: [PATCH 042/102] Detect types for sqlite-utils memory CSV, opt out with --no-detect-types - closes #282 --- docs/cli.rst | 2 ++ sqlite_utils/cli.py | 13 +++++++++++ sqlite_utils/utils.py | 1 + tests/test_cli_memory.py | 49 ++++++++++++++++++++++++++++------------ 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index bad2d51..8a061c3 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -291,6 +291,8 @@ Incoming CSV data will be assumed to use ``utf-8``. If your data uses a differen If you are joining across multiple CSV files they must all use the same encoding. +Column types will be automatically detected in CSV or TSV data, using the same mechanism as ``--detect-types`` described in :ref:`cli_insert_csv_tsv`. You can pass the ``--no-detect-types`` option to disable this automatic type detection and treat all CSV and TSV columns as ``TEXT``. + .. _cli_memory_explicit: Explicitly specifying the format diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 96adaee..4ae1b21 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1167,6 +1167,12 @@ def query( "--encoding", help="Character encoding for CSV input, defaults to utf-8", ) +@click.option( + "-n", + "--no-detect-types", + is_flag=True, + help="Treat all CSV/TSV columns as TEXT", +) @click.option("--dump", is_flag=True, help="Dump SQL for in-memory database") @click.option( "--save", @@ -1189,6 +1195,7 @@ def memory( raw, param, encoding, + no_detect_types, dump, save, load_extension, @@ -1235,7 +1242,13 @@ def memory( csv_table = csv_path.stem csv_fp = csv_path.open("rb") rows = rows_from_file(csv_fp, format=format, encoding=encoding) + tracker = None + if not no_detect_types: + tracker = TypeTracker() + rows = tracker.wrap(rows) db[csv_table].insert_all(rows, alter=True) + if tracker is not None: + db[csv_table].transform(types=tracker.types) # Add convenient t / t1 / t2 views view_names = ["t{}".format(i + 1)] if i == 0: diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index ffc727e..473ad37 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -147,6 +147,7 @@ def rows_from_file( format=None, dialect=None, encoding=None, + detect_types=False, ) -> Generator[dict, None, None]: if format == Format.JSON: decoded = json.load(fp) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index c91beaf..fb0f153 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -33,7 +33,7 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): assert result.exit_code == 0 assert ( result.output.strip() - == '{"id": "1", "name": "Cleo"}\n{"id": "2", "name": "Bants"}' + == '{"rowid": 1, "id": 1, "name": "Cleo"}\n{"rowid": 2, "id": 2, "name": "Bants"}' ) @@ -57,8 +57,8 @@ def test_memory_tsv(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ - {"id": "1", "name": "Cleo"}, - {"id": "2", "name": "Bants"}, + {"rowid": 1, "id": 1, "name": "Cleo"}, + {"rowid": 2, "id": 2, "name": "Bants"}, ] @@ -82,8 +82,8 @@ def test_memory_json(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ - {"name": "Bants", "age": None}, - {"name": "Dori", "age": 1}, + {"rowid": 1, "name": "Bants", "age": None}, + {"rowid": 2, "name": "Dori", "age": 1}, ] @@ -106,7 +106,10 @@ def test_memory_json_nl(tmpdir, use_stdin): input=data, ) assert result.exit_code == 0, result.output - assert json.loads(result.output.strip()) == [{"name": "Bants"}, {"name": "Dori"}] + assert json.loads(result.output.strip()) == [ + {"rowid": 1, "name": "Bants"}, + {"rowid": 2, "name": "Dori"}, + ] @pytest.mark.parametrize("use_stdin", (True, False)) @@ -143,10 +146,11 @@ def test_memory_csv_encoding(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == { + "rowid": 1, "date": "2020-03-04", - "name": "S\u00e3o Paulo", - "latitude": "-23.561", - "longitude": "-46.645", + "name": "São Paulo", + "latitude": -23.561, + "longitude": -46.645, } @@ -160,12 +164,13 @@ def test_memory_dump(extra_args): assert result.exit_code == 0 assert result.output.strip() == ( "BEGIN TRANSACTION;\n" - "CREATE TABLE [stdin] (\n" - " [id] TEXT,\n" + 'CREATE TABLE "stdin" (\n' + " [rowid] INTEGER PRIMARY KEY,\n" + " [id] INTEGER,\n" " [name] TEXT\n" ");\n" - "INSERT INTO \"stdin\" VALUES('1','Cleo');\n" - "INSERT INTO \"stdin\" VALUES('2','Bants');\n" + "INSERT INTO \"stdin\" VALUES(1,1,'Cleo');\n" + "INSERT INTO \"stdin\" VALUES(2,2,'Bants');\n" "CREATE VIEW t1 AS select * from [stdin];\n" "CREATE VIEW t AS select * from [stdin];\n" "COMMIT;" @@ -183,6 +188,20 @@ def test_memory_save(tmpdir, extra_args): assert result.exit_code == 0 db = Database(save_to) assert list(db["stdin"].rows) == [ - {"id": "1", "name": "Cleo"}, - {"id": "2", "name": "Bants"}, + {"rowid": 1, "id": 1, "name": "Cleo"}, + {"rowid": 2, "id": 2, "name": "Bants"}, + ] + + +@pytest.mark.parametrize("option", ("-n", "--no-detect-types")) +def test_memory_no_detect_types(option): + result = CliRunner().invoke( + cli.cli, + ["memory", "-", "select * from stdin"] + [option], + input="id,name,weight\n1,Cleo,45.5\n2,Bants,3.5", + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + {"id": "1", "name": "Cleo", "weight": "45.5"}, + {"id": "2", "name": "Bants", "weight": "3.5"}, ] From dc94f4bb8cfe922bb2f9c89f8f0f29092ea63133 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 07:52:44 -0700 Subject: [PATCH 043/102] Don't detect types on JSON input to memory, closes #283 --- sqlite_utils/cli.py | 4 ++-- sqlite_utils/utils.py | 21 ++++++++++++--------- tests/test_cli_memory.py | 10 +++++----- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4ae1b21..32bd693 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1241,9 +1241,9 @@ def memory( csv_path = pathlib.Path(path) csv_table = csv_path.stem csv_fp = csv_path.open("rb") - rows = rows_from_file(csv_fp, format=format, encoding=encoding) + rows, format_used = rows_from_file(csv_fp, format=format, encoding=encoding) tracker = None - if not no_detect_types: + if format_used in (Format.CSV, Format.TSV) and not no_detect_types: tracker = TypeTracker() rows = tracker.wrap(rows) db[csv_table].insert_all(rows, alter=True) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 473ad37..4dd5f21 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -155,15 +155,18 @@ def rows_from_file( decoded = [decoded] if not isinstance(decoded, list): raise RowsFromFileBadJSON("JSON must be a list or a dictionary") - yield from decoded + return decoded, Format.JSON elif format == Format.NL: - yield from (json.loads(line) for line in fp if line.strip()) + return (json.loads(line) for line in fp if line.strip()), Format.NL elif format == Format.CSV: decoded_fp = io.TextIOWrapper(fp, encoding=encoding or "utf-8-sig") - yield from csv.DictReader(decoded_fp, dialect=dialect) + return csv.DictReader(decoded_fp, dialect=dialect), Format.CSV elif format == Format.TSV: - yield from rows_from_file( - fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding + return ( + rows_from_file( + fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding + )[0], + Format.TSV, ) elif format is None: # Detect the format, then call this recursively @@ -171,12 +174,12 @@ def rows_from_file( first_bytes = buffered.peek(2048).strip() if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"): # TODO: Detect newline-JSON - yield from rows_from_file(buffered, format=Format.JSON) + return rows_from_file(buffered, format=Format.JSON) else: dialect = csv.Sniffer().sniff( first_bytes.decode(encoding or "utf-8-sig", "ignore") ) - yield from rows_from_file( + return rows_from_file( buffered, format=Format.CSV, dialect=dialect, encoding=encoding ) else: @@ -215,14 +218,14 @@ class ValueTracker: try: int(value) return True - except ValueError: + except (ValueError, TypeError): return False def test_float(self, value): try: float(value) return True - except ValueError: + except (ValueError, TypeError): return False def __repr__(self): diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index fb0f153..2a1fb85 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -64,7 +64,7 @@ def test_memory_tsv(tmpdir, use_stdin): @pytest.mark.parametrize("use_stdin", (True, False)) def test_memory_json(tmpdir, use_stdin): - data = '[{"name": "Bants"}, {"name": "Dori", "age": 1}]' + data = '[{"name": "Bants"}, {"name": "Dori", "age": 1, "nested": {"nest": 1}}]' if use_stdin: input = data path = "stdin:json" @@ -82,8 +82,8 @@ def test_memory_json(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ - {"rowid": 1, "name": "Bants", "age": None}, - {"rowid": 2, "name": "Dori", "age": 1}, + {"name": "Bants", "age": None, "nested": None}, + {"name": "Dori", "age": 1, "nested": '{"nest": 1}'}, ] @@ -107,8 +107,8 @@ def test_memory_json_nl(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ - {"rowid": 1, "name": "Bants"}, - {"rowid": 2, "name": "Dori"}, + {"name": "Bants"}, + {"name": "Dori"}, ] From 5b257949d996fe43dc5d218d4308b88796a90740 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 08:12:29 -0700 Subject: [PATCH 044/102] table.use_rowid introspection property, closes #285 --- docs/python-api.rst | 13 +++++++++++++ sqlite_utils/db.py | 4 ++++ tests/test_introspect.py | 8 ++++++++ 3 files changed, 25 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index b25eb21..85f0296 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1549,6 +1549,19 @@ The ``.pks`` property returns a list of strings naming the primary key columns f >>> db["PlantType"].pks ['id'] +If a table has no primary keys but is a `rowid table `__, this property will return ``['rowid']``. + +.. _python_api_introspection_use_rowid: + +.use_rowid +---------- + +Almost all SQLite tables have a ``rowid`` column, but a table with no explicitly defined primary keys must use that ``rowid`` as the primary key for identifying individual rows. The ``.use_rowid`` property checks to see if a table needs to use the ``rowid`` in this way - it returns ``True`` if the table has no explicitly defined primary keys and ``False`` otherwise. + + >>> db["PlantType"].use_rowid + False + + .. _python_api_introspection_foreign_keys: .foreign_keys diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 31d4cee..2d8fb24 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -809,6 +809,10 @@ class Table(Queryable): names = ["rowid"] return names + @property + def use_rowid(self): + return not any(column for column in self.columns if column.is_pk) + def get(self, pk_values): if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] diff --git a/tests/test_introspect.py b/tests/test_introspect.py index d54ca88..6ca578c 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -237,3 +237,11 @@ def test_virtual_table_using(sql, expected_name, expected_using): db = Database(memory=True) db.execute(sql) assert db[expected_name].virtual_table_using == expected_using + + +def test_use_rowid(): + db = Database(memory=True) + db["rowid_table"].insert({"name": "Cleo"}) + db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id") + assert db["rowid_table"].use_rowid + assert not db["regular_table"].use_rowid From 0e797033f96a1c61b173a3d8af2ff36905687a2e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 08:28:26 -0700 Subject: [PATCH 045/102] .transform() on rowid (non-pk) tables bug fix, closes #284 --- sqlite_utils/db.py | 4 ++- tests/test_cli.py | 4 +-- tests/test_cli_memory.py | 17 ++++----- tests/test_extract.py | 15 +++++++- tests/test_transform.py | 76 ++++++++++++++++++++++++++++++++++++++-- 5 files changed, 100 insertions(+), 16 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 2d8fb24..9da1b97 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1016,7 +1016,9 @@ class Table(Queryable): sqls = [] if pk is DEFAULT: - pks_renamed = tuple(rename.get(p) or p for p in self.pks) + pks_renamed = tuple( + rename.get(p.name) or p.name for p in self.columns if p.is_pk + ) if len(pks_renamed) == 1: pk = pks_renamed[0] else: diff --git a/tests/test_cli.py b/tests/test_cli.py index 8ddf71b..8d59c0c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2088,8 +2088,8 @@ def test_insert_detect_types(tmpdir, option_or_env_var): assert result.exit_code == 0 db = Database(db_path) assert list(db["creatures"].rows) == [ - {"rowid": 1, "name": "Cleo", "age": 6, "weight": 45.5}, - {"rowid": 2, "name": "Dori", "age": 1, "weight": 3.5}, + {"name": "Cleo", "age": 6, "weight": 45.5}, + {"name": "Dori", "age": 1, "weight": 3.5}, ] if option_or_env_var is None: diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index 2a1fb85..6966d05 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -32,8 +32,7 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): ) assert result.exit_code == 0 assert ( - result.output.strip() - == '{"rowid": 1, "id": 1, "name": "Cleo"}\n{"rowid": 2, "id": 2, "name": "Bants"}' + result.output.strip() == '{"id": 1, "name": "Cleo"}\n{"id": 2, "name": "Bants"}' ) @@ -57,8 +56,8 @@ def test_memory_tsv(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ - {"rowid": 1, "id": 1, "name": "Cleo"}, - {"rowid": 2, "id": 2, "name": "Bants"}, + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Bants"}, ] @@ -146,7 +145,6 @@ def test_memory_csv_encoding(tmpdir, use_stdin): ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == { - "rowid": 1, "date": "2020-03-04", "name": "São Paulo", "latitude": -23.561, @@ -165,12 +163,11 @@ def test_memory_dump(extra_args): assert result.output.strip() == ( "BEGIN TRANSACTION;\n" 'CREATE TABLE "stdin" (\n' - " [rowid] INTEGER PRIMARY KEY,\n" " [id] INTEGER,\n" " [name] TEXT\n" ");\n" - "INSERT INTO \"stdin\" VALUES(1,1,'Cleo');\n" - "INSERT INTO \"stdin\" VALUES(2,2,'Bants');\n" + "INSERT INTO \"stdin\" VALUES(1,'Cleo');\n" + "INSERT INTO \"stdin\" VALUES(2,'Bants');\n" "CREATE VIEW t1 AS select * from [stdin];\n" "CREATE VIEW t AS select * from [stdin];\n" "COMMIT;" @@ -188,8 +185,8 @@ def test_memory_save(tmpdir, extra_args): assert result.exit_code == 0 db = Database(save_to) assert list(db["stdin"].rows) == [ - {"rowid": 1, "id": 1, "name": "Cleo"}, - {"rowid": 2, "id": 2, "name": "Bants"}, + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Bants"}, ] diff --git a/tests/test_extract.py b/tests/test_extract.py index 9eae704..280c59c 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -126,12 +126,25 @@ def test_extract_rowid_table(fresh_db): fresh_db["tree"].extract(["common_name", "latin_name"]) assert fresh_db["tree"].schema == ( 'CREATE TABLE "tree" (\n' - " [rowid] INTEGER PRIMARY KEY,\n" " [name] TEXT,\n" " [common_name_latin_name_id] INTEGER,\n" " FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n" ")" ) + assert ( + fresh_db.execute( + """ + select + tree.name, + common_name_latin_name.common_name, + common_name_latin_name.latin_name + from tree + join common_name_latin_name + on tree.common_name_latin_name_id = common_name_latin_name.id + """ + ).fetchall() + == [("Tree 1", "Palm", "Arecaceae")] + ) def test_reuse_lookup_table(fresh_db): diff --git a/tests/test_transform.py b/tests/test_transform.py index b3ef009..19e447e 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -89,7 +89,9 @@ import pytest ], ) @pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) -def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys): +def test_transform_sql_table_with_primary_key( + fresh_db, params, expected_sql, use_pragma_foreign_keys +): captured = [] tracer = lambda sql, params: captured.append((sql, params)) dogs = fresh_db["dogs"] @@ -111,7 +113,77 @@ def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys): assert ("PRAGMA foreign_keys=1;", None) not in captured -def test_transform_sql_rowid_to_id(fresh_db): +@pytest.mark.parametrize( + "params,expected_sql", + [ + # Identity transform - nothing changes + ( + {}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + ], + ), + # Change column type + ( + {"types": {"age": int}}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] INTEGER\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + ], + ), + # Rename a column + ( + {"rename": {"age": "dog_age"}}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [dog_age] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + ], + ), + # Make ID a primary key + ( + {"pk": "id"}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + ], + ), + ], +) +@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) +def test_transform_sql_table_with_no_primary_key( + fresh_db, params, expected_sql, use_pragma_foreign_keys +): + captured = [] + tracer = lambda sql, params: captured.append((sql, params)) + dogs = fresh_db["dogs"] + if use_pragma_foreign_keys: + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) + sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}}) + assert sql == expected_sql + # Check that .transform() runs without exceptions: + with fresh_db.tracer(tracer): + dogs.transform(**params) + # If use_pragma_foreign_keys, check that we did the right thing + if use_pragma_foreign_keys: + assert ("PRAGMA foreign_keys=0;", None) in captured + assert captured[-2] == ("PRAGMA foreign_key_check;", None) + assert captured[-1] == ("PRAGMA foreign_keys=1;", None) + else: + assert ("PRAGMA foreign_keys=0;", None) not in captured + assert ("PRAGMA foreign_keys=1;", None) not in captured + + +def test_transform_sql_with_no_primary_key_to_primary_key_of_id(fresh_db): dogs = fresh_db["dogs"] dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) assert ( From 13e76b375ac3e3448df5d705ba65fadaaf9887d6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 09:01:39 -0700 Subject: [PATCH 046/102] Release 3.10 Refs #272, #274, #275, #276, #282, #284, #285 --- docs/changelog.rst | 68 ++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 879d19b..cb9ab6d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,74 @@ Changelog =========== +.. _3.10: + +3.10 (2021-06-19) +----------------- + +This release introduces the ``sqlite-utils memory`` command, which can be used to load CSV or JSON data into a temporary in-memory database and run SQL queries (including joins across multiple files) directly against that data. + +Also new: ``sqlite-utils insert --detect-types``, ``sqlite-utils dump``, ``table.use_rowid`` plus some smaller fixes. + +sqlite-utils memory +~~~~~~~~~~~~~~~~~~~ + +This example of ``sqlite-utils memory`` retrieves information about the all of the repositories in the `Dogsheep `__ organization on GitHub using `this JSON API `__, sorts them by their number of stars and outputs a table of the top five (using ``-t``):: + + $ curl -s 'https://api.github.com/users/dogsheep/repos' \ + | sqlite-utils memory - ' + select full_name, forks_count, stargazers_count + from stdin order by stargazers_count desc limit 5 + ' -t + full_name forks_count stargazers_count + --------------------------------- ------------- ------------------ + dogsheep/twitter-to-sqlite 12 225 + dogsheep/github-to-sqlite 14 139 + dogsheep/dogsheep-photos 5 116 + dogsheep/dogsheep.github.io 7 90 + dogsheep/healthkit-to-sqlite 4 85 + +The tool works against files on disk as well. This example joins data from two CSV files:: + + $ cat creatures.csv + species_id,name + 1,Cleo + 2,Bants + 2,Dori + 2,Azi + $ cat species.csv + id,species_name + 1,Dog + 2,Chicken + $ sqlite-utils memory species.csv creatures.csv ' + select * from creatures join species on creatures.species_id = species.id + ' + [{"species_id": 1, "name": "Cleo", "id": 1, "species_name": "Dog"}, + {"species_id": 2, "name": "Bants", "id": 2, "species_name": "Chicken"}, + {"species_id": 2, "name": "Dori", "id": 2, "species_name": "Chicken"}, + {"species_id": 2, "name": "Azi", "id": 2, "species_name": "Chicken"}] + +Here the ``species.csv`` file becomes the ``species`` table, the ``creatures.csv`` file becomes the ``creatures`` table and the output is JSON, the default output format. + +You can also use the ``--attach`` option to attach existing SQLite database files to the in-memory database, in order to join data from CSV or JSON directly against your existing tables. + +Full documentation of this new feature is available in :ref:`cli_memory`. (`#272 `__) + +sqlite-utils insert \-\-detect-types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`sqlite-utils insert ` command can be used to insert data from JSON, CSV or TSV files into a SQLite database file. The new ``--detect-types`` option (shortcut ``-d``), when used in conjunction with a CSV or TSV import, will automatically detect if columns in the file are integers or floating point numbers as opposed to treating everything as a text column and create the new table with the corresponding schema. See :ref:`cli_insert_csv_tsv` for details. (`#282 `__) + +Other changes +~~~~~~~~~~~~~ + +- **Bug fix**: ``table.transform()``, when run against a table without explicit primary keys, would incorrectly create a new version of the table with an explicit primary key column called ``rowid``. (`#284 `__) +- New ``table.use_rowid`` introspection property, see :ref:`python_api_introspection_use_rowid`. (`#285 `__) +- The new ``sqlite-utils dump file.db`` command outputs a SQL dump that can be used to recreate a database. (`#274 `__) +- ``-h`` now works as a shortcut for ``--help``, thanks Loren McIntyre. (`#276 `__) +- Now using `pytest-cov `__ and `Codecov `__ to track test coverage - currently at 96%. (`#275 `__) +- SQL errors that occur when using ``sqlite-utils query`` are now displayed as CLI errors. + .. _3.9.1: 3.9.1 (2021-06-12) diff --git a/setup.py b/setup.py index 9c73ff4..408055c 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.9.1" +VERSION = "3.10" def get_long_description(): From eb18b6e42c6d10aca6193204bc907490d2f56547 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 09:09:29 -0700 Subject: [PATCH 047/102] Disabling macos-latest for the moment GitHub seems to have run out of workers right now. --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9f05567..2ea0d33 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, windows-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From e71c41d39ba32785772bfbaf62aad4cc63839124 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 13:36:16 -0700 Subject: [PATCH 048/102] Fixed broken anchors --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cb9ab6d..e358994 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog =========== -.. _3.10: +.. _3_10: 3.10 (2021-06-19) ----------------- @@ -70,7 +70,7 @@ Other changes - Now using `pytest-cov `__ and `Codecov `__ to track test coverage - currently at 96%. (`#275 `__) - SQL errors that occur when using ``sqlite-utils query`` are now displayed as CLI errors. -.. _3.9.1: +.. _3_9_1: 3.9.1 (2021-06-12) ------------------ From dce9bb05b697e6f5caebf2f46dcd7cec83055bcb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 19 Jun 2021 14:43:04 -0700 Subject: [PATCH 049/102] Really fix the anchors this time --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e358994..bbbd5f2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog =========== -.. _3_10: +.. _v3_10: 3.10 (2021-06-19) ----------------- @@ -70,7 +70,7 @@ Other changes - Now using `pytest-cov `__ and `Codecov `__ to track test coverage - currently at 96%. (`#275 `__) - SQL errors that occur when using ``sqlite-utils query`` are now displayed as CLI errors. -.. _3_9_1: +.. _v3_9_1: 3.9.1 (2021-06-12) ------------------ From 933be66eba1203a3287ebaacd69f694d12f6f0a0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Jun 2021 11:25:21 -0700 Subject: [PATCH 050/102] sqlite-utils memory --schema, closes #288 Also updated some rowid examples, closes #287 --- docs/cli.rst | 96 +++++++++++++++++++++------------------- sqlite_utils/cli.py | 8 +++- tests/test_cli_memory.py | 18 ++++++++ 3 files changed, 76 insertions(+), 46 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 8a061c3..a404333 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -326,24 +326,33 @@ Here the ``--attach trees trees.db`` option makes the ``trees.db`` database avai The CSV data that was piped into the script is available in the ``stdin`` table, so ``... where rowid in (select id from stdin)`` can be used to return rows from the ``trees`` table that match IDs that were piped in as CSV content. -.. _cli_memory_dump_save: +.. _cli_memory_schema_dump_save: -\-\-dump and \-\-save ---------------------- +\-\-schema, \-\-dump and \-\-save +--------------------------------- -You can dump out the SQL used for the temporary in-memory database, complete with all imported data, using the ``--dump`` option:: +To see the schema that will be created for a file or multiple files, use ``--schema``:: + + % sqlite-utils memory dogs.csv --schema + CREATE TABLE [dogs] ( + [id] INTEGER, + [age] INTEGER, + [name] TEXT + ); + CREATE VIEW t1 AS select * from [dogs]; + CREATE VIEW t AS select * from [dogs]; + +You can output SQL that will both create the tables and insert the full data used to populate the in-memory database using ``--dump``:: % sqlite-utils memory dogs.csv --dump BEGIN TRANSACTION; CREATE TABLE [dogs] ( - [rowid] TEXT, - [id] TEXT, - [dog_age] TEXT, + [id] INTEGER, + [age] INTEGER, [name] TEXT ); - INSERT INTO "dogs" VALUES('1','1','4','Cleo'); - INSERT INTO "dogs" VALUES('2','2','2','Pancakes'); - INSERT INTO "dogs" VALUES('3','2','3','Pancakes'); + INSERT INTO "dogs" VALUES('1','4','Cleo'); + INSERT INTO "dogs" VALUES('2','2','Pancakes'); CREATE VIEW t1 AS select * from [dogs]; CREATE VIEW t AS select * from [dogs]; COMMIT; @@ -354,7 +363,6 @@ Passing ``--save other.db`` will instead use that SQL to populate a new database These features are mainly intented as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`. - .. _cli_rows: Returning all rows in a table @@ -738,13 +746,12 @@ For example, given a ``creatures.csv`` file containing this:: The following command:: - $ sqlite-utils insert creatures.db creatures creatures.tsv --csv --detect-types + $ sqlite-utils insert creatures.db creatures creatures.csv --csv --detect-types Will produce this schema:: $ sqlite-utils schema creatures.db CREATE TABLE "creatures" ( - [rowid] INTEGER PRIMARY KEY, [name] TEXT, [age] INTEGER, [weight] FLOAT @@ -1108,45 +1115,44 @@ Here's a more complex example that makes use of these options. It converts `this --fk-column country_id \ --rename country_long name -After running the above, the command ``sqlite3 global.db .schema`` reveals the following schema: +After running the above, the command ``sqlite-utils schema global.db`` reveals the following schema: .. code-block:: sql CREATE TABLE [countries] ( - [id] INTEGER PRIMARY KEY, - [country] TEXT, - [name] TEXT + [id] INTEGER PRIMARY KEY, + [country] TEXT, + [name] TEXT + ); + CREATE TABLE "power_plants" ( + [country_id] INTEGER, + [name] TEXT, + [gppd_idnr] TEXT, + [capacity_mw] TEXT, + [latitude] TEXT, + [longitude] TEXT, + [primary_fuel] TEXT, + [other_fuel1] TEXT, + [other_fuel2] TEXT, + [other_fuel3] TEXT, + [commissioning_year] TEXT, + [owner] TEXT, + [source] TEXT, + [url] TEXT, + [geolocation_source] TEXT, + [wepp_id] TEXT, + [year_of_capacity_data] TEXT, + [generation_gwh_2013] TEXT, + [generation_gwh_2014] TEXT, + [generation_gwh_2015] TEXT, + [generation_gwh_2016] TEXT, + [generation_gwh_2017] TEXT, + [generation_data_source] TEXT, + [estimated_generation_gwh] TEXT, + FOREIGN KEY([country_id]) REFERENCES [countries]([id]) ); CREATE UNIQUE INDEX [idx_countries_country_name] ON [countries] ([country], [name]); - CREATE TABLE IF NOT EXISTS "power_plants" ( - [rowid] INTEGER PRIMARY KEY, - [country_id] INTEGER, - [name] TEXT, - [gppd_idnr] TEXT, - [capacity_mw] TEXT, - [latitude] TEXT, - [longitude] TEXT, - [primary_fuel] TEXT, - [other_fuel1] TEXT, - [other_fuel2] TEXT, - [other_fuel3] TEXT, - [commissioning_year] TEXT, - [owner] TEXT, - [source] TEXT, - [url] TEXT, - [geolocation_source] TEXT, - [wepp_id] TEXT, - [year_of_capacity_data] TEXT, - [generation_gwh_2013] TEXT, - [generation_gwh_2014] TEXT, - [generation_gwh_2015] TEXT, - [generation_gwh_2016] TEXT, - [generation_gwh_2017] TEXT, - [generation_data_source] TEXT, - [estimated_generation_gwh] TEXT, - FOREIGN KEY(country_id) REFERENCES countries(id) - ); .. _cli_create_view: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 32bd693..8043c98 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1173,6 +1173,7 @@ def query( is_flag=True, help="Treat all CSV/TSV columns as TEXT", ) +@click.option("--schema", is_flag=True, help="Show SQL schema for in-memory database") @click.option("--dump", is_flag=True, help="Dump SQL for in-memory database") @click.option( "--save", @@ -1196,6 +1197,7 @@ def memory( param, encoding, no_detect_types, + schema, dump, save, load_extension, @@ -1224,7 +1226,7 @@ def memory( """ db = sqlite_utils.Database(memory=True) # If --dump or --save used but no paths detected, assume SQL query is a path: - if (dump or save) and not paths: + if (dump or save or schema) and not paths: paths = [sql] sql = None for i, path in enumerate(paths): @@ -1262,6 +1264,10 @@ def memory( click.echo(line) return + if schema: + click.echo(db.schema) + return + if save: db2 = sqlite_utils.Database(save) for line in db.conn.iterdump(): diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index 6966d05..e42e1cb 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -174,6 +174,24 @@ def test_memory_dump(extra_args): ) +@pytest.mark.parametrize("extra_args", ([], ["select 1"])) +def test_memory_schema(extra_args): + result = CliRunner().invoke( + cli.cli, + ["memory", "-"] + extra_args + ["--schema"], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + assert result.output.strip() == ( + 'CREATE TABLE "stdin" (\n' + " [id] INTEGER,\n" + " [name] TEXT\n" + ");\n" + "CREATE VIEW t1 AS select * from [stdin];\n" + "CREATE VIEW t AS select * from [stdin];" + ) + + @pytest.mark.parametrize("extra_args", ([], ["select 1"])) def test_memory_save(tmpdir, extra_args): save_to = str(tmpdir / "save.db") From dbcba6c597fb199d3a7ea0dd732909ec86b616cf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Jun 2021 11:47:09 -0700 Subject: [PATCH 051/102] Added installation instructions, closes #286 --- docs/index.rst | 1 + docs/installation.rst | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 docs/installation.rst diff --git a/docs/index.rst b/docs/index.rst index 571020f..b1bfc0e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,6 +29,7 @@ Contents .. toctree:: :maxdepth: 3 + installation cli python-api changelog diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..099dd16 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,40 @@ +.. _installation: + +============== + Installation +============== + +``sqlite-utils`` is tested on Linux, macOS and Windows. + +.. _installation_homebrew: + +Using Homebrew +============== + +The :ref:`sqlite-utils commad-line tool ` can be installed on macOS using Homebrew:: + + brew install sqlite-utils + +If you have it installed and want to upgrade to the most recent release, you ran run:: + + brew upgrade sqlite-utils + +Then run ``sqlite-utils --version`` to confirm the installed version. + +.. _installation_pip: + +Using pip +========= + +The `sqlite-utils package `__ on PyPI includes both the :ref:`sqlite_utils Python library ` and the ``sqlite-utils`` command-line tool. You can install them using ``pip`` like so:: + + pip install sqlite-utils + +.. _installation_pipx: + +Using pipx +========== + +`pipx `__ is a tool for installing Python command-line applications in their own isolated environments. You can use ``pipx`` to install the ``sqlite-utils`` command-line tool like this:: + + pipx install sqlite-utils From a25a5845b8b4dd79a3c3ff681b7ddd1f9a608b58 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Jun 2021 11:50:48 -0700 Subject: [PATCH 052/102] Release 3.11 Refs #286, #287, #288 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index bbbd5f2..0c31e79 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v3_11: + +3.11 (2021-06-20) +----------------- + +- New ``sqlite-utils memory data.csv --schema`` option, for outputting the schema of the in-memory database generated from one or more files. See :ref:`cli_memory_schema_dump_save`. (`#288 `__) +- Added :ref:`installation instructions `. (`#286 `__) + .. _v3_10: 3.10 (2021-06-19) diff --git a/setup.py b/setup.py index 408055c..626409e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.10" +VERSION = "3.11" def get_long_description(): From adcd32a866dd8828002b9fdc77b4f674fa80c46e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Jun 2021 11:52:15 -0700 Subject: [PATCH 053/102] Re-enable publish testing on macos-latest --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2ea0d33..c0bd779 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 8cedc6a8b29180e68326f6b76f249d5e39e4b591 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 20 Jun 2021 13:44:02 -0700 Subject: [PATCH 054/102] Typo fix Thanks, https://twitter.com/garrettc/status/1406705348648525830 --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 099dd16..aa3234d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -15,7 +15,7 @@ The :ref:`sqlite-utils commad-line tool ` can be installed on macOS using H brew install sqlite-utils -If you have it installed and want to upgrade to the most recent release, you ran run:: +If you have it installed and want to upgrade to the most recent release, you can run:: brew upgrade sqlite-utils From 9faeef230bf84c2f9b2859e5a4544f5ec50adf68 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Jun 2021 21:03:59 -0700 Subject: [PATCH 055/102] New db.query() method, refs #290 --- sqlite_utils/db.py | 8 ++++++-- tests/test_cli.py | 43 +++++++++++++++++++++---------------------- tests/test_create.py | 8 ++++---- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9da1b97..1a88dba 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -359,10 +359,14 @@ class Database: for table in tables ) - def execute_returning_dicts(self, sql, params=None): + def query(self, sql, params=None): cursor = self.execute(sql, params or tuple()) keys = [d[0] for d in cursor.description] - return [dict(zip(keys, row)) for row in cursor.fetchall()] + for row in cursor: + yield dict(zip(keys, row)) + + def execute_returning_dicts(self, sql, params=None): + return list(self.query(sql, params)) def resolve_foreign_keys(self, name, foreign_keys): # foreign_keys may be a list of strcolumn names, a list of ForeignKey tuples, diff --git a/tests/test_cli.py b/tests/test_cli.py index 8d59c0c..5028458 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -613,8 +613,8 @@ def test_insert_simple(tmpdir): open(json_path, "w").write(json.dumps({"name": "Cleo", "age": 4})) result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path]) assert 0 == result.exit_code - assert [{"age": 4, "name": "Cleo"}] == Database(db_path).execute_returning_dicts( - "select * from dogs" + assert [{"age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") ) db = Database(db_path) assert ["dogs"] == db.table_names() @@ -629,8 +629,8 @@ def test_insert_from_stdin(tmpdir): input=json.dumps({"name": "Cleo", "age": 4}), ) assert 0 == result.exit_code - assert [{"age": 4, "name": "Cleo"}] == Database(db_path).execute_returning_dicts( - "select * from dogs" + assert [{"age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") ) @@ -655,9 +655,9 @@ def test_insert_with_primary_key(db_path, tmpdir): cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] ) assert 0 == result.exit_code - assert [{"id": 1, "age": 4, "name": "Cleo"}] == Database( - db_path - ).execute_returning_dicts("select * from dogs") + assert [{"id": 1, "age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") + ) db = Database(db_path) assert ["id"] == db["dogs"].pks @@ -671,7 +671,7 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir): ) assert 0 == result.exit_code db = Database(db_path) - assert dogs == db.execute_returning_dicts("select * from dogs order by id") + assert dogs == list(db.query("select * from dogs order by id")) assert ["id"] == db["dogs"].pks @@ -687,7 +687,7 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): ) assert 0 == result.exit_code db = Database(db_path) - assert dogs == db.execute_returning_dicts("select * from dogs order by breed, id") + assert dogs == list(db.query("select * from dogs order by breed, id")) assert {"breed", "id"} == set(db["dogs"].pks) assert ( "CREATE TABLE [dogs] (\n" @@ -732,7 +732,7 @@ def test_insert_binary_base64(db_path): ) assert 0 == result.exit_code, result.output db = Database(db_path) - actual = db.execute_returning_dicts("select content from files") + actual = list(db.query("select content from files")) assert actual == [{"content": b"hello"}] @@ -747,7 +747,7 @@ def test_insert_newline_delimited(db_path): assert [ {"foo": "bar", "n": 1}, {"foo": "baz", "n": 2}, - ] == db.execute_returning_dicts("select foo, n from from_json_nl") + ] == list(db.query("select foo, n from from_json_nl")) def test_insert_ignore(db_path, tmpdir): @@ -766,9 +766,7 @@ def test_insert_ignore(db_path, tmpdir): ) assert 0 == result.exit_code, result.output # ... but it should actually have no effect - assert [{"id": 1, "name": "Cleo"}] == db.execute_returning_dicts( - "select * from dogs" - ) + assert [{"id": 1, "name": "Cleo"}] == list(db.query("select * from dogs")) @pytest.mark.parametrize( @@ -831,8 +829,9 @@ def test_insert_replace(db_path, tmpdir): ) assert 0 == result.exit_code, result.output assert 21 == db["dogs"].count - assert insert_replace_dogs == db.execute_returning_dicts( - "select * from dogs where id in (1, 2, 21) order by id" + assert ( + list(db.query("select * from dogs where id in (1, 2, 21) order by id")) + == insert_replace_dogs ) @@ -847,7 +846,7 @@ def test_insert_truncate(db_path): assert [ {"foo": "bar", "n": 1}, {"foo": "baz", "n": 2}, - ] == db.execute_returning_dicts("select foo, n from from_json_nl") + ] == list(db.query("select foo, n from from_json_nl")) # Truncate and insert new rows result = CliRunner().invoke( cli.cli, @@ -866,7 +865,7 @@ def test_insert_truncate(db_path): assert [ {"foo": "bam", "n": 3}, {"foo": "bat", "n": 4}, - ] == db.execute_returning_dicts("select foo, n from from_json_nl") + ] == list(db.query("select foo, n from from_json_nl")) def test_insert_alter(db_path, tmpdir): @@ -897,7 +896,7 @@ def test_insert_alter(db_path, tmpdir): {"foo": "bar", "n": 1, "baz": None}, {"foo": "baz", "n": 2, "baz": None}, {"foo": "bar", "baz": 5, "n": None}, - ] == db.execute_returning_dicts("select foo, n, baz from from_json_nl") + ] == list(db.query("select foo, n, baz from from_json_nl")) @pytest.mark.parametrize( @@ -1168,7 +1167,7 @@ def test_upsert(db_path, tmpdir): assert [ {"id": 1, "name": "Cleo", "age": 4}, {"id": 2, "name": "Nixie", "age": 4}, - ] == db.execute_returning_dicts("select * from dogs order by id") + ] == list(db.query("select * from dogs order by id")) def test_upsert_alter(db_path, tmpdir): @@ -1195,7 +1194,7 @@ def test_upsert_alter(db_path, tmpdir): assert 0 == result.exit_code assert [ {"id": 1, "name": "Cleo", "age": 5}, - ] == db.execute_returning_dicts("select * from dogs order by id") + ] == list(db.query("select * from dogs order by id")) @pytest.mark.parametrize( @@ -1549,7 +1548,7 @@ def test_query_update(db_path, args, expected): cli.cli, [db_path, "update dogs set age = 5 where name = 'Cleo'"] + args ) assert expected == result.output.strip() - assert db.execute_returning_dicts("select * from dogs") == [ + assert list(db.query("select * from dogs")) == [ {"id": 1, "age": 5, "name": "Cleo"}, ] diff --git a/tests/test_create.py b/tests/test_create.py index 926a7d1..2cf0b4d 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -827,8 +827,8 @@ def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db + [{"i": 101, "extra": "Should trigger ALTER"}], alter=True, ) - rows = fresh_db.execute_returning_dicts("select * from test where i = 101") - assert [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] == rows + rows = list(fresh_db.query("select * from test where i = 101")) + assert rows == [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] def test_insert_ignore(fresh_db): @@ -839,8 +839,8 @@ def test_insert_ignore(fresh_db): # Using ignore=True should cause our insert to be silently ignored fresh_db["test"].insert({"id": 1, "bar": 3}, pk="id", ignore=True) # Only one row, and it should be bar=2, not bar=3 - rows = fresh_db.execute_returning_dicts("select * from test") - assert [{"id": 1, "bar": 2}] == rows + rows = list(fresh_db.query("select * from test")) + assert rows == [{"id": 1, "bar": 2}] def test_insert_hash_id(fresh_db): From e5d7a2ba3d585303c8e1c861a09e8761ba63678f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 10:43:49 -0700 Subject: [PATCH 056/102] Tests for db.query() method, refs #290 --- tests/test_query.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/test_query.py diff --git a/tests/test_query.py b/tests/test_query.py new file mode 100644 index 0000000..fe79cc0 --- /dev/null +++ b/tests/test_query.py @@ -0,0 +1,17 @@ +import types + + +def test_query(fresh_db): + fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) + results = fresh_db.query("select * from dogs order by name desc") + assert isinstance(results, types.GeneratorType) + assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}] + + +def test_execute_returning_dicts(fresh_db): + # Like db.query() but returns a list, included for backwards compatibility + # see https://github.com/simonw/sqlite-utils/issues/290 + fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") + assert fresh_db.execute_returning_dicts("select * from test") == [ + {"id": 1, "bar": 2} + ] From 3805d1c9731d5355797271bfb292a1a91758db01 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 10:44:12 -0700 Subject: [PATCH 057/102] Removed duplicate vacuum() function, thanks mypy --- sqlite_utils/cli.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8043c98..0a0c0fe 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -260,17 +260,6 @@ def views( ) -@cli.command() -@click.argument( - "path", - type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), - required=True, -) -def vacuum(path): - """Run VACUUM against the database""" - sqlite_utils.Database(path).vacuum() - - @cli.command() @click.argument( "path", From 7b3fdf0fcd553ddf25b8d606b7fc34f9fd7979df Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 11:04:32 -0700 Subject: [PATCH 058/102] mypy annotations for rows_from_file(), run mypy in CI Refs #289, #279 --- .github/workflows/test.yml | 3 +++ sqlite_utils/cli.py | 2 +- sqlite_utils/db.py | 8 ++++---- sqlite_utils/utils.py | 31 ++++++++++++++++++------------- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f8a79ae..012db32 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,11 +26,14 @@ jobs: - name: Install dependencies run: | pip install -e '.[test]' + pip install mypy - name: Optionally install numpy if: matrix.numpy == 1 run: pip install numpy - name: Run tests run: | pytest + - name: run mypy + run: mypy sqlite_utils - name: Check formatting run: black . --check diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0a0c0fe..2de6bf3 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,6 +1,6 @@ import base64 import click -from click_default_group import DefaultGroup +from click_default_group import DefaultGroup # type: ignore from datetime import datetime import hashlib import pathlib diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1a88dba..acd1726 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -11,7 +11,7 @@ import json import os import pathlib import re -from sqlite_fts4 import rank_bm25 +from sqlite_fts4 import rank_bm25 # type: ignore import sys import textwrap import uuid @@ -39,14 +39,14 @@ USING\s+(?P\w+) # e.g. USING FTS5 ) try: - import pandas as pd + import pandas as pd # type: ignore except ImportError: pd = None try: - import numpy as np + import numpy as np # type: ignore except ImportError: - np = None + np = None # type: ignore Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 4dd5f21..4f3c819 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -5,17 +5,18 @@ import enum import io import json import os -from typing import Generator +from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type import click try: - import pysqlite3 as sqlite3 - import pysqlite3.dbapi2 + import pysqlite3 as sqlite3 # type: ignore + import pysqlite3.dbapi2 # type: ignore OperationalError = pysqlite3.dbapi2.OperationalError except ImportError: - import sqlite3 + # https://github.com/python/mypy/issues/1153#issuecomment-253842414 + import sqlite3 # type: ignore OperationalError = sqlite3.OperationalError @@ -143,12 +144,11 @@ class RowsFromFileBadJSON(RowsFromFileError): def rows_from_file( - fp, - format=None, - dialect=None, - encoding=None, - detect_types=False, -) -> Generator[dict, None, None]: + fp: BinaryIO, + format: Optional[Format] = None, + dialect: Optional[Type[csv.Dialect]] = None, + encoding: Optional[str] = None, +) -> Tuple[Iterable[dict], Format]: if format == Format.JSON: decoded = json.load(fp) if isinstance(decoded, dict): @@ -159,8 +159,13 @@ def rows_from_file( elif format == Format.NL: return (json.loads(line) for line in fp if line.strip()), Format.NL elif format == Format.CSV: - decoded_fp = io.TextIOWrapper(fp, encoding=encoding or "utf-8-sig") - return csv.DictReader(decoded_fp, dialect=dialect), Format.CSV + use_encoding: str = encoding or "utf-8-sig" + decoded_fp = io.TextIOWrapper(fp, encoding=use_encoding) + if dialect is not None: + reader = csv.DictReader(decoded_fp, dialect=dialect) + else: + reader = csv.DictReader(decoded_fp) + return reader, Format.CSV elif format == Format.TSV: return ( rows_from_file( @@ -170,7 +175,7 @@ def rows_from_file( ) elif format is None: # Detect the format, then call this recursively - buffered = io.BufferedReader(fp, buffer_size=4096) + buffered = io.BufferedReader(cast(io.RawIOBase, fp), buffer_size=4096) first_bytes = buffered.peek(2048).strip() if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"): # TODO: Detect newline-JSON From 8d1d8013899e110c03c50c1f66a7b9c0b51f8383 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 11:08:21 -0700 Subject: [PATCH 059/102] mypy tweaks, refs #289, #266, #37 --- .github/workflows/test.yml | 3 +-- setup.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 012db32..2e19e61 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,8 +25,7 @@ jobs: ${{ runner.os }}-pip- - name: Install dependencies run: | - pip install -e '.[test]' - pip install mypy + pip install -e '.[test,mypy]' - name: Optionally install numpy if: matrix.numpy == 1 run: pip install numpy diff --git a/setup.py b/setup.py index 626409e..a356f30 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ setup( extras_require={ "test": ["pytest", "black", "hypothesis"], "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], + "mypy": ["mypy", "types-click", "types-tabulate"], }, entry_points=""" [console_scripts] From 90e211e3e2f36d2ff911ecf1afe4470ff45c7c0d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 18:22:08 -0700 Subject: [PATCH 060/102] Now complies with flake8, refs #291 --- setup.cfg | 3 + sqlite_utils/db.py | 6 +- tests/test_analyze_tables.py | 5 +- tests/test_cli.py | 181 +++++++++++++++++++++++++++----- tests/test_cli_memory.py | 4 +- tests/test_constructor.py | 1 - tests/test_conversions.py | 3 - tests/test_create.py | 9 +- tests/test_enable_counts.py | 27 ++++- tests/test_extract.py | 2 +- tests/test_extracts.py | 2 +- tests/test_hypothesis.py | 1 + tests/test_introspect.py | 20 +++- tests/test_recreate.py | 2 +- tests/test_register_function.py | 4 +- tests/test_rows.py | 1 - tests/test_tracer.py | 35 +++++- tests/test_transform.py | 10 +- tests/test_upsert.py | 2 +- tests/test_wal.py | 1 - 20 files changed, 252 insertions(+), 67 deletions(-) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..7a88d6f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 160 +extend-ignore = E203 # for Black diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index acd1726..50fd496 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,5 +1,5 @@ from .utils import sqlite3, OperationalError, suggest_column_types, column_affinity -from collections import namedtuple, OrderedDict +from collections import namedtuple from collections.abc import Mapping import contextlib import datetime @@ -1116,8 +1116,6 @@ class Table(Queryable): ) ) table = table or "_".join(columns) - first_column = columns[0] - pks = self.pks lookup_table = self.db[table] fk_column = fk_column or "{}_id".format(table) magic_lookup_column = "{}_{}".format(fk_column, os.urandom(6).hex()) @@ -1236,7 +1234,7 @@ class Table(Queryable): fk_col_type = None if fk is not None: # fk must be a valid table - if not fk in self.db.table_names(): + if fk not in self.db.table_names(): raise AlterError("table '{}' does not exist".format(fk)) # if fk_col specified, must be a valid column if fk_col is not None: diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index f0af2ca..5795a7a 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -1,10 +1,8 @@ -from sqlite_utils.db import Database, ForeignKey, ColumnDetails +from sqlite_utils.db import Database, ColumnDetails from sqlite_utils import cli -from sqlite_utils.utils import OperationalError from click.testing import CliRunner import pytest import sqlite3 -import textwrap @pytest.fixture @@ -132,6 +130,7 @@ def test_analyze_table_save(db_to_analyze_path): result = CliRunner().invoke( cli.cli, ["analyze-tables", db_to_analyze_path, "--save"] ) + assert result.exit_code == 0 rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows) assert rows == [ { diff --git a/tests/test_cli.py b/tests/test_cli.py index 5028458..37a96a3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -410,7 +410,7 @@ def test_index_foreign_keys(db_path): def test_enable_fts(db_path): db = Database(db_path) - assert None == db["Gosh"].detect_fts() + assert db["Gosh"].detect_fts() is None result = CliRunner().invoke( cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] ) @@ -420,7 +420,7 @@ def test_enable_fts(db_path): # Table names with restricted chars are handled correctly. # colons and dots are restricted characters for table names. db["http://example.com"].create({"c1": str, "c2": str, "c3": str}) - assert None == db["http://example.com"].detect_fts() + assert db["http://example.com"].detect_fts() is None result = CliRunner().invoke( cli.cli, [ @@ -966,7 +966,23 @@ def test_query_json(db_path, sql, args, expected): assert expected == result.output.strip() -LOREM_IPSUM_COMPRESSED = b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8ef\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J\xb9\x97>i\xa9\x11W\xb13\xa5\xde\x96$\x13\xf3I\x9cu\xe8J\xda\xee$EcsI\x8e\x0b$\xea\xab\xf6L&u\xc4emI\xb3foFnT\xf83\xca\x93\xd8QZ\xa8\xf2\xbd1q\xd1\x87\xf3\x85>\x8c\xa4i\x8d\xdaTu\x7f\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\xfb\x8f\xef\x1b\x9b\x06\x83}" +LOREM_IPSUM_COMPRESSED = ( + b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8e" + b"f\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J" + b"\xb9\x97>i\xa9\x11W\xb13\xa5\xde\x96$\x13\xf3I\x9cu\xe8J\xda\xee$EcsI\x8e\x0b" + b"$\xea\xab\xf6L&u\xc4emI\xb3foFnT\xf83\xca\x93\xd8QZ\xa8\xf2\xbd1q\xd1\x87\xf3" + b"\x85>\x8c\xa4i\x8d\xdaTu\x7f\xf0\x81\x0f|\xe0\x03" + b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03" + b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03" + b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03" + b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\xfb\x8f\xef\x1b\x9b\x06\x83}" +) def test_query_json_binary(db_path): @@ -988,7 +1004,18 @@ def test_query_json_binary(db_path): "sz": 16984, "data": { "$base64": True, - "encoded": "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uIjnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3fiCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9", + "encoded": ( + ( + "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH" + "8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+" + "DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I" + "/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uI" + "jnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3f" + "iCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8" + "IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984A" + "Mf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9" + ) + ), }, } ] @@ -1157,17 +1184,17 @@ def test_upsert(db_path, tmpdir): {"id": 1, "age": 5}, {"id": 2, "age": 5}, ] - open(json_path, "w").write(json.dumps(insert_dogs)) + open(json_path, "w").write(json.dumps(upsert_dogs)) result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"], catch_exceptions=False, ) assert 0 == result.exit_code, result.output - assert [ - {"id": 1, "name": "Cleo", "age": 4}, - {"id": 2, "name": "Nixie", "age": 4}, - ] == list(db.query("select * from dogs order by id")) + assert list(db.query("select * from dogs order by id")) == [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Nixie", "age": 5}, + ] def test_upsert_alter(db_path, tmpdir): @@ -1596,47 +1623,112 @@ def test_add_foreign_keys(db_path): [ ( [], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--type", "age", "text"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] TEXT NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] TEXT NOT NULL DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--drop", "age"], - 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)', + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [name] TEXT\n" + ")" + ), ), ( ["--rename", "age", "age2", "--rename", "id", "pk"], - "CREATE TABLE \"dogs\" (\n [pk] INTEGER PRIMARY KEY,\n [age2] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [pk] INTEGER PRIMARY KEY,\n" + " [age2] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--not-null", "name"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT NOT NULL\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT NOT NULL\n" + ")" + ), ), ( ["--not-null-false", "age"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--pk", "name"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT PRIMARY KEY\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT PRIMARY KEY\n" + ")" + ), ), ( ["--pk-none"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--default", "name", "Turnip"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT DEFAULT 'Turnip'\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT DEFAULT 'Turnip'\n" + ")" + ), ), ( ["--default-none", "age"], - 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL,\n [name] TEXT\n)', + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER NOT NULL,\n" + " [name] TEXT\n" + ")" + ), ), ( ["-o", "name", "--column-order", "age", "-o", "id"], - "CREATE TABLE \"dogs\" (\n [name] TEXT,\n [age] INTEGER NOT NULL DEFAULT '1',\n [id] INTEGER PRIMARY KEY\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [name] TEXT,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [id] INTEGER PRIMARY KEY\n" + ")" + ), ), ], ) @@ -1685,9 +1777,13 @@ def test_transform_drop_foreign_key(db_path): print(result.output) assert result.exit_code == 0 schema = db["places"].schema - assert ( - schema - == 'CREATE TABLE "places" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [country] INTEGER,\n [city] INTEGER REFERENCES [city]([id])\n)' + assert schema == ( + 'CREATE TABLE "places" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [name] TEXT,\n" + " [country] INTEGER,\n" + " [city] INTEGER REFERENCES [city]([id])\n" + ")" ) @@ -1701,22 +1797,48 @@ _common_other_schema = ( [ ( [], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)', + ( + 'CREATE TABLE "trees" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [address] TEXT,\n" + " [species_id] INTEGER,\n" + " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n" + ")" + ), _common_other_schema, ), ( ["--table", "custom_table"], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_table_id] INTEGER,\n FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n)', + ( + 'CREATE TABLE "trees" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [address] TEXT,\n" + " [custom_table_id] INTEGER,\n" + " FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n" + ")" + ), "CREATE TABLE [custom_table] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)", ), ( ["--fk-column", "custom_fk"], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_fk] INTEGER,\n FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n)', + ( + 'CREATE TABLE "trees" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [address] TEXT,\n" + " [custom_fk] INTEGER,\n" + " FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n" + ")" + ), _common_other_schema, ), ( ["--rename", "name", "name2"], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)', + 'CREATE TABLE "trees" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [address] TEXT,\n" + " [species_id] INTEGER,\n" + " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n" + ")", "CREATE TABLE [species] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)", ), ], @@ -1902,7 +2024,10 @@ def test_indexes(tmpdir): ] -_TRIGGERS_EXPECTED = '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah AFTER INSERT ON articles\\nBEGIN\\n UPDATE counter SET count = count + 1;\\nEND"}]\n' +_TRIGGERS_EXPECTED = ( + '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah ' + 'AFTER INSERT ON articles\\nBEGIN\\n UPDATE counter SET count = count + 1;\\nEND"}]\n' +) @pytest.mark.parametrize( diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index e42e1cb..e465927 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -52,7 +52,7 @@ def test_memory_tsv(tmpdir, use_stdin): result = CliRunner().invoke( cli.cli, ["memory", path, "select * from {}".format(sql_from)], - input=data, + input=input, ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ @@ -102,7 +102,7 @@ def test_memory_json_nl(tmpdir, use_stdin): result = CliRunner().invoke( cli.cli, ["memory", path, "select * from {}".format(sql_from)], - input=data, + input=input, ) assert result.exit_code == 0, result.output assert json.loads(result.output.strip()) == [ diff --git a/tests/test_constructor.py b/tests/test_constructor.py index b3cd963..924df66 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -1,5 +1,4 @@ from sqlite_utils import Database -import pytest def test_recursive_triggers(): diff --git a/tests/test_conversions.py b/tests/test_conversions.py index ebe2a50..d70f5c8 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -1,6 +1,3 @@ -import pytest - - def test_insert_conversion(fresh_db): table = fresh_db["table"] table.insert({"foo": "bar"}, conversions={"foo": "upper(?)"}) diff --git a/tests/test_create.py b/tests/test_create.py index 2cf0b4d..ff36f90 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -2,7 +2,6 @@ from sqlite_utils.db import ( Index, Database, DescIndex, - ForeignKey, AlterError, NoObviousTable, ForeignKey, @@ -148,8 +147,8 @@ def test_create_table_with_not_null(fresh_db): ) def test_create_table_from_example(fresh_db, example, expected_columns): people_table = fresh_db["people"] - assert None == people_table.last_rowid - assert None == people_table.last_pk + assert people_table.last_rowid is None + assert people_table.last_pk is None people_table.insert(example) assert 1 == people_table.last_rowid assert 1 == people_table.last_pk @@ -515,7 +514,7 @@ def test_insert_row_alter_table( def test_insert_row_alter_table_invalid_column_characters(fresh_db): table = fresh_db["table"] - rowid = table.insert({"foo": "bar"}).last_pk + table.insert({"foo": "bar"}).last_pk with pytest.raises(AssertionError): table.insert({"foo": "baz", "new_col[abc]": 1.2}, alter=True) @@ -870,8 +869,6 @@ def test_works_with_pathlib_path(tmpdir): @pytest.mark.skipif(pd is None, reason="pandas and numpy are not installed") def test_create_table_numpy(fresh_db): - import numpy as np - df = pd.DataFrame({"col 1": range(3), "col 2": range(3)}) fresh_db["pandas"].insert_all(df.to_dict(orient="records")) assert [ diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index b70378e..7a52108 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -14,8 +14,31 @@ def test_enable_counts_specific_table(fresh_db): # Now enable counts foo.enable_counts() assert foo.triggers_dict == { - "foo_counts_insert": "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\nBEGIN\n INSERT OR REPLACE INTO [_counts]\n VALUES (\n 'foo',\n COALESCE(\n (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n 0\n ) + 1\n );\nEND", - "foo_counts_delete": "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\nBEGIN\n INSERT OR REPLACE INTO [_counts]\n VALUES (\n 'foo',\n COALESCE(\n (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n 0\n ) - 1\n );\nEND", + "foo_counts_insert": ( + "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\n" + "BEGIN\n" + " INSERT OR REPLACE INTO [_counts]\n" + " VALUES (\n 'foo',\n" + " COALESCE(\n" + " (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n" + " 0\n" + " ) + 1\n" + " );\n" + "END" + ), + "foo_counts_delete": ( + "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\n" + "BEGIN\n" + " INSERT OR REPLACE INTO [_counts]\n" + " VALUES (\n" + " 'foo',\n" + " COALESCE(\n" + " (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n" + " 0\n" + " ) - 1\n" + " );\n" + "END" + ), } assert fresh_db.table_names() == ["foo", "_counts"] assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}] diff --git a/tests/test_extract.py b/tests/test_extract.py index 280c59c..10b5b09 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import Index, InvalidColumns +from sqlite_utils.db import InvalidColumns import itertools import pytest diff --git a/tests/test_extracts.py b/tests/test_extracts.py index 0edd002..cca16ba 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import Index, ForeignKey +from sqlite_utils.db import Index import pytest diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 5759c5e..f12f865 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -2,6 +2,7 @@ from hypothesis import given import hypothesis.strategies as st import sqlite_utils + # SQLite integers are -(2^63) to 2^63 - 1 @given(st.integers(-9223372036854775808, 9223372036854775807)) def test_roundtrip_integers(integer): diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 6ca578c..cc33c46 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -33,7 +33,7 @@ def test_detect_fts(existing_db): assert "woo_fts" == existing_db["woo_fts"].detect_fts() assert "woo2_fts" == existing_db["woo2"].detect_fts() assert "woo2_fts" == existing_db["woo2_fts"].detect_fts() - assert None == existing_db["foo"].detect_fts() + assert existing_db["foo"].detect_fts() is None def test_tables(existing_db): @@ -175,9 +175,21 @@ def test_triggers_and_triggers_dict(fresh_db): (t.name, t.table) for t in fresh_db["authors"].triggers } expected_triggers = { - "authors_ai": "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND", - "authors_ad": "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\nEND", - "authors_au": "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND", + "authors_ai": ( + "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n" + " INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\n" + "END" + ), + "authors_ad": ( + "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n" + " INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n" + "END" + ), + "authors_au": ( + "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n" + " INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n" + " INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND" + ), } assert authors.triggers_dict == expected_triggers assert fresh_db["other"].triggers == [] diff --git a/tests/test_recreate.py b/tests/test_recreate.py index ddef115..504a0b8 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -15,7 +15,7 @@ def test_recreate_ignored_for_in_memory(): def test_recreate_not_allowed_for_connection(): conn = sqlite3.connect(":memory:") with pytest.raises(AssertionError): - db = Database(conn, recreate=True) + Database(conn, recreate=True) @pytest.mark.parametrize( diff --git a/tests/test_register_function.py b/tests/test_register_function.py index e6d977a..c99477e 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -55,14 +55,14 @@ def test_register_function_replace(fresh_db): # This will fail to replace the function: @fresh_db.register_function() - def one(): + def one(): # noqa: F811 return "two" assert "one" == fresh_db.execute("select one()").fetchone()[0] # This will replace it @fresh_db.register_function(replace=True) - def one(): + def one(): # noqa: F811 return "two" assert "two" == fresh_db.execute("select one()").fetchone()[0] diff --git a/tests/test_rows.py b/tests/test_rows.py index 3ac52f9..a8a4ca0 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -1,4 +1,3 @@ -from sqlite_utils.db import Index, View import pytest diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 094551a..d3ff22d 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -1,4 +1,3 @@ -import pytest from sqlite_utils import Database @@ -32,7 +31,9 @@ def test_tracer(): def test_with_tracer(): collected = [] - tracer = lambda sql, params: collected.append((sql, params)) + + def tracer(sql, params): + return collected.append((sql, params)) db = Database(memory=True) @@ -48,13 +49,39 @@ def test_with_tracer(): assert collected == [ ("select name from sqlite_master where type = 'view'", None), ( - "SELECT name FROM sqlite_master\n WHERE rootpage = 0\n AND (\n sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n OR (\n tbl_name = \"dogs\"\n AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n )\n )", + ( + "SELECT name FROM sqlite_master\n" + " WHERE rootpage = 0\n" + " AND (\n" + " sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n" + " OR (\n" + ' tbl_name = "dogs"\n' + " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" + " )\n" + " )" + ), None, ), ("select name from sqlite_master where type = 'view'", None), ("select sql from sqlite_master where name = ?", ("dogs_fts",)), ( - "with original as (\n select\n rowid,\n *\n from [dogs]\n)\nselect\n [original].*\nfrom\n [original]\n join [dogs_fts] on [original].rowid = [dogs_fts].rowid\nwhere\n [dogs_fts] match :query\norder by\n [dogs_fts].rank", + ( + "with original as (\n" + " select\n" + " rowid,\n" + " *\n" + " from [dogs]\n" + ")\n" + "select\n" + " [original].*\n" + "from\n" + " [original]\n" + " join [dogs_fts] on [original].rowid = [dogs_fts].rowid\n" + "where\n" + " [dogs_fts] match :query\n" + "order by\n" + " [dogs_fts].rank" + ), {"query": "Cleopaws"}, ), ] diff --git a/tests/test_transform.py b/tests/test_transform.py index 19e447e..06e5729 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -93,7 +93,10 @@ def test_transform_sql_table_with_primary_key( fresh_db, params, expected_sql, use_pragma_foreign_keys ): captured = [] - tracer = lambda sql, params: captured.append((sql, params)) + + def tracer(sql, params): + return captured.append((sql, params)) + dogs = fresh_db["dogs"] if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") @@ -163,7 +166,10 @@ def test_transform_sql_table_with_no_primary_key( fresh_db, params, expected_sql, use_pragma_foreign_keys ): captured = [] - tracer = lambda sql, params: captured.append((sql, params)) + + def tracer(sql, params): + return captured.append((sql, params)) + dogs = fresh_db["dogs"] if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 9b1990e..09bdacc 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -54,7 +54,7 @@ def test_upsert_compound_primary_key(fresh_db): ], pk=("species", "id"), ) - assert None == table.last_pk + assert table.last_pk is None table.upsert({"species": "dog", "id": 1, "age": 5}, pk=("species", "id")) assert ("dog", 1) == table.last_pk assert [ diff --git a/tests/test_wal.py b/tests/test_wal.py index 1303eed..23ca144 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -1,6 +1,5 @@ import pytest from sqlite_utils import Database -import sqlite3 @pytest.fixture From 02898bf7af4a4e484ecc8ec852d5fee98463277b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 18:23:12 -0700 Subject: [PATCH 061/102] Run flake8 in CI, refs #291 --- .github/workflows/test.yml | 4 +++- setup.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e19e61..5dc3fe1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: ${{ runner.os }}-pip- - name: Install dependencies run: | - pip install -e '.[test,mypy]' + pip install -e '.[test,mypy,flake8]' - name: Optionally install numpy if: matrix.numpy == 1 run: pip install numpy @@ -34,5 +34,7 @@ jobs: pytest - name: run mypy run: mypy sqlite_utils + - name: run flake8 + run: flake8 - name: Check formatting run: black . --check diff --git a/setup.py b/setup.py index a356f30..f5421ec 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ setup( "test": ["pytest", "black", "hypothesis"], "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], "mypy": ["mypy", "types-click", "types-tabulate"], + "flake8": ["flake8"], }, entry_points=""" [console_scripts] From 1fba60537dcac8be664de0e3ba2c66143cc996bd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 19:06:18 -0700 Subject: [PATCH 062/102] Try more aggressive noqa, refs #291 --- tests/test_register_function.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_register_function.py b/tests/test_register_function.py index c99477e..06f4857 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -55,14 +55,14 @@ def test_register_function_replace(fresh_db): # This will fail to replace the function: @fresh_db.register_function() - def one(): # noqa: F811 + def one(): # noqa return "two" assert "one" == fresh_db.execute("select one()").fetchone()[0] # This will replace it @fresh_db.register_function(replace=True) - def one(): # noqa: F811 + def one(): # noqa return "two" assert "two" == fresh_db.execute("select one()").fetchone()[0] From 93c7fd9868fed3193a1732b39bfac539e5812b0b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Jun 2021 19:08:52 -0700 Subject: [PATCH 063/102] Ignore this entire file, refs #291 --- tests/test_register_function.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_register_function.py b/tests/test_register_function.py index 06f4857..19af0b6 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -1,3 +1,4 @@ +# flake8: noqa import pytest import sys from unittest.mock import MagicMock From 33c9d0087983a99ba18cbe87fe92ea722caa499c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Jun 2021 15:54:38 -0700 Subject: [PATCH 064/102] Documentation for db.query(), closes #290 --- docs/python-api.rst | 77 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 85f0296..7f3cdd5 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1,8 +1,8 @@ .. _python_api: -============ - Python API -============ +============================= + sqlite_utils Python library +============================= .. contents:: :local: @@ -59,12 +59,11 @@ You can attach an additional database using the ``.attach()`` method, providing db = Database("first.db") db.attach("second", "second.db") # Now you can run queries like this one: - cursor = db.execute(""" + print(db.query(""" select * from table_in_first union all select * from second.table_in_second - """) - print(cursor.fetchall()) + """)) You can reference tables in the attached database using the alias value you passed to ``db.attach(alias, filepath)`` as a prefix, for example the ``second.table_in_second`` reference in the SQL query above. @@ -97,27 +96,77 @@ You can also turn on a tracer function temporarily for a block of code using the This example will print queries only for the duration of the ``with`` block. -.. _python_api_execute: +.. _python_api_executing_queries: Executing queries ================= -The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around ``.execute()`` and ``.executescript()`` on the underlying SQLite connection. These wrappers log to the tracer function if one has been registered. +The ``Database`` class offers several methods for directly executing SQL queries. + +.. _python_api_query: + +db.query(sql, params) +--------------------- + +The ``db.query(sql)`` function executes a SQL query and returns an iterator over Python dictionaries representing the resulting rows: + +.. code-block:: python + + db = Database(memory=True) + db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) + for row in db.query("select * from dogs"): + print(row) + # Outputs: + # {'name': 'Cleo'} + # {'name': 'Pancakes'} + +.. _python_api_execute: + +db.execute(sql, params) +----------------------- + +The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around ``.execute()`` and ``.executescript()`` on the underlying SQLite connection. These wrappers log to the :ref:`tracer function ` if one has been registered. + +``db.execute(sql)`` returns a `sqlite3.Cursor `__ that was used to execute the SQL. .. code-block:: python db = Database(memory=True) db["dogs"].insert({"name": "Cleo"}) - db.execute("update dogs set name = 'Cleopaws'") + cursor = db.execute("update dogs set name = 'Cleopaws'") + print(cursor.rowcount) + # Outputs the number of rows affected by the update + # In this case 2 -You can pass parameters as an optional second argument, using either a list or a dictionary. These will be correctly quoted and escaped. +Other cursor methods such as ``.fetchone()`` and ``.fetchall()`` are also available, see the `standard library documentation `__. + +.. _python_api_parameters: + +Passing parameters +------------------ + +Both ``db.query()`` and ``db.execute()`` accept an optional second argument for parameters to be passed to the SQL query. + +This can take the form of either a tuple/list or a dictionary, depending on the type of parameters used in the query. Values passed in this way will be correctly quoted and escaped, helping avoid XSS vulnerabilities. + +``?`` parameters in the SQL query can be filled in using a list: .. code-block:: python - # Using ? and a list: db.execute("update dogs set name = ?", ["Cleopaws"]) - # Or using :name and a dictionary: - db.execute("update dogs set name = :name", {"name": "Cleopaws"}) + # This will rename ALL dogs to be called "Cleopaws" + +Named parameters using ``:name`` can be filled using a dictionary: + +.. code-block:: python + + dog = next(db.query( + "select rowid, name from dogs where name = :name", + {"name": "Cleopaws"} + )) + # dog is now {'rowid': 1, 'name': 'Cleopaws'} + +In this example ``next()`` is used to retrieve the first result in the iterator returned by the ``db.query()`` method. .. _python_api_table: @@ -2222,7 +2271,7 @@ If you want to deliberately replace the registered function with a new implement Quoting strings for use in SQL ============================== -In almost all cases you should pass values to your SQL queries using the optional ``parameters`` argument to ``db.execute()``, as described in :ref:`python_api_execute`. +In almost all cases you should pass values to your SQL queries using the optional ``parameters`` argument to ``db.query()``, as described in :ref:`python_api_parameters`. If that option isn't relevant to your use-case you can to quote a string for use with SQLite using the ``db.quote()`` method, like so: From 747be6057d09a4e5d9d726e29d5cf99b10c59dea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Jun 2021 16:06:12 -0700 Subject: [PATCH 065/102] Added some more types, refs #266, #290 --- sqlite_utils/db.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 50fd496..ae87e27 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -14,6 +14,7 @@ import re from sqlite_fts4 import rank_bm25 # type: ignore import sys import textwrap +from typing import Generator, Iterable, Union, Optional, List import uuid SQLITE_MAX_VARS = 999 @@ -359,13 +360,17 @@ class Database: for table in tables ) - def query(self, sql, params=None): + def query( + self, sql: str, params: Optional[Union[Iterable, dict]] = None + ) -> Generator[dict, None, None]: cursor = self.execute(sql, params or tuple()) keys = [d[0] for d in cursor.description] for row in cursor: yield dict(zip(keys, row)) - def execute_returning_dicts(self, sql, params=None): + def execute_returning_dicts( + self, sql: str, params: Optional[Union[Iterable, dict]] = None + ) -> List[dict]: return list(self.query(sql, params)) def resolve_foreign_keys(self, name, foreign_keys): From fec6cd55cab7ee91046ca4ee278b90cd045a32c2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 25 Jun 2021 10:53:46 -0700 Subject: [PATCH 066/102] Contributing documentation, closes #292 --- docs/contributing.rst | 67 +++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 68 insertions(+) create mode 100644 docs/contributing.rst diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..ec1a4a7 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1,67 @@ +.. _contributing: + +============== + Contributing +============== + +To work on this library locally, first checkout the code. Then create a new virtual environment:: + + git clone git@github.com:simonw/sqlite-utils + cd sqlite-utils + python3 -mvenv venv + source venv/bin/activate + +Or if you are using ``pipenv``:: + + pipenv shell + +Within the virtual environment running ``sqlite-utils`` should run your locally editable version of the tool. You can use ``which sqlite-utils`` to confirm that you are running the version that lives in your virtual environment. + +.. _contributing_tests: + +Running the tests +================= + +To install the dependencies and test dependencies:: + + pip install -e '.[test]' + +To run the tests:: + + pytest + +.. _contributing_docs: + +Building the documentation +========================== + +To build the documentation, first install the documentation dependencies:: + + pip install -e '.[docs]' + +Then run ``make livehtml`` from the ``docs/`` directory to start a server on port 8000 that will serve the documentation and live-reload any time you make an edit to a ``.rst`` file:: + + cd docs + make livehtml + +.. _contributing_linting: + +Linting and formatting +====================== + +``sqlite-utils`` uses `Black `__ for code formatting, and `flake8 `__ and `mypy `__ for linting and type checking. + +Black is installed as part of ``pip install -e '.[test]'`` - you can then format your code by running it in the root of the project:: + + black . + +To install ``mypy`` and ``flake8`` run the following:: + + pip install -e '.[flake8,mypy]' + +Both commands can then be run in the root of the project like this:: + + flake8 + mypy sqlite_utils + +All three of these tools are run by our CI mechanism against every commit and pull request. diff --git a/docs/index.rst b/docs/index.rst index b1bfc0e..93b0bc0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,7 @@ Contents installation cli python-api + contributing changelog Take a look at `this script `_ for an example of this library in action. From 8981b9c1f3e4e3865924861ee63922f696078f6c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 25 Jun 2021 10:59:05 -0700 Subject: [PATCH 067/102] Release 3.12 Refs #290, #291, #292 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0c31e79..57bff4a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v3_12: + +3.12 (2021-06-25) +----------------- + +- New :ref:`db.query(sql, params) ` method, which executes a SQL query and returns the results as an iterator over Python dictionaries. (`#290 `__) +- This project now uses ``flake8`` and has started to use ``mypy``. (`#291 `__) +- New documentation on :ref:`contributing ` to this project. (`#292 `__) + .. _v3_11: 3.11 (2021-06-20) diff --git a/setup.py b/setup.py index f5421ec..02224b3 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.11" +VERSION = "3.12" def get_long_description(): From 8286a66413bc466db11b3b7e0e75826efbd7850e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Jun 2021 09:35:01 -0700 Subject: [PATCH 068/102] sqlite-utils memory --help now mentions --schema --- sqlite_utils/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 2de6bf3..8a4910f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1212,6 +1212,10 @@ def memory( cat animals.csv | sqlite-utils memory stdin:csv places.dat:nl \\ "select * from stdin where place_id in (select id from places)" + Use --schema to view the SQL schema of any imported files: + + \b + sqlite-utils memory animals.csv --schema """ db = sqlite_utils.Database(memory=True) # If --dump or --save used but no paths detected, assume SQL query is a path: From ab8d4aad0c42f905640981f6f24bc1e37205ae62 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 24 Jul 2021 15:08:36 -0700 Subject: [PATCH 069/102] sqlite-utils schema now takes optional tables, closes #299 --- docs/cli.rst | 5 ++++ sqlite_utils/cli.py | 10 ++++++-- tests/test_cli.py | 57 +++++++++++++++++++++++++++++++++------------ 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index a404333..d8993e4 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -525,6 +525,11 @@ The ``sqlite-utils schema`` command shows the full SQL schema for the database:: [name] TEXT ); +This will show the schema for every table and index in the database. To view the schema just for a specified subset of tables pass those as additional arguments:: + + $ sqlite-utils schema dogs.db dogs chickens + ... + .. _cli_analyze_tables: Analyzing tables diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8a4910f..b168ec0 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1550,15 +1550,21 @@ def indexes( type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), required=True, ) +@click.argument("tables", nargs=-1, required=False) @load_extension_option def schema( path, + tables, load_extension, ): - "Show full schema for this database" + "Show full schema for this database or for specified tables" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - click.echo(db.schema) + if tables: + for table in tables: + click.echo(db[table].schema) + else: + click.echo(db.schema) @cli.command() diff --git a/tests/test_cli.py b/tests/test_cli.py index 37a96a3..e84453a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2064,7 +2064,46 @@ def test_triggers(tmpdir, extra_args, expected): assert result.output == expected -def test_schema(tmpdir): +@pytest.mark.parametrize( + "options,expected", + ( + ( + [], + ( + "CREATE TABLE [dogs] (\n" + " [id] INTEGER,\n" + " [name] TEXT\n" + ");\n" + "CREATE TABLE [chickens] (\n" + " [id] INTEGER,\n" + " [name] TEXT,\n" + " [breed] TEXT\n" + ");\n" + "CREATE INDEX [idx_chickens_breed]\n" + " ON [chickens] ([breed]);\n" + ), + ), + ( + ["dogs"], + ("CREATE TABLE [dogs] (\n" " [id] INTEGER,\n" " [name] TEXT\n" ")\n"), + ), + ( + ["chickens", "dogs"], + ( + "CREATE TABLE [chickens] (\n" + " [id] INTEGER,\n" + " [name] TEXT,\n" + " [breed] TEXT\n" + ")\n" + "CREATE TABLE [dogs] (\n" + " [id] INTEGER,\n" + " [name] TEXT\n" + ")\n" + ), + ), + ), +) +def test_schema(tmpdir, options, expected): db_path = str(tmpdir / "test.db") db = Database(db_path) db["dogs"].create({"id": int, "name": str}) @@ -2072,23 +2111,11 @@ def test_schema(tmpdir): db["chickens"].create_index(["breed"]) result = CliRunner().invoke( cli.cli, - ["schema", db_path], + ["schema", db_path] + options, catch_exceptions=False, ) assert result.exit_code == 0 - assert result.output == ( - "CREATE TABLE [dogs] (\n" - " [id] INTEGER,\n" - " [name] TEXT\n" - ");\n" - "CREATE TABLE [chickens] (\n" - " [id] INTEGER,\n" - " [name] TEXT,\n" - " [breed] TEXT\n" - ");\n" - "CREATE INDEX [idx_chickens_breed]\n" - " ON [chickens] ([breed]);\n" - ) + assert result.output == expected def test_long_csv_column_value(tmpdir): From c7e8d72be9fe8fe0811f685a18eebc637662d41b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 24 Jul 2021 15:15:27 -0700 Subject: [PATCH 070/102] Release 3.13 Refs #299 --- docs/changelog.rst | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 57bff4a..9d17c1e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,14 @@ Changelog =========== +.. _v3_13: + +3.13 (2021-07-24) +----------------- + +- ``sqlite-utils schema my.db table1 table2`` command now accepts optional table names. (`#299 `__) +- ``sqlite-utils memory --help`` now describes the ``--schema`` option. + .. _v3_12: 3.12 (2021-06-25) diff --git a/setup.py b/setup.py index 02224b3..eeee83b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.12" +VERSION = "3.13" def get_long_description(): From 5ec6686153e29ae10d4921a1ad4c841f192f20e2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 1 Aug 2021 21:47:39 -0700 Subject: [PATCH 071/102] sqlite-utils convert command and db[table].convert(...) method Closes #251, closes #302. --- docs/cli.rst | 106 +++++++++ docs/python-api.rst | 41 +++- setup.py | 10 +- sqlite_utils/cli.py | 137 +++++++++++- sqlite_utils/db.py | 111 +++++++++- sqlite_utils/recipes.py | 19 ++ sqlite_utils/utils.py | 19 +- tests/test_cli_convert.py | 441 ++++++++++++++++++++++++++++++++++++++ tests/test_convert.py | 77 +++++++ tests/test_docs.py | 33 ++- tests/test_recipes.py | 108 ++++++++++ 11 files changed, 1093 insertions(+), 9 deletions(-) create mode 100644 sqlite_utils/recipes.py create mode 100644 tests/test_cli_convert.py create mode 100644 tests/test_convert.py create mode 100644 tests/test_recipes.py diff --git a/docs/cli.rst b/docs/cli.rst index d8993e4..7bc7a3b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -920,6 +920,112 @@ The ``-`` argument indicates data should be read from standard input. The string When inserting data from standard input only the following column definitions are supported: ``name``, ``path``, ``content``, ``sha256``, ``md5`` and ``size``. +.. _cli_convert: + +Converting data in columns +========================== + +The ``convert`` command can be used to transform the data in a specified column - for example to parse a date string into an ISO timestamp, or to split a string of tags into a JSON array. + +The command accepts a database, table, one or more columns and a string of Python code to be executed against the values from those columns. The following example would replace the values in the ``headline`` column in the ``articles`` table with an upper-case version:: + + $ sqlite-utils convert content.db articles headline 'value.upper()' + +The Python code is passed as a string. Within that Python code the ``value`` variable will be the value of the current column. + +The code you provide will be compiled into a function that takes ``value`` as a single argument. If you break your function body into multiple lines the last line should be a ``return`` statement:: + + $ sqlite-utils convert content.db articles headline ' + value = str(value) + return value.upper()' + +You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options:: + + $ sqlite-utils convert content.db articles content \ + '"\n".join(textwrap.wrap(value, 10))' \ + --import=textwrap + +The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. + +.. _cli_convert_recipes: + +sqlite-utils convert recipes +---------------------------- + +Various built-in recipe functions are available for common operations. These are: + +``r.jsonsplit(value, delimiter=',', type=)`` + Convert a string like ``a,b,c`` into a JSON array ``["a", "b", "c"]`` + + The ``delimiter`` parameter can be used to specify a different delimiter. + + The ``type`` parameter can be set to ``float`` or ``int`` to produce a JSON array of different types, for example if the column's string value was ``1.2,3,4`` the following:: + + r.jsonsplit(value, type=float) + + Would produce an array like this: ``[1.2, 3.0, 4.5]`` + +``r.parsedate(value, dayfirst=False, yearfirst=False)`` + Parse a date and convert it to ISO date format: ``yyyy-mm-dd`` + + In the case of dates such as ``03/04/05`` U.S. ``MM/DD/YY`` format is assumed - you can use ``dayfirst=True`` or ``yearfirst=True`` to change how these ambiguous dates are interpreted. + +``r.parsedatetime(value, dayfirst=False, yearfirst=False)`` + Parse a datetime and convert it to ISO datetime format: ``yyyy-mm-ddTHH:MM:SS`` + +These recipes can be used in the code passed to ``sqlite-utils convert`` like this:: + + $ sqlite-utils convert my.db mytable mycolumn \ + 'r.jsonsplit(value, delimiter=":")' + +.. _cli_convert_output: + +Saving the result to a different column +--------------------------------------- + +The ``--output`` and ``--output-type`` options can be used to save the result of the conversion to a separate column, which will be created if that column does not already exist:: + + $ sqlite-utils convert content.db articles headline 'value.upper()' \ + --output headline_upper + +The type of the created column defaults to ``text``, but a different column type can be specified using ``--output-type``. This example will create a new floating point column called ``id_as_a_float`` with a copy of each item's ID increased by 0.5:: + + $ sqlite-utils convert content.db articles id 'float(value) + 0.5' \ + --output id_as_a_float \ + --output-type float + +You can drop the original column at the end of the operation by adding ``--drop``. + +.. _cli_convert_multi: + +Converting a column into multiple columns +----------------------------------------- + +Sometimes you may wish to convert a single column into multiple derived columns. For example, you may have a ``location`` column containing ``latitude,longitude`` values which you wish to split out into separate ``latitude`` and ``longitude`` columns. + +You can achieve this using the ``--multi`` option to ``sqlite-utils convert``. This option expects your Python code to return a Python dictionary: new columns well be created and populated for each of the keys in that dictionary. + +For the ``latitude,longitude`` example you would use the following:: + + $ sqlite-utils convert demo.db places location \ + 'bits = value.split(",") + return { + "latitude": float(bits[0]), + "longitude": float(bits[1]), + }' --multi + +The type of the returned values will be taken into account when creating the new columns. In this example, the resulting database schema will look like this: + +.. code-block:: sql + + CREATE TABLE [places] ( + [location] TEXT, + [latitude] FLOAT, + [longitude] FLOAT + ); + +The code function can also return ``None``, in which case its output will be ignored. You can drop the original column at the end of the operation by adding ``--drop``. + .. _cli_create_table: Creating tables diff --git a/docs/python-api.rst b/docs/python-api.rst index 7f3cdd5..7f490ab 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -702,7 +702,7 @@ You can delete all records in a table that match a specific WHERE statement usin >>> db = sqlite_utils.Database("dogs.db") >>> # Delete every dog with age less than 3 - >>> db["dogs"].delete_where("age < ?", [3]): + >>> db["dogs"].delete_where("age < ?", [3]) Calling ``table.delete_where()`` with no other arguments will delete every row in the table. @@ -736,6 +736,45 @@ An ``upsert_all()`` method is also available, which behaves like ``insert_all()` .. note:: ``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 `__ for details of this change. +.. _python_api_convert: + +Converting data in columns +========================== + +The ``table.convert(...)`` method can be used to apply a conversion function to the values in a column, either to update that column or to populate new columns. It is the Python library equivalent of the :ref:`sqlite-utils convert ` command. + +This feature works by registering a custom SQLite function that applies a Python transformation, then running a SQL query equivalent to ``UPDATE table SET column = convert_value(column);`` + +To transform a specific column to uppercase, you would use the following: + +.. code-block:: python + + db["dogs"].convert("name", lambda value: value.upper()) + +You can pass a list of columns, in which case the transformation will be applied to each one: + +.. code-block:: python + + db["dogs"].convert(["name", "twitter"], lambda value: value.upper()) + +To save the output to of the transformation to a different column, use the ``output=`` parameter: + +.. code-block:: python + + db["dogs"].convert("name", lambda value: value.upper(), output="name_upper") + +This will add the new column, if it does not already exist. You can pass ``output_type=int`` or some other type to control the type of the new column - otherwise it will default to text. + +If you want to drop the original column after saving the results in a separate output column, pass ``drop=True``. + +You can create multiple new columns from a single input column by passing ``multi=True`` and a conversion function that returns a Python dictionary. This example creates new ``upper`` and ``lower`` columns populated from the single ``title`` column: + +.. code-block:: python + + table.convert( + "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True + ) + .. _python_api_lookup_tables: Working with lookup tables diff --git a/setup.py b/setup.py index eeee83b..5009736 100644 --- a/setup.py +++ b/setup.py @@ -22,12 +22,18 @@ setup( version=VERSION, license="Apache License, Version 2.0", packages=find_packages(exclude=["tests", "tests.*"]), - install_requires=["sqlite-fts4", "click", "click-default-group", "tabulate"], + install_requires=[ + "sqlite-fts4", + "click", + "click-default-group", + "tabulate", + "dateutils", + ], setup_requires=["pytest-runner"], extras_require={ "test": ["pytest", "black", "hypothesis"], "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], - "mypy": ["mypy", "types-click", "types-tabulate"], + "mypy": ["mypy", "types-click", "types-tabulate", "types-python-dateutil"], "flake8": ["flake8"], }, entry_points=""" diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index b168ec0..e1b770f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -5,8 +5,10 @@ from datetime import datetime import hashlib import pathlib import sqlite_utils -from sqlite_utils.db import AlterError, DescIndex +from sqlite_utils.db import AlterError, BadMultiValues, DescIndex +from sqlite_utils import recipes import textwrap +import inspect import io import itertools import json @@ -1903,6 +1905,139 @@ def analyze_tables( click.echo(details) +def _generate_convert_help(): + help = textwrap.dedent( + """ + Convert columns using Python code you supply. For example: + + \b + $ sqlite-utils convert my.db mytable mycolumn \\ + '"\\n".join(textwrap.wrap(value, 10))' \\ + --import=textwrap + + "value" is a variable with the column value to be converted. + + The following common operations are available as recipe functions: + """ + ).strip() + recipe_names = [ + n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser") + ] + for name in recipe_names: + fn = getattr(recipes, name) + help += "\n\nr.{}{}\n\n {}".format( + name, str(inspect.signature(fn)), fn.__doc__ + ) + help += "\n\n" + help += textwrap.dedent( + """ + You can use these recipes like so: + + \b + $ sqlite-utils convert my.db mytable mycolumn \\ + 'r.jsonsplit(value, delimiter=":")' + """ + ).strip() + return help + + +@cli.command(help=_generate_convert_help()) +@click.argument( + "db_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table", type=str) +@click.argument("columns", type=str, nargs=-1, required=True) +@click.argument("code", type=str) +@click.option( + "--import", "imports", type=str, multiple=True, help="Python modules to import" +) +@click.option( + "--dry-run", is_flag=True, help="Show results of running this against first 10 rows" +) +@click.option( + "--multi", is_flag=True, help="Populate columns for keys in returned dictionary" +) +@click.option("--output", help="Optional separate column to populate with the output") +@click.option( + "--output-type", + help="Column type to use for the output column", + default="text", + type=click.Choice(["integer", "float", "blob", "text"]), +) +@click.option("--drop", is_flag=True, help="Drop original column afterwards") +@click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") +def convert( + db_path, + table, + columns, + code, + imports, + dry_run, + multi, + output, + output_type, + drop, + silent, +): + sqlite3.enable_callback_tracebacks(True) + db = sqlite_utils.Database(db_path) + if output is not None and len(columns) > 1: + raise click.ClickException("Cannot use --output with more than one column") + if multi and len(columns) > 1: + raise click.ClickException("Cannot use --multi with more than one column") + if drop and not (output or multi): + raise click.ClickException("--drop can only be used with --output or --multi") + # If single line and no 'return', add the return + if "\n" not in code and not code.strip().startswith("return "): + code = "return {}".format(code) + # Compile the code into a function body called fn(value) + new_code = ["def fn(value):"] + for line in code.split("\n"): + new_code.append(" {}".format(line)) + code_o = compile("\n".join(new_code), "", "exec") + locals = {} + globals = {"r": recipes, "recipes": recipes} + for import_ in imports: + globals[import_] = __import__(import_) + exec(code_o, globals, locals) + fn = locals["fn"] + if dry_run: + # Pull first 20 values for first column and preview them + db.conn.create_function("preview_transform", 1, lambda v: fn(v) if v else v) + sql = """ + select + [{column}] as value, + preview_transform([{column}]) as preview + from [{table}] limit 10 + """.format( + column=columns[0], table=table + ) + for row in db.conn.execute(sql).fetchall(): + click.echo(str(row[0])) + click.echo(" --- becomes:") + click.echo(str(row[1])) + click.echo() + else: + try: + db[table].convert( + columns, + fn, + output=output, + output_type=output_type, + drop=drop, + multi=multi, + show_progress=not silent, + ) + except BadMultiValues as e: + raise click.ClickException( + "When using --multi code must return a Python dictionary - returned: {}".format( + repr(e.values) + ) + ) + + def _render_common(title, values): if values is None: return "" diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ae87e27..eb714e5 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,4 +1,11 @@ -from .utils import sqlite3, OperationalError, suggest_column_types, column_affinity +from .utils import ( + sqlite3, + OperationalError, + suggest_column_types, + types_for_column_types, + column_affinity, + progressbar, +) from collections import namedtuple from collections.abc import Mapping import contextlib @@ -153,6 +160,13 @@ class DescIndex(str): pass +class BadMultiValues(Exception): + "With multi=True code must return a Python dictionary" + + def __init__(self, values): + self.values = values + + _COUNTS_TABLE_CREATE_SQL = """ CREATE TABLE IF NOT EXISTS [{}]( [table] TEXT PRIMARY KEY, @@ -1697,6 +1711,101 @@ class Table(Queryable): self.last_pk = pk_values[0] if len(pks) == 1 else pk_values return self + def convert( + self, + columns, + fn, + output=None, + output_type=None, + drop=False, + multi=False, + show_progress=False, + ): + if isinstance(columns, str): + columns = [columns] + + if multi: + return self._convert_multi( + columns[0], fn, drop=drop, show_progress=show_progress + ) + + if output is not None: + assert len(columns) == 1, "output= can only be used with a single column" + if output not in self.columns_dict: + self.add_column(output, output_type or "text") + + todo_count = self.count * len(columns) + with progressbar(length=todo_count, silent=not show_progress) as bar: + + def convert_value(v): + bar.update(1) + if not v: + return v + return fn(v) + + self.db.register_function(convert_value) + sql = "update [{table}] set {sets};".format( + table=self.name, + sets=", ".join( + [ + "[{output_column}] = convert_value([{column}])".format( + output_column=output or column, column=column + ) + for column in columns + ] + ), + ) + with self.db.conn: + self.db.execute(sql) + if drop: + self.transform(drop=columns) + return self + + def _convert_multi(self, column, fn, drop, show_progress): + # First we execute the function + pk_to_values = {} + new_column_types = {} + pks = [column.name for column in self.columns if column.is_pk] + if not pks: + pks = ["rowid"] + + with progressbar( + length=self.count, silent=not show_progress, label="1: Evaluating" + ) as bar: + for row in self.rows_where( + select=", ".join( + "[{}]".format(column_name) for column_name in (pks + [column]) + ) + ): + row_pk = tuple(row[pk] for pk in pks) + if len(row_pk) == 1: + row_pk = row_pk[0] + values = fn(row[column]) + if values is not None and not isinstance(values, dict): + raise BadMultiValues(values) + if values: + for key, value in values.items(): + new_column_types.setdefault(key, set()).add(type(value)) + pk_to_values[row_pk] = values + bar.update(1) + + # Add any new columns + columns_to_create = types_for_column_types(new_column_types) + for column_name, column_type in columns_to_create.items(): + if column_name not in self.columns_dict: + self.add_column(column_name, column_type) + + # Run the updates + with progressbar( + length=self.count, silent=not show_progress, label="2: Updating" + ) as bar: + with self.db.conn: + for pk, updates in pk_to_values.items(): + self.update(pk, updates) + bar.update(1) + if drop: + self.transform(drop=(column,)) + def build_insert_queries_and_params( self, extracts, diff --git a/sqlite_utils/recipes.py b/sqlite_utils/recipes.py new file mode 100644 index 0000000..6918661 --- /dev/null +++ b/sqlite_utils/recipes.py @@ -0,0 +1,19 @@ +from dateutil import parser +import json + + +def parsedate(value, dayfirst=False, yearfirst=False): + "Parse a date and convert it to ISO date format: yyyy-mm-dd" + return ( + parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).date().isoformat() + ) + + +def parsedatetime(value, dayfirst=False, yearfirst=False): + "Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS" + return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat() + + +def jsonsplit(value, delimiter=",", type=str): + 'Convert a string like a,b,c into a JSON array ["a", "b", "c"]' + return json.dumps([type(s.strip()) for s in value.split(delimiter)]) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 4f3c819..a781469 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -31,8 +31,11 @@ def suggest_column_types(records): for record in records: for key, value in record.items(): all_column_types.setdefault(key, set()).add(type(value)) - column_types = {} + return types_for_column_types(all_column_types) + +def types_for_column_types(all_column_types): + column_types = {} for key, types in all_column_types.items(): # Ignore null values if at least one other type present: if len(types) > 1: @@ -254,3 +257,17 @@ class ValueTracker: not_these.append(name) for key in not_these: del self.couldbe[key] + + +class NullProgressBar: + def update(self, value): + pass + + +@contextlib.contextmanager +def progressbar(silent=False, **kwargs): + if silent: + yield NullProgressBar() + else: + with click.progressbar(**kwargs) as bar: + yield bar diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py new file mode 100644 index 0000000..51634cd --- /dev/null +++ b/tests/test_cli_convert.py @@ -0,0 +1,441 @@ +from click.testing import CliRunner +from sqlite_utils import cli +import sqlite_utils +import json +import textwrap +import pathlib +import pytest + + +@pytest.fixture +def test_db_and_path(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["example"].insert_all( + [ + {"id": 1, "dt": "5th October 2019 12:04"}, + {"id": 2, "dt": "6th October 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ], + pk="id", + ) + return db, db_path + + +@pytest.fixture +def fresh_db_and_path(tmpdir): + db_path = str(pathlib.Path(tmpdir) / "data.db") + db = sqlite_utils.Database(db_path) + return db, db_path + + +@pytest.mark.parametrize( + "code", + [ + "return value.replace('October', 'Spooktober')", + # Return is optional: + "value.replace('October', 'Spooktober')", + ], +) +def test_convert_single_line(test_db_and_path, code): + db, db_path = test_db_and_path + result = CliRunner().invoke(cli.cli, ["convert", db_path, "example", "dt", code]) + assert 0 == result.exit_code, result.output + assert [ + {"id": 1, "dt": "5th Spooktober 2019 12:04"}, + {"id": 2, "dt": "6th Spooktober 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] == list(db["example"].rows) + + +def test_convert_multiple_lines(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "v = value.replace('October', 'Spooktober')\nreturn v.upper()", + ], + ) + assert 0 == result.exit_code, result.output + assert [ + {"id": 1, "dt": "5TH SPOOKTOBER 2019 12:04"}, + {"id": 2, "dt": "6TH SPOOKTOBER 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] == list(db["example"].rows) + + +def test_convert_import(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "return re.sub('O..', 'OXX', value)", + "--import", + "re", + ], + ) + assert 0 == result.exit_code, result.output + assert [ + {"id": 1, "dt": "5th OXXober 2019 12:04"}, + {"id": 2, "dt": "6th OXXober 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] == list(db["example"].rows) + + +def test_convert_dryrun(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "return re.sub('O..', 'OXX', value)", + "--import", + "re", + "--dry-run", + ], + ) + assert result.exit_code == 0 + assert result.output.strip() == ( + "5th October 2019 12:04\n" + " --- becomes:\n" + "5th OXXober 2019 12:04\n" + "\n" + "6th October 2019 00:05:06\n" + " --- becomes:\n" + "6th OXXober 2019 00:05:06\n" + "\n" + "\n" + " --- becomes:\n" + "\n" + "\n" + "None\n" + " --- becomes:\n" + "None" + ) + # But it should not have actually modified the table data + assert list(db["example"].rows) == [ + {"id": 1, "dt": "5th October 2019 12:04"}, + {"id": 2, "dt": "6th October 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] + + +@pytest.mark.parametrize("drop", (True, False)) +def test_convert_output_column(test_db_and_path, drop): + db, db_path = test_db_and_path + args = [ + "convert", + db_path, + "example", + "dt", + "value.replace('October', 'Spooktober')", + "--output", + "newcol", + ] + if drop: + args += ["--drop"] + result = CliRunner().invoke(cli.cli, args) + assert 0 == result.exit_code, result.output + expected = [ + { + "id": 1, + "dt": "5th October 2019 12:04", + "newcol": "5th Spooktober 2019 12:04", + }, + { + "id": 2, + "dt": "6th October 2019 00:05:06", + "newcol": "6th Spooktober 2019 00:05:06", + }, + {"id": 3, "dt": "", "newcol": ""}, + {"id": 4, "dt": None, "newcol": None}, + ] + if drop: + for row in expected: + del row["dt"] + assert list(db["example"].rows) == expected + + +@pytest.mark.parametrize( + "output_type,expected", + ( + ("text", [(1, "1"), (2, "2"), (3, "3"), (4, "4")]), + ("float", [(1, 1.0), (2, 2.0), (3, 3.0), (4, 4.0)]), + ("integer", [(1, 1), (2, 2), (3, 3), (4, 4)]), + (None, [(1, "1"), (2, "2"), (3, "3"), (4, "4")]), + ), +) +def test_convert_output_column_output_type(test_db_and_path, output_type, expected): + db, db_path = test_db_and_path + args = [ + "convert", + db_path, + "example", + "id", + "value", + "--output", + "new_id", + ] + if output_type: + args += ["--output-type", output_type] + result = CliRunner().invoke( + cli.cli, + args, + ) + assert 0 == result.exit_code, result.output + assert expected == list(db.execute("select id, new_id from example")) + + +@pytest.mark.parametrize( + "options,expected_error", + [ + ( + [ + "dt", + "id", + "value.replace('October', 'Spooktober')", + "--output", + "newcol", + ], + "Cannot use --output with more than one column", + ), + ( + [ + "dt", + "value.replace('October', 'Spooktober')", + "--output", + "newcol", + "--output-type", + "invalid", + ], + "Error: Invalid value for '--output-type'", + ), + ( + [ + "value.replace('October', 'Spooktober')", + ], + "Missing argument 'COLUMNS...'", + ), + ], +) +def test_convert_output_error(test_db_and_path, options, expected_error): + db_path = test_db_and_path[1] + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + ] + + options, + ) + assert result.exit_code != 0 + assert expected_error in result.output + + +@pytest.mark.parametrize("drop", (True, False)) +def test_convert_multi(fresh_db_and_path, drop): + db, db_path = fresh_db_and_path + db["creatures"].insert_all( + [ + {"id": 1, "name": "Simon"}, + {"id": 2, "name": "Cleo"}, + ], + pk="id", + ) + args = [ + "convert", + db_path, + "creatures", + "name", + "--multi", + '{"upper": value.upper(), "lower": value.lower()}', + ] + if drop: + args += ["--drop"] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 0, result.output + expected = [ + {"id": 1, "name": "Simon", "upper": "SIMON", "lower": "simon"}, + {"id": 2, "name": "Cleo", "upper": "CLEO", "lower": "cleo"}, + ] + if drop: + for row in expected: + del row["name"] + assert list(db["creatures"].rows) == expected + + +def test_convert_multi_complex_column_types(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["rows"].insert_all( + [ + {"id": 1}, + {"id": 2}, + {"id": 3}, + {"id": 4}, + ], + pk="id", + ) + code = textwrap.dedent( + """ + if value == 1: + return {"is_str": "", "is_float": 1.2, "is_int": None} + elif value == 2: + return {"is_float": 1, "is_int": 12} + elif value == 3: + return {"is_bytes": b"blah"} + """ + ) + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "rows", + "id", + "--multi", + code, + ], + ) + assert result.exit_code == 0, result.output + assert list(db["rows"].rows) == [ + {"id": 1, "is_str": "", "is_float": 1.2, "is_int": None, "is_bytes": None}, + {"id": 2, "is_str": None, "is_float": 1.0, "is_int": 12, "is_bytes": None}, + { + "id": 3, + "is_str": None, + "is_float": None, + "is_int": None, + "is_bytes": b"blah", + }, + {"id": 4, "is_str": None, "is_float": None, "is_int": None, "is_bytes": None}, + ] + assert db["rows"].schema == ( + "CREATE TABLE [rows] (\n" + " [id] INTEGER PRIMARY KEY\n" + ", [is_str] TEXT, [is_float] FLOAT, [is_int] INTEGER, [is_bytes] BLOB)" + ) + + +@pytest.mark.parametrize("delimiter", [None, ";", "-"]) +def test_recipe_jsonsplit(tmpdir, delimiter): + db_path = str(pathlib.Path(tmpdir) / "data.db") + db = sqlite_utils.Database(db_path) + db["example"].insert_all( + [ + {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])}, + {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])}, + ], + pk="id", + ) + code = "r.jsonsplit(value)" + if delimiter: + code = 'recipes.jsonsplit(value, delimiter="{}")'.format(delimiter) + args = ["convert", db_path, "example", "tags", code] + result = CliRunner().invoke(cli.cli, args) + assert 0 == result.exit_code, result.output + assert list(db["example"].rows) == [ + {"id": 1, "tags": '["foo", "bar"]'}, + {"id": 2, "tags": '["bar", "baz"]'}, + ] + + +@pytest.mark.parametrize( + "type,expected_array", + ( + (None, ["1", "2", "3"]), + ("float", [1.0, 2.0, 3.0]), + ("int", [1, 2, 3]), + ), +) +def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array): + db, db_path = fresh_db_and_path + db["example"].insert_all( + [ + {"id": 1, "records": "1,2,3"}, + ], + pk="id", + ) + code = "r.jsonsplit(value)" + if type: + code = "recipes.jsonsplit(value, type={})".format(type) + args = ["convert", db_path, "example", "records", code] + result = CliRunner().invoke(cli.cli, args) + assert 0 == result.exit_code, result.output + assert json.loads(db["example"].get(1)["records"]) == expected_array + + +@pytest.mark.parametrize("drop", (True, False)) +def test_recipe_jsonsplit_output(fresh_db_and_path, drop): + db, db_path = fresh_db_and_path + db["example"].insert_all( + [ + {"id": 1, "records": "1,2,3"}, + ], + pk="id", + ) + code = "r.jsonsplit(value)" + args = ["convert", db_path, "example", "records", code, "--output", "tags"] + if drop: + args += ["--drop"] + result = CliRunner().invoke(cli.cli, args) + assert 0 == result.exit_code, result.output + expected = { + "id": 1, + "records": "1,2,3", + "tags": '["1", "2", "3"]', + } + if drop: + del expected["records"] + assert db["example"].get(1) == expected + + +def test_cannot_use_drop_without_multi_or_output(fresh_db_and_path): + args = ["convert", fresh_db_and_path[1], "example", "records", "value", "--drop"] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 1, result.output + assert "Error: --drop can only be used with --output or --multi" in result.output + + +def test_cannot_use_multi_with_more_than_one_column(fresh_db_and_path): + args = [ + "convert", + fresh_db_and_path[1], + "example", + "records", + "othercol", + "value", + "--multi", + ] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 1, result.output + assert "Error: Cannot use --multi with more than one column" in result.output + + +def test_multi_with_bad_function(test_db_and_path): + args = [ + "convert", + test_db_and_path[1], + "example", + "dt", + "value.upper()", + "--multi", + ] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 1, result.output + assert "When using --multi code must return a Python dictionary" in result.output diff --git a/tests/test_convert.py b/tests/test_convert.py new file mode 100644 index 0000000..34e98f1 --- /dev/null +++ b/tests/test_convert.py @@ -0,0 +1,77 @@ +from sqlite_utils.db import BadMultiValues +import pytest + + +@pytest.mark.parametrize( + "columns,fn,expected", + ( + ( + "title", + lambda value: value.upper(), + {"title": "MIXED CASE", "abstract": "Abstract"}, + ), + ( + ["title", "abstract"], + lambda value: value.upper(), + {"title": "MIXED CASE", "abstract": "ABSTRACT"}, + ), + ), +) +def test_convert(fresh_db, columns, fn, expected): + table = fresh_db["table"] + table.insert({"title": "Mixed Case", "abstract": "Abstract"}) + table.convert(columns, fn) + assert list(table.rows) == [expected] + + +@pytest.mark.parametrize( + "drop,expected", + ( + (False, {"title": "Mixed Case", "other": "MIXED CASE"}), + (True, {"other": "MIXED CASE"}), + ), +) +def test_convert_output(fresh_db, drop, expected): + table = fresh_db["table"] + table.insert({"title": "Mixed Case"}) + table.convert("title", lambda v: v.upper(), output="other", drop=drop) + assert list(table.rows) == [expected] + + +def test_convert_output_multiple_column_error(fresh_db): + table = fresh_db["table"] + with pytest.raises(AssertionError) as excinfo: + table.convert(["title", "other"], lambda v: v, output="out") + assert "output= can only be used with a single column" in str(excinfo.value) + + +@pytest.mark.parametrize( + "type,expected", + ( + (int, {"other": 123}), + (float, {"other": 123.0}), + ), +) +def test_convert_output_type(fresh_db, type, expected): + table = fresh_db["table"] + table.insert({"number": "123"}) + table.convert("number", lambda v: v, output="other", output_type=type, drop=True) + assert list(table.rows) == [expected] + + +def test_convert_multi(fresh_db): + table = fresh_db["table"] + table.insert({"title": "Mixed Case"}) + table.convert( + "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True + ) + assert list(table.rows) == [ + {"title": "Mixed Case", "upper": "MIXED CASE", "lower": "mixed case"} + ] + + +def test_convert_multi_exception(fresh_db): + table = fresh_db["table"] + table.insert({"title": "Mixed Case"}) + with pytest.raises(BadMultiValues): + table.convert("title", lambda v: v.upper(), multi=True) diff --git a/tests/test_docs.py b/tests/test_docs.py index 87b685e..d760629 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,10 +1,12 @@ -from sqlite_utils import cli +from click.testing import CliRunner +from sqlite_utils import cli, recipes from pathlib import Path import pytest import re docs_path = Path(__file__).parent.parent / "docs" commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+) ") +recipes_re = re.compile(r"r\.(\w+)\(") @pytest.fixture(scope="session") @@ -17,11 +19,36 @@ def documented_commands(): } +@pytest.fixture(scope="session") +def documented_recipes(): + rst = (docs_path / "cli.rst").read_text() + return set(recipes_re.findall(rst)) + + @pytest.mark.parametrize("command", cli.cli.commands.keys()) def test_commands_are_documented(documented_commands, command): assert command in documented_commands @pytest.mark.parametrize("command", cli.cli.commands.values()) -def test_commands_have_docstrings(command): - assert command.__doc__, "{} is missing a docstring".format(command) +def test_commands_have_help(command): + assert command.help, "{} is missing its help".format(command) + + +def test_convert_help(): + result = CliRunner().invoke(cli.cli, ["convert", "--help"]) + assert result.exit_code == 0 + for expected in ( + "r.jsonsplit(value, ", + "r.parsedate(value, ", + "r.parsedatetime(value, ", + ): + assert expected in result.output + + +@pytest.mark.parametrize( + "recipe", + [n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")], +) +def test_recipes_are_documented(documented_recipes, recipe): + assert recipe in documented_recipes diff --git a/tests/test_recipes.py b/tests/test_recipes.py new file mode 100644 index 0000000..89240a2 --- /dev/null +++ b/tests/test_recipes.py @@ -0,0 +1,108 @@ +from sqlite_utils import recipes +import json +import pytest + + +@pytest.fixture +def dates_db(fresh_db): + fresh_db["example"].insert_all( + [ + {"id": 1, "dt": "5th October 2019 12:04"}, + {"id": 2, "dt": "6th October 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ], + pk="id", + ) + return fresh_db + + +def test_parsedate(dates_db): + dates_db["example"].convert("dt", recipes.parsedate) + assert list(dates_db["example"].rows) == [ + {"id": 1, "dt": "2019-10-05"}, + {"id": 2, "dt": "2019-10-06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] + + +def test_parsedatetime(dates_db): + dates_db["example"].convert("dt", recipes.parsedatetime) + assert list(dates_db["example"].rows) == [ + {"id": 1, "dt": "2019-10-05T12:04:00"}, + {"id": 2, "dt": "2019-10-06T00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] + + +@pytest.mark.parametrize( + "recipe,kwargs,expected", + ( + ("parsedate", {}, "2005-03-04"), + ("parsedate", {"dayfirst": True}, "2005-04-03"), + ("parsedatetime", {}, "2005-03-04T00:00:00"), + ("parsedatetime", {"dayfirst": True}, "2005-04-03T00:00:00"), + ), +) +def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected): + fresh_db["example"].insert_all( + [ + {"id": 1, "dt": "03/04/05"}, + ], + pk="id", + ) + fresh_db["example"].convert( + "dt", lambda value: getattr(recipes, recipe)(value, **kwargs) + ) + assert list(fresh_db["example"].rows) == [ + {"id": 1, "dt": expected}, + ] + + +@pytest.mark.parametrize("delimiter", [None, ";", "-"]) +def test_jsonsplit(fresh_db, delimiter): + fresh_db["example"].insert_all( + [ + {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])}, + {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])}, + ], + pk="id", + ) + fn = recipes.jsonsplit + if delimiter is not None: + + def fn(value): + return recipes.jsonsplit(value, delimiter=delimiter) + + fresh_db["example"].convert("tags", fn) + assert list(fresh_db["example"].rows) == [ + {"id": 1, "tags": '["foo", "bar"]'}, + {"id": 2, "tags": '["bar", "baz"]'}, + ] + + +@pytest.mark.parametrize( + "type,expected", + ( + (None, ["1", "2", "3"]), + (float, [1.0, 2.0, 3.0]), + (int, [1, 2, 3]), + ), +) +def test_jsonsplit_type(fresh_db, type, expected): + fresh_db["example"].insert_all( + [ + {"id": 1, "records": "1,2,3"}, + ], + pk="id", + ) + fn = recipes.jsonsplit + if type is not None: + + def fn(value): + return recipes.jsonsplit(value, type=type) + + fresh_db["example"].convert("records", fn) + assert json.loads(fresh_db["example"].get(1)["records"]) == expected From 4823aff4c33dd979bf61dcade2d3a6006d248372 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 1 Aug 2021 22:05:03 -0700 Subject: [PATCH 072/102] table.count_where() method, closes #305 --- docs/python-api.rst | 12 +++++++++++- sqlite_utils/db.py | 32 +++++++++++++++++++++++++------- tests/test_enable_counts.py | 2 +- tests/test_introspect.py | 9 ++++++++- 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 7f490ab..aba43ad 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -286,6 +286,16 @@ This method also accepts ``offset=`` and ``limit=`` arguments, for specifying an ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} +.. _python_api_rows_count_where: + +Counting rows +------------- + +To count the number of rows that would be returned by a where filter, use ``.count_where(where, where_args)``: + + >>> db["dogs"].count_where("age > ?", [1]): + 2 + .. _python_api_pks_and_rows_where: Listing rows with their primary keys @@ -1602,7 +1612,7 @@ The ``.count`` property shows the current number of rows (``select count(*) from >>> db["Street_Tree_List"].count 189144 -This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.execute_count()`` instead of accessing the property. +This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.count_where()`` instead of accessing the property. .. _python_api_introspection_columns: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index eb714e5..b5da058 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -682,14 +682,23 @@ class Queryable: self.db = db self.name = name + def count_where( + self, + where=None, + where_args=None, + ): + sql = "select count(*) from [{}]".format(self.name) + if where is not None: + sql += " where " + where + return self.db.execute(sql, where_args or []).fetchone()[0] + def execute_count(self): - return self.db.execute( - "select count(*) from [{}]".format(self.name) - ).fetchone()[0] + # Backwards compatibility, see https://github.com/simonw/sqlite-utils/issues/305#issuecomment-890713185 + return self.count_where() @property def count(self): - return self.execute_count() + return self.count_where() @property def rows(self): @@ -820,7 +829,7 @@ class Table(Queryable): counts = self.db.cached_counts([self.name]) if counts: return next(iter(counts.values())) - return self.execute_count() + return self.count_where() def exists(self): return self.name in self.db.table_names() @@ -1719,6 +1728,8 @@ class Table(Queryable): output_type=None, drop=False, multi=False, + where=None, + where_args=None, show_progress=False, ): if isinstance(columns, str): @@ -1726,7 +1737,12 @@ class Table(Queryable): if multi: return self._convert_multi( - columns[0], fn, drop=drop, show_progress=show_progress + columns[0], + fn, + drop=drop, + where=where, + where_args=where_args, + show_progress=show_progress, ) if output is not None: @@ -1761,7 +1777,9 @@ class Table(Queryable): self.transform(drop=columns) return self - def _convert_multi(self, column, fn, drop, show_progress): + def _convert_multi( + self, column, fn, drop, show_progress, where=None, where_args=None + ): # First we execute the function pk_to_values = {} new_column_types = {} diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index 7a52108..d724e80 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -132,7 +132,7 @@ def test_uses_counts_after_enable_counts(counts_db_path): assert db["foo"].count == 1 assert logged == [ ("select name from sqlite_master where type = 'view'", None), - ("select count(*) from [foo]", None), + ("select count(*) from [foo]", []), ] logged.clear() assert not db.use_counts_table diff --git a/tests/test_introspect.py b/tests/test_introspect.py index cc33c46..dce8afc 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -52,7 +52,14 @@ def test_views(fresh_db): def test_count(existing_db): - assert 3 == existing_db["foo"].count + assert existing_db["foo"].count == 3 + assert existing_db["foo"].count_where() == 3 + assert existing_db["foo"].execute_count() == 3 + + +def test_count_where(existing_db): + assert existing_db["foo"].count_where("text != ?", ["two"]) == 2 + assert existing_db["foo"].count_where("text != :t", {"t": "two"}) == 2 def test_columns(existing_db): From 69c7da5ec9698dabeb23379cc08d012b0cd8e6d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 11:33:56 -0700 Subject: [PATCH 073/102] Implemented .convert(..., where=, where_args=), refs #304 --- docs/python-api.rst | 8 ++++++++ sqlite_utils/db.py | 11 +++++++---- tests/test_convert.py | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index aba43ad..c372a62 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -785,6 +785,14 @@ You can create multiple new columns from a single input column by passing ``mult "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True ) +The ``.convert()`` method accepts optional ``where=`` and ``where_args=`` parameters which can be used to apply the conversion to a subset of rows specified by a where clause. Here's how to apply the conversion only to rows with an ``id`` that is higher than 20: + +.. code-block:: python + + table.convert("title", lambda v: v.upper(), where="id > :id", where_args={"id": 20}) + +These behave the same as the corresponding parameters to the :ref:`.rows_where() ` method, so you can use ``?`` placeholders and a list of values instead of ``:named`` placeholders with a dictionary. + .. _python_api_lookup_tables: Working with lookup tables diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b5da058..a21acc4 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1750,7 +1750,7 @@ class Table(Queryable): if output not in self.columns_dict: self.add_column(output, output_type or "text") - todo_count = self.count * len(columns) + todo_count = self.count_where(where, where_args) * len(columns) with progressbar(length=todo_count, silent=not show_progress) as bar: def convert_value(v): @@ -1760,7 +1760,7 @@ class Table(Queryable): return fn(v) self.db.register_function(convert_value) - sql = "update [{table}] set {sets};".format( + sql = "update [{table}] set {sets}{where};".format( table=self.name, sets=", ".join( [ @@ -1770,9 +1770,10 @@ class Table(Queryable): for column in columns ] ), + where=" where {}".format(where) if where is not None else "", ) with self.db.conn: - self.db.execute(sql) + self.db.execute(sql, where_args or []) if drop: self.transform(drop=columns) return self @@ -1793,7 +1794,9 @@ class Table(Queryable): for row in self.rows_where( select=", ".join( "[{}]".format(column_name) for column_name in (pks + [column]) - ) + ), + where=where, + where_args=where_args, ): row_pk = tuple(row[pk] for pk in pks) if len(row_pk) == 1: diff --git a/tests/test_convert.py b/tests/test_convert.py index 34e98f1..796a08f 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -24,6 +24,24 @@ def test_convert(fresh_db, columns, fn, expected): assert list(table.rows) == [expected] +@pytest.mark.parametrize( + "where,where_args", (("id > 1", None), ("id > :id", {"id": 1}), ("id > ?", [1])) +) +def test_convert_where(fresh_db, where, where_args): + table = fresh_db["table"] + table.insert_all( + [ + {"id": 1, "title": "One"}, + {"id": 2, "title": "Two"}, + ], + pk="id", + ) + table.convert( + "title", lambda value: value.upper(), where=where, where_args=where_args + ) + assert list(table.rows) == [{"id": 1, "title": "One"}, {"id": 2, "title": "TWO"}] + + @pytest.mark.parametrize( "drop,expected", ( @@ -70,6 +88,28 @@ def test_convert_multi(fresh_db): ] +def test_convert_multi_where(fresh_db): + table = fresh_db["table"] + table.insert_all( + [ + {"id": 1, "title": "One"}, + {"id": 2, "title": "Two"}, + ], + pk="id", + ) + table.convert( + "title", + lambda v: {"upper": v.upper(), "lower": v.lower()}, + multi=True, + where="id > ?", + where_args=[1], + ) + assert list(table.rows) == [ + {"id": 1, "lower": None, "title": "One", "upper": None}, + {"id": 2, "lower": "two", "title": "Two", "upper": "TWO"}, + ] + + def test_convert_multi_exception(fresh_db): table = fresh_db["table"] table.insert({"title": "Mixed Case"}) From d83b2568131f2b1cc01228419bb08c96d843d65d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 11:58:05 -0700 Subject: [PATCH 074/102] --where and -p options for sqlite-utils convert, closes #304 --- docs/cli.rst | 11 ++++++ sqlite_utils/cli.py | 26 +++++++++++-- tests/test_cli_convert.py | 78 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 111 insertions(+), 4 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 7bc7a3b..72c8bbf 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -945,6 +945,17 @@ You can specify Python modules that should be imported and made available to you '"\n".join(textwrap.wrap(value, 10))' \ --import=textwrap +The transformation will be applied to every row in the specified table. You can limit that to just rows that match a ``WHERE`` clause using ``--where``:: + + $ sqlite-utils convert content.db articles headline 'value.upper()' \ + --where "headline like '%cat%'" + +You can include named parameters in your where clause and populate them using one or more ``--param`` options:: + + $ sqlite-utils convert content.db articles headline 'value.upper()' \ + --where "headline like :like" \ + --param like '%cat%' + The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. .. _cli_convert_recipes: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index e1b770f..3f32413 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1959,6 +1959,14 @@ def _generate_convert_help(): @click.option( "--multi", is_flag=True, help="Populate columns for keys in returned dictionary" ) +@click.option("--where", help="Optional where clause") +@click.option( + "-p", + "--param", + multiple=True, + type=(str, str), + help="Named :parameters for where clause", +) @click.option("--output", help="Optional separate column to populate with the output") @click.option( "--output-type", @@ -1976,6 +1984,8 @@ def convert( imports, dry_run, multi, + where, + param, output, output_type, drop, @@ -1992,6 +2002,7 @@ def convert( # If single line and no 'return', add the return if "\n" not in code and not code.strip().startswith("return "): code = "return {}".format(code) + where_args = dict(param) if param else [] # Compile the code into a function body called fn(value) new_code = ["def fn(value):"] for line in code.split("\n"): @@ -2010,20 +2021,29 @@ def convert( select [{column}] as value, preview_transform([{column}]) as preview - from [{table}] limit 10 + from [{table}]{where} limit 10 """.format( - column=columns[0], table=table + column=columns[0], + table=table, + where=" where {}".format(where) if where is not None else "", ) - for row in db.conn.execute(sql).fetchall(): + for row in db.conn.execute(sql, where_args).fetchall(): click.echo(str(row[0])) click.echo(" --- becomes:") click.echo(str(row[1])) click.echo() + count = db[table].count_where( + where=where, + where_args=where_args, + ) + click.echo("Would affect {} row{}".format(count, "" if count == 1 else "s")) else: try: db[table].convert( columns, fn, + where=where, + where_args=where_args, output=output, output_type=output_type, drop=drop, diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 51634cd..8ce5203 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -124,7 +124,8 @@ def test_convert_dryrun(test_db_and_path): "\n" "None\n" " --- becomes:\n" - "None" + "None\n\n" + "Would affect 4 rows" ) # But it should not have actually modified the table data assert list(db["example"].rows) == [ @@ -133,6 +134,27 @@ def test_convert_dryrun(test_db_and_path): {"id": 3, "dt": ""}, {"id": 4, "dt": None}, ] + # Test with a where clause too + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "return re.sub('O..', 'OXX', value)", + "--import", + "re", + "--dry-run", + "--where", + "id = :id", + "-p", + "id", + "4", + ], + ) + assert result.exit_code == 0 + assert result.output.strip().split("\n")[-1] == "Would affect 1 row" @pytest.mark.parametrize("drop", (True, False)) @@ -439,3 +461,57 @@ def test_multi_with_bad_function(test_db_and_path): result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 1, result.output assert "When using --multi code must return a Python dictionary" in result.output + + +def test_convert_where(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "str(value).upper()", + "--where", + "id = :id", + "-p", + "id", + 2, + ], + ) + assert result.exit_code == 0, result.output + assert list(db["example"].rows) == [ + {"id": 1, "dt": "5th October 2019 12:04"}, + {"id": 2, "dt": "6TH OCTOBER 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] + + +def test_convert_where_multi(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["names"].insert_all( + [{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Bants"}], pk="id" + ) + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "names", + "name", + '{"upper": value.upper()}', + "--where", + "id = :id", + "-p", + "id", + 2, + "--multi", + ], + ) + assert 0 == result.exit_code, result.output + assert list(db["names"].rows) == [ + {"id": 1, "name": "Cleo", "upper": None}, + {"id": 2, "name": "Bants", "upper": "BANTS"}, + ] From 60dea99ef78c748dedabb6e5f50510a1750fecec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 12:12:16 -0700 Subject: [PATCH 075/102] --silent option for sqlite-utils insert-files, closes #301 --- sqlite_utils/cli.py | 16 ++++++++++++++-- sqlite_utils/utils.py | 13 ++++++++++--- tests/test_insert_files.py | 9 +++++++-- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 3f32413..bd063f6 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -21,6 +21,7 @@ from .utils import ( find_spatialite, sqlite3, decode_base64_values, + progressbar, rows_from_file, Format, TypeTracker, @@ -1744,9 +1745,20 @@ def extract( @click.option("--replace", is_flag=True, help="Replace files with matching primary key") @click.option("--upsert", is_flag=True, help="Upsert files with matching primary key") @click.option("--name", type=str, help="File name to use") +@click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") @load_extension_option def insert_files( - path, table, file_or_dir, column, pk, alter, replace, upsert, name, load_extension + path, + table, + file_or_dir, + column, + pk, + alter, + replace, + upsert, + name, + silent, + load_extension, ): """ Insert one or more files using BLOB columns in the specified table @@ -1783,7 +1795,7 @@ def insert_files( # Load all paths so we can show a progress bar paths_and_relative_paths = list(yield_paths_and_relative_paths()) - with click.progressbar(paths_and_relative_paths) as bar: + with progressbar(paths_and_relative_paths, silent=silent) as bar: def to_insert(): for path, relative_path in bar: diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index a781469..00a3c02 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -260,14 +260,21 @@ class ValueTracker: class NullProgressBar: + def __init__(self, *args): + self.args = args + + def __iter__(self): + yield from self.args[0] + def update(self, value): pass @contextlib.contextmanager -def progressbar(silent=False, **kwargs): +def progressbar(*args, **kwargs): + silent = kwargs.pop("silent") if silent: - yield NullProgressBar() + yield NullProgressBar(*args) else: - with click.progressbar(**kwargs) as bar: + with click.progressbar(*args, **kwargs) as bar: yield bar diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 1a11d2c..1e30a8d 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -2,9 +2,11 @@ from sqlite_utils import cli, Database from click.testing import CliRunner import os import pathlib +import pytest -def test_insert_files(): +@pytest.mark.parametrize("silent", (False, True)) +def test_insert_files(silent): runner = CliRunner() with runner.isolated_filesystem(): tmpdir = pathlib.Path(".") @@ -34,7 +36,10 @@ def test_insert_files(): cols += ["-c", "{}:{}".format(coltype, coltype)] result = runner.invoke( cli.cli, - ["insert-files", db_path, "files", str(tmpdir)] + cols + ["--pk", "path"], + ["insert-files", db_path, "files", str(tmpdir)] + + cols + + ["--pk", "path"] + + (["--silent"] if silent else []), catch_exceptions=False, ) assert result.exit_code == 0, result.stdout From 59032b00bb252c6d8cc43cbc490e0492423f61b1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 12:29:55 -0700 Subject: [PATCH 076/102] Fixed incorrect example in documentation --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 72c8bbf..32405c6 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -970,7 +970,7 @@ Various built-in recipe functions are available for common operations. These are The ``delimiter`` parameter can be used to specify a different delimiter. - The ``type`` parameter can be set to ``float`` or ``int`` to produce a JSON array of different types, for example if the column's string value was ``1.2,3,4`` the following:: + The ``type`` parameter can be set to ``float`` or ``int`` to produce a JSON array of different types, for example if the column's string value was ``1.2,3,4.5`` the following:: r.jsonsplit(value, type=float) From 723ee35344fa9f5e49dca578170cc5f5eb7223ce Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 14:18:01 -0700 Subject: [PATCH 077/102] Release 3.14 Refs #251, #301, #302, #303, #304, #305 --- docs/changelog.rst | 29 +++++++++++++++++++++++++++++ docs/cli.rst | 4 ++-- setup.py | 2 +- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9d17c1e..d9a125d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,35 @@ Changelog =========== +.. _v3_14: + +3.14 (2021-08-02) +----------------- + +This release introduces the new :ref:`sqlite-utils convert command ` (`#251 `__) and corresponding :ref:`table.convert(...) ` Python method (`#302 `__). These tools can be used to apply a Python conversion function to one or more columns of a table, either updating the column in place or using transformed data from that column to populate one or more other columns. + +This command-line example uses the Python standard library `textwrap module `__ to wrap the content of the ``content`` column in the ``articles`` table to 100 characters:: + + $ sqlite-utils convert content.db articles content \ + '"\n".join(textwrap.wrap(value, 100))' \ + --import=textwrap + +The same operation in Python code looks like this: + +.. code-block:: python + + import sqlite_utils, textwrap + + db = sqlite_utils.Database("content.db") + db["articles"].convert("content", lambda v: "\n".join(textwrap.wrap(v, 100))) + +See the full documentation for the :ref:`sqlite-utils convert command ` and the :ref:`table.convert(...) ` Python method for more details. + +Also in this release: + +- The new ``table.count_where(...)`` method, for counting rows in a table that match a specific SQL ``WHERE`` clause. (`#305 `__) +- New ``--silent`` option for the :ref:`sqlite-utils insert-files command ` to hide the terminal progress bar, consistent with the ``--silent`` option for ``sqlite-utils convert``. (`#301 `__) + .. _v3_13: 3.13 (2021-07-24) diff --git a/docs/cli.rst b/docs/cli.rst index 32405c6..3802963 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -939,10 +939,10 @@ The code you provide will be compiled into a function that takes ``value`` as a value = str(value) return value.upper()' -You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options:: +You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options. This example uses the ``textwrap`` module to wrap the ``content`` column at 100 characters:: $ sqlite-utils convert content.db articles content \ - '"\n".join(textwrap.wrap(value, 10))' \ + '"\n".join(textwrap.wrap(value, 100))' \ --import=textwrap The transformation will be applied to every row in the specified table. You can limit that to just rows that match a ``WHERE`` clause using ``--where``:: diff --git a/setup.py b/setup.py index 5009736..8aa9aeb 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.13" +VERSION = "3.14" def get_long_description(): From e83aef951bd3e8c179511faddb607239a5fa8682 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 14:29:00 -0700 Subject: [PATCH 078/102] New :issue: macro, closes #306 --- docs/changelog.rst | 286 ++++++++++++++++++++++----------------------- docs/conf.py | 6 +- 2 files changed, 148 insertions(+), 144 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d9a125d..e03dbd6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,7 @@ 3.14 (2021-08-02) ----------------- -This release introduces the new :ref:`sqlite-utils convert command ` (`#251 `__) and corresponding :ref:`table.convert(...) ` Python method (`#302 `__). These tools can be used to apply a Python conversion function to one or more columns of a table, either updating the column in place or using transformed data from that column to populate one or more other columns. +This release introduces the new :ref:`sqlite-utils convert command ` (:issue:`251`) and corresponding :ref:`table.convert(...) ` Python method (:issue:`302`). These tools can be used to apply a Python conversion function to one or more columns of a table, either updating the column in place or using transformed data from that column to populate one or more other columns. This command-line example uses the Python standard library `textwrap module `__ to wrap the content of the ``content`` column in the ``articles`` table to 100 characters:: @@ -28,15 +28,15 @@ See the full documentation for the :ref:`sqlite-utils convert command `__) -- New ``--silent`` option for the :ref:`sqlite-utils insert-files command ` to hide the terminal progress bar, consistent with the ``--silent`` option for ``sqlite-utils convert``. (`#301 `__) +- The new ``table.count_where(...)`` method, for counting rows in a table that match a specific SQL ``WHERE`` clause. (:issue:`305`) +- New ``--silent`` option for the :ref:`sqlite-utils insert-files command ` to hide the terminal progress bar, consistent with the ``--silent`` option for ``sqlite-utils convert``. (:issue:`301`) .. _v3_13: 3.13 (2021-07-24) ----------------- -- ``sqlite-utils schema my.db table1 table2`` command now accepts optional table names. (`#299 `__) +- ``sqlite-utils schema my.db table1 table2`` command now accepts optional table names. (:issue:`299`) - ``sqlite-utils memory --help`` now describes the ``--schema`` option. .. _v3_12: @@ -44,17 +44,17 @@ Also in this release: 3.12 (2021-06-25) ----------------- -- New :ref:`db.query(sql, params) ` method, which executes a SQL query and returns the results as an iterator over Python dictionaries. (`#290 `__) -- This project now uses ``flake8`` and has started to use ``mypy``. (`#291 `__) -- New documentation on :ref:`contributing ` to this project. (`#292 `__) +- New :ref:`db.query(sql, params) ` method, which executes a SQL query and returns the results as an iterator over Python dictionaries. (:issue:`290`) +- This project now uses ``flake8`` and has started to use ``mypy``. (:issue:`291`) +- New documentation on :ref:`contributing ` to this project. (:issue:`292`) .. _v3_11: 3.11 (2021-06-20) ----------------- -- New ``sqlite-utils memory data.csv --schema`` option, for outputting the schema of the in-memory database generated from one or more files. See :ref:`cli_memory_schema_dump_save`. (`#288 `__) -- Added :ref:`installation instructions `. (`#286 `__) +- New ``sqlite-utils memory data.csv --schema`` option, for outputting the schema of the in-memory database generated from one or more files. See :ref:`cli_memory_schema_dump_save`. (:issue:`288`) +- Added :ref:`installation instructions `. (:issue:`286`) .. _v3_10: @@ -107,21 +107,21 @@ Here the ``species.csv`` file becomes the ``species`` table, the ``creatures.csv You can also use the ``--attach`` option to attach existing SQLite database files to the in-memory database, in order to join data from CSV or JSON directly against your existing tables. -Full documentation of this new feature is available in :ref:`cli_memory`. (`#272 `__) +Full documentation of this new feature is available in :ref:`cli_memory`. (:issue:`272`) sqlite-utils insert \-\-detect-types ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The :ref:`sqlite-utils insert ` command can be used to insert data from JSON, CSV or TSV files into a SQLite database file. The new ``--detect-types`` option (shortcut ``-d``), when used in conjunction with a CSV or TSV import, will automatically detect if columns in the file are integers or floating point numbers as opposed to treating everything as a text column and create the new table with the corresponding schema. See :ref:`cli_insert_csv_tsv` for details. (`#282 `__) +The :ref:`sqlite-utils insert ` command can be used to insert data from JSON, CSV or TSV files into a SQLite database file. The new ``--detect-types`` option (shortcut ``-d``), when used in conjunction with a CSV or TSV import, will automatically detect if columns in the file are integers or floating point numbers as opposed to treating everything as a text column and create the new table with the corresponding schema. See :ref:`cli_insert_csv_tsv` for details. (:issue:`282`) Other changes ~~~~~~~~~~~~~ -- **Bug fix**: ``table.transform()``, when run against a table without explicit primary keys, would incorrectly create a new version of the table with an explicit primary key column called ``rowid``. (`#284 `__) -- New ``table.use_rowid`` introspection property, see :ref:`python_api_introspection_use_rowid`. (`#285 `__) -- The new ``sqlite-utils dump file.db`` command outputs a SQL dump that can be used to recreate a database. (`#274 `__) -- ``-h`` now works as a shortcut for ``--help``, thanks Loren McIntyre. (`#276 `__) -- Now using `pytest-cov `__ and `Codecov `__ to track test coverage - currently at 96%. (`#275 `__) +- **Bug fix**: ``table.transform()``, when run against a table without explicit primary keys, would incorrectly create a new version of the table with an explicit primary key column called ``rowid``. (:issue:`284`) +- New ``table.use_rowid`` introspection property, see :ref:`python_api_introspection_use_rowid`. (:issue:`285`) +- The new ``sqlite-utils dump file.db`` command outputs a SQL dump that can be used to recreate a database. (:issue:`274`) +- ``-h`` now works as a shortcut for ``--help``, thanks Loren McIntyre. (:issue:`276`) +- Now using `pytest-cov `__ and `Codecov `__ to track test coverage - currently at 96%. (:issue:`275`) - SQL errors that occur when using ``sqlite-utils query`` are now displayed as CLI errors. .. _v3_9_1: @@ -129,14 +129,14 @@ Other changes 3.9.1 (2021-06-12) ------------------ -- Fixed bug when using ``table.upsert_all()`` to create a table with only a single column that is treated as the primary key. (`#271 `__) +- Fixed bug when using ``table.upsert_all()`` to create a table with only a single column that is treated as the primary key. (:issue:`271`) .. _v3_9: 3.9 (2021-06-11) ---------------- -- New ``sqlite-utils schema`` command showing the full SQL schema for a database, see :ref:`Showing the schema (CLI)`. (`#268 `__) +- New ``sqlite-utils schema`` command showing the full SQL schema for a database, see :ref:`Showing the schema (CLI)`. (:issue:`268`) - ``db.schema`` introspection property exposing the same feature to the Python library, see :ref:`Showing the schema (Python library) `. .. _v3_8: @@ -144,22 +144,22 @@ Other changes 3.8 (2021-06-02) ---------------- -- New ``sqlite-utils indexes`` command to list indexes in a database, see :ref:`cli_indexes`. (`#263 `__) -- ``table.xindexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (`#261 `__) +- New ``sqlite-utils indexes`` command to list indexes in a database, see :ref:`cli_indexes`. (:issue:`263`) +- ``table.xindexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (:issue:`261`) .. _v3_7: 3.7 (2021-05-28) ---------------- -- New ``table.pks_and_rows_where()`` method returning ``(primary_key, row_dictionary)`` tuples - see :ref:`python_api_pks_and_rows_where`. (`#240 `__) -- Fixed bug with ``table.add_foreign_key()`` against columns containing spaces. (`#238 `__) -- ``table_or_view.drop(ignore=True)`` option for avoiding errors if the table or view does not exist. (`#237 `__) -- ``sqlite-utils drop-view --ignore`` and ``sqlite-utils drop-table --ignore`` options. (`#237 `__) -- Fixed a bug with inserts of nested JSON containing non-ascii strings - thanks, Dylan Wu. (`#257 `__) -- Suggest ``--alter`` if an error occurs caused by a missing column. (`#259 `__) -- Support creating indexes with columns in descending order, see :ref:`API documentation ` and :ref:`CLI documentation `. (`#260 `__) -- Correctly handle CSV files that start with a UTF-8 BOM. (`#250 `__) +- New ``table.pks_and_rows_where()`` method returning ``(primary_key, row_dictionary)`` tuples - see :ref:`python_api_pks_and_rows_where`. (:issue:`240`) +- Fixed bug with ``table.add_foreign_key()`` against columns containing spaces. (:issue:`238`) +- ``table_or_view.drop(ignore=True)`` option for avoiding errors if the table or view does not exist. (:issue:`237`) +- ``sqlite-utils drop-view --ignore`` and ``sqlite-utils drop-table --ignore`` options. (:issue:`237`) +- Fixed a bug with inserts of nested JSON containing non-ascii strings - thanks, Dylan Wu. (:issue:`257`) +- Suggest ``--alter`` if an error occurs caused by a missing column. (:issue:`259`) +- Support creating indexes with columns in descending order, see :ref:`API documentation ` and :ref:`CLI documentation `. (:issue:`260`) +- Correctly handle CSV files that start with a UTF-8 BOM. (:issue:`250`) .. _v3_6: @@ -168,73 +168,73 @@ Other changes This release adds the ability to execute queries joining data from more than one database file - similar to the cross database querying feature introduced in `Datasette 0.55 `__. -- The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation `. (`#113 `__) -- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (`#236 `__) +- The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation `. (:issue:`113`) +- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (:issue:`236`) .. _v3_5: 3.5 (2021-02-14) ---------------- -- ``sqlite-utils insert --sniff`` option for detecting the delimiter and quote character used by a CSV file, see :ref:`cli_insert_csv_tsv_delimiter`. (`#230 `__) -- The ``table.rows_where()``, ``table.search()`` and ``table.search_sql()`` methods all now take optional ``offset=`` and ``limit=`` arguments. (`#231 `__) -- New ``--no-headers`` option for ``sqlite-utils insert --csv`` to handle CSV files that are missing the header row, see :ref:`cli_insert_csv_tsv_no_header`. (`#228 `__) -- Fixed bug where inserting data with extra columns in subsequent chunks would throw an error. Thanks `@nieuwenhoven `__ for the fix. (`#234 `__) -- Fixed bug importing CSV files with columns containing more than 128KB of data. (`#229 `__) -- Test suite now runs in CI against Ubuntu, macOS and Windows. Thanks `@nieuwenhoven `__ for the Windows test fixes. (`#232 `__) +- ``sqlite-utils insert --sniff`` option for detecting the delimiter and quote character used by a CSV file, see :ref:`cli_insert_csv_tsv_delimiter`. (:issue:`230`) +- The ``table.rows_where()``, ``table.search()`` and ``table.search_sql()`` methods all now take optional ``offset=`` and ``limit=`` arguments. (:issue:`231`) +- New ``--no-headers`` option for ``sqlite-utils insert --csv`` to handle CSV files that are missing the header row, see :ref:`cli_insert_csv_tsv_no_header`. (:issue:`228`) +- Fixed bug where inserting data with extra columns in subsequent chunks would throw an error. Thanks `@nieuwenhoven `__ for the fix. (:issue:`234`) +- Fixed bug importing CSV files with columns containing more than 128KB of data. (:issue:`229`) +- Test suite now runs in CI against Ubuntu, macOS and Windows. Thanks `@nieuwenhoven `__ for the Windows test fixes. (:issue:`232`) .. _v3_4_1: 3.4.1 (2021-02-05) ------------------ -- Fixed a code import bug that slipped in to 3.4. (`#226 `__) +- Fixed a code import bug that slipped in to 3.4. (:issue:`226`) .. _v3_4: 3.4 (2021-02-05) ---------------- -- ``sqlite-utils insert --csv`` now accepts optional ``--delimiter`` and ``--quotechar`` options. See :ref:`cli_insert_csv_tsv_delimiter`. (`#223 `__) +- ``sqlite-utils insert --csv`` now accepts optional ``--delimiter`` and ``--quotechar`` options. See :ref:`cli_insert_csv_tsv_delimiter`. (:issue:`223`) .. _v3_3: 3.3 (2021-01-17) ---------------- -- The ``table.m2m()`` method now accepts an optional ``alter=True`` argument to specify that any missing columns should be added to the referenced table. See :ref:`python_api_m2m`. (`#222 `__) +- The ``table.m2m()`` method now accepts an optional ``alter=True`` argument to specify that any missing columns should be added to the referenced table. See :ref:`python_api_m2m`. (:issue:`222`) .. _v3_2_1: 3.2.1 (2021-01-12) ------------------ -- Fixed a bug where ``.add_missing_columns()`` failed to take case insensitive column names into account. (`#221 `__) +- Fixed a bug where ``.add_missing_columns()`` failed to take case insensitive column names into account. (:issue:`221`) .. _v3_2: 3.2 (2021-01-03) ---------------- -This release introduces a new mechanism for speeding up ``count(*)`` queries using cached table counts, stored in a ``_counts`` table and updated by triggers. This mechanism is described in :ref:`python_api_cached_table_counts`, and can be enabled using Python API methods or the new ``enable-counts`` CLI command. (`#212 `__) +This release introduces a new mechanism for speeding up ``count(*)`` queries using cached table counts, stored in a ``_counts`` table and updated by triggers. This mechanism is described in :ref:`python_api_cached_table_counts`, and can be enabled using Python API methods or the new ``enable-counts`` CLI command. (:issue:`212`) - ``table.enable_counts()`` method for enabling these triggers on a specific table. -- ``db.enable_counts()`` method for enabling triggers on every table in the database. (`#213 `__) -- New ``sqlite-utils enable-counts my.db`` command for enabling counts on all or specific tables, see :ref:`cli_enable_counts`. (`#214 `__) -- New ``sqlite-utils triggers`` command for listing the triggers defined for a database or specific tables, see :ref:`cli_triggers`. (`#218 `__) -- New ``db.use_counts_table`` property which, if ``True``, causes ``table.count`` to read from the ``_counts`` table. (`#215 `__) +- ``db.enable_counts()`` method for enabling triggers on every table in the database. (:issue:`213`) +- New ``sqlite-utils enable-counts my.db`` command for enabling counts on all or specific tables, see :ref:`cli_enable_counts`. (:issue:`214`) +- New ``sqlite-utils triggers`` command for listing the triggers defined for a database or specific tables, see :ref:`cli_triggers`. (:issue:`218`) +- New ``db.use_counts_table`` property which, if ``True``, causes ``table.count`` to read from the ``_counts`` table. (:issue:`215`) - ``table.has_counts_triggers`` property revealing if a table has been configured with the new ``_counts`` database triggers. -- ``db.reset_counts()`` method and ``sqlite-utils reset-counts`` command for resetting the values in the ``_counts`` table. (`#219 `__) -- The previously undocumented ``db.escape()`` method has been renamed to ``db.quote()`` and is now covered by the documentation: :ref:`python_api_quote`. (`#217 `__) -- New ``table.triggers_dict`` and ``db.triggers_dict`` introspection properties. (`#211 `__, `#216 `__) -- ``sqlite-utils insert`` now shows a more useful error message for invalid JSON. (`#206 `__) +- ``db.reset_counts()`` method and ``sqlite-utils reset-counts`` command for resetting the values in the ``_counts`` table. (:issue:`219`) +- The previously undocumented ``db.escape()`` method has been renamed to ``db.quote()`` and is now covered by the documentation: :ref:`python_api_quote`. (:issue:`217`) +- New ``table.triggers_dict`` and ``db.triggers_dict`` introspection properties. (:issue:`211`, :issue:`216`) +- ``sqlite-utils insert`` now shows a more useful error message for invalid JSON. (:issue:`206`) .. _v3_1_1: 3.1.1 (2021-01-01) ------------------ -- Fixed failing test caused by ``optimize`` sometimes creating larger database files. (`#209 `__) +- Fixed failing test caused by ``optimize`` sometimes creating larger database files. (:issue:`209`) - Documentation now lives on https://sqlite-utils.datasette.io/ - README now includes ``brew install sqlite-utils`` installation method. @@ -243,7 +243,7 @@ This release introduces a new mechanism for speeding up ``count(*)`` queries usi 3.1 (2020-12-12) ---------------- -- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (`#207 `__) +- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (:issue:`207`) - New ``table.analyze_column(column)`` Python method used by the ``analyze-tables`` command - see :ref:`python_api_analyze_column`. - The ``table.update()`` method now correctly handles values that should be stored as JSON. Thanks, Andreas Madsack. (`#204 `__) @@ -252,28 +252,28 @@ This release introduces a new mechanism for speeding up ``count(*)`` queries usi 3.0 (2020-11-08) ---------------- -This release introduces a new ``sqlite-utils search`` command for searching tables, see :ref:`cli_search`. (`#192 `__) +This release introduces a new ``sqlite-utils search`` command for searching tables, see :ref:`cli_search`. (:issue:`192`) -The ``table.search()`` method has been redesigned, see :ref:`python_api_fts_search`. (`#197 `__) +The ``table.search()`` method has been redesigned, see :ref:`python_api_fts_search`. (:issue:`197`) The release includes minor backwards-incompatible changes, hence the version bump to 3.0. Those changes, which should not affect most users, are: - The ``-c`` shortcut option for outputting CSV is no longer available. The full ``--csv`` option is required instead. - The ``-f`` shortcut for ``--fmt`` has also been removed - use ``--fmt``. -- The ``table.search()`` method now defaults to sorting by relevance, not sorting by ``rowid``. (`#198 `__) +- The ``table.search()`` method now defaults to sorting by relevance, not sorting by ``rowid``. (:issue:`198`) - The ``table.search()`` method now returns a generator over a list of Python dictionaries. It previously returned a list of tuples. Also in this release: -- The ``query``, ``tables``, ``rows`` and ``search`` CLI commands now accept a new ``--tsv`` option which outputs the results in TSV. (`#193 `__) -- A new ``table.virtual_table_using`` property reveals if a table is a virtual table, and returns the upper case type of virtual table (e.g. ``FTS4`` or ``FTS5``) if it is. It returns ``None`` if the table is not a virtual table. (`#196 `__) +- The ``query``, ``tables``, ``rows`` and ``search`` CLI commands now accept a new ``--tsv`` option which outputs the results in TSV. (:issue:`193`) +- A new ``table.virtual_table_using`` property reveals if a table is a virtual table, and returns the upper case type of virtual table (e.g. ``FTS4`` or ``FTS5``) if it is. It returns ``None`` if the table is not a virtual table. (:issue:`196`) - The new ``table.search_sql()`` method returns the SQL for searching a table, see :ref:`python_api_fts_search_sql`. -- ``sqlite-utils rows`` now accepts multiple optional ``-c`` parameters specifying the columns to return. (`#200 `__) +- ``sqlite-utils rows`` now accepts multiple optional ``-c`` parameters specifying the columns to return. (:issue:`200`) Changes since the 3.0a0 alpha release: - The ``sqlite-utils search`` command now defaults to returning every result, unless you add a ``--limit 20`` option. -- The ``sqlite-utils search -c`` and ``table.search(columns=[])`` options are now fully respected. (`#201 `__) +- The ``sqlite-utils search -c`` and ``table.search(columns=[])`` options are now fully respected. (:issue:`201`) .. _v2_23: @@ -281,30 +281,30 @@ Changes since the 3.0a0 alpha release: ----------------- - ``table.m2m(other_table, records)`` method now takes any iterable, not just a list or tuple. Thanks, Adam Wolf. (`#189 `__) -- ``sqlite-utils insert`` now displays a progress bar for CSV or TSV imports. (`#173 `__) -- New ``@db.register_function(deterministic=True)`` option for registering deterministic SQLite functions in Python 3.8 or higher. (`#191 `__) +- ``sqlite-utils insert`` now displays a progress bar for CSV or TSV imports. (:issue:`173`) +- New ``@db.register_function(deterministic=True)`` option for registering deterministic SQLite functions in Python 3.8 or higher. (:issue:`191`) .. _v2_22: 2.22 (2020-10-16) ----------------- -- New ``--encoding`` option for processing CSV and TSV files that use a non-utf-8 encoding, for both the ``insert`` and ``update`` commands. (`#182 `__) -- The ``--load-extension`` option is now available to many more commands. (`#137 `__) -- ``--load-extension=spatialite`` can be used to load SpatiaLite from common installation locations, if it is available. (`#136 `__) -- Tests now also run against Python 3.9. (`#184 `__) -- Passing ``pk=["id"]`` now has the same effect as passing ``pk="id"``. (`#181 `__) +- New ``--encoding`` option for processing CSV and TSV files that use a non-utf-8 encoding, for both the ``insert`` and ``update`` commands. (:issue:`182`) +- The ``--load-extension`` option is now available to many more commands. (:issue:`137`) +- ``--load-extension=spatialite`` can be used to load SpatiaLite from common installation locations, if it is available. (:issue:`136`) +- Tests now also run against Python 3.9. (:issue:`184`) +- Passing ``pk=["id"]`` now has the same effect as passing ``pk="id"``. (:issue:`181`) .. _v2_21: 2.21 (2020-09-24) ----------------- -- ``table.extract()`` and ``sqlite-utils extract`` now apply much, much faster - one example operation reduced from twelve minutes to just four seconds! (`#172 `__) +- ``table.extract()`` and ``sqlite-utils extract`` now apply much, much faster - one example operation reduced from twelve minutes to just four seconds! (:issue:`172`) - ``sqlite-utils extract`` no longer shows a progress bar, because it's fast enough not to need one. -- New ``column_order=`` option for ``table.transform()`` which can be used to alter the order of columns in a table. (`#175 `__) -- ``sqlite-utils transform --column-order=`` option (with a ``-o`` shortcut) for changing column order. (`#176 `__) -- The ``table.transform(drop_foreign_keys=)`` parameter and the ``sqlite-utils transform --drop-foreign-key`` option have changed. They now accept just the name of the column rather than requiring all three of the column, other table and other column. This is technically a backwards-incompatible change but I chose not to bump the major version number because the transform feature is so new. (`#177 `__) +- New ``column_order=`` option for ``table.transform()`` which can be used to alter the order of columns in a table. (:issue:`175`) +- ``sqlite-utils transform --column-order=`` option (with a ``-o`` shortcut) for changing column order. (:issue:`176`) +- The ``table.transform(drop_foreign_keys=)`` parameter and the ``sqlite-utils transform --drop-foreign-key`` option have changed. They now accept just the name of the column rather than requiring all three of the column, other table and other column. This is technically a backwards-incompatible change but I chose not to bump the major version number because the transform feature is so new. (:issue:`177`) - The table ``.disable_fts()``, ``.rebuild_fts()``, ``.delete()``, ``.delete_where()`` and ``.add_missing_columns()`` methods all now ``return self``, which means they can be chained together with other table operations. .. _v2_20: @@ -312,7 +312,7 @@ Changes since the 3.0a0 alpha release: 2.20 (2020-09-22) ----------------- -This release introduces two key new capabilities: **transform** (`#114 `__) and **extract** (`#42 `__). +This release introduces two key new capabilities: **transform** (:issue:`114`) and **extract** (:issue:`42`). Transform ~~~~~~~~~ @@ -333,7 +333,7 @@ The Python library :ref:`extract() documentation ` describes Other changes ~~~~~~~~~~~~~ -- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref:`python_api_register_function`. (`#162 `__) +- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref:`python_api_register_function`. (:issue:`162`) - The ``table.rows_where()`` method now accepts an optional ``select=`` argument for specifying which columns should be selected, see :ref:`python_api_rows`. .. _v2_19: @@ -341,31 +341,31 @@ Other changes 2.19 (2020-09-20) ----------------- -- New ``sqlite-utils add-foreign-keys`` command for :ref:`cli_add_foreign_keys`. (`#157 `__) -- New ``table.enable_fts(..., replace=True)`` argument for replacing an existing FTS table with a new configuration. (`#160 `__) -- New ``table.add_foreign_key(..., ignore=True)`` argument for ignoring a foreign key if it already exists. (`#112 `__) +- New ``sqlite-utils add-foreign-keys`` command for :ref:`cli_add_foreign_keys`. (:issue:`157`) +- New ``table.enable_fts(..., replace=True)`` argument for replacing an existing FTS table with a new configuration. (:issue:`160`) +- New ``table.add_foreign_key(..., ignore=True)`` argument for ignoring a foreign key if it already exists. (:issue:`112`) .. _v2_18: 2.18 (2020-09-08) ----------------- -- ``table.rebuild_fts()`` method for rebuilding a FTS index, see :ref:`python_api_fts_rebuild`. (`#155 `__) -- ``sqlite-utils rebuild-fts data.db`` command for rebuilding FTS indexes across all tables, or just specific tables. (`#155 `__) +- ``table.rebuild_fts()`` method for rebuilding a FTS index, see :ref:`python_api_fts_rebuild`. (:issue:`155`) +- ``sqlite-utils rebuild-fts data.db`` command for rebuilding FTS indexes across all tables, or just specific tables. (:issue:`155`) - ``table.optimize()`` method no longer deletes junk rows from the ``*_fts_docsize`` table. This was added in 2.17 but it turns out running ``table.rebuild_fts()`` is a better solution to this problem. -- Fixed a bug where rows with additional columns that are inserted after the first batch of records could cause an error due to breaking SQLite's maximum number of parameters. Thanks, Simon Wiles. (`#145 `__) +- Fixed a bug where rows with additional columns that are inserted after the first batch of records could cause an error due to breaking SQLite's maximum number of parameters. Thanks, Simon Wiles. (:issue:`145`) .. _v2_17: 2.17 (2020-09-07) ----------------- -This release handles a bug where replacing rows in FTS tables could result in growing numbers of unnecessary rows in the associated ``*_fts_docsize`` table. (`#149 `__) +This release handles a bug where replacing rows in FTS tables could result in growing numbers of unnecessary rows in the associated ``*_fts_docsize`` table. (:issue:`149`) -- ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (`#152 `__) -- ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (`#153 `__) -- New tracer method for tracking underlying SQL queries, see :ref:`python_api_tracing`. (`#150 `__) -- Neater indentation for schema SQL. (`#148 `__) +- ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (:issue:`152`) +- ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (:issue:`153`) +- New tracer method for tracking underlying SQL queries, see :ref:`python_api_tracing`. (:issue:`150`) +- Neater indentation for schema SQL. (:issue:`148`) - Documentation for ``sqlite_utils.AlterError`` exception thrown by in ``add_foreign_keys()``. .. _v2_16_1: @@ -373,23 +373,23 @@ This release handles a bug where replacing rows in FTS tables could result in gr 2.16.1 (2020-08-28) ------------------- -- ``insert_all(..., alter=True)`` now works for columns introduced after the first 100 records. Thanks, Simon Wiles! (`#139 `__) -- Continuous Integration is now powered by GitHub Actions. (`#143 `__) +- ``insert_all(..., alter=True)`` now works for columns introduced after the first 100 records. Thanks, Simon Wiles! (:issue:`139`) +- Continuous Integration is now powered by GitHub Actions. (:issue:`143`) .. _v2_16: 2.16 (2020-08-21) ----------------- -- ``--load-extension`` option for ``sqlite-utils query`` for loading SQLite extensions. (`#134 `__) -- New ``sqlite_utils.utils.find_spatialite()`` function for finding SpatiaLite in common locations. (`#135 `__) +- ``--load-extension`` option for ``sqlite-utils query`` for loading SQLite extensions. (:issue:`134`) +- New ``sqlite_utils.utils.find_spatialite()`` function for finding SpatiaLite in common locations. (:issue:`135`) .. _v2_15_1: 2.15.1 (2020-08-12) ------------------- -- Now available as a ``sdist`` package on PyPI in addition to a wheel. (`#133 `__) +- Now available as a ``sdist`` package on PyPI in addition to a wheel. (:issue:`133`) .. _v2_15: @@ -397,7 +397,7 @@ This release handles a bug where replacing rows in FTS tables could result in gr ----------------- - New ``db.enable_wal()`` and ``db.disable_wal()`` methods for enabling and disabling `Write-Ahead Logging `__ for a database file - see :ref:`python_api_wal` in the Python API documentation. -- Also ``sqlite-utils enable-wal file.db`` and ``sqlite-utils disable-wal file.db`` commands for doing the same thing on the command-line, see :ref:`WAL mode (CLI) `. (`#132 `__) +- Also ``sqlite-utils enable-wal file.db`` and ``sqlite-utils disable-wal file.db`` commands for doing the same thing on the command-line, see :ref:`WAL mode (CLI) `. (:issue:`132`) .. _v2_14_1: @@ -411,8 +411,8 @@ This release handles a bug where replacing rows in FTS tables could result in gr 2.14 (2020-08-01) ----------------- -- The :ref:`insert-files command ` can now read from standard input: ``cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg``. (`#127 `__) -- You can now specify a full-text search tokenizer using the new ``tokenize=`` parameter to :ref:`enable_fts() `. This means you can enable Porter stemming on a table by running ``db["articles"].enable_fts(["headline", "body"], tokenize="porter")``. (`#130 `__) +- The :ref:`insert-files command ` can now read from standard input: ``cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg``. (:issue:`127`) +- You can now specify a full-text search tokenizer using the new ``tokenize=`` parameter to :ref:`enable_fts() `. This means you can enable Porter stemming on a table by running ``db["articles"].enable_fts(["headline", "body"], tokenize="porter")``. (:issue:`130`) - You can also set a custom tokenizer using the :ref:`sqlite-utils enable-fts ` CLI command, via the new ``--tokenize`` option. .. _v2_13: @@ -420,7 +420,7 @@ This release handles a bug where replacing rows in FTS tables could result in gr 2.13 (2020-07-29) ----------------- -- ``memoryview`` and ``uuid.UUID`` objects are now supported. ``memoryview`` objects will be stored using ``BLOB`` and ``uuid.UUID`` objects will be stored using ``TEXT``. (`#128 `__) +- ``memoryview`` and ``uuid.UUID`` objects are now supported. ``memoryview`` objects will be stored using ``BLOB`` and ``uuid.UUID`` objects will be stored using ``TEXT``. (:issue:`128`) .. _v2_12: @@ -429,11 +429,11 @@ This release handles a bug where replacing rows in FTS tables could result in gr The theme of this release is better tools for working with binary data. The new ``insert-files`` command can be used to insert binary files directly into a database table, and other commands have been improved with better support for BLOB columns. -- ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (`#122 `__) -- ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (`#123 `__) -- JSON output now encodes BLOB values as special base64 objects - see :ref:`cli_query_json`. (`#125 `__) -- The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (`#126 `__) -- The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (`#124 `__) +- ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (:issue:`122`) +- ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (:issue:`123`) +- JSON output now encodes BLOB values as special base64 objects - see :ref:`cli_query_json`. (:issue:`125`) +- The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (:issue:`126`) +- The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (:issue:`124`) .. _v2_11: @@ -448,14 +448,14 @@ The theme of this release is better tools for working with binary data. The new 2.10.1 (2020-06-23) ------------------- -- Added documentation for the ``table.pks`` introspection property. (`#116 `__) +- Added documentation for the ``table.pks`` introspection property. (:issue:`116`) .. _v2_10: 2.10 (2020-06-12) ----------------- -- The ``sqlite-utils`` command now supports UPDATE/INSERT/DELETE in addition to SELECT. (`#115 `__) +- The ``sqlite-utils`` command now supports UPDATE/INSERT/DELETE in addition to SELECT. (:issue:`115`) .. _v2_9_1: @@ -469,77 +469,77 @@ The theme of this release is better tools for working with binary data. The new 2.9 (2020-05-10) ---------------- -- New ``sqlite-utils drop-table`` command, see :ref:`cli_drop_table`. (`#111 `__) +- New ``sqlite-utils drop-table`` command, see :ref:`cli_drop_table`. (:issue:`111`) - New ``sqlite-utils drop-view`` command, see :ref:`cli_drop_view`. -- Python ``decimal.Decimal`` objects are now stored as ``FLOAT``. (`#110 `__) +- Python ``decimal.Decimal`` objects are now stored as ``FLOAT``. (:issue:`110`) .. _v2_8: 2.8 (2020-05-03) ---------------- -- New ``sqlite-utils create-table`` command, see :ref:`cli_create_table`. (`#27 `__) -- New ``sqlite-utils create-view`` command, see :ref:`cli_create_view`. (`#107 `__) +- New ``sqlite-utils create-table`` command, see :ref:`cli_create_table`. (:issue:`27`) +- New ``sqlite-utils create-view`` command, see :ref:`cli_create_view`. (:issue:`107`) .. _v2_7.2: 2.7.2 (2020-05-02) ------------------ -- ``db.create_view(...)`` now has additional parameters ``ignore=True`` or ``replace=True``, see :ref:`python_api_create_view`. (`#106 `__) +- ``db.create_view(...)`` now has additional parameters ``ignore=True`` or ``replace=True``, see :ref:`python_api_create_view`. (:issue:`106`) .. _v2_7.1: 2.7.1 (2020-05-01) ------------------ -- New ``sqlite-utils views my.db`` command for listing views in a database, see :ref:`cli_views`. (`#105 `__) -- ``sqlite-utils tables`` (and ``views``) has a new ``--schema`` option which outputs the table/view schema, see :ref:`cli_tables`. (`#104 `__) -- Nested structures containing invalid JSON values (e.g. Python bytestrings) are now serialized using ``repr()`` instead of throwing an error. (`#102 `__) +- New ``sqlite-utils views my.db`` command for listing views in a database, see :ref:`cli_views`. (:issue:`105`) +- ``sqlite-utils tables`` (and ``views``) has a new ``--schema`` option which outputs the table/view schema, see :ref:`cli_tables`. (:issue:`104`) +- Nested structures containing invalid JSON values (e.g. Python bytestrings) are now serialized using ``repr()`` instead of throwing an error. (:issue:`102`) .. _v2_7: 2.7 (2020-04-17) ---------------- -- New ``columns=`` argument for the ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()`` methods, for over-riding the auto-detected types for columns and specifying additional columns that should be added when the table is created. See :ref:`python_api_custom_columns`. (`#100 `__) +- New ``columns=`` argument for the ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()`` methods, for over-riding the auto-detected types for columns and specifying additional columns that should be added when the table is created. See :ref:`python_api_custom_columns`. (:issue:`100`) .. _v2_6: 2.6 (2020-04-15) ---------------- -- New ``table.rows_where(..., order_by="age desc")`` argument, see :ref:`python_api_rows`. (`#76 `__) +- New ``table.rows_where(..., order_by="age desc")`` argument, see :ref:`python_api_rows`. (:issue:`76`) .. _v2_5: 2.5 (2020-04-12) ---------------- -- Panda's Timestamp is now stored as a SQLite TEXT column. Thanks, b0b5h4rp13! (`#96 `__) -- ``table.last_pk`` is now only available for inserts or upserts of a single record. (`#98 `__) -- New ``Database(filepath, recreate=True)`` parameter for deleting and recreating the database. (`#97 `__) +- Panda's Timestamp is now stored as a SQLite TEXT column. Thanks, b0b5h4rp13! (:issue:`96`) +- ``table.last_pk`` is now only available for inserts or upserts of a single record. (:issue:`98`) +- New ``Database(filepath, recreate=True)`` parameter for deleting and recreating the database. (:issue:`97`) .. _v2_4_4: 2.4.4 (2020-03-23) ------------------ -- Fixed bug where columns with only null values were not correctly created. (`#95 `__) +- Fixed bug where columns with only null values were not correctly created. (:issue:`95`) .. _v2_4_3: 2.4.3 (2020-03-23) ------------------ -- Column type suggestion code is no longer confused by null values. (`#94 `__) +- Column type suggestion code is no longer confused by null values. (:issue:`94`) .. _v2_4_2: 2.4.2 (2020-03-14) ------------------ -- ``table.column_dicts`` now works with all column types - previously it would throw errors on types other than ``TEXT``, ``BLOB``, ``INTEGER`` or ``FLOAT``. (`#92 `__) +- ``table.column_dicts`` now works with all column types - previously it would throw errors on types other than ``TEXT``, ``BLOB``, ``INTEGER`` or ``FLOAT``. (:issue:`92`) - Documentation for ``NotFoundError`` thrown by ``table.get(pk)`` - see :ref:`python_api_get`. .. _v2_4_1: @@ -547,45 +547,45 @@ The theme of this release is better tools for working with binary data. The new 2.4.1 (2020-03-01) ------------------ -- ``table.enable_fts()`` now works with columns that contain spaces. (`#90 `__) +- ``table.enable_fts()`` now works with columns that contain spaces. (:issue:`90`) .. _v2_4: 2.4 (2020-02-26) ---------------- -- ``table.disable_fts()`` can now be used to remove FTS tables and triggers that were created using ``table.enable_fts(...)``. (`#88 `__) -- The ``sqlite-utils disable-fts`` command can be used to remove FTS tables and triggers from the command-line. (`#88 `__) -- Trying to create table columns with square braces ([ or ]) in the name now raises an error. (`#86 `__) -- Subclasses of ``dict``, ``list`` and ``tuple`` are now detected as needing a JSON column. (`#87 `__) +- ``table.disable_fts()`` can now be used to remove FTS tables and triggers that were created using ``table.enable_fts(...)``. (:issue:`88`) +- The ``sqlite-utils disable-fts`` command can be used to remove FTS tables and triggers from the command-line. (:issue:`88`) +- Trying to create table columns with square braces ([ or ]) in the name now raises an error. (:issue:`86`) +- Subclasses of ``dict``, ``list`` and ``tuple`` are now detected as needing a JSON column. (:issue:`87`) .. _v2_3_1: 2.3.1 (2020-02-10) ------------------ -``table.create_index()`` now works for columns that contain spaces. (`#85 `__) +``table.create_index()`` now works for columns that contain spaces. (:issue:`85`) .. _v2_3: 2.3 (2020-02-08) ---------------- -``table.exists()`` is now a method, not a property. This was not a documented part of the API before so I'm considering this a non-breaking change. (`#83 `__) +``table.exists()`` is now a method, not a property. This was not a documented part of the API before so I'm considering this a non-breaking change. (:issue:`83`) .. _v2_2_1: 2.2.1 (2020-02-06) ------------------ -Fixed a bug where ``.upsert(..., hash_id="pk")`` threw an error (`#84 `__). +Fixed a bug where ``.upsert(..., hash_id="pk")`` threw an error (:issue:`84`). .. _v2_2: 2.2 (2020-02-01) ---------------- -New feature: ``sqlite_utils.suggest_column_types([records])`` returns the suggested column types for a list of records. See :ref:`python_api_suggest_column_types`. (`#81 `__). +New feature: ``sqlite_utils.suggest_column_types([records])`` returns the suggested column types for a list of records. See :ref:`python_api_suggest_column_types`. (:issue:`81`). This replaces the undocumented ``table.detect_column_types()`` method. @@ -601,7 +601,7 @@ New feature: ``conversions={...}`` can be passed to the ``.insert()`` family of 2.0.1 (2020-01-05) ------------------ -The ``.upsert()`` and ``.upsert_all()`` methods now raise a ``sqlite_utils.db.PrimaryKeyRequired`` exception if you call them without specifying the primary key column using ``pk=`` (`#73 `__). +The ``.upsert()`` and ``.upsert_all()`` methods now raise a ``sqlite_utils.db.PrimaryKeyRequired`` exception if you call them without specifying the primary key column using ``pk=`` (:issue:`73`). .. _v2: @@ -623,14 +623,14 @@ For full background on this change, see `issue #66 `__) +- Fixed error thrown when ``.insert_all()`` and ``.upsert_all()`` were called with empty lists (:issue:`52`) .. _v1_12: 1.12 (2019-11-04) ----------------- -Python library utilities for deleting records (`#62 `__) +Python library utilities for deleting records (:issue:`62`) - ``db["tablename"].delete(4)`` to delete by primary key, see :ref:`python_api_delete` - ``db["tablename"].delete_where("id > ?", [3])`` to delete by a where clause, see :ref:`python_api_delete_where` @@ -644,14 +644,14 @@ Option to create triggers to automatically keep FTS tables up-to-date with newly - ``sqlite-utils enable-fts ... --create-triggers`` - see :ref:`Configuring full-text search using the CLI ` - ``db["tablename"].enable_fts(..., create_triggers=True)`` - see :ref:`Configuring full-text search using the Python library ` -- Support for introspecting triggers for a database or table - see :ref:`python_api_introspection` (`#59 `__) +- Support for introspecting triggers for a database or table - see :ref:`python_api_introspection` (:issue:`59`) .. _v1_10: 1.10 (2019-08-23) ----------------- -Ability to introspect and run queries against views (`#54 `__) +Ability to introspect and run queries against views (:issue:`54`) - ``db.view_names()`` method and and ``db.views`` property - Separate ``View`` and ``Table`` classes, both subclassing new ``Queryable`` class @@ -664,21 +664,21 @@ See :ref:`python_api_views`. 1.9 (2019-08-04) ---------------- -- ``table.m2m(...)`` method for creating many-to-many relationships: :ref:`python_api_m2m` (`#23 `__) +- ``table.m2m(...)`` method for creating many-to-many relationships: :ref:`python_api_m2m` (:issue:`23`) .. _v1_8: 1.8 (2019-07-28) ---------------- -- ``table.update(pk, values)`` method: :ref:`python_api_update` (`#35 `__) +- ``table.update(pk, values)`` method: :ref:`python_api_update` (:issue:`35`) .. _v1_7_1: 1.7.1 (2019-07-28) ------------------ -- Fixed bug where inserting records with 11 columns in a batch of 100 triggered a "too many SQL variables" error (`#50 `__) +- Fixed bug where inserting records with 11 columns in a batch of 100 triggered a "too many SQL variables" error (:issue:`50`) - Documentation and tests for ``table.drop()`` method: :ref:`python_api_drop` .. _v1_7: @@ -688,8 +688,8 @@ See :ref:`python_api_views`. Support for lookup tables. -- New ``table.lookup({...})`` utility method for building and querying lookup tables - see :ref:`python_api_lookup_tables` (`#44 `__) -- New ``extracts=`` table configuration option, see :ref:`python_api_extracts` (`#46 `__) +- New ``table.lookup({...})`` utility method for building and querying lookup tables - see :ref:`python_api_lookup_tables` (:issue:`44`) +- New ``extracts=`` table configuration option, see :ref:`python_api_extracts` (:issue:`46`) - Use `pysqlite3 `__ if it is available, otherwise use ``sqlite3`` from the standard library - Table options can now be passed to the new ``db.table(name, **options)`` factory function in addition to being passed to ``insert_all(records, **options)`` and friends - see :ref:`python_api_table_configuration` - In-memory databases can now be created using ``db = Database(memory=True)`` @@ -699,19 +699,19 @@ Support for lookup tables. 1.6 (2019-07-18) ---------------- -- ``sqlite-utils insert`` can now accept TSV data via the new ``--tsv`` option (`#41 `__) +- ``sqlite-utils insert`` can now accept TSV data via the new ``--tsv`` option (:issue:`41`) .. _v1_5: 1.5 (2019-07-14) ---------------- -- Support for compound primary keys (`#36 `__) +- Support for compound primary keys (:issue:`36`) - Configure these using the CLI tool by passing ``--pk`` multiple times - In Python, pass a tuple of columns to the ``pk=(..., ...)`` argument: :ref:`python_api_compound_primary_keys` -- New ``table.get()`` method for retrieving a record by its primary key: :ref:`python_api_get` (`#39 `__) +- New ``table.get()`` method for retrieving a record by its primary key: :ref:`python_api_get` (:issue:`39`) .. _v1_4_1: @@ -725,14 +725,14 @@ Support for lookup tables. 1.4 (2019-06-30) ---------------- -- Added ``sqlite-utils index-foreign-keys`` command (:ref:`docs `) and ``db.index_foreign_keys()`` method (:ref:`docs `) (`#33 `__) +- Added ``sqlite-utils index-foreign-keys`` command (:ref:`docs `) and ``db.index_foreign_keys()`` method (:ref:`docs `) (:issue:`33`) .. _v1_3: 1.3 (2019-06-28) ---------------- -- New mechanism for adding multiple foreign key constraints at once: :ref:`db.add_foreign_keys() documentation ` (`#31 `__) +- New mechanism for adding multiple foreign key constraints at once: :ref:`db.add_foreign_keys() documentation ` (:issue:`31`) .. _v1_2_2: @@ -746,15 +746,15 @@ Support for lookup tables. 1.2.1 (2019-06-20) ------------------ -- Check the column exists before attempting to add a foreign key (`#29 `__) +- Check the column exists before attempting to add a foreign key (:issue:`29`) .. _v1_2: 1.2 (2019-06-12) ---------------- -- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by introspecting the database. See :ref:`python_api_add_foreign_key` for details. (`#25 `__) -- Ability to set ``NOT NULL`` constraints and ``DEFAULT`` values when creating tables (`#24 `__). Documentation: :ref:`Setting defaults and not null constraints (Python API) `, :ref:`Setting defaults and not null constraints (CLI) ` +- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by introspecting the database. See :ref:`python_api_add_foreign_key` for details. (:issue:`25`) +- Ability to set ``NOT NULL`` constraints and ``DEFAULT`` values when creating tables (:issue:`24`). Documentation: :ref:`Setting defaults and not null constraints (Python API) `, :ref:`Setting defaults and not null constraints (CLI) ` - Support for ``not_null_default=X`` / ``--not-null-default`` for setting a ``NOT NULL DEFAULT 'x'`` when adding a new column. Documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` .. _v1_1: @@ -762,8 +762,8 @@ Support for lookup tables. 1.1 (2019-05-28) ---------------- -- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key already exists (`#21 `__) - documentation: :ref:`Inserting data (Python API) `, :ref:`Inserting data (CLI) ` -- Ability to add a column that is a foreign key reference using ``fk=...`` / ``--fk`` (`#16 `__) - documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` +- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key already exists (:issue:`21`) - documentation: :ref:`Inserting data (Python API) `, :ref:`Inserting data (CLI) ` +- Ability to add a column that is a foreign key reference using ``fk=...`` / ``--fk`` (:issue:`16`) - documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` .. _v1_0_1: diff --git a/docs/conf.py b/docs/conf.py index 929a41e..b4c7f44 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,11 @@ from subprocess import Popen, PIPE # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = ["sphinx.ext.extlinks"] + +extlinks = { + "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"), +} # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] From ec50e5eebc502c85448a2d3db74985c3b0c630c5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 14:53:44 -0700 Subject: [PATCH 079/102] sqlite3.enable_callback_tracebacks(True) in docs, closes #300 --- docs/python-api.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index c372a62..7ff3727 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2323,6 +2323,18 @@ If you want to deliberately replace the registered function with a new implement def reverse_string(s): return s[::-1] +Exceptions that occur inside a user-defined function default to returning the following error:: + + Unexpected error: user-defined function raised exception + +You can cause ``sqlite3`` to return more useful errors, including the traceback from the custom function, by executing the following before your custom fuctions are executed: + +.. code-block:: python + + from sqlite_utils.utils import sqlite3 + + sqlite3.enable_callback_tracebacks(True) + .. _python_api_quote: Quoting strings for use in SQL From d83f624a3124ef9489014fc1f023646f082fdc55 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 15:39:16 -0700 Subject: [PATCH 080/102] Clarified documentation for convert recipes, refs #251 --- docs/cli.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 3802963..125dc87 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -986,6 +986,11 @@ Various built-in recipe functions are available for common operations. These are These recipes can be used in the code passed to ``sqlite-utils convert`` like this:: + $ sqlite-utils convert my.db mytable mycolumn \ + 'r.jsonsplit(value)' + +To use any of the documented parameters, do this:: + $ sqlite-utils convert my.db mytable mycolumn \ 'r.jsonsplit(value, delimiter=":")' From f7c8c78cd0916bd9bca043a665d7c1ec7ae5da7d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Aug 2021 23:05:45 -0700 Subject: [PATCH 081/102] Fixed typo: commad --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index aa3234d..f4f132e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -11,7 +11,7 @@ Using Homebrew ============== -The :ref:`sqlite-utils commad-line tool ` can be installed on macOS using Homebrew:: +The :ref:`sqlite-utils command-line tool ` can be installed on macOS using Homebrew:: brew install sqlite-utils From 991cf56ae2840aaefda2af828a5c40396d2506ca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Aug 2021 09:48:37 -0700 Subject: [PATCH 082/102] Check spelling with codespell, closes #307 --- .github/workflows/spellcheck.yml | 25 +++++++++++++++++++++++++ docs/cli.rst | 4 ++-- docs/codespell-ignore-words.txt | 1 + docs/python-api.rst | 4 ++-- setup.py | 2 +- 5 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/spellcheck.yml create mode 100644 docs/codespell-ignore-words.txt diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml new file mode 100644 index 0000000..d498e17 --- /dev/null +++ b/.github/workflows/spellcheck.yml @@ -0,0 +1,25 @@ +name: Check spelling in documentation + +on: [push, pull_request] + +jobs: + spellcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install dependencies + run: | + pip install -e '.[docs]' + - name: Check spelling + run: codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt diff --git a/docs/cli.rst b/docs/cli.rst index 125dc87..e766d73 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -361,7 +361,7 @@ Passing ``--save other.db`` will instead use that SQL to populate a new database % sqlite-utils memory dogs.csv --save dogs.db -These features are mainly intented as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`. +These features are mainly intended as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`. .. _cli_rows: @@ -617,7 +617,7 @@ The ``_analyze_tables_`` table has the following schema:: PRIMARY KEY ([table], [column]) ); -The ``most_common`` and ``least_common`` columns will contain nested JSON arrays of the most commond and least common values that look like this:: +The ``most_common`` and ``least_common`` columns will contain nested JSON arrays of the most common and least common values that look like this:: [ ["Del Libertador, Av", 5068], diff --git a/docs/codespell-ignore-words.txt b/docs/codespell-ignore-words.txt new file mode 100644 index 0000000..a625cde --- /dev/null +++ b/docs/codespell-ignore-words.txt @@ -0,0 +1 @@ +AddWordsToIgnoreHere diff --git a/docs/python-api.rst b/docs/python-api.rst index 7ff3727..70baf0c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1096,7 +1096,7 @@ Here's an example of this mechanism in action: ]) db["books"].add_foreign_key("author_id", "authors", "id") -The ``table.add_foreign_key(column, other_table, other_column)`` method takes the name of the column, the table that is being referenced and the key column within that other table. If you ommit the ``other_column`` argument the primary key from that table will be used automatically. If you omit the ``other_table`` argument the table will be guessed based on some simple rules: +The ``table.add_foreign_key(column, other_table, other_column)`` method takes the name of the column, the table that is being referenced and the key column within that other table. If you omit the ``other_column`` argument the primary key from that table will be used automatically. If you omit the ``other_table`` argument the table will be guessed based on some simple rules: - If the column is of format ``author_id``, look for tables called ``author`` or ``authors`` - If the column does not end in ``_id``, try looking for a table with the exact name of the column or that name with an added ``s`` @@ -2327,7 +2327,7 @@ Exceptions that occur inside a user-defined function default to returning the fo Unexpected error: user-defined function raised exception -You can cause ``sqlite3`` to return more useful errors, including the traceback from the custom function, by executing the following before your custom fuctions are executed: +You can cause ``sqlite3`` to return more useful errors, including the traceback from the custom function, by executing the following before your custom functions are executed: .. code-block:: python diff --git a/setup.py b/setup.py index 8aa9aeb..5b13427 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "test": ["pytest", "black", "hypothesis"], - "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], + "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], "mypy": ["mypy", "types-click", "types-tabulate", "types-python-dateutil"], "flake8": ["flake8"], }, From cff6afcc43bb96a1e028aca69b67f7d758820150 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Aug 2021 10:06:08 -0700 Subject: [PATCH 083/102] Run codespell against source code too, refs #307 --- .github/workflows/spellcheck.yml | 4 +++- docs/codespell-ignore-words.txt | 2 +- sqlite_utils/cli.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index d498e17..8a86cd2 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -22,4 +22,6 @@ jobs: run: | pip install -e '.[docs]' - name: Check spelling - run: codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + run: | + codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + codespell sqlite_utils --ignore-words docs/codespell-ignore-words.txt diff --git a/docs/codespell-ignore-words.txt b/docs/codespell-ignore-words.txt index a625cde..f8418c4 100644 --- a/docs/codespell-ignore-words.txt +++ b/docs/codespell-ignore-words.txt @@ -1 +1 @@ -AddWordsToIgnoreHere +doub diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index bd063f6..c4f8501 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -44,7 +44,7 @@ It's often worth trying: --encoding=latin-1 """.strip() -# Increase CSV field size limit to maximim possible +# Increase CSV field size limit to maximum possible # https://stackoverflow.com/a/15063941 field_size_limit = sys.maxsize From cc90745f4e8bb1ac57d8ee973863cfe00c2e4fe5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 4 Aug 2021 13:34:30 -0700 Subject: [PATCH 084/102] Started a Jupyter notebook tutorial, refs #308 --- docs/tutorial.ipynb | 1053 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1053 insertions(+) create mode 100644 docs/tutorial.ipynb diff --git a/docs/tutorial.ipynb b/docs/tutorial.ipynb new file mode 100644 index 0000000..aa22461 --- /dev/null +++ b/docs/tutorial.ipynb @@ -0,0 +1,1053 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "27ae18ec", + "metadata": {}, + "source": [ + "# The sqlite-utils tutorial\n", + "\n", + "[sqlite-utils](https://sqlite-utils.datasette.io/en/stable/python-api.html) is a Python library (and [command-line tool](https://sqlite-utils.datasette.io/en/stable/cli.html) for quickly creating and manipulating SQLite database files.\n", + "\n", + "This tutorial will show you how to use the Python library to manipulate data.\n", + "\n", + "## Installation\n", + "\n", + "To install the library, run:\n", + "\n", + " pip install sqlite-utils\n", + "\n", + "You can run this in a Jupyter notebook cell by executing:\n", + "\n", + " %pip install sqlite-utils\n", + " \n", + "Or use `pip install -U sqlite-utils` to ensure you have upgraded to the most recent version." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "bddee0d2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: sqlite_utils in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (3.14)\n", + "Requirement already satisfied: click-default-group in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (1.2.2)\n", + "Requirement already satisfied: sqlite-fts4 in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (1.0.1)\n", + "Requirement already satisfied: click in /Users/simon/Library/Python/3.9/lib/python/site-packages (from sqlite_utils) (7.1.2)\n", + "Requirement already satisfied: tabulate in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (0.8.7)\n", + "Requirement already satisfied: dateutils in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from sqlite_utils) (0.6.12)\n", + "Requirement already satisfied: python-dateutil in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from dateutils->sqlite_utils) (2.8.1)\n", + "Requirement already satisfied: pytz in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from dateutils->sqlite_utils) (2021.1)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from python-dateutil->dateutils->sqlite_utils) (1.16.0)\n", + "\u001b[33mWARNING: You are using pip version 21.1.1; however, version 21.2.2 is available.\n", + "You should consider upgrading via the '/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/bin/python3.9 -m pip install --upgrade pip' command.\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -U sqlite_utils" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "050e85a8", + "metadata": {}, + "outputs": [], + "source": [ + "import sqlite_utils" + ] + }, + { + "cell_type": "markdown", + "id": "348bcbfc", + "metadata": {}, + "source": [ + "You can use the library with a database file on disk by running:\n", + "\n", + " db = sqlite_utils.Database(\"path/to/my/database.db\")\n", + "\n", + "In this tutorial we will use an in-memory database. This is a quick way to try out new things, though you should note that when you close the notebook the data store in the in-memory database will be lost." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4b2aee7e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + ">" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db = sqlite_utils.Database(memory=True)\n", + "db" + ] + }, + { + "cell_type": "markdown", + "id": "1598ab43", + "metadata": {}, + "source": [ + "## Creating a table\n", + "\n", + "We are going to create a new table in our database called `creatures` by passing in a Python list of dictionaries.\n", + "\n", + "`db[name_of_table]` will access a database table object with that name.\n", + "\n", + "Inserting data into that table will create it if it does not already exist." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "4a0ac420", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db[\"creatures\"].insert_all([{\n", + " \"name\": \"Cleo\",\n", + " \"species\": \"dog\",\n", + " \"age\": 6\n", + "}, {\n", + " \"name\": \"Lila\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}, {\n", + " \"name\": \"Bants\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}])" + ] + }, + { + "cell_type": "markdown", + "id": "049d110b", + "metadata": {}, + "source": [ + "Let's grab a `table` reference to the new creatures table:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "8d84ad9c", + "metadata": {}, + "outputs": [], + "source": [ + "table = db[\"creatures\"]" + ] + }, + { + "cell_type": "markdown", + "id": "ffe45750", + "metadata": {}, + "source": [ + "`sqlite-utils` automatically creates a table schema that matches the keys and data types of the dictionaries that were passed to `.insert_all()`.\n", + "\n", + "We can see that schema using `table.schema`:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "136cee1e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE [creatures] (\n", + " [name] TEXT,\n", + " [species] TEXT,\n", + " [age] FLOAT\n", + ")\n" + ] + } + ], + "source": [ + "print(table.schema)" + ] + }, + { + "cell_type": "markdown", + "id": "9e5c3ae9", + "metadata": {}, + "source": [ + "## Accessing data\n", + "\n", + "The `table.rows` property lets us loop through the rows in the table, returning each one as a Python dictionary:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "f812914d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'name': 'Cleo', 'species': 'dog', 'age': 6.0}\n", + "{'name': 'Lila', 'species': 'chicken', 'age': 0.8}\n", + "{'name': 'Bants', 'species': 'chicken', 'age': 0.8}\n" + ] + } + ], + "source": [ + "for row in table.rows:\n", + " print(row)" + ] + }, + { + "cell_type": "markdown", + "id": "60bc6b2c", + "metadata": {}, + "source": [ + "The `db.query(sql)` method can be used to execute SQL queries and return the results as dictionaries:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "eaadd85f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'name': 'Bants', 'species': 'chicken', 'age': 0.8}]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select * from creatures\"))" + ] + }, + { + "cell_type": "markdown", + "id": "6614467b", + "metadata": {}, + "source": [ + "Or in a loop:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "88fdd52e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cleo is a dog\n", + "Lila is a chicken\n", + "Bants is a chicken\n" + ] + } + ], + "source": [ + "for row in db.query(\"select name, species from creatures\"):\n", + " print(f'{row[\"name\"]} is a {row[\"species\"]}')" + ] + }, + { + "cell_type": "markdown", + "id": "b81c031c", + "metadata": {}, + "source": [ + "### SQL parameters\n", + "\n", + "You can run a parameterized query using `?` as placeholders and passing a list of variables. The variables you pass will be correctly quoted, protecting your code from SQL injection vulnerabilities." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "267035d9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'Cleo', 'species': 'dog', 'age': 6.0}]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select * from creatures where age > ?\", [1.0]))" + ] + }, + { + "cell_type": "markdown", + "id": "87cb301b", + "metadata": {}, + "source": [ + "As an alternative to question marks we can use `:name` parameters and feed in the values using a dictionary:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "83be9a80", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'name': 'Bants', 'species': 'chicken', 'age': 0.8}]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select * from creatures where species = :species\", {\"species\": \"chicken\"}))" + ] + }, + { + "cell_type": "markdown", + "id": "5e5179cc", + "metadata": {}, + "source": [ + "### Primary keys\n", + "\n", + "When we created this table we did not specify a primary key. SQLite automatically creates a primary key called `rowid` if no other primary key is defined.\n", + "\n", + "We can run `select rowid, * from creatures` to see this hidden primary key:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "c9d963df", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'rowid': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'rowid': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'rowid': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8}]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select rowid, * from creatures\"))" + ] + }, + { + "cell_type": "markdown", + "id": "0f87cdfb", + "metadata": {}, + "source": [ + "We can also see that using `table.pks_and_rows_where()`:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "d365e405", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 {'rowid': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0}\n", + "2 {'rowid': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8}\n", + "3 {'rowid': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8}\n" + ] + } + ], + "source": [ + "for pk, row in table.pks_and_rows_where():\n", + " print(pk, row)" + ] + }, + { + "cell_type": "markdown", + "id": "5b0e9b74", + "metadata": {}, + "source": [ + "Let's recreate the table with our own primary key, which we will call `id`.\n", + "\n", + "`table.drop()` drops the table:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "568a0e29", + "metadata": {}, + "outputs": [], + "source": [ + "table.drop()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "13ebd3ab", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table" + ] + }, + { + "cell_type": "markdown", + "id": "522aa6d0", + "metadata": {}, + "source": [ + "We can see a list of tables in the database using `db.tables`:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "f3e62678", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db.tables" + ] + }, + { + "cell_type": "markdown", + "id": "6b80d523", + "metadata": {}, + "source": [ + "We'll create the table again, this time with an `id` column.\n", + "\n", + "We use `pk=\"id\"` to specify that the `id` column should be treated as the primary key for the table:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "c9ee8b9f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db[\"creatures\"].insert_all([{\n", + " \"id\": 1,\n", + " \"name\": \"Cleo\",\n", + " \"species\": \"dog\",\n", + " \"age\": 6\n", + "}, {\n", + " \"id\": 2,\n", + " \"name\": \"Lila\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}, {\n", + " \"id\": 3,\n", + " \"name\": \"Bants\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}], pk=\"id\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "523e01ab", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE [creatures] (\n", + " [id] INTEGER PRIMARY KEY,\n", + " [name] TEXT,\n", + " [species] TEXT,\n", + " [age] FLOAT\n", + ")\n" + ] + } + ], + "source": [ + "print(table.schema)" + ] + }, + { + "cell_type": "markdown", + "id": "811bea70", + "metadata": {}, + "source": [ + "## Inserting more records\n", + "\n", + "We can call `.insert_all()` again to insert more records. Let's add two more chickens." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "716df161", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.insert_all([{\n", + " \"id\": 4,\n", + " \"name\": \"Azi\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}, {\n", + " \"id\": 5,\n", + " \"name\": \"Snowy\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.9,\n", + "}], pk=\"id\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "4b1b2476", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'id': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 4, 'name': 'Azi', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 5, 'name': 'Snowy', 'species': 'chicken', 'age': 0.9}]" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(table.rows)" + ] + }, + { + "cell_type": "markdown", + "id": "2af4ae75", + "metadata": {}, + "source": [ + "Since the `id` column is an integer primary key, we can insert a record without specifying an ID and one will be automatically added.\n", + "\n", + "Since we are only adding one record we will use `.insert()` instead of `.insert_all()`." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "246c6dd5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.insert({\"name\": \"Blue\", \"species\": \"chicken\", \"age\": 0.9})" + ] + }, + { + "cell_type": "markdown", + "id": "d7c28e4d", + "metadata": {}, + "source": [ + "We can use `table.last_pk` to see the ID of the record we just added." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "de012e1e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "6" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.last_pk" + ] + }, + { + "cell_type": "markdown", + "id": "c38edaf4", + "metadata": {}, + "source": [ + "Here's the full list of rows again:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "7c27075e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'id': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 4, 'name': 'Azi', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 5, 'name': 'Snowy', 'species': 'chicken', 'age': 0.9},\n", + " {'id': 6, 'name': 'Blue', 'species': 'chicken', 'age': 0.9}]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(table.rows)" + ] + }, + { + "cell_type": "markdown", + "id": "64931bd0", + "metadata": {}, + "source": [ + "If you try to add a new record with an existing ID, you will get an `IntegrityError`:" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "36327794", + "metadata": {}, + "outputs": [ + { + "ename": "IntegrityError", + "evalue": "UNIQUE constraint failed: creatures.id", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mIntegrityError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"id\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m6\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"name\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"Red\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"species\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"chicken\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"age\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m0.9\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/sqlite_utils/db.py\u001b[0m in \u001b[0;36minsert\u001b[0;34m(self, record, pk, foreign_keys, column_order, not_null, defaults, hash_id, alter, ignore, replace, extracts, conversions, columns)\u001b[0m\n\u001b[1;32m 2027\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mDEFAULT\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2028\u001b[0m ):\n\u001b[0;32m-> 2029\u001b[0;31m return self.insert_all(\n\u001b[0m\u001b[1;32m 2030\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2031\u001b[0m \u001b[0mpk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpk\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/sqlite_utils/db.py\u001b[0m in \u001b[0;36minsert_all\u001b[0;34m(self, records, pk, foreign_keys, column_order, not_null, defaults, batch_size, hash_id, alter, ignore, replace, truncate, extracts, conversions, columns, upsert)\u001b[0m\n\u001b[1;32m 2143\u001b[0m \u001b[0mfirst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2145\u001b[0;31m self.insert_chunk(\n\u001b[0m\u001b[1;32m 2146\u001b[0m \u001b[0malter\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2147\u001b[0m \u001b[0mextracts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/sqlite_utils/db.py\u001b[0m in \u001b[0;36minsert_chunk\u001b[0;34m(self, alter, extracts, chunk, all_columns, hash_id, upsert, pk, conversions, num_records_processed, replace, ignore)\u001b[0m\n\u001b[1;32m 1955\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mqueries_and_params\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1956\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1957\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1958\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mOperationalError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1959\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0malter\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\" column\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/sqlite_utils/db.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, sql, parameters)\u001b[0m\n\u001b[1;32m 255\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tracer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 256\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mparameters\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 257\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 258\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 259\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mIntegrityError\u001b[0m: UNIQUE constraint failed: creatures.id" + ] + } + ], + "source": [ + "table.insert({\"id\": 6, \"name\": \"Red\", \"species\": \"chicken\", \"age\": 0.9})" + ] + }, + { + "cell_type": "markdown", + "id": "2e00692f", + "metadata": {}, + "source": [ + "You can use `replace=True` to replace the matching record with a new one:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "2be75589", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.insert({\"id\": 6, \"name\": \"Red\", \"species\": \"chicken\", \"age\": 0.9}, replace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "83281675", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'id': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 4, 'name': 'Azi', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 5, 'name': 'Snowy', 'species': 'chicken', 'age': 0.9},\n", + " {'id': 6, 'name': 'Red', 'species': 'chicken', 'age': 0.9}]" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(table.rows)" + ] + }, + { + "cell_type": "markdown", + "id": "d7122b76", + "metadata": {}, + "source": [ + "## Updating a record\n", + "\n", + "We will rename that row back to `Blue`, this time using the `table.update(pk, updates)` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "43df156d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.update(6, {\"name\": \"Blue\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "0b8f8422", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 6, 'name': 'Blue', 'species': 'chicken', 'age': 0.9}]" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select * from creatures where id = ?\", [6]))" + ] + }, + { + "cell_type": "markdown", + "id": "58142b86", + "metadata": {}, + "source": [ + "## Extracting one of the columns into another table\n", + "\n", + "Our current table has a `species` column with a string in it - let's pull that out into a separate table.\n", + "\n", + "We can do that using the [table.extract() method](https://sqlite-utils.datasette.io/en/stable/python-api.html#extracting-columns-into-a-separate-table)." + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "6ab69111", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.extract(\"species\")" + ] + }, + { + "cell_type": "markdown", + "id": "dca327b2", + "metadata": {}, + "source": [ + "We now have a new table called `species`, which we can see using the `db.tables` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "76e95b36", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[
,
]" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db.tables" + ] + }, + { + "cell_type": "markdown", + "id": "5ea43bf5", + "metadata": {}, + "source": [ + "Our creatures table has been modified - instead of a `species` column it now has `species_id` which is a foreign key to the new table:" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "c0438bff", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE \"creatures\" (\n", + " [id] INTEGER PRIMARY KEY,\n", + " [name] TEXT,\n", + " [species_id] INTEGER,\n", + " [age] FLOAT,\n", + " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n", + ")\n", + "[{'id': 1, 'name': 'Cleo', 'species_id': 1, 'age': 6.0}, {'id': 2, 'name': 'Lila', 'species_id': 2, 'age': 0.8}, {'id': 3, 'name': 'Bants', 'species_id': 2, 'age': 0.8}, {'id': 4, 'name': 'Azi', 'species_id': 2, 'age': 0.8}, {'id': 5, 'name': 'Snowy', 'species_id': 2, 'age': 0.9}, {'id': 6, 'name': 'Blue', 'species_id': 2, 'age': 0.9}]\n" + ] + } + ], + "source": [ + "print(db[\"creatures\"].schema)\n", + "print(list(db[\"creatures\"].rows))" + ] + }, + { + "cell_type": "markdown", + "id": "0452c201", + "metadata": {}, + "source": [ + "The new `species` table has been created and populated too:" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "5d38c3a8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE [species] (\n", + " [id] INTEGER PRIMARY KEY,\n", + " [species] TEXT\n", + ")\n", + "[{'id': 1, 'species': 'dog'}, {'id': 2, 'species': 'chicken'}]\n" + ] + } + ], + "source": [ + "print(db[\"species\"].schema)\n", + "print(list(db[\"species\"].rows))" + ] + }, + { + "cell_type": "markdown", + "id": "a0312d1e", + "metadata": {}, + "source": [ + "We can use a join SQL query to combine data from these two tables:" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "6734ed5d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 'Cleo', 'age': 6.0, 'species_id': 1, 'species': 'dog'},\n", + " {'id': 2, 'name': 'Lila', 'age': 0.8, 'species_id': 2, 'species': 'chicken'},\n", + " {'id': 3, 'name': 'Bants', 'age': 0.8, 'species_id': 2, 'species': 'chicken'},\n", + " {'id': 4, 'name': 'Azi', 'age': 0.8, 'species_id': 2, 'species': 'chicken'},\n", + " {'id': 5, 'name': 'Snowy', 'age': 0.9, 'species_id': 2, 'species': 'chicken'},\n", + " {'id': 6, 'name': 'Blue', 'age': 0.9, 'species_id': 2, 'species': 'chicken'}]" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"\"\"\n", + " select\n", + " creatures.id,\n", + " creatures.name,\n", + " creatures.age,\n", + " species.id as species_id,\n", + " species.species\n", + " from creatures\n", + " join species on creatures.species_id = species.id\n", + "\"\"\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c4802ac", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From f67327abf0a9f018e1764660e190c5bbf9556ec2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 14:44:03 -0700 Subject: [PATCH 085/102] sqlite-utils insert --flatten option, closes #310 --- docs/cli.rst | 43 +++++++++++++++++++++++++++++++++++ sqlite_utils/cli.py | 19 ++++++++++++++++ tests/test_cli.py | 55 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index e766d73..28bd3f7 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -722,6 +722,49 @@ This also means you pipe ``sqlite-utils`` together to easily create a new SQLite 207368,920 Kirkham St,37.760210314285,-122.47073935813 188702,1501 Evans Ave,37.7422086702947,-122.387293152263 +.. _cli_inserting_data_flatten: + +Flattening nested JSON objects +------------------------------ + +``sqlite-utils insert`` expects incoming data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table. + +If your data is nested you can use the `--flatten` object to create columns that are derived from the nested data. + +Consider this example document, in a file called ``log.json``:: + + { + "httpRequest": { + "latency": "0.112114537s", + "requestMethod": "GET", + "requestSize": "534", + "status": 200 + }, + "insertId": "6111722f000b5b4c4d4071e2", + "labels": { + "service": "datasette-io" + } + } + +Inserting this into a table using ``sqlite-utils insert logs.db log log.json`` will create a table with the following schema:: + + CREATE TABLE [logs] ( + [httpRequest] TEXT, + [insertId] TEXT, + [labels] TEXT + ); + +With the ``--flatten`` option columns will be created using ``topkey_nextkey`` column names - so running ``sqlite-utils insert logs.db log log.json --flatten`` will create the following schema instead:: + + CREATE TABLE [logs] ( + [httpRequest_latency] TEXT, + [httpRequest_requestMethod] TEXT, + [httpRequest_requestSize] TEXT, + [httpRequest_status] INTEGER, + [insertId] TEXT, + [labels_service] TEXT + ); + .. _cli_insert_csv_tsv: Inserting CSV or TSV data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c4f8501..2f32d0a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -643,6 +643,7 @@ def insert_upsert_options(fn): "--pk", help="Columns to use as the primary key, e.g. id", multiple=True ), click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), + click.option("--flatten", is_flag=True, help="Flatten nested JSON objets"), click.option("-c", "--csv", is_flag=True, help="Expect CSV"), click.option("--tsv", is_flag=True, help="Expect TSV"), click.option("--delimiter", help="Delimiter to use for CSV files"), @@ -697,6 +698,7 @@ def insert_upsert_implementation( json_file, pk, nl, + flatten, csv, tsv, delimiter, @@ -722,6 +724,8 @@ def insert_upsert_implementation( csv = True if (nl + csv + tsv) >= 2: raise click.ClickException("Use just one of --nl, --csv or --tsv") + if (csv or tsv) and flatten: + raise click.ClickException("--flatten cannot be used with --csv or --tsv") if encoding and not (csv or tsv): raise click.ClickException("--encoding must be used with --csv or --tsv") if pk and len(pk) == 1: @@ -766,6 +770,8 @@ def insert_upsert_implementation( raise click.ClickException( "Invalid JSON - use --csv for CSV or --tsv for TSV files" ) + if flatten: + docs = (dict(_flatten(doc)) for doc in docs) extra_kwargs = {"ignore": ignore, "replace": replace, "truncate": truncate} if not_null: @@ -790,6 +796,15 @@ def insert_upsert_implementation( db[table].transform(types=tracker.types) +def _flatten(d): + for key, value in d.items(): + if isinstance(value, dict): + for key2, value2 in _flatten(value): + yield key + "_" + key2, value2 + else: + yield key, value + + @cli.command() @insert_upsert_options @click.option( @@ -813,6 +828,7 @@ def insert( json_file, pk, nl, + flatten, csv, tsv, delimiter, @@ -844,6 +860,7 @@ def insert( json_file, pk, nl, + flatten, csv, tsv, delimiter, @@ -875,6 +892,7 @@ def upsert( json_file, pk, nl, + flatten, csv, tsv, batch_size, @@ -902,6 +920,7 @@ def upsert( json_file, pk, nl, + flatten, csv, tsv, delimiter, diff --git a/tests/test_cli.py b/tests/test_cli.py index e84453a..e6f6683 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -648,6 +648,34 @@ def test_insert_invalid_json_error(tmpdir): ) +def test_insert_json_flatten(tmpdir): + db_path = str(tmpdir / "flat.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-", "--flatten"], + input=json.dumps({"nested": {"data": 4}}), + ) + assert result.exit_code == 0 + assert list(Database(db_path).query("select * from items")) == [{"nested_data": 4}] + + +def test_insert_json_flatten_nl(tmpdir): + db_path = str(tmpdir / "flat.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-", "--flatten", "--nl"], + input="\n".join( + json.dumps(item) + for item in [{"nested": {"data": 4}}, {"nested": {"other": 3}}] + ), + ) + assert result.exit_code == 0 + assert list(Database(db_path).query("select * from items")) == [ + {"nested_data": 4, "nested_other": None}, + {"nested_data": None, "nested_other": 3}, + ] + + def test_insert_with_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dog.json") open(json_path, "w").write(json.dumps({"id": 1, "name": "Cleo", "age": 4})) @@ -1197,6 +1225,21 @@ def test_upsert(db_path, tmpdir): ] +def test_upsert_flatten(tmpdir): + db_path = str(tmpdir / "flat.db") + db = Database(db_path) + db["upsert_me"].insert({"id": 1, "name": "Example"}, pk="id") + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "upsert_me", "-", "--flatten", "--pk", "id", "--alter"], + input=json.dumps({"id": 1, "nested": {"two": 2}}), + ) + assert result.exit_code == 0 + assert list(db.query("select * from upsert_me")) == [ + {"id": 1, "name": "Example", "nested_two": 2} + ] + + def test_upsert_alter(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") db = Database(db_path) @@ -2249,3 +2292,15 @@ def test_insert_detect_types(tmpdir, option_or_env_var): _test() else: _test() + + +@pytest.mark.parametrize( + "input,expected", + ( + ({"foo": {"bar": 1}}, {"foo_bar": 1}), + ({"foo": {"bar": [1, 2, {"baz": 3}]}}, {"foo_bar": [1, 2, {"baz": 3}]}), + ({"foo": {"bar": 1, "baz": {"three": 3}}}, {"foo_bar": 1, "foo_baz_three": 3}), + ), +) +def test_flatten_helper(input, expected): + assert dict(cli._flatten(input)) == expected From 15758d02fd437004fd9f84c9d4a8bf49f0793e13 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 14:45:39 -0700 Subject: [PATCH 086/102] Fixed spelling of objects, refs #310 --- sqlite_utils/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 2f32d0a..79103bc 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -643,7 +643,7 @@ def insert_upsert_options(fn): "--pk", help="Columns to use as the primary key, e.g. id", multiple=True ), click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), - click.option("--flatten", is_flag=True, help="Flatten nested JSON objets"), + click.option("--flatten", is_flag=True, help="Flatten nested JSON objects"), click.option("-c", "--csv", is_flag=True, help="Expect CSV"), click.option("--tsv", is_flag=True, help="Expect TSV"), click.option("--delimiter", help="Delimiter to use for CSV files"), From 3fb1034e869090876cab0247146a312be993210f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 14:46:47 -0700 Subject: [PATCH 087/102] option, not object, refs #310 --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 28bd3f7..063b9e9 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -729,7 +729,7 @@ Flattening nested JSON objects ``sqlite-utils insert`` expects incoming data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table. -If your data is nested you can use the `--flatten` object to create columns that are derived from the nested data. +If your data is nested you can use the ``--flatten`` option to create columns that are derived from the nested data. Consider this example document, in a file called ``log.json``:: From 14f643d9e91f5557d5e46251dadac481f4b41021 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 15:25:52 -0700 Subject: [PATCH 088/102] Better error messages in CLI, closes #309 --- sqlite_utils/cli.py | 31 ++++++++++++++++++++++++++++--- tests/test_cli.py | 21 ++++++++++++++++++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 79103bc..0acb252 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -786,12 +786,25 @@ def insert_upsert_implementation( db[table].insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs ) - except sqlite3.OperationalError as e: - if e.args and "has no column named" in e.args[0]: + except Exception as e: + if ( + isinstance(e, sqlite3.OperationalError) + and e.args + and "has no column named" in e.args[0] + ): raise click.ClickException( "{}\n\nTry using --alter to add additional columns".format(e.args[0]) ) - raise + # If we can find sql= and params= arguments, show those + variables = _find_variables(e.__traceback__, ["sql", "params"]) + if "sql" in variables and "params" in variables: + raise click.ClickException( + "{}\n\nsql = {}\nparams={}".format( + str(e), variables["sql"], variables["params"] + ) + ) + else: + raise if tracker is not None: db[table].transform(types=tracker.types) @@ -805,6 +818,18 @@ def _flatten(d): yield key, value +def _find_variables(tb, vars): + to_find = list(vars) + found = {} + for var in to_find: + if var in tb.tb_frame.f_locals: + vars.remove(var) + found[var] = tb.tb_frame.f_locals[var] + if vars and tb.tb_next: + found.update(_find_variables(tb.tb_next, vars)) + return found + + @cli.command() @insert_upsert_options @click.option( diff --git a/tests/test_cli.py b/tests/test_cli.py index e6f6683..319bd40 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1256,7 +1256,11 @@ def test_upsert_alter(db_path, tmpdir): cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"] ) assert 1 == result.exit_code - assert "no such column: age" == str(result.exception) + assert ( + "Error: no such column: age\n\n" + "sql = UPDATE [dogs] SET [age] = ? WHERE [id] = ?\n" + "params=[5, 1]" + ) == result.output.strip() # Should succeed with --alter result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id", "--alter"] @@ -2304,3 +2308,18 @@ def test_insert_detect_types(tmpdir, option_or_env_var): ) def test_flatten_helper(input, expected): assert dict(cli._flatten(input)) == expected + + +def test_integer_overflow_error(tmpdir): + db_path = str(tmpdir / "test.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-"], + input=json.dumps({"bignumber": 34223049823094832094802398430298048240}), + ) + assert result.exit_code == 1 + assert result.output == ( + "Error: Python int too large to convert to SQLite INTEGER\n\n" + "sql = INSERT INTO [items] ([bignumber]) VALUES (?);\n" + "params=[34223049823094832094802398430298048240]\n" + ) From a6567ec507e235fd4d313c6b1570d5a4f45e4b86 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 15:33:33 -0700 Subject: [PATCH 089/102] Capture parameters= not params=, refs #309 --- sqlite_utils/cli.py | 10 +++++----- tests/test_cli.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0acb252..d94229d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -795,12 +795,12 @@ def insert_upsert_implementation( raise click.ClickException( "{}\n\nTry using --alter to add additional columns".format(e.args[0]) ) - # If we can find sql= and params= arguments, show those - variables = _find_variables(e.__traceback__, ["sql", "params"]) - if "sql" in variables and "params" in variables: + # If we can find sql= and parameters= arguments, show those + variables = _find_variables(e.__traceback__, ["sql", "parameters"]) + if "sql" in variables and "parameters" in variables: raise click.ClickException( - "{}\n\nsql = {}\nparams={}".format( - str(e), variables["sql"], variables["params"] + "{}\n\nsql = {}\nparameters = {}".format( + str(e), variables["sql"], variables["parameters"] ) ) else: diff --git a/tests/test_cli.py b/tests/test_cli.py index 319bd40..d4801ac 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1259,7 +1259,7 @@ def test_upsert_alter(db_path, tmpdir): assert ( "Error: no such column: age\n\n" "sql = UPDATE [dogs] SET [age] = ? WHERE [id] = ?\n" - "params=[5, 1]" + "parameters = [5, 1]" ) == result.output.strip() # Should succeed with --alter result = CliRunner().invoke( @@ -2321,5 +2321,5 @@ def test_integer_overflow_error(tmpdir): assert result.output == ( "Error: Python int too large to convert to SQLite INTEGER\n\n" "sql = INSERT INTO [items] ([bignumber]) VALUES (?);\n" - "params=[34223049823094832094802398430298048240]\n" + "parameters = [34223049823094832094802398430298048240]\n" ) From d5ef91212022ea3ed85258af605bdc28e5799ff1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 15:42:06 -0700 Subject: [PATCH 090/102] Release 3.15 Refs #309, #310 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e03dbd6..9a1669d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v3_15: + +3.15 (2021-08-09) +----------------- + +- ``sqlite-utils insert --flatten`` option for :ref:`flattening nested JSON objects ` to create tables with column names like ``topkey_nestedkey``. (:issue:`310`) +- Fixed several spelling mistakes in the documentation, spotted `using codespell `__. +- Errors that occur while using the ``sqlite-utils`` CLI tool now show the responsible SQL and query parameters, if possible. (:issue:`309`) + .. _v3_14: 3.14 (2021-08-02) diff --git a/setup.py b/setup.py index 5b13427..c8c58ac 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.14" +VERSION = "3.15" def get_long_description(): From 8757de84b27cedf494ee917ce2daf773d2c3f877 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 15:44:26 -0700 Subject: [PATCH 091/102] Link to stable docs, not latest --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4c738ab..bac6369 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # sqlite-utils [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/) -[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/latest/changelog.html) +[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/stable/changelog.html) [![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/) [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest) -[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.datasette.io/en/latest/?badge=latest) +[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=stable)](http://sqlite-utils.datasette.io/en/stable/?badge=stable) [![codecov](https://codecov.io/gh/simonw/sqlite-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/simonw/sqlite-utils) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE) From ee469e3122d6f5973ec2584c1580d930daca2e7c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 9 Aug 2021 15:50:53 -0700 Subject: [PATCH 092/102] Corrected tiny mistake in --flatten examples --- docs/cli.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 063b9e9..0c08526 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -746,7 +746,7 @@ Consider this example document, in a file called ``log.json``:: } } -Inserting this into a table using ``sqlite-utils insert logs.db log log.json`` will create a table with the following schema:: +Inserting this into a table using ``sqlite-utils insert logs.db logs log.json`` will create a table with the following schema:: CREATE TABLE [logs] ( [httpRequest] TEXT, @@ -754,7 +754,7 @@ Inserting this into a table using ``sqlite-utils insert logs.db log log.json`` w [labels] TEXT ); -With the ``--flatten`` option columns will be created using ``topkey_nextkey`` column names - so running ``sqlite-utils insert logs.db log log.json --flatten`` will create the following schema instead:: +With the ``--flatten`` option columns will be created using ``topkey_nextkey`` column names - so running ``sqlite-utils insert logs.db logs log.json --flatten`` will create the following schema instead:: CREATE TABLE [logs] ( [httpRequest_latency] TEXT, From 6155da72c8939b5d9bdacb7853e5e8d1767ce1d5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 10 Aug 2021 16:09:28 -0700 Subject: [PATCH 093/102] Add reference page to documentation using Sphinx autodoc (#312) * Docstrings and type annotations for almost every method * New /reference API documentation page using Sphink autodoc * Custom Read The Docs config, to get autodoc working * Fix for #313 (add_foreign_keys() doesn't reject being called with a View) * Fixed #315 (.delete_where() returns [] when it should return self) --- .readthedocs.yaml | 12 + docs/Makefile | 2 +- docs/conf.py | 3 +- docs/index.rst | 1 + docs/python-api.rst | 2 +- docs/reference.rst | 69 ++++ sqlite_utils/db.py | 802 ++++++++++++++++++++++++++++++++++---------- 7 files changed, 711 insertions(+), 180 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 docs/reference.rst diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..ce66cbe --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,12 @@ +version: 2 + +sphinx: + configuration: docs/conf.py + +python: + version: "3.8" + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/docs/Makefile b/docs/Makefile index a279768..5578ae3 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -20,4 +20,4 @@ help: @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) livehtml: - sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) + sphinx-autobuild -a -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) --watch ../sqlite_utils diff --git a/docs/conf.py b/docs/conf.py index b4c7f44..1f5a158 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,8 @@ from subprocess import Popen, PIPE # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.extlinks"] +extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"] +autodoc_member_order = "bysource" extlinks = { "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"), diff --git a/docs/index.rst b/docs/index.rst index 93b0bc0..581f306 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,7 @@ Contents installation cli python-api + reference contributing changelog diff --git a/docs/python-api.rst b/docs/python-api.rst index 70baf0c..b793920 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -962,7 +962,7 @@ The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` metho The name of the column ``total_rows`` - The total number of rows in the table` + The total number of rows in the table ``num_null`` The number of rows for which this column is null diff --git a/docs/reference.rst b/docs/reference.rst new file mode 100644 index 0000000..e29f468 --- /dev/null +++ b/docs/reference.rst @@ -0,0 +1,69 @@ +=============== + API Reference +=============== + +.. contents:: :local: + +.. _reference_db_database: + +sqlite_utils.db.Database +======================== + +.. autoclass:: sqlite_utils.db.Database + :members: + :undoc-members: + :show-inheritance: + :special-members: __getitem__ + :exclude-members: use_counts_table, execute_returning_dicts, resolve_foreign_keys + +.. _reference_db_queryable: + +sqlite_utils.db.Queryable +========================= + +:ref:`Table ` and :ref:`View ` are both subclasses of ``Queryable``, providing access to the following methods: + +.. autoclass:: sqlite_utils.db.Queryable + :members: + :undoc-members: + :exclude-members: execute_count + +.. _reference_db_table: + +sqlite_utils.db.Table +===================== + +.. autoclass:: sqlite_utils.db.Table + :members: + :undoc-members: + :show-inheritance: + :exclude-members: guess_foreign_column, value_or_default, build_insert_queries_and_params, insert_chunk, add_missing_columns + +.. _reference_db_view: + +sqlite_utils.db.View +==================== + +.. autoclass:: sqlite_utils.db.View + :members: + :undoc-members: + :show-inheritance: + +.. _reference_db_other: + +Other +===== + +.. _reference_db_other_column: + +sqlite_utils.db.Column +---------------------- + +.. autoclass:: sqlite_utils.db.Column + +.. _reference_db_other_column_details: + +sqlite_utils.db.ColumnDetails +----------------------------- + +.. autoclass:: sqlite_utils.db.ColumnDetails diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a21acc4..c7dc832 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -21,7 +21,19 @@ import re from sqlite_fts4 import rank_bm25 # type: ignore import sys import textwrap -from typing import Generator, Iterable, Union, Optional, List +from typing import ( + cast, + Any, + Callable, + Dict, + Generator, + Iterable, + Union, + Optional, + List, + Set, + Tuple, +) import uuid SQLITE_MAX_VARS = 999 @@ -41,7 +53,7 @@ _virtual_table_using_re = re.compile( ) ) \s+(IF\s+NOT\s+EXISTS\s+)? # IF NOT EXISTS (optional) -USING\s+(?P\w+) # e.g. USING FTS5 +USING\s+(?P\w+) # for example USING FTS5 """, re.VERBOSE | re.IGNORECASE, ) @@ -49,7 +61,7 @@ USING\s+(?P\w+) # e.g. USING FTS5 try: import pandas as pd # type: ignore except ImportError: - pd = None + pd = None # type: ignore try: import numpy as np # type: ignore @@ -59,6 +71,28 @@ except ImportError: Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") ) +Column.__doc__ = """ +Describes a SQLite column returned by the :attr:`.Table.columns` property. + +``cid`` + Column index + +``name`` + Column name + +``type`` + Column type + +``notnull`` + Does the column have a ``not null` constraint + +``default_value`` + Default value for this column + +``is_pk`` + Is this column part of the primary key +""" + ColumnDetails = namedtuple( "ColumnDetails", ( @@ -72,6 +106,34 @@ ColumnDetails = namedtuple( "least_common", ), ) +ColumnDetails.__doc__ = """ +Summary information about a column, see :ref:`python_api_analyze_column`. + +``table`` + The name of the table + +``column`` + The name of the column + +``total_rows`` + The total number of rows in the table + +``num_null`` + The number of rows for which this column is null + +``num_blank`` + The number of rows for which this column is blank (the empty string) + +``num_distinct`` + The number of distinct values in this column + +``most_common`` + The ``N`` most common values as a list of ``(value, count)`` tuples`, or ``None`` if the table consists entirely of distinct values + +``least_common`` + The ``N`` least common values as a list of ``(value, count)`` tuples`, or ``None`` if the table is entirely distinct + or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) +""" ForeignKey = namedtuple( "ForeignKey", ("table", "column", "other_table", "other_column") ) @@ -83,7 +145,11 @@ XIndexColumn = namedtuple( Trigger = namedtuple("Trigger", ("name", "table", "sql")) -DEFAULT = object() +class Default: + pass + + +DEFAULT = Default() COLUMN_TYPE_MAPPING = { float: "FLOAT", @@ -133,26 +199,32 @@ if pd: class AlterError(Exception): + "Error altering table" pass class NoObviousTable(Exception): + "Could not tell which table this operation refers to" pass class BadPrimaryKey(Exception): + "Table does not have a single obvious primary key" pass class NotFoundError(Exception): + "Record not found" pass class PrimaryKeyRequired(Exception): + "Primary key needs to be specified" pass class InvalidColumns(Exception): + "Specified columns do not exist" pass @@ -176,17 +248,32 @@ CREATE TABLE IF NOT EXISTS [{}]( class Database: + """ + Wrapper for a SQLite database connection that adds a variety of useful utility methods. + + - ``filename_or_conn`` - String path to a file, or a ``pathlib.Path`` object, or a + ``sqlite3`` connection + - ``memory`` - set to ``True`` to create an in-memory database + - ``recreate`` - set to ``True`` to delete and recreate a file database (**dangerous**) + - ``recursive_triggers`` - defaults to ``True``, which sets ``PRAGMA recursive_triggers=on;`` - + set to ``False`` to avoid setting this pragma + - ``tracer`` - set a tracer function (``print`` works for this) which will be called with + ``sql, parameters`` every time a SQL query is executed + - ``use_counts_table`` - set to ``True`` to use a cached counts table, if available. See + :ref:`python_api_cached_table_counts`. + """ + _counts_table_name = "_counts" use_counts_table = False def __init__( self, filename_or_conn=None, - memory=False, - recreate=False, - recursive_triggers=True, - tracer=None, - use_counts_table=False, + memory: bool = False, + recreate: bool = False, + recursive_triggers: bool = True, + tracer: Callable = None, + use_counts_table: bool = False, ): assert (filename_or_conn is not None and not memory) or ( filename_or_conn is None and memory @@ -203,11 +290,24 @@ class Database: self._tracer = tracer if recursive_triggers: self.execute("PRAGMA recursive_triggers=on;") - self._registered_functions = set() + self._registered_functions: set = set() self.use_counts_table = use_counts_table @contextlib.contextmanager - def tracer(self, tracer=None): + def tracer(self, tracer: Callable = None): + """ + Context manager to temporarily set a tracer function - all executed SQL queries will + be passed to this. + + The tracer function should accept two arguments: ``sql`` and ``parameters`` + + Example usage:: + + with db.tracer(print): + db["creatures"].insert({"name": "Cleo"}) + + See :ref:`python_api_tracing`. + """ prev_tracer = self._tracer self._tracer = tracer or print try: @@ -215,13 +315,39 @@ class Database: finally: self._tracer = prev_tracer - def __getitem__(self, table_name): + def __getitem__(self, table_name: str) -> Union["Table", "View"]: + """ + ``db[table_name]`` returns a :class:`.Table` object for the table with the specified name. + If the table does not exist yet it will be created the first time data is inserted into it. + """ return self.table(table_name) def __repr__(self): return "".format(self.conn) - def register_function(self, fn=None, deterministic=None, replace=False): + def register_function( + self, fn: Callable = None, deterministic: bool = False, replace: bool = False + ): + """ + ``fn`` will be made available as a function within SQL, with the same name and number + of arguments. Can be used as a decorator:: + + @db.register + def upper(value): + return str(value).upper() + + The decorator can take arguments:: + + @db.register(deterministic=True, replace=True) + def upper(value): + return str(value).upper() + + - ``deterministic`` - set ``True`` for functions that always returns the same output for a given input + - ``replace`` - set ``True`` to replace an existing function with the same name - otherwise throw an error + + See :ref:`python_api_register_function`. + """ + def register(fn): name = fn.__name__ arity = len(inspect.signature(fn).parameters) @@ -240,9 +366,15 @@ class Database: register(fn) def register_fts4_bm25(self): + "Register the ``rank_bm25(match_info)`` function used for calculating relevance with SQLite FTS4." self.register_function(rank_bm25, deterministic=True) - def attach(self, alias, filepath): + def attach(self, alias: str, filepath: Union[str, pathlib.Path]): + """ + Attach another SQLite database file to this connection with the specified alias, equivalent to:: + + ATTACH DATABASE 'filepath.db' AS alias + """ attach_sql = """ ATTACH DATABASE '{}' AS [{}]; """.format( @@ -250,7 +382,19 @@ class Database: ).strip() self.execute(attach_sql) - def execute(self, sql, parameters=None): + def query( + self, sql: str, params: Optional[Union[Iterable, dict]] = None + ) -> Generator[dict, None, None]: + "Execute ``sql`` and return an iterable of dictionaries representing each row." + cursor = self.execute(sql, params or tuple()) + keys = [d[0] for d in cursor.description] + for row in cursor: + yield dict(zip(keys, row)) + + def execute( + self, sql: str, parameters: Optional[Union[Iterable, dict]] = None + ) -> sqlite3.Cursor: + "Execute SQL query and return a ``sqlite3.Cursor``." if self._tracer: self._tracer(sql, parameters) if parameters is not None: @@ -258,16 +402,19 @@ class Database: else: return self.conn.execute(sql) - def executescript(self, sql): + def executescript(self, sql: str) -> sqlite3.Cursor: + "Execute multiple SQL statements separated by ; and return the ``sqlite3.Cursor``." if self._tracer: self._tracer(sql, None) return self.conn.executescript(sql) - def table(self, table_name, **kwargs): + def table(self, table_name: str, **kwargs) -> Union["Table", "View"]: + "Return a table object, optionally configured with default options." klass = View if table_name in self.view_names() else Table return klass(self, table_name, **kwargs) - def quote(self, value): + def quote(self, value: str) -> str: + "Apply SQLite string quoting to a value, including wrappping it in single quotes." # Normally we would use .execute(sql, [params]) for escaping, but # occasionally that isn't available - most notable when we need # to include a "... DEFAULT 'value'" in a column definition. @@ -277,7 +424,8 @@ class Database: {"value": value}, ).fetchone()[0] - def table_names(self, fts4=False, fts5=False): + def table_names(self, fts4: bool = False, fts5: bool = False) -> List[str]: + "A list of string table names in this database." where = ["type = 'table'"] if fts4: where.append("sql like '%USING FTS4%'") @@ -286,7 +434,8 @@ class Database: sql = "select name from sqlite_master where {}".format(" AND ".join(where)) return [r[0] for r in self.execute(sql).fetchall()] - def view_names(self): + def view_names(self) -> List[str]: + "A list of string view names in this database." return [ r[0] for r in self.execute( @@ -295,15 +444,18 @@ class Database: ] @property - def tables(self): - return [self[name] for name in self.table_names()] + def tables(self) -> List["Table"]: + "A list of Table objects in this database." + return cast(List["Table"], [self[name] for name in self.table_names()]) @property - def views(self): - return [self[name] for name in self.view_names()] + def views(self) -> List["View"]: + "A list of View objects in this database." + return cast(List["View"], [self[name] for name in self.view_names()]) @property - def triggers(self): + def triggers(self) -> List[Trigger]: + "A list of ``(name, table_name, sql)`` tuples representing triggers in this database." return [ Trigger(*r) for r in self.execute( @@ -312,12 +464,13 @@ class Database: ] @property - def triggers_dict(self): - "Returns {trigger_name: sql} dictionary" + def triggers_dict(self) -> Dict[str, str]: + "A ``{trigger_name: sql}`` dictionary of triggers in this database." return {trigger.name: trigger.sql for trigger in self.triggers} @property - def schema(self): + def schema(self) -> str: + "SQL schema for this database" sqls = [] for row in self.execute( "select sql from sqlite_master where sql is not null" @@ -329,14 +482,17 @@ class Database: return "\n".join(sqls) @property - def journal_mode(self): + def journal_mode(self) -> str: + "Current ``journal_mode`` of this database." return self.execute("PRAGMA journal_mode;").fetchone()[0] def enable_wal(self): + "Set ``journal_mode`` to ``'wal'`` to enable Write-Ahead Log mode." if self.journal_mode != "wal": self.execute("PRAGMA journal_mode=wal;") def disable_wal(self): + "Set ``journal_mode`` back to ``'delete'`` to disable Write-Ahead Log mode." if self.journal_mode != "delete": self.execute("PRAGMA journal_mode=delete;") @@ -345,6 +501,10 @@ class Database: self.execute(_COUNTS_TABLE_CREATE_SQL.format(self._counts_table_name)) def enable_counts(self): + """ + Enable trigger-based count caching for every table in the database, see + :ref:`python_api_cached_table_counts`. + """ self._ensure_counts_table() for table in self.tables: if ( @@ -354,7 +514,11 @@ class Database: table.enable_counts() self.use_counts_table = True - def cached_counts(self, tables=None): + def cached_counts(self, tables: Optional[Iterable[str]] = None) -> Dict[str, int]: + """ + Return ``{table_name: count}`` dictionary of cached counts for specified tables, or + all tables if ``tables`` not provided. + """ sql = "select [table], count from {}".format(self._counts_table_name) if tables: sql += " where [table] in ({})".format(", ".join("?" for table in tables)) @@ -364,6 +528,7 @@ class Database: return {} def reset_counts(self): + "Re-calculate cached counts for tables." tables = [table for table in self.tables if table.has_counts_triggers] with self.conn: self._ensure_counts_table() @@ -374,14 +539,6 @@ class Database: for table in tables ) - def query( - self, sql: str, params: Optional[Union[Iterable, dict]] = None - ) -> Generator[dict, None, None]: - cursor = self.execute(sql, params or tuple()) - keys = [d[0] for d in cursor.description] - for row in cursor: - yield dict(zip(keys, row)) - def execute_returning_dicts( self, sql: str, params: Optional[Union[Iterable, dict]] = None ) -> List[dict]: @@ -430,16 +587,17 @@ class Database: def create_table_sql( self, - name, - columns, - pk=None, + name: str, + columns: Dict[str, Any], + pk: Optional[Any] = None, foreign_keys=None, column_order=None, not_null=None, defaults=None, hash_id=None, extracts=None, - ): + ) -> str: + "Returns the SQL ``CREATE TABLE`` statement for creating the specified table." foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) foreign_keys_by_column = {fk.column: fk for fk in foreign_keys} # any extracts will be treated as integer columns with a foreign key @@ -536,16 +694,21 @@ class Database: def create_table( self, - name, - columns, - pk=None, + name: str, + columns: Dict[str, Any], + pk: Optional[Any] = None, foreign_keys=None, column_order=None, not_null=None, defaults=None, hash_id=None, extracts=None, - ): + ) -> "Table": + """ + Create a table with the specified name and the specified ``{column_name: type}`` columns. + + See :ref:`python_api_explicit_create`. + """ sql = self.create_table_sql( name=name, columns=columns, @@ -558,7 +721,7 @@ class Database: extracts=extracts, ) self.execute(sql) - return self.table( + table = self.table( name, pk=pk, foreign_keys=foreign_keys, @@ -567,8 +730,17 @@ class Database: defaults=defaults, hash_id=hash_id, ) + return cast(Table, table) - def create_view(self, name, sql, ignore=False, replace=False): + def create_view( + self, name: str, sql: str, ignore: bool = False, replace: bool = False + ): + """ + Create a new SQL view with the specified name - ``sql`` should start with ``SELECT ...``. + + - ``ignore`` - set to ``True`` to do nothing if a view with this name already exists + - ``replace`` - set to ``True`` to do replace the view if one with this name already exists + """ assert not ( ignore and replace ), "Use one or the other of ignore/replace, not both" @@ -586,18 +758,28 @@ class Database: self.execute(create_sql) return self - def m2m_table_candidates(self, table, other_table): - "Returns potential m2m tables for arguments, based on FKs" + def m2m_table_candidates(self, table: str, other_table: str) -> List[str]: + """ + Given two table names returns the name of tables that could define a + many-to-many relationship between those two tables, based on having + foreign keys to both of the provided tables. + """ candidates = [] tables = {table, other_table} - for table in self.tables: + for table_obj in self.tables: # Does it have foreign keys to both table and other_table? - has_fks_to = {fk.other_table for fk in table.foreign_keys} + has_fks_to = {fk.other_table for fk in table_obj.foreign_keys} if has_fks_to.issuperset(tables): - candidates.append(table.name) + candidates.append(table_obj.name) return candidates - def add_foreign_keys(self, foreign_keys): + def add_foreign_keys(self, foreign_keys: Iterable[Tuple[str, str, str, str]]): + """ + See :ref:`python_api_add_foreign_keys`. + + ``foreign_keys`` should be a list of ``(table, column, other_table, other_column)`` + tuples, see :ref:`python_api_add_foreign_keys`. + """ # foreign_keys is a list of explicit 4-tuples assert all( len(fk) == 4 and isinstance(fk, (list, tuple)) for fk in foreign_keys @@ -609,7 +791,11 @@ class Database: for table, column, other_table, other_column in foreign_keys: if not self[table].exists(): raise AlterError("No such table: {}".format(table)) - if column not in self[table].columns_dict: + table_obj = self[table] + if not isinstance(table_obj, Table): + raise AlterError("Must be a table, not a view: {}".format(table)) + table_obj = cast(Table, table_obj) + if column not in table_obj.columns_dict: raise AlterError("No such column: {} in {}".format(column, table)) if not self[other_table].exists(): raise AlterError("No such other_table: {}".format(other_table)) @@ -623,7 +809,7 @@ class Database: # We will silently skip foreign keys that exist already if not any( fk - for fk in self[table].foreign_keys + for fk in table_obj.foreign_keys if fk.column == column and fk.other_table == other_table and fk.other_column == other_column @@ -633,7 +819,7 @@ class Database: ) # Construct SQL for use with "UPDATE sqlite_master SET sql = ? WHERE name = ?" - table_sql = {} + table_sql: Dict[str, str] = {} for table, column, other_table, other_column in foreign_keys_to_create: old_sql = table_sql.get(table, self[table].schema) extra_sql = ",\n FOREIGN KEY([{column}]) REFERENCES [{other_table}]([{other_column}])\n".format( @@ -661,6 +847,7 @@ class Database: self.vacuum() def index_foreign_keys(self): + "Create indexes for every foreign key column on every table in the database." for table_name in self.table_names(): table = self[table_name] existing_indexes = { @@ -671,11 +858,13 @@ class Database: table.create_index([fk.column]) def vacuum(self): + "Run a SQLite ``VACUUM`` against the database." self.execute("VACUUM;") class Queryable: - def exists(self): + def exists(self) -> bool: + "Does this table or view exist yet?" return False def __init__(self, db, name): @@ -684,9 +873,10 @@ class Queryable: def count_where( self, - where=None, - where_args=None, - ): + where: str = None, + where_args: Optional[Union[Iterable, dict]] = None, + ) -> int: + "Executes ``SELECT count(*) FROM table WHERE ...`` and returns a count." sql = "select count(*) from [{}]".format(self.name) if where is not None: sql += " where " + where @@ -697,24 +887,38 @@ class Queryable: return self.count_where() @property - def count(self): + def count(self) -> int: + "A count of the rows in this table or view." return self.count_where() @property - def rows(self): + def rows(self) -> Generator[dict, None, None]: + "Iterate over every dictionaries for each row in this table or view." return self.rows_where() def rows_where( self, - where=None, - where_args=None, - order_by=None, - select="*", - limit=None, - offset=None, - ): + where: str = None, + where_args: Optional[Union[Iterable, dict]] = None, + order_by: str = None, + select: str = "*", + limit: int = None, + offset: int = None, + ) -> Generator[dict, None, None]: + """ + Iterate over every row in this table or view that matches the specified where clause. + + - ``where`` - a SQL fragment to use as a ``WHERE`` clause, for example ``age > ?`` or ``age > :age``. + - ``where_args`` - a list of arguments (if using ``?``) or a dictionary (if using ``:age``). + - ``order_by`` - optional column or fragment of SQL to order by. + - ``select`` - optional comma-separated list of columns to select. + - ``limit`` - optional integer number of rows to limit to. + - ``offset`` - optional integer for SQL offset. + + Returns each row as a dictionary. See :ref:`python_api_rows` for more details. + """ if not self.exists(): - return [] + return sql = "select {} from [{}]".format(select, self.name) if where is not None: sql += " where " + where @@ -731,13 +935,13 @@ class Queryable: def pks_and_rows_where( self, - where=None, - where_args=None, - order_by=None, - limit=None, - offset=None, - ): - "Like .rows_where() but returns (pk, row) pairs - pk can be a single value or tuple" + where: str = None, + where_args: Optional[Union[Iterable, dict]] = None, + order_by: str = None, + limit: int = None, + offset: int = None, + ) -> Generator[Tuple[Any, Dict], None, None]: + "Like ``.rows_where()`` but returns ``(pk, row)`` pairs - ``pk`` can be a single value or tuple." column_names = [column.name for column in self.columns] pks = [column.name for column in self.columns if column.is_pk] if not pks: @@ -758,32 +962,37 @@ class Queryable: yield row_pk, row @property - def columns(self): + def columns(self) -> List["Column"]: + "List of :ref:`Columns ` representing the columns in this table or view." if not self.exists(): return [] rows = self.db.execute("PRAGMA table_info([{}])".format(self.name)).fetchall() return [Column(*row) for row in rows] @property - def columns_dict(self): - "Returns {column: python-type} dictionary" + def columns_dict(self) -> Dict[str, Any]: + "``{column_name: python-type}`` dictionary representing columns in this table or view." return {column.name: column_affinity(column.type) for column in self.columns} @property - def schema(self): + def schema(self) -> str: + "SQL schema for this table or view." return self.db.execute( "select sql from sqlite_master where name = ?", (self.name,) ).fetchone()[0] class Table(Queryable): - last_rowid = None - last_pk = None + "Tables should usually be initialized using the ``db.table(table_name)`` or ``db[table_name]`` methods." + #: The ``rowid`` of the last inserted, updated or selected row.` + last_rowid: Optional[int] = None + #: The primary key of the last inserted, updated or selected row.` + last_pk: Optional[Any] = None def __init__( self, - db, - name, + db: Database, + name: str, pk=None, foreign_keys=None, column_order=None, @@ -815,7 +1024,7 @@ class Table(Queryable): columns=columns, ) - def __repr__(self): + def __repr__(self) -> str: return "
".format( self.name, " (does not exist yet)" @@ -824,7 +1033,8 @@ class Table(Queryable): ) @property - def count(self): + def count(self) -> int: + "Count of the rows in this table - optionally from the table count cache, if configured." if self.db.use_counts_table: counts = self.db.cached_counts([self.name]) if counts: @@ -835,17 +1045,26 @@ class Table(Queryable): return self.name in self.db.table_names() @property - def pks(self): + def pks(self) -> List[str]: + "Primary key columns for this table." names = [column.name for column in self.columns if column.is_pk] if not names: names = ["rowid"] return names @property - def use_rowid(self): + def use_rowid(self) -> bool: + "Does this table use ``rowid`` for its primary key (no other primary keys are specified)?" return not any(column for column in self.columns if column.is_pk) - def get(self, pk_values): + def get(self, pk_values: Union[list, tuple, str, int]) -> dict: + """ + Return row (as dictionary) for the specified primary key. + + Primary key can be a single value, or a tuple for tables with a compound primary key. + + Raises ``NotFoundError`` if a matching row cannot be found. + """ if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] pks = self.pks @@ -867,7 +1086,8 @@ class Table(Queryable): raise NotFoundError @property - def foreign_keys(self): + def foreign_keys(self) -> List["ForeignKey"]: + "List of foreign keys defined on this table." fks = [] for row in self.db.execute( "PRAGMA foreign_key_list([{}])".format(self.name) @@ -885,15 +1105,16 @@ class Table(Queryable): return fks @property - def virtual_table_using(self): - "Returns type of virtual table or None if this is not a virtual table" + def virtual_table_using(self) -> Optional[str]: + "Type of virtual table, or ``None`` if this is not a virtual table." match = _virtual_table_using_re.match(self.schema) if match is None: return None return match.groupdict()["using"].upper() @property - def indexes(self): + def indexes(self) -> List[Index]: + "List of indexes defined on this table." sql = 'PRAGMA index_list("{}")'.format(self.name) indexes = [] for row in self.db.execute_returning_dicts(sql): @@ -916,7 +1137,8 @@ class Table(Queryable): return indexes @property - def xindexes(self): + def xindexes(self) -> List[XIndex]: + "List of indexes defined on this table using the more detailed ``XIndex`` format." sql = 'PRAGMA index_list("{}")'.format(self.name) indexes = [] for row in self.db.execute_returning_dicts(sql): @@ -934,7 +1156,8 @@ class Table(Queryable): return indexes @property - def triggers(self): + def triggers(self) -> List[Trigger]: + "List of triggers defined on this table." return [ Trigger(*r) for r in self.db.execute( @@ -945,8 +1168,8 @@ class Table(Queryable): ] @property - def triggers_dict(self): - "Returns {trigger_name: sql} dictionary" + def triggers_dict(self) -> Dict[str, str]: + "``{trigger_name: sql}`` dictionary of triggers defined on this table." return {trigger.name: trigger.sql for trigger in self.triggers} def create( @@ -959,7 +1182,12 @@ class Table(Queryable): defaults=None, hash_id=None, extracts=None, - ): + ) -> "Table": + """ + Create a table with the specified columns. + + See :ref:`python_api_explicit_create` for full details. + """ columns = {name: value for (name, value) in columns.items()} with self.db.conn: self.db.create_table( @@ -986,7 +1214,13 @@ class Table(Queryable): defaults=None, drop_foreign_keys=None, column_order=None, - ): + ) -> "Table": + """ + Apply an advanced alter table, including operations that are not supported by + ``ALTER TABLE`` in SQLite itself. + + See :ref:`python_api_transform` for full details. + """ assert self.exists(), "Cannot transform a table that doesn't exist yet" sqls = self.transform_sql( types=types, @@ -1027,7 +1261,8 @@ class Table(Queryable): drop_foreign_keys=None, column_order=None, tmp_suffix=None, - ): + ) -> List[str]: + "Returns a list of SQL statements that would be executed in order to apply this transformation." types = types or {} rename = rename or {} drop = drop or set() @@ -1133,7 +1368,18 @@ class Table(Queryable): ) return sqls - def extract(self, columns, table=None, fk_column=None, rename=None): + def extract( + self, + columns: Union[str, Iterable[str]], + table: Optional[str] = None, + fk_column: Optional[str] = None, + rename: Optional[Dict[str, str]] = None, + ) -> "Table": + """ + Extract specified columns into a separate table. + + See :ref:`python_api_extract` for details. + """ rename = rename or {} if isinstance(columns, str): columns = [columns] @@ -1225,7 +1471,24 @@ class Table(Queryable): self.add_foreign_key(fk_column, table, "id") return self - def create_index(self, columns, index_name=None, unique=False, if_not_exists=False): + def create_index( + self, + columns: Iterable[Union[str, DescIndex]], + index_name: Optional[str] = None, + unique: bool = False, + if_not_exists: bool = False, + ): + """ + Create an index on this table. + + - ``columns`` - a single columns or list of columns to index. These can be strings or, + to create an index using the column in descending order, ``db.DescIndex(column_name)`` objects. + - ``index_name`` - the name to use for the new index. Defaults to the column names joined on ``_``. + - ``unique`` - should the index be marked as unique, forcing unique values? + - ``if_not_exists`` - only create the index if one with that name does not already exist. + + See :ref:`python_api_create_index`. + """ if index_name is None: index_name = "idx_{}_{}".format( self.name.replace(" ", "_"), "_".join(columns) @@ -1257,8 +1520,9 @@ class Table(Queryable): return self def add_column( - self, col_name, col_type=None, fk=None, fk_col=None, not_null_default=None + self, col_name: str, col_type=None, fk=None, fk_col=None, not_null_default=None ): + "Add a column to this table. See :ref:`python_api_add_column`." fk_col_type = None if fk is not None: # fk must be a valid table @@ -1293,14 +1557,24 @@ class Table(Queryable): self.add_foreign_key(col_name, fk, fk_col) return self - def drop(self, ignore=False): + def drop(self, ignore: bool = False): + "Drop this table. ``ignore=True`` means errors will be ignored." try: self.db.execute("DROP TABLE [{}]".format(self.name)) except sqlite3.OperationalError: if not ignore: raise - def guess_foreign_table(self, column): + def guess_foreign_table(self, column: str) -> str: + """ + For a given column, suggest another table that might be referenced by this + column should it be used as a foreign key. + + For example, a column called ``tag_id`` or ``tag`` or ``tags`` might suggest + a ``tag`` table, if one exists. + + If no candidates can be found, raises a ``NoObviousTable`` exception. + """ column = column.lower() possibilities = [column] if column.endswith("_id"): @@ -1321,7 +1595,7 @@ class Table(Queryable): ) ) - def guess_foreign_column(self, other_table): + def guess_foreign_column(self, other_table: str): pks = [c for c in self.db[other_table].columns if c.is_pk] if len(pks) != 1: raise BadPrimaryKey( @@ -1331,8 +1605,20 @@ class Table(Queryable): return pks[0].name def add_foreign_key( - self, column, other_table=None, other_column=None, ignore=False + self, + column: str, + other_table: Optional[str] = None, + other_column: Optional[str] = None, + ignore: bool = False, ): + """ + Alter the schema to mark the specified column as a foreign key to another table. + + - ``column`` - the column to mark as a foreign key. + - ``other_table`` - the table it refers to - if omitted, will be guessed based on the column name. + - ``other_column`` - the column on the other table it - if omitted, will be guessed. + - ``ignore`` - set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError` will be raised. + """ # Ensure column exists if column not in self.columns_dict: raise AlterError("No such column: {}".format(column)) @@ -1369,6 +1655,11 @@ class Table(Queryable): return self def enable_counts(self): + """ + Set up triggers to update a cache of the count of rows in this table. + + See :ref:`python_api_cached_table_counts` for details. + """ sql = ( textwrap.dedent( """ @@ -1413,7 +1704,8 @@ class Table(Queryable): self.db.use_counts_table = True @property - def has_counts_triggers(self): + def has_counts_triggers(self) -> bool: + "Does this table have triggers setup to update cached counts?" trigger_names = { "{table}{counts_table}_{suffix}".format( counts_table=self.db._counts_table_name, table=self.name, suffix=suffix @@ -1424,13 +1716,23 @@ class Table(Queryable): def enable_fts( self, - columns, - fts_version="FTS5", - create_triggers=False, - tokenize=None, - replace=False, + columns: Iterable[str], + fts_version: str = "FTS5", + create_triggers: bool = False, + tokenize: Optional[str] = None, + replace: bool = False, ): - "Enables FTS on the specified columns." + """ + Enable SQLite full-text search against the specified columns. + + - ``columns`` - list of column names to include in the search index. + - ``fts_version`` - FTS version to use - defaults to ``FTS5`` but you may want ``FTS4`` for older SQLite versions. + - ``create_triggers`` - should triggers be created to keep the search index up-to-date? Defaults to ``False``. + - ``tokenize`` - custom SQLite tokenizer to use, for example ``"porter"`` to enable Porter stemming. + - ``replace`` - should any existing FTS index for this table be replaced by the new one? + + See :ref:`python_api_fts` for more details. + """ create_fts_sql = ( textwrap.dedent( """ @@ -1498,7 +1800,11 @@ class Table(Queryable): self.db.executescript(triggers) return self - def populate_fts(self, columns): + def populate_fts(self, columns: Iterable[str]) -> "Table": + """ + Update the associated SQLite full-text search index with the latest data from the + table for the specified columns. + """ sql = ( textwrap.dedent( """ @@ -1514,7 +1820,8 @@ class Table(Queryable): self.db.executescript(sql) return self - def disable_fts(self): + def disable_fts(self) -> "Table": + "Remove any full-text search index and related triggers configured for this table." fts_table = self.detect_fts() if fts_table: self.db[fts_table].drop() @@ -1539,6 +1846,7 @@ class Table(Queryable): return self def rebuild_fts(self): + "Run the ``rebuild`` operation against the associated full-text search index table." fts_table = self.detect_fts() if fts_table is None: # Assume this is itself an FTS table @@ -1550,7 +1858,7 @@ class Table(Queryable): ) return self - def detect_fts(self): + def detect_fts(self) -> Optional[str]: "Detect if table has a corresponding FTS virtual table and return it" sql = ( textwrap.dedent( @@ -1575,7 +1883,8 @@ class Table(Queryable): else: return rows[0][0] - def optimize(self): + def optimize(self) -> "Table": + "Run the ``optimize`` operation against the associated full-text search index table." fts_table = self.detect_fts() if fts_table is not None: self.db.execute( @@ -1587,7 +1896,8 @@ class Table(Queryable): ) return self - def search_sql(self, columns=None, order_by=None, limit=None, offset=None): + def search_sql(self, columns=None, order_by=None, limit=None, offset=None) -> str: + "Return SQL string that can be used to execute searches against this table." # Pick names for table and rank column that don't clash original = "original_" if self.name == "original" else "original" columns_sql = "*" @@ -1644,7 +1954,26 @@ class Table(Queryable): limit_offset=limit_offset.strip(), ).strip() - def search(self, q, order_by=None, columns=None, limit=None, offset=None): + def search( + self, + q: str, + order_by: Optional[str] = None, + columns: Optional[List[str]] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Generator[dict, None, None]: + """ + Execute a search against this table using SQLite full-text search, returning a sequence of + dictionaries for each row. + + - ``q`` - words to search for + - ``order_by`` - defaults to order by rank, or specify a column here. + - ``columns`` - list of columns to return, defaults to all columns. + - ``limit`` - optional integer limit for returned rows. + - ``offset`` - optional integer SQL offset. + + See :ref:`python_api_fts_search`. + """ cursor = self.db.execute( self.search_sql( order_by=order_by, @@ -1661,7 +1990,8 @@ class Table(Queryable): def value_or_default(self, key, value): return self._defaults[key] if value is DEFAULT else value - def delete(self, pk_values): + def delete(self, pk_values: Union[list, tuple, str, int, float]) -> "Table": + "Delete row matching the specified primary key." if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] self.get(pk_values) @@ -1673,16 +2003,37 @@ class Table(Queryable): self.db.execute(sql, pk_values) return self - def delete_where(self, where=None, where_args=None): + def delete_where( + self, where: str = None, where_args: Optional[Union[Iterable, dict]] = None + ) -> "Table": + "Delete rows matching specified where clause, or delete all rows in the table." if not self.exists(): - return [] + return self sql = "delete from [{}]".format(self.name) if where is not None: sql += " where " + where self.db.execute(sql, where_args or []) return self - def update(self, pk_values, updates=None, alter=False, conversions=None): + def update( + self, + pk_values: Union[list, tuple, str, int, float], + updates: Optional[dict] = None, + alter: bool = False, + conversions: Optional[dict] = None, + ) -> "Table": + """ + Execute a SQL ``UPDATE`` against the specified row. + + - ``pk_values`` - the primary key of an individual record - can be a tuple if the + table has a compound primary key. + - ``updates`` - a dictionary mapping columns to their updated values. + - ``alter``` - set to ``True`` to add any missing columns. + - ``conversions`` - optional dictionary of SQL functions to apply during the update, for example + ``{"mycolumn": "upper(?)"}``. + + See :ref:`python_api_update`. + """ updates = updates or {} conversions = conversions or {} if not isinstance(pk_values, (list, tuple)): @@ -1722,16 +2073,34 @@ class Table(Queryable): def convert( self, - columns, - fn, - output=None, - output_type=None, - drop=False, - multi=False, - where=None, - where_args=None, - show_progress=False, + columns: Union[str, List[str]], + fn: Callable, + output: Optional[str] = None, + output_type: Optional[Any] = None, + drop: bool = False, + multi: bool = False, + where: Optional[str] = None, + where_args: Optional[Union[Iterable, dict]] = None, + show_progress: bool = False, ): + """ + Apply conversion function ``fn`` to every value in the specified columns. + + - ``columns`` - a single column or list of string column names to convert. + - ``fn`` - a callable that takes a single argument, ``value``, and returns it converted. + - ``output`` - optional string column name to write the results to (defaults to the input column). + - ``output_type`` - if the output column needs to be created, this is the type that will be used + for the new column. + - ``drop`` - boolean, should the original column be dropped once the conversion is complete? + - ``multi`` - boolean, if ``True`` the return value of ``fn(value)`` will be expected to be a + dictionary, and new columns will be created for each key of that dictionary. + - ``where`` - a SQL fragment to use as a ``WHERE`` clause to limit the rows to which the conversion + is applied, for example ``age > ?`` or ``age > :age``. + - ``where_args`` - a list of arguments (if using ``?``) or a dictionary (if using ``:age``). + - ``show_progress`` - boolean, should a progress bar be displayed? + + See :ref:`python_api_convert`. + """ if isinstance(columns, str): columns = [columns] @@ -2012,20 +2381,51 @@ class Table(Queryable): def insert( self, - record, + record: Dict[str, Any], pk=DEFAULT, foreign_keys=DEFAULT, - column_order=DEFAULT, - not_null=DEFAULT, - defaults=DEFAULT, - hash_id=DEFAULT, - alter=DEFAULT, - ignore=DEFAULT, - replace=DEFAULT, - extracts=DEFAULT, - conversions=DEFAULT, - columns=DEFAULT, - ): + column_order: Optional[Union[List[str], Default]] = DEFAULT, + not_null: Optional[Union[Set[str], Default]] = DEFAULT, + defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT, + hash_id: Optional[Union[str, Default]] = DEFAULT, + alter: Optional[Union[bool, Default]] = DEFAULT, + ignore: Optional[Union[bool, Default]] = DEFAULT, + replace: Optional[Union[bool, Default]] = DEFAULT, + extracts: Optional[Union[Dict[str, str], List[str], Default]] = DEFAULT, + conversions: Optional[Union[Dict[str, str], Default]] = DEFAULT, + columns: Optional[Union[Dict[str, Any], Default]] = DEFAULT, + ) -> "Table": + """ + Insert a single record into the table. The table will be created with a schema that matches + the inserted record if it does not already exist, see :ref:`python_api_creating_tables`. + + - ``record`` - required: a dictionary representing the record to be inserted. + + The other parameters are optional, and mostly influence how the new table will be created if + that table does not exist yet. + + Each of them defaults to ``DEFAULT``, which indicates that the default setting for the current + ``Table`` object (specified in the table constructor) should be used. + + - ``pk`` - if creating the table, which column should be the primary key. + - ``foreign_keys`` - see :ref:`python_api_foreign_keys`. + - ``column_order`` - optional list of strings specifying a full or partial column order + to use when creating the table. + - ``not_null`` - optional set of strings specifying columns that should be ``NOT NULL``. + - ``defaults`` - optional dictionary specifying default values for specific columns. + - ``hash_id`` - optional name of a column to create and use as a primary key, where the + value of thet primary key will be derived as a SHA1 hash of the other column values + in the record. ``hash_id="id"`` is a common column name used for this. + - ``alter`` - boolean, should any missing columns be added automatically? + - ``ignore`` - boolean, if a record already exists with this primary key, ignore this insert. + - ``replace`` - boolean, if a record already exists with this primary key, replace it with this new record. + - ``extracts`` - a list of columns to extract to other tables, or a dictionary that maps + ``{column_name: other_table_name}``. See :ref:`python_api_extracts`. + - ``conversions`` - dictionary specifying SQL conversion functions to be applied to the data while it + is being inserted, for example ``{"name": "upper(?)"}``. See :ref:`python_api_conversions`. + - ``columns`` - dictionary over-riding the detected types used for the columns, for example + ``{"age": int, "weight": float}``. + """ return self.insert_all( [record], pk=pk, @@ -2060,11 +2460,10 @@ class Table(Queryable): conversions=DEFAULT, columns=DEFAULT, upsert=False, - ): + ) -> "Table": """ - Like .insert() but takes a list of records and ensures that the table - that it creates (if table does not exist) has columns for ALL of that - data + Like ``.insert()`` but takes a list of records and ensures that the table + that it creates (if table does not exist) has columns for ALL of that data. """ pk = self.value_or_default("pk", pk) foreign_keys = self.value_or_default("foreign_keys", foreign_keys) @@ -2089,7 +2488,7 @@ class Table(Queryable): assert not ( ignore and replace ), "Use either ignore=True or replace=True, not both" - all_columns = None + all_columns = [] first = True num_records_processed = 0 # We can only handle a max of 999 variables in a SQL insert, so @@ -2127,10 +2526,10 @@ class Table(Queryable): hash_id=hash_id, extracts=extracts, ) - all_columns = set() + all_columns_set = set() for record in chunk: - all_columns.update(record.keys()) - all_columns = list(sorted(all_columns)) + all_columns_set.update(record.keys()) + all_columns = list(sorted(all_columns_set)) if hash_id: all_columns.insert(0, hash_id) else: @@ -2171,7 +2570,13 @@ class Table(Queryable): extracts=DEFAULT, conversions=DEFAULT, columns=DEFAULT, - ): + ) -> "Table": + """ + Like ``.insert()`` but performs an ``UPSERT``, where records are inserted if they do + not exist and updated if they DO exist, based on matching against their primary key. + + See :ref:`python_api_upsert`. + """ return self.upsert_all( [record], pk=pk, @@ -2200,7 +2605,10 @@ class Table(Queryable): extracts=DEFAULT, conversions=DEFAULT, columns=DEFAULT, - ): + ) -> "Table": + """ + Like ``.upsert()`` but can be applied to a list of records. + """ return self.insert_all( records, pk=pk, @@ -2217,7 +2625,7 @@ class Table(Queryable): upsert=True, ) - def add_missing_columns(self, records): + def add_missing_columns(self, records: Iterable[Dict[str, Any]]) -> "Table": needed_columns = suggest_column_types(records) current_columns = {c.lower() for c in self.columns_dict} for col_name, col_type in needed_columns.items(): @@ -2225,7 +2633,20 @@ class Table(Queryable): self.add_column(col_name, col_type) return self - def lookup(self, column_values): + def lookup(self, column_values: Dict[str, Any]): + """ + Create or populate a lookup table with the specified values. + + ``db["Species"].lookup({"name": "Palm"})`` will create a table called ``Species`` + (if one does not already exist) with two columns: ``id`` and ``name``. It will + set up a unique constraint on the ``name`` column to guarantee it will not + contain duplicate rows. + + It well then inserts a new row with the ``name`` set to ``Palm`` and return the + new integer primary key value. + + See :ref:`python_api_lookup_tables` for more details. + """ # lookups is a dictionary - all columns will be used for a unique index assert isinstance(column_values, dict) if self.exists(): @@ -2250,15 +2671,38 @@ class Table(Queryable): def m2m( self, - other_table, - record_or_iterable=None, - pk=DEFAULT, - lookup=None, - m2m_table=None, - alter=False, + other_table: Union[str, "Table"], + record_or_iterable: Optional[ + Union[Iterable[Dict[str, Any]], Dict[str, Any]] + ] = None, + pk: Optional[Union[Any, Default]] = DEFAULT, + lookup: Optional[Dict[str, Any]] = None, + m2m_table: Optional[str] = None, + alter: bool = False, ): + """ + After inserting a record in a table, create one or more records in some other + table and then create many-to-many records linking the original record and the + newly created records together. + + For example:: + + db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id").m2m( + "humans", {"id": 1, "name": "Natalie"}, pk="id" + ) + See :ref:`python_api_m2m` for details. + + - ``other_table`` - the name of the table to insert the new records into. + - ``record_or_iterable`` - a single dictionary record to insert, or a list of records. + - ``pk`` - the primary key to use if creating ``other_table``. + - ``lookup`` - same dictionary as for ``.lookup()``, to create a many-to-many lookup table. + - ``m2m_table`` - the string name to use for the many-to-many table, defaults to creating + this automatically based on the names of the two tables. + - ``alter``` - set to ``True`` to add any missing columns on ``other_table`` if that table + already exists. + """ if isinstance(other_table, str): - other_table = self.db.table(other_table, pk=pk) + other_table = cast(Table, self.db.table(other_table, pk=pk)) our_id = self.last_pk if lookup is not None: assert record_or_iterable is None, "Provide lookup= or record, not both" @@ -2282,20 +2726,19 @@ class Table(Queryable): else: # If not, create a new table m2m_table_name = m2m_table or "{}_{}".format(*tables) - m2m_table = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) + m2m_table_obj = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) if lookup is None: # if records is only one record, put the record in a list - records = ( - [record_or_iterable] - if isinstance(record_or_iterable, Mapping) - else record_or_iterable - ) + if isinstance(record_or_iterable, Mapping): + records = [record_or_iterable] + else: + records = cast(List, record_or_iterable) # Ensure each record exists in other table for record in records: id = other_table.insert( - record, pk=pk, replace=True, alter=alter + cast(dict, record), pk=pk, replace=True, alter=alter ).last_pk - m2m_table.insert( + m2m_table_obj.insert( { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, @@ -2304,7 +2747,7 @@ class Table(Queryable): ) else: id = other_table.lookup(lookup) - m2m_table.insert( + m2m_table_obj.insert( { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, @@ -2314,8 +2757,13 @@ class Table(Queryable): return self def analyze_column( - self, column, common_limit=10, value_truncate=None, total_rows=None - ): + self, column: str, common_limit: int = 10, value_truncate=None, total_rows=None + ) -> "ColumnDetails": + """ + Return statistics about the specified column. + + See :ref:`python_api_analyze_column`. + """ db = self.db table = self.name if total_rows is None: From 86fc9fb5c8073af8e20acc6af25974b89ec4720a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 10 Aug 2021 16:51:59 -0700 Subject: [PATCH 094/102] Release 3.15.1 Refs #311, #312, #313, #315 --- docs/changelog.rst | 10 ++++++++++ docs/reference.rst | 2 ++ setup.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9a1669d..da6e746 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,16 @@ Changelog =========== +.. _v3_15.1: + +3.15.1 (2021-08-10) +------------------- + +- Python library now includes type annotations on almost all of the methods, plus detailed docstrings describing each one. (:issue:`311`) +- New :ref:`reference` documentation page, powered by those docstrings. +- Fixed bug where ``.add_foreign_keys()`` failed to raise an error if called against a ``View``. (:issue:`313`) +- Fixed bug where ``.delete_where()`` returned a ``[]`` instead of returning ``self`` if called against a non-existant table. (:issue:`315`) + .. _v3_15: 3.15 (2021-08-09) diff --git a/docs/reference.rst b/docs/reference.rst index e29f468..8331b1e 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -1,3 +1,5 @@ +.. _reference: + =============== API Reference =============== diff --git a/setup.py b/setup.py index c8c58ac..65fe6f7 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.15" +VERSION = "3.15.1" def get_long_description(): From bde372525734bd41d94251675141422b0fd56bda Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 10 Aug 2021 16:55:12 -0700 Subject: [PATCH 095/102] Fixed spelling existent --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index da6e746..ab424ed 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ - Python library now includes type annotations on almost all of the methods, plus detailed docstrings describing each one. (:issue:`311`) - New :ref:`reference` documentation page, powered by those docstrings. - Fixed bug where ``.add_foreign_keys()`` failed to raise an error if called against a ``View``. (:issue:`313`) -- Fixed bug where ``.delete_where()`` returned a ``[]`` instead of returning ``self`` if called against a non-existant table. (:issue:`315`) +- Fixed bug where ``.delete_where()`` returned a ``[]`` instead of returning ``self`` if called against a non-existent table. (:issue:`315`) .. _v3_15: From 3091bec4f7bab85c94fe2879a36c96474e152230 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 11 Aug 2021 04:54:00 -0700 Subject: [PATCH 096/102] Don't show inheritance for Database class --- docs/reference.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/reference.rst b/docs/reference.rst index 8331b1e..ea1203e 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -14,7 +14,6 @@ sqlite_utils.db.Database .. autoclass:: sqlite_utils.db.Database :members: :undoc-members: - :show-inheritance: :special-members: __getitem__ :exclude-members: use_counts_table, execute_returning_dicts, resolve_foreign_keys From af89c5f8513ad6c4228e5f8c8b6c9b5c98c12f63 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 11 Aug 2021 04:56:54 -0700 Subject: [PATCH 097/102] How to create a Database instance --- sqlite_utils/db.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c7dc832..7ba4ecb 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -251,6 +251,13 @@ class Database: """ Wrapper for a SQLite database connection that adds a variety of useful utility methods. + To create an instance:: + + # create data.db file, or open existing: + db = Database("data.db") + # Create an in-memory database: + dB = Database(memory=True) + - ``filename_or_conn`` - String path to a file, or a ``pathlib.Path`` object, or a ``sqlite3`` connection - ``memory`` - set to ``True`` to create an in-memory database From 6de0a5d46a00a66d827c32deaca5cbd0ad2103ad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 11 Aug 2021 05:03:07 -0700 Subject: [PATCH 098/102] Typo fix --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 7ba4ecb..55067d0 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -746,7 +746,7 @@ class Database: Create a new SQL view with the specified name - ``sql`` should start with ``SELECT ...``. - ``ignore`` - set to ``True`` to do nothing if a view with this name already exists - - ``replace`` - set to ``True`` to do replace the view if one with this name already exists + - ``replace`` - set to ``True`` to replace the view if one with this name already exists """ assert not ( ignore and replace From b966c44ef81bc6acbc4be95942afcf33b31e876f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 13 Aug 2021 04:32:40 -0700 Subject: [PATCH 099/102] Minor markup fix --- sqlite_utils/db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 55067d0..ec399fa 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -991,9 +991,9 @@ class Queryable: class Table(Queryable): "Tables should usually be initialized using the ``db.table(table_name)`` or ``db[table_name]`` methods." - #: The ``rowid`` of the last inserted, updated or selected row.` + #: The ``rowid`` of the last inserted, updated or selected row. last_rowid: Optional[int] = None - #: The primary key of the last inserted, updated or selected row.` + #: The primary key of the last inserted, updated or selected row. last_pk: Optional[Any] = None def __init__( From 7ee7b628e101863c73c2a95911bd2213de00fb1f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 13 Aug 2021 22:10:47 -0700 Subject: [PATCH 100/102] Fixed some rogue backticks, closes #316 --- sqlite_utils/db.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ec399fa..520008f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -84,7 +84,7 @@ Describes a SQLite column returned by the :attr:`.Table.columns` property. Column type ``notnull`` - Does the column have a ``not null` constraint + Does the column have a ``not null`` constraint ``default_value`` Default value for this column @@ -128,10 +128,10 @@ Summary information about a column, see :ref:`python_api_analyze_column`. The number of distinct values in this column ``most_common`` - The ``N`` most common values as a list of ``(value, count)`` tuples`, or ``None`` if the table consists entirely of distinct values + The ``N`` most common values as a list of ``(value, count)`` tuples, or ``None`` if the table consists entirely of distinct values ``least_common`` - The ``N`` least common values as a list of ``(value, count)`` tuples`, or ``None`` if the table is entirely distinct + The ``N`` least common values as a list of ``(value, count)`` tuples, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) """ ForeignKey = namedtuple( @@ -1624,7 +1624,7 @@ class Table(Queryable): - ``column`` - the column to mark as a foreign key. - ``other_table`` - the table it refers to - if omitted, will be guessed based on the column name. - ``other_column`` - the column on the other table it - if omitted, will be guessed. - - ``ignore`` - set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError` will be raised. + - ``ignore`` - set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. """ # Ensure column exists if column not in self.columns_dict: @@ -2035,7 +2035,7 @@ class Table(Queryable): - ``pk_values`` - the primary key of an individual record - can be a tuple if the table has a compound primary key. - ``updates`` - a dictionary mapping columns to their updated values. - - ``alter``` - set to ``True`` to add any missing columns. + - ``alter`` - set to ``True`` to add any missing columns. - ``conversions`` - optional dictionary of SQL functions to apply during the update, for example ``{"mycolumn": "upper(?)"}``. @@ -2705,7 +2705,7 @@ class Table(Queryable): - ``lookup`` - same dictionary as for ``.lookup()``, to create a many-to-many lookup table. - ``m2m_table`` - the string name to use for the many-to-many table, defaults to creating this automatically based on the names of the two tables. - - ``alter``` - set to ``True`` to add any missing columns on ``other_table`` if that table + - ``alter`` - set to ``True`` to add any missing columns on ``other_table`` if that table already exists. """ if isinstance(other_table, str): From 7a19822ac9ee24be2fbb4c2326a0bf2f3d2d9c4d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 17 Aug 2021 08:42:02 -0700 Subject: [PATCH 101/102] Updated tagline --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 581f306..91e7431 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,7 +13,7 @@ .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg :target: https://github.com/simonw/sqlite-utils/blob/main/LICENSE -*Python utility functions for manipulating SQLite databases* +*CLI tool and Python utility functions for manipulating SQLite databases* This library and command-line utility helps create SQLite databases from an existing collection of data. From 1fe73c898b44695052f1a9ca832818d50cecf662 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Aug 2021 11:31:20 -0700 Subject: [PATCH 102/102] Remove link to older code example --- docs/index.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 91e7431..0629e0e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,5 +35,3 @@ Contents reference contributing changelog - -Take a look at `this script `_ for an example of this library in action.