From c5f4f0f70ce394dfec6054c3c5aaedf330887093 Mon Sep 17 00:00:00 2001 From: Andreas Madsack Date: Tue, 8 Dec 2020 18:49:42 +0100 Subject: [PATCH 1/5] Use jsonify_if_need for sql updates (#204) * add failing tests for update with json values * use jsonify_if_needed in for sql updates Thanks, @mfa --- sqlite_utils/db.py | 2 +- tests/test_update.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c91ef82..8849680 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1426,7 +1426,7 @@ class Table(Queryable): validate_column_names(updates.keys()) for key, value in updates.items(): sets.append("[{}] = {}".format(key, conversions.get(key, "?"))) - args.append(value) + args.append(jsonify_if_needed(value)) wheres = ["[{}] = ?".format(pk_name) for pk_name in pks] args.extend(pk_values) sql = "update [{table}] set {sets} where {wheres}".format( diff --git a/tests/test_update.py b/tests/test_update.py index 3f4c4ba..7e3e0ab 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -1,6 +1,10 @@ -from sqlite_utils.db import NotFoundError +import collections +import json + import pytest +from sqlite_utils.db import NotFoundError + def test_update_rowid_table(fresh_db): table = fresh_db["table"] @@ -82,3 +86,27 @@ def test_update_with_no_values_sets_last_pk(fresh_db): assert 2 == table.last_pk with pytest.raises(NotFoundError): table.update(3) + + +@pytest.mark.parametrize( + "data_structure", + ( + ["list with one item"], + ["list with", "two items"], + {"dictionary": "simple"}, + {"dictionary": {"nested": "complex"}}, + collections.OrderedDict( + [ + ("key1", {"nested": "complex"}), + ("key2", "foo"), + ] + ), + [{"list": "of"}, {"two": "dicts"}], + ), +) +def test_update_dictionaries_and_lists_as_json(fresh_db, data_structure): + fresh_db["test"].insert({"id": 1, "data": ""}, pk="id") + fresh_db["test"].update(1, {"data": data_structure}) + row = fresh_db.execute("select id, data from test").fetchone() + assert row[0] == 1 + assert data_structure == json.loads(row[1]) From 69a121e08847acbf95abf0c2df1759fc73dc81b8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 23:20:11 -0800 Subject: [PATCH 2/5] sqlite-utils analyze-tables command and table.analyze_column() method Closes #207 --- docs/cli.rst | 87 ++++++++++++++++++ docs/python-api.rst | 35 ++++++++ sqlite_utils/cli.py | 88 ++++++++++++++++++ sqlite_utils/db.py | 79 +++++++++++++++++ tests/test_analyze_tables.py | 167 +++++++++++++++++++++++++++++++++++ 5 files changed, 456 insertions(+) create mode 100644 tests/test_analyze_tables.py diff --git a/docs/cli.rst b/docs/cli.rst index 4e9710d..68ca429 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -279,6 +279,93 @@ It takes the same options as the ``tables`` command: * ``--tsv`` * ``--table`` +.. _cli_analyze_tables: + +Analyzing tables +================ + +When working with a new database it can be useful to get an idea of the shape of the data. The ``sqlite-utils analyze-tables`` command inspects specified tables (or all tables) and calculates some useful details about each of the columns in those tables. + +To inspect the ``tags`` table in the ``github.db`` database, run the following:: + + $ sqlite-utils analyze-tables github.db tags + tags.repo: (1/3) + + Total rows: 261 + Null rows: 0 + Blank rows: 0 + + Distinct values: 14 + + Most common: + 88: 107914493 + 75: 140912432 + 27: 206156866 + + Least common: + 1: 209590345 + 2: 206649770 + 2: 303218369 + + tags.name: (2/3) + + Total rows: 261 + Null rows: 0 + Blank rows: 0 + + Distinct values: 175 + + Most common: + 10: 0.2 + 9: 0.1 + 7: 0.3 + + Least common: + 1: 0.1.1 + 1: 0.11.1 + 1: 0.1a2 + + tags.sha: (3/3) + + Total rows: 261 + Null rows: 0 + Blank rows: 0 + + Distinct values: 261 + +For each column this tool dispalys the number of null rows, the number of blank rows (rows that contain an empty string), the number of distinct values and, for columns that are not entirely distinct, the most common and least common values. + +If you do not specify any tables every table in the database will be analyzed:: + + $ sqlite-utils analyze-tables github.db + +If you wish to analyze one or more specific columns, use the ``-c`` option:: + + $ sqlite-utils analyze-tables github.db tags -c sha + +.. _cli_analyze_tables_save: + +Saving the analyzed table details +--------------------------------- + +``analyze-tables`` can take quite a while to run for large database files. You can save the results of the analysis to a database table called ``_analyze_tables_`` using the ``--save`` option:: + + $ sqlite-utils analyze-tables github.db --save + +The ``_analyze_tables_`` table has the following schema:: + + CREATE TABLE [_analyze_tables_] ( + [table] TEXT, + [column] TEXT, + [total_rows] INTEGER, + [num_null] INTEGER, + [num_blank] INTEGER, + [num_distinct] INTEGER, + [most_common] TEXT, + [least_common] TEXT, + PRIMARY KEY ([table], [column]) + ); + .. _cli_inserting_data: Inserting JSON data diff --git a/docs/python-api.rst b/docs/python-api.rst index 8dead7d..130c4cb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -747,6 +747,41 @@ You can inspect the database to see the results like this:: PRIMARY KEY ([characteristics_id], [dogs_id]) ) +.. _python_api_analyze_column: + +Analyzing a column +================== + +The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` method is used by the :ref:`analyze-tables ` CLI command. It returns a ``ColumnDetails`` named tuple with the following fields: + +``table`` + The name of the table + +``column`` + The name of the column + +``total_rows`` + The total number of rows in the table` + +``num_null`` + The number of rows for which this column is null + +``num_blank`` + The number of rows for which this column is blank (the empty string) + +``num_distinct`` + The number of distinct values in this column + +``most_common`` + The ``N`` most common values as a list of ``(value, count)`` tuples`, or ``None`` if the table consists entirely of distinct values + +``least_common`` + The ``N`` least common values as a list of ``(value, count)`` tuples`, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) + +``N`` defaults to 10, or you can pass a custom ``N`` using the ``common_limit`` parameter. + +You can use the ``value_truncate`` parameter to truncate values in the ``most_common`` and ``least_common`` lists to a specified number of characters. + .. _python_api_add_column: Adding columns diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cfc57c6..6312327 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -7,6 +7,7 @@ import hashlib import pathlib import sqlite_utils from sqlite_utils.db import AlterError +import textwrap import itertools import json import os @@ -1354,6 +1355,93 @@ def insert_files( ) +@cli.command(name="analyze-tables") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False, exists=True), + required=True, +) +@click.argument("tables", nargs=-1) +@click.option( + "-c", + "--column", + "columns", + type=str, + multiple=True, + help="Specific columns to analyze", +) +@click.option("--save", is_flag=True, help="Save results to _analyze_tables table") +@load_extension_option +def analyze_tables( + path, + tables, + columns, + save, + load_extension, +): + "Analyze the columns in one or more tables" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + if not tables: + tables = db.table_names() + todo = [] + table_counts = {} + for table in tables: + table_counts[table] = db[table].count + for column in db[table].columns: + if not columns or column.name in columns: + todo.append((table, column.name)) + # Now we now how many we need to do + for i, (table, column) in enumerate(todo): + column_details = db[table].analyze_column( + column, total_rows=table_counts[table], value_truncate=80 + ) + if save: + db["_analyze_tables_"].insert( + column_details._asdict(), pk=("table", "column"), replace=True + ) + most_common_rendered = _render_common( + "\n\n Most common:", column_details.most_common + ) + least_common_rendered = _render_common( + "\n\n Least common:", column_details.least_common + ) + details = ( + ( + textwrap.dedent( + """ + {table}.{column}: ({i}/{total}) + + Total rows: {total_rows} + Null rows: {num_null} + Blank rows: {num_blank} + + Distinct values: {num_distinct}{most_common_rendered}{least_common_rendered} + """ + ) + .strip() + .format( + i=i + 1, + total=len(todo), + most_common_rendered=most_common_rendered, + least_common_rendered=least_common_rendered, + **column_details._asdict() + ) + ) + + "\n" + ) + click.echo(details) + + +def _render_common(title, values): + if values is None: + return "" + lines = [title] + for value, count in values: + lines.append(" {}: {}".format(count, value)) + return "\n".join(lines) + + FILE_COLUMNS = { "name": lambda p: p.name, "path": lambda p: str(p), diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 8849680..90427bf 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -51,6 +51,19 @@ except ImportError: Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") ) +ColumnDetails = namedtuple( + "ColumnDetails", + ( + "table", + "column", + "total_rows", + "num_null", + "num_blank", + "num_distinct", + "most_common", + "least_common", + ), +) ForeignKey = namedtuple( "ForeignKey", ("table", "column", "other_table", "other_column") ) @@ -1930,6 +1943,72 @@ class Table(Queryable): ) return self + def analyze_column( + self, column, common_limit=10, value_truncate=None, total_rows=None + ): + db = self.db + table = self.name + if total_rows is None: + total_rows = db[table].count + + def truncate(value): + if value_truncate is None or isinstance(value, (float, int)): + return value + value = str(value) + if len(value) > value_truncate: + value = value[:value_truncate] + "..." + return value + + num_null = db.execute( + "select count(*) from [{}] where [{}] is null".format(table, column) + ).fetchone()[0] + num_blank = db.execute( + "select count(*) from [{}] where [{}] = ''".format(table, column) + ).fetchone()[0] + num_distinct = db.execute( + "select count(distinct [{}]) from [{}]".format(column, table) + ).fetchone()[0] + most_common = None + least_common = None + if num_distinct == 1: + value = db.execute( + "select [{}] from [{}] limit 1".format(column, table) + ).fetchone()[0] + most_common = [(truncate(value), total_rows)] + elif num_distinct != total_rows: + most_common = [ + (truncate(r[0]), r[1]) + for r in db.execute( + "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format( + column, table, column, column, common_limit + ) + ).fetchall() + ] + most_common.sort(key=lambda p: (p[1], p[0]), reverse=True) + if num_distinct <= common_limit: + # No need to run the query if it will just return the results in revers order + least_common = None + else: + least_common = [ + (truncate(r[0]), r[1]) + for r in db.execute( + "select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format( + column, table, column, column, common_limit + ) + ).fetchall() + ] + least_common.sort(key=lambda p: (p[1], p[0])) + return ColumnDetails( + self.name, + column, + total_rows, + num_null, + num_blank, + num_distinct, + most_common, + least_common, + ) + class View(Queryable): def exists(self): diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py new file mode 100644 index 0000000..f0af2ca --- /dev/null +++ b/tests/test_analyze_tables.py @@ -0,0 +1,167 @@ +from sqlite_utils.db import Database, ForeignKey, ColumnDetails +from sqlite_utils import cli +from sqlite_utils.utils import OperationalError +from click.testing import CliRunner +import pytest +import sqlite3 +import textwrap + + +@pytest.fixture +def db_to_analyze(fresh_db): + stuff = fresh_db["stuff"] + stuff.insert_all( + [ + {"id": 1, "owner": "Terryterryterry", "size": 5}, + {"id": 2, "owner": "Joan", "size": 4}, + {"id": 3, "owner": "Kumar", "size": 5}, + {"id": 4, "owner": "Anne", "size": 5}, + {"id": 5, "owner": "Terryterryterry", "size": 5}, + {"id": 6, "owner": "Joan", "size": 4}, + {"id": 7, "owner": "Kumar", "size": 5}, + {"id": 8, "owner": "Joan", "size": 4}, + ], + pk="id", + ) + return fresh_db + + +@pytest.mark.parametrize( + "column,expected", + [ + ( + "id", + ColumnDetails( + table="stuff", + column="id", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=8, + most_common=None, + least_common=None, + ), + ), + ( + "owner", + ColumnDetails( + table="stuff", + column="owner", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=4, + most_common=[("Joan", 3), ("Kumar", 2)], + least_common=[("Anne", 1), ("Terry...", 2)], + ), + ), + ( + "size", + ColumnDetails( + table="stuff", + column="size", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=2, + most_common=[(5, 5), (4, 3)], + least_common=None, + ), + ), + ], +) +def test_analyze_column(db_to_analyze, column, expected): + assert ( + db_to_analyze["stuff"].analyze_column(column, common_limit=2, value_truncate=5) + == expected + ) + + +@pytest.fixture +def db_to_analyze_path(db_to_analyze, tmpdir): + path = str(tmpdir / "test.db") + db = sqlite3.connect(path) + db.executescript("\n".join(db_to_analyze.conn.iterdump())) + return path + + +def test_analyze_table(db_to_analyze_path): + result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path]) + assert ( + result.output.strip() + == ( + """ +stuff.id: (1/3) + + Total rows: 8 + Null rows: 0 + Blank rows: 0 + + Distinct values: 8 + +stuff.owner: (2/3) + + Total rows: 8 + Null rows: 0 + Blank rows: 0 + + Distinct values: 4 + + Most common: + 3: Joan + 2: Terryterryterry + 2: Kumar + 1: Anne + +stuff.size: (3/3) + + Total rows: 8 + Null rows: 0 + Blank rows: 0 + + Distinct values: 2 + + Most common: + 5: 5 + 3: 4""" + ).strip() + ) + + +def test_analyze_table_save(db_to_analyze_path): + result = CliRunner().invoke( + cli.cli, ["analyze-tables", db_to_analyze_path, "--save"] + ) + rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows) + assert rows == [ + { + "table": "stuff", + "column": "id", + "total_rows": 8, + "num_null": 0, + "num_blank": 0, + "num_distinct": 8, + "most_common": None, + "least_common": None, + }, + { + "table": "stuff", + "column": "owner", + "total_rows": 8, + "num_null": 0, + "num_blank": 0, + "num_distinct": 4, + "most_common": '[["Joan", 3], ["Terryterryterry", 2], ["Kumar", 2], ["Anne", 1]]', + "least_common": None, + }, + { + "table": "stuff", + "column": "size", + "total_rows": 8, + "num_null": 0, + "num_blank": 0, + "num_distinct": 2, + "most_common": "[[5, 5], [4, 3]]", + "least_common": None, + }, + ] From 1ce96d8c0854476e84216af5e5af71bcebbddb78 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 23:28:20 -0800 Subject: [PATCH 3/5] Release 3.1 Refs #204, #207, #208 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 76b6b5d..b0c3da1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v3_1: + +3.1 (2020-12-12) +---------------- + +- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (`#207 `__) +- New ``table.analyze_column(column)`` Python method used by the ``analyze-tables`` command - see :ref:`python_api_analyze_column`. +- The ``table.update()`` method now correctly handles values that should be stored as JSON. Thanks, Andreas Madsack. (`#204 `__) + .. _v3_0: 3.0 (2020-11-08) diff --git a/setup.py b/setup.py index 9d80bbd..a507a1b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.0" +VERSION = "3.1" def get_long_description(): From 5e06026e76cdda4ffdf89b7369b0e50be398d8fe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 13 Dec 2020 16:19:51 -0800 Subject: [PATCH 4/5] Typo fix --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 68ca429..bf67a11 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -333,7 +333,7 @@ To inspect the ``tags`` table in the ``github.db`` database, run the following:: Distinct values: 261 -For each column this tool dispalys the number of null rows, the number of blank rows (rows that contain an empty string), the number of distinct values and, for columns that are not entirely distinct, the most common and least common values. +For each column this tool displays the number of null rows, the number of blank rows (rows that contain an empty string), the number of distinct values and, for columns that are not entirely distinct, the most common and least common values. If you do not specify any tables every table in the database will be analyzed:: From f1277f638f3a54a821db6e03cb980adad2f2fa35 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 13 Dec 2020 20:52:24 -0800 Subject: [PATCH 5/5] Added Homebrew installation instructions --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 618edf6..85d9b4a 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ sqlite-utils: a Python library and CLI tool for building SQLite databases](https pip install sqlite-utils +Or if you use [Homebrew](https://brew.sh/) for macOS: + + brew install sqlite-utils + ## Using as a CLI tool Now you can do things with the CLI utility like this: