From 69a121e08847acbf95abf0c2df1759fc73dc81b8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 23:20:11 -0800 Subject: [PATCH] sqlite-utils analyze-tables command and table.analyze_column() method Closes #207 --- docs/cli.rst | 87 ++++++++++++++++++ docs/python-api.rst | 35 ++++++++ sqlite_utils/cli.py | 88 ++++++++++++++++++ sqlite_utils/db.py | 79 +++++++++++++++++ tests/test_analyze_tables.py | 167 +++++++++++++++++++++++++++++++++++ 5 files changed, 456 insertions(+) create mode 100644 tests/test_analyze_tables.py diff --git a/docs/cli.rst b/docs/cli.rst index 4e9710d..68ca429 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -279,6 +279,93 @@ It takes the same options as the ``tables`` command: * ``--tsv`` * ``--table`` +.. _cli_analyze_tables: + +Analyzing tables +================ + +When working with a new database it can be useful to get an idea of the shape of the data. The ``sqlite-utils analyze-tables`` command inspects specified tables (or all tables) and calculates some useful details about each of the columns in those tables. + +To inspect the ``tags`` table in the ``github.db`` database, run the following:: + + $ sqlite-utils analyze-tables github.db tags + tags.repo: (1/3) + + Total rows: 261 + Null rows: 0 + Blank rows: 0 + + Distinct values: 14 + + Most common: + 88: 107914493 + 75: 140912432 + 27: 206156866 + + Least common: + 1: 209590345 + 2: 206649770 + 2: 303218369 + + tags.name: (2/3) + + Total rows: 261 + Null rows: 0 + Blank rows: 0 + + Distinct values: 175 + + Most common: + 10: 0.2 + 9: 0.1 + 7: 0.3 + + Least common: + 1: 0.1.1 + 1: 0.11.1 + 1: 0.1a2 + + tags.sha: (3/3) + + Total rows: 261 + Null rows: 0 + Blank rows: 0 + + Distinct values: 261 + +For each column this tool dispalys the number of null rows, the number of blank rows (rows that contain an empty string), the number of distinct values and, for columns that are not entirely distinct, the most common and least common values. + +If you do not specify any tables every table in the database will be analyzed:: + + $ sqlite-utils analyze-tables github.db + +If you wish to analyze one or more specific columns, use the ``-c`` option:: + + $ sqlite-utils analyze-tables github.db tags -c sha + +.. _cli_analyze_tables_save: + +Saving the analyzed table details +--------------------------------- + +``analyze-tables`` can take quite a while to run for large database files. You can save the results of the analysis to a database table called ``_analyze_tables_`` using the ``--save`` option:: + + $ sqlite-utils analyze-tables github.db --save + +The ``_analyze_tables_`` table has the following schema:: + + CREATE TABLE [_analyze_tables_] ( + [table] TEXT, + [column] TEXT, + [total_rows] INTEGER, + [num_null] INTEGER, + [num_blank] INTEGER, + [num_distinct] INTEGER, + [most_common] TEXT, + [least_common] TEXT, + PRIMARY KEY ([table], [column]) + ); + .. _cli_inserting_data: Inserting JSON data diff --git a/docs/python-api.rst b/docs/python-api.rst index 8dead7d..130c4cb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -747,6 +747,41 @@ You can inspect the database to see the results like this:: PRIMARY KEY ([characteristics_id], [dogs_id]) ) +.. _python_api_analyze_column: + +Analyzing a column +================== + +The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` method is used by the :ref:`analyze-tables ` CLI command. It returns a ``ColumnDetails`` named tuple with the following fields: + +``table`` + The name of the table + +``column`` + The name of the column + +``total_rows`` + The total number of rows in the table` + +``num_null`` + The number of rows for which this column is null + +``num_blank`` + The number of rows for which this column is blank (the empty string) + +``num_distinct`` + The number of distinct values in this column + +``most_common`` + The ``N`` most common values as a list of ``(value, count)`` tuples`, or ``None`` if the table consists entirely of distinct values + +``least_common`` + The ``N`` least common values as a list of ``(value, count)`` tuples`, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) + +``N`` defaults to 10, or you can pass a custom ``N`` using the ``common_limit`` parameter. + +You can use the ``value_truncate`` parameter to truncate values in the ``most_common`` and ``least_common`` lists to a specified number of characters. + .. _python_api_add_column: Adding columns diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cfc57c6..6312327 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -7,6 +7,7 @@ import hashlib import pathlib import sqlite_utils from sqlite_utils.db import AlterError +import textwrap import itertools import json import os @@ -1354,6 +1355,93 @@ def insert_files( ) +@cli.command(name="analyze-tables") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False, exists=True), + required=True, +) +@click.argument("tables", nargs=-1) +@click.option( + "-c", + "--column", + "columns", + type=str, + multiple=True, + help="Specific columns to analyze", +) +@click.option("--save", is_flag=True, help="Save results to _analyze_tables table") +@load_extension_option +def analyze_tables( + path, + tables, + columns, + save, + load_extension, +): + "Analyze the columns in one or more tables" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + if not tables: + tables = db.table_names() + todo = [] + table_counts = {} + for table in tables: + table_counts[table] = db[table].count + for column in db[table].columns: + if not columns or column.name in columns: + todo.append((table, column.name)) + # Now we now how many we need to do + for i, (table, column) in enumerate(todo): + column_details = db[table].analyze_column( + column, total_rows=table_counts[table], value_truncate=80 + ) + if save: + db["_analyze_tables_"].insert( + column_details._asdict(), pk=("table", "column"), replace=True + ) + most_common_rendered = _render_common( + "\n\n Most common:", column_details.most_common + ) + least_common_rendered = _render_common( + "\n\n Least common:", column_details.least_common + ) + details = ( + ( + textwrap.dedent( + """ + {table}.{column}: ({i}/{total}) + + Total rows: {total_rows} + Null rows: {num_null} + Blank rows: {num_blank} + + Distinct values: {num_distinct}{most_common_rendered}{least_common_rendered} + """ + ) + .strip() + .format( + i=i + 1, + total=len(todo), + most_common_rendered=most_common_rendered, + least_common_rendered=least_common_rendered, + **column_details._asdict() + ) + ) + + "\n" + ) + click.echo(details) + + +def _render_common(title, values): + if values is None: + return "" + lines = [title] + for value, count in values: + lines.append(" {}: {}".format(count, value)) + return "\n".join(lines) + + FILE_COLUMNS = { "name": lambda p: p.name, "path": lambda p: str(p), diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 8849680..90427bf 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -51,6 +51,19 @@ except ImportError: Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") ) +ColumnDetails = namedtuple( + "ColumnDetails", + ( + "table", + "column", + "total_rows", + "num_null", + "num_blank", + "num_distinct", + "most_common", + "least_common", + ), +) ForeignKey = namedtuple( "ForeignKey", ("table", "column", "other_table", "other_column") ) @@ -1930,6 +1943,72 @@ class Table(Queryable): ) return self + def analyze_column( + self, column, common_limit=10, value_truncate=None, total_rows=None + ): + db = self.db + table = self.name + if total_rows is None: + total_rows = db[table].count + + def truncate(value): + if value_truncate is None or isinstance(value, (float, int)): + return value + value = str(value) + if len(value) > value_truncate: + value = value[:value_truncate] + "..." + return value + + num_null = db.execute( + "select count(*) from [{}] where [{}] is null".format(table, column) + ).fetchone()[0] + num_blank = db.execute( + "select count(*) from [{}] where [{}] = ''".format(table, column) + ).fetchone()[0] + num_distinct = db.execute( + "select count(distinct [{}]) from [{}]".format(column, table) + ).fetchone()[0] + most_common = None + least_common = None + if num_distinct == 1: + value = db.execute( + "select [{}] from [{}] limit 1".format(column, table) + ).fetchone()[0] + most_common = [(truncate(value), total_rows)] + elif num_distinct != total_rows: + most_common = [ + (truncate(r[0]), r[1]) + for r in db.execute( + "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format( + column, table, column, column, common_limit + ) + ).fetchall() + ] + most_common.sort(key=lambda p: (p[1], p[0]), reverse=True) + if num_distinct <= common_limit: + # No need to run the query if it will just return the results in revers order + least_common = None + else: + least_common = [ + (truncate(r[0]), r[1]) + for r in db.execute( + "select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format( + column, table, column, column, common_limit + ) + ).fetchall() + ] + least_common.sort(key=lambda p: (p[1], p[0])) + return ColumnDetails( + self.name, + column, + total_rows, + num_null, + num_blank, + num_distinct, + most_common, + least_common, + ) + class View(Queryable): def exists(self): diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py new file mode 100644 index 0000000..f0af2ca --- /dev/null +++ b/tests/test_analyze_tables.py @@ -0,0 +1,167 @@ +from sqlite_utils.db import Database, ForeignKey, ColumnDetails +from sqlite_utils import cli +from sqlite_utils.utils import OperationalError +from click.testing import CliRunner +import pytest +import sqlite3 +import textwrap + + +@pytest.fixture +def db_to_analyze(fresh_db): + stuff = fresh_db["stuff"] + stuff.insert_all( + [ + {"id": 1, "owner": "Terryterryterry", "size": 5}, + {"id": 2, "owner": "Joan", "size": 4}, + {"id": 3, "owner": "Kumar", "size": 5}, + {"id": 4, "owner": "Anne", "size": 5}, + {"id": 5, "owner": "Terryterryterry", "size": 5}, + {"id": 6, "owner": "Joan", "size": 4}, + {"id": 7, "owner": "Kumar", "size": 5}, + {"id": 8, "owner": "Joan", "size": 4}, + ], + pk="id", + ) + return fresh_db + + +@pytest.mark.parametrize( + "column,expected", + [ + ( + "id", + ColumnDetails( + table="stuff", + column="id", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=8, + most_common=None, + least_common=None, + ), + ), + ( + "owner", + ColumnDetails( + table="stuff", + column="owner", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=4, + most_common=[("Joan", 3), ("Kumar", 2)], + least_common=[("Anne", 1), ("Terry...", 2)], + ), + ), + ( + "size", + ColumnDetails( + table="stuff", + column="size", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=2, + most_common=[(5, 5), (4, 3)], + least_common=None, + ), + ), + ], +) +def test_analyze_column(db_to_analyze, column, expected): + assert ( + db_to_analyze["stuff"].analyze_column(column, common_limit=2, value_truncate=5) + == expected + ) + + +@pytest.fixture +def db_to_analyze_path(db_to_analyze, tmpdir): + path = str(tmpdir / "test.db") + db = sqlite3.connect(path) + db.executescript("\n".join(db_to_analyze.conn.iterdump())) + return path + + +def test_analyze_table(db_to_analyze_path): + result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path]) + assert ( + result.output.strip() + == ( + """ +stuff.id: (1/3) + + Total rows: 8 + Null rows: 0 + Blank rows: 0 + + Distinct values: 8 + +stuff.owner: (2/3) + + Total rows: 8 + Null rows: 0 + Blank rows: 0 + + Distinct values: 4 + + Most common: + 3: Joan + 2: Terryterryterry + 2: Kumar + 1: Anne + +stuff.size: (3/3) + + Total rows: 8 + Null rows: 0 + Blank rows: 0 + + Distinct values: 2 + + Most common: + 5: 5 + 3: 4""" + ).strip() + ) + + +def test_analyze_table_save(db_to_analyze_path): + result = CliRunner().invoke( + cli.cli, ["analyze-tables", db_to_analyze_path, "--save"] + ) + rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows) + assert rows == [ + { + "table": "stuff", + "column": "id", + "total_rows": 8, + "num_null": 0, + "num_blank": 0, + "num_distinct": 8, + "most_common": None, + "least_common": None, + }, + { + "table": "stuff", + "column": "owner", + "total_rows": 8, + "num_null": 0, + "num_blank": 0, + "num_distinct": 4, + "most_common": '[["Joan", 3], ["Terryterryterry", 2], ["Kumar", 2], ["Anne", 1]]', + "least_common": None, + }, + { + "table": "stuff", + "column": "size", + "total_rows": 8, + "num_null": 0, + "num_blank": 0, + "num_distinct": 2, + "most_common": "[[5, 5], [4, 3]]", + "least_common": None, + }, + ]