From ca2ccccac28bffbb0d4e345121919712cdef3ec6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Dec 2020 21:27:00 -0800 Subject: [PATCH 01/16] Initial prototype of sqlite-utils analyze-tables, refs #207 --- sqlite_utils/cli.py | 34 ++++++++++++++++++++++++ sqlite_utils/db.py | 63 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cfc57c6..e779bef 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1354,6 +1354,40 @@ def insert_files( ) +@cli.command(name="analyze-tables") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("tables", nargs=-1) +@click.option("--save", is_flag=True, help="Save results to _analyze_tables table") +@load_extension_option +def analyze_tables( + path, + tables, + save, + load_extension, +): + "Analyze the columns in one or more tables" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + if not tables: + tables = db.table_names() + todo = [] + for table in tables: + for column in db[table].columns: + todo.append((table, column.name)) + # Now we now how many we need to do + for i, (table, column) in enumerate(todo): + column_details = db[table].analyze_column(column) + if save: + db["_analyze_tables"].insert( + column_details._asdict(), pk=("table", "column"), replace=True + ) + click.echo("{}/{}: {}".format(i + 1, len(todo), column_details)) + + FILE_COLUMNS = { "name": lambda p: p.name, "path": lambda p: str(p), diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 8849680..fdb2e92 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -51,6 +51,18 @@ except ImportError: Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") ) +ColumnDetails = namedtuple( + "ColumnDetails", + ( + "table", + "column", + "num_null", + "num_blank", + "num_distinct", + "most_common", + "least_common", + ), +) ForeignKey = namedtuple( "ForeignKey", ("table", "column", "other_table", "other_column") ) @@ -1930,6 +1942,57 @@ class Table(Queryable): ) return self + def analyze_column(self, column, common_limit=10): + db = self.db + table = self.name + num_null = db.execute( + "select count(*) from [{}] where [{}] is null".format(table, column) + ).fetchone()[0] + num_blank = db.execute( + "select count(*) from [{}] where [{}] = ''".format(table, column) + ).fetchone()[0] + num_distinct = db.execute( + "select count(distinct [{}]) from [{}]".format(column, table) + ).fetchone()[0] + most_common = None + least_common = None + if num_distinct == 1: + value = db.execute( + "select [{}] from [{}] limit 1".format(column, table) + ).fetchone()[0] + most_common = [value] + least_common = [value] + else: + most_common = [ + (r[0], r[1]) + for r in db.execute( + "select [{}], count(*) from [{}] group by [{}] order by count(*) desc limit {}".format( + column, table, column, common_limit + ) + ).fetchall() + ] + if num_distinct <= common_limit: + # No need to run the query if it will just return the results in revers order + least_common = most_common[::-1] + else: + least_common = [ + (r[0], r[1]) + for r in db.execute( + "select [{}], count(*) from [{}] group by [{}] order by count(*) limit {}".format( + column, table, column, common_limit + ) + ).fetchall() + ] + return ColumnDetails( + self.name, + column, + num_null, + num_blank, + num_distinct, + most_common, + least_common, + ) + class View(Queryable): def exists(self): From 5c176ccbe06317a394f8abc6cae49a211115f2c8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Dec 2020 21:30:44 -0800 Subject: [PATCH 02/16] If only one distinct value, record that in most_commond and leave least_common as null --- sqlite_utils/db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index fdb2e92..460cb9a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1961,7 +1961,6 @@ class Table(Queryable): "select [{}] from [{}] limit 1".format(column, table) ).fetchone()[0] most_common = [value] - least_common = [value] else: most_common = [ (r[0], r[1]) From d4b8d9e7a5357b8a417a744c0d83792ab86010ce Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Dec 2020 21:41:07 -0800 Subject: [PATCH 03/16] Improvements to most/least common values * Record total_rows for each column * Record (value, count) if there is just a single distinct value * Do not calculate most/least common if all values are distinct * Calculate table count once per table, not once per column --- sqlite_utils/cli.py | 6 ++++-- sqlite_utils/db.py | 10 +++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index e779bef..6cd23f7 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1357,7 +1357,7 @@ def insert_files( @cli.command(name="analyze-tables") @click.argument( "path", - type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False, exists=True), required=True, ) @click.argument("tables", nargs=-1) @@ -1375,12 +1375,14 @@ def analyze_tables( if not tables: tables = db.table_names() todo = [] + table_counts = {} for table in tables: + table_counts[table] = db[table].count for column in db[table].columns: todo.append((table, column.name)) # Now we now how many we need to do for i, (table, column) in enumerate(todo): - column_details = db[table].analyze_column(column) + column_details = db[table].analyze_column(column, total_rows=table_counts[table]) if save: db["_analyze_tables"].insert( column_details._asdict(), pk=("table", "column"), replace=True diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 460cb9a..ae77f6a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -56,6 +56,7 @@ ColumnDetails = namedtuple( ( "table", "column", + "total_rows", "num_null", "num_blank", "num_distinct", @@ -1942,9 +1943,11 @@ class Table(Queryable): ) return self - def analyze_column(self, column, common_limit=10): + def analyze_column(self, column, common_limit=10, total_rows=None): db = self.db table = self.name + if total_rows is None: + total_rows = db[table].count num_null = db.execute( "select count(*) from [{}] where [{}] is null".format(table, column) ).fetchone()[0] @@ -1960,8 +1963,8 @@ class Table(Queryable): value = db.execute( "select [{}] from [{}] limit 1".format(column, table) ).fetchone()[0] - most_common = [value] - else: + most_common = [(value, total_rows)] + elif num_distinct != total_rows: most_common = [ (r[0], r[1]) for r in db.execute( @@ -1985,6 +1988,7 @@ class Table(Queryable): return ColumnDetails( self.name, column, + total_rows, num_null, num_blank, num_distinct, From 61461b98814ebf43e37c1213b7ac2083d11bfc63 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Dec 2020 21:45:37 -0800 Subject: [PATCH 04/16] Don't bother with least_common if less than 10 values --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ae77f6a..b7f6084 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1975,7 +1975,7 @@ class Table(Queryable): ] if num_distinct <= common_limit: # No need to run the query if it will just return the results in revers order - least_common = most_common[::-1] + least_common = None else: least_common = [ (r[0], r[1]) From ef219d6b4df6cfcbefd5fd989a9c16ec3fcdb99c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Dec 2020 21:48:13 -0800 Subject: [PATCH 05/16] -c / --column option --- sqlite_utils/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 6cd23f7..7e653e6 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1361,11 +1361,13 @@ def insert_files( required=True, ) @click.argument("tables", nargs=-1) +@click.option("-c", "--column", "columns", type=str, multiple=True, help="Specific columns to analyze") @click.option("--save", is_flag=True, help="Save results to _analyze_tables table") @load_extension_option def analyze_tables( path, tables, + columns, save, load_extension, ): @@ -1379,7 +1381,8 @@ def analyze_tables( for table in tables: table_counts[table] = db[table].count for column in db[table].columns: - todo.append((table, column.name)) + if (not columns or column.name in columns): + todo.append((table, column.name)) # Now we now how many we need to do for i, (table, column) in enumerate(todo): column_details = db[table].analyze_column(column, total_rows=table_counts[table]) From f7efe77ccb5f444dd4bc90a9df1ed95697823a9c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 20:46:13 -0800 Subject: [PATCH 06/16] Tests for analyze tables --- sqlite_utils/cli.py | 15 ++++-- tests/test_analyze_tables.py | 91 ++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 tests/test_analyze_tables.py diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 7e653e6..fe052d2 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1361,7 +1361,14 @@ def insert_files( required=True, ) @click.argument("tables", nargs=-1) -@click.option("-c", "--column", "columns", type=str, multiple=True, help="Specific columns to analyze") +@click.option( + "-c", + "--column", + "columns", + type=str, + multiple=True, + help="Specific columns to analyze", +) @click.option("--save", is_flag=True, help="Save results to _analyze_tables table") @load_extension_option def analyze_tables( @@ -1381,11 +1388,13 @@ def analyze_tables( for table in tables: table_counts[table] = db[table].count for column in db[table].columns: - if (not columns or column.name in columns): + if not columns or column.name in columns: todo.append((table, column.name)) # Now we now how many we need to do for i, (table, column) in enumerate(todo): - column_details = db[table].analyze_column(column, total_rows=table_counts[table]) + column_details = db[table].analyze_column( + column, total_rows=table_counts[table] + ) if save: db["_analyze_tables"].insert( column_details._asdict(), pk=("table", "column"), replace=True diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py new file mode 100644 index 0000000..8842ee7 --- /dev/null +++ b/tests/test_analyze_tables.py @@ -0,0 +1,91 @@ +from sqlite_utils.db import ForeignKey, ColumnDetails +from sqlite_utils import cli +from sqlite_utils.utils import OperationalError +from click.testing import CliRunner +import pytest +import sqlite3 +import textwrap + + +@pytest.fixture +def db_to_analyze(fresh_db): + stuff = fresh_db["stuff"] + stuff.insert_all( + [ + {"id": 1, "owner": "Terry", "size": 5}, + {"id": 2, "owner": "Joan", "size": 4}, + {"id": 3, "owner": "Kumar", "size": 5}, + {"id": 4, "owner": "Anne", "size": 5}, + {"id": 5, "owner": "Terry", "size": 5}, + {"id": 6, "owner": "Joan", "size": 4}, + {"id": 7, "owner": "Kumar", "size": 5}, + {"id": 8, "owner": "Joan", "size": 4}, + ], + pk="id", + ) + return fresh_db + + +@pytest.mark.parametrize( + "column,expected", + [ + ( + "id", + ColumnDetails( + table="stuff", + column="id", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=8, + most_common=None, + least_common=None, + ), + ), + ( + "owner", + ColumnDetails( + table="stuff", + column="owner", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=4, + most_common=[("Joan", 3), ("Terry", 2)], + least_common=[("Anne", 1), ("Kumar", 2)], + ), + ), + ( + "size", + ColumnDetails( + table="stuff", + column="size", + total_rows=8, + num_null=0, + num_blank=0, + num_distinct=2, + most_common=[(5, 5), (4, 3)], + least_common=None, + ), + ), + ], +) +def test_analyze_column(db_to_analyze, column, expected): + assert db_to_analyze["stuff"].analyze_column(column, common_limit=2) == expected + + +def test_analyze_table(db_to_analyze, tmpdir): + path = str(tmpdir / "test.db") + db = sqlite3.connect(path) + db.executescript("\n".join(db_to_analyze.conn.iterdump())) + result = CliRunner().invoke(cli.cli, ["analyze-tables", path]) + assert ( + result.output.strip() + == textwrap.dedent( + """ + 1/3: ColumnDetails(table='stuff', column='id', total_rows=8, num_null=0, num_blank=0, num_distinct=8, most_common=None, least_common=None) + 2/3: ColumnDetails(table='stuff', column='owner', total_rows=8, num_null=0, num_blank=0, num_distinct=4, most_common=[('Joan', 3), ('Terry', 2), ('Kumar', 2), ('Anne', 1)], least_common=None) + 3/3: ColumnDetails(table='stuff', column='size', total_rows=8, num_null=0, num_blank=0, num_distinct=2, most_common=[(5, 5), (4, 3)], least_common=None) + """ + ).strip() + ) From c0d21956bc88b1964e25524a50f3b17ecdbbb7da Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 20:51:40 -0800 Subject: [PATCH 07/16] Test for analyze-tables --save --- sqlite_utils/cli.py | 3 ++- tests/test_analyze_tables.py | 50 +++++++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index fe052d2..02111a9 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1396,9 +1396,10 @@ def analyze_tables( column, total_rows=table_counts[table] ) if save: - db["_analyze_tables"].insert( + db["_analyze_tables_"].insert( column_details._asdict(), pk=("table", "column"), replace=True ) + click.echo("{}/{}: {}".format(i + 1, len(todo), column_details)) diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index 8842ee7..946ddc7 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import ForeignKey, ColumnDetails +from sqlite_utils.db import Database, ForeignKey, ColumnDetails from sqlite_utils import cli from sqlite_utils.utils import OperationalError from click.testing import CliRunner @@ -74,11 +74,16 @@ def test_analyze_column(db_to_analyze, column, expected): assert db_to_analyze["stuff"].analyze_column(column, common_limit=2) == expected -def test_analyze_table(db_to_analyze, tmpdir): +@pytest.fixture +def db_to_analyze_path(db_to_analyze, tmpdir): path = str(tmpdir / "test.db") db = sqlite3.connect(path) db.executescript("\n".join(db_to_analyze.conn.iterdump())) - result = CliRunner().invoke(cli.cli, ["analyze-tables", path]) + return path + + +def test_analyze_table(db_to_analyze_path): + result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path]) assert ( result.output.strip() == textwrap.dedent( @@ -89,3 +94,42 @@ def test_analyze_table(db_to_analyze, tmpdir): """ ).strip() ) + + +def test_analyze_table_save(db_to_analyze_path): + result = CliRunner().invoke( + cli.cli, ["analyze-tables", db_to_analyze_path, "--save"] + ) + rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows) + assert rows == [ + { + "table": "stuff", + "column": "id", + "total_rows": 8, + "num_null": 0, + "num_blank": 0, + "num_distinct": 8, + "most_common": None, + "least_common": None, + }, + { + "table": "stuff", + "column": "owner", + "total_rows": 8, + "num_null": 0, + "num_blank": 0, + "num_distinct": 4, + "most_common": '[["Joan", 3], ["Terry", 2], ["Kumar", 2], ["Anne", 1]]', + "least_common": None, + }, + { + "table": "stuff", + "column": "size", + "total_rows": 8, + "num_null": 0, + "num_blank": 0, + "num_distinct": 2, + "most_common": "[[5, 5], [4, 3]]", + "least_common": None, + }, + ] From 955daaaf056030fe93721ad30ad8a7c44afe2a50 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 20:58:05 -0800 Subject: [PATCH 08/16] Truncate values longer than 100 characters --- sqlite_utils/db.py | 19 +++++++++++++++---- tests/test_analyze_tables.py | 15 +++++++++------ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b7f6084..1974860 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1943,11 +1943,22 @@ class Table(Queryable): ) return self - def analyze_column(self, column, common_limit=10, total_rows=None): + def analyze_column( + self, column, common_limit=10, value_truncate=100, total_rows=None + ): db = self.db table = self.name if total_rows is None: total_rows = db[table].count + + def truncate(value): + if value_truncate is None or isinstance(value, (float, int)): + return value + value = str(value) + if len(value) > value_truncate: + value = value[:value_truncate] + "..." + return value + num_null = db.execute( "select count(*) from [{}] where [{}] is null".format(table, column) ).fetchone()[0] @@ -1963,10 +1974,10 @@ class Table(Queryable): value = db.execute( "select [{}] from [{}] limit 1".format(column, table) ).fetchone()[0] - most_common = [(value, total_rows)] + most_common = [(truncate(value), total_rows)] elif num_distinct != total_rows: most_common = [ - (r[0], r[1]) + (truncate(r[0]), r[1]) for r in db.execute( "select [{}], count(*) from [{}] group by [{}] order by count(*) desc limit {}".format( column, table, column, common_limit @@ -1978,7 +1989,7 @@ class Table(Queryable): least_common = None else: least_common = [ - (r[0], r[1]) + (truncate(r[0]), r[1]) for r in db.execute( "select [{}], count(*) from [{}] group by [{}] order by count(*) limit {}".format( column, table, column, common_limit diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index 946ddc7..5aa8269 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -12,11 +12,11 @@ def db_to_analyze(fresh_db): stuff = fresh_db["stuff"] stuff.insert_all( [ - {"id": 1, "owner": "Terry", "size": 5}, + {"id": 1, "owner": "Terryterryterry", "size": 5}, {"id": 2, "owner": "Joan", "size": 4}, {"id": 3, "owner": "Kumar", "size": 5}, {"id": 4, "owner": "Anne", "size": 5}, - {"id": 5, "owner": "Terry", "size": 5}, + {"id": 5, "owner": "Terryterryterry", "size": 5}, {"id": 6, "owner": "Joan", "size": 4}, {"id": 7, "owner": "Kumar", "size": 5}, {"id": 8, "owner": "Joan", "size": 4}, @@ -51,7 +51,7 @@ def db_to_analyze(fresh_db): num_null=0, num_blank=0, num_distinct=4, - most_common=[("Joan", 3), ("Terry", 2)], + most_common=[("Joan", 3), ("Terry...", 2)], least_common=[("Anne", 1), ("Kumar", 2)], ), ), @@ -71,7 +71,10 @@ def db_to_analyze(fresh_db): ], ) def test_analyze_column(db_to_analyze, column, expected): - assert db_to_analyze["stuff"].analyze_column(column, common_limit=2) == expected + assert ( + db_to_analyze["stuff"].analyze_column(column, common_limit=2, value_truncate=5) + == expected + ) @pytest.fixture @@ -89,7 +92,7 @@ def test_analyze_table(db_to_analyze_path): == textwrap.dedent( """ 1/3: ColumnDetails(table='stuff', column='id', total_rows=8, num_null=0, num_blank=0, num_distinct=8, most_common=None, least_common=None) - 2/3: ColumnDetails(table='stuff', column='owner', total_rows=8, num_null=0, num_blank=0, num_distinct=4, most_common=[('Joan', 3), ('Terry', 2), ('Kumar', 2), ('Anne', 1)], least_common=None) + 2/3: ColumnDetails(table='stuff', column='owner', total_rows=8, num_null=0, num_blank=0, num_distinct=4, most_common=[('Joan', 3), ('Terryterryterry', 2), ('Kumar', 2), ('Anne', 1)], least_common=None) 3/3: ColumnDetails(table='stuff', column='size', total_rows=8, num_null=0, num_blank=0, num_distinct=2, most_common=[(5, 5), (4, 3)], least_common=None) """ ).strip() @@ -119,7 +122,7 @@ def test_analyze_table_save(db_to_analyze_path): "num_null": 0, "num_blank": 0, "num_distinct": 4, - "most_common": '[["Joan", 3], ["Terry", 2], ["Kumar", 2], ["Anne", 1]]', + "most_common": '[["Joan", 3], ["Terryterryterry", 2], ["Kumar", 2], ["Anne", 1]]', "least_common": None, }, { From 736dca6289fd087ba8d810456e4c7f9f49a584e5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 21:28:43 -0800 Subject: [PATCH 09/16] Much improved design for CLI output of analyze-tables --- sqlite_utils/cli.py | 41 +++++++++++++++++++++++++++++++++++- sqlite_utils/db.py | 2 +- tests/test_analyze_tables.py | 39 +++++++++++++++++++++++++++++----- 3 files changed, 75 insertions(+), 7 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 02111a9..81454dc 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -7,6 +7,7 @@ import hashlib import pathlib import sqlite_utils from sqlite_utils.db import AlterError +import textwrap import itertools import json import os @@ -1399,8 +1400,46 @@ def analyze_tables( db["_analyze_tables_"].insert( column_details._asdict(), pk=("table", "column"), replace=True ) + most_common_rendered = _render_common( + "\n\n Most common:", column_details.most_common + ) + least_common_rendered = _render_common( + "\n\n Least common:", column_details.least_common + ) + details = ( + ( + textwrap.dedent( + """ + {table}.{column}: ({i}/{total}) - click.echo("{}/{}: {}".format(i + 1, len(todo), column_details)) + Total rows: {total_rows} + Null rows: {num_null} + Blank rows: {num_blank} + + Distinct values: {num_distinct}{most_common_rendered}{least_common_rendered} + """ + ) + .strip() + .format( + i=i + 1, + total=len(todo), + most_common_rendered=most_common_rendered, + least_common_rendered=least_common_rendered, + **column_details._asdict() + ) + ) + + "\n" + ) + click.echo(details) + + +def _render_common(title, values): + if values is None: + return "" + lines = [title] + for value, count in values: + lines.append(" {}: {}".format(value, count)) + return "\n".join(lines) FILE_COLUMNS = { diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1974860..9f2653c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1944,7 +1944,7 @@ class Table(Queryable): return self def analyze_column( - self, column, common_limit=10, value_truncate=100, total_rows=None + self, column, common_limit=10, value_truncate=80, total_rows=None ): db = self.db table = self.name diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index 5aa8269..17ec966 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -89,12 +89,41 @@ def test_analyze_table(db_to_analyze_path): result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path]) assert ( result.output.strip() - == textwrap.dedent( + == ( """ - 1/3: ColumnDetails(table='stuff', column='id', total_rows=8, num_null=0, num_blank=0, num_distinct=8, most_common=None, least_common=None) - 2/3: ColumnDetails(table='stuff', column='owner', total_rows=8, num_null=0, num_blank=0, num_distinct=4, most_common=[('Joan', 3), ('Terryterryterry', 2), ('Kumar', 2), ('Anne', 1)], least_common=None) - 3/3: ColumnDetails(table='stuff', column='size', total_rows=8, num_null=0, num_blank=0, num_distinct=2, most_common=[(5, 5), (4, 3)], least_common=None) - """ +stuff.id: (1/3) + + Total rows: 8 + Null rows: 0 + Blank rows: 0 + + Distinct values: 8 + +stuff.owner: (2/3) + + Total rows: 8 + Null rows: 0 + Blank rows: 0 + + Distinct values: 4 + + Most common: + Joan: 3 + Terryterryterry: 2 + Kumar: 2 + Anne: 1 + +stuff.size: (3/3) + + Total rows: 8 + Null rows: 0 + Blank rows: 0 + + Distinct values: 2 + + Most common: + 5: 5 + 4: 3""" ).strip() ) From f65fe8ad5c8184477130cfe41ff515907b53f0ab Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 21:29:20 -0800 Subject: [PATCH 10/16] Documentation for analyze-tables --- docs/cli.rst | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 4e9710d..677f4b1 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -279,6 +279,93 @@ It takes the same options as the ``tables`` command: * ``--tsv`` * ``--table`` +.. _cli_analyze_tables: + +Analyzing tables +================ + +When working with a new database it can be useful to get an idea of the shape of the data. The ``sqlite-utils analyze-tables`` command inspects specified tables (or all tables) and calculates some useful details about each of the columns in those tables. + +To inspect the ``tags`` table in the ``github.db`` database, run the following:: + + $ sqlite-utils analyze-tables github.db tags + tags.repo: (1/3) + + Total rows: 261 + Null rows: 0 + Blank rows: 0 + + Distinct values: 14 + + Most common: + 107914493: 88 + 140912432: 75 + 206156866: 27 + + Least common: + 209590345: 1 + 206649770: 2 + 303218369: 2 + + tags.name: (2/3) + + Total rows: 261 + Null rows: 0 + Blank rows: 0 + + Distinct values: 175 + + Most common: + 0.2: 10 + 0.1: 9 + 0.3: 7 + + Least common: + 0.1.1: 1 + 0.11.1: 1 + 0.1a2: 1 + + tags.sha: (3/3) + + Total rows: 261 + Null rows: 0 + Blank rows: 0 + + Distinct values: 261 + +For each column this tool dispalys the number of null rows, the number of blank rows (rows that contain an empty string), the number of distinct values and, for columns that are not entirely distinct, the most common and least common values. + +If you do not specify any tables every table in the database will be analyzed:: + + $ sqlite-utils analyze-tables github.db + +If you wish to analyze one or more specific columns, use the ``-c`` option:: + + $ sqlite-utils analyze-tables github.db tags -c sha + +.. _cli_analyze_tables_save: + +Saving the analyzed table details +--------------------------------- + +``analyze-tables`` can take quite a while to run for large database files. You can save the results of the analysis to a database table called ``_analyze_tables_`` using the ``--save`` option:: + + $ sqlite-utils analyze-tables github.db --save + +The ``_analyze_tables_`` table has the following schema:: + + CREATE TABLE [_analyze_tables_] ( + [table] TEXT, + [column] TEXT, + [total_rows] INTEGER, + [num_null] INTEGER, + [num_blank] INTEGER, + [num_distinct] INTEGER, + [most_common] TEXT, + [least_common] TEXT, + PRIMARY KEY ([table], [column]) + ); + .. _cli_inserting_data: Inserting JSON data From 14914674441b48abf7698f7d6da20ae679166dab Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 21:39:53 -0800 Subject: [PATCH 11/16] Swap count and value in analyze output --- docs/cli.rst | 24 ++++++++++++------------ sqlite_utils/cli.py | 4 ++-- sqlite_utils/db.py | 2 +- tests/test_analyze_tables.py | 10 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 677f4b1..68ca429 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -298,14 +298,14 @@ To inspect the ``tags`` table in the ``github.db`` database, run the following:: Distinct values: 14 Most common: - 107914493: 88 - 140912432: 75 - 206156866: 27 + 88: 107914493 + 75: 140912432 + 27: 206156866 Least common: - 209590345: 1 - 206649770: 2 - 303218369: 2 + 1: 209590345 + 2: 206649770 + 2: 303218369 tags.name: (2/3) @@ -316,14 +316,14 @@ To inspect the ``tags`` table in the ``github.db`` database, run the following:: Distinct values: 175 Most common: - 0.2: 10 - 0.1: 9 - 0.3: 7 + 10: 0.2 + 9: 0.1 + 7: 0.3 Least common: - 0.1.1: 1 - 0.11.1: 1 - 0.1a2: 1 + 1: 0.1.1 + 1: 0.11.1 + 1: 0.1a2 tags.sha: (3/3) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 81454dc..6312327 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1394,7 +1394,7 @@ def analyze_tables( # Now we now how many we need to do for i, (table, column) in enumerate(todo): column_details = db[table].analyze_column( - column, total_rows=table_counts[table] + column, total_rows=table_counts[table], value_truncate=80 ) if save: db["_analyze_tables_"].insert( @@ -1438,7 +1438,7 @@ def _render_common(title, values): return "" lines = [title] for value, count in values: - lines.append(" {}: {}".format(value, count)) + lines.append(" {}: {}".format(count, value)) return "\n".join(lines) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9f2653c..d8bd221 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1944,7 +1944,7 @@ class Table(Queryable): return self def analyze_column( - self, column, common_limit=10, value_truncate=80, total_rows=None + self, column, common_limit=10, value_truncate=None, total_rows=None ): db = self.db table = self.name diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index 17ec966..7a104e7 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -108,10 +108,10 @@ stuff.owner: (2/3) Distinct values: 4 Most common: - Joan: 3 - Terryterryterry: 2 - Kumar: 2 - Anne: 1 + 3: Joan + 2: Terryterryterry + 2: Kumar + 1: Anne stuff.size: (3/3) @@ -123,7 +123,7 @@ stuff.size: (3/3) Most common: 5: 5 - 4: 3""" + 3: 4""" ).strip() ) From faedf951e41e937da952f0c17c0d912af85dfa2f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 21:40:18 -0800 Subject: [PATCH 12/16] table.analyze_column() documentation --- docs/python-api.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 8dead7d..130c4cb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -747,6 +747,41 @@ You can inspect the database to see the results like this:: PRIMARY KEY ([characteristics_id], [dogs_id]) ) +.. _python_api_analyze_column: + +Analyzing a column +================== + +The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` method is used by the :ref:`analyze-tables ` CLI command. It returns a ``ColumnDetails`` named tuple with the following fields: + +``table`` + The name of the table + +``column`` + The name of the column + +``total_rows`` + The total number of rows in the table` + +``num_null`` + The number of rows for which this column is null + +``num_blank`` + The number of rows for which this column is blank (the empty string) + +``num_distinct`` + The number of distinct values in this column + +``most_common`` + The ``N`` most common values as a list of ``(value, count)`` tuples`, or ``None`` if the table consists entirely of distinct values + +``least_common`` + The ``N`` least common values as a list of ``(value, count)`` tuples`, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) + +``N`` defaults to 10, or you can pass a custom ``N`` using the ``common_limit`` parameter. + +You can use the ``value_truncate`` parameter to truncate values in the ``most_common`` and ``least_common`` lists to a specified number of characters. + .. _python_api_add_column: Adding columns From 5a813bae6d6d72ae9a9d10dcdacce59656472a92 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 21:43:52 -0800 Subject: [PATCH 13/16] Ensure reliable sort order for tests --- sqlite_utils/db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d8bd221..c7a7ed4 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1984,6 +1984,7 @@ class Table(Queryable): ) ).fetchall() ] + most_common.sort(key=lambda p: (p[1], p[0]), reverse=True) if num_distinct <= common_limit: # No need to run the query if it will just return the results in revers order least_common = None @@ -1996,6 +1997,7 @@ class Table(Queryable): ) ).fetchall() ] + least_common.sort(key=lambda p: (p[1], p[0])) return ColumnDetails( self.name, column, From 95a966bb6216215d40d0016d673a28b04428db43 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 23:18:49 -0800 Subject: [PATCH 14/16] Make least_common/most_common order dependable for tests --- sqlite_utils/db.py | 8 ++++---- tests/test_analyze_tables.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c7a7ed4..90427bf 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1979,8 +1979,8 @@ class Table(Queryable): most_common = [ (truncate(r[0]), r[1]) for r in db.execute( - "select [{}], count(*) from [{}] group by [{}] order by count(*) desc limit {}".format( - column, table, column, common_limit + "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format( + column, table, column, column, common_limit ) ).fetchall() ] @@ -1992,8 +1992,8 @@ class Table(Queryable): least_common = [ (truncate(r[0]), r[1]) for r in db.execute( - "select [{}], count(*) from [{}] group by [{}] order by count(*) limit {}".format( - column, table, column, common_limit + "select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format( + column, table, column, column, common_limit ) ).fetchall() ] diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index 7a104e7..f0af2ca 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -51,8 +51,8 @@ def db_to_analyze(fresh_db): num_null=0, num_blank=0, num_distinct=4, - most_common=[("Joan", 3), ("Terry...", 2)], - least_common=[("Anne", 1), ("Kumar", 2)], + most_common=[("Joan", 3), ("Kumar", 2)], + least_common=[("Anne", 1), ("Terry...", 2)], ), ), ( From 6eea94a49dcf2c098a14f4218cf1b80e4e6626db Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 23:28:20 -0800 Subject: [PATCH 15/16] Release 3.1 Refs #204, #207, #208 --- docs/changelog.rst | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 76b6b5d..63e5833 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog =========== +.. _v3_1: + +3.1 (2020-12-12) +---------------- + +- New command: ``sqlite-utils analyze-tables my.db`` outputs useful inforamtion about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (`#207 `__) +- New ``table.analyze_column(column)`` Python method used by the ``analyze-tables`` command - see :ref:`python_api_analyze_column`. +- The ``table.update()`` method now correctly handles values that should be stored as JSON. Thanks, Andreas Madsack. (`#204 `__) + .. _v3_0: 3.0 (2020-11-08) diff --git a/setup.py b/setup.py index 9d80bbd..a507a1b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.0" +VERSION = "3.1" def get_long_description(): From cede53a37d7c86318befa107f4d2f38bbcbc30aa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 12 Dec 2020 23:29:45 -0800 Subject: [PATCH 16/16] Fixed typo in release notes --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 63e5833..b0c3da1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,7 @@ 3.1 (2020-12-12) ---------------- -- New command: ``sqlite-utils analyze-tables my.db`` outputs useful inforamtion about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (`#207 `__) +- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (`#207 `__) - New ``table.analyze_column(column)`` Python method used by the ``analyze-tables`` command - see :ref:`python_api_analyze_column`. - The ``table.update()`` method now correctly handles values that should be stored as JSON. Thanks, Andreas Madsack. (`#204 `__)