sqlite-utils analyze-tables command and table.analyze_column() method

Closes #207

sqlite_utils/cli.py

@@ -7,6 +7,7 @@ import hashlib
 import pathlib
 import sqlite_utils
 from sqlite_utils.db import AlterError
+import textwrap
 import itertools
 import json
 import os
@@ -1354,6 +1355,93 @@ def insert_files(
             )
 
 
+@cli.command(name="analyze-tables")
+@click.argument(
+    "path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False, exists=True),
+    required=True,
+)
+@click.argument("tables", nargs=-1)
+@click.option(
+    "-c",
+    "--column",
+    "columns",
+    type=str,
+    multiple=True,
+    help="Specific columns to analyze",
+)
+@click.option("--save", is_flag=True, help="Save results to _analyze_tables table")
+@load_extension_option
+def analyze_tables(
+    path,
+    tables,
+    columns,
+    save,
+    load_extension,
+):
+    "Analyze the columns in one or more tables"
+    db = sqlite_utils.Database(path)
+    _load_extensions(db, load_extension)
+    if not tables:
+        tables = db.table_names()
+    todo = []
+    table_counts = {}
+    for table in tables:
+        table_counts[table] = db[table].count
+        for column in db[table].columns:
+            if not columns or column.name in columns:
+                todo.append((table, column.name))
+    # Now we now how many we need to do
+    for i, (table, column) in enumerate(todo):
+        column_details = db[table].analyze_column(
+            column, total_rows=table_counts[table], value_truncate=80
+        )
+        if save:
+            db["_analyze_tables_"].insert(
+                column_details._asdict(), pk=("table", "column"), replace=True
+            )
+        most_common_rendered = _render_common(
+            "\n\n  Most common:", column_details.most_common
+        )
+        least_common_rendered = _render_common(
+            "\n\n  Least common:", column_details.least_common
+        )
+        details = (
+            (
+                textwrap.dedent(
+                    """
+        {table}.{column}: ({i}/{total})
+
+          Total rows: {total_rows}
+          Null rows: {num_null}
+          Blank rows: {num_blank}
+
+          Distinct values: {num_distinct}{most_common_rendered}{least_common_rendered}
+        """
+                )
+                .strip()
+                .format(
+                    i=i + 1,
+                    total=len(todo),
+                    most_common_rendered=most_common_rendered,
+                    least_common_rendered=least_common_rendered,
+                    **column_details._asdict()
+                )
+            )
+            + "\n"
+        )
+        click.echo(details)
+
+
+def _render_common(title, values):
+    if values is None:
+        return ""
+    lines = [title]
+    for value, count in values:
+        lines.append("    {}: {}".format(count, value))
+    return "\n".join(lines)
+
+
 FILE_COLUMNS = {
     "name": lambda p: p.name,
     "path": lambda p: str(p),

sqlite_utils/db.py

@@ -51,6 +51,19 @@ except ImportError:
 Column = namedtuple(
     "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk")
 )
+ColumnDetails = namedtuple(
+    "ColumnDetails",
+    (
+        "table",
+        "column",
+        "total_rows",
+        "num_null",
+        "num_blank",
+        "num_distinct",
+        "most_common",
+        "least_common",
+    ),
+)
 ForeignKey = namedtuple(
     "ForeignKey", ("table", "column", "other_table", "other_column")
 )
@@ -1930,6 +1943,72 @@ class Table(Queryable):
             )
         return self
 
+    def analyze_column(
+        self, column, common_limit=10, value_truncate=None, total_rows=None
+    ):
+        db = self.db
+        table = self.name
+        if total_rows is None:
+            total_rows = db[table].count
+
+        def truncate(value):
+            if value_truncate is None or isinstance(value, (float, int)):
+                return value
+            value = str(value)
+            if len(value) > value_truncate:
+                value = value[:value_truncate] + "..."
+            return value
+
+        num_null = db.execute(
+            "select count(*) from [{}] where [{}] is null".format(table, column)
+        ).fetchone()[0]
+        num_blank = db.execute(
+            "select count(*) from [{}] where [{}] = ''".format(table, column)
+        ).fetchone()[0]
+        num_distinct = db.execute(
+            "select count(distinct [{}]) from [{}]".format(column, table)
+        ).fetchone()[0]
+        most_common = None
+        least_common = None
+        if num_distinct == 1:
+            value = db.execute(
+                "select [{}] from [{}] limit 1".format(column, table)
+            ).fetchone()[0]
+            most_common = [(truncate(value), total_rows)]
+        elif num_distinct != total_rows:
+            most_common = [
+                (truncate(r[0]), r[1])
+                for r in db.execute(
+                    "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format(
+                        column, table, column, column, common_limit
+                    )
+                ).fetchall()
+            ]
+            most_common.sort(key=lambda p: (p[1], p[0]), reverse=True)
+            if num_distinct <= common_limit:
+                # No need to run the query if it will just return the results in revers order
+                least_common = None
+            else:
+                least_common = [
+                    (truncate(r[0]), r[1])
+                    for r in db.execute(
+                        "select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format(
+                            column, table, column, column, common_limit
+                        )
+                    ).fetchall()
+                ]
+                least_common.sort(key=lambda p: (p[1], p[0]))
+        return ColumnDetails(
+            self.name,
+            column,
+            total_rows,
+            num_null,
+            num_blank,
+            num_distinct,
+            most_common,
+            least_common,
+        )
+
 
 class View(Queryable):
     def exists(self):