From ca2ccccac28bffbb0d4e345121919712cdef3ec6 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Fri, 11 Dec 2020 21:27:00 -0800
Subject: [PATCH] Initial prototype of sqlite-utils analyze-tables, refs #207

---
 sqlite_utils/cli.py | 34 ++++++++++++++++++++++++
 sqlite_utils/db.py  | 63 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 97 insertions(+)

diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py
index cfc57c6..e779bef 100644
--- a/sqlite_utils/cli.py
+++ b/sqlite_utils/cli.py
@@ -1354,6 +1354,40 @@ def insert_files(
             )
 
 
+@cli.command(name="analyze-tables")
+@click.argument(
+    "path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("tables", nargs=-1)
+@click.option("--save", is_flag=True, help="Save results to _analyze_tables table")
+@load_extension_option
+def analyze_tables(
+    path,
+    tables,
+    save,
+    load_extension,
+):
+    "Analyze the columns in one or more tables"
+    db = sqlite_utils.Database(path)
+    _load_extensions(db, load_extension)
+    if not tables:
+        tables = db.table_names()
+    todo = []
+    for table in tables:
+        for column in db[table].columns:
+            todo.append((table, column.name))
+    # Now we now how many we need to do
+    for i, (table, column) in enumerate(todo):
+        column_details = db[table].analyze_column(column)
+        if save:
+            db["_analyze_tables"].insert(
+                column_details._asdict(), pk=("table", "column"), replace=True
+            )
+        click.echo("{}/{}: {}".format(i + 1, len(todo), column_details))
+
+
 FILE_COLUMNS = {
     "name": lambda p: p.name,
     "path": lambda p: str(p),
diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py
index 8849680..fdb2e92 100644
--- a/sqlite_utils/db.py
+++ b/sqlite_utils/db.py
@@ -51,6 +51,18 @@ except ImportError:
 Column = namedtuple(
     "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk")
 )
+ColumnDetails = namedtuple(
+    "ColumnDetails",
+    (
+        "table",
+        "column",
+        "num_null",
+        "num_blank",
+        "num_distinct",
+        "most_common",
+        "least_common",
+    ),
+)
 ForeignKey = namedtuple(
     "ForeignKey", ("table", "column", "other_table", "other_column")
 )
@@ -1930,6 +1942,57 @@ class Table(Queryable):
             )
         return self
 
+    def analyze_column(self, column, common_limit=10):
+        db = self.db
+        table = self.name
+        num_null = db.execute(
+            "select count(*) from [{}] where [{}] is null".format(table, column)
+        ).fetchone()[0]
+        num_blank = db.execute(
+            "select count(*) from [{}] where [{}] = ''".format(table, column)
+        ).fetchone()[0]
+        num_distinct = db.execute(
+            "select count(distinct [{}]) from [{}]".format(column, table)
+        ).fetchone()[0]
+        most_common = None
+        least_common = None
+        if num_distinct == 1:
+            value = db.execute(
+                "select [{}] from [{}] limit 1".format(column, table)
+            ).fetchone()[0]
+            most_common = [value]
+            least_common = [value]
+        else:
+            most_common = [
+                (r[0], r[1])
+                for r in db.execute(
+                    "select [{}], count(*) from [{}] group by [{}] order by count(*) desc limit {}".format(
+                        column, table, column, common_limit
+                    )
+                ).fetchall()
+            ]
+            if num_distinct <= common_limit:
+                # No need to run the query if it will just return the results in revers order
+                least_common = most_common[::-1]
+            else:
+                least_common = [
+                    (r[0], r[1])
+                    for r in db.execute(
+                        "select [{}], count(*) from [{}] group by [{}] order by count(*) limit {}".format(
+                            column, table, column, common_limit
+                        )
+                    ).fetchall()
+                ]
+        return ColumnDetails(
+            self.name,
+            column,
+            num_null,
+            num_blank,
+            num_distinct,
+            most_common,
+            least_common,
+        )
+
 
 class View(Queryable):
     def exists(self):