From 955daaaf056030fe93721ad30ad8a7c44afe2a50 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sat, 12 Dec 2020 20:58:05 -0800
Subject: [PATCH] Truncate values longer than 100 characters

---
 sqlite_utils/db.py           | 19 +++++++++++++++----
 tests/test_analyze_tables.py | 15 +++++++++------
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py
index b7f6084..1974860 100644
--- a/sqlite_utils/db.py
+++ b/sqlite_utils/db.py
@@ -1943,11 +1943,22 @@ class Table(Queryable):
             )
         return self
 
-    def analyze_column(self, column, common_limit=10, total_rows=None):
+    def analyze_column(
+        self, column, common_limit=10, value_truncate=100, total_rows=None
+    ):
         db = self.db
         table = self.name
         if total_rows is None:
             total_rows = db[table].count
+
+        def truncate(value):
+            if value_truncate is None or isinstance(value, (float, int)):
+                return value
+            value = str(value)
+            if len(value) > value_truncate:
+                value = value[:value_truncate] + "..."
+            return value
+
         num_null = db.execute(
             "select count(*) from [{}] where [{}] is null".format(table, column)
         ).fetchone()[0]
@@ -1963,10 +1974,10 @@ class Table(Queryable):
             value = db.execute(
                 "select [{}] from [{}] limit 1".format(column, table)
             ).fetchone()[0]
-            most_common = [(value, total_rows)]
+            most_common = [(truncate(value), total_rows)]
         elif num_distinct != total_rows:
             most_common = [
-                (r[0], r[1])
+                (truncate(r[0]), r[1])
                 for r in db.execute(
                     "select [{}], count(*) from [{}] group by [{}] order by count(*) desc limit {}".format(
                         column, table, column, common_limit
@@ -1978,7 +1989,7 @@ class Table(Queryable):
                 least_common = None
             else:
                 least_common = [
-                    (r[0], r[1])
+                    (truncate(r[0]), r[1])
                     for r in db.execute(
                         "select [{}], count(*) from [{}] group by [{}] order by count(*) limit {}".format(
                             column, table, column, common_limit
diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py
index 946ddc7..5aa8269 100644
--- a/tests/test_analyze_tables.py
+++ b/tests/test_analyze_tables.py
@@ -12,11 +12,11 @@ def db_to_analyze(fresh_db):
     stuff = fresh_db["stuff"]
     stuff.insert_all(
         [
-            {"id": 1, "owner": "Terry", "size": 5},
+            {"id": 1, "owner": "Terryterryterry", "size": 5},
             {"id": 2, "owner": "Joan", "size": 4},
             {"id": 3, "owner": "Kumar", "size": 5},
             {"id": 4, "owner": "Anne", "size": 5},
-            {"id": 5, "owner": "Terry", "size": 5},
+            {"id": 5, "owner": "Terryterryterry", "size": 5},
             {"id": 6, "owner": "Joan", "size": 4},
             {"id": 7, "owner": "Kumar", "size": 5},
             {"id": 8, "owner": "Joan", "size": 4},
@@ -51,7 +51,7 @@ def db_to_analyze(fresh_db):
                 num_null=0,
                 num_blank=0,
                 num_distinct=4,
-                most_common=[("Joan", 3), ("Terry", 2)],
+                most_common=[("Joan", 3), ("Terry...", 2)],
                 least_common=[("Anne", 1), ("Kumar", 2)],
             ),
         ),
@@ -71,7 +71,10 @@ def db_to_analyze(fresh_db):
     ],
 )
 def test_analyze_column(db_to_analyze, column, expected):
-    assert db_to_analyze["stuff"].analyze_column(column, common_limit=2) == expected
+    assert (
+        db_to_analyze["stuff"].analyze_column(column, common_limit=2, value_truncate=5)
+        == expected
+    )
 
 
 @pytest.fixture
@@ -89,7 +92,7 @@ def test_analyze_table(db_to_analyze_path):
         == textwrap.dedent(
             """
     1/3: ColumnDetails(table='stuff', column='id', total_rows=8, num_null=0, num_blank=0, num_distinct=8, most_common=None, least_common=None)
-    2/3: ColumnDetails(table='stuff', column='owner', total_rows=8, num_null=0, num_blank=0, num_distinct=4, most_common=[('Joan', 3), ('Terry', 2), ('Kumar', 2), ('Anne', 1)], least_common=None)
+    2/3: ColumnDetails(table='stuff', column='owner', total_rows=8, num_null=0, num_blank=0, num_distinct=4, most_common=[('Joan', 3), ('Terryterryterry', 2), ('Kumar', 2), ('Anne', 1)], least_common=None)
     3/3: ColumnDetails(table='stuff', column='size', total_rows=8, num_null=0, num_blank=0, num_distinct=2, most_common=[(5, 5), (4, 3)], least_common=None)
     """
         ).strip()
@@ -119,7 +122,7 @@ def test_analyze_table_save(db_to_analyze_path):
             "num_null": 0,
             "num_blank": 0,
             "num_distinct": 4,
-            "most_common": '[["Joan", 3], ["Terry", 2], ["Kumar", 2], ["Anne", 1]]',
+            "most_common": '[["Joan", 3], ["Terryterryterry", 2], ["Kumar", 2], ["Anne", 1]]',
             "least_common": None,
         },
         {