remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-07-24 09:54:31 +02:00
Code Issues Projects Releases Packages Wiki Activity

sqlite-utils analyze-tables command and table.analyze_column() method

Browse source 
Closes #207
This commit is contained in:
Simon Willison 2020-12-12 23:20:11 -08:00 • committed by GitHub
commit 69a121e088
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 456 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

79
sqlite_utils/db.py
View file

@ -51,6 +51,19 @@ except ImportError:
Column = namedtuple(
"Column", ("cid", "name", "type", "notnull", "default_value", "is_pk")
)
ColumnDetails = namedtuple(
"ColumnDetails",
(
"table",
"column",
"total_rows",
"num_null",
"num_blank",
"num_distinct",
"most_common",
"least_common",
),
)
ForeignKey = namedtuple(
"ForeignKey", ("table", "column", "other_table", "other_column")
)
@ -1930,6 +1943,72 @@ class Table(Queryable):
)
return self
def analyze_column(
self, column, common_limit=10, value_truncate=None, total_rows=None
):
db = self.db
table = self.name
if total_rows is None:
total_rows = db[table].count
def truncate(value):
if value_truncate is None or isinstance(value, (float, int)):
return value
value = str(value)
if len(value) > value_truncate:
value = value[:value_truncate] + "..."
return value
num_null = db.execute(
"select count(*) from [{}] where [{}] is null".format(table, column)
).fetchone()[0]
num_blank = db.execute(
"select count(*) from [{}] where [{}] = ''".format(table, column)
).fetchone()[0]
num_distinct = db.execute(
"select count(distinct [{}]) from [{}]".format(column, table)
).fetchone()[0]
most_common = None
least_common = None
if num_distinct == 1:
value = db.execute(
"select [{}] from [{}] limit 1".format(column, table)
).fetchone()[0]
most_common = [(truncate(value), total_rows)]
elif num_distinct != total_rows:
most_common = [
(truncate(r[0]), r[1])
for r in db.execute(
"select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format(
column, table, column, column, common_limit
)
).fetchall()
]
most_common.sort(key=lambda p: (p[1], p[0]), reverse=True)
if num_distinct <= common_limit:
# No need to run the query if it will just return the results in revers order
least_common = None
else:
least_common = [
(truncate(r[0]), r[1])
for r in db.execute(
"select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format(
column, table, column, column, common_limit
)
).fetchall()
]
least_common.sort(key=lambda p: (p[1], p[0]))
return ColumnDetails(
self.name,
column,
total_rows,
num_null,
num_blank,
num_distinct,
most_common,
least_common,
)
class View(Queryable):
def exists(self):