remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-07-25 10:24:32 +02:00
Code Issues Projects Releases Packages Wiki Activity

Analyze tables options: --common-limit, --no-most, --no-least

Browse source 
Closes #544
This commit is contained in:
Simon Willison 2023-05-21 09:19:30 -07:00 • committed by GitHub
commit d2a7b15b2b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 178 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

54
sqlite_utils/db.py
View file

@ -3419,7 +3419,13 @@ class Table(Queryable):
self.db.analyze(self.name)
def analyze_column(
self, column: str, common_limit: int = 10, value_truncate=None, total_rows=None
self,
column: str,
common_limit: int = 10,
value_truncate=None,
total_rows=None,
most_common: bool = True,
least_common: bool = True,
) -> "ColumnDetails":
"""
Return statistics about the specified column.
@ -3430,6 +3436,8 @@ class Table(Queryable):
:param common_limit: Show this many column values
:param value_truncate: Truncate display of common values to this many characters
:param total_rows: Optimization - pass the total number of rows in the table to save running a fresh ``count(*)`` query
:param most_common: If ``True``, calculate the most common values
:param least_common: If ``True``, calculate the least common values
"""
db = self.db
table = self.name
@ -3453,36 +3461,38 @@ class Table(Queryable):
num_distinct = db.execute(
"select count(distinct [{}]) from [{}]".format(column, table)
).fetchone()[0]
most_common = None
least_common = None
most_common_results = None
least_common_results = None
if num_distinct == 1:
value = db.execute(
"select [{}] from [{}] limit 1".format(column, table)
).fetchone()[0]
most_common = [(truncate(value), total_rows)]
most_common_results = [(truncate(value), total_rows)]
elif num_distinct != total_rows:
most_common = [
(truncate(r[0]), r[1])
for r in db.execute(
"select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format(
column, table, column, column, common_limit
)
).fetchall()
]
most_common.sort(key=lambda p: (p[1], p[0]), reverse=True)
if num_distinct <= common_limit:
# No need to run the query if it will just return the results in revers order
least_common = None
else:
least_common = [
if most_common:
most_common_results = [
(truncate(r[0]), r[1])
for r in db.execute(
"select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format(
"select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format(
column, table, column, column, common_limit
)
).fetchall()
]
least_common.sort(key=lambda p: (p[1], p[0]))
most_common_results.sort(key=lambda p: (p[1], p[0]), reverse=True)
if least_common:
if num_distinct <= common_limit:
# No need to run the query if it will just return the results in revers order
least_common_results = None
else:
least_common_results = [
(truncate(r[0]), r[1])
for r in db.execute(
"select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format(
column, table, column, column, common_limit
)
).fetchall()
]
least_common_results.sort(key=lambda p: (p[1], p[0]))
return ColumnDetails(
self.name,
column,
@ -3490,8 +3500,8 @@ class Table(Queryable):
num_null,
num_blank,
num_distinct,
most_common,
least_common,
most_common_results,
least_common_results,
)
def add_geometry_column(