remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-07-25 18:34:32 +02:00
Code Issues Projects Releases Packages Wiki Activity

New options for analyze-tables - refs #544

Browse source 
This change adds three new options and modifies the `analyze_tables` function and `analyze_column` method in the `cli.py` and `db.py` modules respectively. The new options are `common_limit`, `no_most`, and `no_least`.

The `common_limit` option specifies how many common values should be returned by `analyze_column` method, by default it's set to 10. The `no_most` and `no_least` options, when set to True, skip returning the most and least common values, respectively.

The `analyze_tables` function was modified to pass the new options to the `_analyze` method, and the `analyze_column` method in the `db.py` module was modified to use the newly added options. Now, when `analyze_column` is called, it checks if `most_common` or `least_common` options are set to True before running the corresponding query. If the `num_distinct` value is less than or equal to `common_limit`, it doesn't run the least common query. The results from the most and least common queries are sorted before being returned.
This commit is contained in:
Simon Willison 2023-05-21 07:02:02 -07:00
commit 1c1991b447
2 changed files with 46 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

19
sqlite_utils/cli.py
View file

@ -2639,12 +2639,20 @@ def insert_files(
help="Specific columns to analyze",
)
@click.option("--save", is_flag=True, help="Save results to _analyze_tables table")
@click.option("--common-limit", type=int, default=10, help="How many common values")
@click.option("--no-most", is_flag=True, default=False, help="Skip most common values")
@click.option(
"--no-least", is_flag=True, default=False, help="Skip least common values"
)
@load_extension_option
def analyze_tables(
path,
tables,
columns,
save,
common_limit,
no_most,
no_least,
load_extension,
):
"""Analyze the columns in one or more tables
@ -2656,10 +2664,10 @@ def analyze_tables(
"""
db = sqlite_utils.Database(path)
_load_extensions(db, load_extension)
_analyze(db, tables, columns, save)
_analyze(db, tables, columns, save, common_limit, no_most, no_least)
def _analyze(db, tables, columns, save):
def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least=False):
if not tables:
tables = db.table_names()
todo = []
@ -2672,7 +2680,12 @@ def _analyze(db, tables, columns, save):
# Now we now how many we need to do
for i, (table, column) in enumerate(todo):
column_details = db[table].analyze_column(
column, total_rows=table_counts[table], value_truncate=80
column,
common_limit=common_limit,
total_rows=table_counts[table],
value_truncate=80,
most_common=not no_most,
least_common=not no_least,
)
if save:
db["_analyze_tables_"].insert(

52
sqlite_utils/db.py
View file

@ -3419,7 +3419,13 @@ class Table(Queryable):
self.db.analyze(self.name)
def analyze_column(
self, column: str, common_limit: int = 10, value_truncate=None, total_rows=None
self,
column: str,
common_limit: int = 10,
value_truncate=None,
total_rows=None,
most_common: bool = True,
least_common: bool = True,
) -> "ColumnDetails":
"""
Return statistics about the specified column.
@ -3453,36 +3459,38 @@ class Table(Queryable):
num_distinct = db.execute(
"select count(distinct [{}]) from [{}]".format(column, table)
).fetchone()[0]
most_common = None
least_common = None
most_common_results = None
least_common_results = None
if num_distinct == 1:
value = db.execute(
"select [{}] from [{}] limit 1".format(column, table)
).fetchone()[0]
most_common = [(truncate(value), total_rows)]
most_common_results = [(truncate(value), total_rows)]
elif num_distinct != total_rows:
most_common = [
(truncate(r[0]), r[1])
for r in db.execute(
"select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format(
column, table, column, column, common_limit
)
).fetchall()
]
most_common.sort(key=lambda p: (p[1], p[0]), reverse=True)
if num_distinct <= common_limit:
# No need to run the query if it will just return the results in revers order
least_common = None
else:
least_common = [
if most_common:
most_common_results = [
(truncate(r[0]), r[1])
for r in db.execute(
"select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format(
"select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format(
column, table, column, column, common_limit
)
).fetchall()
]
least_common.sort(key=lambda p: (p[1], p[0]))
most_common_results.sort(key=lambda p: (p[1], p[0]), reverse=True)
if least_common:
if num_distinct <= common_limit:
# No need to run the query if it will just return the results in revers order
least_common_results = None
else:
least_common_results = [
(truncate(r[0]), r[1])
for r in db.execute(
"select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format(
column, table, column, column, common_limit
)
).fetchall()
]
least_common_results.sort(key=lambda p: (p[1], p[0]))
return ColumnDetails(
self.name,
column,
@ -3490,8 +3498,8 @@ class Table(Queryable):
num_null,
num_blank,
num_distinct,
most_common,
least_common,
most_common_results,
least_common_results,
)
def add_geometry_column(