remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-07-26 02:44:33 +02:00
Code Issues Projects Releases Packages Wiki Activity

Improvements to most/least common values

Browse source 
* Record total_rows for each column
* Record (value, count) if there is just a single distinct value
* Do not calculate most/least common if all values are distinct
* Calculate table count once per table, not once per column
This commit is contained in:
Simon Willison 2020-12-11 21:41:07 -08:00
commit d4b8d9e7a5
2 changed files with 11 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

6
sqlite_utils/cli.py
View file

@ -1357,7 +1357,7 @@ def insert_files(
@cli.command(name="analyze-tables")
@click.argument(
"path",
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False, exists=True),
required=True,
)
@click.argument("tables", nargs=-1)
@ -1375,12 +1375,14 @@ def analyze_tables(
if not tables:
tables = db.table_names()
todo = []
table_counts = {}
for table in tables:
table_counts[table] = db[table].count
for column in db[table].columns:
todo.append((table, column.name))
# Now we now how many we need to do
for i, (table, column) in enumerate(todo):
column_details = db[table].analyze_column(column)
column_details = db[table].analyze_column(column, total_rows=table_counts[table])
if save:
db["_analyze_tables"].insert(
column_details._asdict(), pk=("table", "column"), replace=True

10
sqlite_utils/db.py
View file

@ -56,6 +56,7 @@ ColumnDetails = namedtuple(
(
"table",
"column",
"total_rows",
"num_null",
"num_blank",
"num_distinct",
@ -1942,9 +1943,11 @@ class Table(Queryable):
)
return self
def analyze_column(self, column, common_limit=10):
def analyze_column(self, column, common_limit=10, total_rows=None):
db = self.db
table = self.name
if total_rows is None:
total_rows = db[table].count
num_null = db.execute(
"select count(*) from [{}] where [{}] is null".format(table, column)
).fetchone()[0]
@ -1960,8 +1963,8 @@ class Table(Queryable):
value = db.execute(
"select [{}] from [{}] limit 1".format(column, table)
).fetchone()[0]
most_common = [value]
else:
most_common = [(value, total_rows)]
elif num_distinct != total_rows:
most_common = [
(r[0], r[1])
for r in db.execute(
@ -1985,6 +1988,7 @@ class Table(Queryable):
return ColumnDetails(
self.name,
column,
total_rows,
num_null,
num_blank,
num_distinct,