remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-09-19 06:54:25 +02:00
Code Issues Projects Releases Packages Wiki Activity

Improvements to most/least common values

Browse source 
* Record total_rows for each column
* Record (value, count) if there is just a single distinct value
* Do not calculate most/least common if all values are distinct
* Calculate table count once per table, not once per column
This commit is contained in:
Simon Willison 2020-12-11 21:41:07 -08:00
commit d4b8d9e7a5
2 changed files with 11 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

6
sqlite_utils/cli.py
View file

@ -1357,7 +1357,7 @@ def insert_files(
@cli.command(name="analyze-tables") @cli.command(name="analyze-tables")
@click.argument( @click.argument(
"path", "path",
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), type=click.Path(file_okay=True, dir_okay=False, allow_dash=False, exists=True),
required=True, required=True,
) )
@click.argument("tables", nargs=-1) @click.argument("tables", nargs=-1)
@ -1375,12 +1375,14 @@ def analyze_tables(
if not tables: if not tables:
tables = db.table_names() tables = db.table_names()
todo = [] todo = []
table_counts = {}
for table in tables: for table in tables:
table_counts[table] = db[table].count
for column in db[table].columns: for column in db[table].columns:
todo.append((table, column.name)) todo.append((table, column.name))
# Now we now how many we need to do # Now we now how many we need to do
for i, (table, column) in enumerate(todo): for i, (table, column) in enumerate(todo):
column_details = db[table].analyze_column(column) column_details = db[table].analyze_column(column, total_rows=table_counts[table])
if save: if save:
db["_analyze_tables"].insert( db["_analyze_tables"].insert(
column_details._asdict(), pk=("table", "column"), replace=True column_details._asdict(), pk=("table", "column"), replace=True

10
sqlite_utils/db.py
View file

@ -56,6 +56,7 @@ ColumnDetails = namedtuple(
( (
"table", "table",
"column", "column",
"total_rows",
"num_null", "num_null",
"num_blank", "num_blank",
"num_distinct", "num_distinct",
@ -1942,9 +1943,11 @@ class Table(Queryable):
) )
return self return self
def analyze_column(self, column, common_limit=10): def analyze_column(self, column, common_limit=10, total_rows=None):
db = self.db db = self.db
table = self.name table = self.name
if total_rows is None:
total_rows = db[table].count
num_null = db.execute( num_null = db.execute(
"select count(*) from [{}] where [{}] is null".format(table, column) "select count(*) from [{}] where [{}] is null".format(table, column)
).fetchone()[0] ).fetchone()[0]
@ -1960,8 +1963,8 @@ class Table(Queryable):
value = db.execute( value = db.execute(
"select [{}] from [{}] limit 1".format(column, table) "select [{}] from [{}] limit 1".format(column, table)
).fetchone()[0] ).fetchone()[0]
most_common = [value] most_common = [(value, total_rows)]
else: elif num_distinct != total_rows:
most_common = [ most_common = [
(r[0], r[1]) (r[0], r[1])
for r in db.execute( for r in db.execute(
@ -1985,6 +1988,7 @@ class Table(Queryable):
return ColumnDetails( return ColumnDetails(
self.name, self.name,
column, column,
total_rows,
num_null, num_null,
num_blank, num_blank,
num_distinct, num_distinct,