remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-09-27 12:24:11 +02:00
Code Issues Projects Releases Packages Wiki Activity

No need to show common values if everything is null

Browse source 
Closes #547
This commit is contained in:
Simon Willison 2023-05-21 10:19:16 -07:00
commit 6027f3ea69
3 changed files with 44 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

8
sqlite_utils/cli.py
View file

@ -2691,9 +2691,11 @@ def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least
db["_analyze_tables_"].insert( db["_analyze_tables_"].insert(
column_details._asdict(), pk=("table", "column"), replace=True column_details._asdict(), pk=("table", "column"), replace=True
) )
most_common_rendered = _render_common( most_common_rendered = ""
"\n\n Most common:", column_details.most_common if column_details.num_null != column_details.total_rows:
) most_common_rendered = _render_common(
"\n\n Most common:", column_details.most_common
)
least_common_rendered = _render_common( least_common_rendered = _render_common(
"\n\n Least common:", column_details.least_common "\n\n Least common:", column_details.least_common
) )

24
sqlite_utils/db.py
View file

@ -3470,18 +3470,22 @@ class Table(Queryable):
most_common_results = [(truncate(value), total_rows)] most_common_results = [(truncate(value), total_rows)]
elif num_distinct != total_rows: elif num_distinct != total_rows:
if most_common: if most_common:
most_common_results = [ # Optimization - if all rows are null, don't run this query
(truncate(r[0]), r[1]) if num_null == total_rows:
for r in db.execute( most_common_results = [(None, total_rows)]
"select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format( else:
column, table, column, column, common_limit most_common_results = [
) (truncate(r[0]), r[1])
).fetchall() for r in db.execute(
] "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format(
most_common_results.sort(key=lambda p: (p[1], p[0]), reverse=True) column, table, column, column, common_limit
)
).fetchall()
]
most_common_results.sort(key=lambda p: (p[1], p[0]), reverse=True)
if least_common: if least_common:
if num_distinct <= common_limit: if num_distinct <= common_limit:
# No need to run the query if it will just return the results in revers order # No need to run the query if it will just return the results in reverse order
least_common_results = None least_common_results = None
else: else:
least_common_results = [ least_common_results = [

26
tests/test_analyze_tables.py
View file

@ -40,6 +40,7 @@ def big_db_to_analyze_path(tmpdir):
to_insert.append( to_insert.append(
{ {
"category": category, "category": category,
"all_null": None,
} }
) )
db["stuff"].insert_all(to_insert) db["stuff"].insert_all(to_insert)
@ -233,7 +234,15 @@ def test_analyze_table_save(db_to_analyze_path):
def test_analyze_table_save_no_most_no_least_options( def test_analyze_table_save_no_most_no_least_options(
no_most, no_least, big_db_to_analyze_path no_most, no_least, big_db_to_analyze_path
): ):
args = ["analyze-tables", big_db_to_analyze_path, "--save", "--common-limit", "2"] args = [
"analyze-tables",
big_db_to_analyze_path,
"--save",
"--common-limit",
"2",
"--column",
"category",
]
if no_most: if no_most:
args.append("--no-most") args.append("--no-most")
if no_least: if no_least:
@ -257,3 +266,18 @@ def test_analyze_table_save_no_most_no_least_options(
expected["least_common"] = '[["D", 10], ["C", 20]]' expected["least_common"] = '[["D", 10], ["C", 20]]'
assert rows == [expected] assert rows == [expected]
def test_analyze_table_column_all_nulls(big_db_to_analyze_path):
result = CliRunner().invoke(
cli.cli,
["analyze-tables", big_db_to_analyze_path, "stuff", "--column", "all_null"],
)
assert result.exit_code == 0
assert result.output == (
"stuff.all_null: (1/1)\n\n Total rows: 100\n"
" Null rows: 100\n"
" Blank rows: 0\n"
"\n"
" Distinct values: 0\n\n"
)