From 1c1991b447a1ddd3d61d9d4a8a1d6a9da47ced20 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 May 2023 07:02:02 -0700 Subject: [PATCH] New options for analyze-tables - refs #544 This change adds three new options and modifies the `analyze_tables` function and `analyze_column` method in the `cli.py` and `db.py` modules respectively. The new options are `common_limit`, `no_most`, and `no_least`. The `common_limit` option specifies how many common values should be returned by `analyze_column` method, by default it's set to 10. The `no_most` and `no_least` options, when set to True, skip returning the most and least common values, respectively. The `analyze_tables` function was modified to pass the new options to the `_analyze` method, and the `analyze_column` method in the `db.py` module was modified to use the newly added options. Now, when `analyze_column` is called, it checks if `most_common` or `least_common` options are set to True before running the corresponding query. If the `num_distinct` value is less than or equal to `common_limit`, it doesn't run the least common query. The results from the most and least common queries are sorted before being returned. --- sqlite_utils/cli.py | 19 ++++++++++++++--- sqlite_utils/db.py | 52 ++++++++++++++++++++++++++------------------- 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index ce354e0..0c91a8f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2639,12 +2639,20 @@ def insert_files( help="Specific columns to analyze", ) @click.option("--save", is_flag=True, help="Save results to _analyze_tables table") +@click.option("--common-limit", type=int, default=10, help="How many common values") +@click.option("--no-most", is_flag=True, default=False, help="Skip most common values") +@click.option( + "--no-least", is_flag=True, default=False, help="Skip least common values" +) @load_extension_option def analyze_tables( path, tables, columns, save, + common_limit, + no_most, + no_least, load_extension, ): """Analyze the columns in one or more tables @@ -2656,10 +2664,10 @@ def analyze_tables( """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - _analyze(db, tables, columns, save) + _analyze(db, tables, columns, save, common_limit, no_most, no_least) -def _analyze(db, tables, columns, save): +def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least=False): if not tables: tables = db.table_names() todo = [] @@ -2672,7 +2680,12 @@ def _analyze(db, tables, columns, save): # Now we now how many we need to do for i, (table, column) in enumerate(todo): column_details = db[table].analyze_column( - column, total_rows=table_counts[table], value_truncate=80 + column, + common_limit=common_limit, + total_rows=table_counts[table], + value_truncate=80, + most_common=not no_most, + least_common=not no_least, ) if save: db["_analyze_tables_"].insert( diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ec4bbfc..bd349e6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3419,7 +3419,13 @@ class Table(Queryable): self.db.analyze(self.name) def analyze_column( - self, column: str, common_limit: int = 10, value_truncate=None, total_rows=None + self, + column: str, + common_limit: int = 10, + value_truncate=None, + total_rows=None, + most_common: bool = True, + least_common: bool = True, ) -> "ColumnDetails": """ Return statistics about the specified column. @@ -3453,36 +3459,38 @@ class Table(Queryable): num_distinct = db.execute( "select count(distinct [{}]) from [{}]".format(column, table) ).fetchone()[0] - most_common = None - least_common = None + most_common_results = None + least_common_results = None if num_distinct == 1: value = db.execute( "select [{}] from [{}] limit 1".format(column, table) ).fetchone()[0] - most_common = [(truncate(value), total_rows)] + most_common_results = [(truncate(value), total_rows)] elif num_distinct != total_rows: - most_common = [ - (truncate(r[0]), r[1]) - for r in db.execute( - "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format( - column, table, column, column, common_limit - ) - ).fetchall() - ] - most_common.sort(key=lambda p: (p[1], p[0]), reverse=True) - if num_distinct <= common_limit: - # No need to run the query if it will just return the results in revers order - least_common = None - else: - least_common = [ + if most_common: + most_common_results = [ (truncate(r[0]), r[1]) for r in db.execute( - "select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format( + "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format( column, table, column, column, common_limit ) ).fetchall() ] - least_common.sort(key=lambda p: (p[1], p[0])) + most_common_results.sort(key=lambda p: (p[1], p[0]), reverse=True) + if least_common: + if num_distinct <= common_limit: + # No need to run the query if it will just return the results in revers order + least_common_results = None + else: + least_common_results = [ + (truncate(r[0]), r[1]) + for r in db.execute( + "select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format( + column, table, column, column, common_limit + ) + ).fetchall() + ] + least_common_results.sort(key=lambda p: (p[1], p[0])) return ColumnDetails( self.name, column, @@ -3490,8 +3498,8 @@ class Table(Queryable): num_null, num_blank, num_distinct, - most_common, - least_common, + most_common_results, + least_common_results, ) def add_geometry_column(