remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-07-23 09:24:31 +02:00
Code Issues Projects Releases Packages Wiki Activity

sqlite-utils analyze-tables command and table.analyze_column() method

Browse source 
Closes #207
This commit is contained in:
Simon Willison 2020-12-12 23:20:11 -08:00 • committed by GitHub
commit 69a121e088
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 456 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

87
docs/cli.rst
View file

@ -279,6 +279,93 @@ It takes the same options as the ``tables`` command:
* ``--tsv``
* ``--table``
.. _cli_analyze_tables:
Analyzing tables
================
When working with a new database it can be useful to get an idea of the shape of the data. The ``sqlite-utils analyze-tables`` command inspects specified tables (or all tables) and calculates some useful details about each of the columns in those tables.
To inspect the ``tags`` table in the ``github.db`` database, run the following::
$ sqlite-utils analyze-tables github.db tags
tags.repo: (1/3)
Total rows: 261
Null rows: 0
Blank rows: 0
Distinct values: 14
Most common:
88: 107914493
75: 140912432
27: 206156866
Least common:
1: 209590345
2: 206649770
2: 303218369
tags.name: (2/3)
Total rows: 261
Null rows: 0
Blank rows: 0
Distinct values: 175
Most common:
10: 0.2
9: 0.1
7: 0.3
Least common:
1: 0.1.1
1: 0.11.1
1: 0.1a2
tags.sha: (3/3)
Total rows: 261
Null rows: 0
Blank rows: 0
Distinct values: 261
For each column this tool dispalys the number of null rows, the number of blank rows (rows that contain an empty string), the number of distinct values and, for columns that are not entirely distinct, the most common and least common values.
If you do not specify any tables every table in the database will be analyzed::
$ sqlite-utils analyze-tables github.db
If you wish to analyze one or more specific columns, use the ``-c`` option::
$ sqlite-utils analyze-tables github.db tags -c sha
.. _cli_analyze_tables_save:
Saving the analyzed table details
---------------------------------
``analyze-tables`` can take quite a while to run for large database files. You can save the results of the analysis to a database table called ``_analyze_tables_`` using the ``--save`` option::
$ sqlite-utils analyze-tables github.db --save
The ``_analyze_tables_`` table has the following schema::
CREATE TABLE [_analyze_tables_] (
[table] TEXT,
[column] TEXT,
[total_rows] INTEGER,
[num_null] INTEGER,
[num_blank] INTEGER,
[num_distinct] INTEGER,
[most_common] TEXT,
[least_common] TEXT,
PRIMARY KEY ([table], [column])
);
.. _cli_inserting_data:
Inserting JSON data

35
docs/python-api.rst
View file

@ -747,6 +747,41 @@ You can inspect the database to see the results like this::
PRIMARY KEY ([characteristics_id], [dogs_id])
)
.. _python_api_analyze_column:
Analyzing a column
==================
The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` method is used by the :ref:`analyze-tables <cli_analyze_tables>` CLI command. It returns a ``ColumnDetails`` named tuple with the following fields:
``table``
The name of the table
``column``
The name of the column
``total_rows``
The total number of rows in the table`
``num_null``
The number of rows for which this column is null
``num_blank``
The number of rows for which this column is blank (the empty string)
``num_distinct``
The number of distinct values in this column
``most_common``
The ``N`` most common values as a list of ``(value, count)`` tuples`, or ``None`` if the table consists entirely of distinct values
``least_common``
The ``N`` least common values as a list of ``(value, count)`` tuples`, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``)
``N`` defaults to 10, or you can pass a custom ``N`` using the ``common_limit`` parameter.
You can use the ``value_truncate`` parameter to truncate values in the ``most_common`` and ``least_common`` lists to a specified number of characters.
.. _python_api_add_column:
Adding columns

88
sqlite_utils/cli.py
View file

@ -7,6 +7,7 @@ import hashlib
import pathlib
import sqlite_utils
from sqlite_utils.db import AlterError
import textwrap
import itertools
import json
import os
@ -1354,6 +1355,93 @@ def insert_files(
)
@cli.command(name="analyze-tables")
@click.argument(
"path",
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False, exists=True),
required=True,
)
@click.argument("tables", nargs=-1)
@click.option(
"-c",
"--column",
"columns",
type=str,
multiple=True,
help="Specific columns to analyze",
)
@click.option("--save", is_flag=True, help="Save results to _analyze_tables table")
@load_extension_option
def analyze_tables(
path,
tables,
columns,
save,
load_extension,
):
"Analyze the columns in one or more tables"
db = sqlite_utils.Database(path)
_load_extensions(db, load_extension)
if not tables:
tables = db.table_names()
todo = []
table_counts = {}
for table in tables:
table_counts[table] = db[table].count
for column in db[table].columns:
if not columns or column.name in columns:
todo.append((table, column.name))
# Now we now how many we need to do
for i, (table, column) in enumerate(todo):
column_details = db[table].analyze_column(
column, total_rows=table_counts[table], value_truncate=80
)
if save:
db["_analyze_tables_"].insert(
column_details._asdict(), pk=("table", "column"), replace=True
)
most_common_rendered = _render_common(
"\n\n Most common:", column_details.most_common
)
least_common_rendered = _render_common(
"\n\n Least common:", column_details.least_common
)
details = (
(
textwrap.dedent(
"""
{table}.{column}: ({i}/{total})
Total rows: {total_rows}
Null rows: {num_null}
Blank rows: {num_blank}
Distinct values: {num_distinct}{most_common_rendered}{least_common_rendered}
"""
)
.strip()
.format(
i=i + 1,
total=len(todo),
most_common_rendered=most_common_rendered,
least_common_rendered=least_common_rendered,
**column_details._asdict()
)
)
+ "\n"
)
click.echo(details)
def _render_common(title, values):
if values is None:
return ""
lines = [title]
for value, count in values:
lines.append(" {}: {}".format(count, value))
return "\n".join(lines)
FILE_COLUMNS = {
"name": lambda p: p.name,
"path": lambda p: str(p),

79
sqlite_utils/db.py
View file

@ -51,6 +51,19 @@ except ImportError:
Column = namedtuple(
"Column", ("cid", "name", "type", "notnull", "default_value", "is_pk")
)
ColumnDetails = namedtuple(
"ColumnDetails",
(
"table",
"column",
"total_rows",
"num_null",
"num_blank",
"num_distinct",
"most_common",
"least_common",
),
)
ForeignKey = namedtuple(
"ForeignKey", ("table", "column", "other_table", "other_column")
)
@ -1930,6 +1943,72 @@ class Table(Queryable):
)
return self
def analyze_column(
self, column, common_limit=10, value_truncate=None, total_rows=None
):
db = self.db
table = self.name
if total_rows is None:
total_rows = db[table].count
def truncate(value):
if value_truncate is None or isinstance(value, (float, int)):
return value
value = str(value)
if len(value) > value_truncate:
value = value[:value_truncate] + "..."
return value
num_null = db.execute(
"select count(*) from [{}] where [{}] is null".format(table, column)
).fetchone()[0]
num_blank = db.execute(
"select count(*) from [{}] where [{}] = ''".format(table, column)
).fetchone()[0]
num_distinct = db.execute(
"select count(distinct [{}]) from [{}]".format(column, table)
).fetchone()[0]
most_common = None
least_common = None
if num_distinct == 1:
value = db.execute(
"select [{}] from [{}] limit 1".format(column, table)
).fetchone()[0]
most_common = [(truncate(value), total_rows)]
elif num_distinct != total_rows:
most_common = [
(truncate(r[0]), r[1])
for r in db.execute(
"select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format(
column, table, column, column, common_limit
)
).fetchall()
]
most_common.sort(key=lambda p: (p[1], p[0]), reverse=True)
if num_distinct <= common_limit:
# No need to run the query if it will just return the results in revers order
least_common = None
else:
least_common = [
(truncate(r[0]), r[1])
for r in db.execute(
"select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format(
column, table, column, column, common_limit
)
).fetchall()
]
least_common.sort(key=lambda p: (p[1], p[0]))
return ColumnDetails(
self.name,
column,
total_rows,
num_null,
num_blank,
num_distinct,
most_common,
least_common,
)
class View(Queryable):
def exists(self):

167
tests/test_analyze_tables.py Normal file
View file

@ -0,0 +1,167 @@
from sqlite_utils.db import Database, ForeignKey, ColumnDetails
from sqlite_utils import cli
from sqlite_utils.utils import OperationalError
from click.testing import CliRunner
import pytest
import sqlite3
import textwrap
@pytest.fixture
def db_to_analyze(fresh_db):
stuff = fresh_db["stuff"]
stuff.insert_all(
[
{"id": 1, "owner": "Terryterryterry", "size": 5},
{"id": 2, "owner": "Joan", "size": 4},
{"id": 3, "owner": "Kumar", "size": 5},
{"id": 4, "owner": "Anne", "size": 5},
{"id": 5, "owner": "Terryterryterry", "size": 5},
{"id": 6, "owner": "Joan", "size": 4},
{"id": 7, "owner": "Kumar", "size": 5},
{"id": 8, "owner": "Joan", "size": 4},
],
pk="id",
)
return fresh_db
@pytest.mark.parametrize(
"column,expected",
[
(
"id",
ColumnDetails(
table="stuff",
column="id",
total_rows=8,
num_null=0,
num_blank=0,
num_distinct=8,
most_common=None,
least_common=None,
),
),
(
"owner",
ColumnDetails(
table="stuff",
column="owner",
total_rows=8,
num_null=0,
num_blank=0,
num_distinct=4,
most_common=[("Joan", 3), ("Kumar", 2)],
least_common=[("Anne", 1), ("Terry...", 2)],
),
),
(
"size",
ColumnDetails(
table="stuff",
column="size",
total_rows=8,
num_null=0,
num_blank=0,
num_distinct=2,
most_common=[(5, 5), (4, 3)],
least_common=None,
),
),
],
)
def test_analyze_column(db_to_analyze, column, expected):
assert (
db_to_analyze["stuff"].analyze_column(column, common_limit=2, value_truncate=5)
== expected
)
@pytest.fixture
def db_to_analyze_path(db_to_analyze, tmpdir):
path = str(tmpdir / "test.db")
db = sqlite3.connect(path)
db.executescript("\n".join(db_to_analyze.conn.iterdump()))
return path
def test_analyze_table(db_to_analyze_path):
result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path])
assert (
result.output.strip()
== (
"""
stuff.id: (1/3)
Total rows: 8
Null rows: 0
Blank rows: 0
Distinct values: 8
stuff.owner: (2/3)
Total rows: 8
Null rows: 0
Blank rows: 0
Distinct values: 4
Most common:
3: Joan
2: Terryterryterry
2: Kumar
1: Anne
stuff.size: (3/3)
Total rows: 8
Null rows: 0
Blank rows: 0
Distinct values: 2
Most common:
5: 5
3: 4"""
).strip()
)
def test_analyze_table_save(db_to_analyze_path):
result = CliRunner().invoke(
cli.cli, ["analyze-tables", db_to_analyze_path, "--save"]
)
rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows)
assert rows == [
{
"table": "stuff",
"column": "id",
"total_rows": 8,
"num_null": 0,
"num_blank": 0,
"num_distinct": 8,
"most_common": None,
"least_common": None,
},
{
"table": "stuff",
"column": "owner",
"total_rows": 8,
"num_null": 0,
"num_blank": 0,
"num_distinct": 4,
"most_common": '[["Joan", 3], ["Terryterryterry", 2], ["Kumar", 2], ["Anne", 1]]',
"least_common": None,
},
{
"table": "stuff",
"column": "size",
"total_rows": 8,
"num_null": 0,
"num_blank": 0,
"num_distinct": 2,
"most_common": "[[5, 5], [4, 3]]",
"least_common": None,
},
]