sqlite-utils memory --analyze, closes #320

This commit is contained in:
Simon Willison 2021-08-22 08:44:25 -07:00
commit 9258f4bd84
3 changed files with 67 additions and 5 deletions

View file

@ -328,10 +328,10 @@ The CSV data that was piped into the script is available in the ``stdin`` table,
.. _cli_memory_schema_dump_save:
\-\-schema, \-\-dump and \-\-save
---------------------------------
\-\-schema, \-\-analyze, \-\-dump and \-\-save
----------------------------------------------
To see the schema that will be created for a file or multiple files, use ``--schema``::
To see the in-memory datbase schema that would be used for a file or for multiple files, use ``--schema``::
% sqlite-utils memory dogs.csv --schema
CREATE TABLE [dogs] (
@ -342,6 +342,33 @@ To see the schema that will be created for a file or multiple files, use ``--sch
CREATE VIEW t1 AS select * from [dogs];
CREATE VIEW t AS select * from [dogs];
You can run the equivalent of the :ref:`analyze-tables <cli_analyze_tables>` command using ``--analyze``::
% sqlite-utils memory dogs.csv --analyze
dogs.id: (1/3)
Total rows: 2
Null rows: 0
Blank rows: 0
Distinct values: 2
dogs.name: (2/3)
Total rows: 2
Null rows: 0
Blank rows: 0
Distinct values: 2
dogs.age: (3/3)
Total rows: 2
Null rows: 0
Blank rows: 0
Distinct values: 2
You can output SQL that will both create the tables and insert the full data used to populate the in-memory database using ``--dump``::
% sqlite-utils memory dogs.csv --dump

View file

@ -1216,6 +1216,11 @@ def query(
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
help="Save in-memory database to this file",
)
@click.option(
"--analyze",
is_flag=True,
help="Analyze resulting tables and output results",
)
@load_extension_option
def memory(
paths,
@ -1236,6 +1241,7 @@ def memory(
schema,
dump,
save,
analyze,
load_extension,
):
"""Execute SQL query against an in-memory database, optionally populated by imported data
@ -1265,8 +1271,8 @@ def memory(
sqlite-utils memory animals.csv --schema
"""
db = sqlite_utils.Database(memory=True)
# If --dump or --save used but no paths detected, assume SQL query is a path:
if (dump or save or schema) and not paths:
# If --dump or --save or --analyze used but no paths detected, assume SQL query is a path:
if (dump or save or schema or analyze) and not paths:
paths = [sql]
sql = None
for i, path in enumerate(paths):
@ -1299,6 +1305,10 @@ def memory(
if not db[view_name].exists():
db.create_view(view_name, "select * from [{}]".format(csv_table))
if analyze:
_analyze(db, tables=None, columns=None, save=False)
return
if dump:
for line in db.conn.iterdump():
click.echo(line)
@ -1922,6 +1932,10 @@ def analyze_tables(
"Analyze the columns in one or more tables"
db = sqlite_utils.Database(path)
_load_extensions(db, load_extension)
_analyze(db, tables, columns, save)
def _analyze(db, tables, columns, save):
if not tables:
tables = db.table_names()
todo = []

View file

@ -220,3 +220,24 @@ def test_memory_no_detect_types(option):
{"id": "1", "name": "Cleo", "weight": "45.5"},
{"id": "2", "name": "Bants", "weight": "3.5"},
]
def test_memory_analyze():
result = CliRunner().invoke(
cli.cli,
["memory", "-", "--analyze"],
input="id,name\n1,Cleo\n2,Bants",
)
assert result.exit_code == 0
assert result.output == (
"stdin.id: (1/2)\n\n"
" Total rows: 2\n"
" Null rows: 0\n"
" Blank rows: 0\n\n"
" Distinct values: 2\n\n"
"stdin.name: (2/2)\n\n"
" Total rows: 2\n"
" Null rows: 0\n"
" Blank rows: 0\n\n"
" Distinct values: 2\n\n"
)