Fixed typo in release notes

Refs #204, #207, #208

docs/changelog.rst

@@ -2,6 +2,15 @@
  Changelog
 ===========
 
+.. _v3_1:
+
+3.1 (2020-12-12)
+----------------
+
+- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (`#207 <https://github.com/simonw/sqlite-utils/issues/207>`__)
+- New ``table.analyze_column(column)`` Python method used by the ``analyze-tables`` command - see :ref:`python_api_analyze_column`.
+- The ``table.update()`` method now correctly handles values that should be stored as JSON. Thanks, Andreas Madsack. (`#204 <https://github.com/simonw/sqlite-utils/pull/204>`__)
+
 .. _v3_0:
 
 3.0 (2020-11-08)

docs/cli.rst

@@ -279,6 +279,93 @@ It takes the same options as the ``tables`` command:
 * ``--tsv``
 * ``--table``
 
+.. _cli_analyze_tables:
+
+Analyzing tables
+================
+
+When working with a new database it can be useful to get an idea of the shape of the data. The ``sqlite-utils analyze-tables`` command inspects specified tables (or all tables) and calculates some useful details about each of the columns in those tables.
+
+To inspect the ``tags`` table in the ``github.db`` database, run the following::
+
+    $ sqlite-utils analyze-tables github.db tags
+    tags.repo: (1/3)
+
+      Total rows: 261
+      Null rows: 0
+      Blank rows: 0
+
+      Distinct values: 14
+
+      Most common:
+        88: 107914493
+        75: 140912432
+        27: 206156866
+
+      Least common:
+        1: 209590345
+        2: 206649770
+        2: 303218369
+
+    tags.name: (2/3)
+
+      Total rows: 261
+      Null rows: 0
+      Blank rows: 0
+
+      Distinct values: 175
+
+      Most common:
+        10: 0.2
+        9: 0.1
+        7: 0.3
+
+      Least common:
+        1: 0.1.1
+        1: 0.11.1
+        1: 0.1a2
+
+    tags.sha: (3/3)
+
+      Total rows: 261
+      Null rows: 0
+      Blank rows: 0
+
+      Distinct values: 261
+
+For each column this tool dispalys the number of null rows, the number of blank rows (rows that contain an empty string), the number of distinct values and, for columns that are not entirely distinct, the most common and least common values.
+
+If you do not specify any tables every table in the database will be analyzed::
+
+    $ sqlite-utils analyze-tables github.db
+
+If you wish to analyze one or more specific columns, use the ``-c`` option::
+
+    $ sqlite-utils analyze-tables github.db tags -c sha
+
+.. _cli_analyze_tables_save:
+
+Saving the analyzed table details
+---------------------------------
+
+``analyze-tables`` can take quite a while to run for large database files. You can save the results of the analysis to a database table called ``_analyze_tables_`` using the ``--save`` option::
+
+    $ sqlite-utils analyze-tables github.db --save
+
+The ``_analyze_tables_`` table has the following schema::
+
+    CREATE TABLE [_analyze_tables_] (
+        [table] TEXT,
+        [column] TEXT,
+        [total_rows] INTEGER,
+        [num_null] INTEGER,
+        [num_blank] INTEGER,
+        [num_distinct] INTEGER,
+        [most_common] TEXT,
+        [least_common] TEXT,
+        PRIMARY KEY ([table], [column])
+    );
+
 .. _cli_inserting_data:
 
 Inserting JSON data

docs/python-api.rst

@@ -747,6 +747,41 @@ You can inspect the database to see the results like this::
         PRIMARY KEY ([characteristics_id], [dogs_id])
     )
 
+.. _python_api_analyze_column:
+
+Analyzing a column
+==================
+
+The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` method is used by the :ref:`analyze-tables <cli_analyze_tables>` CLI command. It returns a ``ColumnDetails`` named tuple with the following fields:
+
+``table``
+    The name of the table
+
+``column``
+    The name of the column
+
+``total_rows``
+    The total number of rows in the table`
+
+``num_null``
+    The number of rows for which this column is null
+
+``num_blank``
+    The number of rows for which this column is blank (the empty string)
+
+``num_distinct``
+    The number of distinct values in this column
+
+``most_common``
+    The ``N`` most common values as a list of ``(value, count)`` tuples`, or ``None`` if the table consists entirely of distinct values
+
+``least_common``
+    The ``N`` least common values as a list of ``(value, count)`` tuples`, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``)
+
+``N`` defaults to 10, or you can pass a custom ``N`` using the ``common_limit`` parameter.
+
+You can use the ``value_truncate`` parameter to truncate values in the ``most_common`` and ``least_common`` lists to a specified number of characters.
+
 .. _python_api_add_column:
 
 Adding columns

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import io
 import os
 
-VERSION = "3.0"
+VERSION = "3.1"
 
 
 def get_long_description():

sqlite_utils/cli.py

@@ -7,6 +7,7 @@ import hashlib
 import pathlib
 import sqlite_utils
 from sqlite_utils.db import AlterError
+import textwrap
 import itertools
 import json
 import os
@@ -1354,6 +1355,93 @@ def insert_files(
             )
 
 
+@cli.command(name="analyze-tables")
+@click.argument(
+    "path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False, exists=True),
+    required=True,
+)
+@click.argument("tables", nargs=-1)
+@click.option(
+    "-c",
+    "--column",
+    "columns",
+    type=str,
+    multiple=True,
+    help="Specific columns to analyze",
+)
+@click.option("--save", is_flag=True, help="Save results to _analyze_tables table")
+@load_extension_option
+def analyze_tables(
+    path,
+    tables,
+    columns,
+    save,
+    load_extension,
+):
+    "Analyze the columns in one or more tables"
+    db = sqlite_utils.Database(path)
+    _load_extensions(db, load_extension)
+    if not tables:
+        tables = db.table_names()
+    todo = []
+    table_counts = {}
+    for table in tables:
+        table_counts[table] = db[table].count
+        for column in db[table].columns:
+            if not columns or column.name in columns:
+                todo.append((table, column.name))
+    # Now we now how many we need to do
+    for i, (table, column) in enumerate(todo):
+        column_details = db[table].analyze_column(
+            column, total_rows=table_counts[table], value_truncate=80
+        )
+        if save:
+            db["_analyze_tables_"].insert(
+                column_details._asdict(), pk=("table", "column"), replace=True
+            )
+        most_common_rendered = _render_common(
+            "\n\n  Most common:", column_details.most_common
+        )
+        least_common_rendered = _render_common(
+            "\n\n  Least common:", column_details.least_common
+        )
+        details = (
+            (
+                textwrap.dedent(
+                    """
+        {table}.{column}: ({i}/{total})
+
+          Total rows: {total_rows}
+          Null rows: {num_null}
+          Blank rows: {num_blank}
+
+          Distinct values: {num_distinct}{most_common_rendered}{least_common_rendered}
+        """
+                )
+                .strip()
+                .format(
+                    i=i + 1,
+                    total=len(todo),
+                    most_common_rendered=most_common_rendered,
+                    least_common_rendered=least_common_rendered,
+                    **column_details._asdict()
+                )
+            )
+            + "\n"
+        )
+        click.echo(details)
+
+
+def _render_common(title, values):
+    if values is None:
+        return ""
+    lines = [title]
+    for value, count in values:
+        lines.append("    {}: {}".format(count, value))
+    return "\n".join(lines)
+
+
 FILE_COLUMNS = {
     "name": lambda p: p.name,
     "path": lambda p: str(p),

sqlite_utils/db.py

@@ -51,6 +51,19 @@ except ImportError:
 Column = namedtuple(
     "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk")
 )
+ColumnDetails = namedtuple(
+    "ColumnDetails",
+    (
+        "table",
+        "column",
+        "total_rows",
+        "num_null",
+        "num_blank",
+        "num_distinct",
+        "most_common",
+        "least_common",
+    ),
+)
 ForeignKey = namedtuple(
     "ForeignKey", ("table", "column", "other_table", "other_column")
 )
@@ -1930,6 +1943,72 @@ class Table(Queryable):
             )
         return self
 
+    def analyze_column(
+        self, column, common_limit=10, value_truncate=None, total_rows=None
+    ):
+        db = self.db
+        table = self.name
+        if total_rows is None:
+            total_rows = db[table].count
+
+        def truncate(value):
+            if value_truncate is None or isinstance(value, (float, int)):
+                return value
+            value = str(value)
+            if len(value) > value_truncate:
+                value = value[:value_truncate] + "..."
+            return value
+
+        num_null = db.execute(
+            "select count(*) from [{}] where [{}] is null".format(table, column)
+        ).fetchone()[0]
+        num_blank = db.execute(
+            "select count(*) from [{}] where [{}] = ''".format(table, column)
+        ).fetchone()[0]
+        num_distinct = db.execute(
+            "select count(distinct [{}]) from [{}]".format(column, table)
+        ).fetchone()[0]
+        most_common = None
+        least_common = None
+        if num_distinct == 1:
+            value = db.execute(
+                "select [{}] from [{}] limit 1".format(column, table)
+            ).fetchone()[0]
+            most_common = [(truncate(value), total_rows)]
+        elif num_distinct != total_rows:
+            most_common = [
+                (truncate(r[0]), r[1])
+                for r in db.execute(
+                    "select [{}], count(*) from [{}] group by [{}] order by count(*) desc, [{}] limit {}".format(
+                        column, table, column, column, common_limit
+                    )
+                ).fetchall()
+            ]
+            most_common.sort(key=lambda p: (p[1], p[0]), reverse=True)
+            if num_distinct <= common_limit:
+                # No need to run the query if it will just return the results in revers order
+                least_common = None
+            else:
+                least_common = [
+                    (truncate(r[0]), r[1])
+                    for r in db.execute(
+                        "select [{}], count(*) from [{}] group by [{}] order by count(*), [{}] desc limit {}".format(
+                            column, table, column, column, common_limit
+                        )
+                    ).fetchall()
+                ]
+                least_common.sort(key=lambda p: (p[1], p[0]))
+        return ColumnDetails(
+            self.name,
+            column,
+            total_rows,
+            num_null,
+            num_blank,
+            num_distinct,
+            most_common,
+            least_common,
+        )
+
 
 class View(Queryable):
     def exists(self):

tests/test_analyze_tables.py

@@ -0,0 +1,167 @@
+from sqlite_utils.db import Database, ForeignKey, ColumnDetails
+from sqlite_utils import cli
+from sqlite_utils.utils import OperationalError
+from click.testing import CliRunner
+import pytest
+import sqlite3
+import textwrap
+
+
+@pytest.fixture
+def db_to_analyze(fresh_db):
+    stuff = fresh_db["stuff"]
+    stuff.insert_all(
+        [
+            {"id": 1, "owner": "Terryterryterry", "size": 5},
+            {"id": 2, "owner": "Joan", "size": 4},
+            {"id": 3, "owner": "Kumar", "size": 5},
+            {"id": 4, "owner": "Anne", "size": 5},
+            {"id": 5, "owner": "Terryterryterry", "size": 5},
+            {"id": 6, "owner": "Joan", "size": 4},
+            {"id": 7, "owner": "Kumar", "size": 5},
+            {"id": 8, "owner": "Joan", "size": 4},
+        ],
+        pk="id",
+    )
+    return fresh_db
+
+
+@pytest.mark.parametrize(
+    "column,expected",
+    [
+        (
+            "id",
+            ColumnDetails(
+                table="stuff",
+                column="id",
+                total_rows=8,
+                num_null=0,
+                num_blank=0,
+                num_distinct=8,
+                most_common=None,
+                least_common=None,
+            ),
+        ),
+        (
+            "owner",
+            ColumnDetails(
+                table="stuff",
+                column="owner",
+                total_rows=8,
+                num_null=0,
+                num_blank=0,
+                num_distinct=4,
+                most_common=[("Joan", 3), ("Kumar", 2)],
+                least_common=[("Anne", 1), ("Terry...", 2)],
+            ),
+        ),
+        (
+            "size",
+            ColumnDetails(
+                table="stuff",
+                column="size",
+                total_rows=8,
+                num_null=0,
+                num_blank=0,
+                num_distinct=2,
+                most_common=[(5, 5), (4, 3)],
+                least_common=None,
+            ),
+        ),
+    ],
+)
+def test_analyze_column(db_to_analyze, column, expected):
+    assert (
+        db_to_analyze["stuff"].analyze_column(column, common_limit=2, value_truncate=5)
+        == expected
+    )
+
+
+@pytest.fixture
+def db_to_analyze_path(db_to_analyze, tmpdir):
+    path = str(tmpdir / "test.db")
+    db = sqlite3.connect(path)
+    db.executescript("\n".join(db_to_analyze.conn.iterdump()))
+    return path
+
+
+def test_analyze_table(db_to_analyze_path):
+    result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path])
+    assert (
+        result.output.strip()
+        == (
+            """
+stuff.id: (1/3)
+
+  Total rows: 8
+  Null rows: 0
+  Blank rows: 0
+
+  Distinct values: 8
+
+stuff.owner: (2/3)
+
+  Total rows: 8
+  Null rows: 0
+  Blank rows: 0
+
+  Distinct values: 4
+
+  Most common:
+    3: Joan
+    2: Terryterryterry
+    2: Kumar
+    1: Anne
+
+stuff.size: (3/3)
+
+  Total rows: 8
+  Null rows: 0
+  Blank rows: 0
+
+  Distinct values: 2
+
+  Most common:
+    5: 5
+    3: 4"""
+        ).strip()
+    )
+
+
+def test_analyze_table_save(db_to_analyze_path):
+    result = CliRunner().invoke(
+        cli.cli, ["analyze-tables", db_to_analyze_path, "--save"]
+    )
+    rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows)
+    assert rows == [
+        {
+            "table": "stuff",
+            "column": "id",
+            "total_rows": 8,
+            "num_null": 0,
+            "num_blank": 0,
+            "num_distinct": 8,
+            "most_common": None,
+            "least_common": None,
+        },
+        {
+            "table": "stuff",
+            "column": "owner",
+            "total_rows": 8,
+            "num_null": 0,
+            "num_blank": 0,
+            "num_distinct": 4,
+            "most_common": '[["Joan", 3], ["Terryterryterry", 2], ["Kumar", 2], ["Anne", 1]]',
+            "least_common": None,
+        },
+        {
+            "table": "stuff",
+            "column": "size",
+            "total_rows": 8,
+            "num_null": 0,
+            "num_blank": 0,
+            "num_distinct": 2,
+            "most_common": "[[5, 5], [4, 3]]",
+            "least_common": None,
+        },
+    ]