"datasette inspect foo.db" now just calculates table counts

Refs #462

* inspect command now just outputs table counts
* test_inspect.py is now only tests for that CLI command
* Updated some relevant documentation
* Removed docs for /-/inspect since that is about to change

datasette/cli.py

@@ -1,3 +1,4 @@
+import asyncio
 import click
 from click import formatting
 from click_default_group import DefaultGroup
@@ -60,7 +61,7 @@ def cli():
 
 @cli.command()
 @click.argument("files", type=click.Path(exists=True), nargs=-1)
-@click.option("--inspect-file", default="inspect-data.json")
+@click.option("--inspect-file", default="-")
 @click.option(
     "sqlite_extensions",
     "--load-extension",
@@ -70,8 +71,31 @@ def cli():
     help="Path to a SQLite extension to load",
 )
 def inspect(files, inspect_file, sqlite_extensions):
-    app = Datasette(files, sqlite_extensions=sqlite_extensions)
+    app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
-    open(inspect_file, "w").write(json.dumps(app.inspect(), indent=2))
+    if inspect_file == "-":
+        out = sys.stdout
+    else:
+        out = open(inspect_file, "w")
+    loop = asyncio.get_event_loop()
+    inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions))
+    out.write(json.dumps(inspect_data, indent=2))
+
+
+async def inspect_(files, sqlite_extensions):
+    app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
+    data = {}
+    for name, database in app.databases.items():
+        counts = await database.table_counts(limit=3600 * 1000)
+        data[name] = {
+            "hash": database.hash,
+            "size": database.size,
+            "file": database.path,
+            "tables": {
+                table_name: {"count": table_count}
+                for table_name, table_count in counts.items()
+            },
+        }
+    return data
 
 
 @cli.group()

docs/full_text_search.rst

@@ -7,7 +7,7 @@ SQLite includes `a powerful mechanism for enabling full-text search <https://www
 
 .. image:: full_text_search.png
 
-Datasette detects which tables have been configured for full-text search when it first inspects the database on startup (or via the ``datasette inspect`` command). You can visit the ``/-/inspect`` page on your Datasette instance to see the results of this inspection. Tables that have been configured for full-text search will have their ``fts_table`` property set to the name of another table (tables without full-text search will have this property set to ``null``).
+Datasette automatically detects which tables have been configured for full-text search.
 
 FTS versions
 ------------
@@ -71,6 +71,8 @@ And then populate it like this:
 
 You can use this technique to populate the full-text search index from any combination of tables and joins that makes sense for your project.
 
+The `sqlite-utils tool <https://sqlite-utils.readthedocs.io/en/latest/cli.html#configuring-full-text-search>`__ provides a command-line mechanism that can be used to implement the above steps.
+
 .. _full_text_search_table_or_view:
 
 Configuring full-text search for a table or view

docs/introspection.rst

@@ -21,40 +21,6 @@ Shows the contents of the ``metadata.json`` file that was passed to ``datasette
         "databases": {...}
     }
 
-.. _JsonDataView_inspect:
-
-/-/inspect
-----------
-
-Shows the result of running ``datasette inspect`` on the currently loaded databases. This is run automatically when Datasette starts up, or can be run as a separate step and passed to ``datasette serve --inspect-file``.
-
-This is an internal implementation detail of Datasette and the format should not be considered stable - it is likely to change in undocumented ways between different releases.
-
-`Inspect example <https://fivethirtyeight.datasettes.com/-/inspect>`_::
-
-    {
-        "fivethirtyeight": {
-            "file": "fivethirtyeight.db",
-            "hash": "5de27e3eceb3f5ba817e0b2e066cea77832592b62d94690b5102a48f385b95fb",
-            "tables": {
-                "./index": {
-                    "columns": [
-                        "dataset_url",
-                        "article_url",
-                        "live"
-                    ],
-                    "count": 125,
-                    "foreign_keys": {
-                        "incoming": [],
-                        "outgoing": []
-                    },
-                    "fts_table": null,
-                    "hidden": false,
-                    "name": "./index",
-                    "primary_keys": []
-                },
-                ...
-
 .. _JsonDataView_versions:
 
 /-/versions

tests/test_inspect.py

@@ -1,92 +1,28 @@
-from datasette.app import Datasette
+from .fixtures import app_client
-from datasette.utils import sqlite3
+from datasette.cli import cli
-import os
+from click.testing import CliRunner
-import pytest
+import json
-import tempfile
 
 
-TABLES = """
+def test_inspect_cli(app_client):
-CREATE TABLE "election_results" (
+    runner = CliRunner()
-  "county" INTEGER,
+    result = runner.invoke(cli, ["inspect", "fixtures.db"])
-  "party" INTEGER,
+    data = json.loads(result.output)
-  "office" INTEGER,
+    assert ["fixtures"] == list(data.keys())
-  "votes" INTEGER,
+    database = data["fixtures"]
-  FOREIGN KEY (county) REFERENCES county(id),
+    assert "fixtures.db" == database["file"]
-  FOREIGN KEY (party) REFERENCES party(id),
+    assert isinstance(database["hash"], str)
-  FOREIGN KEY (office) REFERENCES office(id)
+    assert 64 == len(database["hash"])
- );
+    for table_name, expected_count in {
-
+        "Table With Space In Name": 0,
-CREATE VIRTUAL TABLE "election_results_fts" USING FTS4 ("county", "party");
+        "facetable": 15
-
+    }.items():
-CREATE TABLE "county" (
+        assert expected_count == database["tables"][table_name]["count"]
-  "id" INTEGER PRIMARY KEY ,
-  "name" TEXT
-);
-
-CREATE TABLE "party" (
-  "id" INTEGER PRIMARY KEY ,
-  "name" TEXT
-);
-
-CREATE TABLE "office" (
-  "id" INTEGER PRIMARY KEY ,
-  "name" TEXT
-);
-"""
 
 
-@pytest.fixture(scope="session")
+def test_inspect_cli_writes_to_file(app_client):
-def ds_instance():
+    runner = CliRunner()
-    with tempfile.TemporaryDirectory() as tmpdir:
+    result = runner.invoke(cli, ["inspect", "fixtures.db", "--inspect-file", "foo.json"])
-        filepath = os.path.join(tmpdir, "fixtures.db")
+    assert 0 == result.exit_code, result.output
-        conn = sqlite3.connect(filepath)
+    data = json.load(open("foo.json"))
-        conn.executescript(TABLES)
+    assert ["fixtures"] == list(data.keys())
-        yield Datasette([filepath])
-
-
-def test_inspect_hidden_tables(ds_instance):
-    info = ds_instance.inspect()
-    tables = info["fixtures"]["tables"]
-    expected_hidden = (
-        "election_results_fts",
-        "election_results_fts_content",
-        "election_results_fts_docsize",
-        "election_results_fts_segdir",
-        "election_results_fts_segments",
-        "election_results_fts_stat",
-    )
-    expected_visible = ("election_results", "county", "party", "office")
-    assert sorted(expected_hidden) == sorted(
-        [table for table in tables if tables[table]["hidden"]]
-    )
-    assert sorted(expected_visible) == sorted(
-        [table for table in tables if not tables[table]["hidden"]]
-    )
-
-
-def test_inspect_foreign_keys(ds_instance):
-    info = ds_instance.inspect()
-    tables = info["fixtures"]["tables"]
-    for table_name in ("county", "party", "office"):
-        assert 0 == tables[table_name]["count"]
-        foreign_keys = tables[table_name]["foreign_keys"]
-        assert [] == foreign_keys["outgoing"]
-        assert [
-            {
-                "column": "id",
-                "other_column": table_name,
-                "other_table": "election_results",
-            }
-        ] == foreign_keys["incoming"]
-
-    election_results = tables["election_results"]
-    assert 0 == election_results["count"]
-    assert sorted(
-        [
-            {"column": "county", "other_column": "id", "other_table": "county"},
-            {"column": "party", "other_column": "id", "other_table": "party"},
-            {"column": "office", "other_column": "id", "other_table": "office"},
-        ],
-        key=lambda d: d["column"],
-    ) == sorted(election_results["foreign_keys"]["outgoing"], key=lambda d: d["column"])
-    assert [] == election_results["foreign_keys"]["incoming"]