Index page no longer uses inspect data - refs #420

Also introduced a mechanism whereby table counts are calculated against a time limit
but immutable databases have their table counts calculated on server startup.

datasette/views/index.py

@@ -1,8 +1,14 @@
+import hashlib
 import json
 
 from sanic import response
 
-from datasette.utils import CustomJSONEncoder
+from datasette.utils import (
+    CustomJSONEncoder,
+    InterruptedError,
+    detect_primary_keys,
+    detect_fts,
+)
 from datasette.version import __version__
 
 from .base import HASH_LENGTH, RenderMixin
@@ -16,26 +22,51 @@ class IndexView(RenderMixin):
 
     async def get(self, request, as_format):
         databases = []
-        for key, info in sorted(self.ds.inspect().items()):
-            tables = [t for t in info["tables"].values() if not t["hidden"]]
-            hidden_tables = [t for t in info["tables"].values() if t["hidden"]]
-            database = {
-                "name": key,
-                "hash": info["hash"],
-                "path": self.database_url(key),
+        for name, db in self.ds.databases.items():
+            table_counts = await db.table_counts(5)
+            views = await db.view_names()
+            tables = {}
+            hidden_table_names = set(await db.hidden_table_names())
+            for table in table_counts:
+                table_columns = await self.ds.table_columns(name, table)
+                tables[table] = {
+                    "name": table,
+                    "columns": table_columns,
+                    "primary_keys": await self.ds.execute_against_connection_in_thread(
+                        name, lambda conn: detect_primary_keys(conn, table)
+                    ),
+                    "count": table_counts[table],
+                    "hidden": table in hidden_table_names,
+                    "fts_table": await self.ds.execute_against_connection_in_thread(
+                        name, lambda conn: detect_fts(conn, table)
+                    ),
+                }
+            # Also mark as hidden any tables which start with the name of a hidden table
+            # e.g. "searchable_fts" implies "searchable_fts_content" should be hidden
+            for t in tables.keys():
+                for hidden_table in hidden_table_names:
+                    if t == hidden_table or t.startswith(hidden_table):
+                        tables[t]["hidden"] = True
+                        continue
+            hidden_tables = [t for t in tables.values() if t["hidden"]]
+
+            databases.append({
+                "name": name,
+                "hash": db.hash,
+                "color": db.hash[:6] if db.hash else hashlib.md5(name.encode("utf8")).hexdigest()[:6],
+                "path": self.database_url(name),
                 "tables_truncated": sorted(
-                    tables, key=lambda t: t["count"], reverse=True
+                    tables.values(), key=lambda t: t["count"] or 0, reverse=True
                 )[
                     :5
                 ],
                 "tables_count": len(tables),
                 "tables_more": len(tables) > 5,
-                "table_rows_sum": sum(t["count"] for t in tables),
+                "table_rows_sum": sum((t["count"] or 0) for t in tables.values()),
                 "hidden_table_rows_sum": sum(t["count"] for t in hidden_tables),
                 "hidden_tables_count": len(hidden_tables),
-                "views_count": len(info["views"]),
-            }
-            databases.append(database)
+                "views_count": len(views),
+            })
         if as_format:
             headers = {}
             if self.ds.cors:
@@ -45,7 +76,6 @@ class IndexView(RenderMixin):
                 content_type="application/json",
                 headers=headers,
             )
-
         else:
             return self.render(
                 ["index.html"],