diff --git a/datasette/app.py b/datasette/app.py index 9bc84df0..cc8506e2 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -74,6 +74,7 @@ from .utils.asgi import ( asgi_send_json, asgi_send_redirect, ) +from .utils.schemas import init_schemas, populate_schema_tables from .utils.sqlite import ( sqlite3, using_pysqlite3, @@ -222,6 +223,11 @@ class Datasette: elif memory: self.files = (MEMORY,) + self.files self.databases = collections.OrderedDict() + # memory_name is a random string so that each Datasette instance gets its own + # unique in-memory named database - otherwise unit tests can fail with weird + # errors when different instances accidentally share an in-memory database + self.add_database("_schemas", Database(self, memory_name=secrets.token_hex())) + self._schemas_created = False for file in self.files: path = file is_memory = False @@ -326,6 +332,33 @@ class Datasette: self._root_token = secrets.token_hex(32) self.client = DatasetteClient(self) + async def refresh_schemas(self): + schema_db = self.databases["_schemas"] + if not self._schemas_created: + await init_schemas(schema_db) + self._schemas_created = True + + current_schema_versions = { + row["database_name"]: row["schema_version"] + for row in await schema_db.execute( + "select database_name, schema_version from databases" + ) + } + for database_name, db in self.databases.items(): + schema_version = (await db.execute("PRAGMA schema_version")).first()[0] + # Compare schema versions to see if we should skip it + if schema_version == current_schema_versions.get(database_name): + continue + await schema_db.execute_write( + """ + INSERT OR REPLACE INTO databases (database_name, path, is_memory, schema_version) + VALUES (?, ?, ?, ?) + """, + [database_name, db.path, db.is_memory, schema_version], + block=True, + ) + await populate_schema_tables(schema_db, db) + @property def urls(self): return Urls(self) @@ -342,7 +375,8 @@ class Datasette: def get_database(self, name=None): if name is None: - return next(iter(self.databases.values())) + # Return first no-_schemas database + name = [key for key in self.databases.keys() if key != "_schemas"][0] return self.databases[name] def add_database(self, name, db): @@ -590,7 +624,8 @@ class Datasette: "is_memory": d.is_memory, "hash": d.hash, } - for d in sorted(self.databases.values(), key=lambda d: d.name) + for name, d in sorted(self.databases.items(), key=lambda p: p[1].name) + if name != "_schemas" ] def _versions(self): diff --git a/datasette/cli.py b/datasette/cli.py index 32408d23..50367fb3 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -134,6 +134,9 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): + if name == "_schemas": + # Don't include the in-memory _schemas database + continue counts = await database.table_counts(limit=3600 * 1000) data[name] = { "hash": database.hash, diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 9f1d9c62..62cab83a 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -13,6 +13,8 @@ def permission_allowed(datasette, actor, action, resource): if allow is not None: return actor_matches_allow(actor, allow) elif action == "view-database": + if resource == "_schemas" and (actor is None or actor.get("id") != "root"): + return False database_allow = datasette.metadata("allow", database=resource) if database_allow is None: return None diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 2576090a..ac1d82f7 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1024,11 +1024,12 @@ def find_spatialite(): async def initial_path_for_datasette(datasette): "Return suggested path for opening this Datasette, based on number of DBs and tables" - if len(datasette.databases) == 1: - db_name = next(iter(datasette.databases.keys())) + databases = dict([p for p in datasette.databases.items() if p[0] != "_schemas"]) + if len(databases) == 1: + db_name = next(iter(databases.keys())) path = datasette.urls.database(db_name) # Does this DB only have one table? - db = next(iter(datasette.databases.values())) + db = next(iter(databases.values())) tables = await db.table_names() if len(tables) == 1: path = datasette.urls.table(db_name, tables[0]) diff --git a/datasette/utils/schemas.py b/datasette/utils/schemas.py new file mode 100644 index 00000000..4612e236 --- /dev/null +++ b/datasette/utils/schemas.py @@ -0,0 +1,162 @@ +async def init_schemas(db): + await db.execute_write( + """ + CREATE TABLE databases ( + "database_name" TEXT PRIMARY KEY, + "path" TEXT, + "is_memory" INTEGER, + "schema_version" INTEGER + ) + """, + block=True, + ) + await db.execute_write( + """ + CREATE TABLE tables ( + "database_name" TEXT, + "table_name" TEXT, + "rootpage" INTEGER, + "sql" TEXT, + PRIMARY KEY (database_name, table_name) + ) + """, + block=True, + ) + await db.execute_write( + """ + CREATE TABLE columns ( + "database_name" TEXT, + "table_name" TEXT, + "cid" INTEGER, + "name" TEXT, + "type" TEXT, + "notnull" INTEGER, + "default_value" TEXT, -- renamed from dflt_value + "is_pk" INTEGER, -- renamed from pk + "hidden" INTEGER, + PRIMARY KEY (database_name, table_name, name) + ) + """, + block=True, + ) + await db.execute_write( + """ + CREATE TABLE indexes ( + "database_name" TEXT, + "table_name" TEXT, + "seq" INTEGER, + "name" TEXT, + "unique" INTEGER, + "origin" TEXT, + "partial" INTEGER, + PRIMARY KEY (database_name, table_name, name) + ) + """, + block=True, + ) + await db.execute_write( + """ + CREATE TABLE foreign_keys ( + "database_name" TEXT, + "table_name" TEXT, + "id" INTEGER, + "seq" INTEGER, + "table" TEXT, + "from" TEXT, + "to" TEXT, + "on_update" TEXT, + "on_delete" TEXT, + "match" TEXT + ) + """, + block=True, + ) + + +async def populate_schema_tables(schema_db, db): + database_name = db.name + await schema_db.execute_write( + "delete from tables where database_name = ?", [database_name], block=True + ) + tables = (await db.execute("select * from sqlite_master where type = 'table'")).rows + for table in tables: + table_name = table["name"] + await schema_db.execute_write( + """ + insert into tables (database_name, table_name, rootpage, sql) + values (?, ?, ?, ?) + """, + [database_name, table_name, table["rootpage"], table["sql"]], + block=True, + ) + # And the columns + await schema_db.execute_write( + "delete from columns where database_name = ? and table_name = ?", + [database_name, table_name], + block=True, + ) + columns = await db.table_column_details(table_name) + for column in columns: + params = { + **{"database_name": database_name, "table_name": table_name}, + **column._asdict(), + } + await schema_db.execute_write( + """ + insert into columns ( + database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden + ) VALUES ( + :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden + ) + """, + params, + block=True, + ) + # And the foreign_keys + await schema_db.execute_write( + "delete from foreign_keys where database_name = ? and table_name = ?", + [database_name, table_name], + block=True, + ) + foreign_keys = ( + await db.execute(f"PRAGMA foreign_key_list([{table_name}])") + ).rows + for foreign_key in foreign_keys: + params = { + **{"database_name": database_name, "table_name": table_name}, + **dict(foreign_key), + } + await schema_db.execute_write( + """ + insert into foreign_keys ( + database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match + ) VALUES ( + :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match + ) + """, + params, + block=True, + ) + # And the indexes + await schema_db.execute_write( + "delete from indexes where database_name = ? and table_name = ?", + [database_name, table_name], + block=True, + ) + indexes = (await db.execute(f"PRAGMA index_list([{table_name}])")).rows + for index in indexes: + params = { + **{"database_name": database_name, "table_name": table_name}, + **dict(index), + } + await schema_db.execute_write( + """ + insert into indexes ( + database_name, table_name, seq, name, "unique", origin, partial + ) VALUES ( + :database_name, :table_name, :seq, :name, :unique, :origin, :partial + ) + """, + params, + block=True, + ) diff --git a/datasette/views/base.py b/datasette/views/base.py index 76e03206..73bf9459 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -115,6 +115,8 @@ class BaseView: return Response.text("Method not allowed", status=405) async def dispatch_request(self, request, *args, **kwargs): + if self.ds: + await self.ds.refresh_schemas() handler = getattr(self, request.method.lower(), None) return await handler(request, *args, **kwargs) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 93b444ab..61e7d4b5 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -293,7 +293,7 @@ def test_hook_extra_body_script(app_client, path, expected_extra_body_script): def test_hook_asgi_wrapper(app_client): response = app_client.get("/fixtures") - assert "fixtures" == response.headers["x-databases"] + assert "_schemas, fixtures" == response.headers["x-databases"] def test_hook_extra_template_vars(restore_working_directory): diff --git a/tests/test_schemas.py b/tests/test_schemas.py new file mode 100644 index 00000000..87656784 --- /dev/null +++ b/tests/test_schemas.py @@ -0,0 +1,68 @@ +from .fixtures import app_client +import pytest + + +def test_schemas_only_available_to_root(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + assert app_client.get("/_schemas").status == 403 + assert app_client.get("/_schemas", cookies={"ds_actor": cookie}).status == 200 + + +def test_schemas_databases(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + databases = app_client.get( + "/_schemas/databases.json?_shape=array", cookies={"ds_actor": cookie} + ).json + assert len(databases) == 2 + assert databases[0]["database_name"] == "_schemas" + assert databases[1]["database_name"] == "fixtures" + + +def test_schemas_tables(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + tables = app_client.get( + "/_schemas/tables.json?_shape=array", cookies={"ds_actor": cookie} + ).json + assert len(tables) > 5 + table = tables[0] + assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"} + + +def test_schemas_indexes(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + indexes = app_client.get( + "/_schemas/indexes.json?_shape=array", cookies={"ds_actor": cookie} + ).json + assert len(indexes) > 5 + index = indexes[0] + assert set(index.keys()) == { + "partial", + "name", + "table_name", + "unique", + "seq", + "database_name", + "origin", + } + + +def test_schemas_foreign_keys(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + foreign_keys = app_client.get( + "/_schemas/foreign_keys.json?_shape=array", cookies={"ds_actor": cookie} + ).json + assert len(foreign_keys) > 5 + foreign_key = foreign_keys[0] + assert set(foreign_key.keys()) == { + "table", + "seq", + "on_update", + "on_delete", + "to", + "rowid", + "id", + "match", + "database_name", + "table_name", + "from", + }