From b9716d42780363f8e61ad475ac9cbe8e7c9c6658 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Fri, 10 Jul 2026 10:56:21 -0700 Subject: [PATCH] Add /{database}/-/editor-schema.json endpoint for SQL editor consumers Neutral {database, tables: [{name, view, columns: [{name, type}]}]} shape, gated on view-database + execute-sql with no table-name leak on 403, hidden tables excluded. /-/schema.json was already taken by the DDL endpoint, hence editor-schema.json. _editor_schema() now maps from the shared _schema_tables() introspection helper. Co-Authored-By: Claude Fable 5 --- datasette/app.py | 5 + datasette/views/query_helpers.py | 86 +++++++++++----- datasette/views/special.py | 53 ++++++++++ docs/json_api.rst | 54 ++++++++++ tests/test_schema_endpoints.py | 164 +++++++++++++++++++++++++++++++ 5 files changed, 338 insertions(+), 24 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 4ba5d20f..3d7037ac 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -85,6 +85,7 @@ from .views.special import ( JumpView, InstanceSchemaView, DatabaseSchemaView, + DatabaseEditorSchemaView, TableSchemaView, ) from .views.table import ( @@ -2716,6 +2717,10 @@ class Datasette: DatabaseSchemaView.as_view(self), r"/(?P[^\/\.]+)/-/schema(\.(?Pjson|md))?$", ) + add_route( + DatabaseEditorSchemaView.as_view(self), + r"/(?P[^\/\.]+)/-/editor-schema\.json$", + ) add_route( QueryParametersView.as_view(self), r"/(?P[^\/\.]+)/-/query/parameters$", diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 0e02f2eb..014026f5 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -649,6 +649,52 @@ def _column_completion(name, type_): return completion +async def _schema_tables(datasette, database_name, *, include_hidden=True): + """ + Neutral introspection of a database's tables and views for SQL editors. + + Returns an ordered list of dicts, one per table or view:: + + {"name": str, "view": bool, + "columns": [{"name": str, "type": str}, ...]} + + ``type`` is the SQLite declared column type (empty string when the column + has no declared type). Regular-table columns come from the internal + ``catalog_columns`` catalog; views are absent from that catalog so their + columns are read directly via PRAGMA table_xinfo. Hidden tables (FTS shadow + tables and the like) are excluded unless ``include_hidden`` is True. This is + the shared, serialization-agnostic source for both ``_editor_schema`` (which + maps it to lang-sql Completion objects) and the ``/-/editor-schema.json`` + endpoint (which emits it directly). + """ + internal_db = datasette.get_internal_database() + result = await internal_db.execute( + "select table_name, name, type from catalog_columns where database_name = ?", + [database_name], + ) + table_columns = {} + for row in result.rows: + table_columns.setdefault(row["table_name"], []).append( + {"name": row["name"], "type": row["type"]} + ) + db = datasette.get_database(database_name) + hidden = set() if include_hidden else set(await db.hidden_table_names()) + tables = [] + for table_name, columns in table_columns.items(): + if table_name in hidden: + continue + tables.append({"name": table_name, "view": False, "columns": columns}) + # Views are not represented in catalog_columns, so pull their real columns + # directly (PRAGMA table_xinfo works against views too). + for view_name in await db.view_names(): + columns = [ + {"name": column.name, "type": column.type} + for column in await db.table_column_details(view_name) + ] + tables.append({"name": view_name, "view": True, "columns": columns}) + return tables + + async def _editor_schema(datasette, database_name): """ Build a lang-sql SQLNamespace for the CodeMirror SQL editor autocomplete. @@ -659,29 +705,21 @@ async def _editor_schema(datasette, database_name): container so the popup can label them as views while still completing their real columns. See @codemirror/lang-sql >= 6.6 SQLNamespace / Completion. """ - internal_db = datasette.get_internal_database() - result = await internal_db.execute( - "select table_name, name, type from catalog_columns where database_name = ?", - [database_name], - ) schema = {} - for row in result.rows: - schema.setdefault(row["table_name"], []).append( - _column_completion(row["name"], row["type"]) - ) - # Views are not represented in catalog_columns, so pull their real columns - # directly (PRAGMA table_xinfo works against views too). - db = datasette.get_database(database_name) - for view_name in await db.view_names(): - columns = await db.table_column_details(view_name) - schema[view_name] = { - "self": { - "label": view_name, - "type": "class", - "detail": "view", - }, - "children": [ - _column_completion(column.name, column.type) for column in columns - ], - } + for table in await _schema_tables(datasette, database_name, include_hidden=True): + completions = [ + _column_completion(column["name"], column["type"]) + for column in table["columns"] + ] + if table["view"]: + schema[table["name"]] = { + "self": { + "label": table["name"], + "type": "class", + "detail": "view", + }, + "children": completions, + } + else: + schema[table["name"]] = completions return schema diff --git a/datasette/views/special.py b/datasette/views/special.py index c13191a1..c92ebc8f 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -1345,6 +1345,59 @@ class DatabaseSchemaView(SchemaBaseView): return await self.format_html_response(request, schemas) +class DatabaseEditorSchemaView(BaseView): + """ + JSON introspection of a database's tables, views and columns shaped for SQL + editor autocomplete consumers (the CodeMirror ```` + component and external clients such as datasette-paper). + + Distinct from :class:`DatabaseSchemaView` (``//-/schema.json``), which + returns the raw DDL as a SQL string gated on ``view-database`` alone. This + endpoint returns a neutral structured payload and is gated on both + ``view-database`` and ``execute-sql`` — the same permissions as the inline + editor schema handed to the SQL query page. + """ + + name = "database_editor_schema" + has_json_alternate = False + + async def get(self, request): + from .query_helpers import _schema_tables + + database_name = request.url_vars["database"] + + # view-database is checked first so actors without it cannot + # distinguish an existing database from a missing one, and a denied + # request only ever leaks the permission action name, never table names. + await self.ds.ensure_permission( + action="view-database", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ) + if database_name not in self.ds.databases: + headers = {} + if self.ds.cors: + add_cors_headers(headers) + return Response.json( + error_body("Database not found", 404), status=404, headers=headers + ) + await self.ds.ensure_permission( + action="execute-sql", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ) + + await self.ds.refresh_schemas() + tables = await _schema_tables(self.ds, database_name, include_hidden=False) + + headers = {} + if self.ds.cors: + add_cors_headers(headers) + return Response.json( + {"database": database_name, "tables": tables}, headers=headers + ) + + class TableSchemaView(SchemaBaseView): """ Displays schema for a specific table. diff --git a/docs/json_api.rst b/docs/json_api.rst index a96fd73d..b3eace7b 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -152,6 +152,60 @@ Values for named SQL parameters can be provided as additional query string param The response uses the same default representation described above. +.. _json_api_editor_schema: + +Schema for SQL editors +---------------------- + +The ``/-/editor-schema.json`` endpoint returns a machine-readable description of +a database's tables, views and columns, shaped for SQL editor autocomplete. It +powers Datasette's own CodeMirror SQL editor and is available for external +consumers such as embeddable editor components. + +:: + + GET //-/editor-schema.json + +Access requires both the :ref:`actions_view_database` and +:ref:`actions_execute_sql` permissions for the database - the same gate as the +inline editor schema on the SQL query page. A request that fails either check +receives a ``403`` JSON error that does not reveal any table or column names. + +The response is a neutral structure - a ``database`` name and a list of +``tables``, each with a ``view`` flag (``true`` for SQL views) and a list of +``columns`` carrying the SQLite declared ``type`` (an empty string when the +column has no declared type): + +.. code-block:: json + + { + "database": "fixtures", + "tables": [ + { + "name": "facetable", + "view": false, + "columns": [ + {"name": "pk", "type": "INTEGER"}, + {"name": "state", "type": "TEXT"} + ] + }, + { + "name": "paginated_view", + "view": true, + "columns": [ + {"name": "content", "type": "TEXT"} + ] + } + ] + } + +Hidden tables - such as the shadow tables that back SQLite full-text search - +are excluded from the response. + +This endpoint is distinct from the :ref:`database schema endpoint ` +at ``//-/schema.json``, which returns the raw ``CREATE`` statements as +a SQL string. + .. _json_api_shapes: Different shapes diff --git a/tests/test_schema_endpoints.py b/tests/test_schema_endpoints.py index c95d8614..f4fc8aa0 100644 --- a/tests/test_schema_endpoints.py +++ b/tests/test_schema_endpoints.py @@ -1,3 +1,5 @@ +import json + import pytest import pytest_asyncio from datasette.app import Datasette @@ -245,3 +247,165 @@ async def test_table_not_exists(schema_ds): response = await schema_ds.client.get("/schema_public_db/nonexistent/-/schema.md") assert response.status_code == 404 assert "not found" in response.text.lower() + + +# --------------------------------------------------------------------------- +# //-/editor-schema.json — neutral structured schema for SQL editors +# --------------------------------------------------------------------------- + + +@pytest_asyncio.fixture(scope="module") +async def editor_schema_ds(): + """Datasette instance exercising the editor-schema endpoint. + + - public db: tables + a view + an FTS table (hidden shadow tables) + - private db: gated behind view-database (allow root only) + - noexec db: view-database allowed for anyone, execute-sql denied + """ + ds = Datasette( + config={ + "databases": { + "editor_private_db": {"allow": {"id": "root"}}, + "editor_noexec_db": { + # Everyone may view the database, but nobody may run SQL + "allow_sql": {"id": "root"}, + }, + } + } + ) + + public_db = ds.add_memory_database("editor_public_db") + await public_db.execute_write( + "CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)" + ) + await public_db.execute_write( + "CREATE TABLE posts (id INTEGER PRIMARY KEY, title TEXT, body TEXT)" + ) + await public_db.execute_write( + "CREATE VIEW recent_posts AS SELECT id, title FROM posts ORDER BY id DESC" + ) + # An FTS table produces hidden shadow tables (users_fts_data, etc.) + await public_db.execute_write( + "CREATE VIRTUAL TABLE users_fts USING fts5(name, content=users)" + ) + + private_db = ds.add_memory_database("editor_private_db") + await private_db.execute_write( + "CREATE TABLE secret_data (id INTEGER PRIMARY KEY, value TEXT)" + ) + + noexec_db = ds.add_memory_database("editor_noexec_db") + await noexec_db.execute_write( + "CREATE TABLE locked (id INTEGER PRIMARY KEY, value TEXT)" + ) + + await ds.invoke_startup() + await ds.refresh_schemas() + return ds + + +@pytest.mark.asyncio +async def test_editor_schema_allowed_shape(editor_schema_ds): + """Authorized fetch returns tables, columns, types and views in the + documented neutral shape.""" + response = await editor_schema_ds.client.get( + "/editor_public_db/-/editor-schema.json" + ) + assert response.status_code == 200 + data = response.json() + assert data["database"] == "editor_public_db" + assert isinstance(data["tables"], list) + + by_name = {t["name"]: t for t in data["tables"]} + + # Regular table with columns + declared types + users = by_name["users"] + assert users["view"] is False + assert users["columns"] == [ + {"name": "id", "type": "INTEGER"}, + {"name": "name", "type": "TEXT"}, + ] + + posts = by_name["posts"] + assert posts["view"] is False + assert {c["name"] for c in posts["columns"]} == {"id", "title", "body"} + + # View is flagged and carries its real columns + view = by_name["recent_posts"] + assert view["view"] is True + assert [c["name"] for c in view["columns"]] == ["id", "title"] + + # Whole payload is JSON-serializable and every entry matches the shape + for table in data["tables"]: + assert set(table) == {"name", "view", "columns"} + for column in table["columns"]: + assert set(column) == {"name", "type"} + + +@pytest.mark.asyncio +async def test_editor_schema_excludes_hidden_tables(editor_schema_ds): + """FTS shadow tables (hidden_table_names) must not appear.""" + response = await editor_schema_ds.client.get( + "/editor_public_db/-/editor-schema.json" + ) + assert response.status_code == 200 + names = {t["name"] for t in response.json()["tables"]} + assert not any("_fts_" in name or name.endswith("_fts") for name in names), names + # Sanity: the visible objects are still there + assert {"users", "posts", "recent_posts"} <= names + + +@pytest.mark.asyncio +async def test_editor_schema_denied_view_database_403_no_leak(editor_schema_ds): + """Anonymous user lacking view-database gets a 403 that leaks no names.""" + response = await editor_schema_ds.client.get( + "/editor_private_db/-/editor-schema.json" + ) + assert response.status_code == 403 + body = response.text + assert "secret_data" not in body + data = response.json() + assert data["ok"] is False + assert "secret_data" not in json.dumps(data) + + # The permitted actor can read it + response = await editor_schema_ds.client.get( + "/editor_private_db/-/editor-schema.json", actor={"id": "root"} + ) + assert response.status_code == 200 + names = {t["name"] for t in response.json()["tables"]} + assert "secret_data" in names + + +@pytest.mark.asyncio +async def test_editor_schema_denied_execute_sql_403_no_leak(editor_schema_ds): + """A viewer who lacks execute-sql gets a 403 with no schema data.""" + # Anonymous user may view editor_noexec_db but not run SQL against it + response = await editor_schema_ds.client.get( + "/editor_noexec_db/-/editor-schema.json" + ) + assert response.status_code == 403 + data = response.json() + assert data["ok"] is False + assert "tables" not in data + assert "locked" not in json.dumps(data) + + # The actor granted execute-sql can read the schema + response = await editor_schema_ds.client.get( + "/editor_noexec_db/-/editor-schema.json", actor={"id": "root"} + ) + assert response.status_code == 200 + names = {t["name"] for t in response.json()["tables"]} + assert "locked" in names + + +@pytest.mark.asyncio +async def test_editor_schema_database_not_found(editor_schema_ds): + """A non-existent database returns a 404 JSON error.""" + response = await editor_schema_ds.client.get( + "/nonexistent_db/-/editor-schema.json" + ) + assert response.status_code == 404 + data = response.json() + assert data["ok"] is False + assert "not found" in data["error"].lower()