Add /{database}/-/editor-schema.json endpoint for SQL editor consumers

Neutral {database, tables: [{name, view, columns: [{name, type}]}]}
shape, gated on view-database + execute-sql with no table-name leak on
403, hidden tables excluded. /-/schema.json was already taken by the
DDL endpoint, hence editor-schema.json. _editor_schema() now maps from
the shared _schema_tables() introspection helper.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

datasette/app.py

@@ -85,6 +85,7 @@ from .views.special import (
     JumpView,
     InstanceSchemaView,
     DatabaseSchemaView,
+    DatabaseEditorSchemaView,
     TableSchemaView,
 )
 from .views.table import (
@@ -2716,6 +2717,10 @@ class Datasette:
             DatabaseSchemaView.as_view(self),
             r"/(?P<database>[^\/\.]+)/-/schema(\.(?P<format>json|md))?$",
         )
+        add_route(
+            DatabaseEditorSchemaView.as_view(self),
+            r"/(?P<database>[^\/\.]+)/-/editor-schema\.json$",
+        )
         add_route(
             QueryParametersView.as_view(self),
             r"/(?P<database>[^\/\.]+)/-/query/parameters$",

datasette/views/query_helpers.py

@@ -649,6 +649,52 @@ def _column_completion(name, type_):
     return completion
 
 
+async def _schema_tables(datasette, database_name, *, include_hidden=True):
+    """
+    Neutral introspection of a database's tables and views for SQL editors.
+
+    Returns an ordered list of dicts, one per table or view::
+
+        {"name": str, "view": bool,
+         "columns": [{"name": str, "type": str}, ...]}
+
+    ``type`` is the SQLite declared column type (empty string when the column
+    has no declared type). Regular-table columns come from the internal
+    ``catalog_columns`` catalog; views are absent from that catalog so their
+    columns are read directly via PRAGMA table_xinfo. Hidden tables (FTS shadow
+    tables and the like) are excluded unless ``include_hidden`` is True. This is
+    the shared, serialization-agnostic source for both ``_editor_schema`` (which
+    maps it to lang-sql Completion objects) and the ``/-/editor-schema.json``
+    endpoint (which emits it directly).
+    """
+    internal_db = datasette.get_internal_database()
+    result = await internal_db.execute(
+        "select table_name, name, type from catalog_columns where database_name = ?",
+        [database_name],
+    )
+    table_columns = {}
+    for row in result.rows:
+        table_columns.setdefault(row["table_name"], []).append(
+            {"name": row["name"], "type": row["type"]}
+        )
+    db = datasette.get_database(database_name)
+    hidden = set() if include_hidden else set(await db.hidden_table_names())
+    tables = []
+    for table_name, columns in table_columns.items():
+        if table_name in hidden:
+            continue
+        tables.append({"name": table_name, "view": False, "columns": columns})
+    # Views are not represented in catalog_columns, so pull their real columns
+    # directly (PRAGMA table_xinfo works against views too).
+    for view_name in await db.view_names():
+        columns = [
+            {"name": column.name, "type": column.type}
+            for column in await db.table_column_details(view_name)
+        ]
+        tables.append({"name": view_name, "view": True, "columns": columns})
+    return tables
+
+
 async def _editor_schema(datasette, database_name):
     """
     Build a lang-sql SQLNamespace for the CodeMirror SQL editor autocomplete.
@@ -659,29 +705,21 @@ async def _editor_schema(datasette, database_name):
     container so the popup can label them as views while still completing their
     real columns. See @codemirror/lang-sql >= 6.6 SQLNamespace / Completion.
     """
-    internal_db = datasette.get_internal_database()
-    result = await internal_db.execute(
-        "select table_name, name, type from catalog_columns where database_name = ?",
-        [database_name],
-    )
     schema = {}
-    for row in result.rows:
-        schema.setdefault(row["table_name"], []).append(
-            _column_completion(row["name"], row["type"])
-        )
-    # Views are not represented in catalog_columns, so pull their real columns
-    # directly (PRAGMA table_xinfo works against views too).
-    db = datasette.get_database(database_name)
-    for view_name in await db.view_names():
-        columns = await db.table_column_details(view_name)
-        schema[view_name] = {
-            "self": {
-                "label": view_name,
-                "type": "class",
-                "detail": "view",
-            },
-            "children": [
-                _column_completion(column.name, column.type) for column in columns
-            ],
-        }
+    for table in await _schema_tables(datasette, database_name, include_hidden=True):
+        completions = [
+            _column_completion(column["name"], column["type"])
+            for column in table["columns"]
+        ]
+        if table["view"]:
+            schema[table["name"]] = {
+                "self": {
+                    "label": table["name"],
+                    "type": "class",
+                    "detail": "view",
+                },
+                "children": completions,
+            }
+        else:
+            schema[table["name"]] = completions
     return schema

datasette/views/special.py

@@ -1345,6 +1345,59 @@ class DatabaseSchemaView(SchemaBaseView):
             return await self.format_html_response(request, schemas)
 
 
+class DatabaseEditorSchemaView(BaseView):
+    """
+    JSON introspection of a database's tables, views and columns shaped for SQL
+    editor autocomplete consumers (the CodeMirror ``<datasette-sql-editor>``
+    component and external clients such as datasette-paper).
+
+    Distinct from :class:`DatabaseSchemaView` (``/<db>/-/schema.json``), which
+    returns the raw DDL as a SQL string gated on ``view-database`` alone. This
+    endpoint returns a neutral structured payload and is gated on both
+    ``view-database`` and ``execute-sql`` — the same permissions as the inline
+    editor schema handed to the SQL query page.
+    """
+
+    name = "database_editor_schema"
+    has_json_alternate = False
+
+    async def get(self, request):
+        from .query_helpers import _schema_tables
+
+        database_name = request.url_vars["database"]
+
+        # view-database is checked first so actors without it cannot
+        # distinguish an existing database from a missing one, and a denied
+        # request only ever leaks the permission action name, never table names.
+        await self.ds.ensure_permission(
+            action="view-database",
+            resource=DatabaseResource(database=database_name),
+            actor=request.actor,
+        )
+        if database_name not in self.ds.databases:
+            headers = {}
+            if self.ds.cors:
+                add_cors_headers(headers)
+            return Response.json(
+                error_body("Database not found", 404), status=404, headers=headers
+            )
+        await self.ds.ensure_permission(
+            action="execute-sql",
+            resource=DatabaseResource(database=database_name),
+            actor=request.actor,
+        )
+
+        await self.ds.refresh_schemas()
+        tables = await _schema_tables(self.ds, database_name, include_hidden=False)
+
+        headers = {}
+        if self.ds.cors:
+            add_cors_headers(headers)
+        return Response.json(
+            {"database": database_name, "tables": tables}, headers=headers
+        )
+
+
 class TableSchemaView(SchemaBaseView):
     """
     Displays schema for a specific table.

docs/json_api.rst

@@ -152,6 +152,60 @@ Values for named SQL parameters can be provided as additional query string param
 
 The response uses the same default representation described above.
 
+.. _json_api_editor_schema:
+
+Schema for SQL editors
+----------------------
+
+The ``/-/editor-schema.json`` endpoint returns a machine-readable description of
+a database's tables, views and columns, shaped for SQL editor autocomplete. It
+powers Datasette's own CodeMirror SQL editor and is available for external
+consumers such as embeddable editor components.
+
+::
+
+    GET /<database>/-/editor-schema.json
+
+Access requires both the :ref:`actions_view_database` and
+:ref:`actions_execute_sql` permissions for the database - the same gate as the
+inline editor schema on the SQL query page. A request that fails either check
+receives a ``403`` JSON error that does not reveal any table or column names.
+
+The response is a neutral structure - a ``database`` name and a list of
+``tables``, each with a ``view`` flag (``true`` for SQL views) and a list of
+``columns`` carrying the SQLite declared ``type`` (an empty string when the
+column has no declared type):
+
+.. code-block:: json
+
+    {
+      "database": "fixtures",
+      "tables": [
+        {
+          "name": "facetable",
+          "view": false,
+          "columns": [
+            {"name": "pk", "type": "INTEGER"},
+            {"name": "state", "type": "TEXT"}
+          ]
+        },
+        {
+          "name": "paginated_view",
+          "view": true,
+          "columns": [
+            {"name": "content", "type": "TEXT"}
+          ]
+        }
+      ]
+    }
+
+Hidden tables - such as the shadow tables that back SQLite full-text search -
+are excluded from the response.
+
+This endpoint is distinct from the :ref:`database schema endpoint <DatabaseSchemaView>`
+at ``/<database>/-/schema.json``, which returns the raw ``CREATE`` statements as
+a SQL string.
+
 .. _json_api_shapes:
 
 Different shapes

tests/test_schema_endpoints.py

@@ -1,3 +1,5 @@
+import json
+
 import pytest
 import pytest_asyncio
 from datasette.app import Datasette
@@ -245,3 +247,165 @@ async def test_table_not_exists(schema_ds):
     response = await schema_ds.client.get("/schema_public_db/nonexistent/-/schema.md")
     assert response.status_code == 404
     assert "not found" in response.text.lower()
+
+
+# ---------------------------------------------------------------------------
+# /<database>/-/editor-schema.json — neutral structured schema for SQL editors
+# ---------------------------------------------------------------------------
+
+
+@pytest_asyncio.fixture(scope="module")
+async def editor_schema_ds():
+    """Datasette instance exercising the editor-schema endpoint.
+
+    - public db: tables + a view + an FTS table (hidden shadow tables)
+    - private db: gated behind view-database (allow root only)
+    - noexec db: view-database allowed for anyone, execute-sql denied
+    """
+    ds = Datasette(
+        config={
+            "databases": {
+                "editor_private_db": {"allow": {"id": "root"}},
+                "editor_noexec_db": {
+                    # Everyone may view the database, but nobody may run SQL
+                    "allow_sql": {"id": "root"},
+                },
+            }
+        }
+    )
+
+    public_db = ds.add_memory_database("editor_public_db")
+    await public_db.execute_write(
+        "CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)"
+    )
+    await public_db.execute_write(
+        "CREATE TABLE posts (id INTEGER PRIMARY KEY, title TEXT, body TEXT)"
+    )
+    await public_db.execute_write(
+        "CREATE VIEW recent_posts AS SELECT id, title FROM posts ORDER BY id DESC"
+    )
+    # An FTS table produces hidden shadow tables (users_fts_data, etc.)
+    await public_db.execute_write(
+        "CREATE VIRTUAL TABLE users_fts USING fts5(name, content=users)"
+    )
+
+    private_db = ds.add_memory_database("editor_private_db")
+    await private_db.execute_write(
+        "CREATE TABLE secret_data (id INTEGER PRIMARY KEY, value TEXT)"
+    )
+
+    noexec_db = ds.add_memory_database("editor_noexec_db")
+    await noexec_db.execute_write(
+        "CREATE TABLE locked (id INTEGER PRIMARY KEY, value TEXT)"
+    )
+
+    await ds.invoke_startup()
+    await ds.refresh_schemas()
+    return ds
+
+
+@pytest.mark.asyncio
+async def test_editor_schema_allowed_shape(editor_schema_ds):
+    """Authorized fetch returns tables, columns, types and views in the
+    documented neutral shape."""
+    response = await editor_schema_ds.client.get(
+        "/editor_public_db/-/editor-schema.json"
+    )
+    assert response.status_code == 200
+    data = response.json()
+    assert data["database"] == "editor_public_db"
+    assert isinstance(data["tables"], list)
+
+    by_name = {t["name"]: t for t in data["tables"]}
+
+    # Regular table with columns + declared types
+    users = by_name["users"]
+    assert users["view"] is False
+    assert users["columns"] == [
+        {"name": "id", "type": "INTEGER"},
+        {"name": "name", "type": "TEXT"},
+    ]
+
+    posts = by_name["posts"]
+    assert posts["view"] is False
+    assert {c["name"] for c in posts["columns"]} == {"id", "title", "body"}
+
+    # View is flagged and carries its real columns
+    view = by_name["recent_posts"]
+    assert view["view"] is True
+    assert [c["name"] for c in view["columns"]] == ["id", "title"]
+
+    # Whole payload is JSON-serializable and every entry matches the shape
+    for table in data["tables"]:
+        assert set(table) == {"name", "view", "columns"}
+        for column in table["columns"]:
+            assert set(column) == {"name", "type"}
+
+
+@pytest.mark.asyncio
+async def test_editor_schema_excludes_hidden_tables(editor_schema_ds):
+    """FTS shadow tables (hidden_table_names) must not appear."""
+    response = await editor_schema_ds.client.get(
+        "/editor_public_db/-/editor-schema.json"
+    )
+    assert response.status_code == 200
+    names = {t["name"] for t in response.json()["tables"]}
+    assert not any("_fts_" in name or name.endswith("_fts") for name in names), names
+    # Sanity: the visible objects are still there
+    assert {"users", "posts", "recent_posts"} <= names
+
+
+@pytest.mark.asyncio
+async def test_editor_schema_denied_view_database_403_no_leak(editor_schema_ds):
+    """Anonymous user lacking view-database gets a 403 that leaks no names."""
+    response = await editor_schema_ds.client.get(
+        "/editor_private_db/-/editor-schema.json"
+    )
+    assert response.status_code == 403
+    body = response.text
+    assert "secret_data" not in body
+    data = response.json()
+    assert data["ok"] is False
+    assert "secret_data" not in json.dumps(data)
+
+    # The permitted actor can read it
+    response = await editor_schema_ds.client.get(
+        "/editor_private_db/-/editor-schema.json", actor={"id": "root"}
+    )
+    assert response.status_code == 200
+    names = {t["name"] for t in response.json()["tables"]}
+    assert "secret_data" in names
+
+
+@pytest.mark.asyncio
+async def test_editor_schema_denied_execute_sql_403_no_leak(editor_schema_ds):
+    """A viewer who lacks execute-sql gets a 403 with no schema data."""
+    # Anonymous user may view editor_noexec_db but not run SQL against it
+    response = await editor_schema_ds.client.get(
+        "/editor_noexec_db/-/editor-schema.json"
+    )
+    assert response.status_code == 403
+    data = response.json()
+    assert data["ok"] is False
+    assert "tables" not in data
+    assert "locked" not in json.dumps(data)
+
+    # The actor granted execute-sql can read the schema
+    response = await editor_schema_ds.client.get(
+        "/editor_noexec_db/-/editor-schema.json", actor={"id": "root"}
+    )
+    assert response.status_code == 200
+    names = {t["name"] for t in response.json()["tables"]}
+    assert "locked" in names
+
+
+@pytest.mark.asyncio
+async def test_editor_schema_database_not_found(editor_schema_ds):
+    """A non-existent database returns a 404 JSON error."""
+    response = await editor_schema_ds.client.get(
+        "/nonexistent_db/-/editor-schema.json"
+    )
+    assert response.status_code == 404
+    data = response.json()
+    assert data["ok"] is False
+    assert "not found" in data["error"].lower()