Add datasette-sql-editor ESM primitives module

Single source of truth for the CodeMirror setup: SQLiteDialect,
createSqlEditor() (delegable history with host undo/redo forwarding,
hostChange annotation for echo suppression, submit/escape callbacks,
fixed-tooltip mode, per-editor Compartment updateSchema), and
datasetteSchema() which fetches /-/editor-schema.json and maps it to a
lang-sql SQLNamespace identical to the server-inlined shape.
cm-editor.js is now a thin consumer; rollup emits both the IIFE and an
importable ESM bundle. Submit key is Mod-Enter (Cmd on mac as before,
now also Ctrl elsewhere) plus Shift-Enter.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

datasette/static/cm-editor.js

@@ -1,74 +1,36 @@
-import { EditorView, basicSetup } from "codemirror";
-import { Compartment } from "@codemirror/state";
-import { keymap } from "@codemirror/view";
-import { sql, SQLDialect } from "@codemirror/lang-sql";
+// IIFE entry point for Datasette's own SQL editor pages. Built as
+// cm-editor.bundle.js (global name `cm`) and included by _codemirror.html.
+//
+// This is a thin consumer of the datasette-sql-editor.js primitives so there is
+// a single CodeMirror implementation. rollup inlines the shared module into this
+// bundle.
+import { createSqlEditor, SQLiteDialect } from "./datasette-sql-editor.js";
 
-// A variation of SQLite from lang-sql https://github.com/codemirror/lang-sql/blob/ebf115fffdbe07f91465ccbd82868c587f8182bc/src/sql.ts#L231
-const SQLite = SQLDialect.define({
-  // Based on https://www.sqlite.org/lang_keywords.html based on likely keywords to be used in select queries
-  // https://github.com/simonw/datasette/pull/1893#issuecomment-1316401895:
-  keywords:
-    "and as asc between by case cast count current_date current_time current_timestamp desc distinct each else escape except exists explain filter first for from full generated group having if in index inner intersect into isnull join last left like limit not null or order outer over pragma primary query raise range regexp right rollback row select set table then to union unique using values view virtual when where",
-  // https://www.sqlite.org/datatype3.html
-  types: "null integer real text blob",
-  builtin: "",
-  operatorChars: "*+-%<>!=&|/~",
-  identifierQuotes: '`"',
-  specialVar: "@:?$",
-  caseInsensitiveIdentifiers: true,
-});
+// Re-exported so plugins/pages using the IIFE global can reach the curated
+// dialect (cm.SQLiteDialect) without a second CodeMirror instance.
+export { SQLiteDialect };
 
-// Builds the sql() extension from a {schema, defaultTable, defaultSchema} conf object
-function sqlExtension(conf) {
-  return sql({
-    dialect: SQLite,
+// Utility function from https://codemirror.net/docs/migration/. Wraps a textarea
+// with a CodeMirror SQL editor, mirroring the textarea's value back on submit.
+// Returns the EditorView (with an added updateSchema method) for backwards
+// compatibility with existing callers (window.editor).
+export function editorFromTextArea(textarea, conf = {}) {
+  const submit = (view) => {
+    textarea.value = view.state.doc.toString();
+    textarea.form.submit();
+  };
+
+  const handle = createSqlEditor(null, {
+    doc: textarea.value,
     schema: conf.schema,
     defaultTable: conf.defaultTable,
     defaultSchema: conf.defaultSchema,
+    onSubmit: submit,
   });
-}
+  const view = handle.view;
 
-// Utility function from https://codemirror.net/docs/migration/
-export function editorFromTextArea(textarea, conf = {}) {
-  // Wraps the sql() extension so it can be swapped out later via view.updateSchema()
-  // https://codemirror.net/examples/config/#dynamic-configuration
-  let sqlCompartment = new Compartment();
-
-  let view = new EditorView({
-    doc: textarea.value,
-    extensions: [
-      keymap.of([
-        {
-          key: "Shift-Enter",
-          run: function () {
-            textarea.value = view.state.doc.toString();
-            textarea.form.submit();
-            return true;
-          },
-        },
-        {
-          key: "Meta-Enter",
-          run: function () {
-            textarea.value = view.state.doc.toString();
-            textarea.form.submit();
-            return true;
-          },
-        },
-      ]),
-      // This has to be after the keymap or else the basicSetup keys will prevent
-      // Meta-Enter from running
-      basicSetup,
-      EditorView.lineWrapping,
-      sqlCompartment.of(sqlExtension(conf)),
-    ],
-  });
-
-  // Allows callers (and plugins) to update the schema/defaultTable/defaultSchema
-  // used for autocomplete after the editor has already been created.
-  view.updateSchema = (conf2) =>
-    view.dispatch({
-      effects: sqlCompartment.reconfigure(sqlExtension(conf2)),
-    });
+  // Preserve the historical public surface: callers use view.updateSchema(conf).
+  view.updateSchema = handle.updateSchema;
 
   // Idea taken from https://discuss.codemirror.net/t/resizing-codemirror-6/3265.
   // Using CSS resize: both and scheduling a measurement when the element changes.

datasette/static/datasette-sql-editor.js

@@ -0,0 +1,326 @@
+// datasette-sql-editor: ESM primitives for embedding Datasette's SQL editor.
+//
+// This is the single source of truth for Datasette's CodeMirror setup. The IIFE
+// entry point (cm-editor.js, served as cm-editor.bundle.js for Datasette's own
+// pages) is a thin consumer of these primitives, and plugin authors can import
+// this module directly from /-/static/datasette-sql-editor.js to get a SQL
+// editor that shares ONE CodeMirror instance per page (no duplicate
+// @codemirror/state bug).
+//
+// Built by rollup.config.mjs into datasette-sql-editor.bundle.js.
+
+import {
+  EditorView,
+  keymap,
+  lineNumbers,
+  highlightActiveLineGutter,
+  highlightSpecialChars,
+  drawSelection,
+  dropCursor,
+  rectangularSelection,
+  crosshairCursor,
+  highlightActiveLine,
+  tooltips,
+} from "@codemirror/view";
+import { EditorState, Compartment, Annotation, Prec } from "@codemirror/state";
+import {
+  foldGutter,
+  indentOnInput,
+  syntaxHighlighting,
+  defaultHighlightStyle,
+  bracketMatching,
+  foldKeymap,
+} from "@codemirror/language";
+import { history, defaultKeymap, historyKeymap } from "@codemirror/commands";
+import { highlightSelectionMatches, searchKeymap } from "@codemirror/search";
+import {
+  closeBrackets,
+  autocompletion,
+  closeBracketsKeymap,
+  completionKeymap,
+} from "@codemirror/autocomplete";
+import { lintKeymap } from "@codemirror/lint";
+import { sql, SQLDialect } from "@codemirror/lang-sql";
+
+// A curated variation of SQLite from lang-sql:
+// https://github.com/codemirror/lang-sql/blob/ebf115fffdbe07f91465ccbd82868c587f8182bc/src/sql.ts#L231
+export const SQLiteDialect = SQLDialect.define({
+  // Based on https://www.sqlite.org/lang_keywords.html, restricted to likely
+  // keywords used in select queries.
+  // https://github.com/simonw/datasette/pull/1893#issuecomment-1316401895:
+  keywords:
+    "and as asc between by case cast count current_date current_time current_timestamp desc distinct each else escape except exists explain filter first for from full generated group having if in index inner intersect into isnull join last left like limit not null or order outer over pragma primary query raise range regexp right rollback row select set table then to union unique using values view virtual when where",
+  // https://www.sqlite.org/datatype3.html
+  types: "null integer real text blob",
+  builtin: "",
+  operatorChars: "*+-%<>!=&|/~",
+  identifierQuotes: '`"',
+  specialVar: "@:?$",
+  caseInsensitiveIdentifiers: true,
+});
+
+// Annotation used to tag host-originated changes (e.g. a ProseMirror/collab host
+// pushing edits into the editor, or the exported `value` setter). Changes tagged
+// with this annotation do NOT re-fire onChange, so hosts can suppress the echo of
+// their own edits. Mirrors datasette-paper's `fromPM` pattern.
+export const hostChange = Annotation.define();
+
+// Builds the sql() language extension from a {schema, defaultTable, defaultSchema}
+// conf object. Undefined fields are fine - lang-sql ignores them.
+function sqlExtension(conf = {}) {
+  return sql({
+    dialect: SQLiteDialect,
+    schema: conf.schema,
+    defaultTable: conf.defaultTable,
+    defaultSchema: conf.defaultSchema,
+  });
+}
+
+// Replicates codemirror's basicSetup (node_modules/codemirror/dist/index.js) as a
+// plain array so we can drop the undo history when `withHistory` is false. When
+// history is off we optionally forward Mod-z / Mod-y / Mod-Shift-z to the host so
+// an external undo stack (ProseMirror, collab) can own undo/redo.
+function baseSetup(withHistory, onHostUndo, onHostRedo) {
+  const setup = [
+    lineNumbers(),
+    highlightActiveLineGutter(),
+    highlightSpecialChars(),
+    foldGutter(),
+    drawSelection(),
+    dropCursor(),
+    EditorState.allowMultipleSelections.of(true),
+    indentOnInput(),
+    syntaxHighlighting(defaultHighlightStyle, { fallback: true }),
+    bracketMatching(),
+    closeBrackets(),
+    autocompletion(),
+    rectangularSelection(),
+    crosshairCursor(),
+    highlightActiveLine(),
+    highlightSelectionMatches(),
+  ];
+  const bindings = [
+    ...closeBracketsKeymap,
+    ...defaultKeymap,
+    ...searchKeymap,
+    ...foldKeymap,
+    ...completionKeymap,
+    ...lintKeymap,
+  ];
+  if (withHistory) {
+    setup.push(history());
+    bindings.push(...historyKeymap);
+  } else {
+    if (onHostUndo) {
+      bindings.push({
+        key: "Mod-z",
+        preventDefault: true,
+        run: () => {
+          onHostUndo();
+          return true;
+        },
+      });
+    }
+    if (onHostRedo) {
+      bindings.push(
+        {
+          key: "Mod-y",
+          mac: "Mod-Shift-z",
+          preventDefault: true,
+          run: () => {
+            onHostRedo();
+            return true;
+          },
+        },
+        {
+          key: "Mod-Shift-z",
+          preventDefault: true,
+          run: () => {
+            onHostRedo();
+            return true;
+          },
+        },
+      );
+    }
+  }
+  setup.push(keymap.of(bindings));
+  return setup;
+}
+
+// createSqlEditor(parent, opts) -> handle
+//
+// opts:
+//   doc            initial document string (default "")
+//   schema         lang-sql SQLNamespace for autocomplete
+//   defaultTable   unqualified-column default table
+//   defaultSchema  default schema/attached-database name
+//   history        include CM undo history (default true); false forwards
+//                  undo/redo to onHostUndo/onHostRedo
+//   onHostUndo     called on Mod-z when history is false
+//   onHostRedo     called on Mod-y / Mod-Shift-z when history is false
+//   extensions     extra CodeMirror extensions to append (default [])
+//   fixedTooltips  use position:"fixed" tooltips for overflow-clipped containers
+//   onChange       called (update) on user edits; host-annotated changes are
+//                  suppressed
+//   onSubmit       called (view) on Mod-Enter / Shift-Enter (highest precedence)
+//   onEscape       called (view) on Escape
+//   lineWrapping   soft-wrap long lines (default true)
+//
+// handle: {view, updateSchema(conf), destroy(), get value(), set value(v)}
+export function createSqlEditor(parent, opts = {}) {
+  const {
+    doc = "",
+    schema,
+    defaultTable,
+    defaultSchema,
+    history: withHistory = true,
+    onHostUndo,
+    onHostRedo,
+    extensions = [],
+    fixedTooltips = false,
+    onChange,
+    onSubmit,
+    onEscape,
+    lineWrapping = true,
+  } = opts;
+
+  const sqlCompartment = new Compartment();
+
+  // Highest-precedence keymap so submit/escape win over the basic keymap.
+  const priorityBindings = [];
+  if (onSubmit) {
+    const runSubmit = () => {
+      onSubmit(view);
+      return true;
+    };
+    priorityBindings.push(
+      { key: "Mod-Enter", run: runSubmit },
+      { key: "Shift-Enter", run: runSubmit },
+    );
+  }
+  if (onEscape) {
+    priorityBindings.push({
+      key: "Escape",
+      run: () => {
+        onEscape(view);
+        return true;
+      },
+    });
+  }
+
+  const editorExtensions = [
+    Prec.highest(keymap.of(priorityBindings)),
+    ...baseSetup(withHistory, onHostUndo, onHostRedo),
+    lineWrapping ? EditorView.lineWrapping : [],
+    fixedTooltips ? tooltips({ position: "fixed" }) : [],
+    sqlCompartment.of(sqlExtension({ schema, defaultTable, defaultSchema })),
+    onChange
+      ? EditorView.updateListener.of((update) => {
+          if (!update.docChanged) return;
+          // Suppress echoes of host-originated changes.
+          if (update.transactions.some((tr) => tr.annotation(hostChange))) {
+            return;
+          }
+          onChange(update);
+        })
+      : [],
+    ...extensions,
+  ];
+
+  let view = new EditorView({
+    doc,
+    extensions: editorExtensions,
+    ...(parent ? { parent } : {}),
+  });
+
+  return {
+    view,
+    // Swap out the schema/defaultTable/defaultSchema used for autocomplete after
+    // the editor has been created.
+    // https://codemirror.net/examples/config/#dynamic-configuration
+    updateSchema(conf) {
+      view.dispatch({
+        effects: sqlCompartment.reconfigure(sqlExtension(conf)),
+      });
+    },
+    destroy() {
+      view.destroy();
+    },
+    get value() {
+      return view.state.doc.toString();
+    },
+    // Host-originated: tagged with hostChange so it does not re-fire onChange.
+    set value(newValue) {
+      view.dispatch({
+        changes: { from: 0, to: view.state.doc.length, insert: newValue },
+        annotations: hostChange.of(true),
+      });
+    },
+  };
+}
+
+// Maps ticket 05's neutral editor-schema shape
+//   {tables: [{name, view: bool, columns: [{name, type}]}]}
+// to a lang-sql SQLNamespace of Completion objects. Kept identical to
+// _editor_schema() / _column_completion() in datasette/views/query_helpers.py so
+// server-inlined and client-fetched schemas behave the same.
+function columnCompletion(name, type) {
+  const completion = { label: name, type: "property", boost: 10 };
+  if (type) {
+    completion.detail = type;
+  }
+  return completion;
+}
+
+function schemaFromTables(tables) {
+  const schema = {};
+  for (const table of tables || []) {
+    const completions = (table.columns || []).map((column) =>
+      columnCompletion(column.name, column.type),
+    );
+    if (table.view) {
+      schema[table.name] = {
+        self: { label: table.name, type: "class", detail: "view" },
+        children: completions,
+      };
+    } else {
+      schema[table.name] = completions;
+    }
+  }
+  return schema;
+}
+
+// datasetteSchema(baseUrl, database) -> Promise<SQLNamespace>
+//
+// Fetches GET {baseUrl}/{database}/-/editor-schema.json (ticket 05) and maps the
+// neutral payload to a lang-sql SQLNamespace ready to pass as opts.schema /
+// updateSchema({schema}). baseUrl is Datasette's base_url (may be "" or "/" or a
+// mount prefix). Throws a descriptive Error on a non-200 response.
+export async function datasetteSchema(baseUrl, database) {
+  const base = (baseUrl || "").replace(/\/+$/, "");
+  const url = `${base}/${encodeURIComponent(database)}/-/editor-schema.json`;
+  const response = await fetch(url, { credentials: "same-origin" });
+  if (!response.ok) {
+    throw new Error(
+      `datasetteSchema: failed to fetch ${url} (${response.status} ${response.statusText})`,
+    );
+  }
+  const data = await response.json();
+  return schemaFromTables(data.tables);
+}
+
+// Re-export the CodeMirror pieces callers need so plugin code shares this
+// module's single CM instance instead of bundling its own.
+export {
+  EditorView,
+  EditorState,
+  Compartment,
+  Annotation,
+  Prec,
+  keymap,
+  tooltips,
+  sql,
+  SQLDialect,
+  autocompletion,
+  completionKeymap,
+};

docs/contributing.rst

@@ -438,6 +438,11 @@ Datasette bundles `CodeMirror <https://codemirror.net/>`__ for the SQL editing i
 
     npm install && npm run build:codemirror
 
-  This runs ``rollup -c`` against the ``rollup.config.mjs`` file at the root of the repository, which reads ``datasette/static/cm-editor.js`` and writes the bundled, minified output to ``datasette/static/cm-editor.bundle.js``. The bundle filename does not include the CodeMirror version number, so no template needs to be updated.
+  This runs ``rollup -c`` against the ``rollup.config.mjs`` file at the root of the repository, which builds two bundles:
 
-* Commit the rebuilt ``datasette/static/cm-editor.bundle.js`` - the bundle is checked into the repository.
+  * ``datasette/static/cm-editor.bundle.js`` - the minified IIFE bundle (global name ``cm``) used by Datasette's own pages, built from ``datasette/static/cm-editor.js``.
+  * ``datasette/static/datasette-sql-editor.bundle.js`` - a self-contained ESM bundle built from ``datasette/static/datasette-sql-editor.js``, which plugin authors can import directly (e.g. ``import {createSqlEditor, datasetteSchema} from "/-/static/datasette-sql-editor.bundle.js"``). ``cm-editor.js`` is a thin consumer of this shared module, so there is a single CodeMirror implementation.
+
+  The bundle filenames do not include the CodeMirror version number, so no template needs to be updated.
+
+* Commit the rebuilt ``datasette/static/cm-editor.bundle.js`` and ``datasette/static/datasette-sql-editor.bundle.js`` - the bundles are checked into the repository.

package-lock.json

@@ -6,8 +6,14 @@
     "": {
       "name": "datasette",
       "dependencies": {
+        "@codemirror/autocomplete": "^6.20.3",
+        "@codemirror/commands": "^6.10.4",
         "@codemirror/lang-sql": "^6.10.0",
+        "@codemirror/language": "^6.12.4",
+        "@codemirror/lint": "^6.9.7",
+        "@codemirror/search": "^6.7.1",
         "@codemirror/state": "^6.7.1",
+        "@codemirror/view": "^6.43.6",
         "@rollup/plugin-node-resolve": "^15.0.1",
         "@rollup/plugin-terser": "^0.1.0",
         "codemirror": "^6.0.2",

package.json

@@ -10,8 +10,14 @@
     "prettier": "prettier 'datasette/static/*[!.min|bundle].js'"
   },
   "dependencies": {
+    "@codemirror/autocomplete": "^6.20.3",
+    "@codemirror/commands": "^6.10.4",
     "@codemirror/lang-sql": "^6.10.0",
+    "@codemirror/language": "^6.12.4",
+    "@codemirror/lint": "^6.9.7",
+    "@codemirror/search": "^6.7.1",
     "@codemirror/state": "^6.7.1",
+    "@codemirror/view": "^6.43.6",
     "@rollup/plugin-node-resolve": "^15.0.1",
     "@rollup/plugin-terser": "^0.1.0",
     "codemirror": "^6.0.2",

rollup.config.mjs

@@ -1,12 +1,30 @@
 import { nodeResolve } from "@rollup/plugin-node-resolve";
 import terser from "@rollup/plugin-terser";
 
-export default {
-  input: "datasette/static/cm-editor.js",
-  output: {
-    file: "datasette/static/cm-editor.bundle.js",
-    format: "iife",
-    name: "cm",
+const plugins = [nodeResolve(), terser()];
+
+export default [
+  // IIFE bundle for Datasette's own pages (global name `cm`, included by
+  // _codemirror.html). The shared datasette-sql-editor.js module is inlined.
+  {
+    input: "datasette/static/cm-editor.js",
+    output: {
+      file: "datasette/static/cm-editor.bundle.js",
+      format: "iife",
+      name: "cm",
+    },
+    plugins,
   },
-  plugins: [nodeResolve(), terser()],
-};
+  // Self-contained ESM bundle for plugin authors to import directly, e.g.
+  // import {createSqlEditor, datasetteSchema} from
+  //   "/-/static/datasette-sql-editor.bundle.js"
+  // No bare specifiers remain; all @codemirror/* deps are inlined.
+  {
+    input: "datasette/static/datasette-sql-editor.js",
+    output: {
+      file: "datasette/static/datasette-sql-editor.bundle.js",
+      format: "es",
+    },
+    plugins,
+  },
+];