From 435ff7fa88fce25f5f2e4fa0aae8834deff7240c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:51:09 -0700 Subject: [PATCH 001/167] Context.documented_fields() and extras doc-metadata enforcement - Context dataclasses now expose documented_fields(), returning ContextField(name, type_name, help) for each field - ExtraRegistry.internal_classes_for_scope() returns the Extra classes that are available to HTML templates but excluded from JSON - Tests enforce that every registered Extra has a description and every DatabaseContext/QueryContext field has help metadata Refs #1510, #2127 Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 10 ++++++++++ datasette/views/__init__.py | 25 +++++++++++++++++++++++++ tests/test_extras.py | 32 ++++++++++++++++++++++++++++++++ tests/test_template_context.py | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 tests/test_template_context.py diff --git a/datasette/extras.py b/datasette/extras.py index 5cab52a4..36014185 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -81,6 +81,16 @@ class ExtraRegistry: def public_classes_for_scope(self, scope): return self.classes_for_scope(scope, include_internal=False) + def internal_classes_for_scope(self, scope): + # Extras that are available to HTML templates but excluded from + # JSON responses - plain Providers are dependency plumbing and + # never surface as keys, so they are not included + return [ + cls + for cls in self.classes_for_scope(scope) + if issubclass(cls, Extra) and not cls.public + ] + def _registry_for_scope(self, scope): registry = self._scope_registries.get(scope) if registry is None: diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index 88106737..de851708 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -1,2 +1,27 @@ +from dataclasses import dataclass +import dataclasses + + +@dataclass(frozen=True) +class ContextField: + name: str + type_name: str + help: str + + class Context: "Base class for all documented contexts" + + @classmethod + def documented_fields(cls): + "List of ContextField describing the documented fields of this context" + documented = [] + for f in dataclasses.fields(cls): + documented.append( + ContextField( + name=f.name, + type_name=getattr(f.type, "__name__", str(f.type)), + help=f.metadata.get("help", ""), + ) + ) + return documented diff --git a/tests/test_extras.py b/tests/test_extras.py index ad8a9f00..73b4965e 100644 --- a/tests/test_extras.py +++ b/tests/test_extras.py @@ -23,6 +23,38 @@ class DependentExtra(Extra): return slow_value + 1 +class InternalOnlyExtra(Extra): + description = "Internal extra for HTML templates only" + scopes = {ExtraScope.TABLE} + public = False + + async def resolve(self, context): + return "internal" + + +def test_internal_classes_for_scope(): + registry = ExtraRegistry([SlowValueExtra, DependentExtra, InternalOnlyExtra]) + assert registry.internal_classes_for_scope(ExtraScope.TABLE) == [InternalOnlyExtra] + assert registry.public_classes_for_scope(ExtraScope.TABLE) == [ + SlowValueExtra, + DependentExtra, + ] + + +def _registered_extra_classes(): + # Plain Providers are internal dependency plumbing, only Extra + # subclasses surface as documented JSON/template keys + from datasette.views.table_extras import table_extra_registry + + return [cls for cls in table_extra_registry.classes if issubclass(cls, Extra)] + + +@pytest.mark.parametrize("cls", _registered_extra_classes(), ids=lambda cls: cls.key()) +def test_registered_extras_have_descriptions(cls): + # Every registered extra is part of the documented template/JSON contract + assert cls.description, "{} is missing a description".format(cls.__name__) + + def test_registry_is_built_once_per_scope(): registry = ExtraRegistry([SlowValueExtra, DependentExtra]) first = registry._registry_for_scope(ExtraScope.TABLE) diff --git a/tests/test_template_context.py b/tests/test_template_context.py new file mode 100644 index 00000000..4ce00a55 --- /dev/null +++ b/tests/test_template_context.py @@ -0,0 +1,32 @@ +""" +Tests for the documented template context - the contract that custom +template authors can rely on for Datasette 1.0. +""" + +from dataclasses import dataclass, field + +import pytest + +from datasette.views import Context +from datasette.views.database import DatabaseContext, QueryContext + + +def test_documented_fields(): + @dataclass + class DemoContext(Context): + name: str = field(metadata={"help": "The name"}) + count: int = field(metadata={"help": "How many there are"}) + + fields = DemoContext.documented_fields() + assert [(f.name, f.type_name, f.help) for f in fields] == [ + ("name", "str", "The name"), + ("count", "int", "How many there are"), + ] + + +@pytest.mark.parametrize("klass", (DatabaseContext, QueryContext)) +def test_context_dataclass_fields_all_have_help(klass): + for context_field in klass.documented_fields(): + assert context_field.help, "{}.{} is missing help metadata".format( + klass.__name__, context_field.name + ) From 6a1b237b39ca11ca8a876f912b23e27f281eb0f4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:55:25 -0700 Subject: [PATCH 002/167] Documented template context manifest with contract tests datasette/template_contexts.py is the source of truth for the template context contract: the variables custom templates can rely on for the database, query, table and row pages, plus the base context that render_template() adds to every page. Documentation for each key comes from the Context dataclass field help (database, query), the Extra class description (table and row extras) or inline docs in the manifest (keys added by view code). Contract tests render each page with template_debug ?_context=1 and assert the real context keys exactly match the documented set, in both directions - an undocumented addition or a removed documented key both fail. Refs #1510, #2127 Co-Authored-By: Claude Fable 5 --- datasette/template_contexts.py | 240 +++++++++++++++++++++++++++++++++ tests/test_template_context.py | 103 ++++++++++++++ 2 files changed, 343 insertions(+) create mode 100644 datasette/template_contexts.py diff --git a/datasette/template_contexts.py b/datasette/template_contexts.py new file mode 100644 index 00000000..1ff1e2a8 --- /dev/null +++ b/datasette/template_contexts.py @@ -0,0 +1,240 @@ +""" +The documented template context for Datasette's core HTML pages. + +This module is the source of truth for the template context contract: +the set of variables that custom templates can rely on for each page. +It is consumed by the contract tests in tests/test_template_context.py, +which assert that the real rendered context for each page exactly +matches what is documented here, and by docs/template_context_doc.py +which generates the documentation in docs/template_context.rst. + +Documentation for each key comes from one of three places: + +- Pages that render a Context dataclass (database, query) use the + ``help`` metadata on each dataclass field +- Keys provided by registered extras use the ``description`` from the + Extra class +- Keys added inline by view code are documented in this module +""" + +from dataclasses import dataclass + +from datasette.extras import ExtraScope +from datasette.views.database import DatabaseContext, QueryContext +from datasette.views.table_extras import table_extra_registry + + +@dataclass(frozen=True) +class TemplateContextKey: + name: str + doc: str + + +def _keys(**docs): + return tuple(TemplateContextKey(name, doc) for name, doc in docs.items()) + + +# Added by Datasette.render_template() to the context for every page, +# including pages rendered by plugins +BASE_CONTEXT_KEYS = _keys( + request="The current Request object, or None", + crumb_items="Async function returning breadcrumb navigation items for the current page", + urls="Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation", + actor="The currently authenticated actor dictionary, or None", + menu_links="Async function returning links for the Datasette application menu, including those added by plugins", + display_actor="Function returning a display string for an actor dictionary", + show_logout="True if the logout link should be shown in the navigation menu", + app_css_hash="Hash of Datasette's app.css contents, used for cache busting", + zip="Python's zip() builtin, made available to template logic", + body_scripts="List of script blocks for the page body contributed by plugins", + format_bytes="Function that formats a number of bytes as a human-readable size", + show_messages="Function returning any messages set for the current user, clearing them in the process", + extra_css_urls="List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration", + extra_js_urls="List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page", + base_url="The configured base_url setting", + csrftoken="Function returning the CSRF token for the current request", + datasette_version="The version of Datasette that is running", +) + + +@dataclass(frozen=True) +class PageContext: + # Identifier used in tests and documentation, e.g. "table" + name: str + title: str + description: str + # The default template used to render this page + template: str + # For pages rendered from a Context dataclass + context_class: type = None + # For pages whose context includes resolved extras + extras_scope: ExtraScope = None + extra_keys: tuple = () + # Keys added inline by the view code, documented here + keys: tuple = () + + def documented_keys(self): + "Every page-specific documented key, excluding BASE_CONTEXT_KEYS" + documented = [] + if self.context_class is not None: + documented.extend( + TemplateContextKey(f.name, f.help) + for f in self.context_class.documented_fields() + ) + for name in self.extra_keys: + cls = table_extra_registry.classes_by_name[name] + documented.append(TemplateContextKey(name, cls.description or "")) + documented.extend(self.keys) + return sorted(documented, key=lambda key: key.name) + + +_SHARED_DOCS = dict( + ok="True if the data for this page was retrieved without errors", + rows="The rows for this page, as a list of dictionaries mapping column name to value", + query_ms="Time taken by the SQL queries for this page, in milliseconds", + select_templates="List of template names that were considered for this page, the one used marked with an asterisk", + settings="Dictionary of Datasette's current settings", + alternate_url_json="URL for the JSON version of this page", + url_csv="URL for the CSV export of this page", + url_csv_path="Path portion of the CSV export URL", + url_csv_hidden_args="(name, value) pairs for hidden form fields used by the CSV export form", + renderers="Dictionary mapping output format names (e.g. json) to their URLs for this page", + display_columns="Column objects formatted for the HTML table display", + display_rows="Row data formatted for the HTML table display", + custom_table_templates="Custom template names that were considered for displaying this table", +) + + +def _shared(*names): + return tuple(TemplateContextKey(name, _SHARED_DOCS[name]) for name in names) + + +PAGES = { + page.name: page + for page in ( + PageContext( + name="database", + title="Database", + description="The page listing the tables, views and queries in a database, e.g. /fixtures", + template="database.html", + context_class=DatabaseContext, + ), + PageContext( + name="query", + title="Query", + description="The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)", + template="query.html", + context_class=QueryContext, + ), + PageContext( + name="table", + title="Table", + description="The page showing the rows in a table or SQL view, e.g. /fixtures/facetable", + template="table.html", + extras_scope=ExtraScope.TABLE, + extra_keys=( + "actions", + "all_columns", + "columns", + "count", + "count_sql", + "custom_table_templates", + "database", + "database_color", + "display_columns", + "display_rows", + "expandable_columns", + "facet_results", + "facets_timed_out", + "filters", + "form_hidden_args", + "human_description_en", + "is_view", + "metadata", + "next_url", + "primary_keys", + "private", + "query", + "renderers", + "set_column_type_ui", + "sorted_facet_results", + "suggested_facets", + "table", + "table_definition", + "view_definition", + ), + keys=_shared( + "ok", + "rows", + "query_ms", + "select_templates", + "settings", + "alternate_url_json", + "url_csv", + "url_csv_path", + "url_csv_hidden_args", + ) + + _keys( + allow_execute_sql="True if the current actor can execute custom SQL against this database", + append_querystring="Function that appends additional querystring arguments to a URL", + count_limit="The maximum number of rows Datasette will count before showing an approximation", + datasette_allow_facet='The string "true" or "false" reflecting the allow_facet setting', + extra_wheres_for_ui="Extra where clauses from ?_where=, with links to remove them", + filter_columns="List of columns offered by the filter interface", + fix_path="Function that applies the base_url prefix to a path", + is_sortable="True if any of the displayed columns can be used to sort", + next="Pagination token for the next page, or None", + path_with_replaced_args="Function for building the current path with modified querystring arguments", + sort="Column the page is sorted by, or None", + sort_desc="Column the page is sorted by in descending order, or None", + supports_search="True if this table has full-text search configured", + top_table="Async function rendering the top_table plugin slot", + ), + ), + PageContext( + name="row", + title="Row", + description="The page showing an individual row, e.g. /fixtures/facetable/1", + template="row.html", + extras_scope=ExtraScope.ROW, + extra_keys=( + "columns", + "database", + "database_color", + "foreign_key_tables", + "metadata", + "primary_keys", + "private", + "table", + ), + keys=_shared( + "ok", + "rows", + "query_ms", + "select_templates", + "settings", + "alternate_url_json", + "url_csv", + "url_csv_path", + "url_csv_hidden_args", + "renderers", + "display_columns", + "display_rows", + "custom_table_templates", + ) + + _keys( + primary_key_values="Values of the primary keys for this row, from the URL", + row_actions="Row actions made available by plugin hooks", + top_row="Async function rendering the top_row plugin slot", + ), + ), + ) +} + + +def documented_context_keys(page_name): + "Set of every documented key for the named page, including base context keys" + page = PAGES[page_name] + return {key.name for key in BASE_CONTEXT_KEYS} | { + key.name for key in page.documented_keys() + } diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 4ce00a55..04002f31 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -3,12 +3,23 @@ Tests for the documented template context - the contract that custom template authors can rely on for Datasette 1.0. """ +import html +import json from dataclasses import dataclass, field import pytest +from datasette.app import Datasette +from datasette.extras import ExtraScope +from datasette.fixtures import write_fixture_database +from datasette.template_contexts import ( + BASE_CONTEXT_KEYS, + PAGES, + documented_context_keys, +) from datasette.views import Context from datasette.views.database import DatabaseContext, QueryContext +from datasette.views.table_extras import table_extra_registry def test_documented_fields(): @@ -30,3 +41,95 @@ def test_context_dataclass_fields_all_have_help(klass): assert context_field.help, "{}.{} is missing help metadata".format( klass.__name__, context_field.name ) + + +@pytest.fixture(scope="module") +def context_ds(tmp_path_factory): + db_path = tmp_path_factory.mktemp("template-context") / "fixtures.db" + write_fixture_database(db_path) + ds = Datasette( + [str(db_path)], + settings={"num_sql_threads": 1, "template_debug": True}, + config={ + "databases": { + "fixtures": { + "queries": { + "neighborhood_search": { + "sql": ( + "select _neighborhood from facetable " + "where _neighborhood like '%' || :text || '%'" + ), + "title": "Search neighborhoods", + } + } + } + } + }, + ) + yield ds + for db in ds.databases.values(): + if not db.is_memory: + db.close() + + +async def get_template_context(ds, path): + sep = "&" if "?" in path else "?" + response = await ds.client.get(path + sep + "_context=1") + assert response.status_code == 200, path + body = html.unescape( + response.text.removeprefix("
").removesuffix("
") + ) + return json.loads(body) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "page_name,path", + ( + ("database", "/fixtures"), + ("table", "/fixtures/facetable"), + ("table", "/fixtures/facetable?_city_id__exact=1"), + ("row", "/fixtures/facetable/1"), + ("query", "/fixtures/-/query?sql=select+*+from+facetable"), + ("query", "/fixtures/neighborhood_search?text=cork"), + ), +) +async def test_template_context_matches_documented_contract( + context_ds, page_name, path +): + # The full contract: every key in the rendered template context is + # documented, and every documented key is present in the context + documented = documented_context_keys(page_name) + actual = set(await get_template_context(context_ds, path)) + undocumented = actual - documented + no_longer_present = documented - actual + assert not undocumented, ( + "Undocumented keys in {} template context: {} - document them in " + "datasette/template_contexts.py".format(page_name, sorted(undocumented)) + ) + assert not no_longer_present, ( + "Documented keys missing from {} template context: {} - this would " + "break custom templates".format(page_name, sorted(no_longer_present)) + ) + + +def test_base_context_keys_all_have_docs(): + for key in BASE_CONTEXT_KEYS: + assert key.doc, "Base context key {} is missing docs".format(key.name) + + +@pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) +def test_page_documented_keys_all_have_docs(page): + for key in page.documented_keys(): + assert key.doc, "{} page key {} is missing docs".format(page.name, key.name) + + +@pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) +def test_page_extra_keys_are_registered_extras(page): + for name in page.extra_keys: + cls = table_extra_registry.classes_by_name.get(name) + assert cls is not None, "{} is not a registered extra".format(name) + assert page.extras_scope is not None + assert cls.available_for(page.extras_scope), ( + "{} extra is not available for scope {}".format(name, page.extras_scope) + ) From a55ae2adfc9500dfbb028ac167c379d8207f4676 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 00:00:42 -0700 Subject: [PATCH 003/167] Generated template context documentation, closes #1510 docs/template_context.rst is generated by cog from the manifest in datasette/template_contexts.py, following the json_api_doc.py pattern. It documents the base context available on every page plus the database, query, table and row pages, including the stability policy for custom template authors. Refs #2127 Co-Authored-By: Claude Fable 5 --- datasette/template_contexts.py | 8 +- docs/custom_templates.rst | 5 + docs/index.rst | 1 + docs/template_context.rst | 488 +++++++++++++++++++++++++++++++++ docs/template_context_doc.py | 54 ++++ tests/test_template_context.py | 25 +- 6 files changed, 571 insertions(+), 10 deletions(-) create mode 100644 docs/template_context.rst create mode 100644 docs/template_context_doc.py diff --git a/datasette/template_contexts.py b/datasette/template_contexts.py index 1ff1e2a8..e2854467 100644 --- a/datasette/template_contexts.py +++ b/datasette/template_contexts.py @@ -115,21 +115,21 @@ PAGES = { PageContext( name="database", title="Database", - description="The page listing the tables, views and queries in a database, e.g. /fixtures", + description="The page listing the tables, views and queries in a database, e.g. /fixtures.", template="database.html", context_class=DatabaseContext, ), PageContext( name="query", title="Query", - description="The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)", + description="The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name).", template="query.html", context_class=QueryContext, ), PageContext( name="table", title="Table", - description="The page showing the rows in a table or SQL view, e.g. /fixtures/facetable", + description="The page showing the rows in a table or SQL view, e.g. /fixtures/facetable.", template="table.html", extras_scope=ExtraScope.TABLE, extra_keys=( @@ -194,7 +194,7 @@ PAGES = { PageContext( name="row", title="Row", - description="The page showing an individual row, e.g. /fixtures/facetable/1", + description="The page showing an individual row, e.g. /fixtures/facetable/1.", template="row.html", extras_scope=ExtraScope.ROW, extra_keys=( diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index c324fb79..8066d28f 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -177,6 +177,11 @@ this:: Datasette will now first look for templates in that directory, and fall back on the defaults if no matches are found. +The variables made available to each template are documented on the +:ref:`template_context` page. Variables documented there are a stable API: +custom templates that use them will keep working in future Datasette +releases, up until the next major version. + It is also possible to over-ride templates on a per-database, per-row or per- table basis. diff --git a/docs/index.rst b/docs/index.rst index c76969bc..d494fd17 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -58,6 +58,7 @@ Contents settings introspection custom_templates + template_context plugins writing_plugins javascript_plugins diff --git a/docs/template_context.rst b/docs/template_context.rst new file mode 100644 index 00000000..a487a6e7 --- /dev/null +++ b/docs/template_context.rst @@ -0,0 +1,488 @@ +.. _template_context: + +Template context +================ + +This page documents the variables that are available to custom templates +for each of Datasette's core pages. See :ref:`customization_custom_templates` +for how to provide your own templates. + +The variables documented here are a stable contract: custom templates that +use them will continue to work across Datasette releases, up until the next +major version (Datasette 2.0). Anything present in the template context but +not documented on this page is not part of that contract and may change or +be removed in any release. + +You can inspect the full context for any page by starting Datasette with +``--setting template_debug 1`` and adding ``?_context=1`` to the page URL. + +.. [[[cog + from template_context_doc import template_context + template_context(cog) +.. ]]] + +Base context +------------ + +These variables are available on every page rendered by Datasette, including pages rendered by plugins that use :ref:`datasette.render_template() `. Plugins can add additional variables using the :ref:`plugin_hook_extra_template_vars` hook. + +``request`` + The current Request object, or None + +``crumb_items`` + Async function returning breadcrumb navigation items for the current page + +``urls`` + Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation + +``actor`` + The currently authenticated actor dictionary, or None + +``menu_links`` + Async function returning links for the Datasette application menu, including those added by plugins + +``display_actor`` + Function returning a display string for an actor dictionary + +``show_logout`` + True if the logout link should be shown in the navigation menu + +``app_css_hash`` + Hash of Datasette's app.css contents, used for cache busting + +``zip`` + Python's zip() builtin, made available to template logic + +``body_scripts`` + List of script blocks for the page body contributed by plugins + +``format_bytes`` + Function that formats a number of bytes as a human-readable size + +``show_messages`` + Function returning any messages set for the current user, clearing them in the process + +``extra_css_urls`` + List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration + +``extra_js_urls`` + List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page + +``base_url`` + The configured base_url setting + +``csrftoken`` + Function returning the CSRF token for the current request + +``datasette_version`` + The version of Datasette that is running + +Database page +------------- + +The page listing the tables, views and queries in a database, e.g. /fixtures. Rendered using the ``database.html`` template. + +``allow_download`` - ``bool`` + Boolean indicating if database download is allowed + +``allow_execute_sql`` - ``bool`` + Boolean indicating if custom SQL can be executed + +``alternate_url_json`` - ``str`` + URL for the alternate JSON version of this page + +``attached_databases`` - ``list`` + List of names of attached databases + +``count_limit`` - ``int`` + The maximum number of rows to count + +``database`` - ``str`` + The name of the database + +``database_actions`` - ``callable`` + Callable returning list of action links for the database menu + +``database_color`` - ``str`` + The color assigned to the database + +``editable`` - ``bool`` + Boolean indicating if the database is editable + +``hidden_count`` - ``int`` + Count of hidden tables + +``metadata`` - ``dict`` + Metadata for the database + +``path`` - ``str`` + The URL path to this database + +``private`` - ``bool`` + Boolean indicating if this is a private database + +``queries`` - ``list`` + List of stored query objects + +``queries_count`` - ``int`` + Count of visible stored queries + +``queries_more`` - ``bool`` + Boolean indicating if more stored queries are available + +``select_templates`` - ``list`` + List of templates that were considered for rendering this page + +``show_hidden`` - ``str`` + Value of _show_hidden query parameter + +``size`` - ``int`` + The size of the database in bytes + +``table_columns`` - ``dict`` + Dictionary mapping table names to their column lists + +``tables`` - ``list`` + List of table objects in the database + +``top_database`` - ``callable`` + Callable to render the top_database slot + +``views`` - ``list`` + List of view objects in the database + +Query page +---------- + +The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name). Rendered using the ``query.html`` template. + +``allow_execute_sql`` - ``bool`` + Boolean indicating if custom SQL can be executed + +``alternate_url_json`` - ``str`` + URL for alternate JSON version of this page + +``columns`` - ``list`` + List of column names + +``database`` - ``str`` + The name of the database being queried + +``database_color`` - ``str`` + The color of the database + +``db_is_immutable`` - ``bool`` + Boolean indicating if this database is immutable + +``display_rows`` - ``list`` + List of result rows to display + +``edit_sql_url`` - ``str`` + URL to edit the SQL for a stored query + +``editable`` - ``bool`` + Boolean indicating if the SQL can be edited + +``error`` - ``str`` + Any query error message + +``hide_sql`` - ``bool`` + Boolean indicating if the SQL should be hidden + +``metadata`` - ``dict`` + Metadata about the database or the stored query + +``named_parameter_values`` - ``dict`` + Dictionary of parameter names/values + +``private`` - ``bool`` + Boolean indicating if this is a private database + +``query`` - ``dict`` + The SQL query object containing the `sql` string + +``query_actions`` - ``callable`` + Callable returning a list of links for the query action menu + +``renderers`` - ``dict`` + Dictionary of renderer name to URL + +``save_query_url`` - ``str`` + URL to save the current arbitrary SQL as a query + +``select_templates`` - ``list`` + List of templates that were considered for rendering this page + +``show_hide_hidden`` - ``str`` + Hidden input field for the _show_sql parameter + +``show_hide_link`` - ``str`` + The URL to toggle showing/hiding the SQL + +``show_hide_text`` - ``str`` + The text for the show/hide SQL link + +``stored_query`` - ``str`` + The name of the stored query if this is a stored query + +``stored_query_write`` - ``bool`` + Boolean indicating if this is a stored query that allows writes + +``table_columns`` - ``dict`` + Dictionary of table name to list of column names + +``tables`` - ``list`` + List of table objects in the database + +``top_query`` - ``callable`` + Callable to render the top_query slot + +``top_stored_query`` - ``callable`` + Callable to render the top_stored_query slot + +``url_csv`` - ``str`` + URL for CSV export + +Table page +---------- + +The page showing the rows in a table or SQL view, e.g. /fixtures/facetable. Rendered using the ``table.html`` template. + +Many of these keys are shared with the :ref:`JSON API ` for this page. + +``actions`` + Table or view actions made available by plugin hooks + +``all_columns`` + All columns in the table, regardless of _col/_nocol filtering + +``allow_execute_sql`` + True if the current actor can execute custom SQL against this database + +``alternate_url_json`` + URL for the JSON version of this page + +``append_querystring`` + Function that appends additional querystring arguments to a URL + +``columns`` + Column names returned by this query + +``count`` + Total count of rows matching these filters + +``count_limit`` + The maximum number of rows Datasette will count before showing an approximation + +``count_sql`` + SQL query used to calculate the total count + +``custom_table_templates`` + Custom template names considered for this table + +``database`` + Database name + +``database_color`` + Color assigned to the database + +``datasette_allow_facet`` + The string "true" or "false" reflecting the allow_facet setting + +``display_columns`` + Column metadata used by the HTML table display + +``display_rows`` + Row data formatted for the HTML table display + +``expandable_columns`` + Foreign key columns that can be expanded with labels + +``extra_wheres_for_ui`` + Extra where clauses from ?_where=, with links to remove them + +``facet_results`` + Results of facets calculated against this data + +``facets_timed_out`` + Facet calculations that timed out + +``filter_columns`` + List of columns offered by the filter interface + +``filters`` + Filters object used by the HTML table interface + +``fix_path`` + Function that applies the base_url prefix to a path + +``form_hidden_args`` + Hidden form arguments used by the HTML table interface + +``human_description_en`` + Human-readable description of the filters + +``is_sortable`` + True if any of the displayed columns can be used to sort + +``is_view`` + Whether this resource is a view instead of a table + +``metadata`` + Metadata about the table, database or stored query + +``next`` + Pagination token for the next page, or None + +``next_url`` + Full URL for the next page of results + +``ok`` + True if the data for this page was retrieved without errors + +``path_with_replaced_args`` + Function for building the current path with modified querystring arguments + +``primary_keys`` + Primary keys for this table + +``private`` + Whether this resource is private to the current actor + +``query`` + Details of the underlying SQL query + +``query_ms`` + Time taken by the SQL queries for this page, in milliseconds + +``renderers`` + Alternative output renderers available for this table + +``rows`` + The rows for this page, as a list of dictionaries mapping column name to value + +``select_templates`` + List of template names that were considered for this page, the one used marked with an asterisk + +``set_column_type_ui`` + Column type UI metadata for this table + +``settings`` + Dictionary of Datasette's current settings + +``sort`` + Column the page is sorted by, or None + +``sort_desc`` + Column the page is sorted by in descending order, or None + +``sorted_facet_results`` + Facet results sorted for display + +``suggested_facets`` + Suggestions for facets that might return interesting results + +``supports_search`` + True if this table has full-text search configured + +``table`` + Table name + +``table_definition`` + SQL definition for this table + +``top_table`` + Async function rendering the top_table plugin slot + +``url_csv`` + URL for the CSV export of this page + +``url_csv_hidden_args`` + (name, value) pairs for hidden form fields used by the CSV export form + +``url_csv_path`` + Path portion of the CSV export URL + +``view_definition`` + SQL definition for this view + +Row page +-------- + +The page showing an individual row, e.g. /fixtures/facetable/1. Rendered using the ``row.html`` template. + +Many of these keys are shared with the :ref:`JSON API ` for this page. + +``alternate_url_json`` + URL for the JSON version of this page + +``columns`` + Column names returned by this query + +``custom_table_templates`` + Custom template names that were considered for displaying this table + +``database`` + Database name + +``database_color`` + Color assigned to the database + +``display_columns`` + Column objects formatted for the HTML table display + +``display_rows`` + Row data formatted for the HTML table display + +``foreign_key_tables`` + Tables that link to this row using foreign keys + +``metadata`` + Metadata about the table, database or stored query + +``ok`` + True if the data for this page was retrieved without errors + +``primary_key_values`` + Values of the primary keys for this row, from the URL + +``primary_keys`` + Primary keys for this table + +``private`` + Whether this resource is private to the current actor + +``query_ms`` + Time taken by the SQL queries for this page, in milliseconds + +``renderers`` + Dictionary mapping output format names (e.g. json) to their URLs for this page + +``row_actions`` + Row actions made available by plugin hooks + +``rows`` + The rows for this page, as a list of dictionaries mapping column name to value + +``select_templates`` + List of template names that were considered for this page, the one used marked with an asterisk + +``settings`` + Dictionary of Datasette's current settings + +``table`` + Table name + +``top_row`` + Async function rendering the top_row plugin slot + +``url_csv`` + URL for the CSV export of this page + +``url_csv_hidden_args`` + (name, value) pairs for hidden form fields used by the CSV export form + +``url_csv_path`` + Path portion of the CSV export URL + +.. [[[end]]] diff --git a/docs/template_context_doc.py b/docs/template_context_doc.py new file mode 100644 index 00000000..1539dc5e --- /dev/null +++ b/docs/template_context_doc.py @@ -0,0 +1,54 @@ +""" +Cog helpers for generating docs/template_context.rst from the manifest +in datasette/template_contexts.py - same pattern as json_api_doc.py. +""" + + +def template_context(cog): + from datasette.template_contexts import BASE_CONTEXT_KEYS, PAGES + + cog.out("\n") + _section( + cog, + "Base context", + ( + "These variables are available on every page rendered by " + "Datasette, including pages rendered by plugins that use " + ":ref:`datasette.render_template() `. " + "Plugins can add additional variables using the " + ":ref:`plugin_hook_extra_template_vars` hook." + ), + ) + _untyped_keys(cog, BASE_CONTEXT_KEYS) + + for page in PAGES.values(): + _section( + cog, + "{} page".format(page.title), + "{} Rendered using the ``{}`` template.".format( + page.description, page.template + ), + ) + if page.context_class is not None: + for f in sorted( + page.context_class.documented_fields(), key=lambda f: f.name + ): + cog.out("``{}`` - ``{}``\n".format(f.name, f.type_name)) + cog.out(" {}\n\n".format(f.help)) + else: + cog.out( + "Many of these keys are shared with the :ref:`JSON API " + "` for this page.\n\n" + ) + _untyped_keys(cog, page.documented_keys()) + + +def _section(cog, title, intro): + cog.out("{}\n{}\n\n".format(title, "-" * len(title))) + cog.out("{}\n\n".format(intro)) + + +def _untyped_keys(cog, keys): + for key in keys: + cog.out("``{}``\n".format(key.name)) + cog.out(" {}\n\n".format(key.doc)) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 04002f31..73e1d6c1 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -5,6 +5,7 @@ template authors can rely on for Datasette 1.0. import html import json +import pathlib from dataclasses import dataclass, field import pytest @@ -76,9 +77,7 @@ async def get_template_context(ds, path): sep = "&" if "?" in path else "?" response = await ds.client.get(path + sep + "_context=1") assert response.status_code == 200, path - body = html.unescape( - response.text.removeprefix("
").removesuffix("
") - ) + body = html.unescape(response.text.removeprefix("
").removesuffix("
")) return json.loads(body) @@ -124,12 +123,26 @@ def test_page_documented_keys_all_have_docs(page): assert key.doc, "{} page key {} is missing docs".format(page.name, key.name) +def test_template_context_docs_cover_every_documented_key(): + docs_path = pathlib.Path(__file__).parent.parent / "docs" / "template_context.rst" + assert docs_path.exists(), "docs/template_context.rst is missing" + docs = docs_path.read_text() + for key in BASE_CONTEXT_KEYS: + assert "``{}``".format(key.name) in docs, key.name + for page in PAGES.values(): + assert page.title in docs, page.title + for key in page.documented_keys(): + assert "``{}``".format(key.name) in docs, "{} ({} page)".format( + key.name, page.name + ) + + @pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) def test_page_extra_keys_are_registered_extras(page): for name in page.extra_keys: cls = table_extra_registry.classes_by_name.get(name) assert cls is not None, "{} is not a registered extra".format(name) assert page.extras_scope is not None - assert cls.available_for(page.extras_scope), ( - "{} extra is not available for scope {}".format(name, page.extras_scope) - ) + assert cls.available_for( + page.extras_scope + ), "{} extra is not available for scope {}".format(name, page.extras_scope) From 3ea7ed86065d3071fa76c755bdfa63e6776ea7cc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 00:06:56 -0700 Subject: [PATCH 004/167] Isolate test plugins from template context contract tests Datasette instances created with plugins_dir register their plugins on the global plugin manager for the rest of the process, so the contract tests could see extra_template_vars keys leaked from earlier test modules (e.g. the session-scoped ds_client fixture). A fixture now unregisters non-default plugins implementing extra_template_vars for the duration of each contract test and restores them afterwards. Co-Authored-By: Claude Fable 5 --- tests/test_template_context.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 73e1d6c1..3c6b13de 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -44,6 +44,27 @@ def test_context_dataclass_fields_all_have_help(klass): ) +@pytest.fixture +def isolate_extra_template_vars_plugins(): + # Datasette instances created with plugins_dir (e.g. the session-scoped + # ds_client fixture) register their plugins on the global plugin manager + # for the rest of the process. The contract documents plugin-free + # Datasette core, so unregister any non-default plugin that adds + # template variables via the extra_template_vars hook + from datasette.plugins import pm, DEFAULT_PLUGINS + + hook_plugins = {impl.plugin for impl in pm.hook.extra_template_vars.get_hookimpls()} + removed = [] + for plugin in list(pm.get_plugins()): + name = pm.get_name(plugin) + if name not in DEFAULT_PLUGINS and plugin in hook_plugins: + pm.unregister(plugin) + removed.append((plugin, name)) + yield + for plugin, name in removed: + pm.register(plugin, name) + + @pytest.fixture(scope="module") def context_ds(tmp_path_factory): db_path = tmp_path_factory.mktemp("template-context") / "fixtures.db" @@ -94,7 +115,7 @@ async def get_template_context(ds, path): ), ) async def test_template_context_matches_documented_contract( - context_ds, page_name, path + context_ds, isolate_extra_template_vars_plugins, page_name, path ): # The full contract: every key in the rendered template context is # documented, and every documented key is present in the context From 63995ce823124569968e9dfdcc94221c59a83810 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 06:51:54 -0700 Subject: [PATCH 005/167] extra_field() - Context fields documented by their Extra class A Context dataclass field declared with extra_field() takes its documentation from the description on the registered Extra of the same name, validated against the class's extras_scope. This keeps doc strings next to the resolve() code instead of duplicating them on the dataclass, ahead of introducing TableContext and RowContext. Co-Authored-By: Claude Fable 5 --- datasette/views/__init__.py | 47 +++++++++++++++++++++++++++++++++- tests/test_template_context.py | 45 ++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index de851708..238b9bb3 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -7,21 +7,66 @@ class ContextField: name: str type_name: str help: str + from_extra: bool = False + + +def extra_field(): + """ + Declare a Context dataclass field whose value comes from a registered + Extra of the same name - its documentation is the Extra description, + so the doc string lives next to the resolve() code rather than being + duplicated on the dataclass. + """ + return dataclasses.field(metadata={"from_extra": True}) class Context: "Base class for all documented contexts" + # Set on subclasses whose extra_field() fields should be resolved + # against the extras registry for this scope + extras_scope = None + @classmethod def documented_fields(cls): "List of ContextField describing the documented fields of this context" documented = [] for f in dataclasses.fields(cls): + from_extra = bool(f.metadata.get("from_extra")) + if from_extra: + help_text = cls._extra_description(f.name) + else: + help_text = f.metadata.get("help", "") documented.append( ContextField( name=f.name, type_name=getattr(f.type, "__name__", str(f.type)), - help=f.metadata.get("help", ""), + help=help_text, + from_extra=from_extra, ) ) return documented + + @classmethod + def _extra_description(cls, name): + # Imported lazily - table_extras is not needed just to define + # Context subclasses + from datasette.views.table_extras import table_extra_registry + + try: + extra_class = table_extra_registry.classes_by_name[name] + except KeyError: + raise KeyError( + "{}.{} is declared with extra_field() but there is no " + "registered extra of that name".format(cls.__name__, name) + ) + if cls.extras_scope is not None and not extra_class.available_for( + cls.extras_scope + ): + raise ValueError( + "{}.{} is declared with extra_field() but the {} extra is " + "not available for scope {}".format( + cls.__name__, name, name, cls.extras_scope + ) + ) + return extra_class.description or "" diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 3c6b13de..c385097a 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -44,6 +44,51 @@ def test_context_dataclass_fields_all_have_help(klass): ) +def test_extra_field_documentation_comes_from_the_extra_class(): + from datasette.views import extra_field + from datasette.views.table_extras import CountExtra + + @dataclass + class DemoContext(Context): + extras_scope = ExtraScope.TABLE + + count: int = extra_field() + name: str = field(metadata={"help": "The name"}) + + fields = {f.name: f for f in DemoContext.documented_fields()} + assert fields["count"].help == CountExtra.description + assert fields["count"].from_extra + assert fields["name"].help == "The name" + assert not fields["name"].from_extra + + +def test_extra_field_must_match_a_registered_extra(): + from datasette.views import extra_field + + @dataclass + class BadContext(Context): + extras_scope = ExtraScope.TABLE + + not_a_real_extra: str = extra_field() + + with pytest.raises(KeyError): + BadContext.documented_fields() + + +def test_extra_field_must_be_available_for_the_scope(): + from datasette.views import extra_field + + @dataclass + class WrongScopeContext(Context): + extras_scope = ExtraScope.ROW + + # count is a TABLE-scope extra, not available for ROW + count: int = extra_field() + + with pytest.raises(ValueError): + WrongScopeContext.documented_fields() + + @pytest.fixture def isolate_extra_template_vars_plugins(): # Datasette instances created with plugins_dir (e.g. the session-scoped From 8b89a3aca85bdf8eee93b9f4677dd85b2599dcb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 06:57:51 -0700 Subject: [PATCH 006/167] TableContext - table page now renders a documented Context dataclass The table HTML view constructs a TableContext instead of an ad-hoc dict, matching how the database and query pages already work. Fields resolved by registered extras are declared with extra_field() so their documentation lives on the Extra classes in table_extras.py; fields added by the view code carry help metadata next to the view. render_template() now converts Context dataclasses shallowly instead of via dataclasses.asdict(), which deep-copied every value and would fail on values like sqlite3.Row. Keys not declared on TableContext - extras requested with ?_extra= on the HTML page, or extra filter context from filters_from_request plugins - are now dropped from the HTML template context rather than passed through undocumented. Refs #2127 Co-Authored-By: Claude Fable 5 --- datasette/app.py | 6 +- datasette/views/table.py | 136 +++++++++++++++++++++++++++++- tests/test_internals_datasette.py | 25 ++++++ tests/test_template_context.py | 8 ++ 4 files changed, 172 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 81d23acb..275baae4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2162,7 +2162,11 @@ class Datasette: templates = [templates] template = self.get_jinja_environment(request).select_template(templates) if dataclasses.is_dataclass(context): - context = dataclasses.asdict(context) + # Shallow conversion - asdict() would deep-copy values, which + # is wasteful and fails on values like sqlite3.Row + context = { + f.name: getattr(context, f.name) for f in dataclasses.fields(context) + } body_scripts = [] # pylint: disable=no-member for extra_script in pm.hook.extra_body_script( diff --git a/datasette/views/table.py b/datasette/views/table.py index 65388c9c..356247ff 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -43,6 +43,10 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response from datasette.filters import Filters import sqlite_utils +from dataclasses import dataclass, field, fields + +from datasette.extras import ExtraScope +from . import Context, extra_field from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView from .table_extras import ( @@ -52,6 +56,129 @@ from .table_extras import ( table_extra_registry, ) + +@dataclass +class TableContext(Context): + "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable" + + extras_scope = ExtraScope.TABLE + + # Fields resolved by registered extras - their documentation comes + # from the description on each Extra class in table_extras.py + actions: callable = extra_field() + all_columns: list = extra_field() + columns: list = extra_field() + count: int = extra_field() + count_sql: str = extra_field() + custom_table_templates: list = extra_field() + database: str = extra_field() + database_color: str = extra_field() + display_columns: list = extra_field() + display_rows: list = extra_field() + expandable_columns: list = extra_field() + facet_results: dict = extra_field() + facets_timed_out: list = extra_field() + filters: Filters = extra_field() + form_hidden_args: list = extra_field() + human_description_en: str = extra_field() + is_view: bool = extra_field() + metadata: dict = extra_field() + next_url: str = extra_field() + primary_keys: list = extra_field() + private: bool = extra_field() + query: dict = extra_field() + renderers: dict = extra_field() + set_column_type_ui: dict = extra_field() + sorted_facet_results: list = extra_field() + suggested_facets: list = extra_field() + table: str = extra_field() + table_definition: str = extra_field() + view_definition: str = extra_field() + + # Fields added by the view code + ok: bool = field( + metadata={"help": "True if the data for this page was retrieved without errors"} + ) + next: str = field(metadata={"help": "Pagination token for the next page, or None"}) + rows: list = field( + metadata={ + "help": "The rows for this page, as a list of dictionaries mapping column name to value" + } + ) + filter_columns: list = field( + metadata={"help": "List of columns offered by the filter interface"} + ) + supports_search: bool = field( + metadata={"help": "True if this table has full-text search configured"} + ) + extra_wheres_for_ui: list = field( + metadata={ + "help": "Extra where clauses from ?_where=, with links to remove them" + } + ) + url_csv: str = field(metadata={"help": "URL for the CSV export of this page"}) + url_csv_path: str = field(metadata={"help": "Path portion of the CSV export URL"}) + url_csv_hidden_args: list = field( + metadata={ + "help": "(name, value) pairs for hidden form fields used by the CSV export form" + } + ) + sort: str = field(metadata={"help": "Column the page is sorted by, or None"}) + sort_desc: str = field( + metadata={"help": "Column the page is sorted by in descending order, or None"} + ) + append_querystring: callable = field( + metadata={ + "help": "Function that appends additional querystring arguments to a URL" + } + ) + path_with_replaced_args: callable = field( + metadata={ + "help": "Function for building the current path with modified querystring arguments" + } + ) + fix_path: callable = field( + metadata={"help": "Function that applies the base_url prefix to a path"} + ) + settings: dict = field( + metadata={"help": "Dictionary of Datasette's current settings"} + ) + alternate_url_json: str = field( + metadata={"help": "URL for the JSON version of this page"} + ) + datasette_allow_facet: str = field( + metadata={ + "help": 'The string "true" or "false" reflecting the allow_facet setting' + } + ) + is_sortable: bool = field( + metadata={"help": "True if any of the displayed columns can be used to sort"} + ) + allow_execute_sql: bool = field( + metadata={ + "help": "True if the current actor can execute custom SQL against this database" + } + ) + query_ms: float = field( + metadata={ + "help": "Time taken by the SQL queries for this page, in milliseconds" + } + ) + select_templates: list = field( + metadata={ + "help": "List of template names that were considered for this page, the one used marked with an asterisk" + } + ) + top_table: callable = field( + metadata={"help": "Async function rendering the top_table plugin slot"} + ) + count_limit: int = field( + metadata={ + "help": "The maximum number of rows Datasette will count before showing an approximation" + } + ) + + LINK_WITH_LABEL = ( '{label} {id}' ) @@ -1084,11 +1211,16 @@ async def table_view_traced(datasette, request): ) } ) + # Only keys declared on TableContext are part of the documented + # template contract - anything else in data (e.g. extras requested + # with ?_extra= on the HTML page, or extra filter context added by + # filters_from_request plugins) is dropped here + declared_fields = {f.name for f in fields(TableContext)} r = Response.html( await datasette.render_template( template, - dict( - data, + TableContext( + **{k: v for k, v in data.items() if k in declared_fields}, append_querystring=append_querystring, path_with_replaced_args=path_with_replaced_args, fix_path=datasette.urls.path, diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 3f867eb0..e9c78ecc 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -344,3 +344,28 @@ async def test_datasette_close_continues_past_db_error(): ds.close() assert good._closed assert ds._internal_database._closed + + +@pytest.mark.asyncio +async def test_datasette_render_template_dataclass_values_not_deep_copied(): + # display_rows can contain values like sqlite3.Row that cannot be + # deep-copied, so render_template must convert Context dataclasses + # shallowly - https://github.com/simonw/datasette/issues/2127 + class RefusesDeepCopy: + def __deepcopy__(self, memo): + raise RuntimeError("deepcopy not supported") + + def __str__(self): + return "shallow-copied-value" + + @dataclasses.dataclass + class ExampleContext(Context): + title: str + status: int + error: RefusesDeepCopy + + context = ExampleContext(title="Hello", status=200, error=RefusesDeepCopy()) + ds = Datasette(memory=True) + await ds.invoke_startup() + rendered = await ds.render_template("error.html", context) + assert "shallow-copied-value" in rendered diff --git a/tests/test_template_context.py b/tests/test_template_context.py index c385097a..4f08c476 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -178,6 +178,14 @@ async def test_template_context_matches_documented_contract( ) +def test_table_context_fields_match_documented_contract(): + from datasette.views.table import TableContext + + assert {f.name for f in TableContext.documented_fields()} == { + key.name for key in PAGES["table"].documented_keys() + } + + def test_base_context_keys_all_have_docs(): for key in BASE_CONTEXT_KEYS: assert key.doc, "Base context key {} is missing docs".format(key.name) From 3cc0fc07b49da3c9757ad33c73187f4fe49af557 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:01:57 -0700 Subject: [PATCH 007/167] RowContext - row page now renders a documented Context dataclass RowView declares context_class = RowContext; BaseView.render() constructs the dataclass from the assembled context, dropping any keys not declared on the class, after select_templates and alternate_url_json have been added. Extras-named fields use extra_field() so their documentation comes from the Extra classes; view-added fields carry help metadata next to the view code. Refs #2127 Co-Authored-By: Claude Fable 5 --- datasette/views/base.py | 9 ++++ datasette/views/row.py | 81 +++++++++++++++++++++++++++++++++- tests/test_template_context.py | 8 ++++ 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 2e2a5443..a3a207bd 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -1,5 +1,6 @@ import asyncio import csv +import dataclasses import hashlib import sys import textwrap @@ -88,6 +89,9 @@ class View: class BaseView: ds = None has_json_alternate = True + # Set to a Context subclass to render a documented template context - + # keys not declared on the class are dropped before rendering + context_class = None def __init__(self, datasette): self.ds = datasette @@ -169,6 +173,11 @@ class BaseView: ) } ) + if self.context_class is not None: + declared = {f.name for f in dataclasses.fields(self.context_class)} + template_context = self.context_class( + **{k: v for k, v in template_context.items() if k in declared} + ) return Response.html( await self.ds.render_template( template, diff --git a/datasette/views/row.py b/datasette/views/row.py index c300758b..f7475117 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -11,16 +11,95 @@ from datasette.utils import ( escape_sqlite, ) from datasette.plugins import pm +from dataclasses import dataclass, field import json import markupsafe import sqlite_utils -from datasette.extras import extra_names_from_request +from datasette.extras import extra_names_from_request, ExtraScope +from . import Context, extra_field from .table import display_columns_and_rows from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry +@dataclass +class RowContext(Context): + "The page showing an individual row, e.g. /fixtures/facetable/1" + + extras_scope = ExtraScope.ROW + + # Fields resolved by registered extras - their documentation comes + # from the description on each Extra class in table_extras.py + columns: list = extra_field() + database: str = extra_field() + database_color: str = extra_field() + foreign_key_tables: list = extra_field() + metadata: dict = extra_field() + primary_keys: list = extra_field() + private: bool = extra_field() + table: str = extra_field() + + # Fields added by the view code + ok: bool = field( + metadata={"help": "True if the data for this page was retrieved without errors"} + ) + rows: list = field( + metadata={ + "help": "The rows for this page, as a list of dictionaries mapping column name to value" + } + ) + primary_key_values: list = field( + metadata={"help": "Values of the primary keys for this row, from the URL"} + ) + query_ms: float = field( + metadata={ + "help": "Time taken by the SQL queries for this page, in milliseconds" + } + ) + display_columns: list = field( + metadata={"help": "Column objects formatted for the HTML table display"} + ) + display_rows: list = field( + metadata={"help": "Row data formatted for the HTML table display"} + ) + custom_table_templates: list = field( + metadata={ + "help": "Custom template names that were considered for displaying this table" + } + ) + row_actions: list = field( + metadata={"help": "Row actions made available by plugin hooks"} + ) + top_row: callable = field( + metadata={"help": "Async function rendering the top_row plugin slot"} + ) + renderers: dict = field( + metadata={ + "help": "Dictionary mapping output format names (e.g. json) to their URLs for this page" + } + ) + url_csv: str = field(metadata={"help": "URL for the CSV export of this page"}) + url_csv_path: str = field(metadata={"help": "Path portion of the CSV export URL"}) + url_csv_hidden_args: list = field( + metadata={ + "help": "(name, value) pairs for hidden form fields used by the CSV export form" + } + ) + settings: dict = field( + metadata={"help": "Dictionary of Datasette's current settings"} + ) + select_templates: list = field( + metadata={ + "help": "List of template names that were considered for this page, the one used marked with an asterisk" + } + ) + alternate_url_json: str = field( + metadata={"help": "URL for the JSON version of this page"} + ) + + class RowView(DataView): name = "row" + context_class = RowContext async def data(self, request, default_labels=False): resolved = await self.ds.resolve_row(request) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 4f08c476..0c694b2e 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -186,6 +186,14 @@ def test_table_context_fields_match_documented_contract(): } +def test_row_context_fields_match_documented_contract(): + from datasette.views.row import RowContext + + assert {f.name for f in RowContext.documented_fields()} == { + key.name for key in PAGES["row"].documented_keys() + } + + def test_base_context_keys_all_have_docs(): for key in BASE_CONTEXT_KEYS: assert key.doc, "Base context key {} is missing docs".format(key.name) From 8e01542fe98c237eff1390353c9c1287050f2edd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:08:21 -0700 Subject: [PATCH 008/167] One consistent pattern: every page context is a Context dataclass datasette/template_contexts.py is now a thin index with no documentation strings of its own - the docs live next to the code: - Each page's Context class (DatabaseContext, QueryContext, TableContext, RowContext) carries a docstring, its template name and help metadata on view-added fields, in the view module itself - extra_field() fields document themselves from the Extra classes - The keys render_template() adds to every page are documented in TEMPLATE_BASE_CONTEXT in app.py, next to the code that adds them, with the contract tests keeping the two in sync docs/template_context.rst is regenerated from the dataclasses, so the table and row pages now include field types like the others. Refs #2127 Co-Authored-By: Claude Fable 5 --- datasette/app.py | 28 ++++ datasette/template_contexts.py | 250 ++++----------------------------- datasette/views/database.py | 8 ++ datasette/views/row.py | 3 +- datasette/views/table.py | 3 +- docs/template_context.rst | 152 ++++++++++---------- docs/template_context_doc.py | 41 +++--- tests/test_template_context.py | 78 +++------- 8 files changed, 180 insertions(+), 383 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 275baae4..b683969a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -313,6 +313,32 @@ def _to_string(value): return json.dumps(value, default=str) +# Documentation for the variables Datasette.render_template() adds to the +# context for every page. This is part of the documented template contract: +# keys added in render_template() must be documented here - the contract +# tests in tests/test_template_context.py enforce this, and the docs in +# docs/template_context.rst are generated from it. +TEMPLATE_BASE_CONTEXT = { + "request": "The current Request object, or None", + "crumb_items": "Async function returning breadcrumb navigation items for the current page", + "urls": "Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation", + "actor": "The currently authenticated actor dictionary, or None", + "menu_links": "Async function returning links for the Datasette application menu, including those added by plugins", + "display_actor": "Function returning a display string for an actor dictionary", + "show_logout": "True if the logout link should be shown in the navigation menu", + "app_css_hash": "Hash of Datasette's app.css contents, used for cache busting", + "zip": "Python's zip() builtin, made available to template logic", + "body_scripts": "List of script blocks for the page body contributed by plugins", + "format_bytes": "Function that formats a number of bytes as a human-readable size", + "show_messages": "Function returning any messages set for the current user, clearing them in the process", + "extra_css_urls": "List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration", + "extra_js_urls": "List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page", + "base_url": "The configured base_url setting", + "csrftoken": "Function returning the CSRF token for the current request", + "datasette_version": "The version of Datasette that is running", +} + + class Datasette: # Message constants: INFO = 1 @@ -2216,6 +2242,8 @@ class Datasette: links.extend(extra_links) return links + # Keys added here must be documented in TEMPLATE_BASE_CONTEXT - + # the contract tests fail otherwise template_context = { **context, **{ diff --git a/datasette/template_contexts.py b/datasette/template_contexts.py index e2854467..5fe6a80e 100644 --- a/datasette/template_contexts.py +++ b/datasette/template_contexts.py @@ -1,240 +1,40 @@ """ -The documented template context for Datasette's core HTML pages. +Index of the documented template contexts for Datasette's core HTML pages. -This module is the source of truth for the template context contract: -the set of variables that custom templates can rely on for each page. -It is consumed by the contract tests in tests/test_template_context.py, -which assert that the real rendered context for each page exactly -matches what is documented here, and by docs/template_context_doc.py -which generates the documentation in docs/template_context.rst. +This module deliberately contains no documentation strings of its own - +the documentation lives next to the code it describes: -Documentation for each key comes from one of three places: +- Every page renders a Context dataclass defined in its view module + (DatabaseContext, QueryContext in views/database.py, TableContext in + views/table.py, RowContext in views/row.py). Fields added by view code + carry ``help`` metadata; fields declared with extra_field() take their + documentation from the description on the matching Extra class in + views/table_extras.py. +- The keys render_template() adds to every page are documented in + TEMPLATE_BASE_CONTEXT in datasette/app.py, next to the code that adds + them. -- Pages that render a Context dataclass (database, query) use the - ``help`` metadata on each dataclass field -- Keys provided by registered extras use the ``description`` from the - Extra class -- Keys added inline by view code are documented in this module +The contract tests in tests/test_template_context.py assert that the real +rendered context for each page exactly matches what is documented, and +docs/template_context_doc.py generates docs/template_context.rst from the +same classes. """ -from dataclasses import dataclass - -from datasette.extras import ExtraScope +from datasette.app import TEMPLATE_BASE_CONTEXT from datasette.views.database import DatabaseContext, QueryContext -from datasette.views.table_extras import table_extra_registry - - -@dataclass(frozen=True) -class TemplateContextKey: - name: str - doc: str - - -def _keys(**docs): - return tuple(TemplateContextKey(name, doc) for name, doc in docs.items()) - - -# Added by Datasette.render_template() to the context for every page, -# including pages rendered by plugins -BASE_CONTEXT_KEYS = _keys( - request="The current Request object, or None", - crumb_items="Async function returning breadcrumb navigation items for the current page", - urls="Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation", - actor="The currently authenticated actor dictionary, or None", - menu_links="Async function returning links for the Datasette application menu, including those added by plugins", - display_actor="Function returning a display string for an actor dictionary", - show_logout="True if the logout link should be shown in the navigation menu", - app_css_hash="Hash of Datasette's app.css contents, used for cache busting", - zip="Python's zip() builtin, made available to template logic", - body_scripts="List of script blocks for the page body contributed by plugins", - format_bytes="Function that formats a number of bytes as a human-readable size", - show_messages="Function returning any messages set for the current user, clearing them in the process", - extra_css_urls="List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration", - extra_js_urls="List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page", - base_url="The configured base_url setting", - csrftoken="Function returning the CSRF token for the current request", - datasette_version="The version of Datasette that is running", -) - - -@dataclass(frozen=True) -class PageContext: - # Identifier used in tests and documentation, e.g. "table" - name: str - title: str - description: str - # The default template used to render this page - template: str - # For pages rendered from a Context dataclass - context_class: type = None - # For pages whose context includes resolved extras - extras_scope: ExtraScope = None - extra_keys: tuple = () - # Keys added inline by the view code, documented here - keys: tuple = () - - def documented_keys(self): - "Every page-specific documented key, excluding BASE_CONTEXT_KEYS" - documented = [] - if self.context_class is not None: - documented.extend( - TemplateContextKey(f.name, f.help) - for f in self.context_class.documented_fields() - ) - for name in self.extra_keys: - cls = table_extra_registry.classes_by_name[name] - documented.append(TemplateContextKey(name, cls.description or "")) - documented.extend(self.keys) - return sorted(documented, key=lambda key: key.name) - - -_SHARED_DOCS = dict( - ok="True if the data for this page was retrieved without errors", - rows="The rows for this page, as a list of dictionaries mapping column name to value", - query_ms="Time taken by the SQL queries for this page, in milliseconds", - select_templates="List of template names that were considered for this page, the one used marked with an asterisk", - settings="Dictionary of Datasette's current settings", - alternate_url_json="URL for the JSON version of this page", - url_csv="URL for the CSV export of this page", - url_csv_path="Path portion of the CSV export URL", - url_csv_hidden_args="(name, value) pairs for hidden form fields used by the CSV export form", - renderers="Dictionary mapping output format names (e.g. json) to their URLs for this page", - display_columns="Column objects formatted for the HTML table display", - display_rows="Row data formatted for the HTML table display", - custom_table_templates="Custom template names that were considered for displaying this table", -) - - -def _shared(*names): - return tuple(TemplateContextKey(name, _SHARED_DOCS[name]) for name in names) - +from datasette.views.row import RowContext +from datasette.views.table import TableContext PAGES = { - page.name: page - for page in ( - PageContext( - name="database", - title="Database", - description="The page listing the tables, views and queries in a database, e.g. /fixtures.", - template="database.html", - context_class=DatabaseContext, - ), - PageContext( - name="query", - title="Query", - description="The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name).", - template="query.html", - context_class=QueryContext, - ), - PageContext( - name="table", - title="Table", - description="The page showing the rows in a table or SQL view, e.g. /fixtures/facetable.", - template="table.html", - extras_scope=ExtraScope.TABLE, - extra_keys=( - "actions", - "all_columns", - "columns", - "count", - "count_sql", - "custom_table_templates", - "database", - "database_color", - "display_columns", - "display_rows", - "expandable_columns", - "facet_results", - "facets_timed_out", - "filters", - "form_hidden_args", - "human_description_en", - "is_view", - "metadata", - "next_url", - "primary_keys", - "private", - "query", - "renderers", - "set_column_type_ui", - "sorted_facet_results", - "suggested_facets", - "table", - "table_definition", - "view_definition", - ), - keys=_shared( - "ok", - "rows", - "query_ms", - "select_templates", - "settings", - "alternate_url_json", - "url_csv", - "url_csv_path", - "url_csv_hidden_args", - ) - + _keys( - allow_execute_sql="True if the current actor can execute custom SQL against this database", - append_querystring="Function that appends additional querystring arguments to a URL", - count_limit="The maximum number of rows Datasette will count before showing an approximation", - datasette_allow_facet='The string "true" or "false" reflecting the allow_facet setting', - extra_wheres_for_ui="Extra where clauses from ?_where=, with links to remove them", - filter_columns="List of columns offered by the filter interface", - fix_path="Function that applies the base_url prefix to a path", - is_sortable="True if any of the displayed columns can be used to sort", - next="Pagination token for the next page, or None", - path_with_replaced_args="Function for building the current path with modified querystring arguments", - sort="Column the page is sorted by, or None", - sort_desc="Column the page is sorted by in descending order, or None", - supports_search="True if this table has full-text search configured", - top_table="Async function rendering the top_table plugin slot", - ), - ), - PageContext( - name="row", - title="Row", - description="The page showing an individual row, e.g. /fixtures/facetable/1.", - template="row.html", - extras_scope=ExtraScope.ROW, - extra_keys=( - "columns", - "database", - "database_color", - "foreign_key_tables", - "metadata", - "primary_keys", - "private", - "table", - ), - keys=_shared( - "ok", - "rows", - "query_ms", - "select_templates", - "settings", - "alternate_url_json", - "url_csv", - "url_csv_path", - "url_csv_hidden_args", - "renderers", - "display_columns", - "display_rows", - "custom_table_templates", - ) - + _keys( - primary_key_values="Values of the primary keys for this row, from the URL", - row_actions="Row actions made available by plugin hooks", - top_row="Async function rendering the top_row plugin slot", - ), - ), - ) + "database": DatabaseContext, + "query": QueryContext, + "table": TableContext, + "row": RowContext, } def documented_context_keys(page_name): "Set of every documented key for the named page, including base context keys" - page = PAGES[page_name] - return {key.name for key in BASE_CONTEXT_KEYS} | { - key.name for key in page.documented_keys() + return set(TEMPLATE_BASE_CONTEXT) | { + f.name for f in PAGES[page_name].documented_fields() } diff --git a/datasette/views/database.py b/datasette/views/database.py index f1756863..4f05c804 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -228,6 +228,10 @@ class DatabaseView(View): @dataclass class DatabaseContext(Context): + "The page listing the tables, views and queries in a database, e.g. /fixtures." + + template = "database.html" + database: str = field(metadata={"help": "The name of the database"}) private: bool = field( metadata={"help": "Boolean indicating if this is a private database"} @@ -281,6 +285,10 @@ class DatabaseContext(Context): @dataclass class QueryContext(Context): + "The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)." + + template = "query.html" + database: str = field(metadata={"help": "The name of the database being queried"}) database_color: str = field(metadata={"help": "The color of the database"}) query: dict = field( diff --git a/datasette/views/row.py b/datasette/views/row.py index f7475117..3ce02e21 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -23,8 +23,9 @@ from .table_extras import RowExtraContext, resolve_row_extras, table_extra_regis @dataclass class RowContext(Context): - "The page showing an individual row, e.g. /fixtures/facetable/1" + "The page showing an individual row, e.g. /fixtures/facetable/1." + template = "row.html" extras_scope = ExtraScope.ROW # Fields resolved by registered extras - their documentation comes diff --git a/datasette/views/table.py b/datasette/views/table.py index 356247ff..8ad7fa8c 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -59,8 +59,9 @@ from .table_extras import ( @dataclass class TableContext(Context): - "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable" + "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable." + template = "table.html" extras_scope = ExtraScope.TABLE # Fields resolved by registered extras - their documentation comes diff --git a/docs/template_context.rst b/docs/template_context.rst index a487a6e7..e0a34921 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -250,160 +250,160 @@ The page showing the rows in a table or SQL view, e.g. /fixtures/facetable. Rend Many of these keys are shared with the :ref:`JSON API ` for this page. -``actions`` +``actions`` - ``callable`` Table or view actions made available by plugin hooks -``all_columns`` +``all_columns`` - ``list`` All columns in the table, regardless of _col/_nocol filtering -``allow_execute_sql`` +``allow_execute_sql`` - ``bool`` True if the current actor can execute custom SQL against this database -``alternate_url_json`` +``alternate_url_json`` - ``str`` URL for the JSON version of this page -``append_querystring`` +``append_querystring`` - ``callable`` Function that appends additional querystring arguments to a URL -``columns`` +``columns`` - ``list`` Column names returned by this query -``count`` +``count`` - ``int`` Total count of rows matching these filters -``count_limit`` +``count_limit`` - ``int`` The maximum number of rows Datasette will count before showing an approximation -``count_sql`` +``count_sql`` - ``str`` SQL query used to calculate the total count -``custom_table_templates`` +``custom_table_templates`` - ``list`` Custom template names considered for this table -``database`` +``database`` - ``str`` Database name -``database_color`` +``database_color`` - ``str`` Color assigned to the database -``datasette_allow_facet`` +``datasette_allow_facet`` - ``str`` The string "true" or "false" reflecting the allow_facet setting -``display_columns`` +``display_columns`` - ``list`` Column metadata used by the HTML table display -``display_rows`` +``display_rows`` - ``list`` Row data formatted for the HTML table display -``expandable_columns`` +``expandable_columns`` - ``list`` Foreign key columns that can be expanded with labels -``extra_wheres_for_ui`` +``extra_wheres_for_ui`` - ``list`` Extra where clauses from ?_where=, with links to remove them -``facet_results`` +``facet_results`` - ``dict`` Results of facets calculated against this data -``facets_timed_out`` +``facets_timed_out`` - ``list`` Facet calculations that timed out -``filter_columns`` +``filter_columns`` - ``list`` List of columns offered by the filter interface -``filters`` +``filters`` - ``Filters`` Filters object used by the HTML table interface -``fix_path`` +``fix_path`` - ``callable`` Function that applies the base_url prefix to a path -``form_hidden_args`` +``form_hidden_args`` - ``list`` Hidden form arguments used by the HTML table interface -``human_description_en`` +``human_description_en`` - ``str`` Human-readable description of the filters -``is_sortable`` +``is_sortable`` - ``bool`` True if any of the displayed columns can be used to sort -``is_view`` +``is_view`` - ``bool`` Whether this resource is a view instead of a table -``metadata`` +``metadata`` - ``dict`` Metadata about the table, database or stored query -``next`` +``next`` - ``str`` Pagination token for the next page, or None -``next_url`` +``next_url`` - ``str`` Full URL for the next page of results -``ok`` +``ok`` - ``bool`` True if the data for this page was retrieved without errors -``path_with_replaced_args`` +``path_with_replaced_args`` - ``callable`` Function for building the current path with modified querystring arguments -``primary_keys`` +``primary_keys`` - ``list`` Primary keys for this table -``private`` +``private`` - ``bool`` Whether this resource is private to the current actor -``query`` +``query`` - ``dict`` Details of the underlying SQL query -``query_ms`` +``query_ms`` - ``float`` Time taken by the SQL queries for this page, in milliseconds -``renderers`` +``renderers`` - ``dict`` Alternative output renderers available for this table -``rows`` +``rows`` - ``list`` The rows for this page, as a list of dictionaries mapping column name to value -``select_templates`` +``select_templates`` - ``list`` List of template names that were considered for this page, the one used marked with an asterisk -``set_column_type_ui`` +``set_column_type_ui`` - ``dict`` Column type UI metadata for this table -``settings`` +``settings`` - ``dict`` Dictionary of Datasette's current settings -``sort`` +``sort`` - ``str`` Column the page is sorted by, or None -``sort_desc`` +``sort_desc`` - ``str`` Column the page is sorted by in descending order, or None -``sorted_facet_results`` +``sorted_facet_results`` - ``list`` Facet results sorted for display -``suggested_facets`` +``suggested_facets`` - ``list`` Suggestions for facets that might return interesting results -``supports_search`` +``supports_search`` - ``bool`` True if this table has full-text search configured -``table`` +``table`` - ``str`` Table name -``table_definition`` +``table_definition`` - ``str`` SQL definition for this table -``top_table`` +``top_table`` - ``callable`` Async function rendering the top_table plugin slot -``url_csv`` +``url_csv`` - ``str`` URL for the CSV export of this page -``url_csv_hidden_args`` +``url_csv_hidden_args`` - ``list`` (name, value) pairs for hidden form fields used by the CSV export form -``url_csv_path`` +``url_csv_path`` - ``str`` Path portion of the CSV export URL -``view_definition`` +``view_definition`` - ``str`` SQL definition for this view Row page @@ -413,76 +413,76 @@ The page showing an individual row, e.g. /fixtures/facetable/1. Rendered using t Many of these keys are shared with the :ref:`JSON API ` for this page. -``alternate_url_json`` +``alternate_url_json`` - ``str`` URL for the JSON version of this page -``columns`` +``columns`` - ``list`` Column names returned by this query -``custom_table_templates`` +``custom_table_templates`` - ``list`` Custom template names that were considered for displaying this table -``database`` +``database`` - ``str`` Database name -``database_color`` +``database_color`` - ``str`` Color assigned to the database -``display_columns`` +``display_columns`` - ``list`` Column objects formatted for the HTML table display -``display_rows`` +``display_rows`` - ``list`` Row data formatted for the HTML table display -``foreign_key_tables`` +``foreign_key_tables`` - ``list`` Tables that link to this row using foreign keys -``metadata`` +``metadata`` - ``dict`` Metadata about the table, database or stored query -``ok`` +``ok`` - ``bool`` True if the data for this page was retrieved without errors -``primary_key_values`` +``primary_key_values`` - ``list`` Values of the primary keys for this row, from the URL -``primary_keys`` +``primary_keys`` - ``list`` Primary keys for this table -``private`` +``private`` - ``bool`` Whether this resource is private to the current actor -``query_ms`` +``query_ms`` - ``float`` Time taken by the SQL queries for this page, in milliseconds -``renderers`` +``renderers`` - ``dict`` Dictionary mapping output format names (e.g. json) to their URLs for this page -``row_actions`` +``row_actions`` - ``list`` Row actions made available by plugin hooks -``rows`` +``rows`` - ``list`` The rows for this page, as a list of dictionaries mapping column name to value -``select_templates`` +``select_templates`` - ``list`` List of template names that were considered for this page, the one used marked with an asterisk -``settings`` +``settings`` - ``dict`` Dictionary of Datasette's current settings -``table`` +``table`` - ``str`` Table name -``top_row`` +``top_row`` - ``callable`` Async function rendering the top_row plugin slot -``url_csv`` +``url_csv`` - ``str`` URL for the CSV export of this page -``url_csv_hidden_args`` +``url_csv_hidden_args`` - ``list`` (name, value) pairs for hidden form fields used by the CSV export form -``url_csv_path`` +``url_csv_path`` - ``str`` Path portion of the CSV export URL .. [[[end]]] diff --git a/docs/template_context_doc.py b/docs/template_context_doc.py index 1539dc5e..c3ec77e2 100644 --- a/docs/template_context_doc.py +++ b/docs/template_context_doc.py @@ -1,11 +1,12 @@ """ -Cog helpers for generating docs/template_context.rst from the manifest -in datasette/template_contexts.py - same pattern as json_api_doc.py. +Cog helpers for generating docs/template_context.rst from the Context +dataclasses and TEMPLATE_BASE_CONTEXT - same pattern as json_api_doc.py. """ def template_context(cog): - from datasette.template_contexts import BASE_CONTEXT_KEYS, PAGES + from datasette.app import TEMPLATE_BASE_CONTEXT + from datasette.template_contexts import PAGES cog.out("\n") _section( @@ -19,36 +20,26 @@ def template_context(cog): ":ref:`plugin_hook_extra_template_vars` hook." ), ) - _untyped_keys(cog, BASE_CONTEXT_KEYS) + for name, doc in TEMPLATE_BASE_CONTEXT.items(): + cog.out("``{}``\n".format(name)) + cog.out(" {}\n\n".format(doc)) - for page in PAGES.values(): - _section( - cog, - "{} page".format(page.title), - "{} Rendered using the ``{}`` template.".format( - page.description, page.template - ), + for klass in PAGES.values(): + title = "{} page".format(klass.__name__.removesuffix("Context")) + intro = "{} Rendered using the ``{}`` template.".format( + klass.__doc__, klass.template ) - if page.context_class is not None: - for f in sorted( - page.context_class.documented_fields(), key=lambda f: f.name - ): - cog.out("``{}`` - ``{}``\n".format(f.name, f.type_name)) - cog.out(" {}\n\n".format(f.help)) - else: + _section(cog, title, intro) + if klass.extras_scope is not None: cog.out( "Many of these keys are shared with the :ref:`JSON API " "` for this page.\n\n" ) - _untyped_keys(cog, page.documented_keys()) + for f in sorted(klass.documented_fields(), key=lambda f: f.name): + cog.out("``{}`` - ``{}``\n".format(f.name, f.type_name)) + cog.out(" {}\n\n".format(f.help)) def _section(cog, title, intro): cog.out("{}\n{}\n\n".format(title, "-" * len(title))) cog.out("{}\n\n".format(intro)) - - -def _untyped_keys(cog, keys): - for key in keys: - cog.out("``{}``\n".format(key.name)) - cog.out(" {}\n\n".format(key.doc)) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 0c694b2e..b970cb8a 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -10,17 +10,11 @@ from dataclasses import dataclass, field import pytest -from datasette.app import Datasette +from datasette.app import Datasette, TEMPLATE_BASE_CONTEXT from datasette.extras import ExtraScope from datasette.fixtures import write_fixture_database -from datasette.template_contexts import ( - BASE_CONTEXT_KEYS, - PAGES, - documented_context_keys, -) +from datasette.template_contexts import PAGES, documented_context_keys from datasette.views import Context -from datasette.views.database import DatabaseContext, QueryContext -from datasette.views.table_extras import table_extra_registry def test_documented_fields(): @@ -36,14 +30,20 @@ def test_documented_fields(): ] -@pytest.mark.parametrize("klass", (DatabaseContext, QueryContext)) -def test_context_dataclass_fields_all_have_help(klass): +@pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) +def test_context_class_fields_all_have_help(klass): for context_field in klass.documented_fields(): - assert context_field.help, "{}.{} is missing help metadata".format( + assert context_field.help, "{}.{} is missing documentation".format( klass.__name__, context_field.name ) +@pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) +def test_context_class_has_docstring_and_template(klass): + assert klass.__doc__, "{} is missing a docstring".format(klass.__name__) + assert klass.template, "{} is missing a template".format(klass.__name__) + + def test_extra_field_documentation_comes_from_the_extra_class(): from datasette.views import extra_field from datasette.views.table_extras import CountExtra @@ -169,8 +169,8 @@ async def test_template_context_matches_documented_contract( undocumented = actual - documented no_longer_present = documented - actual assert not undocumented, ( - "Undocumented keys in {} template context: {} - document them in " - "datasette/template_contexts.py".format(page_name, sorted(undocumented)) + "Undocumented keys in {} template context: {} - add them to the " + "page's Context class".format(page_name, sorted(undocumented)) ) assert not no_longer_present, ( "Documented keys missing from {} template context: {} - this would " @@ -178,53 +178,21 @@ async def test_template_context_matches_documented_contract( ) -def test_table_context_fields_match_documented_contract(): - from datasette.views.table import TableContext - - assert {f.name for f in TableContext.documented_fields()} == { - key.name for key in PAGES["table"].documented_keys() - } - - -def test_row_context_fields_match_documented_contract(): - from datasette.views.row import RowContext - - assert {f.name for f in RowContext.documented_fields()} == { - key.name for key in PAGES["row"].documented_keys() - } - - def test_base_context_keys_all_have_docs(): - for key in BASE_CONTEXT_KEYS: - assert key.doc, "Base context key {} is missing docs".format(key.name) - - -@pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) -def test_page_documented_keys_all_have_docs(page): - for key in page.documented_keys(): - assert key.doc, "{} page key {} is missing docs".format(page.name, key.name) + for name, doc in TEMPLATE_BASE_CONTEXT.items(): + assert doc, "Base context key {} is missing docs".format(name) def test_template_context_docs_cover_every_documented_key(): docs_path = pathlib.Path(__file__).parent.parent / "docs" / "template_context.rst" assert docs_path.exists(), "docs/template_context.rst is missing" docs = docs_path.read_text() - for key in BASE_CONTEXT_KEYS: - assert "``{}``".format(key.name) in docs, key.name - for page in PAGES.values(): - assert page.title in docs, page.title - for key in page.documented_keys(): - assert "``{}``".format(key.name) in docs, "{} ({} page)".format( - key.name, page.name + for name in TEMPLATE_BASE_CONTEXT: + assert "``{}``".format(name) in docs, name + for page_name, klass in PAGES.items(): + title = "{} page".format(klass.__name__.removesuffix("Context")) + assert title in docs, title + for context_field in klass.documented_fields(): + assert "``{}``".format(context_field.name) in docs, "{} ({} page)".format( + context_field.name, page_name ) - - -@pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) -def test_page_extra_keys_are_registered_extras(page): - for name in page.extra_keys: - cls = table_extra_registry.classes_by_name.get(name) - assert cls is not None, "{} is not a registered extra".format(name) - assert page.extras_scope is not None - assert cls.available_for( - page.extras_scope - ), "{} extra is not available for scope {}".format(name, page.extras_scope) From 2d3c85dfc0b802dcb6736f17cb15e84f67d4f921 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 16 Jun 2026 18:02:58 -0700 Subject: [PATCH 009/167] Add create table UI Adds a permission-gated database action that opens a create table modal on database pages, backed by the existing create-table JSON API. The modal starts with an id integer primary key column plus a blank text column, supports SQLite type selection, and shows custom column type controls only when the actor can set column types. Selected custom column types are applied after table creation with follow-up set-column-type API calls. Includes styling plus HTML and Playwright coverage for the action payload and create-table flow. --- datasette/static/app.css | 323 ++++++++++++++ datasette/static/edit-tools.js | 679 ++++++++++++++++++++++++++++++ datasette/templates/database.html | 4 + datasette/views/database.py | 120 +++++- tests/test_playwright.py | 53 +++ tests/test_table_html.py | 144 +++++++ 6 files changed, 1303 insertions(+), 20 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 5fe4502d..ce0c46a8 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1749,6 +1749,289 @@ datasette-autocomplete input[type="text"], cursor: not-allowed; } +dialog.table-create-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; + width: min(760px, calc(100vw - 32px)); + max-width: 95vw; + max-height: min(780px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.table-create-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.table-create-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +.table-create-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 12px; + flex-shrink: 0; + min-width: 0; +} + +.table-create-dialog .modal-title { + display: flex; + align-items: center; + min-width: 0; + max-width: 100%; + font-size: 1rem; + font-weight: 600; + color: var(--ink); +} + +.table-create-form { + display: flex; + flex: 1 1 auto; + min-height: 0; + flex-direction: column; +} + +.table-create-error { + border-left: 4px solid #b91c1c; + border-radius: 4px; + background: #fff1f1; + color: #7f1d1d; + font-size: 0.9rem; + margin: 12px 24px 0; + padding: 10px 12px; +} + +.table-create-error:focus { + outline: 3px solid rgba(185, 28, 28, 0.18); + outline-offset: 2px; +} + +.table-create-fields { + display: grid; + gap: 18px; + padding: 16px 24px 24px; + overflow-y: auto; +} + +.table-create-field { + display: grid; + grid-template-columns: minmax(120px, 180px) minmax(0, 1fr); + gap: 12px; + align-items: start; +} + +.table-create-label, +.table-create-columns-heading { + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.82rem; +} + +.table-create-label { + padding-top: 8px; +} + +.table-create-column-label { + border: 0; + clip: rect(0 0 0 0); + height: 1px; + margin: -1px; + overflow: hidden; + padding: 0; + position: absolute; + white-space: nowrap; + width: 1px; +} + +.table-create-input { + box-sizing: border-box; + min-width: 0; + border: 1px solid var(--rule); + border-radius: 5px; + padding: 8px 10px; + color: var(--ink); + background: #fff; + font: inherit; +} + +.table-create-input-placeholder { + color: var(--muted); +} + +.table-create-custom-column-type option { + color: var(--ink); +} + +.table-create-custom-column-type option[value=""] { + color: var(--muted); +} + +.table-create-table-name { + width: 100%; +} + +.table-create-input:focus { + border-color: var(--accent); + outline: 3px solid rgba(26, 86, 219, 0.12); +} + +.table-create-columns { + display: grid; + gap: 10px; +} + +.table-create-columns-heading { + font-weight: 600; +} + +.table-create-column-list { + display: grid; + gap: 8px; +} + +.table-create-column-row { + display: grid; + grid-template-columns: minmax(140px, 1.2fr) minmax(7.5rem, 0.7fr) minmax(12rem, 1fr) minmax(3.5rem, max-content) 32px; + align-items: center; + gap: 8px; + min-width: 0; + position: relative; +} + +.table-create-primary-key { + display: inline-flex; + align-items: center; + gap: 6px; + color: var(--ink); + font-size: 0.85rem; + min-width: 0; + white-space: nowrap; + justify-self: center; +} + +.table-create-primary-key-input { + margin: 0; +} + +.table-create-remove-column { + appearance: none; + border: 1px solid rgba(74, 85, 104, 0.24); + background: transparent; + color: #4a5568; + border-radius: 4px; + cursor: pointer; + display: inline-grid; + place-items: center; + height: 32px; + width: 32px; + padding: 0; +} + +.table-create-remove-column:hover, +.table-create-remove-column:focus { + background: rgba(74, 85, 104, 0.07); +} + +.table-create-remove-column:focus { + outline: 3px solid #b3d4ff; + outline-offset: 1px; +} + +.table-create-remove-column svg { + display: block; +} + +.table-create-add-column { + appearance: none; + justify-self: start; + border: 1px solid var(--rule); + border-radius: 5px; + background: #fff; + color: var(--accent); + cursor: pointer; + font: inherit; + font-size: 0.85rem; + padding: 7px 10px; +} + +.table-create-add-column:hover, +.table-create-add-column:focus { + background: #f8fafc; +} + +.table-create-add-column:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 1px; +} + +.table-create-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +.table-create-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.table-create-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.table-create-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.table-create-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.table-create-dialog .btn-primary:hover { + background: #1949b8; +} + +.table-create-dialog .btn:disabled, +.table-create-add-column:disabled, +.table-create-remove-column:disabled { + opacity: 0.55; + cursor: not-allowed; +} + .row-link-with-actions { display: inline-flex; align-items: center; @@ -1892,6 +2175,46 @@ datasette-autocomplete input[type="text"], padding-right: 18px; } + dialog.table-create-dialog { + width: 95vw; + max-height: 85vh; + border-radius: 0.5rem; + } + + .table-create-dialog .modal-header, + .table-create-fields { + padding-left: 18px; + padding-right: 18px; + } + + .table-create-error { + margin-left: 18px; + margin-right: 18px; + } + + .table-create-field { + grid-template-columns: 1fr; + gap: 5px; + } + + .table-create-label { + padding-top: 0; + } + + .table-create-column-row { + grid-template-columns: minmax(0, 1fr) 8.5rem 3.5rem 32px; + align-items: end; + } + + .table-create-custom-column-type { + grid-column: 1 / -1; + } + + .table-create-dialog .modal-footer { + padding-left: 18px; + padding-right: 18px; + } + .row-inline-action { min-height: 30px; min-width: 30px; diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 8142d02b..4b93c4fd 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -2,6 +2,8 @@ var ROW_DELETE_DIALOG_ID = "row-delete-dialog"; var rowDeleteDialogState = null; var ROW_EDIT_DIALOG_ID = "row-edit-dialog"; var rowEditDialogState = null; +var TABLE_CREATE_DIALOG_ID = "table-create-dialog"; +var tableCreateDialogState = null; function ensureRowMutationStatus(manager) { var status = document.querySelector(".row-mutation-status"); @@ -43,6 +45,682 @@ function hideRowMutationStatus() { status.textContent = ""; } +function databaseCreateTableData() { + return ( + window._datasetteDatabaseData && + window._datasetteDatabaseData.createTable + ); +} + +function tableCreateColumnTypes() { + var data = databaseCreateTableData() || {}; + return data.columnTypes && data.columnTypes.length + ? data.columnTypes + : ["text", "integer", "float", "blob"]; +} + +function tableCreateCustomColumnTypes() { + var data = databaseCreateTableData() || {}; + return data.customColumnTypes || []; +} + +function tableCreateCustomColumnType(name) { + var options = tableCreateCustomColumnTypes(); + for (var i = 0; i < options.length; i += 1) { + if (options[i].name === name) { + return options[i]; + } + } + return null; +} + +function tableCreateCustomTypeAppliesToSqliteType(option, sqliteType) { + return ( + option && + option.sqliteTypes && + option.sqliteTypes.indexOf(sqliteType) !== -1 + ); +} + +function tableCreateDialogSignature(state) { + if (!state || !state.form) { + return ""; + } + var columns = []; + state.columnList + .querySelectorAll(".table-create-column-row") + .forEach(function (row) { + columns.push({ + name: row.querySelector(".table-create-column-name").value, + type: row.querySelector(".table-create-column-type").value, + customType: + ( + row.querySelector(".table-create-custom-column-type") || { + value: "", + } + ).value || "", + pk: row.querySelector(".table-create-primary-key-input").checked, + }); + }); + return JSON.stringify({ + table: state.tableName.value, + columns: columns, + }); +} + +function tableCreateDialogHasChanges(state) { + return ( + !!state && + !state.isSaving && + tableCreateDialogSignature(state) !== state.initialSignature + ); +} + +function clearTableCreateDialogError(state) { + state.error.hidden = true; + state.error.textContent = ""; + state.dialog.removeAttribute("aria-describedby"); +} + +function showTableCreateDialogError(state, message) { + state.error.hidden = false; + state.error.textContent = message; + state.dialog.setAttribute("aria-describedby", "table-create-error"); + state.error.focus(); +} + +function setTableCreateDialogSaving(state, isSaving) { + state.isSaving = isSaving; + state.cancelButton.disabled = isSaving; + state.saveButton.disabled = isSaving; + state.addColumnButton.disabled = isSaving; + state.saveButton.textContent = isSaving ? "Creating..." : "Create table"; + state.columnList + .querySelectorAll("input, select, button") + .forEach(function (control) { + control.disabled = isSaving; + }); +} + +function tableCreateSelectTypeValue(select, type) { + var options = tableCreateColumnTypes(); + options.forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option; + optionElement.textContent = option; + select.appendChild(optionElement); + }); + select.value = options.indexOf(type) === -1 ? options[0] : type; +} + +function updateTableCreateCustomColumnTypePlaceholder(select) { + select.classList.toggle( + "table-create-input-placeholder", + !select.value, + ); +} + +function createTableCustomColumnTypeSelect() { + var options = tableCreateCustomColumnTypes(); + var select = document.createElement("select"); + select.className = "table-create-input table-create-custom-column-type"; + select.setAttribute("aria-label", "Custom column type"); + var blankOption = document.createElement("option"); + blankOption.value = ""; + blankOption.textContent = "- custom type -"; + select.appendChild(blankOption); + options.forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option.name; + optionElement.textContent = option.description + ? option.description + " (" + option.name + ")" + : option.name; + select.appendChild(optionElement); + }); + updateTableCreateCustomColumnTypePlaceholder(select); + return select; +} + +function syncTableCreateCustomTypeForSqliteType(row) { + var typeSelect = row.querySelector(".table-create-column-type"); + var customTypeSelect = row.querySelector(".table-create-custom-column-type"); + if (!typeSelect || !customTypeSelect || !customTypeSelect.value) { + return; + } + var option = tableCreateCustomColumnType(customTypeSelect.value); + if (!tableCreateCustomTypeAppliesToSqliteType(option, typeSelect.value)) { + customTypeSelect.value = ""; + updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + } +} + +function createTableColumnRow(state, column) { + var index = state.nextColumnIndex; + state.nextColumnIndex += 1; + + var row = document.createElement("div"); + row.className = "table-create-column-row"; + + var nameId = "table-create-column-name-" + index; + var nameLabel = document.createElement("label"); + nameLabel.className = "table-create-column-label"; + nameLabel.setAttribute("for", nameId); + nameLabel.textContent = "Column"; + + var nameInput = document.createElement("input"); + nameInput.id = nameId; + nameInput.className = "table-create-input table-create-column-name"; + nameInput.type = "text"; + nameInput.required = true; + nameInput.autocomplete = "off"; + nameInput.value = column && column.name ? column.name : ""; + + var typeSelect = document.createElement("select"); + typeSelect.className = "table-create-input table-create-column-type"; + typeSelect.setAttribute("aria-label", "Column type"); + tableCreateSelectTypeValue(typeSelect, column && column.type); + + var customTypeSelect = createTableCustomColumnTypeSelect(); + if (column && column.customType) { + customTypeSelect.value = column.customType; + } + updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + + var pkLabel = document.createElement("label"); + pkLabel.className = "table-create-primary-key"; + var pkInput = document.createElement("input"); + pkInput.type = "checkbox"; + pkInput.className = "table-create-primary-key-input"; + pkInput.checked = !!(column && column.primaryKey); + var pkText = document.createElement("span"); + pkText.textContent = "PK"; + pkText.title = "Primary key"; + pkLabel.appendChild(pkInput); + pkLabel.appendChild(pkText); + + var removeButton = document.createElement("button"); + removeButton.type = "button"; + removeButton.className = "table-create-remove-column"; + removeButton.setAttribute("aria-label", "Remove column"); + removeButton.title = "Remove column"; + removeButton.innerHTML = + ''; + + row.appendChild(nameLabel); + row.appendChild(nameInput); + row.appendChild(typeSelect); + if (tableCreateCustomColumnTypes().length) { + row.appendChild(customTypeSelect); + } + row.appendChild(pkLabel); + row.appendChild(removeButton); + + removeButton.addEventListener("click", function () { + if (state.isSaving) { + return; + } + row.remove(); + clearTableCreateDialogError(state); + var nextInput = state.columnList.querySelector( + ".table-create-column-name", + ); + if (nextInput) { + nextInput.focus(); + } else { + state.addColumnButton.focus(); + } + }); + + nameInput.addEventListener("input", function () { + clearTableCreateDialogError(state); + }); + typeSelect.addEventListener("change", function () { + clearTableCreateDialogError(state); + syncTableCreateCustomTypeForSqliteType(row); + }); + customTypeSelect.addEventListener("change", function () { + clearTableCreateDialogError(state); + updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + var option = tableCreateCustomColumnType(customTypeSelect.value); + if ( + option && + option.fixedSqliteType && + tableCreateColumnTypes().indexOf(option.fixedSqliteType) !== -1 + ) { + typeSelect.value = option.fixedSqliteType; + } + }); + pkInput.addEventListener("change", function () { + clearTableCreateDialogError(state); + }); + + return row; +} + +function addTableCreateColumn(state, column) { + var row = createTableColumnRow(state, column || { type: "text" }); + state.columnList.appendChild(row); + return row; +} + +function resetTableCreateDialog(state) { + state.nextColumnIndex = 0; + state.tableName.value = ""; + state.columnList.textContent = ""; + addTableCreateColumn(state, { + name: "id", + type: "integer", + primaryKey: true, + }); + addTableCreateColumn(state, { + name: "", + type: "text", + primaryKey: false, + }); + state.initialSignature = tableCreateDialogSignature(state); +} + +function collectTableCreatePayload(state) { + var payload = { + table: state.tableName.value.trim(), + columns: [], + }; + var primaryKeys = []; + state.columnList + .querySelectorAll(".table-create-column-row") + .forEach(function (row) { + var name = row.querySelector(".table-create-column-name").value.trim(); + var type = row.querySelector(".table-create-column-type").value; + payload.columns.push({ name: name, type: type }); + if (row.querySelector(".table-create-primary-key-input").checked) { + primaryKeys.push(name); + } + }); + if (primaryKeys.length === 1) { + payload.pk = primaryKeys[0]; + } else if (primaryKeys.length > 1) { + payload.pks = primaryKeys; + } + return payload; +} + +function collectTableCreateColumnTypeAssignments(state) { + var assignments = []; + state.columnList + .querySelectorAll(".table-create-column-row") + .forEach(function (row) { + var customTypeSelect = row.querySelector( + ".table-create-custom-column-type", + ); + if (!customTypeSelect || !customTypeSelect.value) { + return; + } + assignments.push({ + column: row.querySelector(".table-create-column-name").value.trim(), + columnType: customTypeSelect.value, + sqliteType: row.querySelector(".table-create-column-type").value, + }); + }); + return assignments; +} + +function validateTableCreatePayload(payload) { + if (!payload.table) { + return "Table name is required."; + } + if (payload.table.indexOf("\n") !== -1) { + return "Table name cannot contain newlines."; + } + if (/^sqlite_/i.test(payload.table)) { + return "Table name cannot start with sqlite_."; + } + if (!payload.columns.length) { + return "At least one column is required."; + } + var seen = {}; + var supportedTypes = tableCreateColumnTypes(); + for (var i = 0; i < payload.columns.length; i += 1) { + var column = payload.columns[i]; + if (!column.name) { + return "Column name is required."; + } + if (column.name.indexOf("\n") !== -1) { + return "Column names cannot contain newlines."; + } + var columnKey = column.name.toLowerCase(); + if (seen[columnKey]) { + return "Duplicate column name: " + column.name; + } + seen[columnKey] = true; + if (supportedTypes.indexOf(column.type) === -1) { + return "Unsupported column type: " + column.type; + } + } + return null; +} + +function validateTableCreateColumnTypeAssignments(assignments) { + for (var i = 0; i < assignments.length; i += 1) { + var assignment = assignments[i]; + var option = tableCreateCustomColumnType(assignment.columnType); + if (!option) { + return "Unknown custom column type: " + assignment.columnType; + } + if (!tableCreateCustomTypeAppliesToSqliteType(option, assignment.sqliteType)) { + return ( + "Custom type " + + assignment.columnType + + " cannot be used with SQLite type " + + assignment.sqliteType + + "." + ); + } + } + return null; +} + +function fallbackTableUrl(tableName) { + var data = databaseCreateTableData() || {}; + if (!data.path) { + return null; + } + return data.path.replace(/\/-\/create$/, "/" + encodeURIComponent(tableName)); +} + +function tableCreateSetColumnTypeUrl(responseData, payload) { + var tableUrl = + responseData.table_url || fallbackTableUrl(responseData.table || payload.table); + if (!tableUrl) { + return null; + } + var url = new URL(tableUrl, location.href); + url.hash = ""; + url.search = ""; + url.pathname = url.pathname.replace(/\/$/, "") + "/-/set-column-type"; + return url.toString(); +} + +async function assignTableCreateColumnTypes(responseData, payload, assignments) { + if (!assignments.length) { + return; + } + var url = tableCreateSetColumnTypeUrl(responseData, payload); + if (!url) { + throw new Error("Could not find the set column type URL."); + } + for (var i = 0; i < assignments.length; i += 1) { + var assignment = assignments[i]; + var response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ + column: assignment.column, + column_type: { + type: assignment.columnType, + }, + }), + }); + var data = null; + try { + data = await response.json(); + } catch (_error) { + data = null; + } + if (!response.ok || (data && data.ok === false)) { + var error = rowMutationRequestError(response, data); + throw new Error( + "Created table, but could not set custom type for " + + assignment.column + + ": " + + error.message, + ); + } + } +} + +async function saveTableCreateDialog(state) { + if (state.isSaving) { + return; + } + var data = databaseCreateTableData(); + if (!data || !data.path) { + showTableCreateDialogError(state, "Could not find the create table URL."); + return; + } + clearTableCreateDialogError(state); + var payload = collectTableCreatePayload(state); + var columnTypeAssignments = collectTableCreateColumnTypeAssignments(state); + var validationError = validateTableCreatePayload(payload); + if (validationError) { + showTableCreateDialogError(state, validationError); + return; + } + var columnTypeValidationError = validateTableCreateColumnTypeAssignments( + columnTypeAssignments, + ); + if (columnTypeValidationError) { + showTableCreateDialogError(state, columnTypeValidationError); + return; + } + setTableCreateDialogSaving(state, true); + try { + var response = await fetch(data.path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var responseData = null; + try { + responseData = await response.json(); + } catch (_error) { + responseData = null; + } + if (!response.ok || (responseData && responseData.ok === false)) { + throw rowMutationRequestError(response, responseData); + } + await assignTableCreateColumnTypes( + responseData, + payload, + columnTypeAssignments, + ); + var tableUrl = + responseData.table_url || fallbackTableUrl(responseData.table || payload.table); + state.shouldRestoreFocus = false; + state.dialog.close(); + if (tableUrl) { + location.href = tableUrl; + } else { + location.reload(); + } + } catch (error) { + setTableCreateDialogSaving(state, false); + showTableCreateDialogError( + state, + error.message || "Could not create table", + ); + } +} + +function confirmDiscardTableCreateChanges(state) { + if (!tableCreateDialogHasChanges(state)) { + return true; + } + return window.confirm("Discard this new table?"); +} + +function closeTableCreateDialogIfConfirmed(state) { + if (!state || state.isSaving) { + return false; + } + if (!confirmDiscardTableCreateChanges(state)) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + +function ensureTableCreateDialog(manager) { + if (tableCreateDialogState) { + return tableCreateDialogState; + } + if (!window.HTMLDialogElement) { + return null; + } + + var dialog = document.createElement("dialog"); + dialog.id = TABLE_CREATE_DIALOG_ID; + dialog.className = "table-create-dialog"; + dialog.setAttribute("aria-labelledby", "table-create-title"); + dialog.innerHTML = ` +
+ Create table +
+
+ +
+
+ + +
+
+
Columns
+
+ +
+
+
+ + +
+
+ `; + document.body.appendChild(dialog); + + tableCreateDialogState = { + dialog: dialog, + form: dialog.querySelector(".table-create-form"), + title: dialog.querySelector(".modal-title"), + error: dialog.querySelector(".table-create-error"), + fields: dialog.querySelector(".table-create-fields"), + tableName: dialog.querySelector(".table-create-table-name"), + columnList: dialog.querySelector(".table-create-column-list"), + addColumnButton: dialog.querySelector(".table-create-add-column"), + cancelButton: dialog.querySelector(".table-create-cancel"), + saveButton: dialog.querySelector(".table-create-save"), + currentButton: null, + shouldRestoreFocus: true, + isSaving: false, + initialSignature: "", + nextColumnIndex: 0, + manager: manager, + }; + + tableCreateDialogState.form.addEventListener("submit", function (ev) { + ev.preventDefault(); + saveTableCreateDialog(tableCreateDialogState); + }); + + tableCreateDialogState.addColumnButton.addEventListener("click", function () { + if (tableCreateDialogState.isSaving) { + return; + } + var row = addTableCreateColumn(tableCreateDialogState, { type: "text" }); + clearTableCreateDialogError(tableCreateDialogState); + row.querySelector(".table-create-column-name").focus(); + }); + + tableCreateDialogState.cancelButton.addEventListener("click", function () { + closeTableCreateDialogIfConfirmed(tableCreateDialogState); + }); + + tableCreateDialogState.tableName.addEventListener("input", function () { + clearTableCreateDialogError(tableCreateDialogState); + }); + + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog) { + closeTableCreateDialogIfConfirmed(tableCreateDialogState); + } + }); + + dialog.addEventListener("keydown", function (ev) { + if (ev.key !== "Escape") { + return; + } + ev.preventDefault(); + closeTableCreateDialogIfConfirmed(tableCreateDialogState); + }); + + dialog.addEventListener("cancel", function (ev) { + ev.preventDefault(); + closeTableCreateDialogIfConfirmed(tableCreateDialogState); + }); + + dialog.addEventListener("close", function () { + var state = tableCreateDialogState; + clearTableCreateDialogError(state); + setTableCreateDialogSaving(state, false); + if ( + state.shouldRestoreFocus && + state.currentButton && + document.contains(state.currentButton) + ) { + state.currentButton.focus(); + } + }); + + return tableCreateDialogState; +} + +function openTableCreateDialog(button, manager) { + var data = databaseCreateTableData(); + if (!data) { + return; + } + var state = ensureTableCreateDialog(manager); + if (!state) { + return; + } + + var menu = button.closest("details"); + if (menu) { + menu.open = false; + } + state.manager = manager; + state.currentButton = button; + state.shouldRestoreFocus = true; + state.title.textContent = "Create a table in " + data.databaseName; + clearTableCreateDialogError(state); + resetTableCreateDialog(state); + if (!state.dialog.open) { + state.dialog.showModal(); + } + state.tableName.focus(); +} + +function initTableCreateActions(manager) { + if (!window.fetch || !window.HTMLDialogElement || !databaseCreateTableData()) { + return; + } + document.addEventListener("click", function (ev) { + var button = ev.target.closest( + 'button[data-database-action="create-table"]', + ); + if (!button) { + return; + } + ev.preventDefault(); + openTableCreateDialog(button, manager); + }); +} + function setRowDeleteDialogBusy(state, isBusy) { state.isBusy = isBusy; state.confirmButton.disabled = isBusy; @@ -2017,6 +2695,7 @@ document.addEventListener("datasette_init", function (evt) { const { detail: manager } = evt; registerBuiltinColumnFieldPlugins(manager); + initTableCreateActions(manager); initRowInsertActions(manager); initRowEditActions(manager); initRowDeleteActions(manager); diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 371f6a22..23eeb571 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -6,6 +6,10 @@ {{- super() -}} {% include "_codemirror.html" %} {% include "_sql_parameter_styles.html" %} +{% if database_page_data.createTable %} + + +{% endif %} {% endblock %} {% block body_class %}db db-{{ database|to_css_class }}{% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index cd9565c6..db70b135 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -13,7 +13,8 @@ import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted -from datasette.resources import DatabaseResource, QueryResource +from datasette.column_types import SQLiteType +from datasette.resources import DatabaseResource, QueryResource, TableResource from datasette.stored_queries import stored_query_to_dict from datasette.write_sql import QueryWriteRejected from datasette.utils import ( @@ -46,6 +47,18 @@ from .table_extras import ( ) from . import Context +CREATE_TABLE_COLUMN_TYPES = ["text", "integer", "float", "blob"] +CREATE_TABLE_SQLITE_TYPES = { + "text": SQLiteType.TEXT, + "integer": SQLiteType.INTEGER, + "float": SQLiteType.REAL, + "blob": SQLiteType.BLOB, +} +CREATE_TABLE_TYPE_FOR_SQLITE_TYPE = { + sqlite_type: column_type + for column_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() +} + class DatabaseView(View): async def get(self, request, datasette): @@ -117,21 +130,36 @@ class DatabaseView(View): else len(stored_queries) ) + # Resolve the registered database-level actions for this database in + # one batched query, seeding the request permission cache so allowed() + # calls made inside plugin hooks below are served from the cache. + database_action_permissions = await datasette.allowed_many( + actions=[ + name + for name, action in datasette.actions.items() + if action.resource_class is DatabaseResource + ], + resource=DatabaseResource(database), + actor=request.actor, + ) + create_table_ui = await _database_create_table_ui( + datasette, request, db, database, database_action_permissions + ) + async def database_actions(): - # Resolve the registered database-level actions for this - # database in one batched query, seeding the request permission - # cache so that allowed() calls made inside the plugin hooks - # below are served from the cache - await datasette.allowed_many( - actions=[ - name - for name, action in datasette.actions.items() - if action.resource_class is DatabaseResource - ], - resource=DatabaseResource(database), - actor=request.actor, - ) links = [] + if create_table_ui: + links.append( + { + "type": "button", + "label": "Create table", + "description": "Create a new table in this database.", + "attrs": { + "aria-label": "Create table in {}".format(database), + "data-database-action": "create-table", + }, + } + ) for hook in pm.hook.database_actions( datasette=datasette, database=database, @@ -211,6 +239,9 @@ class DatabaseView(View): ), metadata=metadata, database_color=db.color, + database_page_data=( + {"createTable": create_table_ui} if create_table_ui else {} + ), database_actions=database_actions, show_hidden=request.args.get("_show_hidden"), editable=True, @@ -263,6 +294,9 @@ class DatabaseContext(Context): ) metadata: dict = field(metadata={"help": "Metadata for the database"}) database_color: str = field(metadata={"help": "The color assigned to the database"}) + database_page_data: dict = field( + metadata={"help": "JSON data used by JavaScript on the database page"} + ) database_actions: callable = field( metadata={ "help": "Callable returning list of action links for the database menu" @@ -292,6 +326,57 @@ class DatabaseContext(Context): ) +async def _database_create_table_ui( + datasette, request, db, database_name, database_action_permissions +): + if not db.is_mutable: + return None + if not database_action_permissions.get("create-table"): + return None + data = { + "path": "{}/-/create".format(datasette.urls.database(database_name)), + "databaseName": database_name, + "columnTypes": CREATE_TABLE_COLUMN_TYPES, + } + can_set_column_type = await datasette.allowed( + action="set-column-type", + resource=TableResource(database=database_name, table="__new_table__"), + actor=request.actor, + ) + if can_set_column_type: + data["customColumnTypes"] = _custom_column_type_options_for_create_table( + datasette + ) + return data + + +def _custom_column_type_options_for_create_table(datasette): + options = [] + for name, ct_cls in sorted(datasette._column_types.items()): + sqlite_types = getattr(ct_cls, "sqlite_types", None) + if sqlite_types is None: + option_sqlite_types = CREATE_TABLE_COLUMN_TYPES[:] + else: + option_sqlite_types = [ + create_table_type + for create_table_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() + if sqlite_type in sqlite_types + ] + if not option_sqlite_types: + continue + option = { + "name": name, + "description": ct_cls.description, + "sqliteTypes": option_sqlite_types, + } + if sqlite_types is not None and len(sqlite_types) == 1: + fixed_sqlite_type = CREATE_TABLE_TYPE_FOR_SQLITE_TYPE.get(sqlite_types[0]) + if fixed_sqlite_type is not None: + option["fixedSqliteType"] = fixed_sqlite_type + options.append(option) + return options + + @dataclass class QueryContext(Context): database: str = field(metadata={"help": "The name of the database being queried"}) @@ -1069,12 +1154,7 @@ class TableCreateView(BaseView): "replace", "alter", } - _supported_column_types = { - "text", - "integer", - "float", - "blob", - } + _supported_column_types = set(CREATE_TABLE_COLUMN_TYPES) # Any string that does not contain a newline or start with sqlite_ _table_name_re = re.compile(r"^(?!sqlite_)[^\n]+$") diff --git a/tests/test_playwright.py b/tests/test_playwright.py index a8c5aa4b..0c7042f0 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -117,6 +117,10 @@ def write_playwright_config(config_path): { "databases": { "data": { + "permissions": { + "create-table": True, + "set-column-type": True, + }, "tables": { "projects": { "label_column": "title", @@ -275,6 +279,55 @@ def test_datasette_homepage_contains_datasette(page, datasette_server): assert "Datasette" in page.locator("body").inner_text() +@pytest.mark.playwright +def test_create_table_flow(page, datasette_server): + page.goto(f"{datasette_server}data") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-database-action="create-table"]').click() + + dialog = page.locator("#table-create-dialog") + dialog.wait_for() + assert dialog.locator(".modal-title").inner_text() == "Create a table in data" + placeholder_select = dialog.locator(".table-create-custom-column-type").nth(0) + assert placeholder_select.input_value() == "" + assert ( + placeholder_select.locator("option:checked").inner_text() == "- custom type -" + ) + assert "table-create-input-placeholder" in placeholder_select.get_attribute("class") + dialog.locator('input[name="table"]').fill("playwright_created") + dialog.locator(".table-create-column-name").nth(1).fill("title") + dialog.locator(".table-create-add-column").click() + dialog.locator(".table-create-column-name").nth(2).fill("score") + dialog.locator(".table-create-column-type").nth(2).select_option("integer") + dialog.locator(".table-create-add-column").click() + dialog.locator(".table-create-column-name").nth(3).fill("metadata") + dialog.locator(".table-create-column-type").nth(3).select_option("integer") + dialog.locator(".table-create-custom-column-type").nth(3).select_option("json") + assert dialog.locator(".table-create-column-type").nth(3).input_value() == "text" + assert "table-create-input-placeholder" not in dialog.locator( + ".table-create-custom-column-type" + ).nth(3).get_attribute("class") + + dialog.locator(".table-create-save").click() + page.wait_for_url("**/data/playwright_created") + assert "playwright_created" in page.locator("h1").inner_text() + + response = httpx.get( + f"{datasette_server}data/playwright_created.json?_extra=columns,column_types" + ) + response.raise_for_status() + data = response.json() + assert data["columns"] == [ + "id", + "title", + "score", + "metadata", + ] + assert data["column_types"] == { + "metadata": {"type": "json", "config": None}, + } + + @pytest.mark.playwright def test_navigation_search_tracks_and_renders_recent_items(page, datasette_server): page.goto(datasette_server) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index aa67bb3f..887fdf50 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -23,6 +23,23 @@ def table_data_from_soup(soup): return json.loads(match.group(1)) +def database_data_from_soup(soup): + import json + import re + + database_script = [ + s + for s in soup.find_all("script") + if "_datasetteDatabaseData" in (s.string or "") + ][0] + match = re.search( + r"window\._datasetteDatabaseData\s*=\s*({.*?});", + database_script.string, + re.DOTALL, + ) + return json.loads(match.group(1)) + + @pytest.mark.asyncio @pytest.mark.parametrize( "path,expected_definition_sql", @@ -934,6 +951,133 @@ async def test_row_delete_action_data_attributes(): ds.close() +@pytest.mark.asyncio +async def test_database_create_table_action_button_and_data(): + ds = Datasette( + [], + config={ + "databases": { + "data": { + "permissions": { + "create-table": {"id": "root"}, + }, + }, + }, + }, + ) + try: + db = ds.add_database( + Database(ds, memory_name="test_database_create_table_action"), name="data" + ) + await db.execute_write_script(""" + create table items (id integer primary key, name text); + """) + + response = await ds.client.get("/data", actor={"id": "root"}) + assert response.status_code == 200 + soup = Soup(response.text, "html.parser") + + button = soup.select_one( + 'button.action-menu-button[data-database-action="create-table"]' + ) + assert button is not None + assert button["aria-label"] == "Create table in data" + assert button["role"] == "menuitem" + description = button.find("span", class_="dropdown-description") + assert description.text.strip() == "Create a new table in this database." + description.extract() + assert button.text.strip() == "Create table" + assert any( + "edit-tools.js" in script.get("src", "") + for script in soup.find_all("script") + ) + assert database_data_from_soup(soup) == { + "createTable": { + "path": "/data/-/create", + "databaseName": "data", + "columnTypes": ["text", "integer", "float", "blob"], + }, + } + assert "customColumnTypes" not in database_data_from_soup(soup)["createTable"] + + response_without_permission = await ds.client.get( + "/data", actor={"id": "someone-else"} + ) + assert response_without_permission.status_code == 200 + soup_without_permission = Soup(response_without_permission.text, "html.parser") + assert ( + soup_without_permission.select_one( + 'button[data-database-action="create-table"]' + ) + is None + ) + assert not any( + "_datasetteDatabaseData" in (script.string or "") + for script in soup_without_permission.find_all("script") + ) + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_database_create_table_data_includes_custom_column_types(): + ds = Datasette( + [], + config={ + "databases": { + "data": { + "permissions": { + "create-table": {"id": "root"}, + "set-column-type": {"id": "root"}, + }, + }, + }, + }, + ) + try: + db = ds.add_database( + Database(ds, memory_name="test_database_create_table_custom_types"), + name="data", + ) + await db.execute_write_script(""" + create table items (id integer primary key, name text); + """) + + response = await ds.client.get("/data", actor={"id": "root"}) + assert response.status_code == 200 + create_table_data = database_data_from_soup(Soup(response.text, "html.parser"))[ + "createTable" + ] + assert create_table_data["customColumnTypes"] == [ + { + "name": "email", + "description": "Email address", + "sqliteTypes": ["text"], + "fixedSqliteType": "text", + }, + { + "name": "json", + "description": "JSON data", + "sqliteTypes": ["text"], + "fixedSqliteType": "text", + }, + { + "name": "textarea", + "description": "Multiline text", + "sqliteTypes": ["text"], + "fixedSqliteType": "text", + }, + { + "name": "url", + "description": "URL", + "sqliteTypes": ["text"], + "fixedSqliteType": "text", + }, + ] + finally: + ds.close() + + @pytest.mark.asyncio async def test_table_insert_action_button_and_data(): ds = Datasette( From b40665dd143a81b5cea99ff039b3bc8a41c41d1f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 09:14:19 -0700 Subject: [PATCH 010/167] Add alter table JSON API - Add POST ///-/alter with Pydantic validation and dry-run support. - Support add, rename, alter, drop, primary-key and reorder operations, including allow-listed default expressions. - Document the endpoint and cover schema changes, validation, permissions, events and dry runs. Refs #2788 --- datasette/app.py | 5 + datasette/views/table.py | 353 +++++++++++++++++++++++++++++++++++++++ docs/json_api.rst | 103 ++++++++++++ pyproject.toml | 1 + tests/test_api_write.py | 205 +++++++++++++++++++++++ 5 files changed, 667 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 139e4c34..6ea3d5a4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -84,6 +84,7 @@ from .views.special import ( ) from .views.table import ( TableAutocompleteView, + TableAlterView, TableInsertView, TableUpsertView, TableSetColumnTypeView, @@ -2626,6 +2627,10 @@ class Datasette: TableUpsertView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/upsert$", ) + add_route( + TableAlterView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/alter$", + ) add_route( TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/set-column-type$", diff --git a/datasette/views/table.py b/datasette/views/table.py index c5448c85..11a28323 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,10 +1,12 @@ import asyncio import itertools import json +from typing import Annotated, Any, Literal, Union import urllib import urllib.parse import markupsafe +from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator from datasette.column_types import SQLiteType from datasette.extras import extra_names_from_request @@ -46,6 +48,7 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response from datasette.filters import Filters import sqlite_utils +from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView from .table_extras import ( @@ -649,6 +652,154 @@ async def display_columns_and_rows( return columns, cell_rows +SqliteApiType = Literal["text", "integer", "float", "blob"] +DefaultExpr = Literal["current_timestamp", "current_date", "current_time"] +DEFAULT_EXPR_SQL = { + "current_timestamp": "CURRENT_TIMESTAMP", + "current_date": "CURRENT_DATE", + "current_time": "CURRENT_TIME", +} + + +class _StrictPydanticModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class _DefaultArgsMixin(_StrictPydanticModel): + default: Any | None = None + default_expr: DefaultExpr | None = None + + @model_validator(mode="after") + def validate_default_fields(self): + has_default = "default" in self.model_fields_set + has_default_expr = "default_expr" in self.model_fields_set + if has_default and has_default_expr: + raise ValueError("default and default_expr cannot both be provided") + if has_default_expr and self.default_expr is None: + raise ValueError("default_expr cannot be null") + return self + + +class AddColumnArgs(_DefaultArgsMixin): + name: str + type: SqliteApiType = "text" + not_null: bool = False + + +class RenameColumnArgs(_StrictPydanticModel): + name: str + to: str + + +class AlterColumnArgs(_DefaultArgsMixin): + name: str + type: SqliteApiType | None = None + not_null: bool | None = None + + @model_validator(mode="after") + def require_change(self): + if not ( + {"type", "not_null", "default", "default_expr"} & self.model_fields_set + ): + raise ValueError( + "At least one of type, not_null, default or default_expr must be provided" + ) + return self + + +class DropColumnArgs(_StrictPydanticModel): + name: str + + +class SetPrimaryKeyArgs(_StrictPydanticModel): + columns: list[str] = Field(min_length=1) + + +class ReorderColumnsArgs(_StrictPydanticModel): + columns: list[str] = Field(min_length=1) + + +class AddColumnOperation(_StrictPydanticModel): + op: Literal["add_column"] + args: AddColumnArgs + + +class RenameColumnOperation(_StrictPydanticModel): + op: Literal["rename_column"] + args: RenameColumnArgs + + +class AlterColumnOperation(_StrictPydanticModel): + op: Literal["alter_column"] + args: AlterColumnArgs + + +class DropColumnOperation(_StrictPydanticModel): + op: Literal["drop_column"] + args: DropColumnArgs + + +class SetPrimaryKeyOperation(_StrictPydanticModel): + op: Literal["set_primary_key"] + args: SetPrimaryKeyArgs + + +class ReorderColumnsOperation(_StrictPydanticModel): + op: Literal["reorder_columns"] + args: ReorderColumnsArgs + + +AlterTableOperation = Annotated[ + Union[ + AddColumnOperation, + RenameColumnOperation, + AlterColumnOperation, + DropColumnOperation, + SetPrimaryKeyOperation, + ReorderColumnsOperation, + ], + Field(discriminator="op"), +] + + +class AlterTableRequest(_StrictPydanticModel): + operations: list[AlterTableOperation] = Field(min_length=1) + dry_run: bool = False + + +def _pydantic_errors(validation_error): + errors = [] + for error in validation_error.errors(): + location = ".".join(str(item) for item in error["loc"]) + message = error["msg"] + errors.append("{}: {}".format(location, message) if location else message) + return errors + + +def _table_schema_from_conn(conn, table_name): + row = conn.execute( + "select sql from sqlite_master where type = 'table' and name = ?", + [table_name], + ).fetchone() + return row[0] if row else None + + +def _primary_key_value(columns): + if len(columns) == 1: + return columns[0] + return tuple(columns) + + +def _default_expression_sql(default_expr): + return DEFAULT_EXPR_SQL[default_expr] + + +def _literal_default(db, value): + if isinstance(value, str): + return db.quote(value) + return value + + class TableInsertView(BaseView): name = "table-insert" @@ -946,6 +1097,208 @@ class TableUpsertView(TableInsertView): return await super().post(request, upsert=True) +class TableAlterView(BaseView): + name = "table-alter" + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + try: + resolved = await self.ds.resolve_table(request) + except NotFound as e: + return _error([e.args[0]], 404) + + db = resolved.db + database_name = db.name + table_name = resolved.table + + if not await self.ds.allowed( + action="alter-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return _error(["Permission denied: need alter-table"], 403) + + if not db.is_mutable: + return _error(["Database is immutable"], 403) + + content_type = request.headers.get("content-type") or "" + if not content_type.startswith("application/json"): + return _error(["Invalid content-type, must be application/json"], 400) + + try: + data = await request.json() + except json.JSONDecodeError as e: + return _error(["Invalid JSON: {}".format(e)], 400) + + if not isinstance(data, dict): + return _error(["JSON must be a dictionary"], 400) + + try: + alter_request = AlterTableRequest.model_validate(data) + except ValidationError as e: + return _error(_pydantic_errors(e), 400) + + def alter_table(conn): + before_schema = _table_schema_from_conn(conn, table_name) + + def apply_operations(operation_conn): + db_for_write = sqlite_utils.Database(operation_conn) + table = db_for_write[table_name] + + add_columns = [] + types = {} + rename = {} + drop = set() + not_null = {} + defaults = {} + column_order = None + pk = SQLITE_UTILS_DEFAULT + + for operation in alter_request.operations: + args = operation.args + if operation.op == "add_column": + if args.not_null and not ( + ( + "default" in args.model_fields_set + and args.default is not None + ) + or "default_expr" in args.model_fields_set + ): + raise ValueError( + "add_column args.default or args.default_expr is required when not_null is true" + ) + add_columns.append(args) + if "default" in args.model_fields_set and not args.not_null: + defaults[args.name] = _literal_default( + db_for_write, args.default + ) + if "default_expr" in args.model_fields_set and not args.not_null: + defaults[args.name] = _default_expression_sql( + args.default_expr + ) + elif operation.op == "rename_column": + rename[args.name] = args.to + elif operation.op == "alter_column": + if args.type is not None: + types[args.name] = args.type + if args.not_null is not None: + not_null[args.name] = args.not_null + if "default" in args.model_fields_set: + defaults[args.name] = ( + None + if args.default is None + else _literal_default(db_for_write, args.default) + ) + if "default_expr" in args.model_fields_set: + defaults[args.name] = _default_expression_sql( + args.default_expr + ) + elif operation.op == "drop_column": + drop.add(args.name) + elif operation.op == "set_primary_key": + pk = _primary_key_value(args.columns) + elif operation.op == "reorder_columns": + column_order = args.columns + + with operation_conn: + for column in add_columns: + not_null_default = None + if column.not_null: + if "default_expr" in column.model_fields_set: + not_null_default = _default_expression_sql( + column.default_expr + ) + else: + not_null_default = _literal_default( + db_for_write, column.default + ) + table.add_column( + column.name, + column.type, + not_null_default=not_null_default, + ) + + should_transform = any( + ( + types, + rename, + drop, + not_null, + defaults, + column_order is not None, + pk is not SQLITE_UTILS_DEFAULT, + ) + ) + if should_transform: + table.transform( + types=types or None, + rename=rename or None, + drop=drop or None, + pk=pk, + not_null=not_null or None, + defaults=defaults or None, + column_order=column_order, + ) + + return _table_schema_from_conn(operation_conn, table_name) + + if alter_request.dry_run: + memory_conn = sqlite3.connect(":memory:") + try: + conn.backup(memory_conn) + return before_schema, apply_operations(memory_conn) + finally: + memory_conn.close() + + after_schema = apply_operations(conn) + return before_schema, after_schema + + try: + before_schema, after_schema = await db.execute_write_fn( + alter_table, request=request + ) + except Exception as e: + return _error([str(e)], 400) + + altered = before_schema != after_schema + if altered and not alter_request.dry_run: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=before_schema, + after_schema=after_schema, + ) + ) + + table_url = self.ds.absolute_url( + request, self.ds.urls.table(database_name, table_name) + ) + table_api_url = self.ds.absolute_url( + request, self.ds.urls.table(database_name, table_name, format="json") + ) + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "table_url": table_url, + "table_api_url": table_api_url, + "altered": altered, + "schema": after_schema, + "before_schema": before_schema, + "operations_applied": 0 + if alter_request.dry_run + else len(alter_request.operations), + "dry_run": alter_request.dry_run, + }, + status=200, + ) + + class TableSetColumnTypeView(BaseView): name = "table-set-column-type" diff --git a/docs/json_api.rst b/docs/json_api.rst index f7a0caae..4074b479 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2072,6 +2072,109 @@ To use the ``"replace": true`` option you will also need the :ref:`actions_updat Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`actions_alter_table` permission. +.. _TableAlterView: + +Altering tables +~~~~~~~~~~~~~~~ + +To alter an existing table, make a ``POST`` to ``//
/-/alter``. This requires the :ref:`actions_alter_table` permission. + +:: + + POST //
/-/alter + Content-Type: application/json + Authorization: Bearer dstok_ + +The request body should include an ``operations`` array. Each operation has the same top-level shape: an ``op`` string and an ``args`` object. + +.. code-block:: json + + { + "operations": [ + { + "op": "add_column", + "args": { + "name": "slug", + "type": "text", + "not_null": true, + "default": "" + } + }, + { + "op": "add_column", + "args": { + "name": "created", + "type": "text", + "default_expr": "current_timestamp" + } + }, + { + "op": "rename_column", + "args": { + "name": "title", + "to": "headline" + } + }, + { + "op": "alter_column", + "args": { + "name": "score", + "type": "float" + } + }, + { + "op": "drop_column", + "args": { + "name": "draft_notes" + } + }, + { + "op": "set_primary_key", + "args": { + "columns": ["id"] + } + }, + { + "op": "reorder_columns", + "args": { + "columns": ["id", "headline", "slug", "created", "score"] + } + } + ] + } + +Set ``"dry_run": true`` to validate the operations and return the schema that would be created without modifying the table. + +Supported operations: + +* ``add_column`` adds a new column. ``args`` accepts ``name``, optional ``type`` of ``text``, ``integer``, ``float`` or ``blob``, optional ``not_null``, optional literal ``default`` and optional ``default_expr``. If ``not_null`` is ``true`` either a non-null ``default`` or ``default_expr`` is required. +* ``rename_column`` renames a column. ``args`` accepts ``name`` and ``to``. +* ``alter_column`` changes column properties. ``args`` accepts ``name`` and at least one of ``type``, ``not_null``, literal ``default`` or ``default_expr``. Passing ``"default": null`` removes an existing default. +* ``drop_column`` drops a column. ``args`` accepts ``name``. +* ``set_primary_key`` changes the table primary key. ``args`` accepts ``columns``, a list of one or more column names. +* ``reorder_columns`` reorders columns. ``args`` accepts ``columns``, a list of one or more column names. Columns omitted from this list will appear afterwards in their existing order. + +``default`` is always treated as a literal value. ``default_expr`` accepts one of ``current_timestamp``, ``current_date`` or ``current_time`` and is rendered as the corresponding SQLite default expression. + +A successful response returns the new schema and the previous schema: + +.. code-block:: json + + { + "ok": true, + "database": "data", + "table": "posts", + "table_url": "http://127.0.0.1:8001/data/posts", + "table_api_url": "http://127.0.0.1:8001/data/posts.json", + "altered": true, + "schema": "CREATE TABLE ...", + "before_schema": "CREATE TABLE ...", + "operations_applied": 7, + "dry_run": false + } + +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. + .. _TableSetColumnTypeView: Setting a column type diff --git a/pyproject.toml b/pyproject.toml index a19dc957..38776b2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "asyncinject>=0.7", "setuptools", "pip", + "pydantic>=2", ] [project.urls] diff --git a/tests/test_api_write.py b/tests/test_api_write.py index b7ceb6b2..f117c06e 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -794,6 +794,211 @@ async def test_update_row_alter(ds_write): assert response.json() == {"ok": True} +@pytest.mark.asyncio +async def test_alter_table_operations(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + before_schema = await db.execute_fn( + lambda conn: conn.execute( + "select sql from sqlite_master where type = 'table' and name = 'docs'" + ).fetchone()[0] + ) + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + { + "op": "add_column", + "args": { + "name": "slug", + "type": "text", + "not_null": True, + "default": "", + }, + }, + { + "op": "add_column", + "args": { + "name": "created", + "type": "text", + "default_expr": "current_timestamp", + }, + }, + { + "op": "add_column", + "args": { + "name": "literal_default", + "type": "text", + "default": "hello)", + }, + }, + {"op": "rename_column", "args": {"name": "title", "to": "headline"}}, + { + "op": "alter_column", + "args": {"name": "age", "type": "text", "default": "0"}, + }, + {"op": "drop_column", "args": {"name": "score"}}, + { + "op": "reorder_columns", + "args": { + "columns": [ + "id", + "headline", + "slug", + "created", + "literal_default", + "age", + ] + }, + }, + {"op": "set_primary_key", "args": {"columns": ["id"]}}, + ] + }, + headers=_headers(token), + ) + + assert response.status_code == 200, response.text + data = response.json() + assert data["ok"] is True + assert data["database"] == "data" + assert data["table"] == "docs" + assert data["altered"] is True + assert data["operations_applied"] == 8 + assert data["before_schema"] == before_schema + assert "headline" in data["schema"] + assert "score" not in data["schema"] + assert "DEFAULT CURRENT_TIMESTAMP" in data["schema"] + assert "DEFAULT 'hello)'" in data["schema"] + + columns = ( + await db.execute("select * from pragma_table_info('docs') order by cid") + ).dicts() + assert [column["name"] for column in columns] == [ + "id", + "headline", + "slug", + "created", + "literal_default", + "age", + ] + assert columns[0]["pk"] == 1 + assert columns[2]["notnull"] == 1 + assert columns[2]["dflt_value"] == "''" + assert columns[3]["dflt_value"] == "CURRENT_TIMESTAMP" + assert columns[4]["dflt_value"] == "'hello)'" + assert columns[5]["type"] == "TEXT" + assert columns[5]["dflt_value"] == "'0'" + + event = last_event(ds_write) + assert event.name == "alter-table" + assert event.database == "data" + assert event.table == "docs" + assert event.before_schema == before_schema + assert event.after_schema == data["schema"] + + +@pytest.mark.asyncio +async def test_alter_table_dry_run(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "dry_run": True, + "operations": [ + {"op": "add_column", "args": {"name": "slug", "type": "text"}} + ], + }, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["ok"] is True + assert data["dry_run"] is True + assert data["altered"] is True + assert data["operations_applied"] == 0 + assert "slug" in data["schema"] + columns = ( + await db.execute("select name from pragma_table_info('docs') order by cid") + ).dicts() + assert [column["name"] for column in columns] == ["id", "title", "score", "age"] + assert last_event(ds_write) is None + + +@pytest.mark.asyncio +async def test_alter_table_permission_denied(ds_write): + token = write_token(ds_write, permissions=["ir"]) + response = await ds_write.client.post( + "/data/docs/-/alter", + json={"operations": [{"op": "add_column", "args": {"name": "slug"}}]}, + headers=_headers(token), + ) + assert response.status_code == 403 + assert response.json() == { + "ok": False, + "errors": ["Permission denied: need alter-table"], + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body,expected_error", + ( + ( + {"operations": [{"op": "add_column", "args": {"type": "text"}}]}, + "operations.0.add_column.args.name: Field required", + ), + ( + { + "operations": [ + {"op": "add_column", "args": {"name": "x", "type": "bad"}} + ] + }, + "operations.0.add_column.args.type: Input should be 'text', 'integer', 'float' or 'blob'", + ), + ( + { + "operations": [ + { + "op": "add_column", + "args": { + "name": "x", + "default_expr": "datetime('now')", + }, + } + ] + }, + "operations.0.add_column.args.default_expr: Input should be 'current_timestamp', 'current_date' or 'current_time'", + ), + ( + { + "operations": [ + { + "op": "add_column", + "args": { + "name": "x", + "default": "x", + "default_expr": "current_timestamp", + }, + } + ] + }, + "operations.0.add_column.args: Value error, default and default_expr cannot both be provided", + ), + ), +) +async def test_alter_table_validation_errors(ds_write, body, expected_error): + response = await ds_write.client.post( + "/data/docs/-/alter", + json=body, + headers=_headers(write_token(ds_write, permissions=["at"])), + ) + assert response.status_code == 400 + assert response.json()["ok"] is False + assert response.json()["errors"] == [expected_error] + + @pytest.mark.asyncio async def test_execute_write_form_parameter_called_sql(): ds = Datasette(memory=True, default_deny=True) From fdd1b61a3e4bacba7e792554c5376856160f1dba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 09:21:09 -0700 Subject: [PATCH 011/167] Add alter table modal - Register a built-in table action and expose alter-table metadata to table pages. - Build the client-side modal for editing columns, defaults, ordering, primary keys, and custom column types. - Add a review/apply confirmation flow with HTML and Playwright coverage. Refs #2788 --- datasette/default_table_actions.py | 29 + datasette/plugins.py | 1 + datasette/static/app.css | 499 ++++++++++ datasette/static/edit-tools.js | 1478 +++++++++++++++++++++++++++- datasette/views/row.py | 1 + datasette/views/table.py | 82 +- tests/test_playwright.py | 210 ++++ tests/test_table_html.py | 117 +++ 8 files changed, 2414 insertions(+), 3 deletions(-) create mode 100644 datasette/default_table_actions.py diff --git a/datasette/default_table_actions.py b/datasette/default_table_actions.py new file mode 100644 index 00000000..e41434ef --- /dev/null +++ b/datasette/default_table_actions.py @@ -0,0 +1,29 @@ +from datasette import hookimpl +from datasette.resources import TableResource + + +@hookimpl +def table_actions(datasette, actor, database, table, request): + async def inner(): + db = datasette.get_database(database) + if not db.is_mutable: + return [] + if not await datasette.allowed( + action="alter-table", + resource=TableResource(database=database, table=table), + actor=actor, + ): + return [] + return [ + { + "type": "button", + "label": "Alter table", + "description": "Change columns and primary key for this table.", + "attrs": { + "aria-label": "Alter table {}".format(table), + "data-table-action": "alter-table", + }, + } + ] + + return inner diff --git a/datasette/plugins.py b/datasette/plugins.py index f0fbc7f8..ae2cb17d 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -31,6 +31,7 @@ DEFAULT_PLUGINS = ( "datasette.default_debug_menu", "datasette.default_jump_items", "datasette.default_database_actions", + "datasette.default_table_actions", "datasette.default_query_actions", "datasette.handle_exception", "datasette.forbidden", diff --git a/datasette/static/app.css b/datasette/static/app.css index ce0c46a8..f9ebe5ac 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -2032,6 +2032,505 @@ dialog.table-create-dialog::backdrop { cursor: not-allowed; } +dialog.table-alter-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; + width: min(980px, calc(100vw - 32px)); + max-width: 95vw; + max-height: min(780px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.table-alter-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.table-alter-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +.table-alter-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 12px; + flex-shrink: 0; + min-width: 0; +} + +.table-alter-dialog .modal-title { + display: flex; + align-items: center; + min-width: 0; + max-width: 100%; + font-size: 1rem; + font-weight: 600; + color: var(--ink); +} + +.table-alter-form { + display: flex; + flex: 1 1 auto; + min-height: 0; + flex-direction: column; +} + +.table-alter-error { + border-left: 4px solid #b91c1c; + border-radius: 4px; + background: #fff1f1; + color: #7f1d1d; + font-size: 0.9rem; + margin: 12px 24px 0; + padding: 10px 12px; +} + +.table-alter-error:focus { + outline: 3px solid rgba(185, 28, 28, 0.18); + outline-offset: 2px; +} + +.table-alter-fields { + display: grid; + gap: 18px; + padding: 16px 24px 24px; + overflow-y: auto; +} + +.table-alter-fields[hidden], +.table-alter-dialog .modal-footer [hidden] { + display: none; +} + +.table-alter-review { + display: grid; + gap: 12px; + overflow-y: auto; + padding: 16px 24px 24px; +} + +.table-alter-review[hidden] { + display: none; +} + +.table-alter-review-title { + color: var(--ink); + font-size: 1rem; + line-height: 1.35; + margin: 0; +} + +.table-alter-review-title:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 2px; +} + +.table-alter-review-intro { + color: var(--muted); + font-size: 0.9rem; + margin: 0; +} + +.table-alter-review-warning { + border-left: 4px solid #b91c1c; + border-radius: 4px; + background: #fff1f1; + color: #7f1d1d; + font-size: 0.9rem; + margin: 0; + padding: 10px 12px; +} + +.table-alter-review-list { + display: grid; + gap: 8px; + margin: 0; + padding-left: 1.4rem; +} + +.table-alter-review-list li { + color: var(--ink); + line-height: 1.4; +} + +.table-alter-review-damaging { + font-weight: 600; +} + +.table-alter-review-name { + background: #eef6ff; + border: 1px solid #c9ddf2; + border-radius: 4px; + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.85em; + padding: 1px 4px; + white-space: nowrap; +} + +.table-alter-columns { + display: grid; + gap: 10px; +} + +.table-alter-column-list { + display: grid; + gap: 8px; +} + +.table-alter-column-headings, +.table-alter-column-main { + display: grid; + grid-template-columns: minmax(140px, 1.2fr) minmax(7.5rem, 0.7fr) minmax(12rem, 1fr) max-content 32px; + align-items: center; + gap: 8px; + min-width: 0; +} + +.table-alter-column-row { + display: grid; + gap: 8px; + min-width: 0; +} + +.table-alter-column-headings { + color: var(--muted); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.72rem; + padding: 0 1px; +} + +.table-alter-column-label { + border: 0; + clip: rect(0 0 0 0); + height: 1px; + margin: -1px; + overflow: hidden; + padding: 0; + position: absolute; + white-space: nowrap; + width: 1px; +} + +.table-alter-input { + box-sizing: border-box; + min-width: 0; + border: 1px solid var(--rule); + border-radius: 5px; + padding: 8px 10px; + color: var(--ink); + background: #fff; + font: inherit; +} + +.table-alter-input-placeholder { + color: var(--muted); +} + +.table-alter-default-expr option, +.table-alter-custom-column-type option { + color: var(--ink); +} + +.table-alter-default-expr option[value=""], +.table-alter-custom-column-type option[value=""] { + color: var(--muted); +} + +.table-alter-input:focus { + border-color: var(--accent); + outline: 3px solid rgba(26, 86, 219, 0.12); +} + +.table-alter-column-details { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + align-items: start; + gap: 12px 16px; + padding: 12px; + border-left: 3px solid var(--rule); + background: #f8fafc; +} + +.table-alter-column-details[hidden] { + display: none; +} + +.table-alter-detail-field { + display: grid; + gap: 4px; + min-width: 0; +} + +.table-alter-detail-label { + color: var(--muted); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.72rem; +} + +.table-alter-detail-check { + display: inline-flex; + align-items: flex-start; + gap: 8px; + color: var(--ink); + font-size: 0.85rem; + line-height: 1.35; + min-width: 0; + white-space: normal; +} + +.table-alter-not-null, +.table-alter-primary-key { + grid-column: 1 / -1; +} + +.table-alter-detail-check input { + flex: 0 0 auto; + margin: 0.15rem 0 0; +} + +.table-alter-detail-check span { + min-width: 0; + overflow-wrap: break-word; +} + +.table-alter-move-controls { + display: grid; + grid-template-columns: repeat(4, 32px); + gap: 4px; + justify-content: start; +} + +.table-alter-more-options { + appearance: none; + border: 0; + background: transparent; + color: var(--accent); + cursor: pointer; + font: inherit; + font-size: 0.85rem; + justify-self: start; + padding: 0; + grid-column: 1 / -1; + text-align: left; +} + +.table-alter-more-options:hover, +.table-alter-more-options:focus { + text-decoration: underline; +} + +.table-alter-more-options:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 2px; +} + +.table-alter-more-options:disabled { + color: var(--muted); + cursor: default; + text-decoration: none; +} + +.table-alter-icon-button { + appearance: none; + border: 1px solid rgba(74, 85, 104, 0.24); + background: transparent; + color: #4a5568; + border-radius: 4px; + cursor: pointer; + display: inline-grid; + place-items: center; + height: 32px; + width: 32px; + padding: 0; +} + +.table-alter-icon-button:hover, +.table-alter-icon-button:focus { + background: rgba(74, 85, 104, 0.07); +} + +.table-alter-icon-button:focus { + outline: 3px solid #b3d4ff; + outline-offset: 1px; +} + +.table-alter-icon-button svg { + display: block; +} + +.table-alter-add-column { + appearance: none; + justify-self: start; + border: 1px solid var(--rule); + border-radius: 5px; + background: #fff; + color: var(--accent); + cursor: pointer; + font: inherit; + font-size: 0.85rem; + padding: 7px 10px; +} + +.table-alter-add-column:hover, +.table-alter-add-column:focus { + background: #f8fafc; +} + +.table-alter-add-column:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 1px; +} + +.table-alter-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +.table-alter-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.table-alter-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.table-alter-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.table-alter-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.table-alter-dialog .btn-primary:hover { + background: #1949b8; +} + +.table-alter-dialog .btn-primary:disabled, +.table-alter-dialog .btn-primary:disabled:hover { + background: #a0aec0; + color: #fff; +} + +.table-alter-dialog .btn:disabled, +.table-alter-add-column:disabled, +.table-alter-icon-button:disabled { + opacity: 0.55; + cursor: not-allowed; +} + +@media (max-width: 900px) { + dialog.table-alter-dialog { + width: 95vw; + max-height: 85vh; + border-radius: 0.5rem; + } + + .table-alter-dialog .modal-header, + .table-alter-fields, + .table-alter-review { + padding-left: 18px; + padding-right: 18px; + } + + .table-alter-error { + margin-left: 18px; + margin-right: 18px; + } + + .table-alter-column-headings { + display: none; + } + + .table-alter-column-row { + padding-bottom: 8px; + border-bottom: 1px solid var(--rule); + } + + .table-alter-column-main { + grid-template-columns: minmax(0, 1fr) minmax(7.5rem, 0.8fr) 32px; + align-items: end; + } + + .table-alter-column-name { + grid-column: 1; + grid-row: 1; + } + + .table-alter-column-type { + grid-column: 2; + grid-row: 1; + } + + .table-alter-remove-column { + grid-column: 3; + grid-row: 1; + justify-self: end; + } + + .table-alter-custom-column-type { + grid-column: 1 / 3; + grid-row: 2; + } + + .table-alter-move-controls { + grid-column: 1; + grid-row: 3; + justify-self: start; + } + + .table-alter-more-options { + align-self: center; + grid-column: 2 / 4; + grid-row: 3; + } + + .table-alter-column-details { + grid-template-columns: 1fr; + } + + .table-alter-dialog .modal-footer { + padding-left: 18px; + padding-right: 18px; + } +} + .row-link-with-actions { display: inline-flex; align-items: center; diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 4b93c4fd..284d6bde 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -4,6 +4,8 @@ var ROW_EDIT_DIALOG_ID = "row-edit-dialog"; var rowEditDialogState = null; var TABLE_CREATE_DIALOG_ID = "table-create-dialog"; var tableCreateDialogState = null; +var TABLE_ALTER_DIALOG_ID = "table-alter-dialog"; +var tableAlterDialogState = null; function ensureRowMutationStatus(manager) { var status = document.querySelector(".row-mutation-status"); @@ -59,6 +61,16 @@ function tableCreateColumnTypes() { : ["text", "integer", "float", "blob"]; } +function sqliteColumnTypeLabel(type) { + if (type === "float") { + return "floating point number"; + } + if (type === "blob") { + return "blob - binary data"; + } + return type; +} + function tableCreateCustomColumnTypes() { var data = databaseCreateTableData() || {}; return data.customColumnTypes || []; @@ -147,7 +159,7 @@ function tableCreateSelectTypeValue(select, type) { options.forEach(function (option) { var optionElement = document.createElement("option"); optionElement.value = option; - optionElement.textContent = option; + optionElement.textContent = sqliteColumnTypeLabel(option); select.appendChild(optionElement); }); select.value = options.indexOf(type) === -1 ? options[0] : type; @@ -819,6 +831,1469 @@ function tableInsertData() { return tablePageData().insertRow; } +function tableAlterData() { + return tablePageData().alterTable; +} + +function tableAlterColumnTypes() { + var data = tableAlterData() || {}; + return data.columnTypes && data.columnTypes.length + ? data.columnTypes + : ["text", "integer", "float", "blob"]; +} + +function tableAlterDefaultExpressions() { + var data = tableAlterData() || {}; + return data.defaultExpressions || []; +} + +function tableAlterCustomColumnTypes() { + var data = tableAlterData() || {}; + return data.customColumnTypes || []; +} + +function tableAlterCustomColumnType(name) { + var options = tableAlterCustomColumnTypes(); + for (var i = 0; i < options.length; i += 1) { + if (options[i].name === name) { + return options[i]; + } + } + return null; +} + +function tableAlterCustomTypeAppliesToSqliteType(option, sqliteType) { + return ( + option && + option.sqliteTypes && + option.sqliteTypes.indexOf(sqliteType) !== -1 + ); +} + +function tableAlterDialogRows(state) { + return Array.prototype.slice.call( + state.columnList.querySelectorAll(".table-alter-column-row"), + ); +} + +function tableAlterRowSignature(row) { + return { + existing: row.dataset.existing === "1", + originalName: row.dataset.originalName || "", + name: row.querySelector(".table-alter-column-name").value, + type: row.querySelector(".table-alter-column-type").value, + customType: + ( + row.querySelector(".table-alter-custom-column-type") || { + value: "", + } + ).value || "", + notNull: row.querySelector(".table-alter-not-null-input").checked, + defaultValue: row.querySelector(".table-alter-default").value, + defaultExpr: row.querySelector(".table-alter-default-expr").value, + pk: row.querySelector(".table-alter-primary-key-input").checked, + }; +} + +function tableAlterDialogSignature(state) { + if (!state || !state.form) { + return ""; + } + return JSON.stringify({ + columns: tableAlterDialogRows(state).map(tableAlterRowSignature), + deletedColumns: state.deletedColumns.slice(), + }); +} + +function tableAlterDialogHasChanges(state) { + return ( + !!state && + !state.isSaving && + tableAlterDialogSignature(state) !== state.initialSignature + ); +} + +function updateTableAlterSaveButtonState(state) { + if (!state || !state.saveButton) { + return; + } + state.saveButton.disabled = + state.isSaving || + (state.mode !== "review" && + tableAlterDialogSignature(state) === state.initialSignature); +} + +function tableAlterRowIsPrimaryKey(row) { + var input = row && row.querySelector(".table-alter-primary-key-input"); + return !!(input && input.checked); +} + +function tableAlterFirstNonPrimaryRow(state) { + var rows = tableAlterDialogRows(state); + for (var i = 0; i < rows.length; i += 1) { + if (!tableAlterRowIsPrimaryKey(rows[i])) { + return rows[i]; + } + } + return null; +} + +function updateTableAlterMoveButtons(state) { + if (!state || !state.columnList) { + return; + } + var firstNonPrimary = tableAlterFirstNonPrimaryRow(state); + var rows = tableAlterDialogRows(state); + var hasPrimaryKeys = rows.some(function (row) { + return tableAlterRowIsPrimaryKey(row); + }); + var primaryKeyMoveTitle = "Primary key columns are always listed first"; + rows.forEach(function (row) { + var isPrimaryKey = tableAlterRowIsPrimaryKey(row); + var previous = row.previousElementSibling; + var next = row.nextElementSibling; + row.querySelectorAll(".table-alter-move-controls button").forEach( + function (button) { + button.title = button.dataset.defaultTitle || button.title; + button.disabled = state.isSaving || isPrimaryKey; + if (isPrimaryKey) { + button.title = primaryKeyMoveTitle; + } + }, + ); + if (!isPrimaryKey) { + var topButton = row.querySelector(".table-alter-move-top"); + var upButton = row.querySelector(".table-alter-move-up"); + var downButton = row.querySelector(".table-alter-move-down"); + var bottomButton = row.querySelector(".table-alter-move-bottom"); + topButton.disabled = + state.isSaving || !firstNonPrimary || row === firstNonPrimary; + upButton.disabled = + state.isSaving || !previous || tableAlterRowIsPrimaryKey(previous); + downButton.disabled = state.isSaving || !next; + bottomButton.disabled = state.isSaving || !next; + if (hasPrimaryKeys && row === firstNonPrimary) { + topButton.title = primaryKeyMoveTitle; + upButton.title = primaryKeyMoveTitle; + } + } + }); +} + +function normalizeTableAlterPrimaryKeyRows(state) { + var rows = tableAlterDialogRows(state); + rows + .filter(function (row) { + return tableAlterRowIsPrimaryKey(row); + }) + .concat( + rows.filter(function (row) { + return !tableAlterRowIsPrimaryKey(row); + }), + ) + .forEach(function (row) { + state.columnList.appendChild(row); + }); +} + +function clearTableAlterDialogError(state) { + state.error.hidden = true; + state.error.textContent = ""; + state.dialog.removeAttribute("aria-describedby"); +} + +function showTableAlterDialogError(state, message) { + state.error.hidden = false; + state.error.textContent = message; + state.dialog.setAttribute("aria-describedby", "table-alter-error"); + state.error.focus(); +} + +function setTableAlterDialogSaving(state, isSaving) { + state.isSaving = isSaving; + state.cancelButton.disabled = isSaving; + state.addColumnButton.disabled = isSaving; + state.backButton.disabled = isSaving; + state.saveButton.textContent = isSaving + ? state.mode === "review" + ? "Applying..." + : "Preparing..." + : tableAlterSaveButtonText(state); + state.columnList + .querySelectorAll("input, select, button") + .forEach(function (control) { + control.disabled = isSaving; + }); + if (!isSaving) { + state.columnList + .querySelectorAll(".table-alter-default-expr") + .forEach(function (select) { + syncTableAlterDefaultControls(select.closest(".table-alter-column-row")); + }); + } + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); +} + +function tableAlterSaveButtonText(state) { + return state && state.mode === "review" ? "Apply changes" : "Review changes"; +} + +function tableAlterSelectTypeValue(select, type) { + var options = tableAlterColumnTypes(); + options.forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option; + optionElement.textContent = sqliteColumnTypeLabel(option); + select.appendChild(optionElement); + }); + select.value = options.indexOf(type) === -1 ? options[0] : type; +} + +function updateTableAlterCustomColumnTypePlaceholder(select) { + select.classList.toggle( + "table-alter-input-placeholder", + !select.value, + ); +} + +function createTableAlterCustomColumnTypeSelect() { + var options = tableAlterCustomColumnTypes(); + var select = document.createElement("select"); + select.className = "table-alter-input table-alter-custom-column-type"; + select.setAttribute("aria-label", "Custom column type"); + var blankOption = document.createElement("option"); + blankOption.value = ""; + blankOption.textContent = "- custom type -"; + select.appendChild(blankOption); + options.forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option.name; + optionElement.textContent = option.description + ? option.description + " (" + option.name + ")" + : option.name; + select.appendChild(optionElement); + }); + updateTableAlterCustomColumnTypePlaceholder(select); + return select; +} + +function syncTableAlterCustomTypeForSqliteType(row) { + var typeSelect = row.querySelector(".table-alter-column-type"); + var customTypeSelect = row.querySelector(".table-alter-custom-column-type"); + if (!typeSelect || !customTypeSelect || !customTypeSelect.value) { + return; + } + var option = tableAlterCustomColumnType(customTypeSelect.value); + if (!tableAlterCustomTypeAppliesToSqliteType(option, typeSelect.value)) { + customTypeSelect.value = ""; + updateTableAlterCustomColumnTypePlaceholder(customTypeSelect); + } +} + +function tableAlterSelectDefaultExprValue(select, value) { + var blankOption = document.createElement("option"); + blankOption.value = ""; + blankOption.textContent = "- default expr -"; + select.appendChild(blankOption); + tableAlterDefaultExpressions().forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option; + optionElement.textContent = option.replace(/_/g, " "); + select.appendChild(optionElement); + }); + select.value = value || ""; + updateTableAlterDefaultExprPlaceholder(select); +} + +function updateTableAlterDefaultExprPlaceholder(select) { + select.classList.toggle("table-alter-input-placeholder", !select.value); +} + +function syncTableAlterDefaultControls(row) { + if (!row) { + return; + } + var defaultInput = row.querySelector(".table-alter-default"); + var defaultExprSelect = row.querySelector(".table-alter-default-expr"); + if (!defaultInput || !defaultExprSelect) { + return; + } + defaultInput.disabled = !!defaultExprSelect.value; + updateTableAlterDefaultExprPlaceholder(defaultExprSelect); +} + +function createTableAlterColumnRow(state, column) { + var index = state.nextColumnIndex; + state.nextColumnIndex += 1; + var existing = !!(column && column.existing); + var originalName = existing ? column.name || "" : ""; + var originalCustomType = + existing && column.column_type ? column.column_type.type || "" : ""; + var originalDefault = + existing && column.has_default && column.default !== null + ? String(column.default) + : ""; + + var row = document.createElement("div"); + row.className = "table-alter-column-row"; + row.dataset.existing = existing ? "1" : "0"; + row.dataset.originalName = originalName; + row.dataset.originalType = existing ? column.type || "text" : ""; + row.dataset.originalNotNull = existing && column.notnull ? "1" : "0"; + row.dataset.originalHasDefault = existing && column.has_default ? "1" : "0"; + row.dataset.originalDefault = originalDefault; + row.dataset.originalPk = existing && column.is_pk ? "1" : "0"; + row.dataset.originalCustomType = originalCustomType; + + var main = document.createElement("div"); + main.className = "table-alter-column-main"; + + var details = document.createElement("div"); + details.className = "table-alter-column-details"; + details.id = "table-alter-column-details-" + index; + details.hidden = !(column && column.expanded); + + var expandButton = document.createElement("button"); + expandButton.type = "button"; + expandButton.className = "table-alter-more-options"; + expandButton.setAttribute("aria-label", "Toggle column settings"); + expandButton.setAttribute("aria-controls", details.id); + expandButton.setAttribute( + "aria-expanded", + details.hidden ? "false" : "true", + ); + function updateExpandButton() { + var isExpanded = expandButton.getAttribute("aria-expanded") === "true"; + expandButton.textContent = isExpanded + ? "v Hide options" + : "> Advanced options"; + expandButton.title = isExpanded + ? "Hide column settings" + : "Show column settings"; + } + updateExpandButton(); + + var nameId = "table-alter-column-name-" + index; + var nameLabel = document.createElement("label"); + nameLabel.className = "table-alter-column-label"; + nameLabel.setAttribute("for", nameId); + nameLabel.textContent = "Column"; + + var nameInput = document.createElement("input"); + nameInput.id = nameId; + nameInput.className = "table-alter-input table-alter-column-name"; + nameInput.type = "text"; + nameInput.required = true; + nameInput.autocomplete = "off"; + nameInput.value = column && column.name ? column.name : ""; + + var typeSelect = document.createElement("select"); + typeSelect.className = "table-alter-input table-alter-column-type"; + typeSelect.setAttribute("aria-label", "Column type"); + tableAlterSelectTypeValue(typeSelect, column && column.type); + + var customTypeSelect = null; + if (tableAlterCustomColumnTypes().length) { + customTypeSelect = createTableAlterCustomColumnTypeSelect(); + customTypeSelect.value = originalCustomType; + updateTableAlterCustomColumnTypePlaceholder(customTypeSelect); + } + + var notNullLabel = document.createElement("label"); + notNullLabel.className = "table-alter-detail-check table-alter-not-null"; + var notNullInput = document.createElement("input"); + notNullInput.type = "checkbox"; + notNullInput.className = "table-alter-not-null-input"; + notNullInput.checked = !!(column && column.notnull); + var notNullText = document.createElement("span"); + var notNullStrong = document.createElement("strong"); + notNullStrong.textContent = "Not null"; + notNullText.appendChild(notNullStrong); + notNullText.appendChild( + document.createTextNode(" This value cannot be left unset"), + ); + notNullLabel.appendChild(notNullInput); + notNullLabel.appendChild(notNullText); + + var defaultId = "table-alter-column-default-" + index; + var defaultField = document.createElement("div"); + defaultField.className = "table-alter-detail-field"; + var defaultLabel = document.createElement("label"); + defaultLabel.className = "table-alter-detail-label"; + defaultLabel.setAttribute("for", defaultId); + defaultLabel.textContent = "Default value"; + var defaultInput = document.createElement("input"); + defaultInput.id = defaultId; + defaultInput.className = "table-alter-input table-alter-default"; + defaultInput.type = "text"; + defaultInput.autocomplete = "off"; + defaultInput.placeholder = "default"; + defaultInput.setAttribute("aria-label", "Default value"); + defaultInput.value = originalDefault; + defaultField.appendChild(defaultLabel); + defaultField.appendChild(defaultInput); + + var defaultExprId = "table-alter-column-default-expr-" + index; + var defaultExprField = document.createElement("div"); + defaultExprField.className = "table-alter-detail-field"; + var defaultExprLabel = document.createElement("label"); + defaultExprLabel.className = "table-alter-detail-label"; + defaultExprLabel.setAttribute("for", defaultExprId); + defaultExprLabel.textContent = "or default to a specific time"; + var defaultExprSelect = document.createElement("select"); + defaultExprSelect.id = defaultExprId; + defaultExprSelect.className = + "table-alter-input table-alter-default-expr"; + defaultExprSelect.setAttribute("aria-label", "or default to a specific time"); + tableAlterSelectDefaultExprValue(defaultExprSelect, ""); + defaultExprField.appendChild(defaultExprLabel); + defaultExprField.appendChild(defaultExprSelect); + + var pkLabel = document.createElement("label"); + pkLabel.className = "table-alter-detail-check table-alter-primary-key"; + var pkInput = document.createElement("input"); + pkInput.type = "checkbox"; + pkInput.className = "table-alter-primary-key-input"; + pkInput.checked = !!(column && column.is_pk); + var pkText = document.createElement("span"); + var pkStrong = document.createElement("strong"); + pkStrong.textContent = "Primary key"; + pkText.appendChild(pkStrong); + pkText.appendChild( + document.createTextNode(" An ID that uniquely identifies this record"), + ); + pkLabel.appendChild(pkInput); + pkLabel.appendChild(pkText); + + var moveControls = document.createElement("div"); + moveControls.className = "table-alter-move-controls"; + + var moveTopButton = document.createElement("button"); + moveTopButton.type = "button"; + moveTopButton.className = "table-alter-icon-button table-alter-move-top"; + moveTopButton.setAttribute("aria-label", "Move column to top"); + moveTopButton.title = "Move column to top"; + moveTopButton.dataset.defaultTitle = moveTopButton.title; + moveTopButton.innerHTML = + ''; + + var moveUpButton = document.createElement("button"); + moveUpButton.type = "button"; + moveUpButton.className = "table-alter-icon-button table-alter-move-up"; + moveUpButton.setAttribute("aria-label", "Move column up"); + moveUpButton.title = "Move column up"; + moveUpButton.dataset.defaultTitle = moveUpButton.title; + moveUpButton.innerHTML = + ''; + + var moveDownButton = document.createElement("button"); + moveDownButton.type = "button"; + moveDownButton.className = "table-alter-icon-button table-alter-move-down"; + moveDownButton.setAttribute("aria-label", "Move column down"); + moveDownButton.title = "Move column down"; + moveDownButton.dataset.defaultTitle = moveDownButton.title; + moveDownButton.innerHTML = + ''; + + var moveBottomButton = document.createElement("button"); + moveBottomButton.type = "button"; + moveBottomButton.className = "table-alter-icon-button table-alter-move-bottom"; + moveBottomButton.setAttribute("aria-label", "Move column to bottom"); + moveBottomButton.title = "Move column to bottom"; + moveBottomButton.dataset.defaultTitle = moveBottomButton.title; + moveBottomButton.innerHTML = + ''; + + var removeButton = document.createElement("button"); + removeButton.type = "button"; + removeButton.className = "table-alter-icon-button table-alter-remove-column"; + removeButton.setAttribute( + "aria-label", + existing ? "Drop column" : "Remove column", + ); + removeButton.title = existing ? "Drop column" : "Remove column"; + removeButton.innerHTML = + ''; + + moveControls.appendChild(moveTopButton); + moveControls.appendChild(moveUpButton); + moveControls.appendChild(moveDownButton); + moveControls.appendChild(moveBottomButton); + main.appendChild(nameLabel); + main.appendChild(nameInput); + main.appendChild(typeSelect); + if (customTypeSelect) { + main.appendChild(customTypeSelect); + } + main.appendChild(moveControls); + main.appendChild(removeButton); + main.appendChild(expandButton); + + details.appendChild(notNullLabel); + details.appendChild(defaultField); + details.appendChild(defaultExprField); + details.appendChild(pkLabel); + row.appendChild(main); + row.appendChild(details); + + var controls = [ + nameInput, + typeSelect, + notNullInput, + defaultInput, + defaultExprSelect, + pkInput, + ]; + if (customTypeSelect) { + controls.push(customTypeSelect); + } + controls.forEach(function (control) { + control.addEventListener("input", function () { + clearTableAlterDialogError(state); + updateTableAlterSaveButtonState(state); + }); + control.addEventListener("change", function () { + clearTableAlterDialogError(state); + updateTableAlterSaveButtonState(state); + }); + }); + + defaultInput.addEventListener("input", function () { + if (defaultInput.value) { + defaultExprSelect.value = ""; + syncTableAlterDefaultControls(row); + } + updateTableAlterSaveButtonState(state); + }); + defaultExprSelect.addEventListener("change", function () { + if (defaultExprSelect.value) { + defaultInput.value = ""; + } + syncTableAlterDefaultControls(row); + updateTableAlterSaveButtonState(state); + }); + pkInput.addEventListener("change", function () { + normalizeTableAlterPrimaryKeyRows(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); + }); + + expandButton.addEventListener("click", function () { + var isExpanded = expandButton.getAttribute("aria-expanded") === "true"; + details.hidden = isExpanded; + expandButton.setAttribute("aria-expanded", isExpanded ? "false" : "true"); + updateExpandButton(); + }); + + typeSelect.addEventListener("change", function () { + syncTableAlterCustomTypeForSqliteType(row); + updateTableAlterSaveButtonState(state); + }); + if (customTypeSelect) { + customTypeSelect.addEventListener("change", function () { + updateTableAlterCustomColumnTypePlaceholder(customTypeSelect); + var option = tableAlterCustomColumnType(customTypeSelect.value); + if ( + option && + option.fixedSqliteType && + tableAlterColumnTypes().indexOf(option.fixedSqliteType) !== -1 + ) { + typeSelect.value = option.fixedSqliteType; + } + updateTableAlterSaveButtonState(state); + }); + } + + moveTopButton.addEventListener("click", function () { + var first = tableAlterFirstNonPrimaryRow(state); + if (state.isSaving || tableAlterRowIsPrimaryKey(row) || !first || first === row) { + return; + } + state.columnList.insertBefore(row, first); + clearTableAlterDialogError(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); + row.querySelector(".table-alter-column-name").focus(); + }); + + moveUpButton.addEventListener("click", function () { + var previous = row.previousElementSibling; + if ( + state.isSaving || + tableAlterRowIsPrimaryKey(row) || + !previous || + tableAlterRowIsPrimaryKey(previous) + ) { + return; + } + state.columnList.insertBefore(row, previous); + clearTableAlterDialogError(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); + row.querySelector(".table-alter-column-name").focus(); + }); + + moveDownButton.addEventListener("click", function () { + var next = row.nextElementSibling; + if (state.isSaving || tableAlterRowIsPrimaryKey(row) || !next) { + return; + } + state.columnList.insertBefore(next, row); + clearTableAlterDialogError(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); + row.querySelector(".table-alter-column-name").focus(); + }); + + moveBottomButton.addEventListener("click", function () { + var last = state.columnList.lastElementChild; + if (state.isSaving || tableAlterRowIsPrimaryKey(row) || !last || last === row) { + return; + } + state.columnList.appendChild(row); + clearTableAlterDialogError(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); + row.querySelector(".table-alter-column-name").focus(); + }); + + removeButton.addEventListener("click", function () { + if (state.isSaving) { + return; + } + if (row.dataset.existing === "1") { + state.deletedColumns.push(row.dataset.originalName); + } + row.remove(); + clearTableAlterDialogError(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); + var nextInput = state.columnList.querySelector(".table-alter-column-name"); + if (nextInput) { + nextInput.focus(); + } else { + state.addColumnButton.focus(); + } + }); + + syncTableAlterDefaultControls(row); + return row; +} + +function addTableAlterColumn(state, column) { + var row = createTableAlterColumnRow(state, column || { type: "text" }); + state.columnList.appendChild(row); + return row; +} + +function resetTableAlterDialog(state, data) { + state.nextColumnIndex = 0; + state.deletedColumns = []; + state.originalPrimaryKeys = (data.primaryKeys || []).slice(); + state.originalColumnNames = (data.columns || []).map(function (column) { + return column.name; + }); + state.columnList.textContent = ""; + (data.columns || []).forEach(function (column) { + addTableAlterColumn( + state, + Object.assign({}, column, { + existing: true, + }), + ); + }); + normalizeTableAlterPrimaryKeyRows(state); + state.initialSignature = tableAlterDialogSignature(state); + showTableAlterEditor(state); +} + +function collectTableAlterRows(state) { + return tableAlterDialogRows(state).map(function (row) { + var signature = tableAlterRowSignature(row); + signature.originalType = row.dataset.originalType || ""; + signature.originalNotNull = row.dataset.originalNotNull === "1"; + signature.originalHasDefault = row.dataset.originalHasDefault === "1"; + signature.originalDefault = row.dataset.originalDefault || ""; + signature.originalPk = row.dataset.originalPk === "1"; + signature.originalCustomType = row.dataset.originalCustomType || ""; + return signature; + }); +} + +function validateTableAlterRows(state, rows) { + if (!rows.length) { + return "At least one column is required."; + } + var seen = {}; + var supportedTypes = tableAlterColumnTypes(); + for (var i = 0; i < rows.length; i += 1) { + var row = rows[i]; + var name = row.name.trim(); + if (!name) { + return "Column name is required."; + } + if (name.indexOf("\n") !== -1) { + return "Column names cannot contain newlines."; + } + var columnKey = name.toLowerCase(); + if (seen[columnKey]) { + return "Duplicate column name: " + name; + } + seen[columnKey] = true; + if (supportedTypes.indexOf(row.type) === -1) { + return "Unsupported column type: " + row.type; + } + if (row.customType) { + var option = tableAlterCustomColumnType(row.customType); + if (!option) { + return "Unknown custom column type: " + row.customType; + } + if (!tableAlterCustomTypeAppliesToSqliteType(option, row.type)) { + return ( + "Custom type " + + row.customType + + " cannot be used with SQLite type " + + row.type + + "." + ); + } + } + if (row.defaultValue && row.defaultExpr) { + return "Use either a default value or a default expression."; + } + if (!row.existing && row.notNull && !row.defaultValue && !row.defaultExpr) { + return "New NOT NULL columns need a default or default expression."; + } + } + var pkColumns = rows.filter(function (row) { + return row.pk; + }); + if (state.originalPrimaryKeys.length && !pkColumns.length) { + return "At least one primary key column is required."; + } + return null; +} + +function collectTableAlterColumnTypeAssignments(rows) { + var assignments = []; + if (!tableAlterCustomColumnTypes().length) { + return assignments; + } + rows.forEach(function (row) { + var renamed = row.existing && row.name.trim() !== row.originalName; + if (row.customType === row.originalCustomType && !renamed) { + return; + } + if (!row.customType && !row.originalCustomType) { + return; + } + assignments.push({ + column: row.name.trim(), + columnType: row.customType || null, + sqliteType: row.type, + }); + }); + return assignments; +} + +function tableAlterPkIdentityColumns(rows) { + return rows + .filter(function (row) { + return row.pk; + }) + .map(function (row) { + return row.existing ? row.originalName : row.name.trim(); + }); +} + +function tableAlterPkChanged(state, rows) { + return ( + JSON.stringify(tableAlterPkIdentityColumns(rows)) !== + JSON.stringify(state.originalPrimaryKeys) + ); +} + +function tableAlterNaturalColumnOrder(state, rows) { + var existingRowsByOriginalName = {}; + var newRows = []; + rows.forEach(function (row) { + if (row.existing) { + existingRowsByOriginalName[row.originalName] = row; + } else { + newRows.push(row); + } + }); + var naturalOrder = []; + state.originalColumnNames.forEach(function (originalName) { + var row = existingRowsByOriginalName[originalName]; + if (row) { + naturalOrder.push(row.name.trim()); + } + }); + newRows.forEach(function (row) { + naturalOrder.push(row.name.trim()); + }); + return naturalOrder; +} + +function tableAlterColumnsReordered(state, rows) { + var finalOrder = rows.map(function (row) { + return row.name.trim(); + }); + return ( + JSON.stringify(finalOrder) !== + JSON.stringify(tableAlterNaturalColumnOrder(state, rows)) + ); +} + +function collectTableAlterPayload(state) { + var rows = collectTableAlterRows(state); + var validationError = validateTableAlterRows(state, rows); + if (validationError) { + return { error: validationError }; + } + + var operations = []; + var columnTypeAssignments = collectTableAlterColumnTypeAssignments(rows); + rows.forEach(function (row) { + var name = row.name.trim(); + if (!row.existing) { + var addArgs = { + name: name, + type: row.type, + not_null: row.notNull, + }; + if (row.defaultExpr) { + addArgs.default_expr = row.defaultExpr; + } else if (row.defaultValue || row.notNull) { + addArgs.default = row.defaultValue; + } + operations.push({ op: "add_column", args: addArgs }); + return; + } + + var originalName = row.originalName; + if (name !== originalName) { + operations.push({ + op: "rename_column", + args: { name: originalName, to: name }, + }); + } + + var alterArgs = { name: originalName }; + if (row.type !== row.originalType) { + alterArgs.type = row.type; + } + if (row.notNull !== row.originalNotNull) { + alterArgs.not_null = row.notNull; + } + if (row.defaultExpr) { + alterArgs.default_expr = row.defaultExpr; + } else if (row.originalHasDefault) { + if (row.defaultValue !== row.originalDefault) { + alterArgs.default = row.defaultValue === "" ? null : row.defaultValue; + } + } else if (row.defaultValue) { + alterArgs.default = row.defaultValue; + } + if (Object.keys(alterArgs).length > 1) { + operations.push({ op: "alter_column", args: alterArgs }); + } + }); + + state.deletedColumns.forEach(function (name) { + operations.push({ op: "drop_column", args: { name: name } }); + }); + + var pkColumns = rows + .filter(function (row) { + return row.pk; + }) + .map(function (row) { + return row.name.trim(); + }); + if (tableAlterPkChanged(state, rows)) { + operations.push({ op: "set_primary_key", args: { columns: pkColumns } }); + } + + if (tableAlterColumnsReordered(state, rows)) { + operations.push({ + op: "reorder_columns", + args: { + columns: rows.map(function (row) { + return row.name.trim(); + }), + }, + }); + } + + if (!operations.length && !columnTypeAssignments.length) { + return { error: "No changes to apply." }; + } + return { + payload: operations.length ? { operations: operations } : null, + columnTypeAssignments: columnTypeAssignments, + }; +} + +function tableAlterQuotedName(name) { + return '"' + name + '"'; +} + +function tableAlterReadableDefaultExpression(value) { + return value ? value.replace(/_/g, " ") : ""; +} + +function tableAlterReadableValue(value) { + if (value === null) { + return "NULL"; + } + return '"' + String(value) + '"'; +} + +function tableAlterOperationSummary(operation) { + var args = operation.args || {}; + if (operation.op === "add_column") { + var addDetails = ["as " + args.type]; + if (args.not_null) { + addDetails.push("with values required"); + } + if (args.default_expr) { + addDetails.push( + "defaulting to " + + tableAlterReadableDefaultExpression(args.default_expr), + ); + } else if (Object.prototype.hasOwnProperty.call(args, "default")) { + addDetails.push( + "with default value " + tableAlterReadableValue(args.default), + ); + } + return { + text: + "Add column " + + tableAlterQuotedName(args.name) + + " " + + addDetails.join(", ") + + ".", + damaging: false, + }; + } + if (operation.op === "rename_column") { + return { + text: + "Rename column " + + tableAlterQuotedName(args.name) + + " to " + + tableAlterQuotedName(args.to) + + ".", + damaging: false, + }; + } + if (operation.op === "alter_column") { + var changes = []; + if (args.type) { + changes.push("set type to " + args.type); + } + if (Object.prototype.hasOwnProperty.call(args, "not_null")) { + changes.push( + args.not_null + ? "not null (require values)" + : "allow unset values", + ); + } + if (args.default_expr) { + changes.push( + "default to " + tableAlterReadableDefaultExpression(args.default_expr), + ); + } else if (Object.prototype.hasOwnProperty.call(args, "default")) { + changes.push( + args.default === null + ? "remove the default value" + : "set default value to " + tableAlterReadableValue(args.default), + ); + } + return { + text: + "Change column " + + tableAlterQuotedName(args.name) + + ": " + + changes.join(", ") + + ".", + damaging: false, + }; + } + if (operation.op === "drop_column") { + return { + text: "Drop column " + tableAlterQuotedName(args.name) + ".", + damaging: true, + }; + } + if (operation.op === "set_primary_key") { + return { + text: + "Set primary key to " + + (args.columns || []).map(tableAlterQuotedName).join(", ") + + ".", + damaging: false, + }; + } + if (operation.op === "reorder_columns") { + return { + text: + "Set column order to " + + (args.columns || []).map(tableAlterQuotedName).join(", ") + + ".", + damaging: false, + }; + } + return { + text: "Run " + operation.op + ".", + damaging: false, + }; +} + +function tableAlterColumnTypeAssignmentSummary(assignment) { + return { + text: assignment.columnType + ? "Set custom type for column " + + tableAlterQuotedName(assignment.column) + + " to " + + assignment.columnType + + "." + : "Remove custom type from column " + + tableAlterQuotedName(assignment.column) + + ".", + damaging: false, + }; +} + +function tableAlterReviewItems(result) { + var items = []; + var operations = result.payload ? result.payload.operations || [] : []; + operations.forEach(function (operation) { + items.push(tableAlterOperationSummary(operation)); + }); + (result.columnTypeAssignments || []).forEach(function (assignment) { + items.push(tableAlterColumnTypeAssignmentSummary(assignment)); + }); + return items; +} + +function tableAlterReviewHasDamagingItems(items) { + return items.some(function (item) { + return item.damaging; + }); +} + +function appendTableAlterReviewText(element, text) { + text.split(/("[^"]+")/g).forEach(function (part) { + if (!part) { + return; + } + if (part.charAt(0) === '"' && part.charAt(part.length - 1) === '"') { + var name = document.createElement("code"); + name.className = "table-alter-review-name"; + name.textContent = part.slice(1, -1); + element.appendChild(name); + } else { + element.appendChild(document.createTextNode(part)); + } + }); +} + +function tableAlterSetColumnTypeUrl() { + var data = tableAlterData(); + if (!data || !data.path) { + return null; + } + var url = new URL(data.path, location.href); + url.pathname = url.pathname.replace(/\/-\/alter\/?$/, "/-/set-column-type"); + return url.toString(); +} + +async function assignTableAlterColumnTypes(assignments) { + if (!assignments.length) { + return; + } + var url = tableAlterSetColumnTypeUrl(); + if (!url) { + throw new Error("Could not find the set column type URL."); + } + for (var i = 0; i < assignments.length; i += 1) { + var assignment = assignments[i]; + var response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ + column: assignment.column, + column_type: assignment.columnType + ? { + type: assignment.columnType, + } + : null, + }), + }); + var data = null; + try { + data = await response.json(); + } catch (_error) { + data = null; + } + if (!response.ok || (data && data.ok === false)) { + var error = rowMutationRequestError(response, data); + throw new Error( + "Saved schema changes, but could not set custom type for " + + assignment.column + + ": " + + error.message, + ); + } + } +} + +function showTableAlterEditor(state) { + state.mode = "edit"; + state.reviewResult = null; + state.dialog.classList.remove("table-alter-reviewing"); + state.fields.hidden = false; + state.review.hidden = true; + state.review.textContent = ""; + state.backButton.hidden = true; + state.saveButton.textContent = tableAlterSaveButtonText(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); +} + +function showTableAlterReview(state, result) { + var items = tableAlterReviewItems(result); + state.mode = "review"; + state.reviewResult = result; + state.dialog.classList.add("table-alter-reviewing"); + state.fields.hidden = true; + state.review.hidden = false; + state.review.textContent = ""; + state.backButton.hidden = false; + state.saveButton.textContent = tableAlterSaveButtonText(state); + updateTableAlterSaveButtonState(state); + + var heading = document.createElement("h3"); + heading.className = "table-alter-review-title"; + heading.tabIndex = -1; + heading.textContent = "Review changes"; + state.review.appendChild(heading); + + var intro = document.createElement("p"); + intro.className = "table-alter-review-intro"; + intro.textContent = "These changes will be applied to the table."; + state.review.appendChild(intro); + + if (tableAlterReviewHasDamagingItems(items)) { + var warning = document.createElement("p"); + warning.className = "table-alter-review-warning"; + warning.setAttribute("role", "alert"); + warning.textContent = + "Warning: data in dropped columns will be permanently lost."; + state.review.appendChild(warning); + } + + var list = document.createElement("ol"); + list.className = "table-alter-review-list"; + items.forEach(function (item) { + var listItem = document.createElement("li"); + appendTableAlterReviewText(listItem, item.text); + if (item.damaging) { + listItem.className = "table-alter-review-damaging"; + } + list.appendChild(listItem); + }); + state.review.appendChild(list); + heading.focus(); +} + +async function applyTableAlterChanges(state, result) { + if (state.isSaving) { + return; + } + if (!result) { + showTableAlterDialogError(state, "Could not find the reviewed changes."); + return; + } + var data = tableAlterData(); + if (!data || !data.path) { + showTableAlterDialogError(state, "Could not find the alter table URL."); + return; + } + clearTableAlterDialogError(state); + if (result.error) { + showTableAlterDialogError(state, result.error); + return; + } + setTableAlterDialogSaving(state, true); + try { + if (result.payload) { + var response = await fetch(data.path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(result.payload), + }); + var responseData = null; + try { + responseData = await response.json(); + } catch (_error) { + responseData = null; + } + if (!response.ok || (responseData && responseData.ok === false)) { + throw rowMutationRequestError(response, responseData); + } + } + await assignTableAlterColumnTypes(result.columnTypeAssignments || []); + state.shouldRestoreFocus = false; + state.dialog.close(); + location.reload(); + } catch (error) { + setTableAlterDialogSaving(state, false); + showTableAlterDialogError(state, error.message || "Could not alter table"); + } +} + +async function saveTableAlterDialog(state) { + if (state.isSaving) { + return; + } + if (state.mode === "review") { + if (!state.reviewResult) { + showTableAlterDialogError(state, "Could not find the reviewed changes."); + return; + } + await applyTableAlterChanges(state, state.reviewResult); + return; + } + clearTableAlterDialogError(state); + var result = collectTableAlterPayload(state); + if (result.error) { + showTableAlterDialogError(state, result.error); + return; + } + showTableAlterReview(state, result); +} + +function confirmDiscardTableAlterChanges(state) { + if (!tableAlterDialogHasChanges(state)) { + return true; + } + return window.confirm("Discard table changes?"); +} + +function closeTableAlterDialogIfConfirmed(state) { + if (!state || state.isSaving) { + return false; + } + if (!confirmDiscardTableAlterChanges(state)) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + +function closeTableAlterDialog(state) { + if (!state || state.isSaving) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + +function ensureTableAlterDialog(manager) { + if (tableAlterDialogState) { + return tableAlterDialogState; + } + if (!window.HTMLDialogElement) { + return null; + } + + var dialog = document.createElement("dialog"); + dialog.id = TABLE_ALTER_DIALOG_ID; + dialog.className = "table-alter-dialog"; + dialog.setAttribute("aria-labelledby", "table-alter-title"); + dialog.innerHTML = ` +
+ Alter table +
+
+ +
+
+ +
+ +
+
+ +
+ + + +
+ + `; + document.body.appendChild(dialog); + + tableAlterDialogState = { + dialog: dialog, + form: dialog.querySelector(".table-alter-form"), + title: dialog.querySelector(".modal-title"), + error: dialog.querySelector(".table-alter-error"), + fields: dialog.querySelector(".table-alter-fields"), + review: dialog.querySelector(".table-alter-review"), + columnList: dialog.querySelector(".table-alter-column-list"), + addColumnButton: dialog.querySelector(".table-alter-add-column"), + backButton: dialog.querySelector(".table-alter-back"), + cancelButton: dialog.querySelector(".table-alter-cancel"), + saveButton: dialog.querySelector(".table-alter-save"), + currentButton: null, + shouldRestoreFocus: true, + isSaving: false, + initialSignature: "", + nextColumnIndex: 0, + deletedColumns: [], + originalColumnNames: [], + originalPrimaryKeys: [], + mode: "edit", + reviewResult: null, + manager: manager, + }; + + tableAlterDialogState.form.addEventListener("submit", function (ev) { + ev.preventDefault(); + saveTableAlterDialog(tableAlterDialogState); + }); + + tableAlterDialogState.addColumnButton.addEventListener("click", function () { + if (tableAlterDialogState.isSaving) { + return; + } + var row = addTableAlterColumn(tableAlterDialogState, { + type: "text", + existing: false, + expanded: true, + }); + clearTableAlterDialogError(tableAlterDialogState); + updateTableAlterMoveButtons(tableAlterDialogState); + updateTableAlterSaveButtonState(tableAlterDialogState); + row.querySelector(".table-alter-column-name").focus(); + }); + + tableAlterDialogState.cancelButton.addEventListener("click", function () { + closeTableAlterDialog(tableAlterDialogState); + }); + + tableAlterDialogState.backButton.addEventListener("click", function () { + if (tableAlterDialogState.isSaving) { + return; + } + clearTableAlterDialogError(tableAlterDialogState); + showTableAlterEditor(tableAlterDialogState); + var firstName = tableAlterDialogState.columnList.querySelector( + ".table-alter-column-name", + ); + if (firstName) { + firstName.focus(); + } + }); + + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog) { + closeTableAlterDialogIfConfirmed(tableAlterDialogState); + } + }); + + dialog.addEventListener("keydown", function (ev) { + if (ev.key !== "Escape") { + return; + } + ev.preventDefault(); + closeTableAlterDialogIfConfirmed(tableAlterDialogState); + }); + + dialog.addEventListener("cancel", function (ev) { + ev.preventDefault(); + closeTableAlterDialogIfConfirmed(tableAlterDialogState); + }); + + dialog.addEventListener("close", function () { + var state = tableAlterDialogState; + clearTableAlterDialogError(state); + setTableAlterDialogSaving(state, false); + if ( + state.shouldRestoreFocus && + state.currentButton && + document.contains(state.currentButton) + ) { + state.currentButton.focus(); + } + }); + + return tableAlterDialogState; +} + +function openTableAlterDialog(button, manager) { + var data = tableAlterData(); + if (!data) { + return; + } + var state = ensureTableAlterDialog(manager); + if (!state) { + return; + } + + var menu = button.closest("details"); + if (menu) { + menu.open = false; + } + state.manager = manager; + state.currentButton = button; + state.shouldRestoreFocus = true; + state.title.textContent = "Alter table " + data.tableName; + clearTableAlterDialogError(state); + resetTableAlterDialog(state, data); + if (!state.dialog.open) { + state.dialog.showModal(); + } + var firstName = state.columnList.querySelector(".table-alter-column-name"); + if (firstName) { + firstName.focus(); + } +} + +function initTableAlterActions(manager) { + if (!window.fetch || !window.HTMLDialogElement || !tableAlterData()) { + return; + } + document.addEventListener("click", function (ev) { + var button = ev.target.closest('button[data-table-action="alter-table"]'); + if (!button) { + return; + } + ev.preventDefault(); + openTableAlterDialog(button, manager); + }); +} + function tableForeignKeys() { return tablePageData().foreignKeys || {}; } @@ -2696,6 +4171,7 @@ document.addEventListener("datasette_init", function (evt) { registerBuiltinColumnFieldPlugins(manager); initTableCreateActions(manager); + initTableAlterActions(manager); initRowInsertActions(manager); initRowEditActions(manager); initRowDeleteActions(manager); diff --git a/datasette/views/row.py b/datasette/views/row.py index 4d61eb91..34773d48 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -212,6 +212,7 @@ class RowView(DataView): table, not is_table, None, + None, ), "row_actions": row_actions, "top_row": make_slot_function( diff --git a/datasette/views/table.py b/datasette/views/table.py index 11a28323..5175c7d9 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -50,7 +50,7 @@ from datasette.filters import Filters import sqlite_utils from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT from .base import BaseView, DatasetteError, _error, stream_csv -from .database import QueryView +from .database import QueryView, _custom_column_type_options_for_create_table from .table_extras import ( TABLE_EXTRA_BUNDLES, TableExtraContext, @@ -62,6 +62,13 @@ LINK_WITH_LABEL = ( '{label} {id}' ) LINK_WITH_VALUE = '{id}' +ALTER_TABLE_COLUMN_TYPES = ["text", "integer", "float", "blob"] +ALTER_TABLE_TYPE_FOR_SQLITE_TYPE = { + SQLiteType.TEXT: "text", + SQLiteType.INTEGER: "integer", + SQLiteType.REAL: "float", + SQLiteType.BLOB: "blob", +} class Row: @@ -283,7 +290,14 @@ async def _foreign_key_autocomplete_urls( async def _table_page_data( - datasette, request, db, database_name, table_name, is_view, table_insert_ui + datasette, + request, + db, + database_name, + table_name, + is_view, + table_insert_ui, + table_alter_ui, ): data = { "database": database_name, @@ -292,6 +306,8 @@ async def _table_page_data( } if table_insert_ui: data["insertRow"] = table_insert_ui + if table_alter_ui: + data["alterTable"] = table_alter_ui if not is_view: foreign_keys = await _foreign_key_autocomplete_urls( datasette, request, db, database_name, table_name @@ -354,6 +370,63 @@ async def _table_insert_ui( } +async def _table_alter_ui( + datasette, request, db, database_name, table_name, is_view, pks +): + if is_view or not db.is_mutable: + return None + + if not await datasette.allowed( + action="alter-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return None + + column_types_map = await datasette.get_column_types(database_name, table_name) + columns = [] + for column in await db.table_column_details(table_name): + if column.hidden: + continue + sqlite_type = SQLiteType.from_declared_type(column.type) + column_type = column_types_map.get(column.name) + columns.append( + { + "name": column.name, + "type": ALTER_TABLE_TYPE_FOR_SQLITE_TYPE.get(sqlite_type, "text"), + "sqlite_type": sqlite_type.value, + "notnull": column.notnull, + "default": column.default_value, + "has_default": column.default_value is not None, + "is_pk": column.name in pks, + "column_type": ( + {"type": column_type.name, "config": column_type.config} + if column_type is not None + else None + ), + } + ) + + data = { + "path": "{}/-/alter".format(datasette.urls.table(database_name, table_name)), + "tableName": table_name, + "columns": columns, + "primaryKeys": pks, + "columnTypes": ALTER_TABLE_COLUMN_TYPES, + "defaultExpressions": list(DEFAULT_EXPR_SQL), + } + can_set_column_type = await datasette.allowed( + action="set-column-type", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ) + if can_set_column_type: + data["customColumnTypes"] = _custom_column_type_options_for_create_table( + datasette + ) + return data + + async def display_columns_and_rows( datasette, database_name, @@ -2421,7 +2494,11 @@ async def table_view_data( table_insert_ui = await _table_insert_ui( datasette, request, db, database_name, table_name, is_view, pks ) + table_alter_ui = await _table_alter_ui( + datasette, request, db, database_name, table_name, is_view, pks + ) data["table_insert_ui"] = table_insert_ui + data["table_alter_ui"] = table_alter_ui data["table_page_data"] = await _table_page_data( datasette, request, @@ -2430,6 +2507,7 @@ async def table_view_data( table_name, is_view, table_insert_ui, + table_alter_ui, ) return data, rows[:page_size], columns, expanded_columns, sql, next_url diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 0c7042f0..b7990009 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -130,6 +130,7 @@ def write_playwright_config(config_path): "notes": "textarea", }, "permissions": { + "alter-table": True, "insert-row": True, "update-row": True, "delete-row": True, @@ -328,6 +329,215 @@ def test_create_table_flow(page, datasette_server): } +@pytest.mark.playwright +def test_alter_table_flow(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + assert dialog.locator(".modal-title").inner_text() == "Alter table projects" + assert dialog.locator(".table-alter-save").is_disabled() + type_options = dialog.locator(".table-alter-column-type").first.locator("option") + assert type_options.all_inner_texts() == [ + "text", + "integer", + "floating point number", + "blob - binary data", + ] + first_more_options = dialog.locator(".table-alter-more-options").first + assert first_more_options.inner_text() == "> Advanced options" + first_more_options.click() + assert first_more_options.inner_text() == "v Hide options" + expanded_options_text = dialog.locator(".table-alter-column-details").first.inner_text() + assert dialog.locator(".table-alter-fields").evaluate( + "node => node.scrollWidth <= node.clientWidth + 1" + ) + assert "Not null" in expanded_options_text + assert "This value cannot be left unset" in expanded_options_text + assert "Default value" in expanded_options_text + assert "or default to a specific time" in expanded_options_text + assert "Primary key" in expanded_options_text + assert "An ID that uniquely identifies this record" in expanded_options_text + + dialog.locator(".table-alter-add-column").click() + assert dialog.locator(".table-alter-save").is_enabled() + dialog.locator(".table-alter-column-name").last.fill("status") + dialog.locator(".table-alter-column-type").last.select_option("text") + dialog.locator(".table-alter-default").last.fill("planned") + dialog.locator(".table-alter-save").click() + review = dialog.locator(".table-alter-review") + review.wait_for() + assert not dialog.locator(".table-alter-column-list").is_visible() + review_text = review.inner_text() + assert "Add column status as text, with default value planned." in review_text + assert "Set column order to" not in review_text + assert dialog.locator(".table-alter-back").is_visible() + assert dialog.locator(".table-alter-save").inner_text() == "Apply changes" + dialog.locator(".table-alter-save").click() + + columns = [] + for _ in range(20): + response = httpx.get(f"{datasette_server}data/projects.json?_extra=columns") + response.raise_for_status() + columns = response.json()["columns"] + if "status" in columns: + break + time.sleep(0.1) + assert "status" in columns + + +@pytest.mark.playwright +def test_alter_table_primary_key_columns_stay_at_top(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + rows = dialog.locator(".table-alter-column-row") + assert rows.nth(0).locator(".table-alter-column-name").input_value() == "id" + first_row_move_buttons = rows.nth(0).locator(".table-alter-move-controls button") + for i in range(first_row_move_buttons.count()): + assert first_row_move_buttons.nth(i).is_disabled() + assert ( + first_row_move_buttons.nth(i).get_attribute("title") + == "Primary key columns are always listed first" + ) + + assert rows.nth(1).locator(".table-alter-move-up").is_disabled() + assert rows.nth(1).locator(".table-alter-move-top").get_attribute("title") == ( + "Primary key columns are always listed first" + ) + assert rows.nth(1).locator(".table-alter-move-up").get_attribute("title") == ( + "Primary key columns are always listed first" + ) + last_row = rows.nth(rows.count() - 1) + assert last_row.locator(".table-alter-column-name").input_value() == "score" + last_row.locator(".table-alter-move-top").click() + assert rows.nth(0).locator(".table-alter-column-name").input_value() == "id" + assert rows.nth(1).locator(".table-alter-column-name").input_value() == "score" + + +@pytest.mark.playwright +def test_alter_table_review_rename_primary_key_column(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + save = dialog.locator(".table-alter-save") + assert save.is_disabled() + dialog.locator(".table-alter-column-name").first.fill("id3") + assert save.is_enabled() + save.click() + + review = dialog.locator(".table-alter-review") + review.wait_for() + review_text = review.inner_text() + assert "Rename column id to id3." in review_text + assert "Set primary key to" not in review_text + assert dialog.locator(".table-alter-review-name").all_inner_texts() == [ + "id", + "id3", + ] + + +@pytest.mark.playwright +def test_alter_table_review_not_null_wording(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + dialog.locator(".table-alter-more-options").first.click() + dialog.locator(".table-alter-not-null-input").first.check() + dialog.locator(".table-alter-save").click() + + review = dialog.locator(".table-alter-review") + review.wait_for() + assert "Change column id: not null (require values)." in review.inner_text() + + +@pytest.mark.playwright +def test_alter_table_review_warns_when_dropping_column(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + remove_buttons = dialog.locator(".table-alter-remove-column") + remove_buttons.nth(remove_buttons.count() - 1).click() + dialog.locator(".table-alter-save").click() + + review = dialog.locator(".table-alter-review") + review.wait_for() + assert not dialog.locator(".table-alter-column-list").is_visible() + review_text = review.inner_text() + assert "Warning: data in dropped columns will be permanently lost." in review_text + assert "Drop column score." in review_text + assert "Set column order to" not in review_text + assert dialog.locator(".table-alter-review-damaging").inner_text() == ( + "Drop column score." + ) + + dialog.locator(".table-alter-back").click() + assert dialog.locator(".table-alter-column-list").is_visible() + assert dialog.locator(".table-alter-save").inner_text() == "Review changes" + + +@pytest.mark.playwright +def test_alter_table_cancel_skips_discard_prompt(page, datasette_server): + def open_alter_dialog(): + page.locator("details.actions-menu-links").evaluate("node => node.open = true") + page.locator('button[data-table-action="alter-table"]').click() + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + return dialog + + page.goto(f"{datasette_server}data/projects") + page.evaluate( + """ + () => { + window.__discardConfirmMessages = []; + window.confirm = (message) => { + window.__discardConfirmMessages.push(message); + return false; + }; + } + """ + ) + + dialog = open_alter_dialog() + dialog.locator(".table-alter-add-column").click() + dialog.locator(".table-alter-column-name").last.fill("cancel_me") + dialog.locator(".table-alter-cancel").click() + assert dialog.evaluate("node => node.open") is False + assert page.evaluate("() => window.__discardConfirmMessages") == [] + + dialog = open_alter_dialog() + dialog.locator(".table-alter-add-column").click() + dialog.locator(".table-alter-column-name").last.fill("escape_me") + page.keyboard.press("Escape") + assert page.evaluate("() => window.__discardConfirmMessages") == [ + "Discard table changes?" + ] + assert dialog.evaluate("node => node.open") is True + + page.evaluate("() => window.__discardConfirmMessages = []") + dialog.evaluate( + """node => node.dispatchEvent(new MouseEvent("click", {bubbles: true}))""" + ) + assert page.evaluate("() => window.__discardConfirmMessages") == [ + "Discard table changes?" + ] + assert dialog.evaluate("node => node.open") is True + + @pytest.mark.playwright def test_navigation_search_tracks_and_renders_recent_items(page, datasette_server): page.goto(datasette_server) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 887fdf50..374cc08d 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1078,6 +1078,123 @@ async def test_database_create_table_data_includes_custom_column_types(): ds.close() +@pytest.mark.asyncio +async def test_table_alter_action_button_and_data(): + ds = Datasette( + [], + config={ + "databases": { + "data": { + "tables": { + "items": { + "permissions": { + "alter-table": {"id": "root"}, + "set-column-type": {"id": "root"}, + }, + "column_types": {"name": "textarea"}, + }, + }, + }, + }, + }, + ) + try: + db = ds.add_database( + Database(ds, memory_name="test_table_alter_action"), name="data" + ) + await db.execute_write_script(""" + create table items ( + id integer primary key, + name text not null, + score integer default 5 + ); + """) + response = await ds.client.get("/data/items", actor={"id": "root"}) + assert response.status_code == 200 + soup = Soup(response.text, "html.parser") + + button = soup.select_one( + 'button.action-menu-button[data-table-action="alter-table"]' + ) + assert button is not None + assert button["aria-label"] == "Alter table items" + assert button["role"] == "menuitem" + description = button.find("span", class_="dropdown-description") + assert description.text.strip() == ( + "Change columns and primary key for this table." + ) + description.extract() + assert button.text.strip() == "Alter table" + assert any( + "edit-tools.js" in script.get("src", "") + for script in soup.find_all("script") + ) + + alter_data = table_data_from_soup(soup)["alterTable"] + assert alter_data["path"] == "/data/items/-/alter" + assert alter_data["tableName"] == "items" + assert alter_data["primaryKeys"] == ["id"] + assert alter_data["columnTypes"] == ["text", "integer", "float", "blob"] + assert alter_data["defaultExpressions"] == [ + "current_timestamp", + "current_date", + "current_time", + ] + assert [option["name"] for option in alter_data["customColumnTypes"]] == [ + "email", + "json", + "textarea", + "url", + ] + assert alter_data["columns"] == [ + { + "name": "id", + "type": "integer", + "sqlite_type": "INTEGER", + "notnull": 0, + "default": None, + "has_default": False, + "is_pk": True, + "column_type": None, + }, + { + "name": "name", + "type": "text", + "sqlite_type": "TEXT", + "notnull": 1, + "default": None, + "has_default": False, + "is_pk": False, + "column_type": {"type": "textarea", "config": None}, + }, + { + "name": "score", + "type": "integer", + "sqlite_type": "INTEGER", + "notnull": 0, + "default": "5", + "has_default": True, + "is_pk": False, + "column_type": None, + }, + ] + + response_without_permission = await ds.client.get( + "/data/items", actor={"id": "someone-else"} + ) + assert response_without_permission.status_code == 200 + soup_without_permission = Soup(response_without_permission.text, "html.parser") + assert ( + soup_without_permission.select_one( + 'button[data-table-action="alter-table"]' + ) + is None + ) + assert "alterTable" not in table_data_from_soup(soup_without_permission) + finally: + ds.close() + + @pytest.mark.asyncio async def test_table_insert_action_button_and_data(): ds = Datasette( From 15a3ac58cc10a4004e21e67845e53b2ead26a3b3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 09:24:28 -0700 Subject: [PATCH 012/167] Ran Prettier --- datasette/static/edit-tools.js | 96 +++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 43 deletions(-) diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 284d6bde..cb964365 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -49,8 +49,7 @@ function hideRowMutationStatus() { function databaseCreateTableData() { return ( - window._datasetteDatabaseData && - window._datasetteDatabaseData.createTable + window._datasetteDatabaseData && window._datasetteDatabaseData.createTable ); } @@ -166,10 +165,7 @@ function tableCreateSelectTypeValue(select, type) { } function updateTableCreateCustomColumnTypePlaceholder(select) { - select.classList.toggle( - "table-create-input-placeholder", - !select.value, - ); + select.classList.toggle("table-create-input-placeholder", !select.value); } function createTableCustomColumnTypeSelect() { @@ -273,9 +269,7 @@ function createTableColumnRow(state, column) { } row.remove(); clearTableCreateDialogError(state); - var nextInput = state.columnList.querySelector( - ".table-create-column-name", - ); + var nextInput = state.columnList.querySelector(".table-create-column-name"); if (nextInput) { nextInput.focus(); } else { @@ -418,7 +412,9 @@ function validateTableCreateColumnTypeAssignments(assignments) { if (!option) { return "Unknown custom column type: " + assignment.columnType; } - if (!tableCreateCustomTypeAppliesToSqliteType(option, assignment.sqliteType)) { + if ( + !tableCreateCustomTypeAppliesToSqliteType(option, assignment.sqliteType) + ) { return ( "Custom type " + assignment.columnType + @@ -441,7 +437,8 @@ function fallbackTableUrl(tableName) { function tableCreateSetColumnTypeUrl(responseData, payload) { var tableUrl = - responseData.table_url || fallbackTableUrl(responseData.table || payload.table); + responseData.table_url || + fallbackTableUrl(responseData.table || payload.table); if (!tableUrl) { return null; } @@ -452,7 +449,11 @@ function tableCreateSetColumnTypeUrl(responseData, payload) { return url.toString(); } -async function assignTableCreateColumnTypes(responseData, payload, assignments) { +async function assignTableCreateColumnTypes( + responseData, + payload, + assignments, +) { if (!assignments.length) { return; } @@ -542,7 +543,8 @@ async function saveTableCreateDialog(state) { columnTypeAssignments, ); var tableUrl = - responseData.table_url || fallbackTableUrl(responseData.table || payload.table); + responseData.table_url || + fallbackTableUrl(responseData.table || payload.table); state.shouldRestoreFocus = false; state.dialog.close(); if (tableUrl) { @@ -718,7 +720,11 @@ function openTableCreateDialog(button, manager) { } function initTableCreateActions(manager) { - if (!window.fetch || !window.HTMLDialogElement || !databaseCreateTableData()) { + if ( + !window.fetch || + !window.HTMLDialogElement || + !databaseCreateTableData() + ) { return; } document.addEventListener("click", function (ev) { @@ -952,15 +958,15 @@ function updateTableAlterMoveButtons(state) { var isPrimaryKey = tableAlterRowIsPrimaryKey(row); var previous = row.previousElementSibling; var next = row.nextElementSibling; - row.querySelectorAll(".table-alter-move-controls button").forEach( - function (button) { + row + .querySelectorAll(".table-alter-move-controls button") + .forEach(function (button) { button.title = button.dataset.defaultTitle || button.title; button.disabled = state.isSaving || isPrimaryKey; if (isPrimaryKey) { button.title = primaryKeyMoveTitle; } - }, - ); + }); if (!isPrimaryKey) { var topButton = row.querySelector(".table-alter-move-top"); var upButton = row.querySelector(".table-alter-move-up"); @@ -1028,7 +1034,9 @@ function setTableAlterDialogSaving(state, isSaving) { state.columnList .querySelectorAll(".table-alter-default-expr") .forEach(function (select) { - syncTableAlterDefaultControls(select.closest(".table-alter-column-row")); + syncTableAlterDefaultControls( + select.closest(".table-alter-column-row"), + ); }); } updateTableAlterMoveButtons(state); @@ -1051,10 +1059,7 @@ function tableAlterSelectTypeValue(select, type) { } function updateTableAlterCustomColumnTypePlaceholder(select) { - select.classList.toggle( - "table-alter-input-placeholder", - !select.value, - ); + select.classList.toggle("table-alter-input-placeholder", !select.value); } function createTableAlterCustomColumnTypeSelect() { @@ -1159,10 +1164,7 @@ function createTableAlterColumnRow(state, column) { expandButton.className = "table-alter-more-options"; expandButton.setAttribute("aria-label", "Toggle column settings"); expandButton.setAttribute("aria-controls", details.id); - expandButton.setAttribute( - "aria-expanded", - details.hidden ? "false" : "true", - ); + expandButton.setAttribute("aria-expanded", details.hidden ? "false" : "true"); function updateExpandButton() { var isExpanded = expandButton.getAttribute("aria-expanded") === "true"; expandButton.textContent = isExpanded @@ -1243,8 +1245,7 @@ function createTableAlterColumnRow(state, column) { defaultExprLabel.textContent = "or default to a specific time"; var defaultExprSelect = document.createElement("select"); defaultExprSelect.id = defaultExprId; - defaultExprSelect.className = - "table-alter-input table-alter-default-expr"; + defaultExprSelect.className = "table-alter-input table-alter-default-expr"; defaultExprSelect.setAttribute("aria-label", "or default to a specific time"); tableAlterSelectDefaultExprValue(defaultExprSelect, ""); defaultExprField.appendChild(defaultExprLabel); @@ -1298,7 +1299,8 @@ function createTableAlterColumnRow(state, column) { var moveBottomButton = document.createElement("button"); moveBottomButton.type = "button"; - moveBottomButton.className = "table-alter-icon-button table-alter-move-bottom"; + moveBottomButton.className = + "table-alter-icon-button table-alter-move-bottom"; moveBottomButton.setAttribute("aria-label", "Move column to bottom"); moveBottomButton.title = "Move column to bottom"; moveBottomButton.dataset.defaultTitle = moveBottomButton.title; @@ -1349,15 +1351,15 @@ function createTableAlterColumnRow(state, column) { controls.push(customTypeSelect); } controls.forEach(function (control) { - control.addEventListener("input", function () { - clearTableAlterDialogError(state); - updateTableAlterSaveButtonState(state); - }); - control.addEventListener("change", function () { - clearTableAlterDialogError(state); - updateTableAlterSaveButtonState(state); - }); + control.addEventListener("input", function () { + clearTableAlterDialogError(state); + updateTableAlterSaveButtonState(state); }); + control.addEventListener("change", function () { + clearTableAlterDialogError(state); + updateTableAlterSaveButtonState(state); + }); + }); defaultInput.addEventListener("input", function () { if (defaultInput.value) { @@ -1407,7 +1409,12 @@ function createTableAlterColumnRow(state, column) { moveTopButton.addEventListener("click", function () { var first = tableAlterFirstNonPrimaryRow(state); - if (state.isSaving || tableAlterRowIsPrimaryKey(row) || !first || first === row) { + if ( + state.isSaving || + tableAlterRowIsPrimaryKey(row) || + !first || + first === row + ) { return; } state.columnList.insertBefore(row, first); @@ -1448,7 +1455,12 @@ function createTableAlterColumnRow(state, column) { moveBottomButton.addEventListener("click", function () { var last = state.columnList.lastElementChild; - if (state.isSaving || tableAlterRowIsPrimaryKey(row) || !last || last === row) { + if ( + state.isSaving || + tableAlterRowIsPrimaryKey(row) || + !last || + last === row + ) { return; } state.columnList.appendChild(row); @@ -1797,9 +1809,7 @@ function tableAlterOperationSummary(operation) { } if (Object.prototype.hasOwnProperty.call(args, "not_null")) { changes.push( - args.not_null - ? "not null (require values)" - : "allow unset values", + args.not_null ? "not null (require values)" : "allow unset values", ); } if (args.default_expr) { From c9c79fdfc8395fe49d0e9d262548f2b337bec3bc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 09:58:39 -0700 Subject: [PATCH 013/167] Isolate Unix domain socket test server paths - Use a per-process socket path for the UDS test fixture. - Clean up stale socket files before and after the fixture runs. - Close the HTTP client and wait for the Datasette subprocess to exit. --- tests/conftest.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8860d54c..7ec03146 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -260,8 +260,12 @@ def ds_unix_domain_socket_server(tmp_path_factory): # This used to use tmp_path_factory.mktemp("uds") but that turned out to # produce paths that were too long to use as UDS on macOS, see # https://github.com/simonw/datasette/issues/1407 - so I switched to - # using tempfile.gettempdir() - uds = str(pathlib.Path(tempfile.gettempdir()) / "datasette.sock") + # using tempfile.gettempdir() with a per-process filename. + uds = str(pathlib.Path(tempfile.gettempdir()) / f"datasette-{os.getpid()}.sock") + try: + os.unlink(uds) + except FileNotFoundError: + pass ds_proc = subprocess.Popen( [sys.executable, "-m", "datasette", "--memory", "--uds", uds], stdout=subprocess.PIPE, @@ -271,12 +275,26 @@ def ds_unix_domain_socket_server(tmp_path_factory): # Poll until available transport = httpx.HTTPTransport(uds=uds) client = httpx.Client(transport=transport) - wait_until_responds("http://localhost/_memory.json", client=client) - # Check it started successfully - assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") - yield ds_proc, uds - # Shut it down at the end of the pytest session - ds_proc.terminate() + try: + wait_until_responds( + "http://localhost/_memory.json", timeout=30.0, client=client + ) + # Check it started successfully + assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") + yield ds_proc, uds + finally: + client.close() + # Shut it down at the end of the pytest session + ds_proc.terminate() + try: + ds_proc.wait(timeout=5) + except subprocess.TimeoutExpired: + ds_proc.kill() + ds_proc.wait() + try: + os.unlink(uds) + except FileNotFoundError: + pass # Import fixtures from fixtures.py to make them available From 4115213e17d23cdf3286bc4583e7005991892568 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 10:22:42 -0700 Subject: [PATCH 014/167] Precompute action permissions for table pages - Extract reusable helpers for database and table action permission preloading. - Precompute those permissions before building table-page HTML data. - Document the default table actions plugin. --- datasette/views/table.py | 22 ++++++++++++--- datasette/views/table_extras.py | 49 ++++++++++++++++++++++----------- docs/plugins.rst | 9 ++++++ tests/test_playwright.py | 10 +++---- 4 files changed, 65 insertions(+), 25 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 5175c7d9..82c7e03d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -54,6 +54,8 @@ from .database import QueryView, _custom_column_type_options_for_create_table from .table_extras import ( TABLE_EXTRA_BUNDLES, TableExtraContext, + precompute_database_action_permissions, + precompute_table_action_permissions, resolve_table_extras, table_extra_registry, ) @@ -1247,7 +1249,10 @@ class TableAlterView(BaseView): defaults[args.name] = _literal_default( db_for_write, args.default ) - if "default_expr" in args.model_fields_set and not args.not_null: + if ( + "default_expr" in args.model_fields_set + and not args.not_null + ): defaults[args.name] = _default_expression_sql( args.default_expr ) @@ -1363,9 +1368,9 @@ class TableAlterView(BaseView): "altered": altered, "schema": after_schema, "before_schema": before_schema, - "operations_applied": 0 - if alter_request.dry_run - else len(alter_request.operations), + "operations_applied": ( + 0 if alter_request.dry_run else len(alter_request.operations) + ), "dry_run": alter_request.dry_run, }, status=200, @@ -2068,6 +2073,15 @@ async def table_view_data( if redirect_response: return redirect_response + if context_for_html_hack: + await precompute_database_action_permissions( + datasette, request.actor, database_name + ) + if not is_view: + await precompute_table_action_permissions( + datasette, request.actor, database_name, table_name + ) + # Introspect columns and primary keys for table pks = await db.primary_keys(table_name) table_columns = await db.table_columns(table_name) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index a0308e49..7cb4d8f0 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -367,23 +367,14 @@ class ActionsExtra(Extra): # that allowed() calls made inside the plugin hooks below # are served from the cache datasette = context.datasette - await datasette.allowed_many( - actions=[ - name - for name, action in datasette.actions.items() - if action.resource_class is TableResource - ], - resource=TableResource(context.database_name, context.table_name), - actor=context.request.actor, + await precompute_table_action_permissions( + datasette, + context.request.actor, + context.database_name, + context.table_name, ) - await datasette.allowed_many( - actions=[ - name - for name, action in datasette.actions.items() - if action.resource_class is DatabaseResource - ], - resource=DatabaseResource(context.database_name), - actor=context.request.actor, + await precompute_database_action_permissions( + datasette, context.request.actor, context.database_name ) for hook in method(**kwargs): extra_links = await await_me_maybe(hook) @@ -394,6 +385,32 @@ class ActionsExtra(Extra): return actions +async def precompute_table_action_permissions( + datasette, actor, database_name, table_name +): + await datasette.allowed_many( + actions=[ + name + for name, action in datasette.actions.items() + if action.resource_class is TableResource + ], + resource=TableResource(database_name, table_name), + actor=actor, + ) + + +async def precompute_database_action_permissions(datasette, actor, database_name): + await datasette.allowed_many( + actions=[ + name + for name, action in datasette.actions.items() + if action.resource_class is DatabaseResource + ], + resource=DatabaseResource(database_name), + actor=actor, + ) + + class IsViewExtra(Extra): description = "Whether this resource is a view instead of a table" example = ExtraExample("/fixtures/simple_view.json?_extra=is_view") diff --git a/docs/plugins.rst b/docs/plugins.rst index c2eb282a..d2b5c20a 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -280,6 +280,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "query_actions" ] }, + { + "name": "datasette.default_table_actions", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "table_actions" + ] + }, { "name": "datasette.events", "static": false, diff --git a/tests/test_playwright.py b/tests/test_playwright.py index b7990009..70b5a8c1 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -350,7 +350,9 @@ def test_alter_table_flow(page, datasette_server): assert first_more_options.inner_text() == "> Advanced options" first_more_options.click() assert first_more_options.inner_text() == "v Hide options" - expanded_options_text = dialog.locator(".table-alter-column-details").first.inner_text() + expanded_options_text = dialog.locator( + ".table-alter-column-details" + ).first.inner_text() assert dialog.locator(".table-alter-fields").evaluate( "node => node.scrollWidth <= node.clientWidth + 1" ) @@ -500,8 +502,7 @@ def test_alter_table_cancel_skips_discard_prompt(page, datasette_server): return dialog page.goto(f"{datasette_server}data/projects") - page.evaluate( - """ + page.evaluate(""" () => { window.__discardConfirmMessages = []; window.confirm = (message) => { @@ -509,8 +510,7 @@ def test_alter_table_cancel_skips_discard_prompt(page, datasette_server): return false; }; } - """ - ) + """) dialog = open_alter_dialog() dialog.locator(".table-alter-add-column").click() From 8cec528eebd253994fbb41c85d571ab649d723e9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 10:32:21 -0700 Subject: [PATCH 015/167] Test against pyodide/v314.0.0 Now that we depend on pydantic we need a more recent pyodide in order to load the emscripten build of pydantic-core. Refs https://github.com/simonw/datasette/pull/2789#issuecomment-4733412763 --- test-in-pyodide-with-shot-scraper.sh | 29 +++++++++++++--------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh index 4d1c4968..8d7fa08e 100755 --- a/test-in-pyodide-with-shot-scraper.sh +++ b/test-in-pyodide-with-shot-scraper.sh @@ -1,40 +1,37 @@ #!/bin/bash -set -e -# So the script fails if there are any errors +set -euo pipefail + +read -r -a PYTHON_CMD <<< "${PYTHON:-python3}" +read -r -a SHOT_SCRAPER_CMD <<< "${SHOT_SCRAPER:-shot-scraper}" # Build the wheel -python3 -m build +"${PYTHON_CMD[@]}" -m build -# Find name of wheel, strip off the dist/ -wheel=$(basename $(ls dist/*.whl) | head -n 1) +# Find name of most recently built wheel, strip off the dist/ +wheel=$(basename "$(ls -t dist/*.whl | head -n 1)") # Create a blank index page echo ' - + ' > dist/index.html # Run a server for that dist/ folder -cd dist -python3 -m http.server 8529 & -cd .. +"${PYTHON_CMD[@]}" -m http.server 8529 --directory dist & +server_pid=$! # Register the kill_server function to be called on script exit kill_server() { - pkill -f 'http.server 8529' + kill "$server_pid" 2>/dev/null || true } trap kill_server EXIT -shot-scraper javascript http://localhost:8529/ " +"${SHOT_SCRAPER_CMD[@]}" javascript http://localhost:8529/ " async () => { let pyodide = await loadPyodide(); - await pyodide.loadPackage(['micropip', 'ssl', 'setuptools']); + await pyodide.loadPackage(['micropip', 'setuptools']); let output = await pyodide.runPythonAsync(\` import micropip - await micropip.install('h11==0.12.0') - await micropip.install('httpx==0.23') - # To avoid 'from typing_extensions import deprecated' error: - await micropip.install('typing-extensions>=4.12.2') await micropip.install('http://localhost:8529/$wheel') import ssl import setuptools From 1972ba8952f0e10fc13054c4214df2473c448daf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 10:54:25 -0700 Subject: [PATCH 016/167] Split table create and alter views - Move create-table and alter-table API views into table_create_alter.py. - Keep create and alter schema-editing constants and helpers together. - Rename the create table modal context helper. --- datasette/app.py | 3 +- datasette/views/database.py | 323 +----------- datasette/views/table.py | 371 +------------- datasette/views/table_create_alter.py | 687 ++++++++++++++++++++++++++ 4 files changed, 699 insertions(+), 685 deletions(-) create mode 100644 datasette/views/table_create_alter.py diff --git a/datasette/app.py b/datasette/app.py index 6ea3d5a4..13e25d4e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -47,9 +47,9 @@ from .views import Context from .views.database import ( database_download, DatabaseView, - TableCreateView, QueryView, ) +from .views.table_create_alter import TableAlterView, TableCreateView from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView from .views.stored_queries import ( QueryCreateAnalyzeView, @@ -84,7 +84,6 @@ from .views.special import ( ) from .views.table import ( TableAutocompleteView, - TableAlterView, TableInsertView, TableUpsertView, TableSetColumnTypeView, diff --git a/datasette/views/database.py b/datasette/views/database.py index db70b135..8e9c1361 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -6,15 +6,11 @@ import itertools import json import markupsafe import os -import re -import sqlite_utils import textwrap -from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted -from datasette.column_types import SQLiteType -from datasette.resources import DatabaseResource, QueryResource, TableResource +from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import stored_query_to_dict from datasette.write_sql import QueryWriteRejected from datasette.utils import ( @@ -38,27 +34,16 @@ from datasette.utils import ( from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden from datasette.plugins import pm -from .base import BaseView, DatasetteError, View, _error, stream_csv +from .base import DatasetteError, View, stream_csv from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns from .table_extras import ( QueryExtraContext, resolve_query_extras, table_extra_registry, ) +from .table_create_alter import _create_table_ui_context from . import Context -CREATE_TABLE_COLUMN_TYPES = ["text", "integer", "float", "blob"] -CREATE_TABLE_SQLITE_TYPES = { - "text": SQLiteType.TEXT, - "integer": SQLiteType.INTEGER, - "float": SQLiteType.REAL, - "blob": SQLiteType.BLOB, -} -CREATE_TABLE_TYPE_FOR_SQLITE_TYPE = { - sqlite_type: column_type - for column_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() -} - class DatabaseView(View): async def get(self, request, datasette): @@ -142,7 +127,7 @@ class DatabaseView(View): resource=DatabaseResource(database), actor=request.actor, ) - create_table_ui = await _database_create_table_ui( + create_table_ui = await _create_table_ui_context( datasette, request, db, database, database_action_permissions ) @@ -326,57 +311,6 @@ class DatabaseContext(Context): ) -async def _database_create_table_ui( - datasette, request, db, database_name, database_action_permissions -): - if not db.is_mutable: - return None - if not database_action_permissions.get("create-table"): - return None - data = { - "path": "{}/-/create".format(datasette.urls.database(database_name)), - "databaseName": database_name, - "columnTypes": CREATE_TABLE_COLUMN_TYPES, - } - can_set_column_type = await datasette.allowed( - action="set-column-type", - resource=TableResource(database=database_name, table="__new_table__"), - actor=request.actor, - ) - if can_set_column_type: - data["customColumnTypes"] = _custom_column_type_options_for_create_table( - datasette - ) - return data - - -def _custom_column_type_options_for_create_table(datasette): - options = [] - for name, ct_cls in sorted(datasette._column_types.items()): - sqlite_types = getattr(ct_cls, "sqlite_types", None) - if sqlite_types is None: - option_sqlite_types = CREATE_TABLE_COLUMN_TYPES[:] - else: - option_sqlite_types = [ - create_table_type - for create_table_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() - if sqlite_type in sqlite_types - ] - if not option_sqlite_types: - continue - option = { - "name": name, - "description": ct_cls.description, - "sqliteTypes": option_sqlite_types, - } - if sqlite_types is not None and len(sqlite_types) == 1: - fixed_sqlite_type = CREATE_TABLE_TYPE_FOR_SQLITE_TYPE.get(sqlite_types[0]) - if fixed_sqlite_type is not None: - option["fixedSqliteType"] = fixed_sqlite_type - options.append(option) - return options - - @dataclass class QueryContext(Context): database: str = field(metadata={"help": "The name of the database being queried"}) @@ -1140,255 +1074,6 @@ class MagicParameters(dict): return super().__getitem__(key) -class TableCreateView(BaseView): - name = "table-create" - - _valid_keys = { - "table", - "rows", - "row", - "columns", - "pk", - "pks", - "ignore", - "replace", - "alter", - } - _supported_column_types = set(CREATE_TABLE_COLUMN_TYPES) - # Any string that does not contain a newline or start with sqlite_ - _table_name_re = re.compile(r"^(?!sqlite_)[^\n]+$") - - def __init__(self, datasette): - self.ds = datasette - - async def post(self, request): - db = await self.ds.resolve_database(request) - database_name = db.name - - # Must have create-table permission - if not await self.ds.allowed( - action="create-table", - resource=DatabaseResource(database=database_name), - actor=request.actor, - ): - return _error(["Permission denied"], 403) - - try: - data = await request.json() - except json.JSONDecodeError as e: - return _error(["Invalid JSON: {}".format(e)]) - - if not isinstance(data, dict): - return _error(["JSON must be an object"]) - - invalid_keys = set(data.keys()) - self._valid_keys - if invalid_keys: - return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) - - # ignore and replace are mutually exclusive - if data.get("ignore") and data.get("replace"): - return _error(["ignore and replace are mutually exclusive"]) - - # ignore and replace only allowed with row or rows - if "ignore" in data or "replace" in data: - if not data.get("row") and not data.get("rows"): - return _error(["ignore and replace require row or rows"]) - - # ignore and replace require pk or pks - if "ignore" in data or "replace" in data: - if not data.get("pk") and not data.get("pks"): - return _error(["ignore and replace require pk or pks"]) - - ignore = data.get("ignore") - replace = data.get("replace") - - if replace: - # Must have update-row permission - if not await self.ds.allowed( - action="update-row", - resource=DatabaseResource(database=database_name), - actor=request.actor, - ): - return _error(["Permission denied: need update-row"], 403) - - table_name = data.get("table") - if not table_name: - return _error(["Table is required"]) - - if not self._table_name_re.match(table_name): - return _error(["Invalid table name"]) - - table_exists = await db.table_exists(data["table"]) - columns = data.get("columns") - rows = data.get("rows") - row = data.get("row") - if not columns and not rows and not row: - return _error(["columns, rows or row is required"]) - - if rows and row: - return _error(["Cannot specify both rows and row"]) - - if rows or row: - # Must have insert-row permission - if not await self.ds.allowed( - action="insert-row", - resource=DatabaseResource(database=database_name), - actor=request.actor, - ): - return _error(["Permission denied: need insert-row"], 403) - - alter = False - if rows or row: - if not table_exists: - # if table is being created for the first time, alter=True - alter = True - else: - # alter=True only if they request it AND they have permission - if data.get("alter"): - if not await self.ds.allowed( - action="alter-table", - resource=DatabaseResource(database=database_name), - actor=request.actor, - ): - return _error(["Permission denied: need alter-table"], 403) - alter = True - - if columns: - if rows or row: - return _error(["Cannot specify columns with rows or row"]) - if not isinstance(columns, list): - return _error(["columns must be a list"]) - for column in columns: - if not isinstance(column, dict): - return _error(["columns must be a list of objects"]) - if not column.get("name") or not isinstance(column.get("name"), str): - return _error(["Column name is required"]) - if not column.get("type"): - column["type"] = "text" - if column["type"] not in self._supported_column_types: - return _error( - ["Unsupported column type: {}".format(column["type"])] - ) - # No duplicate column names - dupes = {c["name"] for c in columns if columns.count(c) > 1} - if dupes: - return _error(["Duplicate column name: {}".format(", ".join(dupes))]) - - if row: - rows = [row] - - if rows: - if not isinstance(rows, list): - return _error(["rows must be a list"]) - for row in rows: - if not isinstance(row, dict): - return _error(["rows must be a list of objects"]) - - pk = data.get("pk") - pks = data.get("pks") - - if pk and pks: - return _error(["Cannot specify both pk and pks"]) - if pk: - if not isinstance(pk, str): - return _error(["pk must be a string"]) - if pks: - if not isinstance(pks, list): - return _error(["pks must be a list"]) - for pk in pks: - if not isinstance(pk, str): - return _error(["pks must be a list of strings"]) - - # If table exists already, read pks from that instead - if table_exists: - actual_pks = await db.primary_keys(table_name) - # if pk passed and table already exists check it does not change - bad_pks = False - if len(actual_pks) == 1 and data.get("pk") and data["pk"] != actual_pks[0]: - bad_pks = True - elif ( - len(actual_pks) > 1 - and data.get("pks") - and set(data["pks"]) != set(actual_pks) - ): - bad_pks = True - if bad_pks: - return _error(["pk cannot be changed for existing table"]) - pks = actual_pks - - initial_schema = None - if table_exists: - initial_schema = await db.execute_fn( - lambda conn: sqlite_utils.Database(conn)[table_name].schema - ) - - def create_table(conn): - table = sqlite_utils.Database(conn)[table_name] - if rows: - table.insert_all( - rows, pk=pks or pk, ignore=ignore, replace=replace, alter=alter - ) - else: - table.create( - {c["name"]: c["type"] for c in columns}, - pk=pks or pk, - ) - return table.schema - - try: - schema = await db.execute_write_fn(create_table, request=request) - except Exception as e: - return _error([str(e)]) - - if initial_schema is not None and initial_schema != schema: - await self.ds.track_event( - AlterTableEvent( - request.actor, - database=database_name, - table=table_name, - before_schema=initial_schema, - after_schema=schema, - ) - ) - - table_url = self.ds.absolute_url( - request, self.ds.urls.table(db.name, table_name) - ) - table_api_url = self.ds.absolute_url( - request, self.ds.urls.table(db.name, table_name, format="json") - ) - details = { - "ok": True, - "database": db.name, - "table": table_name, - "table_url": table_url, - "table_api_url": table_api_url, - "schema": schema, - } - if rows: - details["row_count"] = len(rows) - - if not table_exists: - # Only log creation if we created a table - await self.ds.track_event( - CreateTableEvent( - request.actor, database=db.name, table=table_name, schema=schema - ) - ) - if rows: - await self.ds.track_event( - InsertRowsEvent( - request.actor, - database=db.name, - table=table_name, - num_rows=len(rows), - ignore=ignore, - replace=replace, - ) - ) - return Response.json(details, status=201) - - async def display_rows(datasette, database, request, rows, columns): display_rows = [] truncate_cells = datasette.setting("truncate_cells_html") diff --git a/datasette/views/table.py b/datasette/views/table.py index 82c7e03d..6c0d2914 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,12 +1,10 @@ import asyncio import itertools import json -from typing import Annotated, Any, Literal, Union import urllib import urllib.parse import markupsafe -from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator from datasette.column_types import SQLiteType from datasette.extras import extra_names_from_request @@ -48,9 +46,14 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response from datasette.filters import Filters import sqlite_utils -from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT from .base import BaseView, DatasetteError, _error, stream_csv -from .database import QueryView, _custom_column_type_options_for_create_table +from .database import QueryView +from .table_create_alter import ( + ALTER_TABLE_COLUMN_TYPES, + ALTER_TABLE_TYPE_FOR_SQLITE_TYPE, + DEFAULT_EXPR_SQL, + _custom_column_type_options_for_create_table, +) from .table_extras import ( TABLE_EXTRA_BUNDLES, TableExtraContext, @@ -64,13 +67,6 @@ LINK_WITH_LABEL = ( '{label} {id}' ) LINK_WITH_VALUE = '{id}' -ALTER_TABLE_COLUMN_TYPES = ["text", "integer", "float", "blob"] -ALTER_TABLE_TYPE_FOR_SQLITE_TYPE = { - SQLiteType.TEXT: "text", - SQLiteType.INTEGER: "integer", - SQLiteType.REAL: "float", - SQLiteType.BLOB: "blob", -} class Row: @@ -727,154 +723,6 @@ async def display_columns_and_rows( return columns, cell_rows -SqliteApiType = Literal["text", "integer", "float", "blob"] -DefaultExpr = Literal["current_timestamp", "current_date", "current_time"] -DEFAULT_EXPR_SQL = { - "current_timestamp": "CURRENT_TIMESTAMP", - "current_date": "CURRENT_DATE", - "current_time": "CURRENT_TIME", -} - - -class _StrictPydanticModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - -class _DefaultArgsMixin(_StrictPydanticModel): - default: Any | None = None - default_expr: DefaultExpr | None = None - - @model_validator(mode="after") - def validate_default_fields(self): - has_default = "default" in self.model_fields_set - has_default_expr = "default_expr" in self.model_fields_set - if has_default and has_default_expr: - raise ValueError("default and default_expr cannot both be provided") - if has_default_expr and self.default_expr is None: - raise ValueError("default_expr cannot be null") - return self - - -class AddColumnArgs(_DefaultArgsMixin): - name: str - type: SqliteApiType = "text" - not_null: bool = False - - -class RenameColumnArgs(_StrictPydanticModel): - name: str - to: str - - -class AlterColumnArgs(_DefaultArgsMixin): - name: str - type: SqliteApiType | None = None - not_null: bool | None = None - - @model_validator(mode="after") - def require_change(self): - if not ( - {"type", "not_null", "default", "default_expr"} & self.model_fields_set - ): - raise ValueError( - "At least one of type, not_null, default or default_expr must be provided" - ) - return self - - -class DropColumnArgs(_StrictPydanticModel): - name: str - - -class SetPrimaryKeyArgs(_StrictPydanticModel): - columns: list[str] = Field(min_length=1) - - -class ReorderColumnsArgs(_StrictPydanticModel): - columns: list[str] = Field(min_length=1) - - -class AddColumnOperation(_StrictPydanticModel): - op: Literal["add_column"] - args: AddColumnArgs - - -class RenameColumnOperation(_StrictPydanticModel): - op: Literal["rename_column"] - args: RenameColumnArgs - - -class AlterColumnOperation(_StrictPydanticModel): - op: Literal["alter_column"] - args: AlterColumnArgs - - -class DropColumnOperation(_StrictPydanticModel): - op: Literal["drop_column"] - args: DropColumnArgs - - -class SetPrimaryKeyOperation(_StrictPydanticModel): - op: Literal["set_primary_key"] - args: SetPrimaryKeyArgs - - -class ReorderColumnsOperation(_StrictPydanticModel): - op: Literal["reorder_columns"] - args: ReorderColumnsArgs - - -AlterTableOperation = Annotated[ - Union[ - AddColumnOperation, - RenameColumnOperation, - AlterColumnOperation, - DropColumnOperation, - SetPrimaryKeyOperation, - ReorderColumnsOperation, - ], - Field(discriminator="op"), -] - - -class AlterTableRequest(_StrictPydanticModel): - operations: list[AlterTableOperation] = Field(min_length=1) - dry_run: bool = False - - -def _pydantic_errors(validation_error): - errors = [] - for error in validation_error.errors(): - location = ".".join(str(item) for item in error["loc"]) - message = error["msg"] - errors.append("{}: {}".format(location, message) if location else message) - return errors - - -def _table_schema_from_conn(conn, table_name): - row = conn.execute( - "select sql from sqlite_master where type = 'table' and name = ?", - [table_name], - ).fetchone() - return row[0] if row else None - - -def _primary_key_value(columns): - if len(columns) == 1: - return columns[0] - return tuple(columns) - - -def _default_expression_sql(default_expr): - return DEFAULT_EXPR_SQL[default_expr] - - -def _literal_default(db, value): - if isinstance(value, str): - return db.quote(value) - return value - - class TableInsertView(BaseView): name = "table-insert" @@ -1172,211 +1020,6 @@ class TableUpsertView(TableInsertView): return await super().post(request, upsert=True) -class TableAlterView(BaseView): - name = "table-alter" - - def __init__(self, datasette): - self.ds = datasette - - async def post(self, request): - try: - resolved = await self.ds.resolve_table(request) - except NotFound as e: - return _error([e.args[0]], 404) - - db = resolved.db - database_name = db.name - table_name = resolved.table - - if not await self.ds.allowed( - action="alter-table", - resource=TableResource(database=database_name, table=table_name), - actor=request.actor, - ): - return _error(["Permission denied: need alter-table"], 403) - - if not db.is_mutable: - return _error(["Database is immutable"], 403) - - content_type = request.headers.get("content-type") or "" - if not content_type.startswith("application/json"): - return _error(["Invalid content-type, must be application/json"], 400) - - try: - data = await request.json() - except json.JSONDecodeError as e: - return _error(["Invalid JSON: {}".format(e)], 400) - - if not isinstance(data, dict): - return _error(["JSON must be a dictionary"], 400) - - try: - alter_request = AlterTableRequest.model_validate(data) - except ValidationError as e: - return _error(_pydantic_errors(e), 400) - - def alter_table(conn): - before_schema = _table_schema_from_conn(conn, table_name) - - def apply_operations(operation_conn): - db_for_write = sqlite_utils.Database(operation_conn) - table = db_for_write[table_name] - - add_columns = [] - types = {} - rename = {} - drop = set() - not_null = {} - defaults = {} - column_order = None - pk = SQLITE_UTILS_DEFAULT - - for operation in alter_request.operations: - args = operation.args - if operation.op == "add_column": - if args.not_null and not ( - ( - "default" in args.model_fields_set - and args.default is not None - ) - or "default_expr" in args.model_fields_set - ): - raise ValueError( - "add_column args.default or args.default_expr is required when not_null is true" - ) - add_columns.append(args) - if "default" in args.model_fields_set and not args.not_null: - defaults[args.name] = _literal_default( - db_for_write, args.default - ) - if ( - "default_expr" in args.model_fields_set - and not args.not_null - ): - defaults[args.name] = _default_expression_sql( - args.default_expr - ) - elif operation.op == "rename_column": - rename[args.name] = args.to - elif operation.op == "alter_column": - if args.type is not None: - types[args.name] = args.type - if args.not_null is not None: - not_null[args.name] = args.not_null - if "default" in args.model_fields_set: - defaults[args.name] = ( - None - if args.default is None - else _literal_default(db_for_write, args.default) - ) - if "default_expr" in args.model_fields_set: - defaults[args.name] = _default_expression_sql( - args.default_expr - ) - elif operation.op == "drop_column": - drop.add(args.name) - elif operation.op == "set_primary_key": - pk = _primary_key_value(args.columns) - elif operation.op == "reorder_columns": - column_order = args.columns - - with operation_conn: - for column in add_columns: - not_null_default = None - if column.not_null: - if "default_expr" in column.model_fields_set: - not_null_default = _default_expression_sql( - column.default_expr - ) - else: - not_null_default = _literal_default( - db_for_write, column.default - ) - table.add_column( - column.name, - column.type, - not_null_default=not_null_default, - ) - - should_transform = any( - ( - types, - rename, - drop, - not_null, - defaults, - column_order is not None, - pk is not SQLITE_UTILS_DEFAULT, - ) - ) - if should_transform: - table.transform( - types=types or None, - rename=rename or None, - drop=drop or None, - pk=pk, - not_null=not_null or None, - defaults=defaults or None, - column_order=column_order, - ) - - return _table_schema_from_conn(operation_conn, table_name) - - if alter_request.dry_run: - memory_conn = sqlite3.connect(":memory:") - try: - conn.backup(memory_conn) - return before_schema, apply_operations(memory_conn) - finally: - memory_conn.close() - - after_schema = apply_operations(conn) - return before_schema, after_schema - - try: - before_schema, after_schema = await db.execute_write_fn( - alter_table, request=request - ) - except Exception as e: - return _error([str(e)], 400) - - altered = before_schema != after_schema - if altered and not alter_request.dry_run: - await self.ds.track_event( - AlterTableEvent( - request.actor, - database=database_name, - table=table_name, - before_schema=before_schema, - after_schema=after_schema, - ) - ) - - table_url = self.ds.absolute_url( - request, self.ds.urls.table(database_name, table_name) - ) - table_api_url = self.ds.absolute_url( - request, self.ds.urls.table(database_name, table_name, format="json") - ) - return Response.json( - { - "ok": True, - "database": database_name, - "table": table_name, - "table_url": table_url, - "table_api_url": table_api_url, - "altered": altered, - "schema": after_schema, - "before_schema": before_schema, - "operations_applied": ( - 0 if alter_request.dry_run else len(alter_request.operations) - ), - "dry_run": alter_request.dry_run, - }, - status=200, - ) - - class TableSetColumnTypeView(BaseView): name = "table-set-column-type" diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py new file mode 100644 index 00000000..7decfad2 --- /dev/null +++ b/datasette/views/table_create_alter.py @@ -0,0 +1,687 @@ +import json +import re +from typing import Annotated, Any, Literal, Union + +from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator +import sqlite_utils +from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT + +from datasette.column_types import SQLiteType +from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent +from datasette.resources import DatabaseResource, TableResource +from datasette.utils import sqlite3 +from datasette.utils.asgi import NotFound, Response + +from .base import BaseView, _error + +CREATE_TABLE_COLUMN_TYPES = ["text", "integer", "float", "blob"] +CREATE_TABLE_SQLITE_TYPES = { + "text": SQLiteType.TEXT, + "integer": SQLiteType.INTEGER, + "float": SQLiteType.REAL, + "blob": SQLiteType.BLOB, +} +CREATE_TABLE_TYPE_FOR_SQLITE_TYPE = { + sqlite_type: column_type + for column_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() +} +ALTER_TABLE_COLUMN_TYPES = CREATE_TABLE_COLUMN_TYPES +ALTER_TABLE_TYPE_FOR_SQLITE_TYPE = { + SQLiteType.TEXT: "text", + SQLiteType.INTEGER: "integer", + SQLiteType.REAL: "float", + SQLiteType.BLOB: "blob", +} + + +async def _create_table_ui_context( + datasette, request, db, database_name, database_action_permissions +): + if not db.is_mutable: + return None + if not database_action_permissions.get("create-table"): + return None + data = { + "path": "{}/-/create".format(datasette.urls.database(database_name)), + "databaseName": database_name, + "columnTypes": CREATE_TABLE_COLUMN_TYPES, + } + can_set_column_type = await datasette.allowed( + action="set-column-type", + resource=TableResource(database=database_name, table="__new_table__"), + actor=request.actor, + ) + if can_set_column_type: + data["customColumnTypes"] = _custom_column_type_options_for_create_table( + datasette + ) + return data + + +def _custom_column_type_options_for_create_table(datasette): + options = [] + for name, ct_cls in sorted(datasette._column_types.items()): + sqlite_types = getattr(ct_cls, "sqlite_types", None) + if sqlite_types is None: + option_sqlite_types = CREATE_TABLE_COLUMN_TYPES[:] + else: + option_sqlite_types = [ + create_table_type + for create_table_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() + if sqlite_type in sqlite_types + ] + if not option_sqlite_types: + continue + option = { + "name": name, + "description": ct_cls.description, + "sqliteTypes": option_sqlite_types, + } + if sqlite_types is not None and len(sqlite_types) == 1: + fixed_sqlite_type = CREATE_TABLE_TYPE_FOR_SQLITE_TYPE.get(sqlite_types[0]) + if fixed_sqlite_type is not None: + option["fixedSqliteType"] = fixed_sqlite_type + options.append(option) + return options + + +SqliteApiType = Literal["text", "integer", "float", "blob"] +DefaultExpr = Literal["current_timestamp", "current_date", "current_time"] +DEFAULT_EXPR_SQL = { + "current_timestamp": "CURRENT_TIMESTAMP", + "current_date": "CURRENT_DATE", + "current_time": "CURRENT_TIME", +} + + +class _StrictPydanticModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class _DefaultArgsMixin(_StrictPydanticModel): + default: Any | None = None + default_expr: DefaultExpr | None = None + + @model_validator(mode="after") + def validate_default_fields(self): + has_default = "default" in self.model_fields_set + has_default_expr = "default_expr" in self.model_fields_set + if has_default and has_default_expr: + raise ValueError("default and default_expr cannot both be provided") + if has_default_expr and self.default_expr is None: + raise ValueError("default_expr cannot be null") + return self + + +class AddColumnArgs(_DefaultArgsMixin): + name: str + type: SqliteApiType = "text" + not_null: bool = False + + +class RenameColumnArgs(_StrictPydanticModel): + name: str + to: str + + +class AlterColumnArgs(_DefaultArgsMixin): + name: str + type: SqliteApiType | None = None + not_null: bool | None = None + + @model_validator(mode="after") + def require_change(self): + if not ( + {"type", "not_null", "default", "default_expr"} & self.model_fields_set + ): + raise ValueError( + "At least one of type, not_null, default or default_expr must be provided" + ) + return self + + +class DropColumnArgs(_StrictPydanticModel): + name: str + + +class SetPrimaryKeyArgs(_StrictPydanticModel): + columns: list[str] = Field(min_length=1) + + +class ReorderColumnsArgs(_StrictPydanticModel): + columns: list[str] = Field(min_length=1) + + +class AddColumnOperation(_StrictPydanticModel): + op: Literal["add_column"] + args: AddColumnArgs + + +class RenameColumnOperation(_StrictPydanticModel): + op: Literal["rename_column"] + args: RenameColumnArgs + + +class AlterColumnOperation(_StrictPydanticModel): + op: Literal["alter_column"] + args: AlterColumnArgs + + +class DropColumnOperation(_StrictPydanticModel): + op: Literal["drop_column"] + args: DropColumnArgs + + +class SetPrimaryKeyOperation(_StrictPydanticModel): + op: Literal["set_primary_key"] + args: SetPrimaryKeyArgs + + +class ReorderColumnsOperation(_StrictPydanticModel): + op: Literal["reorder_columns"] + args: ReorderColumnsArgs + + +AlterTableOperation = Annotated[ + Union[ + AddColumnOperation, + RenameColumnOperation, + AlterColumnOperation, + DropColumnOperation, + SetPrimaryKeyOperation, + ReorderColumnsOperation, + ], + Field(discriminator="op"), +] + + +class AlterTableRequest(_StrictPydanticModel): + operations: list[AlterTableOperation] = Field(min_length=1) + dry_run: bool = False + + +def _pydantic_errors(validation_error): + errors = [] + for error in validation_error.errors(): + location = ".".join(str(item) for item in error["loc"]) + message = error["msg"] + errors.append("{}: {}".format(location, message) if location else message) + return errors + + +def _table_schema_from_conn(conn, table_name): + row = conn.execute( + "select sql from sqlite_master where type = 'table' and name = ?", + [table_name], + ).fetchone() + return row[0] if row else None + + +def _primary_key_value(columns): + if len(columns) == 1: + return columns[0] + return tuple(columns) + + +def _default_expression_sql(default_expr): + return DEFAULT_EXPR_SQL[default_expr] + + +def _literal_default(db, value): + if isinstance(value, str): + return db.quote(value) + return value + + +class TableCreateView(BaseView): + name = "table-create" + + _valid_keys = { + "table", + "rows", + "row", + "columns", + "pk", + "pks", + "ignore", + "replace", + "alter", + } + _supported_column_types = set(CREATE_TABLE_COLUMN_TYPES) + # Any string that does not contain a newline or start with sqlite_ + _table_name_re = re.compile(r"^(?!sqlite_)[^\n]+$") + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + db = await self.ds.resolve_database(request) + database_name = db.name + + # Must have create-table permission + if not await self.ds.allowed( + action="create-table", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ): + return _error(["Permission denied"], 403) + + try: + data = await request.json() + except json.JSONDecodeError as e: + return _error(["Invalid JSON: {}".format(e)]) + + if not isinstance(data, dict): + return _error(["JSON must be an object"]) + + invalid_keys = set(data.keys()) - self._valid_keys + if invalid_keys: + return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) + + # ignore and replace are mutually exclusive + if data.get("ignore") and data.get("replace"): + return _error(["ignore and replace are mutually exclusive"]) + + # ignore and replace only allowed with row or rows + if "ignore" in data or "replace" in data: + if not data.get("row") and not data.get("rows"): + return _error(["ignore and replace require row or rows"]) + + # ignore and replace require pk or pks + if "ignore" in data or "replace" in data: + if not data.get("pk") and not data.get("pks"): + return _error(["ignore and replace require pk or pks"]) + + ignore = data.get("ignore") + replace = data.get("replace") + + if replace: + # Must have update-row permission + if not await self.ds.allowed( + action="update-row", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ): + return _error(["Permission denied: need update-row"], 403) + + table_name = data.get("table") + if not table_name: + return _error(["Table is required"]) + + if not self._table_name_re.match(table_name): + return _error(["Invalid table name"]) + + table_exists = await db.table_exists(data["table"]) + columns = data.get("columns") + rows = data.get("rows") + row = data.get("row") + if not columns and not rows and not row: + return _error(["columns, rows or row is required"]) + + if rows and row: + return _error(["Cannot specify both rows and row"]) + + if rows or row: + # Must have insert-row permission + if not await self.ds.allowed( + action="insert-row", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ): + return _error(["Permission denied: need insert-row"], 403) + + alter = False + if rows or row: + if not table_exists: + # if table is being created for the first time, alter=True + alter = True + else: + # alter=True only if they request it AND they have permission + if data.get("alter"): + if not await self.ds.allowed( + action="alter-table", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ): + return _error(["Permission denied: need alter-table"], 403) + alter = True + + if columns: + if rows or row: + return _error(["Cannot specify columns with rows or row"]) + if not isinstance(columns, list): + return _error(["columns must be a list"]) + for column in columns: + if not isinstance(column, dict): + return _error(["columns must be a list of objects"]) + if not column.get("name") or not isinstance(column.get("name"), str): + return _error(["Column name is required"]) + if not column.get("type"): + column["type"] = "text" + if column["type"] not in self._supported_column_types: + return _error( + ["Unsupported column type: {}".format(column["type"])] + ) + # No duplicate column names + dupes = {c["name"] for c in columns if columns.count(c) > 1} + if dupes: + return _error(["Duplicate column name: {}".format(", ".join(dupes))]) + + if row: + rows = [row] + + if rows: + if not isinstance(rows, list): + return _error(["rows must be a list"]) + for row in rows: + if not isinstance(row, dict): + return _error(["rows must be a list of objects"]) + + pk = data.get("pk") + pks = data.get("pks") + + if pk and pks: + return _error(["Cannot specify both pk and pks"]) + if pk: + if not isinstance(pk, str): + return _error(["pk must be a string"]) + if pks: + if not isinstance(pks, list): + return _error(["pks must be a list"]) + for pk in pks: + if not isinstance(pk, str): + return _error(["pks must be a list of strings"]) + + # If table exists already, read pks from that instead + if table_exists: + actual_pks = await db.primary_keys(table_name) + # if pk passed and table already exists check it does not change + bad_pks = False + if len(actual_pks) == 1 and data.get("pk") and data["pk"] != actual_pks[0]: + bad_pks = True + elif ( + len(actual_pks) > 1 + and data.get("pks") + and set(data["pks"]) != set(actual_pks) + ): + bad_pks = True + if bad_pks: + return _error(["pk cannot be changed for existing table"]) + pks = actual_pks + + initial_schema = None + if table_exists: + initial_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + + def create_table(conn): + table = sqlite_utils.Database(conn)[table_name] + if rows: + table.insert_all( + rows, pk=pks or pk, ignore=ignore, replace=replace, alter=alter + ) + else: + table.create( + {c["name"]: c["type"] for c in columns}, + pk=pks or pk, + ) + return table.schema + + try: + schema = await db.execute_write_fn(create_table, request=request) + except Exception as e: + return _error([str(e)]) + + if initial_schema is not None and initial_schema != schema: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=initial_schema, + after_schema=schema, + ) + ) + + table_url = self.ds.absolute_url( + request, self.ds.urls.table(db.name, table_name) + ) + table_api_url = self.ds.absolute_url( + request, self.ds.urls.table(db.name, table_name, format="json") + ) + details = { + "ok": True, + "database": db.name, + "table": table_name, + "table_url": table_url, + "table_api_url": table_api_url, + "schema": schema, + } + if rows: + details["row_count"] = len(rows) + + if not table_exists: + # Only log creation if we created a table + await self.ds.track_event( + CreateTableEvent( + request.actor, database=db.name, table=table_name, schema=schema + ) + ) + if rows: + await self.ds.track_event( + InsertRowsEvent( + request.actor, + database=db.name, + table=table_name, + num_rows=len(rows), + ignore=ignore, + replace=replace, + ) + ) + return Response.json(details, status=201) + + +class TableAlterView(BaseView): + name = "table-alter" + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + try: + resolved = await self.ds.resolve_table(request) + except NotFound as e: + return _error([e.args[0]], 404) + + db = resolved.db + database_name = db.name + table_name = resolved.table + + if not await self.ds.allowed( + action="alter-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return _error(["Permission denied: need alter-table"], 403) + + if not db.is_mutable: + return _error(["Database is immutable"], 403) + + content_type = request.headers.get("content-type") or "" + if not content_type.startswith("application/json"): + return _error(["Invalid content-type, must be application/json"], 400) + + try: + data = await request.json() + except json.JSONDecodeError as e: + return _error(["Invalid JSON: {}".format(e)], 400) + + if not isinstance(data, dict): + return _error(["JSON must be a dictionary"], 400) + + try: + alter_request = AlterTableRequest.model_validate(data) + except ValidationError as e: + return _error(_pydantic_errors(e), 400) + + def alter_table(conn): + before_schema = _table_schema_from_conn(conn, table_name) + + def apply_operations(operation_conn): + db_for_write = sqlite_utils.Database(operation_conn) + table = db_for_write[table_name] + + add_columns = [] + types = {} + rename = {} + drop = set() + not_null = {} + defaults = {} + column_order = None + pk = SQLITE_UTILS_DEFAULT + + for operation in alter_request.operations: + args = operation.args + if operation.op == "add_column": + if args.not_null and not ( + ( + "default" in args.model_fields_set + and args.default is not None + ) + or "default_expr" in args.model_fields_set + ): + raise ValueError( + "add_column args.default or args.default_expr is required when not_null is true" + ) + add_columns.append(args) + if "default" in args.model_fields_set and not args.not_null: + defaults[args.name] = _literal_default( + db_for_write, args.default + ) + if ( + "default_expr" in args.model_fields_set + and not args.not_null + ): + defaults[args.name] = _default_expression_sql( + args.default_expr + ) + elif operation.op == "rename_column": + rename[args.name] = args.to + elif operation.op == "alter_column": + if args.type is not None: + types[args.name] = args.type + if args.not_null is not None: + not_null[args.name] = args.not_null + if "default" in args.model_fields_set: + defaults[args.name] = ( + None + if args.default is None + else _literal_default(db_for_write, args.default) + ) + if "default_expr" in args.model_fields_set: + defaults[args.name] = _default_expression_sql( + args.default_expr + ) + elif operation.op == "drop_column": + drop.add(args.name) + elif operation.op == "set_primary_key": + pk = _primary_key_value(args.columns) + elif operation.op == "reorder_columns": + column_order = args.columns + + with operation_conn: + for column in add_columns: + not_null_default = None + if column.not_null: + if "default_expr" in column.model_fields_set: + not_null_default = _default_expression_sql( + column.default_expr + ) + else: + not_null_default = _literal_default( + db_for_write, column.default + ) + table.add_column( + column.name, + column.type, + not_null_default=not_null_default, + ) + + should_transform = any( + ( + types, + rename, + drop, + not_null, + defaults, + column_order is not None, + pk is not SQLITE_UTILS_DEFAULT, + ) + ) + if should_transform: + table.transform( + types=types or None, + rename=rename or None, + drop=drop or None, + pk=pk, + not_null=not_null or None, + defaults=defaults or None, + column_order=column_order, + ) + + return _table_schema_from_conn(operation_conn, table_name) + + if alter_request.dry_run: + memory_conn = sqlite3.connect(":memory:") + try: + conn.backup(memory_conn) + return before_schema, apply_operations(memory_conn) + finally: + memory_conn.close() + + after_schema = apply_operations(conn) + return before_schema, after_schema + + try: + before_schema, after_schema = await db.execute_write_fn( + alter_table, request=request + ) + except Exception as e: + return _error([str(e)], 400) + + altered = before_schema != after_schema + if altered and not alter_request.dry_run: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=before_schema, + after_schema=after_schema, + ) + ) + + table_url = self.ds.absolute_url( + request, self.ds.urls.table(database_name, table_name) + ) + table_api_url = self.ds.absolute_url( + request, self.ds.urls.table(database_name, table_name, format="json") + ) + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "table_url": table_url, + "table_api_url": table_api_url, + "altered": altered, + "schema": after_schema, + "before_schema": before_schema, + "operations_applied": ( + 0 if alter_request.dry_run else len(alter_request.operations) + ), + "dry_run": alter_request.dry_run, + }, + status=200, + ) From 9766a9c0876351a5c60430c4582a7686cb24ad79 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 12:38:51 -0700 Subject: [PATCH 017/167] Add foreign keys to create table API - Add fk_table and optional fk_column support to create-table columns. - Validate create-table requests with Pydantic while preserving existing errors. - Document the API and cover inferred primary-key and validation cases. Refs https://github.com/simonw/datasette/pull/2789#issuecomment-4733544452 --- datasette/views/table_create_alter.py | 287 ++++++++++++++++---------- docs/json_api.rst | 25 +++ tests/test_api_write.py | 115 +++++++++++ 3 files changed, 321 insertions(+), 106 deletions(-) diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 7decfad2..e8264a6f 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -2,7 +2,15 @@ import json import re from typing import Annotated, Any, Literal, Union -from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationError, + field_validator, + model_validator, +) +from pydantic_core import PydanticCustomError import sqlite_utils from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT @@ -25,6 +33,7 @@ CREATE_TABLE_TYPE_FOR_SQLITE_TYPE = { sqlite_type: column_type for column_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() } +TABLE_NAME_RE = re.compile(r"^(?!sqlite_)[^\n]+$") ALTER_TABLE_COLUMN_TYPES = CREATE_TABLE_COLUMN_TYPES ALTER_TABLE_TYPE_FOR_SQLITE_TYPE = { SQLiteType.TEXT: "text", @@ -98,6 +107,137 @@ class _StrictPydanticModel(BaseModel): model_config = ConfigDict(extra="forbid") +class CreateTableColumn(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: Any = None + type: Any = "text" + fk_table: str | None = None + fk_column: str | None = None + + @model_validator(mode="after") + def validate_column(self): + if not self.name or not isinstance(self.name, str): + raise PydanticCustomError("create_table", "Column name is required") + if not self.type: + self.type = "text" + elif self.type not in CREATE_TABLE_COLUMN_TYPES: + raise PydanticCustomError( + "create_table", "Unsupported column type: {type}", {"type": self.type} + ) + if self.fk_column and not self.fk_table: + raise PydanticCustomError( + "create_table_with_location", + "fk_column requires fk_table", + ) + return self + + +class CreateTableRequest(_StrictPydanticModel): + table: Any = None + rows: Any = None + row: Any = None + columns: list[CreateTableColumn] | None = None + pk: Any = None + pks: Any = None + ignore: bool | None = None + replace: bool | None = None + alter: bool | None = None + + @field_validator("columns", mode="before") + @classmethod + def validate_columns_list(cls, value): + if value is None: + return value + if not isinstance(value, list): + raise PydanticCustomError("create_table", "columns must be a list") + if not all(isinstance(column, dict) for column in value): + raise PydanticCustomError( + "create_table", "columns must be a list of objects" + ) + return value + + @model_validator(mode="after") + def validate_request(self): + if not self.table: + raise PydanticCustomError("create_table", "Table is required") + if not isinstance(self.table, str) or not TABLE_NAME_RE.match(self.table): + raise PydanticCustomError("create_table", "Invalid table name") + if not self.columns and not self.rows and not self.row: + raise PydanticCustomError( + "create_table", "columns, rows or row is required" + ) + if self.rows and self.row: + raise PydanticCustomError( + "create_table", "Cannot specify both rows and row" + ) + if self.columns and (self.rows or self.row): + raise PydanticCustomError( + "create_table", "Cannot specify columns with rows or row" + ) + if self.columns is not None: + seen = set() + duplicates = [] + for column in self.columns: + if column.name in seen and column.name not in duplicates: + duplicates.append(column.name) + seen.add(column.name) + if duplicates: + raise PydanticCustomError( + "create_table", + "Duplicate column name: {names}", + {"names": ", ".join(duplicates)}, + ) + if self.rows is not None: + if not isinstance(self.rows, list): + raise PydanticCustomError("create_table", "rows must be a list") + if not all(isinstance(row, dict) for row in self.rows): + raise PydanticCustomError( + "create_table", "rows must be a list of objects" + ) + if self.pk is not None and not isinstance(self.pk, str): + raise PydanticCustomError("create_table", "pk must be a string") + if self.pk and self.pks: + raise PydanticCustomError("create_table", "Cannot specify both pk and pks") + if self.pks is not None: + if not isinstance(self.pks, list): + raise PydanticCustomError("create_table", "pks must be a list") + if not all(isinstance(pk, str) for pk in self.pks): + raise PydanticCustomError( + "create_table", "pks must be a list of strings" + ) + if self.ignore and self.replace: + raise PydanticCustomError( + "create_table", "ignore and replace are mutually exclusive" + ) + if {"ignore", "replace"} & self.model_fields_set: + if not self.row and not self.rows: + raise PydanticCustomError( + "create_table", "ignore and replace require row or rows" + ) + if not self.pk and not self.pks: + raise PydanticCustomError( + "create_table", "ignore and replace require pk or pks" + ) + return self + + @property + def rows_list(self): + return [self.row] if self.row else self.rows + + @property + def foreign_keys(self): + if not self.columns: + return None + foreign_keys = [] + for column in self.columns: + if column.fk_table and column.fk_column: + foreign_keys.append((column.name, column.fk_table, column.fk_column)) + elif column.fk_table: + foreign_keys.append((column.name, column.fk_table)) + return foreign_keys or None + + class _DefaultArgsMixin(_StrictPydanticModel): default: Any | None = None default_expr: DefaultExpr | None = None @@ -209,6 +349,27 @@ def _pydantic_errors(validation_error): return errors +def _create_table_pydantic_errors(validation_error): + errors = validation_error.errors() + invalid_keys = sorted( + str(error["loc"][0]) + for error in errors + if error["type"] == "extra_forbidden" and len(error["loc"]) == 1 + ) + if invalid_keys: + return ["Invalid keys: {}".format(", ".join(invalid_keys))] + + output = [] + for error in errors: + message = error["msg"] + if error["type"] == "create_table": + output.append(message) + continue + location = ".".join(str(item) for item in error["loc"]) + output.append("{}: {}".format(location, message) if location else message) + return output + + def _table_schema_from_conn(conn, table_name): row = conn.execute( "select sql from sqlite_master where type = 'table' and name = ?", @@ -236,21 +397,6 @@ def _literal_default(db, value): class TableCreateView(BaseView): name = "table-create" - _valid_keys = { - "table", - "rows", - "row", - "columns", - "pk", - "pks", - "ignore", - "replace", - "alter", - } - _supported_column_types = set(CREATE_TABLE_COLUMN_TYPES) - # Any string that does not contain a newline or start with sqlite_ - _table_name_re = re.compile(r"^(?!sqlite_)[^\n]+$") - def __init__(self, datasette): self.ds = datasette @@ -274,26 +420,13 @@ class TableCreateView(BaseView): if not isinstance(data, dict): return _error(["JSON must be an object"]) - invalid_keys = set(data.keys()) - self._valid_keys - if invalid_keys: - return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) + try: + create_request = CreateTableRequest.model_validate(data) + except ValidationError as e: + return _error(_create_table_pydantic_errors(e)) - # ignore and replace are mutually exclusive - if data.get("ignore") and data.get("replace"): - return _error(["ignore and replace are mutually exclusive"]) - - # ignore and replace only allowed with row or rows - if "ignore" in data or "replace" in data: - if not data.get("row") and not data.get("rows"): - return _error(["ignore and replace require row or rows"]) - - # ignore and replace require pk or pks - if "ignore" in data or "replace" in data: - if not data.get("pk") and not data.get("pks"): - return _error(["ignore and replace require pk or pks"]) - - ignore = data.get("ignore") - replace = data.get("replace") + ignore = create_request.ignore + replace = create_request.replace if replace: # Must have update-row permission @@ -304,24 +437,12 @@ class TableCreateView(BaseView): ): return _error(["Permission denied: need update-row"], 403) - table_name = data.get("table") - if not table_name: - return _error(["Table is required"]) + table_name = create_request.table + table_exists = await db.table_exists(table_name) + columns = create_request.columns + rows = create_request.rows_list - if not self._table_name_re.match(table_name): - return _error(["Invalid table name"]) - - table_exists = await db.table_exists(data["table"]) - columns = data.get("columns") - rows = data.get("rows") - row = data.get("row") - if not columns and not rows and not row: - return _error(["columns, rows or row is required"]) - - if rows and row: - return _error(["Cannot specify both rows and row"]) - - if rows or row: + if rows: # Must have insert-row permission if not await self.ds.allowed( action="insert-row", @@ -331,13 +452,13 @@ class TableCreateView(BaseView): return _error(["Permission denied: need insert-row"], 403) alter = False - if rows or row: + if rows: if not table_exists: # if table is being created for the first time, alter=True alter = True else: # alter=True only if they request it AND they have permission - if data.get("alter"): + if create_request.alter: if not await self.ds.allowed( action="alter-table", resource=DatabaseResource(database=database_name), @@ -346,64 +467,17 @@ class TableCreateView(BaseView): return _error(["Permission denied: need alter-table"], 403) alter = True - if columns: - if rows or row: - return _error(["Cannot specify columns with rows or row"]) - if not isinstance(columns, list): - return _error(["columns must be a list"]) - for column in columns: - if not isinstance(column, dict): - return _error(["columns must be a list of objects"]) - if not column.get("name") or not isinstance(column.get("name"), str): - return _error(["Column name is required"]) - if not column.get("type"): - column["type"] = "text" - if column["type"] not in self._supported_column_types: - return _error( - ["Unsupported column type: {}".format(column["type"])] - ) - # No duplicate column names - dupes = {c["name"] for c in columns if columns.count(c) > 1} - if dupes: - return _error(["Duplicate column name: {}".format(", ".join(dupes))]) - - if row: - rows = [row] - - if rows: - if not isinstance(rows, list): - return _error(["rows must be a list"]) - for row in rows: - if not isinstance(row, dict): - return _error(["rows must be a list of objects"]) - - pk = data.get("pk") - pks = data.get("pks") - - if pk and pks: - return _error(["Cannot specify both pk and pks"]) - if pk: - if not isinstance(pk, str): - return _error(["pk must be a string"]) - if pks: - if not isinstance(pks, list): - return _error(["pks must be a list"]) - for pk in pks: - if not isinstance(pk, str): - return _error(["pks must be a list of strings"]) + pk = create_request.pk + pks = create_request.pks # If table exists already, read pks from that instead if table_exists: actual_pks = await db.primary_keys(table_name) # if pk passed and table already exists check it does not change bad_pks = False - if len(actual_pks) == 1 and data.get("pk") and data["pk"] != actual_pks[0]: + if len(actual_pks) == 1 and pk and pk != actual_pks[0]: bad_pks = True - elif ( - len(actual_pks) > 1 - and data.get("pks") - and set(data["pks"]) != set(actual_pks) - ): + elif len(actual_pks) > 1 and pks and set(pks) != set(actual_pks): bad_pks = True if bad_pks: return _error(["pk cannot be changed for existing table"]) @@ -423,8 +497,9 @@ class TableCreateView(BaseView): ) else: table.create( - {c["name"]: c["type"] for c in columns}, + {column.name: column.type for column in columns}, pk=pks or pk, + foreign_keys=create_request.foreign_keys, ) return table.schema diff --git a/docs/json_api.rst b/docs/json_api.rst index 4074b479..1b4a196e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -1981,6 +1981,7 @@ The JSON here describes the table that will be created: - ``name`` is the name of the column. This is required. - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. + - ``fk_table`` can be used to create a single-column foreign key constraint referencing another table. ``fk_column`` is optional and can be used to specify the referenced column - if omitted, Datasette will use the single primary key of ``fk_table``. * ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. @@ -1993,6 +1994,30 @@ The JSON here describes the table that will be created: * ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. This requires the :ref:`actions_update_row` permission. * ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`actions_alter_table` permission. +This example creates a foreign key from ``projects.owner_id`` to the single primary key of ``owners``: + +.. code-block:: json + + { + "table": "projects", + "columns": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "owner_id", + "type": "integer", + "fk_table": "owners" + }, + { + "name": "title", + "type": "text" + } + ], + "pk": "id" + } + If the table is successfully created this will return a ``201`` status code and the following response: .. code-block:: json diff --git a/tests/test_api_write.py b/tests/test_api_write.py index f117c06e..627b1ac1 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1614,6 +1614,121 @@ async def test_create_table( assert [e.name for e in events] == expected_events +@pytest.mark.asyncio +async def test_create_table_with_foreign_key(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "owners", + "columns": [ + {"name": "id", "type": "integer"}, + {"name": "name", "type": "text"}, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "projects", + "columns": [ + {"name": "id", "type": "integer"}, + { + "name": "owner_id", + "type": "integer", + "fk_table": "owners", + }, + {"name": "title", "type": "text"}, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + data = response.json() + assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "column,expected_error", + ( + ( + {"name": "owner_id", "type": "integer", "fk_table": "owners"}, + None, + ), + ( + {"name": "owner_id", "type": "integer", "fk_column": "id"}, + "columns.0: fk_column requires fk_table", + ), + ), +) +async def test_create_table_foreign_key_validation(ds_write, column, expected_error): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "projects", + "columns": [column], + }, + headers=_headers(token), + ) + if expected_error: + assert response.status_code == 400 + assert response.json() == {"ok": False, "errors": [expected_error]} + else: + assert response.status_code == 400 + assert response.json() == { + "ok": False, + "errors": ["Could not detect single primary key for table 'owners'"], + } + + +@pytest.mark.asyncio +async def test_create_table_foreign_key_without_fk_column_requires_single_pk(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "accounts", + "columns": [ + {"name": "tenant_id", "type": "integer"}, + {"name": "id", "type": "integer"}, + {"name": "name", "type": "text"}, + ], + "pks": ["tenant_id", "id"], + }, + headers=_headers(token), + ) + assert response.status_code == 201 + + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "projects", + "columns": [ + {"name": "id", "type": "integer"}, + { + "name": "account_id", + "type": "integer", + "fk_table": "accounts", + }, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 400 + assert response.json() == { + "ok": False, + "errors": ["Could not detect single primary key for table 'accounts'"], + } + + @pytest.mark.asyncio @pytest.mark.parametrize( "permissions,body,expected_status,expected_errors", From 9d9a2d3ff379b1f1ee3db4d24b53310ef2a71023 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 12:51:19 -0700 Subject: [PATCH 018/167] Add foreign keys to alter table API - Add add_foreign_key, drop_foreign_key, and set_foreign_keys operations. - Validate flat fk_table and fk_column arguments with Pydantic. - Document the API and cover inferred primary-key and validation cases. --- datasette/views/table_create_alter.py | 67 +++++++++++++++ docs/json_api.rst | 30 +++++++ tests/test_api_write.py | 118 ++++++++++++++++++++++++++ 3 files changed, 215 insertions(+) diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index e8264a6f..20dcc03e 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -292,6 +292,40 @@ class ReorderColumnsArgs(_StrictPydanticModel): columns: list[str] = Field(min_length=1) +class ForeignKeyArgs(_StrictPydanticModel): + column: str + fk_table: str | None = None + fk_column: str | None = None + + @model_validator(mode="after") + def validate_foreign_key(self): + if self.fk_column and not self.fk_table: + raise PydanticCustomError( + "alter_table_foreign_key", + "fk_column requires fk_table", + ) + if not self.fk_table: + raise PydanticCustomError( + "alter_table_foreign_key", + "fk_table is required", + ) + return self + + @property + def tuple(self): + if self.fk_column: + return (self.column, self.fk_table, self.fk_column) + return (self.column, self.fk_table) + + +class DropForeignKeyArgs(_StrictPydanticModel): + column: str + + +class SetForeignKeysArgs(_StrictPydanticModel): + foreign_keys: list[ForeignKeyArgs] + + class AddColumnOperation(_StrictPydanticModel): op: Literal["add_column"] args: AddColumnArgs @@ -322,6 +356,21 @@ class ReorderColumnsOperation(_StrictPydanticModel): args: ReorderColumnsArgs +class AddForeignKeyOperation(_StrictPydanticModel): + op: Literal["add_foreign_key"] + args: ForeignKeyArgs + + +class DropForeignKeyOperation(_StrictPydanticModel): + op: Literal["drop_foreign_key"] + args: DropForeignKeyArgs + + +class SetForeignKeysOperation(_StrictPydanticModel): + op: Literal["set_foreign_keys"] + args: SetForeignKeysArgs + + AlterTableOperation = Annotated[ Union[ AddColumnOperation, @@ -330,6 +379,9 @@ AlterTableOperation = Annotated[ DropColumnOperation, SetPrimaryKeyOperation, ReorderColumnsOperation, + AddForeignKeyOperation, + DropForeignKeyOperation, + SetForeignKeysOperation, ], Field(discriminator="op"), ] @@ -615,6 +667,9 @@ class TableAlterView(BaseView): defaults = {} column_order = None pk = SQLITE_UTILS_DEFAULT + add_foreign_keys = [] + drop_foreign_keys = [] + foreign_keys = None for operation in alter_request.operations: args = operation.args @@ -664,6 +719,12 @@ class TableAlterView(BaseView): pk = _primary_key_value(args.columns) elif operation.op == "reorder_columns": column_order = args.columns + elif operation.op == "add_foreign_key": + add_foreign_keys.append(args.tuple) + elif operation.op == "drop_foreign_key": + drop_foreign_keys.append(args.column) + elif operation.op == "set_foreign_keys": + foreign_keys = [fk.tuple for fk in args.foreign_keys] with operation_conn: for column in add_columns: @@ -692,6 +753,9 @@ class TableAlterView(BaseView): defaults, column_order is not None, pk is not SQLITE_UTILS_DEFAULT, + add_foreign_keys, + drop_foreign_keys, + foreign_keys is not None, ) ) if should_transform: @@ -703,6 +767,9 @@ class TableAlterView(BaseView): not_null=not_null or None, defaults=defaults or None, column_order=column_order, + add_foreign_keys=add_foreign_keys or None, + drop_foreign_keys=drop_foreign_keys or None, + foreign_keys=foreign_keys, ) return _table_schema_from_conn(operation_conn, table_name) diff --git a/docs/json_api.rst b/docs/json_api.rst index 1b4a196e..af16626f 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2159,6 +2159,31 @@ The request body should include an ``operations`` array. Each operation has the "columns": ["id"] } }, + { + "op": "add_foreign_key", + "args": { + "column": "owner_id", + "fk_table": "owners" + } + }, + { + "op": "drop_foreign_key", + "args": { + "column": "old_owner_id" + } + }, + { + "op": "set_foreign_keys", + "args": { + "foreign_keys": [ + { + "column": "owner_id", + "fk_table": "owners", + "fk_column": "id" + } + ] + } + }, { "op": "reorder_columns", "args": { @@ -2177,10 +2202,15 @@ Supported operations: * ``alter_column`` changes column properties. ``args`` accepts ``name`` and at least one of ``type``, ``not_null``, literal ``default`` or ``default_expr``. Passing ``"default": null`` removes an existing default. * ``drop_column`` drops a column. ``args`` accepts ``name``. * ``set_primary_key`` changes the table primary key. ``args`` accepts ``columns``, a list of one or more column names. +* ``add_foreign_key`` adds a single-column foreign key constraint. ``args`` accepts ``column``, ``fk_table`` and optional ``fk_column``. If ``fk_column`` is omitted, Datasette will use the single primary key of ``fk_table``. +* ``drop_foreign_key`` removes the foreign key constraint for a column. ``args`` accepts ``column``. +* ``set_foreign_keys`` replaces all foreign key constraints on the table. ``args`` accepts ``foreign_keys``, a list of objects that each have ``column``, ``fk_table`` and optional ``fk_column``. An empty list removes all foreign key constraints. * ``reorder_columns`` reorders columns. ``args`` accepts ``columns``, a list of one or more column names. Columns omitted from this list will appear afterwards in their existing order. ``default`` is always treated as a literal value. ``default_expr`` accepts one of ``current_timestamp``, ``current_date`` or ``current_time`` and is rendered as the corresponding SQLite default expression. +For foreign key operations that omit ``fk_column``, the referenced ``fk_table`` must have a single-column primary key. Datasette will return an error if it cannot identify a single primary key column for that table. + A successful response returns the new schema and the previous schema: .. code-block:: json diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 627b1ac1..046cf695 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -926,6 +926,124 @@ async def test_alter_table_dry_run(ds_write): assert last_event(ds_write) is None +@pytest.mark.asyncio +async def test_alter_table_foreign_key_operations(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + await db.execute_write("create table owners (id integer primary key)") + await db.execute_write("create table categories (id integer primary key)") + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + {"op": "add_column", "args": {"name": "owner_id", "type": "integer"}}, + { + "op": "add_foreign_key", + "args": {"column": "owner_id", "fk_table": "owners"}, + }, + ] + }, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["operations_applied"] == 2 + assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"] + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [{"op": "drop_foreign_key", "args": {"column": "owner_id"}}] + }, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert "[owner_id] INTEGER REFERENCES" not in data["schema"] + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + { + "op": "set_foreign_keys", + "args": { + "foreign_keys": [ + { + "column": "owner_id", + "fk_table": "categories", + "fk_column": "id", + } + ] + }, + } + ] + }, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert "[owner_id] INTEGER REFERENCES [categories]([id])" in data["schema"] + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={"operations": [{"op": "set_foreign_keys", "args": {"foreign_keys": []}}]}, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert "[owner_id] INTEGER REFERENCES" not in data["schema"] + + +@pytest.mark.asyncio +async def test_alter_table_foreign_key_requires_fk_table_for_fk_column(ds_write): + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + { + "op": "add_foreign_key", + "args": {"column": "age", "fk_column": "id"}, + } + ] + }, + headers=_headers(write_token(ds_write, permissions=["at"])), + ) + assert response.status_code == 400 + assert response.json() == { + "ok": False, + "errors": ["operations.0.add_foreign_key.args: fk_column requires fk_table"], + } + + +@pytest.mark.asyncio +async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + await db.execute_write( + "create table accounts (tenant_id integer, id integer, primary key (tenant_id, id))" + ) + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + { + "op": "add_foreign_key", + "args": {"column": "age", "fk_table": "accounts"}, + } + ] + }, + headers=_headers(token), + ) + assert response.status_code == 400 + assert response.json() == { + "ok": False, + "errors": ["Could not detect single primary key for table 'accounts'"], + } + + @pytest.mark.asyncio async def test_alter_table_permission_denied(ds_write): token = write_token(ds_write, permissions=["ir"]) From 2900efb32d5348fdd4b3ecb33e5f8b1ee836f5c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 14:47:25 -0700 Subject: [PATCH 019/167] /db/table/-/foreign-key-suggestions API Improved version of the implementation datasette-edit-schema --- datasette/app.py | 10 +- datasette/views/table_create_alter.py | 303 +++++++++++++++++++++++++- docs/json_api.rst | 58 +++++ tests/test_api_write.py | 100 +++++++++ 4 files changed, 469 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 13e25d4e..6b9f47ba 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -49,7 +49,11 @@ from .views.database import ( DatabaseView, QueryView, ) -from .views.table_create_alter import TableAlterView, TableCreateView +from .views.table_create_alter import ( + TableAlterView, + TableCreateView, + TableForeignKeySuggestionsView, +) from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView from .views.stored_queries import ( QueryCreateAnalyzeView, @@ -2630,6 +2634,10 @@ class Datasette: TableAlterView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/alter$", ) + add_route( + TableForeignKeySuggestionsView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/foreign-key-suggestions$", + ) add_route( TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/set-column-type$", diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 20dcc03e..2cb59ac1 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -1,7 +1,9 @@ import json import re +import time from typing import Annotated, Any, Literal, Union +from datasette.database import QueryInterrupted from pydantic import ( BaseModel, ConfigDict, @@ -17,8 +19,14 @@ from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT from datasette.column_types import SQLiteType from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.resources import DatabaseResource, TableResource -from datasette.utils import sqlite3 +from datasette.utils import ( + escape_sqlite, + get_outbound_foreign_keys, + sqlite3, + table_column_details, +) from datasette.utils.asgi import NotFound, Response +from datasette.utils.sqlite import sqlite_hidden_table_names from .base import BaseView, _error @@ -41,6 +49,177 @@ ALTER_TABLE_TYPE_FOR_SQLITE_TYPE = { SQLiteType.REAL: "float", SQLiteType.BLOB: "blob", } +FOREIGN_KEY_SUGGESTION_ROW_LIMIT = 500 +FOREIGN_KEY_SUGGESTION_TIME_LIMIT_MS = 50 +FOREIGN_KEY_SUGGESTION_TOTAL_TIME_LIMIT_MS = 200 + + +class ForeignKeySuggestionTimedOut(Exception): + pass + + +def _sqlite_type_affinity(type_name): + type_name = (type_name or "").upper() + if "INT" in type_name: + return "integer" + if any(token in type_name for token in ("CHAR", "CLOB", "TEXT")): + return "text" + if "BLOB" in type_name or not type_name: + return "blob" + if any(token in type_name for token in ("REAL", "FLOA", "DOUB")): + return "real" + return "numeric" + + +def _foreign_key_type_compatible(source_affinity, target_affinity): + if source_affinity == target_affinity: + return True + numeric_affinities = {"integer", "real", "numeric"} + if source_affinity == "numeric": + return target_affinity in numeric_affinities + if target_affinity == "numeric": + return source_affinity in numeric_affinities + return False + + +def _public_foreign_key_target(target): + return { + "fk_table": target["fk_table"], + "fk_column": target["fk_column"], + "type": target["type"], + } + + +def _singular(name): + if name.endswith("ies") and len(name) > 3: + return name[:-3] + "y" + if name.endswith("s") and len(name) > 1: + return name[:-1] + return name + + +def _foreign_key_name_reasons(source_column, target): + source = source_column.lower() + table = target["fk_table"].lower() + singular_table = _singular(table) + column = target["fk_column"].lower() + possible_names = { + "{}_{}".format(table, column), + "{}_{}".format(singular_table, column), + } + if column == "id": + possible_names.update( + { + "{}_id".format(table), + "{}_id".format(singular_table), + } + ) + return ["name_match"] if source in possible_names else [] + + +def _foreign_key_option_sort_key(source_column, target): + has_name_match = bool(_foreign_key_name_reasons(source_column, target)) + return ( + 0 if has_name_match else 1, + target["fk_table"], + target["fk_column"], + ) + + +def _foreign_key_suggestion_metadata(conn, table_name): + hidden_tables = set(sqlite_hidden_table_names(conn)) + source_columns = [ + { + "column": column.name, + "type": (column.type or "").upper(), + "affinity": _sqlite_type_affinity(column.type), + } + for column in table_column_details(conn, table_name) + if not column.hidden + ] + current_by_column = { + fk["column"]: { + "fk_table": fk["other_table"], + "fk_column": fk["other_column"], + } + for fk in get_outbound_foreign_keys(conn, table_name) + } + table_names = [ + row[0] + for row in conn.execute( + "select name from sqlite_master where type = 'table' order by name" + ).fetchall() + if not row[0].startswith("sqlite_") + ] + targets = [] + for candidate_table in table_names: + if candidate_table == table_name or candidate_table in hidden_tables: + continue + columns = [column for column in table_column_details(conn, candidate_table)] + pks = [column for column in columns if column.is_pk and not column.hidden] + pks.sort(key=lambda column: column.is_pk) + if len(pks) != 1: + continue + pk = pks[0] + targets.append( + { + "fk_table": candidate_table, + "fk_column": pk.name, + "type": (pk.type or "").upper(), + "affinity": _sqlite_type_affinity(pk.type), + } + ) + return source_columns, targets, current_by_column + + +async def _foreign_key_suggestion_samples(db, table_name, columns): + if not columns: + return 0, {} + sql = "select {} from {} limit {}".format( + ", ".join(escape_sqlite(column) for column in columns), + escape_sqlite(table_name), + FOREIGN_KEY_SUGGESTION_ROW_LIMIT, + ) + try: + results = await db.execute( + sql, + custom_time_limit=FOREIGN_KEY_SUGGESTION_TIME_LIMIT_MS, + log_sql_errors=False, + ) + except QueryInterrupted as e: + raise ForeignKeySuggestionTimedOut from e + values_by_column = {column: [] for column in columns} + seen_by_column = {column: set() for column in columns} + for row in results.rows: + for column in columns: + value = row[column] + if value is None or value in seen_by_column[column]: + continue + seen_by_column[column].add(value) + values_by_column[column].append(value) + return len(results.rows), values_by_column + + +async def _foreign_key_suggestion_values_exist(db, target, values, time_limit_ms): + if not values: + return False + sql = "select {} from {} where {} in ({})".format( + escape_sqlite(target["fk_column"]), + escape_sqlite(target["fk_table"]), + escape_sqlite(target["fk_column"]), + ", ".join("?" for _ in values), + ) + try: + results = await db.execute( + sql, + params=values, + custom_time_limit=time_limit_ms, + log_sql_errors=False, + ) + except QueryInterrupted as e: + raise ForeignKeySuggestionTimedOut from e + found = {row[0] for row in results.rows} + return all(value in found for value in values) async def _create_table_ui_context( @@ -609,6 +788,128 @@ class TableCreateView(BaseView): return Response.json(details, status=201) +class TableForeignKeySuggestionsView(BaseView): + name = "table-foreign-key-suggestions" + + def __init__(self, datasette): + self.ds = datasette + + async def get(self, request): + try: + resolved = await self.ds.resolve_table(request) + except NotFound as e: + return _error([e.args[0]], 404) + + db = resolved.db + database_name = db.name + table_name = resolved.table + + if resolved.is_view: + return _error(["Cannot suggest foreign keys for a view"], 400) + + if not await self.ds.allowed( + action="alter-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return _error(["Permission denied: need alter-table"], 403) + + source_columns, targets, current_by_column = await db.execute_fn( + lambda conn: _foreign_key_suggestion_metadata(conn, table_name) + ) + + columns = [] + options_by_column = {} + for source_column in source_columns: + options = sorted( + [ + target + for target in targets + if _foreign_key_type_compatible( + source_column["affinity"], target["affinity"] + ) + ], + key=lambda target: _foreign_key_option_sort_key( + source_column["column"], target + ), + ) + options_by_column[source_column["column"]] = options + columns.append( + { + "column": source_column["column"], + "type": source_column["type"], + "affinity": source_column["affinity"], + "current": current_by_column.get(source_column["column"]), + "suggestions": [], + "options": [ + _public_foreign_key_target(option) for option in options + ], + } + ) + + columns_to_sample = [ + column["column"] + for column in columns + if options_by_column[column["column"]] + ] + row_check = { + "attempted": bool(columns_to_sample), + "status": "completed" if columns_to_sample else "skipped", + "row_limit": FOREIGN_KEY_SUGGESTION_ROW_LIMIT, + "sampled_rows": 0, + "checked_options": 0, + } + + try: + sampled_rows, values_by_column = await _foreign_key_suggestion_samples( + db, table_name, columns_to_sample + ) + row_check["sampled_rows"] = sampled_rows + deadline = time.perf_counter() + ( + FOREIGN_KEY_SUGGESTION_TOTAL_TIME_LIMIT_MS / 1000 + ) + for column_info in columns: + values = values_by_column.get(column_info["column"]) or [] + if not values: + continue + for option in options_by_column[column_info["column"]]: + remaining_ms = int((deadline - time.perf_counter()) * 1000) + if remaining_ms <= 0: + raise ForeignKeySuggestionTimedOut + if await _foreign_key_suggestion_values_exist( + db, + option, + values, + min(FOREIGN_KEY_SUGGESTION_TIME_LIMIT_MS, remaining_ms), + ): + reasons = [ + "type_match", + "sample_values_exist", + ] + _foreign_key_name_reasons(column_info["column"], option) + column_info["suggestions"].append( + { + "fk_table": option["fk_table"], + "fk_column": option["fk_column"], + "confidence": "sampled", + "sampled_values": len(values), + "reasons": reasons, + } + ) + row_check["checked_options"] += 1 + except ForeignKeySuggestionTimedOut: + row_check["status"] = "timed_out" + + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "row_check": row_check, + "columns": columns, + } + ) + + class TableAlterView(BaseView): name = "table-alter" diff --git a/docs/json_api.rst b/docs/json_api.rst index af16626f..5b05e920 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2097,6 +2097,64 @@ To use the ``"replace": true`` option you will also need the :ref:`actions_updat Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`actions_alter_table` permission. +.. _TableForeignKeySuggestionsView: + +Table foreign key suggestions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``//
/-/foreign-key-suggestions`` endpoint suggests possible single-column foreign key relationships for a table. This requires the :ref:`actions_alter_table` permission. + +:: + + GET //
/-/foreign-key-suggestions + +The response includes every type-compatible single-column primary key target for each column in ``options``. Datasette also performs a bounded data check against up to 500 rows in the table: if the sampled non-null values for a column all exist in a target primary key, that target is included in ``suggestions``. + +If the bounded check takes too long, the endpoint fails open. It still returns the type-compatible ``options`` for each column, but ``row_check.status`` will be ``"timed_out"`` and there may be no ``suggestions``. + +.. code-block:: json + + { + "ok": true, + "database": "data", + "table": "projects", + "row_check": { + "attempted": true, + "status": "completed", + "row_limit": 500, + "sampled_rows": 3, + "checked_options": 4 + }, + "columns": [ + { + "column": "owner_id", + "type": "INTEGER", + "affinity": "integer", + "current": null, + "suggestions": [ + { + "fk_table": "owners", + "fk_column": "id", + "confidence": "sampled", + "sampled_values": 3, + "reasons": [ + "type_match", + "sample_values_exist", + "name_match" + ] + } + ], + "options": [ + { + "fk_table": "owners", + "fk_column": "id", + "type": "INTEGER" + } + ] + } + ] + } + .. _TableAlterView: Altering tables diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 046cf695..36fe40e9 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1044,6 +1044,106 @@ async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_w } +@pytest.mark.asyncio +async def test_foreign_key_suggestions(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + await db.execute_write("create table owners (id integer primary key)") + await db.execute_write("insert into owners (id) values (1), (2), (3)") + await db.execute_write("create table categories (slug text primary key)") + await db.execute_write("insert into categories (slug) values ('one'), ('two')") + await db.execute_write("create table numbers (id integer primary key)") + await db.execute_write("insert into numbers (id) values (10), (20)") + await db.execute_write("create table weights (id real primary key)") + await db.execute_write("insert into weights (id) values (1.5), (2.5)") + await db.execute_write( + "insert into docs (id, title, score, age) values " + "(1, 'one', 1.5, 1), (2, 'two', 999.5, 2), (3, null, null, null)" + ) + + response = await ds_write.client.get( + "/data/docs/-/foreign-key-suggestions", + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["ok"] is True + assert data["database"] == "data" + assert data["table"] == "docs" + assert data["row_check"]["attempted"] is True + assert data["row_check"]["status"] == "completed" + assert data["row_check"]["row_limit"] == 500 + assert data["row_check"]["sampled_rows"] == 3 + + columns = {column["column"]: column for column in data["columns"]} + assert columns["age"]["options"] == [ + {"fk_table": "numbers", "fk_column": "id", "type": "INTEGER"}, + {"fk_table": "owners", "fk_column": "id", "type": "INTEGER"}, + ] + assert columns["age"]["suggestions"] == [ + { + "fk_table": "owners", + "fk_column": "id", + "confidence": "sampled", + "sampled_values": 2, + "reasons": ["type_match", "sample_values_exist"], + } + ] + assert columns["title"]["options"] == [ + {"fk_table": "categories", "fk_column": "slug", "type": "TEXT"} + ] + assert columns["title"]["suggestions"][0]["fk_table"] == "categories" + assert columns["score"]["options"] == [ + {"fk_table": "weights", "fk_column": "id", "type": "REAL"} + ] + assert columns["score"]["suggestions"] == [] + + +@pytest.mark.asyncio +async def test_foreign_key_suggestions_permission_denied(ds_write): + token = write_token(ds_write, permissions=["ir"]) + response = await ds_write.client.get( + "/data/docs/-/foreign-key-suggestions", + headers=_headers(token), + ) + assert response.status_code == 403 + assert response.json() == { + "ok": False, + "errors": ["Permission denied: need alter-table"], + } + + +@pytest.mark.asyncio +async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + await db.execute_write("create table owners (id integer primary key)") + + async def raise_timeout(*args, **kwargs): + raise table_create_alter.ForeignKeySuggestionTimedOut + + from datasette.views import table_create_alter + + monkeypatch.setattr( + table_create_alter, + "_foreign_key_suggestion_samples", + raise_timeout, + ) + + response = await ds_write.client.get( + "/data/docs/-/foreign-key-suggestions", + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["row_check"]["status"] == "timed_out" + columns = {column["column"]: column for column in data["columns"]} + assert columns["age"]["options"] == [ + {"fk_table": "owners", "fk_column": "id", "type": "INTEGER"} + ] + assert columns["age"]["suggestions"] == [] + + @pytest.mark.asyncio async def test_alter_table_permission_denied(ds_write): token = write_token(ds_write, permissions=["ir"]) From a6ef65f90da8b7789a41972fbe89aba37e1b6a30 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 16:29:59 -0700 Subject: [PATCH 020/167] //-/foreign-key-targets API endpoint Returns a list of tables with a single primary key, and for each one the name of that primary key column and its SQLite type affinity. This will be used by the create table UI to suggest foreign keys. --- datasette/app.py | 5 ++ datasette/views/table_create_alter.py | 60 ++++++++++++++++++++- docs/json_api.rst | 34 ++++++++++++ tests/test_api_write.py | 75 +++++++++++++++++++++++++++ 4 files changed, 173 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index 6b9f47ba..79dffb66 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -50,6 +50,7 @@ from .views.database import ( QueryView, ) from .views.table_create_alter import ( + DatabaseForeignKeyTargetsView, TableAlterView, TableCreateView, TableForeignKeySuggestionsView, @@ -2566,6 +2567,10 @@ class Datasette: r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") + add_route( + DatabaseForeignKeyTargetsView.as_view(self), + r"/(?P[^\/\.]+)/-/foreign-key-targets$", + ) add_route( QueryListView.as_view(self), r"/(?P[^\/\.]+)/-/queries(\.(?Pjson))?$", diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 2cb59ac1..ce43199c 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -52,6 +52,34 @@ ALTER_TABLE_TYPE_FOR_SQLITE_TYPE = { FOREIGN_KEY_SUGGESTION_ROW_LIMIT = 500 FOREIGN_KEY_SUGGESTION_TIME_LIMIT_MS = 50 FOREIGN_KEY_SUGGESTION_TOTAL_TIME_LIMIT_MS = 200 +FOREIGN_KEY_TARGETS_SQL = """ +select + m.name as fk_table, + p.name as fk_column, + case + when upper(coalesce(p.type, '')) like '%INT%' then 'integer' + when upper(coalesce(p.type, '')) like '%CHAR%' + or upper(coalesce(p.type, '')) like '%CLOB%' + or upper(coalesce(p.type, '')) like '%TEXT%' then 'text' + when upper(coalesce(p.type, '')) like '%BLOB%' + or coalesce(p.type, '') = '' then 'blob' + when upper(coalesce(p.type, '')) like '%REAL%' + or upper(coalesce(p.type, '')) like '%FLOA%' + or upper(coalesce(p.type, '')) like '%' || 'DOU' || 'B' || '%' then 'real' + else 'numeric' + end as type +from sqlite_master as m +cross join pragma_table_info(m.name) as p +where m.type = 'table' + and m.name not like 'sqlite_%' + and p.pk > 0 + and ( + select count(*) + from pragma_table_info(m.name) as p2 + where p2.pk > 0 + ) = 1 +order by m.name +""" class ForeignKeySuggestionTimedOut(Exception): @@ -66,7 +94,10 @@ def _sqlite_type_affinity(type_name): return "text" if "BLOB" in type_name or not type_name: return "blob" - if any(token in type_name for token in ("REAL", "FLOA", "DOUB")): + if any( + token in type_name + for token in ("REAL", "FLOA", "DOUB") # codespell:ignore doub + ): return "real" return "numeric" @@ -788,6 +819,33 @@ class TableCreateView(BaseView): return Response.json(details, status=201) +class DatabaseForeignKeyTargetsView(BaseView): + name = "database-foreign-key-targets" + + def __init__(self, datasette): + self.ds = datasette + + async def get(self, request): + db = await self.ds.resolve_database(request) + database_name = db.name + + if not await self.ds.allowed( + action="create-table", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ): + return _error(["Permission denied: need create-table"], 403) + + targets = (await db.execute(FOREIGN_KEY_TARGETS_SQL)).dicts() + return Response.json( + { + "ok": True, + "database": database_name, + "targets": targets, + } + ) + + class TableForeignKeySuggestionsView(BaseView): name = "table-foreign-key-suggestions" diff --git a/docs/json_api.rst b/docs/json_api.rst index 5b05e920..dee98ef2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2097,6 +2097,40 @@ To use the ``"replace": true`` option you will also need the :ref:`actions_updat Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`actions_alter_table` permission. +.. _DatabaseForeignKeyTargetsView: + +Database foreign key targets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``//-/foreign-key-targets`` endpoint returns the list of tables in a database that can be referenced by a single-column foreign key. This requires the :ref:`actions_create_table` permission. + +:: + + GET //-/foreign-key-targets + +The response includes only tables with exactly one primary key column. Tables with compound primary keys and tables with no explicit primary key are omitted. + +Each target includes the normalized SQLite type affinity for the primary key column in ``type``. The type is calculated using SQLite's documented affinity rules: ``INT`` maps to ``integer``; ``CHAR``, ``CLOB`` or ``TEXT`` maps to ``text``; ``BLOB`` or no type maps to ``blob``; ``REAL`` and floating-point declared types map to ``real``; everything else maps to ``numeric``. + +.. code-block:: json + + { + "ok": true, + "database": "data", + "targets": [ + { + "fk_table": "owners", + "fk_column": "id", + "type": "integer" + }, + { + "fk_table": "categories", + "fk_column": "slug", + "type": "text" + } + ] + } + .. _TableForeignKeySuggestionsView: Table foreign key suggestions diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 36fe40e9..18ffe43d 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1144,6 +1144,81 @@ async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch): assert columns["age"]["suggestions"] == [] +@pytest.mark.asyncio +async def test_foreign_key_targets(ds_write): + token = write_token(ds_write, permissions=["ct"]) + db = ds_write.get_database("data") + await db.execute_write("create table owners (id integer primary key)") + await db.execute_write("create table categories (slug varchar(30) primary key)") + await db.execute_write("create table blob_things (hash blob primary key)") + await db.execute_write( + "create table numeric_codes (code decimal(10,5) primary key)" + ) + await db.execute_write( + 'create table floating_point (value "FLOATING POINT" primary key)' + ) + await db.execute_write( + "create table compound (a integer, b integer, primary key (a, b))" + ) + await db.execute_write("create table no_pk (name text)") + + response = await ds_write.client.get( + "/data/-/foreign-key-targets", + headers=_headers(token), + ) + assert response.status_code == 200, response.text + assert response.json() == { + "ok": True, + "database": "data", + "targets": [ + { + "fk_table": "blob_things", + "fk_column": "hash", + "type": "blob", + }, + { + "fk_table": "categories", + "fk_column": "slug", + "type": "text", + }, + { + "fk_table": "docs", + "fk_column": "id", + "type": "integer", + }, + { + "fk_table": "floating_point", + "fk_column": "value", + "type": "integer", + }, + { + "fk_table": "numeric_codes", + "fk_column": "code", + "type": "numeric", + }, + { + "fk_table": "owners", + "fk_column": "id", + "type": "integer", + }, + ], + } + + +@pytest.mark.asyncio +async def test_foreign_key_targets_permission_denied(ds_write): + token = write_token(ds_write, permissions=["ir"]) + response = await ds_write.client.get( + "/data/-/foreign-key-targets", + headers=_headers(token), + ) + assert response.status_code == 403 + assert response.json() == { + "ok": False, + "errors": ["Permission denied: need create-table"], + } + + @pytest.mark.asyncio async def test_alter_table_permission_denied(ds_write): token = write_token(ds_write, permissions=["ir"]) From 21c156dfb1ca76d8e8c2e04c044766a155e22edc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Jun 2026 08:13:28 -0700 Subject: [PATCH 021/167] Expose foreign key targets to create table UI - Add foreignKeyTargetsPath to create table page data - Filter hidden tables from database-level foreign key target results - Update JSON API docs and tests for filtered targets --- datasette/views/table_create_alter.py | 12 +++++++++++- docs/json_api.rst | 2 +- tests/test_api_write.py | 8 ++++++++ tests/test_table_html.py | 1 + 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index ce43199c..f7775479 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -262,6 +262,9 @@ async def _create_table_ui_context( return None data = { "path": "{}/-/create".format(datasette.urls.database(database_name)), + "foreignKeyTargetsPath": "{}/-/foreign-key-targets".format( + datasette.urls.database(database_name) + ), "databaseName": database_name, "columnTypes": CREATE_TABLE_COLUMN_TYPES, } @@ -836,7 +839,14 @@ class DatabaseForeignKeyTargetsView(BaseView): ): return _error(["Permission denied: need create-table"], 403) - targets = (await db.execute(FOREIGN_KEY_TARGETS_SQL)).dicts() + hidden_tables = await db.execute_fn( + lambda conn: set(sqlite_hidden_table_names(conn)) + ) + targets = [ + target + for target in (await db.execute(FOREIGN_KEY_TARGETS_SQL)).dicts() + if target["fk_table"] not in hidden_tables + ] return Response.json( { "ok": True, diff --git a/docs/json_api.rst b/docs/json_api.rst index dee98ef2..1db46dd2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2108,7 +2108,7 @@ The ``//-/foreign-key-targets`` endpoint returns the list of tables in GET //-/foreign-key-targets -The response includes only tables with exactly one primary key column. Tables with compound primary keys and tables with no explicit primary key are omitted. +The response includes only tables with exactly one primary key column. Hidden tables, tables with compound primary keys and tables with no explicit primary key are omitted. Each target includes the normalized SQLite type affinity for the primary key column in ``type``. The type is calculated using SQLite's documented affinity rules: ``INT`` maps to ``integer``; ``CHAR``, ``CLOB`` or ``TEXT`` maps to ``text``; ``BLOB`` or no type maps to ``blob``; ``REAL`` and floating-point declared types map to ``real``; everything else maps to ``numeric``. diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 18ffe43d..1d16ad26 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1161,6 +1161,10 @@ async def test_foreign_key_targets(ds_write): "create table compound (a integer, b integer, primary key (a, b))" ) await db.execute_write("create table no_pk (name text)") + try: + await db.execute_write("create virtual table search_docs using fts5(body)") + except Exception: + pass response = await ds_write.client.get( "/data/-/foreign-key-targets", @@ -1203,6 +1207,10 @@ async def test_foreign_key_targets(ds_write): }, ], } + assert not any( + target["fk_table"].startswith("search_docs_") + for target in response.json()["targets"] + ) @pytest.mark.asyncio diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 374cc08d..fa01c8ec 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -994,6 +994,7 @@ async def test_database_create_table_action_button_and_data(): assert database_data_from_soup(soup) == { "createTable": { "path": "/data/-/create", + "foreignKeyTargetsPath": "/data/-/foreign-key-targets", "databaseName": "data", "columnTypes": ["text", "integer", "float", "blob"], }, From 1f863def5e410cc3f836f1e07404aa68b578db9b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Jun 2026 08:13:37 -0700 Subject: [PATCH 022/167] Add foreign key controls to create table dialog - Add create table advanced controls for foreign keys and first-column primary keys - Share schema dialog row helpers between create and alter dialogs - Move custom type into advanced options and add Add column icons --- datasette/static/app.css | 210 ++++++-- datasette/static/edit-tools.js | 954 ++++++++++++++++++++++++--------- 2 files changed, 862 insertions(+), 302 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index f9ebe5ac..49f070b1 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1760,7 +1760,7 @@ dialog.table-create-dialog { border-radius: var(--modal-border-radius, 0.75rem); padding: 0; margin: auto; - width: min(760px, calc(100vw - 32px)); + width: min(980px, calc(100vw - 32px)); max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); @@ -1839,7 +1839,7 @@ dialog.table-create-dialog::backdrop { } .table-create-label, -.table-create-columns-heading { +.table-create-column-headings { color: var(--ink); font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 0.82rem; @@ -1876,10 +1876,12 @@ dialog.table-create-dialog::backdrop { color: var(--muted); } +.table-create-foreign-key-target option, .table-create-custom-column-type option { color: var(--ink); } +.table-create-foreign-key-target option[value=""], .table-create-custom-column-type option[value=""] { color: var(--muted); } @@ -1898,40 +1900,130 @@ dialog.table-create-dialog::backdrop { gap: 10px; } -.table-create-columns-heading { - font-weight: 600; -} - .table-create-column-list { display: grid; gap: 8px; } -.table-create-column-row { +.table-create-column-headings, +.table-create-column-main { display: grid; - grid-template-columns: minmax(140px, 1.2fr) minmax(7.5rem, 0.7fr) minmax(12rem, 1fr) minmax(3.5rem, max-content) 32px; + grid-template-columns: minmax(140px, 1.2fr) minmax(7.5rem, 0.7fr) max-content 32px; align-items: center; gap: 8px; min-width: 0; - position: relative; } -.table-create-primary-key { - display: inline-flex; - align-items: center; - gap: 6px; - color: var(--ink); - font-size: 0.85rem; +.table-create-column-row { + display: grid; + gap: 8px; min-width: 0; - white-space: nowrap; - justify-self: center; } -.table-create-primary-key-input { +.table-create-column-headings { + color: var(--muted); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.72rem; + padding: 0 1px; +} + +.table-create-column-details { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + align-items: start; + gap: 12px 16px; + padding: 12px; + border-left: 3px solid var(--rule); + background: #f8fafc; +} + +.table-create-column-details[hidden] { + display: none; +} + +.table-create-detail-field { + display: grid; + gap: 4px; + min-width: 0; +} + +.table-create-detail-label { + color: var(--muted); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.72rem; +} + +.table-create-detail-help { + color: var(--muted); + font-size: 0.82rem; + line-height: 1.35; margin: 0; } -.table-create-remove-column { +.table-create-detail-check { + display: inline-flex; + align-items: flex-start; + gap: 8px; + color: var(--ink); + font-size: 0.85rem; + line-height: 1.35; + min-width: 0; + white-space: normal; +} + +.table-create-primary-key, +.table-create-foreign-key-field { + grid-column: 1 / -1; +} + +.table-create-detail-check input { + flex: 0 0 auto; + margin: 0.15rem 0 0; +} + +.table-create-detail-check span { + min-width: 0; + overflow-wrap: break-word; +} + +.table-create-move-controls { + display: grid; + grid-template-columns: repeat(4, 32px); + gap: 4px; + justify-content: start; +} + +.table-create-more-options { + appearance: none; + border: 0; + background: transparent; + color: var(--accent); + cursor: pointer; + font: inherit; + font-size: 0.85rem; + justify-self: start; + padding: 0; + grid-column: 1 / -1; + text-align: left; +} + +.table-create-more-options:hover, +.table-create-more-options:focus { + text-decoration: underline; +} + +.table-create-more-options:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 2px; +} + +.table-create-more-options:disabled { + color: var(--muted); + cursor: default; + text-decoration: none; +} + +.table-create-icon-button { appearance: none; border: 1px solid rgba(74, 85, 104, 0.24); background: transparent; @@ -1945,17 +2037,17 @@ dialog.table-create-dialog::backdrop { padding: 0; } -.table-create-remove-column:hover, -.table-create-remove-column:focus { +.table-create-icon-button:hover, +.table-create-icon-button:focus { background: rgba(74, 85, 104, 0.07); } -.table-create-remove-column:focus { +.table-create-icon-button:focus { outline: 3px solid #b3d4ff; outline-offset: 1px; } -.table-create-remove-column svg { +.table-create-icon-button svg { display: block; } @@ -1967,11 +2059,19 @@ dialog.table-create-dialog::backdrop { background: #fff; color: var(--accent); cursor: pointer; + display: inline-flex; + align-items: center; + gap: 6px; font: inherit; font-size: 0.85rem; padding: 7px 10px; } +.table-create-add-column svg { + display: block; + flex: 0 0 auto; +} + .table-create-add-column:hover, .table-create-add-column:focus { background: #f8fafc; @@ -2027,7 +2127,7 @@ dialog.table-create-dialog::backdrop { .table-create-dialog .btn:disabled, .table-create-add-column:disabled, -.table-create-remove-column:disabled { +.table-create-icon-button:disabled { opacity: 0.55; cursor: not-allowed; } @@ -2198,7 +2298,7 @@ dialog.table-alter-dialog::backdrop { .table-alter-column-headings, .table-alter-column-main { display: grid; - grid-template-columns: minmax(140px, 1.2fr) minmax(7.5rem, 0.7fr) minmax(12rem, 1fr) max-content 32px; + grid-template-columns: minmax(140px, 1.2fr) minmax(7.5rem, 0.7fr) max-content 32px; align-items: center; gap: 8px; min-width: 0; @@ -2384,11 +2484,19 @@ dialog.table-alter-dialog::backdrop { background: #fff; color: var(--accent); cursor: pointer; + display: inline-flex; + align-items: center; + gap: 6px; font: inherit; font-size: 0.85rem; padding: 7px 10px; } +.table-alter-add-column svg { + display: block; + flex: 0 0 auto; +} + .table-alter-add-column:hover, .table-alter-add-column:focus { background: #f8fafc; @@ -2504,21 +2612,16 @@ dialog.table-alter-dialog::backdrop { justify-self: end; } - .table-alter-custom-column-type { - grid-column: 1 / 3; - grid-row: 2; - } - .table-alter-move-controls { grid-column: 1; - grid-row: 3; + grid-row: 2; justify-self: start; } .table-alter-more-options { align-self: center; grid-column: 2 / 4; - grid-row: 3; + grid-row: 2; } .table-alter-column-details { @@ -2700,13 +2803,50 @@ dialog.table-alter-dialog::backdrop { padding-top: 0; } + .table-create-column-headings { + display: none; + } + .table-create-column-row { - grid-template-columns: minmax(0, 1fr) 8.5rem 3.5rem 32px; + padding-bottom: 8px; + border-bottom: 1px solid var(--rule); + } + + .table-create-column-main { + grid-template-columns: minmax(0, 1fr) minmax(7.5rem, 0.8fr) 32px; align-items: end; } - .table-create-custom-column-type { - grid-column: 1 / -1; + .table-create-column-name { + grid-column: 1; + grid-row: 1; + } + + .table-create-column-type { + grid-column: 2; + grid-row: 1; + } + + .table-create-remove-column { + grid-column: 3; + grid-row: 1; + justify-self: end; + } + + .table-create-move-controls { + grid-column: 1; + grid-row: 2; + justify-self: start; + } + + .table-create-more-options { + align-self: center; + grid-column: 2 / 4; + grid-row: 2; + } + + .table-create-column-details { + grid-template-columns: 1fr; } .table-create-dialog .modal-footer { diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index cb964365..5a9d0962 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -64,12 +64,220 @@ function sqliteColumnTypeLabel(type) { if (type === "float") { return "floating point number"; } + if (type === "real") { + return "floating point number"; + } if (type === "blob") { return "blob - binary data"; } return type; } +function populateSqliteColumnTypeSelect(select, type, options) { + options.forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option; + optionElement.textContent = sqliteColumnTypeLabel(option); + select.appendChild(optionElement); + }); + select.value = options.indexOf(type) === -1 ? options[0] : type; +} + +function updateSelectPlaceholder(select, placeholderClass) { + select.classList.toggle(placeholderClass, !select.value); +} + +function createCustomColumnTypeSelect(options, className, placeholderClass) { + var select = document.createElement("select"); + select.className = className; + select.setAttribute("aria-label", "Custom column type"); + var blankOption = document.createElement("option"); + blankOption.value = ""; + blankOption.textContent = "- custom type -"; + select.appendChild(blankOption); + options.forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option.name; + optionElement.textContent = option.description + ? option.description + " (" + option.name + ")" + : option.name; + select.appendChild(optionElement); + }); + updateSelectPlaceholder(select, placeholderClass); + return select; +} + +var COLUMN_MOVE_ICONS = { + top: '', + up: '', + down: '', + bottom: + '', + remove: + '', +}; + +function createSchemaDialogIconButton(prefix, modifier, ariaLabel, title, svg) { + var button = document.createElement("button"); + button.type = "button"; + button.className = prefix + "-icon-button " + prefix + "-" + modifier; + button.setAttribute("aria-label", ariaLabel); + button.title = title; + button.dataset.defaultTitle = title; + button.innerHTML = svg; + return button; +} + +function createSchemaDialogMoveControls(prefix) { + var moveControls = document.createElement("div"); + moveControls.className = prefix + "-move-controls"; + + var moveTopButton = createSchemaDialogIconButton( + prefix, + "move-top", + "Move column to top", + "Move column to top", + COLUMN_MOVE_ICONS.top, + ); + var moveUpButton = createSchemaDialogIconButton( + prefix, + "move-up", + "Move column up", + "Move column up", + COLUMN_MOVE_ICONS.up, + ); + var moveDownButton = createSchemaDialogIconButton( + prefix, + "move-down", + "Move column down", + "Move column down", + COLUMN_MOVE_ICONS.down, + ); + var moveBottomButton = createSchemaDialogIconButton( + prefix, + "move-bottom", + "Move column to bottom", + "Move column to bottom", + COLUMN_MOVE_ICONS.bottom, + ); + + moveControls.appendChild(moveTopButton); + moveControls.appendChild(moveUpButton); + moveControls.appendChild(moveDownButton); + moveControls.appendChild(moveBottomButton); + + return { + controls: moveControls, + topButton: moveTopButton, + upButton: moveUpButton, + downButton: moveDownButton, + bottomButton: moveBottomButton, + }; +} + +function createSchemaDialogMoreOptionsButton(prefix, details) { + var expandButton = document.createElement("button"); + expandButton.type = "button"; + expandButton.className = prefix + "-more-options"; + expandButton.setAttribute("aria-label", "Toggle column settings"); + expandButton.setAttribute("aria-controls", details.id); + expandButton.setAttribute("aria-expanded", details.hidden ? "false" : "true"); + updateSchemaDialogMoreOptionsButton(expandButton); + return expandButton; +} + +function updateSchemaDialogMoreOptionsButton(button) { + var isExpanded = button.getAttribute("aria-expanded") === "true"; + button.textContent = isExpanded ? "v Hide options" : "> Advanced options"; + button.title = isExpanded ? "Hide column settings" : "Show column settings"; +} + +function toggleSchemaDialogMoreOptions(button, details) { + var isExpanded = button.getAttribute("aria-expanded") === "true"; + details.hidden = isExpanded; + button.setAttribute("aria-expanded", isExpanded ? "false" : "true"); + updateSchemaDialogMoreOptionsButton(button); +} + +function schemaDialogRows(state, prefix) { + return Array.prototype.slice.call( + state.columnList.querySelectorAll("." + prefix + "-column-row"), + ); +} + +function schemaDialogRowIsPrimaryKey(row, prefix) { + var input = row && row.querySelector("." + prefix + "-primary-key-input"); + return !!(input && input.checked); +} + +function schemaDialogFirstNonPrimaryRow(state, prefix) { + var rows = schemaDialogRows(state, prefix); + for (var i = 0; i < rows.length; i += 1) { + if (!schemaDialogRowIsPrimaryKey(rows[i], prefix)) { + return rows[i]; + } + } + return null; +} + +function updateSchemaDialogMoveButtons(state, prefix) { + if (!state || !state.columnList) { + return; + } + var firstNonPrimary = schemaDialogFirstNonPrimaryRow(state, prefix); + var rows = schemaDialogRows(state, prefix); + var hasPrimaryKeys = rows.some(function (row) { + return schemaDialogRowIsPrimaryKey(row, prefix); + }); + var primaryKeyMoveTitle = "Primary key columns are always listed first"; + rows.forEach(function (row) { + var isPrimaryKey = schemaDialogRowIsPrimaryKey(row, prefix); + var previous = row.previousElementSibling; + var next = row.nextElementSibling; + row + .querySelectorAll("." + prefix + "-move-controls button") + .forEach(function (button) { + button.title = button.dataset.defaultTitle || button.title; + button.disabled = state.isSaving || isPrimaryKey; + if (isPrimaryKey) { + button.title = primaryKeyMoveTitle; + } + }); + if (!isPrimaryKey) { + var topButton = row.querySelector("." + prefix + "-move-top"); + var upButton = row.querySelector("." + prefix + "-move-up"); + var downButton = row.querySelector("." + prefix + "-move-down"); + var bottomButton = row.querySelector("." + prefix + "-move-bottom"); + topButton.disabled = + state.isSaving || !firstNonPrimary || row === firstNonPrimary; + upButton.disabled = + state.isSaving || !previous || schemaDialogRowIsPrimaryKey(previous, prefix); + downButton.disabled = state.isSaving || !next; + bottomButton.disabled = state.isSaving || !next; + if (hasPrimaryKeys && row === firstNonPrimary) { + topButton.title = primaryKeyMoveTitle; + upButton.title = primaryKeyMoveTitle; + } + } + }); +} + +function normalizeSchemaDialogPrimaryKeyRows(state, prefix) { + var rows = schemaDialogRows(state, prefix); + rows + .filter(function (row) { + return schemaDialogRowIsPrimaryKey(row, prefix); + }) + .concat( + rows.filter(function (row) { + return !schemaDialogRowIsPrimaryKey(row, prefix); + }), + ) + .forEach(function (row) { + state.columnList.appendChild(row); + }); +} + function tableCreateCustomColumnTypes() { var data = databaseCreateTableData() || {}; return data.customColumnTypes || []; @@ -93,15 +301,205 @@ function tableCreateCustomTypeAppliesToSqliteType(option, sqliteType) { ); } +function tableCreateDialogRows(state) { + return schemaDialogRows(state, "table-create"); +} + +function tableCreateRowIsPrimaryKey(row) { + return schemaDialogRowIsPrimaryKey(row, "table-create"); +} + +function tableCreateFirstNonPrimaryRow(state) { + return schemaDialogFirstNonPrimaryRow(state, "table-create"); +} + +function updateTableCreateMoveButtons(state) { + updateSchemaDialogMoveButtons(state, "table-create"); +} + +function tableCreateTypeAffinity(type) { + if (type === "float") { + return "real"; + } + return type; +} + +function foreignKeyTypesCompatible(sourceAffinity, targetAffinity) { + if (sourceAffinity === targetAffinity) { + return true; + } + var numericAffinities = ["integer", "real", "numeric"]; + if (sourceAffinity === "numeric") { + return numericAffinities.indexOf(targetAffinity) !== -1; + } + if (targetAffinity === "numeric") { + return numericAffinities.indexOf(sourceAffinity) !== -1; + } + return false; +} + +function tableCreateForeignKeyTargetKey(target) { + return target.fk_table + "\u001f" + target.fk_column; +} + +function tableCreateForeignKeyTargetLabel(target) { + return ( + target.fk_table + + "." + + target.fk_column + + " (" + + sqliteColumnTypeLabel(target.type) + + ")" + ); +} + +function tableCreateForeignKeyTargetsUrl() { + var data = databaseCreateTableData() || {}; + if (data.foreignKeyTargetsPath) { + return data.foreignKeyTargetsPath; + } + if (!data.path) { + return null; + } + return data.path.replace(/\/-\/create$/, "/-/foreign-key-targets"); +} + +function populateTableCreateForeignKeySelect(select, state, sourceType) { + var previousKey = select.value || select.dataset.selectedKey || ""; + select.textContent = ""; + + var blankOption = document.createElement("option"); + blankOption.value = ""; + blankOption.textContent = "- no foreign key -"; + select.appendChild(blankOption); + + if (state.foreignKeyTargetsLoading) { + var loadingOption = document.createElement("option"); + loadingOption.value = ""; + loadingOption.disabled = true; + loadingOption.textContent = "Loading foreign keys..."; + select.appendChild(loadingOption); + } else if (state.foreignKeyTargetsError) { + var errorOption = document.createElement("option"); + errorOption.value = ""; + errorOption.disabled = true; + errorOption.textContent = "Could not load foreign keys"; + select.appendChild(errorOption); + } else { + var sourceAffinity = tableCreateTypeAffinity(sourceType); + (state.foreignKeyTargets || []).forEach(function (target) { + if (!foreignKeyTypesCompatible(sourceAffinity, target.type)) { + return; + } + var optionElement = document.createElement("option"); + optionElement.value = tableCreateForeignKeyTargetKey(target); + optionElement.dataset.fkTable = target.fk_table; + optionElement.dataset.fkColumn = target.fk_column; + optionElement.textContent = tableCreateForeignKeyTargetLabel(target); + select.appendChild(optionElement); + }); + } + + select.value = previousKey; + if (select.value !== previousKey) { + select.value = ""; + } + select.dataset.selectedKey = select.value; + select.disabled = state.isSaving || select.options.length <= 1; + updateSelectPlaceholder(select, "table-create-input-placeholder"); +} + +function syncTableCreateForeignKeyOptions(row, state) { + var typeSelect = row.querySelector(".table-create-column-type"); + var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); + if (!typeSelect || !foreignKeySelect) { + return; + } + populateTableCreateForeignKeySelect( + foreignKeySelect, + state, + typeSelect.value, + ); +} + +function refreshTableCreateForeignKeyControls(state) { + tableCreateDialogRows(state).forEach(function (row, index) { + if (index > 0) { + syncTableCreateForeignKeyOptions(row, state); + } + }); +} + +function updateTableCreateColumnRules(state) { + tableCreateDialogRows(state).forEach(function (row, index) { + var isFirstColumn = index === 0; + var pkLabel = row.querySelector(".table-create-primary-key"); + var pkInput = row.querySelector(".table-create-primary-key-input"); + var foreignKeyField = row.querySelector(".table-create-foreign-key-field"); + var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); + + if (pkLabel && pkInput) { + pkLabel.hidden = !isFirstColumn; + if (!isFirstColumn) { + pkInput.checked = false; + } + } + + if (foreignKeyField && foreignKeySelect) { + foreignKeyField.hidden = isFirstColumn; + if (isFirstColumn) { + foreignKeySelect.value = ""; + foreignKeySelect.dataset.selectedKey = ""; + foreignKeySelect.disabled = true; + updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); + } else { + syncTableCreateForeignKeyOptions(row, state); + } + } + }); + updateTableCreateMoveButtons(state); +} + +async function loadTableCreateForeignKeyTargets(state) { + var url = tableCreateForeignKeyTargetsUrl(); + if (!url || !window.fetch) { + state.foreignKeyTargets = []; + state.foreignKeyTargetsLoading = false; + refreshTableCreateForeignKeyControls(state); + return; + } + state.foreignKeyTargets = []; + state.foreignKeyTargetsError = null; + state.foreignKeyTargetsLoading = true; + refreshTableCreateForeignKeyControls(state); + try { + var response = await fetch(url, { + headers: { + Accept: "application/json", + }, + }); + var data = await response.json(); + if (!response.ok || data.ok === false) { + throw rowMutationRequestError(response, data); + } + state.foreignKeyTargets = data.targets || []; + } catch (error) { + state.foreignKeyTargets = []; + state.foreignKeyTargetsError = error; + } finally { + state.foreignKeyTargetsLoading = false; + refreshTableCreateForeignKeyControls(state); + } +} + function tableCreateDialogSignature(state) { if (!state || !state.form) { return ""; } - var columns = []; - state.columnList - .querySelectorAll(".table-create-column-row") - .forEach(function (row) { - columns.push({ + return JSON.stringify({ + table: state.tableName.value, + columns: tableCreateDialogRows(state).map(function (row) { + return { name: row.querySelector(".table-create-column-name").value, type: row.querySelector(".table-create-column-type").value, customType: @@ -111,11 +509,14 @@ function tableCreateDialogSignature(state) { } ).value || "", pk: row.querySelector(".table-create-primary-key-input").checked, - }); - }); - return JSON.stringify({ - table: state.tableName.value, - columns: columns, + foreignKey: + ( + row.querySelector(".table-create-foreign-key-target") || { + value: "", + } + ).value || "", + }; + }), }); } @@ -151,42 +552,28 @@ function setTableCreateDialogSaving(state, isSaving) { .forEach(function (control) { control.disabled = isSaving; }); + if (!isSaving) { + updateTableCreateColumnRules(state); + } + updateTableCreateMoveButtons(state); } function tableCreateSelectTypeValue(select, type) { var options = tableCreateColumnTypes(); - options.forEach(function (option) { - var optionElement = document.createElement("option"); - optionElement.value = option; - optionElement.textContent = sqliteColumnTypeLabel(option); - select.appendChild(optionElement); - }); - select.value = options.indexOf(type) === -1 ? options[0] : type; + populateSqliteColumnTypeSelect(select, type, options); } function updateTableCreateCustomColumnTypePlaceholder(select) { - select.classList.toggle("table-create-input-placeholder", !select.value); + updateSelectPlaceholder(select, "table-create-input-placeholder"); } function createTableCustomColumnTypeSelect() { var options = tableCreateCustomColumnTypes(); - var select = document.createElement("select"); - select.className = "table-create-input table-create-custom-column-type"; - select.setAttribute("aria-label", "Custom column type"); - var blankOption = document.createElement("option"); - blankOption.value = ""; - blankOption.textContent = "- custom type -"; - select.appendChild(blankOption); - options.forEach(function (option) { - var optionElement = document.createElement("option"); - optionElement.value = option.name; - optionElement.textContent = option.description - ? option.description + " (" + option.name + ")" - : option.name; - select.appendChild(optionElement); - }); - updateTableCreateCustomColumnTypePlaceholder(select); - return select; + return createCustomColumnTypeSelect( + options, + "table-create-input table-create-custom-column-type", + "table-create-input-placeholder", + ); } function syncTableCreateCustomTypeForSqliteType(row) { @@ -209,6 +596,19 @@ function createTableColumnRow(state, column) { var row = document.createElement("div"); row.className = "table-create-column-row"; + var main = document.createElement("div"); + main.className = "table-create-column-main"; + + var details = document.createElement("div"); + details.className = "table-create-column-details"; + details.id = "table-create-column-details-" + index; + details.hidden = !(column && column.expanded); + + var expandButton = createSchemaDialogMoreOptionsButton( + "table-create", + details, + ); + var nameId = "table-create-column-name-" + index; var nameLabel = document.createElement("label"); nameLabel.className = "table-create-column-label"; @@ -228,40 +628,88 @@ function createTableColumnRow(state, column) { typeSelect.setAttribute("aria-label", "Column type"); tableCreateSelectTypeValue(typeSelect, column && column.type); - var customTypeSelect = createTableCustomColumnTypeSelect(); - if (column && column.customType) { - customTypeSelect.value = column.customType; + var customTypeSelect = null; + var customTypeField = null; + if (tableCreateCustomColumnTypes().length) { + var customTypeId = "table-create-column-custom-type-" + index; + customTypeField = document.createElement("div"); + customTypeField.className = + "table-create-detail-field table-create-custom-type-field"; + var customTypeLabel = document.createElement("label"); + customTypeLabel.className = "table-create-detail-label"; + customTypeLabel.setAttribute("for", customTypeId); + customTypeLabel.textContent = "Custom type"; + customTypeSelect = createTableCustomColumnTypeSelect(); + customTypeSelect.id = customTypeId; + customTypeSelect.value = column && column.customType ? column.customType : ""; + updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + customTypeField.appendChild(customTypeLabel); + customTypeField.appendChild(customTypeSelect); } - updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); var pkLabel = document.createElement("label"); - pkLabel.className = "table-create-primary-key"; + pkLabel.className = "table-create-detail-check table-create-primary-key"; var pkInput = document.createElement("input"); pkInput.type = "checkbox"; pkInput.className = "table-create-primary-key-input"; pkInput.checked = !!(column && column.primaryKey); var pkText = document.createElement("span"); - pkText.textContent = "PK"; - pkText.title = "Primary key"; + var pkStrong = document.createElement("strong"); + pkStrong.textContent = "Primary key"; + pkText.appendChild(pkStrong); + pkText.appendChild( + document.createTextNode(" This ID uniquely identifies the record"), + ); pkLabel.appendChild(pkInput); pkLabel.appendChild(pkText); - var removeButton = document.createElement("button"); - removeButton.type = "button"; - removeButton.className = "table-create-remove-column"; - removeButton.setAttribute("aria-label", "Remove column"); - removeButton.title = "Remove column"; - removeButton.innerHTML = - ''; + var foreignKeyId = "table-create-column-foreign-key-" + index; + var foreignKeyHelpId = "table-create-column-foreign-key-help-" + index; + var foreignKeyField = document.createElement("div"); + foreignKeyField.className = + "table-create-detail-field table-create-foreign-key-field"; + var foreignKeyLabel = document.createElement("label"); + foreignKeyLabel.className = "table-create-detail-label"; + foreignKeyLabel.setAttribute("for", foreignKeyId); + foreignKeyLabel.textContent = "Foreign key"; + var foreignKeyHelp = document.createElement("p"); + foreignKeyHelp.id = foreignKeyHelpId; + foreignKeyHelp.className = "table-create-detail-help"; + foreignKeyHelp.textContent = "Link this column to another table."; + var foreignKeySelect = document.createElement("select"); + foreignKeySelect.id = foreignKeyId; + foreignKeySelect.className = + "table-create-input table-create-foreign-key-target"; + foreignKeySelect.setAttribute("aria-label", "Foreign key target"); + foreignKeySelect.setAttribute("aria-describedby", foreignKeyHelpId); + foreignKeyField.appendChild(foreignKeyLabel); + foreignKeyField.appendChild(foreignKeyHelp); + foreignKeyField.appendChild(foreignKeySelect); - row.appendChild(nameLabel); - row.appendChild(nameInput); - row.appendChild(typeSelect); - if (tableCreateCustomColumnTypes().length) { - row.appendChild(customTypeSelect); + var moveControls = createSchemaDialogMoveControls("table-create"); + + var removeButton = createSchemaDialogIconButton( + "table-create", + "remove-column", + "Remove column", + "Remove column", + COLUMN_MOVE_ICONS.remove, + ); + + main.appendChild(nameLabel); + main.appendChild(nameInput); + main.appendChild(typeSelect); + main.appendChild(moveControls.controls); + main.appendChild(removeButton); + main.appendChild(expandButton); + + if (customTypeField) { + details.appendChild(customTypeField); } - row.appendChild(pkLabel); - row.appendChild(removeButton); + details.appendChild(foreignKeyField); + details.appendChild(pkLabel); + row.appendChild(main); + row.appendChild(details); removeButton.addEventListener("click", function () { if (state.isSaving) { @@ -269,6 +717,7 @@ function createTableColumnRow(state, column) { } row.remove(); clearTableCreateDialogError(state); + updateTableCreateColumnRules(state); var nextInput = state.columnList.querySelector(".table-create-column-name"); if (nextInput) { nextInput.focus(); @@ -283,21 +732,94 @@ function createTableColumnRow(state, column) { typeSelect.addEventListener("change", function () { clearTableCreateDialogError(state); syncTableCreateCustomTypeForSqliteType(row); + syncTableCreateForeignKeyOptions(row, state); }); - customTypeSelect.addEventListener("change", function () { - clearTableCreateDialogError(state); - updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); - var option = tableCreateCustomColumnType(customTypeSelect.value); - if ( - option && - option.fixedSqliteType && - tableCreateColumnTypes().indexOf(option.fixedSqliteType) !== -1 - ) { - typeSelect.value = option.fixedSqliteType; - } - }); + if (customTypeSelect) { + customTypeSelect.addEventListener("change", function () { + clearTableCreateDialogError(state); + updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + var option = tableCreateCustomColumnType(customTypeSelect.value); + if ( + option && + option.fixedSqliteType && + tableCreateColumnTypes().indexOf(option.fixedSqliteType) !== -1 + ) { + typeSelect.value = option.fixedSqliteType; + syncTableCreateForeignKeyOptions(row, state); + } + }); + } pkInput.addEventListener("change", function () { clearTableCreateDialogError(state); + updateTableCreateColumnRules(state); + }); + foreignKeySelect.addEventListener("change", function () { + foreignKeySelect.dataset.selectedKey = foreignKeySelect.value; + clearTableCreateDialogError(state); + updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); + }); + + expandButton.addEventListener("click", function () { + toggleSchemaDialogMoreOptions(expandButton, details); + }); + + moveControls.topButton.addEventListener("click", function () { + var first = tableCreateFirstNonPrimaryRow(state); + if ( + state.isSaving || + tableCreateRowIsPrimaryKey(row) || + !first || + first === row + ) { + return; + } + state.columnList.insertBefore(row, first); + clearTableCreateDialogError(state); + updateTableCreateColumnRules(state); + row.querySelector(".table-create-column-name").focus(); + }); + + moveControls.upButton.addEventListener("click", function () { + var previous = row.previousElementSibling; + if ( + state.isSaving || + tableCreateRowIsPrimaryKey(row) || + !previous || + tableCreateRowIsPrimaryKey(previous) + ) { + return; + } + state.columnList.insertBefore(row, previous); + clearTableCreateDialogError(state); + updateTableCreateColumnRules(state); + row.querySelector(".table-create-column-name").focus(); + }); + + moveControls.downButton.addEventListener("click", function () { + var next = row.nextElementSibling; + if (state.isSaving || tableCreateRowIsPrimaryKey(row) || !next) { + return; + } + state.columnList.insertBefore(next, row); + clearTableCreateDialogError(state); + updateTableCreateColumnRules(state); + row.querySelector(".table-create-column-name").focus(); + }); + + moveControls.bottomButton.addEventListener("click", function () { + var last = state.columnList.lastElementChild; + if ( + state.isSaving || + tableCreateRowIsPrimaryKey(row) || + !last || + last === row + ) { + return; + } + state.columnList.appendChild(row); + clearTableCreateDialogError(state); + updateTableCreateColumnRules(state); + row.querySelector(".table-create-column-name").focus(); }); return row; @@ -306,6 +828,7 @@ function createTableColumnRow(state, column) { function addTableCreateColumn(state, column) { var row = createTableColumnRow(state, column || { type: "text" }); state.columnList.appendChild(row); + updateTableCreateColumnRules(state); return row; } @@ -323,6 +846,7 @@ function resetTableCreateDialog(state) { type: "text", primaryKey: false, }); + updateTableCreateColumnRules(state); state.initialSignature = tableCreateDialogSignature(state); } @@ -332,16 +856,32 @@ function collectTableCreatePayload(state) { columns: [], }; var primaryKeys = []; - state.columnList - .querySelectorAll(".table-create-column-row") - .forEach(function (row) { - var name = row.querySelector(".table-create-column-name").value.trim(); - var type = row.querySelector(".table-create-column-type").value; - payload.columns.push({ name: name, type: type }); - if (row.querySelector(".table-create-primary-key-input").checked) { - primaryKeys.push(name); - } - }); + tableCreateDialogRows(state).forEach(function (row, index) { + var name = row.querySelector(".table-create-column-name").value.trim(); + var type = row.querySelector(".table-create-column-type").value; + var column = { name: name, type: type }; + var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); + var foreignKeyOption = + foreignKeySelect && foreignKeySelect.selectedOptions + ? foreignKeySelect.selectedOptions[0] + : null; + if ( + index > 0 && + foreignKeyOption && + foreignKeyOption.dataset.fkTable && + foreignKeyOption.dataset.fkColumn + ) { + column.fk_table = foreignKeyOption.dataset.fkTable; + column.fk_column = foreignKeyOption.dataset.fkColumn; + } + payload.columns.push(column); + if ( + index === 0 && + row.querySelector(".table-create-primary-key-input").checked + ) { + primaryKeys.push(name); + } + }); if (primaryKeys.length === 1) { payload.pk = primaryKeys[0]; } else if (primaryKeys.length > 1) { @@ -352,21 +892,19 @@ function collectTableCreatePayload(state) { function collectTableCreateColumnTypeAssignments(state) { var assignments = []; - state.columnList - .querySelectorAll(".table-create-column-row") - .forEach(function (row) { - var customTypeSelect = row.querySelector( - ".table-create-custom-column-type", - ); - if (!customTypeSelect || !customTypeSelect.value) { - return; - } - assignments.push({ - column: row.querySelector(".table-create-column-name").value.trim(), - columnType: customTypeSelect.value, - sqliteType: row.querySelector(".table-create-column-type").value, - }); + tableCreateDialogRows(state).forEach(function (row) { + var customTypeSelect = row.querySelector( + ".table-create-custom-column-type", + ); + if (!customTypeSelect || !customTypeSelect.value) { + return; + } + assignments.push({ + column: row.querySelector(".table-create-column-name").value.trim(), + columnType: customTypeSelect.value, + sqliteType: row.querySelector(".table-create-column-type").value, }); + }); return assignments; } @@ -604,9 +1142,14 @@ function ensureTableCreateDialog(manager) {
-
Columns
+
- +
@@ -633,6 +1176,9 @@ function ensureTableCreateDialog(manager) { isSaving: false, initialSignature: "", nextColumnIndex: 0, + foreignKeyTargets: [], + foreignKeyTargetsError: null, + foreignKeyTargetsLoading: false, manager: manager, }; @@ -713,6 +1259,7 @@ function openTableCreateDialog(button, manager) { state.title.textContent = "Create a table in " + data.databaseName; clearTableCreateDialogError(state); resetTableCreateDialog(state); + loadTableCreateForeignKeyTargets(state); if (!state.dialog.open) { state.dialog.showModal(); } @@ -877,9 +1424,7 @@ function tableAlterCustomTypeAppliesToSqliteType(option, sqliteType) { } function tableAlterDialogRows(state) { - return Array.prototype.slice.call( - state.columnList.querySelectorAll(".table-alter-column-row"), - ); + return schemaDialogRows(state, "table-alter"); } function tableAlterRowSignature(row) { @@ -930,76 +1475,19 @@ function updateTableAlterSaveButtonState(state) { } function tableAlterRowIsPrimaryKey(row) { - var input = row && row.querySelector(".table-alter-primary-key-input"); - return !!(input && input.checked); + return schemaDialogRowIsPrimaryKey(row, "table-alter"); } function tableAlterFirstNonPrimaryRow(state) { - var rows = tableAlterDialogRows(state); - for (var i = 0; i < rows.length; i += 1) { - if (!tableAlterRowIsPrimaryKey(rows[i])) { - return rows[i]; - } - } - return null; + return schemaDialogFirstNonPrimaryRow(state, "table-alter"); } function updateTableAlterMoveButtons(state) { - if (!state || !state.columnList) { - return; - } - var firstNonPrimary = tableAlterFirstNonPrimaryRow(state); - var rows = tableAlterDialogRows(state); - var hasPrimaryKeys = rows.some(function (row) { - return tableAlterRowIsPrimaryKey(row); - }); - var primaryKeyMoveTitle = "Primary key columns are always listed first"; - rows.forEach(function (row) { - var isPrimaryKey = tableAlterRowIsPrimaryKey(row); - var previous = row.previousElementSibling; - var next = row.nextElementSibling; - row - .querySelectorAll(".table-alter-move-controls button") - .forEach(function (button) { - button.title = button.dataset.defaultTitle || button.title; - button.disabled = state.isSaving || isPrimaryKey; - if (isPrimaryKey) { - button.title = primaryKeyMoveTitle; - } - }); - if (!isPrimaryKey) { - var topButton = row.querySelector(".table-alter-move-top"); - var upButton = row.querySelector(".table-alter-move-up"); - var downButton = row.querySelector(".table-alter-move-down"); - var bottomButton = row.querySelector(".table-alter-move-bottom"); - topButton.disabled = - state.isSaving || !firstNonPrimary || row === firstNonPrimary; - upButton.disabled = - state.isSaving || !previous || tableAlterRowIsPrimaryKey(previous); - downButton.disabled = state.isSaving || !next; - bottomButton.disabled = state.isSaving || !next; - if (hasPrimaryKeys && row === firstNonPrimary) { - topButton.title = primaryKeyMoveTitle; - upButton.title = primaryKeyMoveTitle; - } - } - }); + updateSchemaDialogMoveButtons(state, "table-alter"); } function normalizeTableAlterPrimaryKeyRows(state) { - var rows = tableAlterDialogRows(state); - rows - .filter(function (row) { - return tableAlterRowIsPrimaryKey(row); - }) - .concat( - rows.filter(function (row) { - return !tableAlterRowIsPrimaryKey(row); - }), - ) - .forEach(function (row) { - state.columnList.appendChild(row); - }); + normalizeSchemaDialogPrimaryKeyRows(state, "table-alter"); } function clearTableAlterDialogError(state) { @@ -1049,38 +1537,20 @@ function tableAlterSaveButtonText(state) { function tableAlterSelectTypeValue(select, type) { var options = tableAlterColumnTypes(); - options.forEach(function (option) { - var optionElement = document.createElement("option"); - optionElement.value = option; - optionElement.textContent = sqliteColumnTypeLabel(option); - select.appendChild(optionElement); - }); - select.value = options.indexOf(type) === -1 ? options[0] : type; + populateSqliteColumnTypeSelect(select, type, options); } function updateTableAlterCustomColumnTypePlaceholder(select) { - select.classList.toggle("table-alter-input-placeholder", !select.value); + updateSelectPlaceholder(select, "table-alter-input-placeholder"); } function createTableAlterCustomColumnTypeSelect() { var options = tableAlterCustomColumnTypes(); - var select = document.createElement("select"); - select.className = "table-alter-input table-alter-custom-column-type"; - select.setAttribute("aria-label", "Custom column type"); - var blankOption = document.createElement("option"); - blankOption.value = ""; - blankOption.textContent = "- custom type -"; - select.appendChild(blankOption); - options.forEach(function (option) { - var optionElement = document.createElement("option"); - optionElement.value = option.name; - optionElement.textContent = option.description - ? option.description + " (" + option.name + ")" - : option.name; - select.appendChild(optionElement); - }); - updateTableAlterCustomColumnTypePlaceholder(select); - return select; + return createCustomColumnTypeSelect( + options, + "table-alter-input table-alter-custom-column-type", + "table-alter-input-placeholder", + ); } function syncTableAlterCustomTypeForSqliteType(row) { @@ -1112,7 +1582,7 @@ function tableAlterSelectDefaultExprValue(select, value) { } function updateTableAlterDefaultExprPlaceholder(select) { - select.classList.toggle("table-alter-input-placeholder", !select.value); + updateSelectPlaceholder(select, "table-alter-input-placeholder"); } function syncTableAlterDefaultControls(row) { @@ -1159,22 +1629,10 @@ function createTableAlterColumnRow(state, column) { details.id = "table-alter-column-details-" + index; details.hidden = !(column && column.expanded); - var expandButton = document.createElement("button"); - expandButton.type = "button"; - expandButton.className = "table-alter-more-options"; - expandButton.setAttribute("aria-label", "Toggle column settings"); - expandButton.setAttribute("aria-controls", details.id); - expandButton.setAttribute("aria-expanded", details.hidden ? "false" : "true"); - function updateExpandButton() { - var isExpanded = expandButton.getAttribute("aria-expanded") === "true"; - expandButton.textContent = isExpanded - ? "v Hide options" - : "> Advanced options"; - expandButton.title = isExpanded - ? "Hide column settings" - : "Show column settings"; - } - updateExpandButton(); + var expandButton = createSchemaDialogMoreOptionsButton( + "table-alter", + details, + ); var nameId = "table-alter-column-name-" + index; var nameLabel = document.createElement("label"); @@ -1196,10 +1654,22 @@ function createTableAlterColumnRow(state, column) { tableAlterSelectTypeValue(typeSelect, column && column.type); var customTypeSelect = null; + var customTypeField = null; if (tableAlterCustomColumnTypes().length) { + var customTypeId = "table-alter-column-custom-type-" + index; + customTypeField = document.createElement("div"); + customTypeField.className = + "table-alter-detail-field table-alter-custom-type-field"; + var customTypeLabel = document.createElement("label"); + customTypeLabel.className = "table-alter-detail-label"; + customTypeLabel.setAttribute("for", customTypeId); + customTypeLabel.textContent = "Custom type"; customTypeSelect = createTableAlterCustomColumnTypeSelect(); + customTypeSelect.id = customTypeId; customTypeSelect.value = originalCustomType; updateTableAlterCustomColumnTypePlaceholder(customTypeSelect); + customTypeField.appendChild(customTypeLabel); + customTypeField.appendChild(customTypeSelect); } var notNullLabel = document.createElement("label"); @@ -1262,76 +1732,30 @@ function createTableAlterColumnRow(state, column) { pkStrong.textContent = "Primary key"; pkText.appendChild(pkStrong); pkText.appendChild( - document.createTextNode(" An ID that uniquely identifies this record"), + document.createTextNode(" This ID uniquely identifies the record"), ); pkLabel.appendChild(pkInput); pkLabel.appendChild(pkText); - var moveControls = document.createElement("div"); - moveControls.className = "table-alter-move-controls"; + var moveControls = createSchemaDialogMoveControls("table-alter"); - var moveTopButton = document.createElement("button"); - moveTopButton.type = "button"; - moveTopButton.className = "table-alter-icon-button table-alter-move-top"; - moveTopButton.setAttribute("aria-label", "Move column to top"); - moveTopButton.title = "Move column to top"; - moveTopButton.dataset.defaultTitle = moveTopButton.title; - moveTopButton.innerHTML = - ''; - - var moveUpButton = document.createElement("button"); - moveUpButton.type = "button"; - moveUpButton.className = "table-alter-icon-button table-alter-move-up"; - moveUpButton.setAttribute("aria-label", "Move column up"); - moveUpButton.title = "Move column up"; - moveUpButton.dataset.defaultTitle = moveUpButton.title; - moveUpButton.innerHTML = - ''; - - var moveDownButton = document.createElement("button"); - moveDownButton.type = "button"; - moveDownButton.className = "table-alter-icon-button table-alter-move-down"; - moveDownButton.setAttribute("aria-label", "Move column down"); - moveDownButton.title = "Move column down"; - moveDownButton.dataset.defaultTitle = moveDownButton.title; - moveDownButton.innerHTML = - ''; - - var moveBottomButton = document.createElement("button"); - moveBottomButton.type = "button"; - moveBottomButton.className = - "table-alter-icon-button table-alter-move-bottom"; - moveBottomButton.setAttribute("aria-label", "Move column to bottom"); - moveBottomButton.title = "Move column to bottom"; - moveBottomButton.dataset.defaultTitle = moveBottomButton.title; - moveBottomButton.innerHTML = - ''; - - var removeButton = document.createElement("button"); - removeButton.type = "button"; - removeButton.className = "table-alter-icon-button table-alter-remove-column"; - removeButton.setAttribute( - "aria-label", + var removeButton = createSchemaDialogIconButton( + "table-alter", + "remove-column", existing ? "Drop column" : "Remove column", + existing ? "Drop column" : "Remove column", + COLUMN_MOVE_ICONS.remove, ); - removeButton.title = existing ? "Drop column" : "Remove column"; - removeButton.innerHTML = - ''; - - moveControls.appendChild(moveTopButton); - moveControls.appendChild(moveUpButton); - moveControls.appendChild(moveDownButton); - moveControls.appendChild(moveBottomButton); main.appendChild(nameLabel); main.appendChild(nameInput); main.appendChild(typeSelect); - if (customTypeSelect) { - main.appendChild(customTypeSelect); - } - main.appendChild(moveControls); + main.appendChild(moveControls.controls); main.appendChild(removeButton); main.appendChild(expandButton); + if (customTypeField) { + details.appendChild(customTypeField); + } details.appendChild(notNullLabel); details.appendChild(defaultField); details.appendChild(defaultExprField); @@ -1382,10 +1806,7 @@ function createTableAlterColumnRow(state, column) { }); expandButton.addEventListener("click", function () { - var isExpanded = expandButton.getAttribute("aria-expanded") === "true"; - details.hidden = isExpanded; - expandButton.setAttribute("aria-expanded", isExpanded ? "false" : "true"); - updateExpandButton(); + toggleSchemaDialogMoreOptions(expandButton, details); }); typeSelect.addEventListener("change", function () { @@ -1407,7 +1828,7 @@ function createTableAlterColumnRow(state, column) { }); } - moveTopButton.addEventListener("click", function () { + moveControls.topButton.addEventListener("click", function () { var first = tableAlterFirstNonPrimaryRow(state); if ( state.isSaving || @@ -1424,7 +1845,7 @@ function createTableAlterColumnRow(state, column) { row.querySelector(".table-alter-column-name").focus(); }); - moveUpButton.addEventListener("click", function () { + moveControls.upButton.addEventListener("click", function () { var previous = row.previousElementSibling; if ( state.isSaving || @@ -1441,7 +1862,7 @@ function createTableAlterColumnRow(state, column) { row.querySelector(".table-alter-column-name").focus(); }); - moveDownButton.addEventListener("click", function () { + moveControls.downButton.addEventListener("click", function () { var next = row.nextElementSibling; if (state.isSaving || tableAlterRowIsPrimaryKey(row) || !next) { return; @@ -1453,7 +1874,7 @@ function createTableAlterColumnRow(state, column) { row.querySelector(".table-alter-column-name").focus(); }); - moveBottomButton.addEventListener("click", function () { + moveControls.bottomButton.addEventListener("click", function () { var last = state.columnList.lastElementChild; if ( state.isSaving || @@ -2145,12 +2566,11 @@ function ensureTableAlterDialog(manager) {
- +
From c77dad910b47c370c2b4c2cf16bebf3d74b51069 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Jun 2026 21:16:56 -0700 Subject: [PATCH 023/167] More robust test_datasette_https_server.sh test --- tests/test_datasette_https_server.sh | 31 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/test_datasette_https_server.sh b/tests/test_datasette_https_server.sh index aee262cc..a544b9a8 100755 --- a/tests/test_datasette_https_server.sh +++ b/tests/test_datasette_https_server.sh @@ -40,22 +40,23 @@ curl -f --cacert client.pem $test_url curl_exit_code=$? # Shut down the server -kill $server_pid -waiting=0 -# show all pids -# | find just the $server_pid -# | | don’t match on the previous grep -# | | | we don’t need the output -# | | | | -until ( ! ps ax | grep $server_pid | grep -v grep > /dev/null ); do - if [ $waiting -eq 4 ]; then - echo "$server_pid does still exist, server failed to stop" - cleanup - exit 1 +kill $server_pid 2>/dev/null || true +( + sleep 5 + if kill -0 $server_pid 2>/dev/null; then + kill -9 $server_pid 2>/dev/null || true fi - let waiting=waiting+1 - sleep 1 -done +) & +killer_pid=$! +wait_status=0 +wait $server_pid 2>/dev/null || wait_status=$? +kill $killer_pid 2>/dev/null || true +wait $killer_pid 2>/dev/null || true +if [ $wait_status -eq 137 ]; then + echo "$server_pid did not stop after SIGTERM, server failed to stop" + cleanup + exit 1 +fi # Clean up the certificates cleanup From e834008075eead04c142a57275dcaa3f08d44d38 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 19 Jun 2026 22:32:37 -0700 Subject: [PATCH 024/167] Make custom type and foreign key mutually exclusive In the create table dialog a column can now have either a custom display type or a foreign key target, but not both - a foreign key column's type is determined by the referenced primary key, so a custom type doesn't apply. Setting one clears and disables the other, and the foreign key select stays disabled on the primary key column and when no targets exist. Also add "Controls how Datasette displays and edits this column" help text (with aria-describedby) under the custom type selector in both the create and alter dialogs, and style the alter dialog help text. --- datasette/static/app.css | 3 +- datasette/static/edit-tools.js | 71 ++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 49f070b1..06919444 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1953,7 +1953,8 @@ dialog.table-create-dialog::backdrop { font-size: 0.72rem; } -.table-create-detail-help { +.table-create-detail-help, +.table-alter-detail-help { color: var(--muted); font-size: 0.82rem; line-height: 1.35; diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 5a9d0962..2a0a29b2 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -422,11 +422,42 @@ function syncTableCreateForeignKeyOptions(row, state) { ); } +function syncTableCreateCustomTypeAndForeignKey(row, state, isFirstColumn) { + var customTypeSelect = row.querySelector(".table-create-custom-column-type"); + var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); + if (!foreignKeySelect) { + return; + } + + var hasCustomType = customTypeSelect && !!customTypeSelect.value; + var hasForeignKey = !!foreignKeySelect.value; + + if (customTypeSelect && hasForeignKey) { + customTypeSelect.value = ""; + updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + hasCustomType = false; + } + + if (isFirstColumn || hasCustomType) { + foreignKeySelect.value = ""; + foreignKeySelect.dataset.selectedKey = ""; + updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); + hasForeignKey = false; + } + + if (customTypeSelect) { + customTypeSelect.disabled = state.isSaving; + } + foreignKeySelect.disabled = + state.isSaving || isFirstColumn || foreignKeySelect.options.length <= 1; +} + function refreshTableCreateForeignKeyControls(state) { tableCreateDialogRows(state).forEach(function (row, index) { if (index > 0) { syncTableCreateForeignKeyOptions(row, state); } + syncTableCreateCustomTypeAndForeignKey(row, state, index === 0); }); } @@ -455,6 +486,7 @@ function updateTableCreateColumnRules(state) { } else { syncTableCreateForeignKeyOptions(row, state); } + syncTableCreateCustomTypeAndForeignKey(row, state, isFirstColumn); } }); updateTableCreateMoveButtons(state); @@ -639,11 +671,19 @@ function createTableColumnRow(state, column) { customTypeLabel.className = "table-create-detail-label"; customTypeLabel.setAttribute("for", customTypeId); customTypeLabel.textContent = "Custom type"; + var customTypeHelpId = "table-create-column-custom-type-help-" + index; + var customTypeHelp = document.createElement("p"); + customTypeHelp.id = customTypeHelpId; + customTypeHelp.className = "table-create-detail-help"; + customTypeHelp.textContent = + "Controls how Datasette displays and edits this column"; customTypeSelect = createTableCustomColumnTypeSelect(); customTypeSelect.id = customTypeId; + customTypeSelect.setAttribute("aria-describedby", customTypeHelpId); customTypeSelect.value = column && column.customType ? column.customType : ""; updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); customTypeField.appendChild(customTypeLabel); + customTypeField.appendChild(customTypeHelp); customTypeField.appendChild(customTypeSelect); } @@ -733,11 +773,20 @@ function createTableColumnRow(state, column) { clearTableCreateDialogError(state); syncTableCreateCustomTypeForSqliteType(row); syncTableCreateForeignKeyOptions(row, state); + syncTableCreateCustomTypeAndForeignKey( + row, + state, + row === state.columnList.firstElementChild, + ); }); if (customTypeSelect) { customTypeSelect.addEventListener("change", function () { clearTableCreateDialogError(state); updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + if (customTypeSelect.value) { + foreignKeySelect.value = ""; + foreignKeySelect.dataset.selectedKey = ""; + } var option = tableCreateCustomColumnType(customTypeSelect.value); if ( option && @@ -747,6 +796,11 @@ function createTableColumnRow(state, column) { typeSelect.value = option.fixedSqliteType; syncTableCreateForeignKeyOptions(row, state); } + syncTableCreateCustomTypeAndForeignKey( + row, + state, + row === state.columnList.firstElementChild, + ); }); } pkInput.addEventListener("change", function () { @@ -757,6 +811,15 @@ function createTableColumnRow(state, column) { foreignKeySelect.dataset.selectedKey = foreignKeySelect.value; clearTableCreateDialogError(state); updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); + if (customTypeSelect && foreignKeySelect.value) { + customTypeSelect.value = ""; + updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + } + syncTableCreateCustomTypeAndForeignKey( + row, + state, + row === state.columnList.firstElementChild, + ); }); expandButton.addEventListener("click", function () { @@ -1664,11 +1727,19 @@ function createTableAlterColumnRow(state, column) { customTypeLabel.className = "table-alter-detail-label"; customTypeLabel.setAttribute("for", customTypeId); customTypeLabel.textContent = "Custom type"; + var customTypeHelpId = "table-alter-column-custom-type-help-" + index; + var customTypeHelp = document.createElement("p"); + customTypeHelp.id = customTypeHelpId; + customTypeHelp.className = "table-alter-detail-help"; + customTypeHelp.textContent = + "Controls how Datasette displays and edits this column"; customTypeSelect = createTableAlterCustomColumnTypeSelect(); customTypeSelect.id = customTypeId; + customTypeSelect.setAttribute("aria-describedby", customTypeHelpId); customTypeSelect.value = originalCustomType; updateTableAlterCustomColumnTypePlaceholder(customTypeSelect); customTypeField.appendChild(customTypeLabel); + customTypeField.appendChild(customTypeHelp); customTypeField.appendChild(customTypeSelect); } From 354780a1366bbfa6c5c9c34fdec3ccf24163e099 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 19 Jun 2026 22:40:14 -0700 Subject: [PATCH 025/167] Drop table button in alter dialog --- datasette/static/app.css | 16 ++++++++ datasette/static/edit-tools.js | 68 ++++++++++++++++++++++++++++++++++ datasette/views/table.py | 14 +++++++ tests/test_table_html.py | 14 ++++++- 4 files changed, 111 insertions(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 06919444..1cca8b80 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -2542,6 +2542,22 @@ dialog.table-alter-dialog::backdrop { color: var(--ink); } +.table-alter-dialog .btn-danger { + background: #b91c1c; + color: #fff; + margin-right: auto; +} + +.table-alter-dialog .btn-danger:hover { + background: #991b1b; +} + +.table-alter-dialog .btn-danger:disabled, +.table-alter-dialog .btn-danger:disabled:hover { + background: #d98c8c; + color: #fff; +} + .table-alter-dialog .btn-primary { background: var(--accent); color: #fff; diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 2a0a29b2..779ec702 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -1571,6 +1571,7 @@ function setTableAlterDialogSaving(state, isSaving) { state.cancelButton.disabled = isSaving; state.addColumnButton.disabled = isSaving; state.backButton.disabled = isSaving; + state.dropButton.disabled = isSaving; state.saveButton.textContent = isSaving ? state.mode === "review" ? "Applying..." @@ -2465,6 +2466,8 @@ function showTableAlterEditor(state) { state.review.hidden = true; state.review.textContent = ""; state.backButton.hidden = true; + var data = tableAlterData(); + state.dropButton.hidden = !(data && data.dropPath); state.saveButton.textContent = tableAlterSaveButtonText(state); updateTableAlterMoveButtons(state); updateTableAlterSaveButtonState(state); @@ -2479,6 +2482,7 @@ function showTableAlterReview(state, result) { state.review.hidden = false; state.review.textContent = ""; state.backButton.hidden = false; + state.dropButton.hidden = true; state.saveButton.textContent = tableAlterSaveButtonText(state); updateTableAlterSaveButtonState(state); @@ -2565,6 +2569,64 @@ async function applyTableAlterChanges(state, result) { } } +function tableAlterDatabaseUrl() { + var data = tableAlterData(); + if (!data || !data.path) { + return null; + } + var url = new URL(data.path, location.href); + url.pathname = url.pathname.replace(/\/[^/]+\/-\/alter\/?$/, ""); + url.search = ""; + url.hash = ""; + return url.toString(); +} + +async function dropTableFromAlterDialog(state) { + if (state.isSaving) { + return; + } + var data = tableAlterData(); + if (!data || !data.dropPath) { + return; + } + if ( + !window.confirm( + 'Permanently delete the table "' + + data.tableName + + '"? This will delete all of its data and cannot be undone.', + ) + ) { + return; + } + clearTableAlterDialogError(state); + setTableAlterDialogSaving(state, true); + try { + var response = await fetch(data.dropPath, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ confirm: true }), + }); + var responseData = null; + try { + responseData = await response.json(); + } catch (_error) { + responseData = null; + } + if (!response.ok || (responseData && responseData.ok === false)) { + throw rowMutationRequestError(response, responseData); + } + state.shouldRestoreFocus = false; + state.dialog.close(); + window.location.href = tableAlterDatabaseUrl() || "/"; + } catch (error) { + setTableAlterDialogSaving(state, false); + showTableAlterDialogError(state, error.message || "Could not drop table"); + } +} + async function saveTableAlterDialog(state) { if (state.isSaving) { return; @@ -2646,6 +2708,7 @@ function ensureTableAlterDialog(manager) {
+ @@ -2664,6 +2727,7 @@ function ensureTableAlterDialog(manager) { columnList: dialog.querySelector(".table-alter-column-list"), addColumnButton: dialog.querySelector(".table-alter-add-column"), backButton: dialog.querySelector(".table-alter-back"), + dropButton: dialog.querySelector(".table-alter-drop"), cancelButton: dialog.querySelector(".table-alter-cancel"), saveButton: dialog.querySelector(".table-alter-save"), currentButton: null, @@ -2703,6 +2767,10 @@ function ensureTableAlterDialog(manager) { closeTableAlterDialog(tableAlterDialogState); }); + tableAlterDialogState.dropButton.addEventListener("click", function () { + dropTableFromAlterDialog(tableAlterDialogState); + }); + tableAlterDialogState.backButton.addEventListener("click", function () { if (tableAlterDialogState.isSaving) { return; diff --git a/datasette/views/table.py b/datasette/views/table.py index 6c0d2914..4adcd181 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -422,6 +422,15 @@ async def _table_alter_ui( data["customColumnTypes"] = _custom_column_type_options_for_create_table( datasette ) + can_drop_table = await datasette.allowed( + action="drop-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ) + if can_drop_table: + data["dropPath"] = "{}/-/drop".format( + datasette.urls.table(database_name, table_name) + ) return data @@ -1193,6 +1202,11 @@ class TableDropView(BaseView): actor=request.actor, database=database_name, table=table_name ) ) + self.ds.add_message( + request, + "Table {} dropped".format(table_name), + self.ds.WARNING, + ) return Response.json({"ok": True}, status=200) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index fa01c8ec..4c80691c 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1089,8 +1089,9 @@ async def test_table_alter_action_button_and_data(): "tables": { "items": { "permissions": { - "alter-table": {"id": "root"}, + "alter-table": {"id": ["root", "alter-only"]}, "set-column-type": {"id": "root"}, + "drop-table": {"id": "root"}, }, "column_types": {"name": "textarea"}, }, @@ -1147,6 +1148,7 @@ async def test_table_alter_action_button_and_data(): "textarea", "url", ] + assert alter_data["dropPath"] == "/data/items/-/drop" assert alter_data["columns"] == [ { "name": "id", @@ -1192,6 +1194,16 @@ async def test_table_alter_action_button_and_data(): is None ) assert "alterTable" not in table_data_from_soup(soup_without_permission) + + # An actor that can alter but not drop should not get a dropPath + response_alter_only = await ds.client.get( + "/data/items", actor={"id": "alter-only"} + ) + assert response_alter_only.status_code == 200 + alter_only_data = table_data_from_soup( + Soup(response_alter_only.text, "html.parser") + )["alterTable"] + assert "dropPath" not in alter_only_data finally: ds.close() From a87c4ac555ed763b4e4e48716a7d4402dd439b99 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 09:47:39 -0700 Subject: [PATCH 026/167] Object not chain of ifs Refs https://github.com/simonw/datasette/pull/2789/changes#r3453964430 --- datasette/static/edit-tools.js | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 779ec702..4c6d7d10 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -60,17 +60,14 @@ function tableCreateColumnTypes() { : ["text", "integer", "float", "blob"]; } +var SQLITE_COLUMN_TYPE_LABELS = { + float: "floating point number", + real: "floating point number", + blob: "blob - binary data", +}; + function sqliteColumnTypeLabel(type) { - if (type === "float") { - return "floating point number"; - } - if (type === "real") { - return "floating point number"; - } - if (type === "blob") { - return "blob - binary data"; - } - return type; + return SQLITE_COLUMN_TYPE_LABELS[type] || type; } function populateSqliteColumnTypeSelect(select, type, options) { From b02999bca6a4cf9aa3712b3ba424f42b0cbee35b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 09:59:29 -0700 Subject: [PATCH 027/167] Fix broken Playwright tests --- tests/test_playwright.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 70b5a8c1..94dcbd64 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -303,6 +303,7 @@ def test_create_table_flow(page, datasette_server): dialog.locator(".table-create-add-column").click() dialog.locator(".table-create-column-name").nth(3).fill("metadata") dialog.locator(".table-create-column-type").nth(3).select_option("integer") + dialog.locator(".table-create-more-options").nth(3).click() dialog.locator(".table-create-custom-column-type").nth(3).select_option("json") assert dialog.locator(".table-create-column-type").nth(3).input_value() == "text" assert "table-create-input-placeholder" not in dialog.locator( @@ -361,7 +362,7 @@ def test_alter_table_flow(page, datasette_server): assert "Default value" in expanded_options_text assert "or default to a specific time" in expanded_options_text assert "Primary key" in expanded_options_text - assert "An ID that uniquely identifies this record" in expanded_options_text + assert "This ID uniquely identifies the record" in expanded_options_text dialog.locator(".table-alter-add-column").click() assert dialog.locator(".table-alter-save").is_enabled() From 17876ccf45b35a025c5fa0186427685629b66492 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 10:01:39 -0700 Subject: [PATCH 028/167] Keyword arguments for readability --- datasette/views/row.py | 16 ++++++++-------- datasette/views/table.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index 34773d48..e3575b6c 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -205,14 +205,14 @@ class RowView(DataView): ], "row_mutation_ui": any(row_action_permissions.values()), "table_page_data": await _table_page_data( - self.ds, - request, - db, - database, - table, - not is_table, - None, - None, + datasette=self.ds, + request=request, + db=db, + database_name=database, + table_name=table, + is_view=not is_table, + table_insert_ui=None, + table_alter_ui=None, ), "row_actions": row_actions, "top_row": make_slot_function( diff --git a/datasette/views/table.py b/datasette/views/table.py index 4adcd181..988d3808 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -2171,14 +2171,14 @@ async def table_view_data( data["table_insert_ui"] = table_insert_ui data["table_alter_ui"] = table_alter_ui data["table_page_data"] = await _table_page_data( - datasette, - request, - db, - database_name, - table_name, - is_view, - table_insert_ui, - table_alter_ui, + datasette=datasette, + request=request, + db=db, + database_name=database_name, + table_name=table_name, + is_view=is_view, + table_insert_ui=table_insert_ui, + table_alter_ui=table_alter_ui, ) return data, rows[:page_size], columns, expanded_columns, sql, next_url From c4aead65ee5a37a40f9fa8abb9fc45a03979e535 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 10:02:42 -0700 Subject: [PATCH 029/167] Ran prettier --- datasette/static/edit-tools.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 4c6d7d10..c4982e54 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -248,7 +248,9 @@ function updateSchemaDialogMoveButtons(state, prefix) { topButton.disabled = state.isSaving || !firstNonPrimary || row === firstNonPrimary; upButton.disabled = - state.isSaving || !previous || schemaDialogRowIsPrimaryKey(previous, prefix); + state.isSaving || + !previous || + schemaDialogRowIsPrimaryKey(previous, prefix); downButton.disabled = state.isSaving || !next; bottomButton.disabled = state.isSaving || !next; if (hasPrimaryKeys && row === firstNonPrimary) { @@ -464,7 +466,9 @@ function updateTableCreateColumnRules(state) { var pkLabel = row.querySelector(".table-create-primary-key"); var pkInput = row.querySelector(".table-create-primary-key-input"); var foreignKeyField = row.querySelector(".table-create-foreign-key-field"); - var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); + var foreignKeySelect = row.querySelector( + ".table-create-foreign-key-target", + ); if (pkLabel && pkInput) { pkLabel.hidden = !isFirstColumn; @@ -677,7 +681,8 @@ function createTableColumnRow(state, column) { customTypeSelect = createTableCustomColumnTypeSelect(); customTypeSelect.id = customTypeId; customTypeSelect.setAttribute("aria-describedby", customTypeHelpId); - customTypeSelect.value = column && column.customType ? column.customType : ""; + customTypeSelect.value = + column && column.customType ? column.customType : ""; updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); customTypeField.appendChild(customTypeLabel); customTypeField.appendChild(customTypeHelp); @@ -920,7 +925,9 @@ function collectTableCreatePayload(state) { var name = row.querySelector(".table-create-column-name").value.trim(); var type = row.querySelector(".table-create-column-type").value; var column = { name: name, type: type }; - var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); + var foreignKeySelect = row.querySelector( + ".table-create-foreign-key-target", + ); var foreignKeyOption = foreignKeySelect && foreignKeySelect.selectedOptions ? foreignKeySelect.selectedOptions[0] From 084df1fba2076eec2124309360e05bfa2a497a28 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 10:18:01 -0700 Subject: [PATCH 030/167] Removed the alter table dry run feature It works by doing conn.backup(memory_conn) which could use a lot of memory for a large database. --- datasette/views/table_create_alter.py | 17 ++---------- docs/json_api.rst | 5 +--- tests/test_api_write.py | 37 +++++++-------------------- 3 files changed, 12 insertions(+), 47 deletions(-) diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index f7775479..88bdafc4 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -22,7 +22,6 @@ from datasette.resources import DatabaseResource, TableResource from datasette.utils import ( escape_sqlite, get_outbound_foreign_keys, - sqlite3, table_column_details, ) from datasette.utils.asgi import NotFound, Response @@ -602,7 +601,6 @@ AlterTableOperation = Annotated[ class AlterTableRequest(_StrictPydanticModel): operations: list[AlterTableOperation] = Field(min_length=1) - dry_run: bool = False def _pydantic_errors(validation_error): @@ -1143,14 +1141,6 @@ class TableAlterView(BaseView): return _table_schema_from_conn(operation_conn, table_name) - if alter_request.dry_run: - memory_conn = sqlite3.connect(":memory:") - try: - conn.backup(memory_conn) - return before_schema, apply_operations(memory_conn) - finally: - memory_conn.close() - after_schema = apply_operations(conn) return before_schema, after_schema @@ -1162,7 +1152,7 @@ class TableAlterView(BaseView): return _error([str(e)], 400) altered = before_schema != after_schema - if altered and not alter_request.dry_run: + if altered: await self.ds.track_event( AlterTableEvent( request.actor, @@ -1189,10 +1179,7 @@ class TableAlterView(BaseView): "altered": altered, "schema": after_schema, "before_schema": before_schema, - "operations_applied": ( - 0 if alter_request.dry_run else len(alter_request.operations) - ), - "dry_run": alter_request.dry_run, + "operations_applied": len(alter_request.operations), }, status=200, ) diff --git a/docs/json_api.rst b/docs/json_api.rst index 1db46dd2..a42175f7 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2285,8 +2285,6 @@ The request body should include an ``operations`` array. Each operation has the ] } -Set ``"dry_run": true`` to validate the operations and return the schema that would be created without modifying the table. - Supported operations: * ``add_column`` adds a new column. ``args`` accepts ``name``, optional ``type`` of ``text``, ``integer``, ``float`` or ``blob``, optional ``not_null``, optional literal ``default`` and optional ``default_expr``. If ``not_null`` is ``true`` either a non-null ``default`` or ``default_expr`` is required. @@ -2316,8 +2314,7 @@ A successful response returns the new schema and the previous schema: "altered": true, "schema": "CREATE TABLE ...", "before_schema": "CREATE TABLE ...", - "operations_applied": 7, - "dry_run": false + "operations_applied": 7 } Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 1d16ad26..49f38cde 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -898,34 +898,6 @@ async def test_alter_table_operations(ds_write): assert event.after_schema == data["schema"] -@pytest.mark.asyncio -async def test_alter_table_dry_run(ds_write): - token = write_token(ds_write, permissions=["at"]) - db = ds_write.get_database("data") - response = await ds_write.client.post( - "/data/docs/-/alter", - json={ - "dry_run": True, - "operations": [ - {"op": "add_column", "args": {"name": "slug", "type": "text"}} - ], - }, - headers=_headers(token), - ) - assert response.status_code == 200, response.text - data = response.json() - assert data["ok"] is True - assert data["dry_run"] is True - assert data["altered"] is True - assert data["operations_applied"] == 0 - assert "slug" in data["schema"] - columns = ( - await db.execute("select name from pragma_table_info('docs') order by cid") - ).dicts() - assert [column["name"] for column in columns] == ["id", "title", "score", "age"] - assert last_event(ds_write) is None - - @pytest.mark.asyncio async def test_alter_table_foreign_key_operations(ds_write): token = write_token(ds_write, permissions=["at"]) @@ -1246,6 +1218,15 @@ async def test_alter_table_permission_denied(ds_write): @pytest.mark.parametrize( "body,expected_error", ( + ( + { + "dry_run": True, + "operations": [ + {"op": "add_column", "args": {"name": "slug", "type": "text"}} + ], + }, + "dry_run: Extra inputs are not permitted", + ), ( {"operations": [{"op": "add_column", "args": {"type": "text"}}]}, "operations.0.add_column.args.name: Field required", From a2e75967ce19445d2862ac40cd9306447f2b4524 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 11:03:49 -0700 Subject: [PATCH 031/167] sqlite-utils>=3.30,<4.0 So we don't get test failures from reformatted SQL. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 38776b2c..215b2cca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "PyYAML>=5.3", "mergedeep>=1.1.1", "itsdangerous>=1.1", - "sqlite-utils>=3.30", + "sqlite-utils>=3.30,<4.0", "asyncinject>=0.7", "setuptools", "pip", From 87354cf94e9921262d1fb50dfb9b25afd0cc028f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 11:04:19 -0700 Subject: [PATCH 032/167] not_null, default and default_exr support for create table API columns --- datasette/views/table_create_alter.py | 53 ++++++++++++------- docs/json_api.rst | 14 ++++- tests/test_api_write.py | 73 ++++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 21 deletions(-) diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 88bdafc4..d4259287 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -319,13 +319,28 @@ class _StrictPydanticModel(BaseModel): model_config = ConfigDict(extra="forbid") -class CreateTableColumn(BaseModel): - model_config = ConfigDict(extra="forbid") +class _DefaultArgsMixin(_StrictPydanticModel): + default: Any | None = None + default_expr: DefaultExpr | None = None + + @model_validator(mode="after") + def validate_default_fields(self): + has_default = "default" in self.model_fields_set + has_default_expr = "default_expr" in self.model_fields_set + if has_default and has_default_expr: + raise ValueError("default and default_expr cannot both be provided") + if has_default_expr and self.default_expr is None: + raise ValueError("default_expr cannot be null") + return self + + +class CreateTableColumn(_DefaultArgsMixin): name: Any = None type: Any = "text" fk_table: str | None = None fk_column: str | None = None + not_null: bool = False @model_validator(mode="after") def validate_column(self): @@ -450,21 +465,6 @@ class CreateTableRequest(_StrictPydanticModel): return foreign_keys or None -class _DefaultArgsMixin(_StrictPydanticModel): - default: Any | None = None - default_expr: DefaultExpr | None = None - - @model_validator(mode="after") - def validate_default_fields(self): - has_default = "default" in self.model_fields_set - has_default_expr = "default_expr" in self.model_fields_set - if has_default and has_default_expr: - raise ValueError("default and default_expr cannot both be provided") - if has_default_expr and self.default_expr is None: - raise ValueError("default_expr cannot be null") - return self - - class AddColumnArgs(_DefaultArgsMixin): name: str type: SqliteApiType = "text" @@ -753,16 +753,33 @@ class TableCreateView(BaseView): ) def create_table(conn): - table = sqlite_utils.Database(conn)[table_name] + db_for_write = sqlite_utils.Database(conn) + table = db_for_write[table_name] if rows: table.insert_all( rows, pk=pks or pk, ignore=ignore, replace=replace, alter=alter ) else: + not_null = [column.name for column in columns if column.not_null] + defaults = {} + for column in columns: + if "default_expr" in column.model_fields_set: + defaults[column.name] = _default_expression_sql( + column.default_expr + ) + elif ( + "default" in column.model_fields_set + and column.default is not None + ): + defaults[column.name] = _literal_default( + db_for_write, column.default + ) table.create( {column.name: column.type for column in columns}, pk=pks or pk, foreign_keys=create_request.foreign_keys, + not_null=not_null or None, + defaults=defaults or None, ) return table.schema diff --git a/docs/json_api.rst b/docs/json_api.rst index a42175f7..b2927e38 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -1968,7 +1968,14 @@ To create a table, make a ``POST`` to ``//-/create``. This requires th }, { "name": "title", - "type": "text" + "type": "text", + "not_null": true, + "default": "Untitled" + }, + { + "name": "created", + "type": "text", + "default_expr": "current_timestamp" } ], "pk": "id" @@ -1981,6 +1988,9 @@ The JSON here describes the table that will be created: - ``name`` is the name of the column. This is required. - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. + - ``not_null`` can be set to ``true`` to create this column with a ``NOT NULL`` constraint. + - ``default`` can be used to set a literal default value for this column. + - ``default_expr`` can be used instead of ``default`` to set a SQLite default expression. The supported values are ``current_timestamp``, ``current_date`` and ``current_time``. - ``fk_table`` can be used to create a single-column foreign key constraint referencing another table. ``fk_column`` is optional and can be used to specify the referenced column - if omitted, Datasette will use the single primary key of ``fk_table``. * ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. @@ -2028,7 +2038,7 @@ If the table is successfully created this will return a ``201`` status code and "table": "name_of_new_table", "table_url": "http://127.0.0.1:8001/data/name_of_new_table", "table_api_url": "http://127.0.0.1:8001/data/name_of_new_table.json", - "schema": "CREATE TABLE [name_of_new_table] (\n [id] INTEGER PRIMARY KEY,\n [title] TEXT\n)" + "schema": "CREATE TABLE [name_of_new_table] (\n [id] INTEGER PRIMARY KEY,\n [title] TEXT NOT NULL DEFAULT 'Untitled',\n [created] TEXT DEFAULT CURRENT_TIMESTAMP\n)" } .. _TableCreateView_example: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 49f38cde..32be8905 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1935,6 +1935,60 @@ async def test_create_table_with_foreign_key(ds_write): assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"] +@pytest.mark.asyncio +async def test_create_table_with_column_constraints(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "constrained", + "columns": [ + {"name": "id", "type": "integer"}, + { + "name": "title", + "type": "text", + "not_null": True, + "default": "Untitled", + }, + { + "name": "created", + "type": "text", + "default_expr": "current_timestamp", + }, + {"name": "score", "type": "integer", "default": 0}, + {"name": "literal_default", "type": "text", "default": "hello)"}, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201, response.text + data = response.json() + assert data["ok"] is True + assert "NOT NULL DEFAULT 'Untitled'" in data["schema"] + assert "DEFAULT CURRENT_TIMESTAMP" in data["schema"] + assert "DEFAULT 0" in data["schema"] + assert "DEFAULT 'hello)'" in data["schema"] + + db = ds_write.get_database("data") + columns = ( + await db.execute("select * from pragma_table_info('constrained') order by cid") + ).dicts() + assert [column["name"] for column in columns] == [ + "id", + "title", + "created", + "score", + "literal_default", + ] + assert columns[0]["pk"] == 1 + assert columns[1]["notnull"] == 1 + assert columns[1]["dflt_value"] == "'Untitled'" + assert columns[2]["dflt_value"] == "CURRENT_TIMESTAMP" + assert columns[3]["dflt_value"] == "0" + assert columns[4]["dflt_value"] == "'hello)'" + + @pytest.mark.asyncio @pytest.mark.parametrize( "column,expected_error", @@ -1947,9 +2001,26 @@ async def test_create_table_with_foreign_key(ds_write): {"name": "owner_id", "type": "integer", "fk_column": "id"}, "columns.0: fk_column requires fk_table", ), + ( + { + "name": "created", + "type": "text", + "default_expr": "datetime('now')", + }, + "columns.0.default_expr: Input should be 'current_timestamp', 'current_date' or 'current_time'", + ), + ( + { + "name": "created", + "type": "text", + "default": "x", + "default_expr": "current_timestamp", + }, + "columns.0: Value error, default and default_expr cannot both be provided", + ), ), ) -async def test_create_table_foreign_key_validation(ds_write, column, expected_error): +async def test_create_table_column_validation(ds_write, column, expected_error): token = write_token(ds_write) response = await ds_write.client.post( "/data/-/create", From 4b219be8bd14b445b1a48206050d2c34a6943185 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 12:02:51 -0700 Subject: [PATCH 033/167] Alter table API can now rename tables, refs #2788 Refs https://github.com/simonw/datasette/pull/2789#issuecomment-4771774289 --- datasette/views/table_create_alter.py | 53 +++++++++++++++++++++++---- docs/json_api.rst | 17 ++++++--- tests/test_api_write.py | 53 +++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 13 deletions(-) diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index d4259287..07f13e9d 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -476,6 +476,20 @@ class RenameColumnArgs(_StrictPydanticModel): to: str +class RenameTableArgs(_StrictPydanticModel): + to: str + + @field_validator("to") + @classmethod + def validate_table_name(cls, v): + if not TABLE_NAME_RE.match(v): + raise PydanticCustomError( + "alter_table_rename_table", + "Invalid table name", + ) + return v + + class AlterColumnArgs(_DefaultArgsMixin): name: str type: SqliteApiType | None = None @@ -548,6 +562,11 @@ class RenameColumnOperation(_StrictPydanticModel): args: RenameColumnArgs +class RenameTableOperation(_StrictPydanticModel): + op: Literal["rename_table"] + args: RenameTableArgs + + class AlterColumnOperation(_StrictPydanticModel): op: Literal["alter_column"] args: AlterColumnArgs @@ -587,6 +606,7 @@ AlterTableOperation = Annotated[ Union[ AddColumnOperation, RenameColumnOperation, + RenameTableOperation, AlterColumnOperation, DropColumnOperation, SetPrimaryKeyOperation, @@ -1042,10 +1062,12 @@ class TableAlterView(BaseView): def apply_operations(operation_conn): db_for_write = sqlite_utils.Database(operation_conn) table = db_for_write[table_name] + current_table_name = table_name add_columns = [] types = {} rename = {} + rename_table_to = None drop = set() not_null = {} defaults = {} @@ -1080,6 +1102,8 @@ class TableAlterView(BaseView): defaults[args.name] = _default_expression_sql( args.default_expr ) + elif operation.op == "rename_table": + rename_table_to = args.to elif operation.op == "rename_column": rename[args.name] = args.to elif operation.op == "alter_column": @@ -1155,14 +1179,27 @@ class TableAlterView(BaseView): drop_foreign_keys=drop_foreign_keys or None, foreign_keys=foreign_keys, ) + if ( + rename_table_to is not None + and rename_table_to != current_table_name + ): + operation_conn.execute( + "alter table {} rename to {}".format( + escape_sqlite(current_table_name), + escape_sqlite(rename_table_to), + ) + ) + current_table_name = rename_table_to - return _table_schema_from_conn(operation_conn, table_name) + return current_table_name, _table_schema_from_conn( + operation_conn, current_table_name + ) - after_schema = apply_operations(conn) - return before_schema, after_schema + after_table_name, after_schema = apply_operations(conn) + return before_schema, after_schema, after_table_name try: - before_schema, after_schema = await db.execute_write_fn( + before_schema, after_schema, after_table_name = await db.execute_write_fn( alter_table, request=request ) except Exception as e: @@ -1174,23 +1211,23 @@ class TableAlterView(BaseView): AlterTableEvent( request.actor, database=database_name, - table=table_name, + table=after_table_name, before_schema=before_schema, after_schema=after_schema, ) ) table_url = self.ds.absolute_url( - request, self.ds.urls.table(database_name, table_name) + request, self.ds.urls.table(database_name, after_table_name) ) table_api_url = self.ds.absolute_url( - request, self.ds.urls.table(database_name, table_name, format="json") + request, self.ds.urls.table(database_name, after_table_name, format="json") ) return Response.json( { "ok": True, "database": database_name, - "table": table_name, + "table": after_table_name, "table_url": table_url, "table_api_url": table_api_url, "altered": altered, diff --git a/docs/json_api.rst b/docs/json_api.rst index b2927e38..849a8acf 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2242,6 +2242,12 @@ The request body should include an ``operations`` array. Each operation has the "to": "headline" } }, + { + "op": "rename_table", + "args": { + "to": "published_posts" + } + }, { "op": "alter_column", "args": { @@ -2299,6 +2305,7 @@ Supported operations: * ``add_column`` adds a new column. ``args`` accepts ``name``, optional ``type`` of ``text``, ``integer``, ``float`` or ``blob``, optional ``not_null``, optional literal ``default`` and optional ``default_expr``. If ``not_null`` is ``true`` either a non-null ``default`` or ``default_expr`` is required. * ``rename_column`` renames a column. ``args`` accepts ``name`` and ``to``. +* ``rename_table`` renames the table. ``args`` accepts ``to``, the new table name. If combined with other operations, Datasette applies the column, primary key, foreign key and column order changes before renaming the table. * ``alter_column`` changes column properties. ``args`` accepts ``name`` and at least one of ``type``, ``not_null``, literal ``default`` or ``default_expr``. Passing ``"default": null`` removes an existing default. * ``drop_column`` drops a column. ``args`` accepts ``name``. * ``set_primary_key`` changes the table primary key. ``args`` accepts ``columns``, a list of one or more column names. @@ -2311,20 +2318,20 @@ Supported operations: For foreign key operations that omit ``fk_column``, the referenced ``fk_table`` must have a single-column primary key. Datasette will return an error if it cannot identify a single primary key column for that table. -A successful response returns the new schema and the previous schema: +A successful response returns the new schema and the previous schema. If the request used ``rename_table``, ``table``, ``table_url`` and ``table_api_url`` will use the new table name. Renaming a table through this endpoint triggers the :class:`~datasette.events.RenameTableEvent` event. .. code-block:: json { "ok": true, "database": "data", - "table": "posts", - "table_url": "http://127.0.0.1:8001/data/posts", - "table_api_url": "http://127.0.0.1:8001/data/posts.json", + "table": "published_posts", + "table_url": "http://127.0.0.1:8001/data/published_posts", + "table_api_url": "http://127.0.0.1:8001/data/published_posts.json", "altered": true, "schema": "CREATE TABLE ...", "before_schema": "CREATE TABLE ...", - "operations_applied": 7 + "operations_applied": 11 } Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 32be8905..ca8a46f5 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1,4 +1,5 @@ from datasette.app import Datasette +from datasette.events import RenameTableEvent from datasette.utils import sqlite3 from .utils import last_event import pytest @@ -898,6 +899,58 @@ async def test_alter_table_operations(ds_write): assert event.after_schema == data["schema"] +@pytest.mark.asyncio +async def test_alter_table_rename_table(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + before_schema = await db.execute_fn( + lambda conn: conn.execute( + "select sql from sqlite_master where type = 'table' and name = 'docs'" + ).fetchone()[0] + ) + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + {"op": "rename_table", "args": {"to": "documents"}}, + ] + }, + headers=_headers(token), + ) + + assert response.status_code == 200, response.text + data = response.json() + assert data["ok"] is True + assert data["database"] == "data" + assert data["table"] == "documents" + assert data["table_url"].endswith("/data/documents") + assert data["table_api_url"].endswith("/data/documents.json") + assert data["altered"] is True + assert data["operations_applied"] == 1 + assert data["before_schema"] == before_schema + assert 'CREATE TABLE "documents"' in data["schema"] + + tables = ( + await db.execute( + "select name from sqlite_master where type = 'table' order by name" + ) + ).dicts() + table_names = [table["name"] for table in tables] + assert "docs" not in table_names + assert "documents" in table_names + + rename_events = [ + event + for event in ds_write._tracked_events + if isinstance(event, RenameTableEvent) + ] + assert len(rename_events) == 1 + assert rename_events[0].database == "data" + assert rename_events[0].old_table == "docs" + assert rename_events[0].new_table == "documents" + + @pytest.mark.asyncio async def test_alter_table_foreign_key_operations(ds_write): token = write_token(ds_write, permissions=["at"]) From dada4de172c974f677c51855e8234ffb22c6259a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 12:35:34 -0700 Subject: [PATCH 034/167] Fix for Safari select box heights Refs https://github.com/simonw/datasette/pull/2789#issuecomment-4772241681 --- datasette/static/app.css | 12 ++++++++++++ tests/test_playwright.py | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index 1cca8b80..90d7e7e9 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1864,12 +1864,18 @@ dialog.table-create-dialog::backdrop { .table-create-input { box-sizing: border-box; min-width: 0; + min-height: 46px; border: 1px solid var(--rule); border-radius: 5px; padding: 8px 10px; color: var(--ink); background: #fff; font: inherit; + line-height: 1.35; +} + +select.table-create-input { + height: 46px; } .table-create-input-placeholder { @@ -2333,12 +2339,18 @@ dialog.table-alter-dialog::backdrop { .table-alter-input { box-sizing: border-box; min-width: 0; + min-height: 46px; border: 1px solid var(--rule); border-radius: 5px; padding: 8px 10px; color: var(--ink); background: #fff; font: inherit; + line-height: 1.35; +} + +select.table-alter-input { + height: 46px; } .table-alter-input-placeholder { diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 94dcbd64..eefec118 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -295,6 +295,15 @@ def test_create_table_flow(page, datasette_server): placeholder_select.locator("option:checked").inner_text() == "- custom type -" ) assert "table-create-input-placeholder" in placeholder_select.get_attribute("class") + assert dialog.locator(".table-create-column-main").first.evaluate("""node => { + const inputHeight = node.querySelector( + ".table-create-column-name" + ).getBoundingClientRect().height; + const selectHeight = node.querySelector( + ".table-create-column-type" + ).getBoundingClientRect().height; + return Math.abs(inputHeight - selectHeight) <= 1; + }""") dialog.locator('input[name="table"]').fill("playwright_created") dialog.locator(".table-create-column-name").nth(1).fill("title") dialog.locator(".table-create-add-column").click() @@ -340,6 +349,15 @@ def test_alter_table_flow(page, datasette_server): dialog.wait_for() assert dialog.locator(".modal-title").inner_text() == "Alter table projects" assert dialog.locator(".table-alter-save").is_disabled() + assert dialog.locator(".table-alter-column-main").first.evaluate("""node => { + const inputHeight = node.querySelector( + ".table-alter-column-name" + ).getBoundingClientRect().height; + const selectHeight = node.querySelector( + ".table-alter-column-type" + ).getBoundingClientRect().height; + return Math.abs(inputHeight - selectHeight) <= 1; + }""") type_options = dialog.locator(".table-alter-column-type").first.locator("option") assert type_options.all_inner_texts() == [ "text", From 063b04ad83f88f642053bd88c5368e9ad19f13af Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 12:47:02 -0700 Subject: [PATCH 035/167] Expose foreign key data to alter table UI Include current foreign key metadata in the alter table page data and allow the foreign-key-targets endpoint to be read by actors with alter-table permission for a specific table. Add API and HTML data tests for the new alter-table foreign key support. --- datasette/views/table.py | 18 ++++++++++++++++++ datasette/views/table_create_alter.py | 13 +++++++++++-- tests/test_api_write.py | 11 +++++++++++ tests/test_table_html.py | 6 ++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 988d3808..dbca3a18 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -382,6 +382,19 @@ async def _table_alter_ui( return None column_types_map = await datasette.get_column_types(database_name, table_name) + foreign_keys_by_column = {} + for fk in await db.foreign_keys_for_table(table_name): + other_column = fk["other_column"] + if other_column is None and await db.table_exists(fk["other_table"]): + other_pks = await db.primary_keys(fk["other_table"]) + if len(other_pks) == 1: + other_column = other_pks[0] + if other_column is None: + continue + foreign_keys_by_column[fk["column"]] = { + "fk_table": fk["other_table"], + "fk_column": other_column, + } columns = [] for column in await db.table_column_details(table_name): if column.hidden: @@ -397,6 +410,7 @@ async def _table_alter_ui( "default": column.default_value, "has_default": column.default_value is not None, "is_pk": column.name in pks, + "foreign_key": foreign_keys_by_column.get(column.name), "column_type": ( {"type": column_type.name, "config": column_type.config} if column_type is not None @@ -412,6 +426,10 @@ async def _table_alter_ui( "primaryKeys": pks, "columnTypes": ALTER_TABLE_COLUMN_TYPES, "defaultExpressions": list(DEFAULT_EXPR_SQL), + "foreignKeyTargetsPath": "{}/-/foreign-key-targets?table={}".format( + datasette.urls.database(database_name), + urllib.parse.quote(table_name, safe=""), + ), } can_set_column_type = await datasette.allowed( action="set-column-type", diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 07f13e9d..fa89aae4 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -867,11 +867,20 @@ class DatabaseForeignKeyTargetsView(BaseView): db = await self.ds.resolve_database(request) database_name = db.name - if not await self.ds.allowed( + table_name = request.args.get("table") + can_create_table = await self.ds.allowed( action="create-table", resource=DatabaseResource(database=database_name), actor=request.actor, - ): + ) + can_alter_table = False + if table_name and await db.table_exists(table_name): + can_alter_table = await self.ds.allowed( + action="alter-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ) + if not (can_create_table or can_alter_table): return _error(["Permission denied: need create-table"], 403) hidden_tables = await db.execute_fn( diff --git a/tests/test_api_write.py b/tests/test_api_write.py index ca8a46f5..6e8c4d4b 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1252,6 +1252,17 @@ async def test_foreign_key_targets_permission_denied(ds_write): } +@pytest.mark.asyncio +async def test_foreign_key_targets_allowed_for_alter_table(ds_write): + token = write_token(ds_write, permissions=["at"]) + response = await ds_write.client.get( + "/data/-/foreign-key-targets?table=docs", + headers=_headers(token), + ) + assert response.status_code == 200, response.text + assert response.json()["ok"] is True + + @pytest.mark.asyncio async def test_alter_table_permission_denied(ds_write): token = write_token(ds_write, permissions=["ir"]) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 4c80691c..ba9c03a5 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1137,6 +1137,9 @@ async def test_table_alter_action_button_and_data(): assert alter_data["tableName"] == "items" assert alter_data["primaryKeys"] == ["id"] assert alter_data["columnTypes"] == ["text", "integer", "float", "blob"] + assert alter_data["foreignKeyTargetsPath"] == ( + "/data/-/foreign-key-targets?table=items" + ) assert alter_data["defaultExpressions"] == [ "current_timestamp", "current_date", @@ -1158,6 +1161,7 @@ async def test_table_alter_action_button_and_data(): "default": None, "has_default": False, "is_pk": True, + "foreign_key": None, "column_type": None, }, { @@ -1168,6 +1172,7 @@ async def test_table_alter_action_button_and_data(): "default": None, "has_default": False, "is_pk": False, + "foreign_key": None, "column_type": {"type": "textarea", "config": None}, }, { @@ -1178,6 +1183,7 @@ async def test_table_alter_action_button_and_data(): "default": "5", "has_default": True, "is_pk": False, + "foreign_key": None, "column_type": None, }, ] From fa43aba30920eb268557897b2eb1d7d6a41acade Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 12:50:57 -0700 Subject: [PATCH 036/167] Unify create and alter table modal controls Share default value controls between the create and alter table dialogs and expose create-table default expressions to the frontend. Add create-table not-null/default handling and align the shared foreign key picker behavior across both dialogs. --- datasette/static/app.css | 60 ++- datasette/static/edit-tools.js | 716 ++++++++++++++++++++------ datasette/views/table_create_alter.py | 1 + tests/test_playwright.py | 99 +++- tests/test_table_html.py | 5 + 5 files changed, 709 insertions(+), 172 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 90d7e7e9..32b7a4c2 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1883,12 +1883,14 @@ select.table-create-input { } .table-create-foreign-key-target option, -.table-create-custom-column-type option { +.table-create-custom-column-type option, +.table-create-default-expr option { color: var(--ink); } .table-create-foreign-key-target option[value=""], -.table-create-custom-column-type option[value=""] { +.table-create-custom-column-type option[value=""], +.table-create-default-expr option[value=""] { color: var(--muted); } @@ -1978,11 +1980,44 @@ select.table-create-input { white-space: normal; } +.table-create-not-null, .table-create-primary-key, -.table-create-foreign-key-field { +.table-create-foreign-key-field, +.table-create-default-options { grid-column: 1 / -1; } +.table-create-default-options, +.table-alter-default-options { + border: 1px solid var(--rule); + border-radius: 5px; + background: #fff; + color: var(--ink); + min-width: 0; +} + +.table-create-default-options > summary, +.table-alter-default-options > summary { + cursor: pointer; + color: var(--accent); + font-size: 0.85rem; + padding: 8px 10px; +} + +.table-create-default-options > summary:focus, +.table-alter-default-options > summary:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 1px; +} + +.table-create-default-grid, +.table-alter-default-grid { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + gap: 12px 16px; + padding: 0 10px 10px; +} + .table-create-detail-check input { flex: 0 0 auto; margin: 0.15rem 0 0; @@ -2221,6 +2256,7 @@ dialog.table-alter-dialog::backdrop { overflow-y: auto; } + .table-alter-fields[hidden], .table-alter-dialog .modal-footer [hidden] { display: none; @@ -2358,12 +2394,14 @@ select.table-alter-input { } .table-alter-default-expr option, -.table-alter-custom-column-type option { +.table-alter-custom-column-type option, +.table-alter-foreign-key-target option { color: var(--ink); } .table-alter-default-expr option[value=""], -.table-alter-custom-column-type option[value=""] { +.table-alter-custom-column-type option[value=""], +.table-alter-foreign-key-target option[value=""] { color: var(--muted); } @@ -2410,7 +2448,9 @@ select.table-alter-input { } .table-alter-not-null, -.table-alter-primary-key { +.table-alter-primary-key, +.table-alter-foreign-key-field, +.table-alter-default-options { grid-column: 1 / -1; } @@ -2657,6 +2697,10 @@ select.table-alter-input { grid-template-columns: 1fr; } + .table-alter-default-grid { + grid-template-columns: 1fr; + } + .table-alter-dialog .modal-footer { padding-left: 18px; padding-right: 18px; @@ -2878,6 +2922,10 @@ select.table-alter-input { grid-template-columns: 1fr; } + .table-create-default-grid { + grid-template-columns: 1fr; + } + .table-create-dialog .modal-footer { padding-left: 18px; padding-right: 18px; diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index c4982e54..2e4331e6 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -60,6 +60,11 @@ function tableCreateColumnTypes() { : ["text", "integer", "float", "blob"]; } +function tableCreateDefaultExpressions() { + var data = databaseCreateTableData() || {}; + return data.defaultExpressions || []; +} + var SQLITE_COLUMN_TYPE_LABELS = { float: "floating point number", real: "floating point number", @@ -104,6 +109,112 @@ function createCustomColumnTypeSelect(options, className, placeholderClass) { return select; } +var DEFAULT_EXPRESSION_LABELS = { + current_timestamp: "Current timestamp in UTC, e.g. 2026-05-01 13:34:00", + current_date: "Current date in UTC, e.g. 2026-05-01", + current_time: "Current time in UTC, e.g. 13:34:00", +}; + +function defaultExpressionLabel(value) { + return DEFAULT_EXPRESSION_LABELS[value] || value.replace(/_/g, " "); +} + +function createDefaultExpressionSelect( + options, + className, + placeholderClass, + value, +) { + var select = document.createElement("select"); + select.className = className; + var blankOption = document.createElement("option"); + blankOption.value = ""; + blankOption.textContent = "- default expr -"; + select.appendChild(blankOption); + options.forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option; + optionElement.textContent = defaultExpressionLabel(option); + select.appendChild(optionElement); + }); + select.value = value || ""; + updateSelectPlaceholder(select, placeholderClass); + return select; +} + +function createSchemaDialogDefaultControls(prefix, index, expressions, column) { + var defaultDetails = document.createElement("details"); + defaultDetails.className = prefix + "-default-options"; + defaultDetails.open = !!( + column && + (column.defaultValue || column.defaultExpr) + ); + var summary = document.createElement("summary"); + summary.textContent = "Set a default value"; + defaultDetails.appendChild(summary); + + var defaultGrid = document.createElement("div"); + defaultGrid.className = prefix + "-default-grid"; + + var defaultExprId = prefix + "-column-default-expr-" + index; + var defaultExprField = document.createElement("div"); + defaultExprField.className = prefix + "-detail-field"; + var defaultExprLabel = document.createElement("label"); + defaultExprLabel.className = prefix + "-detail-label"; + defaultExprLabel.setAttribute("for", defaultExprId); + defaultExprLabel.textContent = "Default expression"; + var defaultExprSelect = createDefaultExpressionSelect( + expressions, + prefix + "-input " + prefix + "-default-expr", + prefix + "-input-placeholder", + column && column.defaultExpr, + ); + defaultExprSelect.id = defaultExprId; + defaultExprSelect.setAttribute("aria-label", "Default expression"); + defaultExprField.appendChild(defaultExprLabel); + defaultExprField.appendChild(defaultExprSelect); + + var defaultId = prefix + "-column-default-" + index; + var defaultField = document.createElement("div"); + defaultField.className = prefix + "-detail-field"; + var defaultLabel = document.createElement("label"); + defaultLabel.className = prefix + "-detail-label"; + defaultLabel.setAttribute("for", defaultId); + defaultLabel.textContent = "or default to a specific value"; + var defaultInput = document.createElement("input"); + defaultInput.id = defaultId; + defaultInput.className = prefix + "-input " + prefix + "-default"; + defaultInput.type = "text"; + defaultInput.autocomplete = "off"; + defaultInput.placeholder = "default"; + defaultInput.setAttribute("aria-label", "or default to a specific value"); + defaultInput.value = column && column.defaultValue ? column.defaultValue : ""; + defaultField.appendChild(defaultLabel); + defaultField.appendChild(defaultInput); + + defaultGrid.appendChild(defaultExprField); + defaultGrid.appendChild(defaultField); + defaultDetails.appendChild(defaultGrid); + + return { + controls: defaultDetails, + defaultInput: defaultInput, + defaultExprSelect: defaultExprSelect, + }; +} + +function syncSchemaDialogDefaultControls(row, prefix) { + if (!row) { + return; + } + var defaultInput = row.querySelector("." + prefix + "-default"); + var defaultExprSelect = row.querySelector("." + prefix + "-default-expr"); + if (!defaultInput || !defaultExprSelect) { + return; + } + updateSelectPlaceholder(defaultExprSelect, prefix + "-input-placeholder"); +} + var COLUMN_MOVE_ICONS = { top: '', up: '', @@ -316,7 +427,7 @@ function updateTableCreateMoveButtons(state) { updateSchemaDialogMoveButtons(state, "table-create"); } -function tableCreateTypeAffinity(type) { +function schemaDialogTypeAffinity(type) { if (type === "float") { return "real"; } @@ -337,11 +448,11 @@ function foreignKeyTypesCompatible(sourceAffinity, targetAffinity) { return false; } -function tableCreateForeignKeyTargetKey(target) { +function foreignKeyTargetKey(target) { return target.fk_table + "\u001f" + target.fk_column; } -function tableCreateForeignKeyTargetLabel(target) { +function foreignKeyTargetLabel(target) { return ( target.fk_table + "." + @@ -352,6 +463,54 @@ function tableCreateForeignKeyTargetLabel(target) { ); } +function appendForeignKeyTargetOption(select, target) { + var optionElement = document.createElement("option"); + optionElement.value = foreignKeyTargetKey(target); + optionElement.dataset.fkTable = target.fk_table; + optionElement.dataset.fkColumn = target.fk_column; + optionElement.dataset.fkType = target.type; + optionElement.textContent = foreignKeyTargetLabel(target); + select.appendChild(optionElement); + return optionElement; +} + +function sqliteColumnTypeForForeignKeyTarget(type) { + var affinity = schemaDialogTypeAffinity(type); + if (affinity === "real" || affinity === "numeric") { + return "float"; + } + if (["text", "integer", "blob"].indexOf(affinity) !== -1) { + return affinity; + } + return ""; +} + +function selectedSchemaDialogForeignKeyOption(foreignKeySelect) { + return foreignKeySelect && foreignKeySelect.selectedOptions + ? foreignKeySelect.selectedOptions[0] + : null; +} + +function setBlankSchemaDialogColumnNameFromForeignKey( + row, + prefix, + foreignKeyOption, +) { + var nameInput = row.querySelector("." + prefix + "-column-name"); + if ( + nameInput && + !nameInput.value.trim() && + foreignKeyOption && + foreignKeyOption.dataset.fkTable && + foreignKeyOption.dataset.fkColumn + ) { + nameInput.value = + foreignKeyOption.dataset.fkTable + + "_" + + foreignKeyOption.dataset.fkColumn; + } +} + function tableCreateForeignKeyTargetsUrl() { var data = databaseCreateTableData() || {}; if (data.foreignKeyTargetsPath) { @@ -363,7 +522,14 @@ function tableCreateForeignKeyTargetsUrl() { return data.path.replace(/\/-\/create$/, "/-/foreign-key-targets"); } -function populateTableCreateForeignKeySelect(select, state, sourceType) { +function populateSchemaDialogForeignKeySelect( + select, + state, + prefix, + sourceType, + options, +) { + options = options || {}; var previousKey = select.value || select.dataset.selectedKey || ""; select.textContent = ""; @@ -385,45 +551,64 @@ function populateTableCreateForeignKeySelect(select, state, sourceType) { errorOption.textContent = "Could not load foreign keys"; select.appendChild(errorOption); } else { - var sourceAffinity = tableCreateTypeAffinity(sourceType); + var sourceAffinity = schemaDialogTypeAffinity(sourceType); (state.foreignKeyTargets || []).forEach(function (target) { - if (!foreignKeyTypesCompatible(sourceAffinity, target.type)) { + if ( + options.filterByType !== false && + !foreignKeyTypesCompatible(sourceAffinity, target.type) + ) { return; } - var optionElement = document.createElement("option"); - optionElement.value = tableCreateForeignKeyTargetKey(target); - optionElement.dataset.fkTable = target.fk_table; - optionElement.dataset.fkColumn = target.fk_column; - optionElement.textContent = tableCreateForeignKeyTargetLabel(target); - select.appendChild(optionElement); + appendForeignKeyTargetOption(select, target); }); } select.value = previousKey; + if ( + previousKey && + select.value !== previousKey && + select.dataset.currentFkTable && + select.dataset.currentFkColumn + ) { + appendForeignKeyTargetOption(select, { + fk_table: select.dataset.currentFkTable, + fk_column: select.dataset.currentFkColumn, + type: select.dataset.currentFkType || sourceType, + }); + select.value = previousKey; + } if (select.value !== previousKey) { select.value = ""; } select.dataset.selectedKey = select.value; select.disabled = state.isSaving || select.options.length <= 1; - updateSelectPlaceholder(select, "table-create-input-placeholder"); + updateSelectPlaceholder(select, prefix + "-input-placeholder"); } -function syncTableCreateForeignKeyOptions(row, state) { - var typeSelect = row.querySelector(".table-create-column-type"); - var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); +function syncSchemaDialogForeignKeyOptions(row, state, prefix, options) { + var typeSelect = row.querySelector("." + prefix + "-column-type"); + var foreignKeySelect = row.querySelector( + "." + prefix + "-foreign-key-target", + ); if (!typeSelect || !foreignKeySelect) { return; } - populateTableCreateForeignKeySelect( + populateSchemaDialogForeignKeySelect( foreignKeySelect, state, + prefix, typeSelect.value, + options, ); } -function syncTableCreateCustomTypeAndForeignKey(row, state, isFirstColumn) { - var customTypeSelect = row.querySelector(".table-create-custom-column-type"); - var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); +function syncSchemaDialogCustomTypeAndForeignKey(row, state, prefix) { + var customTypeSelect = row.querySelector( + "." + prefix + "-custom-column-type", + ); + var foreignKeySelect = row.querySelector( + "." + prefix + "-foreign-key-target", + ); if (!foreignKeySelect) { return; } @@ -433,14 +618,14 @@ function syncTableCreateCustomTypeAndForeignKey(row, state, isFirstColumn) { if (customTypeSelect && hasForeignKey) { customTypeSelect.value = ""; - updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + updateSelectPlaceholder(customTypeSelect, prefix + "-input-placeholder"); hasCustomType = false; } - if (isFirstColumn || hasCustomType) { + if (hasCustomType) { foreignKeySelect.value = ""; foreignKeySelect.dataset.selectedKey = ""; - updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); + updateSelectPlaceholder(foreignKeySelect, prefix + "-input-placeholder"); hasForeignKey = false; } @@ -448,63 +633,73 @@ function syncTableCreateCustomTypeAndForeignKey(row, state, isFirstColumn) { customTypeSelect.disabled = state.isSaving; } foreignKeySelect.disabled = - state.isSaving || isFirstColumn || foreignKeySelect.options.length <= 1; + state.isSaving || foreignKeySelect.options.length <= 1; } -function refreshTableCreateForeignKeyControls(state) { - tableCreateDialogRows(state).forEach(function (row, index) { - if (index > 0) { - syncTableCreateForeignKeyOptions(row, state); - } - syncTableCreateCustomTypeAndForeignKey(row, state, index === 0); - }); -} +function handleSchemaDialogForeignKeyChange(row, state, prefix, options) { + options = options || {}; + var foreignKeySelect = row.querySelector( + "." + prefix + "-foreign-key-target", + ); + var typeSelect = row.querySelector("." + prefix + "-column-type"); + var customTypeSelect = row.querySelector( + "." + prefix + "-custom-column-type", + ); + if (!foreignKeySelect) { + return; + } + foreignKeySelect.dataset.selectedKey = foreignKeySelect.value; + updateSelectPlaceholder(foreignKeySelect, prefix + "-input-placeholder"); -function updateTableCreateColumnRules(state) { - tableCreateDialogRows(state).forEach(function (row, index) { - var isFirstColumn = index === 0; - var pkLabel = row.querySelector(".table-create-primary-key"); - var pkInput = row.querySelector(".table-create-primary-key-input"); - var foreignKeyField = row.querySelector(".table-create-foreign-key-field"); - var foreignKeySelect = row.querySelector( - ".table-create-foreign-key-target", + var foreignKeyOption = selectedSchemaDialogForeignKeyOption(foreignKeySelect); + setBlankSchemaDialogColumnNameFromForeignKey(row, prefix, foreignKeyOption); + + var columnTypes = options.columnTypes || []; + var foreignKeyColumnType = + foreignKeyOption && foreignKeyOption.dataset.fkType + ? sqliteColumnTypeForForeignKeyTarget(foreignKeyOption.dataset.fkType) + : ""; + if ( + options.matchType && + typeSelect && + foreignKeyColumnType && + columnTypes.indexOf(foreignKeyColumnType) !== -1 && + typeSelect.value !== foreignKeyColumnType + ) { + typeSelect.value = foreignKeyColumnType; + syncSchemaDialogForeignKeyOptions( + row, + state, + prefix, + options.foreignKeyOptions, ); + } - if (pkLabel && pkInput) { - pkLabel.hidden = !isFirstColumn; - if (!isFirstColumn) { - pkInput.checked = false; - } - } - - if (foreignKeyField && foreignKeySelect) { - foreignKeyField.hidden = isFirstColumn; - if (isFirstColumn) { - foreignKeySelect.value = ""; - foreignKeySelect.dataset.selectedKey = ""; - foreignKeySelect.disabled = true; - updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); - } else { - syncTableCreateForeignKeyOptions(row, state); - } - syncTableCreateCustomTypeAndForeignKey(row, state, isFirstColumn); - } - }); - updateTableCreateMoveButtons(state); + if (customTypeSelect && foreignKeySelect.value) { + customTypeSelect.value = ""; + updateSelectPlaceholder(customTypeSelect, prefix + "-input-placeholder"); + } + syncSchemaDialogCustomTypeAndForeignKey(row, state, prefix); } -async function loadTableCreateForeignKeyTargets(state) { - var url = tableCreateForeignKeyTargetsUrl(); +function refreshSchemaDialogForeignKeyControls(state, prefix, options) { + schemaDialogRows(state, prefix).forEach(function (row) { + syncSchemaDialogForeignKeyOptions(row, state, prefix, options); + syncSchemaDialogCustomTypeAndForeignKey(row, state, prefix); + }); +} + +async function loadSchemaDialogForeignKeyTargets(state, prefix, url, options) { if (!url || !window.fetch) { state.foreignKeyTargets = []; state.foreignKeyTargetsLoading = false; - refreshTableCreateForeignKeyControls(state); + refreshSchemaDialogForeignKeyControls(state, prefix, options); return; } state.foreignKeyTargets = []; state.foreignKeyTargetsError = null; state.foreignKeyTargetsLoading = true; - refreshTableCreateForeignKeyControls(state); + refreshSchemaDialogForeignKeyControls(state, prefix, options); try { var response = await fetch(url, { headers: { @@ -521,10 +716,46 @@ async function loadTableCreateForeignKeyTargets(state) { state.foreignKeyTargetsError = error; } finally { state.foreignKeyTargetsLoading = false; - refreshTableCreateForeignKeyControls(state); + refreshSchemaDialogForeignKeyControls(state, prefix, options); } } +function syncTableCreateForeignKeyOptions(row, state) { + syncSchemaDialogForeignKeyOptions(row, state, "table-create", { + filterByType: false, + }); +} + +function syncTableCreateCustomTypeAndForeignKey(row, state) { + syncSchemaDialogCustomTypeAndForeignKey(row, state, "table-create"); +} + +function refreshTableCreateForeignKeyControls(state) { + tableCreateDialogRows(state).forEach(function (row) { + syncTableCreateForeignKeyOptions(row, state); + syncTableCreateCustomTypeAndForeignKey(row, state); + }); +} + +function updateTableCreateColumnRules(state) { + normalizeSchemaDialogPrimaryKeyRows(state, "table-create"); + tableCreateDialogRows(state).forEach(function (row) { + syncTableCreateForeignKeyOptions(row, state); + syncTableCreateCustomTypeAndForeignKey(row, state); + syncSchemaDialogDefaultControls(row, "table-create"); + }); + updateTableCreateMoveButtons(state); +} + +async function loadTableCreateForeignKeyTargets(state) { + return loadSchemaDialogForeignKeyTargets( + state, + "table-create", + tableCreateForeignKeyTargetsUrl(), + { filterByType: false }, + ); +} + function tableCreateDialogSignature(state) { if (!state || !state.form) { return ""; @@ -542,6 +773,9 @@ function tableCreateDialogSignature(state) { } ).value || "", pk: row.querySelector(".table-create-primary-key-input").checked, + notNull: row.querySelector(".table-create-not-null-input").checked, + defaultValue: row.querySelector(".table-create-default").value, + defaultExpr: row.querySelector(".table-create-default-expr").value, foreignKey: ( row.querySelector(".table-create-foreign-key-target") || { @@ -654,6 +888,7 @@ function createTableColumnRow(state, column) { nameInput.type = "text"; nameInput.required = true; nameInput.autocomplete = "off"; + nameInput.placeholder = "column name"; nameInput.value = column && column.name ? column.name : ""; var typeSelect = document.createElement("select"); @@ -728,6 +963,32 @@ function createTableColumnRow(state, column) { foreignKeyField.appendChild(foreignKeyHelp); foreignKeyField.appendChild(foreignKeySelect); + var notNullLabel = document.createElement("label"); + notNullLabel.className = "table-create-detail-check table-create-not-null"; + var notNullInput = document.createElement("input"); + notNullInput.type = "checkbox"; + notNullInput.className = "table-create-not-null-input"; + notNullInput.checked = !!(column && column.notNull); + var notNullText = document.createElement("span"); + var notNullStrong = document.createElement("strong"); + notNullStrong.textContent = "Not null"; + notNullText.appendChild(notNullStrong); + notNullText.appendChild( + document.createTextNode(" This value cannot be left unset"), + ); + notNullLabel.appendChild(notNullInput); + notNullLabel.appendChild(notNullText); + + var defaultControls = createSchemaDialogDefaultControls( + "table-create", + index, + tableCreateDefaultExpressions(), + { + defaultValue: column && column.defaultValue, + defaultExpr: column && column.defaultExpr, + }, + ); + var moveControls = createSchemaDialogMoveControls("table-create"); var removeButton = createSchemaDialogIconButton( @@ -748,8 +1009,10 @@ function createTableColumnRow(state, column) { if (customTypeField) { details.appendChild(customTypeField); } - details.appendChild(foreignKeyField); + details.appendChild(defaultControls.controls); + details.appendChild(notNullLabel); details.appendChild(pkLabel); + details.appendChild(foreignKeyField); row.appendChild(main); row.appendChild(details); @@ -775,11 +1038,7 @@ function createTableColumnRow(state, column) { clearTableCreateDialogError(state); syncTableCreateCustomTypeForSqliteType(row); syncTableCreateForeignKeyOptions(row, state); - syncTableCreateCustomTypeAndForeignKey( - row, - state, - row === state.columnList.firstElementChild, - ); + syncTableCreateCustomTypeAndForeignKey(row, state); }); if (customTypeSelect) { customTypeSelect.addEventListener("change", function () { @@ -798,30 +1057,37 @@ function createTableColumnRow(state, column) { typeSelect.value = option.fixedSqliteType; syncTableCreateForeignKeyOptions(row, state); } - syncTableCreateCustomTypeAndForeignKey( - row, - state, - row === state.columnList.firstElementChild, - ); + syncTableCreateCustomTypeAndForeignKey(row, state); }); } pkInput.addEventListener("change", function () { clearTableCreateDialogError(state); updateTableCreateColumnRules(state); }); - foreignKeySelect.addEventListener("change", function () { - foreignKeySelect.dataset.selectedKey = foreignKeySelect.value; + notNullInput.addEventListener("change", function () { clearTableCreateDialogError(state); - updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); - if (customTypeSelect && foreignKeySelect.value) { - customTypeSelect.value = ""; - updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + }); + defaultControls.defaultInput.addEventListener("input", function () { + if (defaultControls.defaultInput.value) { + defaultControls.defaultExprSelect.value = ""; + syncSchemaDialogDefaultControls(row, "table-create"); } - syncTableCreateCustomTypeAndForeignKey( - row, - state, - row === state.columnList.firstElementChild, - ); + clearTableCreateDialogError(state); + }); + defaultControls.defaultExprSelect.addEventListener("change", function () { + if (defaultControls.defaultExprSelect.value) { + defaultControls.defaultInput.value = ""; + } + syncSchemaDialogDefaultControls(row, "table-create"); + clearTableCreateDialogError(state); + }); + foreignKeySelect.addEventListener("change", function () { + clearTableCreateDialogError(state); + handleSchemaDialogForeignKeyChange(row, state, "table-create", { + columnTypes: tableCreateColumnTypes(), + foreignKeyOptions: { filterByType: false }, + matchType: true, + }); }); expandButton.addEventListener("click", function () { @@ -887,6 +1153,7 @@ function createTableColumnRow(state, column) { row.querySelector(".table-create-column-name").focus(); }); + syncSchemaDialogDefaultControls(row, "table-create"); return row; } @@ -921,7 +1188,7 @@ function collectTableCreatePayload(state) { columns: [], }; var primaryKeys = []; - tableCreateDialogRows(state).forEach(function (row, index) { + tableCreateDialogRows(state).forEach(function (row) { var name = row.querySelector(".table-create-column-name").value.trim(); var type = row.querySelector(".table-create-column-type").value; var column = { name: name, type: type }; @@ -933,7 +1200,6 @@ function collectTableCreatePayload(state) { ? foreignKeySelect.selectedOptions[0] : null; if ( - index > 0 && foreignKeyOption && foreignKeyOption.dataset.fkTable && foreignKeyOption.dataset.fkColumn @@ -941,11 +1207,18 @@ function collectTableCreatePayload(state) { column.fk_table = foreignKeyOption.dataset.fkTable; column.fk_column = foreignKeyOption.dataset.fkColumn; } + if (row.querySelector(".table-create-not-null-input").checked) { + column.not_null = true; + } + var defaultExpr = row.querySelector(".table-create-default-expr").value; + var defaultValue = row.querySelector(".table-create-default").value; + if (defaultExpr) { + column.default_expr = defaultExpr; + } else if (defaultValue) { + column.default = defaultValue; + } payload.columns.push(column); - if ( - index === 0 && - row.querySelector(".table-create-primary-key-input").checked - ) { + if (row.querySelector(".table-create-primary-key-input").checked) { primaryKeys.push(name); } }); @@ -1006,6 +1279,9 @@ function validateTableCreatePayload(payload) { if (supportedTypes.indexOf(column.type) === -1) { return "Unsupported column type: " + column.type; } + if (column.default && column.default_expr) { + return "Use either a default value or a default expression."; + } } return null; } @@ -1467,6 +1743,11 @@ function tableAlterDefaultExpressions() { return data.defaultExpressions || []; } +function tableAlterForeignKeyTargetsUrl() { + var data = tableAlterData() || {}; + return data.foreignKeyTargetsPath || null; +} + function tableAlterCustomColumnTypes() { var data = tableAlterData() || {}; return data.customColumnTypes || []; @@ -1494,7 +1775,21 @@ function tableAlterDialogRows(state) { return schemaDialogRows(state, "table-alter"); } +function syncTableAlterForeignKeyOptions(row, state) { + syncSchemaDialogForeignKeyOptions(row, state, "table-alter", { + filterByType: false, + }); +} + function tableAlterRowSignature(row) { + var foreignKeySelect = row.querySelector(".table-alter-foreign-key-target"); + var foreignKeyValue = foreignKeySelect + ? foreignKeySelect.value || foreignKeySelect.dataset.selectedKey || "" + : ""; + var foreignKeyOption = + foreignKeySelect && foreignKeySelect.selectedOptions + ? foreignKeySelect.selectedOptions[0] + : null; return { existing: row.dataset.existing === "1", originalName: row.dataset.originalName || "", @@ -1510,6 +1805,15 @@ function tableAlterRowSignature(row) { defaultValue: row.querySelector(".table-alter-default").value, defaultExpr: row.querySelector(".table-alter-default-expr").value, pk: row.querySelector(".table-alter-primary-key-input").checked, + foreignKey: foreignKeyValue, + foreignKeyTable: + foreignKeyOption && foreignKeyOption.dataset.fkTable + ? foreignKeyOption.dataset.fkTable + : "", + foreignKeyColumn: + foreignKeyOption && foreignKeyOption.dataset.fkColumn + ? foreignKeyOption.dataset.fkColumn + : "", }; } @@ -1581,7 +1885,7 @@ function setTableAlterDialogSaving(state, isSaving) { ? "Applying..." : "Preparing..." : tableAlterSaveButtonText(state); - state.columnList + state.fields .querySelectorAll("input, select, button") .forEach(function (control) { control.disabled = isSaving; @@ -1590,10 +1894,12 @@ function setTableAlterDialogSaving(state, isSaving) { state.columnList .querySelectorAll(".table-alter-default-expr") .forEach(function (select) { - syncTableAlterDefaultControls( + syncSchemaDialogDefaultControls( select.closest(".table-alter-column-row"), + "table-alter", ); }); + refreshSchemaDialogForeignKeyControls(state, "table-alter"); } updateTableAlterMoveButtons(state); updateTableAlterSaveButtonState(state); @@ -1634,38 +1940,6 @@ function syncTableAlterCustomTypeForSqliteType(row) { } } -function tableAlterSelectDefaultExprValue(select, value) { - var blankOption = document.createElement("option"); - blankOption.value = ""; - blankOption.textContent = "- default expr -"; - select.appendChild(blankOption); - tableAlterDefaultExpressions().forEach(function (option) { - var optionElement = document.createElement("option"); - optionElement.value = option; - optionElement.textContent = option.replace(/_/g, " "); - select.appendChild(optionElement); - }); - select.value = value || ""; - updateTableAlterDefaultExprPlaceholder(select); -} - -function updateTableAlterDefaultExprPlaceholder(select) { - updateSelectPlaceholder(select, "table-alter-input-placeholder"); -} - -function syncTableAlterDefaultControls(row) { - if (!row) { - return; - } - var defaultInput = row.querySelector(".table-alter-default"); - var defaultExprSelect = row.querySelector(".table-alter-default-expr"); - if (!defaultInput || !defaultExprSelect) { - return; - } - defaultInput.disabled = !!defaultExprSelect.value; - updateTableAlterDefaultExprPlaceholder(defaultExprSelect); -} - function createTableAlterColumnRow(state, column) { var index = state.nextColumnIndex; state.nextColumnIndex += 1; @@ -1677,6 +1951,10 @@ function createTableAlterColumnRow(state, column) { existing && column.has_default && column.default !== null ? String(column.default) : ""; + var originalForeignKey = + existing && column.foreign_key + ? foreignKeyTargetKey(column.foreign_key) + : ""; var row = document.createElement("div"); row.className = "table-alter-column-row"; @@ -1688,6 +1966,7 @@ function createTableAlterColumnRow(state, column) { row.dataset.originalDefault = originalDefault; row.dataset.originalPk = existing && column.is_pk ? "1" : "0"; row.dataset.originalCustomType = originalCustomType; + row.dataset.originalForeignKey = originalForeignKey; var main = document.createElement("div"); main.className = "table-alter-column-main"; @@ -1714,6 +1993,7 @@ function createTableAlterColumnRow(state, column) { nameInput.type = "text"; nameInput.required = true; nameInput.autocomplete = "off"; + nameInput.placeholder = "column name"; nameInput.value = column && column.name ? column.name : ""; var typeSelect = document.createElement("select"); @@ -1764,38 +2044,51 @@ function createTableAlterColumnRow(state, column) { notNullLabel.appendChild(notNullInput); notNullLabel.appendChild(notNullText); - var defaultId = "table-alter-column-default-" + index; - var defaultField = document.createElement("div"); - defaultField.className = "table-alter-detail-field"; - var defaultLabel = document.createElement("label"); - defaultLabel.className = "table-alter-detail-label"; - defaultLabel.setAttribute("for", defaultId); - defaultLabel.textContent = "Default value"; - var defaultInput = document.createElement("input"); - defaultInput.id = defaultId; - defaultInput.className = "table-alter-input table-alter-default"; - defaultInput.type = "text"; - defaultInput.autocomplete = "off"; - defaultInput.placeholder = "default"; - defaultInput.setAttribute("aria-label", "Default value"); - defaultInput.value = originalDefault; - defaultField.appendChild(defaultLabel); - defaultField.appendChild(defaultInput); + var defaultControls = createSchemaDialogDefaultControls( + "table-alter", + index, + tableAlterDefaultExpressions(), + { + defaultValue: originalDefault, + defaultExpr: "", + }, + ); + var defaultInput = defaultControls.defaultInput; + var defaultExprSelect = defaultControls.defaultExprSelect; - var defaultExprId = "table-alter-column-default-expr-" + index; - var defaultExprField = document.createElement("div"); - defaultExprField.className = "table-alter-detail-field"; - var defaultExprLabel = document.createElement("label"); - defaultExprLabel.className = "table-alter-detail-label"; - defaultExprLabel.setAttribute("for", defaultExprId); - defaultExprLabel.textContent = "or default to a specific time"; - var defaultExprSelect = document.createElement("select"); - defaultExprSelect.id = defaultExprId; - defaultExprSelect.className = "table-alter-input table-alter-default-expr"; - defaultExprSelect.setAttribute("aria-label", "or default to a specific time"); - tableAlterSelectDefaultExprValue(defaultExprSelect, ""); - defaultExprField.appendChild(defaultExprLabel); - defaultExprField.appendChild(defaultExprSelect); + var foreignKeyId = "table-alter-column-foreign-key-" + index; + var foreignKeyHelpId = "table-alter-column-foreign-key-help-" + index; + var foreignKeyField = document.createElement("div"); + foreignKeyField.className = + "table-alter-detail-field table-alter-foreign-key-field"; + var foreignKeyLabel = document.createElement("label"); + foreignKeyLabel.className = "table-alter-detail-label"; + foreignKeyLabel.setAttribute("for", foreignKeyId); + foreignKeyLabel.textContent = "Foreign key"; + var foreignKeyHelp = document.createElement("p"); + foreignKeyHelp.id = foreignKeyHelpId; + foreignKeyHelp.className = "table-alter-detail-help"; + foreignKeyHelp.textContent = "Link this column to another table."; + var foreignKeySelect = document.createElement("select"); + foreignKeySelect.id = foreignKeyId; + foreignKeySelect.className = + "table-alter-input table-alter-foreign-key-target"; + foreignKeySelect.setAttribute("aria-label", "Foreign key target"); + foreignKeySelect.setAttribute("aria-describedby", foreignKeyHelpId); + foreignKeySelect.dataset.selectedKey = originalForeignKey; + if (column && column.foreign_key) { + foreignKeySelect.dataset.currentFkTable = column.foreign_key.fk_table; + foreignKeySelect.dataset.currentFkColumn = column.foreign_key.fk_column; + appendForeignKeyTargetOption(foreignKeySelect, { + fk_table: column.foreign_key.fk_table, + fk_column: column.foreign_key.fk_column, + type: column.type || "text", + }); + foreignKeySelect.value = originalForeignKey; + } + foreignKeyField.appendChild(foreignKeyLabel); + foreignKeyField.appendChild(foreignKeyHelp); + foreignKeyField.appendChild(foreignKeySelect); var pkLabel = document.createElement("label"); pkLabel.className = "table-alter-detail-check table-alter-primary-key"; @@ -1832,10 +2125,10 @@ function createTableAlterColumnRow(state, column) { if (customTypeField) { details.appendChild(customTypeField); } + details.appendChild(defaultControls.controls); details.appendChild(notNullLabel); - details.appendChild(defaultField); - details.appendChild(defaultExprField); details.appendChild(pkLabel); + details.appendChild(foreignKeyField); row.appendChild(main); row.appendChild(details); @@ -1846,6 +2139,7 @@ function createTableAlterColumnRow(state, column) { defaultInput, defaultExprSelect, pkInput, + foreignKeySelect, ]; if (customTypeSelect) { controls.push(customTypeSelect); @@ -1864,7 +2158,7 @@ function createTableAlterColumnRow(state, column) { defaultInput.addEventListener("input", function () { if (defaultInput.value) { defaultExprSelect.value = ""; - syncTableAlterDefaultControls(row); + syncSchemaDialogDefaultControls(row, "table-alter"); } updateTableAlterSaveButtonState(state); }); @@ -1872,7 +2166,7 @@ function createTableAlterColumnRow(state, column) { if (defaultExprSelect.value) { defaultInput.value = ""; } - syncTableAlterDefaultControls(row); + syncSchemaDialogDefaultControls(row, "table-alter"); updateTableAlterSaveButtonState(state); }); pkInput.addEventListener("change", function () { @@ -1887,6 +2181,8 @@ function createTableAlterColumnRow(state, column) { typeSelect.addEventListener("change", function () { syncTableAlterCustomTypeForSqliteType(row); + syncTableAlterForeignKeyOptions(row, state); + syncSchemaDialogCustomTypeAndForeignKey(row, state, "table-alter"); updateTableAlterSaveButtonState(state); }); if (customTypeSelect) { @@ -1899,10 +2195,20 @@ function createTableAlterColumnRow(state, column) { tableAlterColumnTypes().indexOf(option.fixedSqliteType) !== -1 ) { typeSelect.value = option.fixedSqliteType; + syncTableAlterForeignKeyOptions(row, state); } + syncSchemaDialogCustomTypeAndForeignKey(row, state, "table-alter"); updateTableAlterSaveButtonState(state); }); } + foreignKeySelect.addEventListener("change", function () { + handleSchemaDialogForeignKeyChange(row, state, "table-alter", { + columnTypes: tableAlterColumnTypes(), + foreignKeyOptions: { filterByType: false }, + matchType: true, + }); + updateTableAlterSaveButtonState(state); + }); moveControls.topButton.addEventListener("click", function () { var first = tableAlterFirstNonPrimaryRow(state); @@ -1986,7 +2292,9 @@ function createTableAlterColumnRow(state, column) { } }); - syncTableAlterDefaultControls(row); + syncSchemaDialogDefaultControls(row, "table-alter"); + syncTableAlterForeignKeyOptions(row, state); + syncSchemaDialogCustomTypeAndForeignKey(row, state, "table-alter"); return row; } @@ -2026,6 +2334,7 @@ function collectTableAlterRows(state) { signature.originalDefault = row.dataset.originalDefault || ""; signature.originalPk = row.dataset.originalPk === "1"; signature.originalCustomType = row.dataset.originalCustomType || ""; + signature.originalForeignKey = row.dataset.originalForeignKey || ""; return signature; }); } @@ -2156,6 +2465,46 @@ function tableAlterColumnsReordered(state, rows) { ); } +function tableAlterForeignKeyIdentity(row) { + return [ + row.name.trim(), + row.foreignKeyTable || "", + row.foreignKeyColumn || "", + ].join("\u001f"); +} + +function tableAlterOriginalForeignKeyIdentity(row) { + return [row.originalName || "", row.originalForeignKey].join("\u001f"); +} + +function tableAlterForeignKeyRows(rows) { + return rows + .filter(function (row) { + return row.foreignKey && row.foreignKeyTable && row.foreignKeyColumn; + }) + .map(function (row) { + return { + column: row.name.trim(), + fk_table: row.foreignKeyTable, + fk_column: row.foreignKeyColumn, + }; + }); +} + +function tableAlterForeignKeysChanged(rows) { + var original = rows + .filter(function (row) { + return row.existing && row.originalForeignKey; + }) + .map(tableAlterOriginalForeignKeyIdentity); + var final = rows + .filter(function (row) { + return row.foreignKey && row.foreignKeyTable && row.foreignKeyColumn; + }) + .map(tableAlterForeignKeyIdentity); + return JSON.stringify(original) !== JSON.stringify(final); +} + function collectTableAlterPayload(state) { var rows = collectTableAlterRows(state); var validationError = validateTableAlterRows(state, rows); @@ -2237,6 +2586,15 @@ function collectTableAlterPayload(state) { }); } + if (tableAlterForeignKeysChanged(rows)) { + operations.push({ + op: "set_foreign_keys", + args: { + foreign_keys: tableAlterForeignKeyRows(rows), + }, + }); + } + if (!operations.length && !columnTypeAssignments.length) { return { error: "No changes to apply." }; } @@ -2354,6 +2712,27 @@ function tableAlterOperationSummary(operation) { damaging: false, }; } + if (operation.op === "set_foreign_keys") { + var foreignKeys = args.foreign_keys || []; + return { + text: foreignKeys.length + ? "Set foreign keys to " + + foreignKeys + .map(function (foreignKey) { + return ( + tableAlterQuotedName(foreignKey.column) + + " -> " + + foreignKey.fk_table + + "." + + foreignKey.fk_column + ); + }) + .join(", ") + + "." + : "Remove all foreign keys.", + damaging: false, + }; + } return { text: "Run " + operation.op + ".", damaging: false, @@ -2742,6 +3121,9 @@ function ensureTableAlterDialog(manager) { deletedColumns: [], originalColumnNames: [], originalPrimaryKeys: [], + foreignKeyTargets: [], + foreignKeyTargetsError: null, + foreignKeyTargetsLoading: false, mode: "edit", reviewResult: null, manager: manager, @@ -2844,6 +3226,12 @@ function openTableAlterDialog(button, manager) { state.title.textContent = "Alter table " + data.tableName; clearTableAlterDialogError(state); resetTableAlterDialog(state, data); + loadSchemaDialogForeignKeyTargets( + state, + "table-alter", + tableAlterForeignKeyTargetsUrl(), + { filterByType: false }, + ); if (!state.dialog.open) { state.dialog.showModal(); } diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index fa89aae4..111dfab1 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -266,6 +266,7 @@ async def _create_table_ui_context( ), "databaseName": database_name, "columnTypes": CREATE_TABLE_COLUMN_TYPES, + "defaultExpressions": list(DEFAULT_EXPR_SQL), } can_set_column_type = await datasette.allowed( action="set-column-type", diff --git a/tests/test_playwright.py b/tests/test_playwright.py index eefec118..5719039a 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -295,6 +295,10 @@ def test_create_table_flow(page, datasette_server): placeholder_select.locator("option:checked").inner_text() == "- custom type -" ) assert "table-create-input-placeholder" in placeholder_select.get_attribute("class") + assert ( + dialog.locator(".table-create-column-name").nth(0).get_attribute("placeholder") + == "column name" + ) assert dialog.locator(".table-create-column-main").first.evaluate("""node => { const inputHeight = node.querySelector( ".table-create-column-name" @@ -306,6 +310,22 @@ def test_create_table_flow(page, datasette_server): }""") dialog.locator('input[name="table"]').fill("playwright_created") dialog.locator(".table-create-column-name").nth(1).fill("title") + dialog.locator(".table-create-more-options").nth(1).click() + dialog.locator(".table-create-not-null-input").nth(1).check() + title_defaults = dialog.locator(".table-create-default-options").nth(1) + assert title_defaults.locator("summary").inner_text() == "Set a default value" + title_defaults.locator("summary").click() + assert "or default to a specific value" in title_defaults.inner_text() + title_default_expr = title_defaults.locator(".table-create-default-expr") + title_default_input = title_defaults.locator(".table-create-default") + assert ( + "Current timestamp in UTC, e.g. 2026-05-01 13:34:00" + in title_default_expr.locator("option").nth(1).inner_text() + ) + title_default_expr.select_option("current_timestamp") + assert title_default_input.is_enabled() + title_default_input.fill("Untitled") + assert title_default_expr.input_value() == "" dialog.locator(".table-create-add-column").click() dialog.locator(".table-create-column-name").nth(2).fill("score") dialog.locator(".table-create-column-type").nth(2).select_option("integer") @@ -337,6 +357,63 @@ def test_create_table_flow(page, datasette_server): assert data["column_types"] == { "metadata": {"type": "json", "config": None}, } + schema_response = httpx.get( + f"{datasette_server}data/-/query.json", + params={ + "sql": ( + "select sql from sqlite_master where type = 'table' " + "and name = 'playwright_created'" + ) + }, + ) + schema_response.raise_for_status() + schema = schema_response.json()["rows"][0]["sql"] + assert "title" in schema + assert "NOT NULL DEFAULT 'Untitled'" in schema + + +@pytest.mark.playwright +def test_create_table_foreign_key_selection_updates_column_type(page, datasette_server): + page.goto(f"{datasette_server}data") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-database-action="create-table"]').click() + + dialog = page.locator("#table-create-dialog") + dialog.wait_for() + dialog.locator(".table-create-more-options").nth(1).click() + + column_name = dialog.locator(".table-create-column-name").nth(1) + type_select = dialog.locator(".table-create-column-type").nth(1) + foreign_key_select = dialog.locator(".table-create-foreign-key-target").nth(1) + assert column_name.input_value() == "" + assert type_select.input_value() == "text" + + foreign_key_select.select_option("projects\u001fid") + assert column_name.input_value() == "projects_id" + assert type_select.input_value() == "integer" + assert foreign_key_select.input_value() == "projects\u001fid" + + +@pytest.mark.playwright +def test_alter_table_foreign_key_selection_updates_blank_column(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + dialog.locator(".table-alter-add-column").click() + + column_name = dialog.locator(".table-alter-column-name").last + type_select = dialog.locator(".table-alter-column-type").last + foreign_key_select = dialog.locator(".table-alter-foreign-key-target").last + assert column_name.input_value() == "" + assert type_select.input_value() == "text" + + foreign_key_select.select_option("projects\u001fid") + assert column_name.input_value() == "projects_id" + assert type_select.input_value() == "integer" + assert foreign_key_select.input_value() == "projects\u001fid" @pytest.mark.playwright @@ -349,6 +426,10 @@ def test_alter_table_flow(page, datasette_server): dialog.wait_for() assert dialog.locator(".modal-title").inner_text() == "Alter table projects" assert dialog.locator(".table-alter-save").is_disabled() + assert ( + dialog.locator(".table-alter-column-name").first.get_attribute("placeholder") + == "column name" + ) assert dialog.locator(".table-alter-column-main").first.evaluate("""node => { const inputHeight = node.querySelector( ".table-alter-column-name" @@ -377,15 +458,29 @@ def test_alter_table_flow(page, datasette_server): ) assert "Not null" in expanded_options_text assert "This value cannot be left unset" in expanded_options_text - assert "Default value" in expanded_options_text - assert "or default to a specific time" in expanded_options_text + assert "Set a default value" in expanded_options_text assert "Primary key" in expanded_options_text assert "This ID uniquely identifies the record" in expanded_options_text + assert "Foreign key" in expanded_options_text + first_defaults = dialog.locator(".table-alter-default-options").first + first_defaults.locator("summary").click() + assert "or default to a specific value" in first_defaults.inner_text() + first_default_expr = first_defaults.locator(".table-alter-default-expr") + first_default_input = first_defaults.locator(".table-alter-default") + assert ( + "Current timestamp in UTC, e.g. 2026-05-01 13:34:00" + in first_default_expr.locator("option").nth(1).inner_text() + ) + first_default_expr.select_option("current_timestamp") + assert first_default_input.is_enabled() + first_default_input.fill("manual") + assert first_default_expr.input_value() == "" dialog.locator(".table-alter-add-column").click() assert dialog.locator(".table-alter-save").is_enabled() dialog.locator(".table-alter-column-name").last.fill("status") dialog.locator(".table-alter-column-type").last.select_option("text") + dialog.locator(".table-alter-default-options").last.locator("summary").click() dialog.locator(".table-alter-default").last.fill("planned") dialog.locator(".table-alter-save").click() review = dialog.locator(".table-alter-review") diff --git a/tests/test_table_html.py b/tests/test_table_html.py index ba9c03a5..9d1d1245 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -997,6 +997,11 @@ async def test_database_create_table_action_button_and_data(): "foreignKeyTargetsPath": "/data/-/foreign-key-targets", "databaseName": "data", "columnTypes": ["text", "integer", "float", "blob"], + "defaultExpressions": [ + "current_timestamp", + "current_date", + "current_time", + ], }, } assert "customColumnTypes" not in database_data_from_soup(soup)["createTable"] From 2ebae5ed71887cfd65af03aa88514170bdf4af17 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 12:51:23 -0700 Subject: [PATCH 037/167] Add rename table controls to alter table dialog Add a collapsed rename-table section to the alter table modal and include rename_table operations in the review/apply flow. Redirect to the renamed table URL after applying changes and cover the review text in Playwright. --- datasette/static/app.css | 22 ++++++++++ datasette/static/edit-tools.js | 75 +++++++++++++++++++++++++++++++--- tests/test_playwright.py | 30 ++++++++++++++ 3 files changed, 121 insertions(+), 6 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 32b7a4c2..f7cd97b0 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -2256,6 +2256,28 @@ dialog.table-alter-dialog::backdrop { overflow-y: auto; } +.table-alter-table-options { + border-top: 1px solid var(--rule); + padding-top: 12px; +} + +.table-alter-table-options > summary { + color: var(--ink); + cursor: pointer; + font-weight: 600; +} + +.table-alter-table-options > summary:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 3px; +} + +.table-alter-table-name-field { + display: grid; + gap: 4px; + margin-top: 10px; + max-width: 24rem; +} .table-alter-fields[hidden], .table-alter-dialog .modal-footer [hidden] { diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 2e4331e6..fd8d9a0c 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -1822,6 +1822,7 @@ function tableAlterDialogSignature(state) { return ""; } return JSON.stringify({ + tableName: state.tableNameInput ? state.tableNameInput.value.trim() : "", columns: tableAlterDialogRows(state).map(tableAlterRowSignature), deletedColumns: state.deletedColumns.slice(), }); @@ -2307,6 +2308,9 @@ function addTableAlterColumn(state, column) { function resetTableAlterDialog(state, data) { state.nextColumnIndex = 0; state.deletedColumns = []; + state.originalTableName = data.tableName || ""; + state.tableNameInput.value = state.originalTableName; + state.tableOptions.open = false; state.originalPrimaryKeys = (data.primaryKeys || []).slice(); state.originalColumnNames = (data.columns || []).map(function (column) { return column.name; @@ -2340,6 +2344,16 @@ function collectTableAlterRows(state) { } function validateTableAlterRows(state, rows) { + var tableName = state.tableNameInput.value.trim(); + if (!tableName) { + return "Table name is required."; + } + if (tableName.indexOf("\n") !== -1) { + return "Table names cannot contain newlines."; + } + if (/^sqlite_/.test(tableName)) { + return "Table names cannot start with sqlite_."; + } if (!rows.length) { return "At least one column is required."; } @@ -2513,6 +2527,13 @@ function collectTableAlterPayload(state) { } var operations = []; + var tableName = state.tableNameInput.value.trim(); + if (tableName !== state.originalTableName) { + operations.push({ + op: "rename_table", + args: { to: tableName }, + }); + } var columnTypeAssignments = collectTableAlterColumnTypeAssignments(rows); rows.forEach(function (row) { var name = row.name.trim(); @@ -2621,6 +2642,12 @@ function tableAlterReadableValue(value) { function tableAlterOperationSummary(operation) { var args = operation.args || {}; + if (operation.op === "rename_table") { + return { + text: "Rename table to " + tableAlterQuotedName(args.to) + ".", + damaging: false, + }; + } if (operation.op === "add_column") { var addDetails = ["as " + args.type]; if (args.not_null) { @@ -2788,7 +2815,10 @@ function appendTableAlterReviewText(element, text) { }); } -function tableAlterSetColumnTypeUrl() { +function tableAlterSetColumnTypeUrl(tableUrl) { + if (tableUrl) { + return tableUrl.replace(/\/$/, "") + "/-/set-column-type"; + } var data = tableAlterData(); if (!data || !data.path) { return null; @@ -2798,11 +2828,11 @@ function tableAlterSetColumnTypeUrl() { return url.toString(); } -async function assignTableAlterColumnTypes(assignments) { +async function assignTableAlterColumnTypes(assignments, tableUrl) { if (!assignments.length) { return; } - var url = tableAlterSetColumnTypeUrl(); + var url = tableAlterSetColumnTypeUrl(tableUrl); if (!url) { throw new Error("Could not find the set column type URL."); } @@ -2841,6 +2871,16 @@ async function assignTableAlterColumnTypes(assignments) { } } +function tableAlterResultRenamesTable(result) { + return !!( + result && + result.payload && + (result.payload.operations || []).some(function (operation) { + return operation.op === "rename_table"; + }) + ); +} + function showTableAlterEditor(state) { state.mode = "edit"; state.reviewResult = null; @@ -2923,6 +2963,7 @@ async function applyTableAlterChanges(state, result) { } setTableAlterDialogSaving(state, true); try { + var responseData = null; if (result.payload) { var response = await fetch(data.path, { method: "POST", @@ -2932,7 +2973,6 @@ async function applyTableAlterChanges(state, result) { }, body: JSON.stringify(result.payload), }); - var responseData = null; try { responseData = await response.json(); } catch (_error) { @@ -2942,10 +2982,18 @@ async function applyTableAlterChanges(state, result) { throw rowMutationRequestError(response, responseData); } } - await assignTableAlterColumnTypes(result.columnTypeAssignments || []); + var tableUrl = responseData && responseData.table_url; + await assignTableAlterColumnTypes( + result.columnTypeAssignments || [], + tableUrl, + ); state.shouldRestoreFocus = false; state.dialog.close(); - location.reload(); + if (tableAlterResultRenamesTable(result) && tableUrl) { + window.location.href = tableUrl; + } else { + location.reload(); + } } catch (error) { setTableAlterDialogSaving(state, false); showTableAlterDialogError(state, error.message || "Could not alter table"); @@ -3088,6 +3136,13 @@ function ensureTableAlterDialog(manager) {
+
+ Rename table +
+ + +
+
@@ -3106,6 +3161,8 @@ function ensureTableAlterDialog(manager) { title: dialog.querySelector(".modal-title"), error: dialog.querySelector(".table-alter-error"), fields: dialog.querySelector(".table-alter-fields"), + tableOptions: dialog.querySelector(".table-alter-table-options"), + tableNameInput: dialog.querySelector(".table-alter-table-name"), review: dialog.querySelector(".table-alter-review"), columnList: dialog.querySelector(".table-alter-column-list"), addColumnButton: dialog.querySelector(".table-alter-add-column"), @@ -3117,6 +3174,7 @@ function ensureTableAlterDialog(manager) { shouldRestoreFocus: true, isSaving: false, initialSignature: "", + originalTableName: "", nextColumnIndex: 0, deletedColumns: [], originalColumnNames: [], @@ -3134,6 +3192,11 @@ function ensureTableAlterDialog(manager) { saveTableAlterDialog(tableAlterDialogState); }); + tableAlterDialogState.tableNameInput.addEventListener("input", function () { + clearTableAlterDialogError(tableAlterDialogState); + updateTableAlterSaveButtonState(tableAlterDialogState); + }); + tableAlterDialogState.addColumnButton.addEventListener("click", function () { if (tableAlterDialogState.isSaving) { return; diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 5719039a..fcf97a90 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -561,6 +561,36 @@ def test_alter_table_review_rename_primary_key_column(page, datasette_server): ] +@pytest.mark.playwright +def test_alter_table_review_rename_table(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + save = dialog.locator(".table-alter-save") + rename_details = dialog.locator(".table-alter-table-options") + assert rename_details.locator("summary").inner_text() == "Rename table" + assert not dialog.locator(".table-alter-table-name").is_visible() + assert save.is_disabled() + + rename_details.locator("summary").click() + table_name = dialog.locator(".table-alter-table-name") + assert table_name.input_value() == "projects" + assert table_name.get_attribute("placeholder") == "table name" + table_name.fill("projects_archive") + assert save.is_enabled() + save.click() + + review = dialog.locator(".table-alter-review") + review.wait_for() + assert "Rename table to projects_archive." in review.inner_text() + assert dialog.locator(".table-alter-review-name").all_inner_texts() == [ + "projects_archive", + ] + + @pytest.mark.playwright def test_alter_table_review_not_null_wording(page, datasette_server): page.goto(f"{datasette_server}data/projects") From 1717ab02f720286aea8e4634a1047e6a6f2a1d45 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 22 Jun 2026 13:22:56 -0700 Subject: [PATCH 038/167] Replace row edit icon SVG in table.py Closes #2796 --- datasette/views/table.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index c5448c85..0e291524 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -455,8 +455,8 @@ async def display_columns_and_rows( 'xmlns="http://www.w3.org/2000/svg" width="14" height="14" ' 'viewBox="0 0 24 24" fill="none" stroke="currentColor" ' 'stroke-width="2" stroke-linecap="round" stroke-linejoin="round">' - '' - '' + '' + '' "" ) delete_icon = ( From b932d0dc78a3ddc466409a6d21a31124a20d8c17 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 13:42:35 -0700 Subject: [PATCH 039/167] current_unixtime and current_unixtime_ms default_expr options Plus tweaked how alter table changing those works a bit. --- datasette/static/edit-tools.js | 119 +++++++++++++++++++++++--- datasette/views/table.py | 41 +++++---- datasette/views/table_create_alter.py | 116 +++++++++++++++++++++++-- docs/json_api.rst | 32 ++++++- tests/test_api_write.py | 119 +++++++++++++++++++++++++- tests/test_playwright.py | 81 ++++++++++++++++++ tests/test_table_html.py | 69 ++++++++++++--- 7 files changed, 522 insertions(+), 55 deletions(-) diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index fd8d9a0c..9f4f89b9 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -109,14 +109,64 @@ function createCustomColumnTypeSelect(options, className, placeholderClass) { return select; } -var DEFAULT_EXPRESSION_LABELS = { - current_timestamp: "Current timestamp in UTC, e.g. 2026-05-01 13:34:00", - current_date: "Current date in UTC, e.g. 2026-05-01", - current_time: "Current time in UTC, e.g. 13:34:00", -}; +function normalizeDefaultExpressionOption(option) { + if (typeof option === "string") { + return { + value: option, + label: option.replace(/_/g, " "), + sqliteType: "", + }; + } + option = option || {}; + return { + value: option.value || "", + label: option.label || option.value || "", + sqliteType: option.sqliteType || "", + }; +} -function defaultExpressionLabel(value) { - return DEFAULT_EXPRESSION_LABELS[value] || value.replace(/_/g, " "); +function defaultExpressionOptionForValue(options, value) { + var match = null; + (options || []).some(function (option) { + var normalized = normalizeDefaultExpressionOption(option); + if (normalized.value === value) { + match = normalized; + return true; + } + return false; + }); + return match; +} + +function defaultExpressionLabelForValue(options, value) { + var option = defaultExpressionOptionForValue(options, value); + return option && option.label + ? option.label + : value + ? value.replace(/_/g, " ") + : ""; +} + +function applyDefaultExpressionColumnType(row, prefix, options, columnTypes) { + var defaultExprSelect = row.querySelector("." + prefix + "-default-expr"); + var typeSelect = row.querySelector("." + prefix + "-column-type"); + if (!defaultExprSelect || !typeSelect || !defaultExprSelect.value) { + return false; + } + var option = defaultExpressionOptionForValue( + options, + defaultExprSelect.value, + ); + if ( + option && + option.sqliteType && + (columnTypes || []).indexOf(option.sqliteType) !== -1 && + typeSelect.value !== option.sqliteType + ) { + typeSelect.value = option.sqliteType; + return true; + } + return false; } function createDefaultExpressionSelect( @@ -132,9 +182,13 @@ function createDefaultExpressionSelect( blankOption.textContent = "- default expr -"; select.appendChild(blankOption); options.forEach(function (option) { + var normalized = normalizeDefaultExpressionOption(option); + if (!normalized.value) { + return; + } var optionElement = document.createElement("option"); - optionElement.value = option; - optionElement.textContent = defaultExpressionLabel(option); + optionElement.value = normalized.value; + optionElement.textContent = normalized.label; select.appendChild(optionElement); }); select.value = value || ""; @@ -1078,6 +1132,18 @@ function createTableColumnRow(state, column) { if (defaultControls.defaultExprSelect.value) { defaultControls.defaultInput.value = ""; } + if ( + applyDefaultExpressionColumnType( + row, + "table-create", + tableCreateDefaultExpressions(), + tableCreateColumnTypes(), + ) + ) { + syncTableCreateCustomTypeForSqliteType(row); + syncTableCreateForeignKeyOptions(row, state); + syncTableCreateCustomTypeAndForeignKey(row, state); + } syncSchemaDialogDefaultControls(row, "table-create"); clearTableCreateDialogError(state); }); @@ -1948,8 +2014,15 @@ function createTableAlterColumnRow(state, column) { var originalName = existing ? column.name || "" : ""; var originalCustomType = existing && column.column_type ? column.column_type.type || "" : ""; + var originalDefaultExpr = + existing && column.has_default && column.default_expr + ? column.default_expr + : ""; var originalDefault = - existing && column.has_default && column.default !== null + existing && + column.has_default && + !originalDefaultExpr && + column.default !== null ? String(column.default) : ""; var originalForeignKey = @@ -1965,6 +2038,7 @@ function createTableAlterColumnRow(state, column) { row.dataset.originalNotNull = existing && column.notnull ? "1" : "0"; row.dataset.originalHasDefault = existing && column.has_default ? "1" : "0"; row.dataset.originalDefault = originalDefault; + row.dataset.originalDefaultExpr = originalDefaultExpr; row.dataset.originalPk = existing && column.is_pk ? "1" : "0"; row.dataset.originalCustomType = originalCustomType; row.dataset.originalForeignKey = originalForeignKey; @@ -2051,7 +2125,7 @@ function createTableAlterColumnRow(state, column) { tableAlterDefaultExpressions(), { defaultValue: originalDefault, - defaultExpr: "", + defaultExpr: originalDefaultExpr, }, ); var defaultInput = defaultControls.defaultInput; @@ -2167,6 +2241,18 @@ function createTableAlterColumnRow(state, column) { if (defaultExprSelect.value) { defaultInput.value = ""; } + if ( + applyDefaultExpressionColumnType( + row, + "table-alter", + tableAlterDefaultExpressions(), + tableAlterColumnTypes(), + ) + ) { + syncTableAlterCustomTypeForSqliteType(row); + syncTableAlterForeignKeyOptions(row, state); + syncSchemaDialogCustomTypeAndForeignKey(row, state, "table-alter"); + } syncSchemaDialogDefaultControls(row, "table-alter"); updateTableAlterSaveButtonState(state); }); @@ -2336,6 +2422,7 @@ function collectTableAlterRows(state) { signature.originalNotNull = row.dataset.originalNotNull === "1"; signature.originalHasDefault = row.dataset.originalHasDefault === "1"; signature.originalDefault = row.dataset.originalDefault || ""; + signature.originalDefaultExpr = row.dataset.originalDefaultExpr || ""; signature.originalPk = row.dataset.originalPk === "1"; signature.originalCustomType = row.dataset.originalCustomType || ""; signature.originalForeignKey = row.dataset.originalForeignKey || ""; @@ -2567,8 +2654,12 @@ function collectTableAlterPayload(state) { if (row.notNull !== row.originalNotNull) { alterArgs.not_null = row.notNull; } - if (row.defaultExpr) { - alterArgs.default_expr = row.defaultExpr; + if (row.defaultExpr !== row.originalDefaultExpr) { + if (row.defaultExpr) { + alterArgs.default_expr = row.defaultExpr; + } else { + alterArgs.default = row.defaultValue === "" ? null : row.defaultValue; + } } else if (row.originalHasDefault) { if (row.defaultValue !== row.originalDefault) { alterArgs.default = row.defaultValue === "" ? null : row.defaultValue; @@ -2630,7 +2721,7 @@ function tableAlterQuotedName(name) { } function tableAlterReadableDefaultExpression(value) { - return value ? value.replace(/_/g, " ") : ""; + return defaultExpressionLabelForValue(tableAlterDefaultExpressions(), value); } function tableAlterReadableValue(value) { diff --git a/datasette/views/table.py b/datasette/views/table.py index dbca3a18..80e1c514 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -51,8 +51,9 @@ from .database import QueryView from .table_create_alter import ( ALTER_TABLE_COLUMN_TYPES, ALTER_TABLE_TYPE_FOR_SQLITE_TYPE, - DEFAULT_EXPR_SQL, _custom_column_type_options_for_create_table, + default_expr_for_sql, + default_expression_options, ) from .table_extras import ( TABLE_EXTRA_BUNDLES, @@ -401,23 +402,25 @@ async def _table_alter_ui( continue sqlite_type = SQLiteType.from_declared_type(column.type) column_type = column_types_map.get(column.name) - columns.append( - { - "name": column.name, - "type": ALTER_TABLE_TYPE_FOR_SQLITE_TYPE.get(sqlite_type, "text"), - "sqlite_type": sqlite_type.value, - "notnull": column.notnull, - "default": column.default_value, - "has_default": column.default_value is not None, - "is_pk": column.name in pks, - "foreign_key": foreign_keys_by_column.get(column.name), - "column_type": ( - {"type": column_type.name, "config": column_type.config} - if column_type is not None - else None - ), - } - ) + default_expr = default_expr_for_sql(column.default_value) + column_data = { + "name": column.name, + "type": ALTER_TABLE_TYPE_FOR_SQLITE_TYPE.get(sqlite_type, "text"), + "sqlite_type": sqlite_type.value, + "notnull": column.notnull, + "default": None if default_expr else column.default_value, + "has_default": column.default_value is not None, + "is_pk": column.name in pks, + "foreign_key": foreign_keys_by_column.get(column.name), + "column_type": ( + {"type": column_type.name, "config": column_type.config} + if column_type is not None + else None + ), + } + if default_expr: + column_data["default_expr"] = default_expr + columns.append(column_data) data = { "path": "{}/-/alter".format(datasette.urls.table(database_name, table_name)), @@ -425,7 +428,7 @@ async def _table_alter_ui( "columns": columns, "primaryKeys": pks, "columnTypes": ALTER_TABLE_COLUMN_TYPES, - "defaultExpressions": list(DEFAULT_EXPR_SQL), + "defaultExpressions": default_expression_options(), "foreignKeyTargetsPath": "{}/-/foreign-key-targets?table={}".format( datasette.urls.database(database_name), urllib.parse.quote(table_name, safe=""), diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 111dfab1..ffeb7f14 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -266,7 +266,7 @@ async def _create_table_ui_context( ), "databaseName": database_name, "columnTypes": CREATE_TABLE_COLUMN_TYPES, - "defaultExpressions": list(DEFAULT_EXPR_SQL), + "defaultExpressions": default_expression_options(), } can_set_column_type = await datasette.allowed( action="set-column-type", @@ -308,12 +308,109 @@ def _custom_column_type_options_for_create_table(datasette): SqliteApiType = Literal["text", "integer", "float", "blob"] -DefaultExpr = Literal["current_timestamp", "current_date", "current_time"] -DEFAULT_EXPR_SQL = { - "current_timestamp": "CURRENT_TIMESTAMP", - "current_date": "CURRENT_DATE", - "current_time": "CURRENT_TIME", +DEFAULT_EXPRESSIONS = { + "current_timestamp": { + "sql": "CURRENT_TIMESTAMP", + "label": "Current timestamp in UTC, e.g. 2026-05-01 13:34:00", + "sqliteType": "text", + }, + "current_date": { + "sql": "CURRENT_DATE", + "label": "Current date in UTC, e.g. 2026-05-01", + "sqliteType": "text", + }, + "current_time": { + "sql": "CURRENT_TIME", + "label": "Current time in UTC, e.g. 13:34:00", + "sqliteType": "text", + }, + "current_unixtime": { + "sql": "(CAST(strftime('%s', 'now') AS INTEGER))", + "label": "Current Unix time, integer seconds since the epoch", + "sqliteType": "integer", + }, + "current_unixtime_ms": { + "sql": "(CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER))", + "label": "Current Unix time, integer milliseconds since the epoch", + "sqliteType": "integer", + }, } +DefaultExpr = str +DEFAULT_EXPR_SQL = { + name: metadata["sql"] for name, metadata in DEFAULT_EXPRESSIONS.items() +} + + +def _strip_wrapping_parentheses(expression): + expression = expression.strip() + while expression.startswith("(") and expression.endswith(")"): + depth = 0 + in_single_quote = False + wraps_whole_expression = True + i = 0 + while i < len(expression): + char = expression[i] + if char == "'": + if ( + in_single_quote + and i + 1 < len(expression) + and expression[i + 1] == "'" + ): + i += 2 + continue + in_single_quote = not in_single_quote + elif not in_single_quote: + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0 and i != len(expression) - 1: + wraps_whole_expression = False + break + i += 1 + if not wraps_whole_expression or depth != 0 or in_single_quote: + break + expression = expression[1:-1].strip() + return expression + + +def _default_expression_lookup_key(expression): + return re.sub(r"\s+", " ", _strip_wrapping_parentheses(expression)).lower() + + +DEFAULT_EXPR_BY_SQL = { + _default_expression_lookup_key(sql): name for name, sql in DEFAULT_EXPR_SQL.items() +} + + +def default_expr_for_sql(expression): + if expression is None: + return None + return DEFAULT_EXPR_BY_SQL.get(_default_expression_lookup_key(expression)) + + +def _quoted_options(options): + if len(options) == 1: + return "'{}'".format(options[0]) + return "{} or '{}'".format( + ", ".join("'{}'".format(option) for option in options[:-1]), + options[-1], + ) + + +def _default_expr_error_message(): + return "Input should be {}".format(_quoted_options(list(DEFAULT_EXPRESSIONS))) + + +def default_expression_options(): + return [ + { + "value": value, + "label": metadata["label"], + "sqliteType": metadata["sqliteType"], + } + for value, metadata in DEFAULT_EXPRESSIONS.items() + ] class _StrictPydanticModel(BaseModel): @@ -324,6 +421,13 @@ class _DefaultArgsMixin(_StrictPydanticModel): default: Any | None = None default_expr: DefaultExpr | None = None + @field_validator("default_expr") + @classmethod + def validate_default_expr_value(cls, value): + if value is not None and value not in DEFAULT_EXPRESSIONS: + raise PydanticCustomError("default_expr", _default_expr_error_message()) + return value + @model_validator(mode="after") def validate_default_fields(self): has_default = "default" in self.model_fields_set diff --git a/docs/json_api.rst b/docs/json_api.rst index 849a8acf..b8632bc3 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -44,7 +44,7 @@ looks like this: ``"ok"`` is always ``true`` if an error did not occur. -The ``"rows"`` key is a list of objects, each one representing a row. +The ``"rows"`` key is a list of objects, each one representing a row. The ``"truncated"`` key lets you know if the query was truncated. This can happen if a SQL query returns more than 1,000 results (or the :ref:`setting_max_returned_rows` setting). @@ -1990,7 +1990,7 @@ The JSON here describes the table that will be created: - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. - ``not_null`` can be set to ``true`` to create this column with a ``NOT NULL`` constraint. - ``default`` can be used to set a literal default value for this column. - - ``default_expr`` can be used instead of ``default`` to set a SQLite default expression. The supported values are ``current_timestamp``, ``current_date`` and ``current_time``. + - ``default_expr`` can be used instead of ``default`` to set a SQLite default expression. See :ref:`default_expr values `. - ``fk_table`` can be used to create a single-column foreign key constraint referencing another table. ``fk_column`` is optional and can be used to specify the referenced column - if omitted, Datasette will use the single primary key of ``fk_table``. * ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. @@ -2004,6 +2004,32 @@ The JSON here describes the table that will be created: * ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. This requires the :ref:`actions_update_row` permission. * ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`actions_alter_table` permission. +.. _json_api_default_expr_values: + +``default_expr`` accepts these values: + +.. list-table:: + :header-rows: 1 + + * - Value + - Recommended column type + - Example inserted value + * - ``current_timestamp`` + - ``text`` + - ``2026-05-01 13:34:00`` + * - ``current_date`` + - ``text`` + - ``2026-05-01`` + * - ``current_time`` + - ``text`` + - ``13:34:00`` + * - ``current_unixtime`` + - ``integer`` + - ``1777642440`` + * - ``current_unixtime_ms`` + - ``integer`` + - ``1777642440000`` + This example creates a foreign key from ``projects.owner_id`` to the single primary key of ``owners``: .. code-block:: json @@ -2314,7 +2340,7 @@ Supported operations: * ``set_foreign_keys`` replaces all foreign key constraints on the table. ``args`` accepts ``foreign_keys``, a list of objects that each have ``column``, ``fk_table`` and optional ``fk_column``. An empty list removes all foreign key constraints. * ``reorder_columns`` reorders columns. ``args`` accepts ``columns``, a list of one or more column names. Columns omitted from this list will appear afterwards in their existing order. -``default`` is always treated as a literal value. ``default_expr`` accepts one of ``current_timestamp``, ``current_date`` or ``current_time`` and is rendered as the corresponding SQLite default expression. +``default`` is always treated as a literal value. ``default_expr`` accepts the values shown in :ref:`default_expr values ` and is rendered as the corresponding SQLite default expression. For foreign key operations that omit ``fk_column``, the referenced ``fk_table`` must have a single-column primary key. Datasette will return an error if it cannot identify a single primary key column for that table. diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 6e8c4d4b..b3a2f6a3 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -899,6 +899,64 @@ async def test_alter_table_operations(ds_write): assert event.after_schema == data["schema"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "default_expr,minimum_value,expected_schema", + ( + ( + "current_unixtime", + 1_600_000_000, + "strftime('%s', 'now')", + ), + ( + "current_unixtime_ms", + 1_600_000_000_000, + "julianday('now')", + ), + ), +) +async def test_alter_table_integer_default_expr( + ds_write, default_expr, minimum_value, expected_schema +): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + { + "op": "add_column", + "args": { + "name": "created", + "type": "integer", + "default_expr": default_expr, + }, + } + ] + }, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert expected_schema in data["schema"] + + columns = await db.execute("select * from pragma_table_info('docs')") + created_column = [ + column for column in columns.dicts() if column["name"] == "created" + ][0] + assert created_column["type"] == "INTEGER" + assert expected_schema in created_column["dflt_value"] + + row = await db.execute_write_fn( + lambda conn: conn.execute( + "insert into docs (title) values ('with default') " + "returning created, typeof(created)" + ).fetchone() + ) + assert row[0] > minimum_value + assert row[1] == "integer" + + @pytest.mark.asyncio async def test_alter_table_rename_table(ds_write): token = write_token(ds_write, permissions=["at"]) @@ -1315,7 +1373,7 @@ async def test_alter_table_permission_denied(ds_write): } ] }, - "operations.0.add_column.args.default_expr: Input should be 'current_timestamp', 'current_date' or 'current_time'", + "operations.0.add_column.args.default_expr: Input should be 'current_timestamp', 'current_date', 'current_time', 'current_unixtime' or 'current_unixtime_ms'", ), ( { @@ -2053,6 +2111,63 @@ async def test_create_table_with_column_constraints(ds_write): assert columns[4]["dflt_value"] == "'hello)'" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "default_expr,minimum_value,expected_schema", + ( + ( + "current_unixtime", + 1_600_000_000, + "strftime('%s', 'now')", + ), + ( + "current_unixtime_ms", + 1_600_000_000_000, + "julianday('now')", + ), + ), +) +async def test_create_table_integer_default_expr( + ds_write, default_expr, minimum_value, expected_schema +): + token = write_token(ds_write) + table = "default_{}".format(default_expr) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": table, + "columns": [ + {"name": "id", "type": "integer"}, + { + "name": "created", + "type": "integer", + "default_expr": default_expr, + }, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201, response.text + data = response.json() + assert expected_schema in data["schema"] + + db = ds_write.get_database("data") + columns = (await db.execute("select * from pragma_table_info(?)", [table])).dicts() + assert columns[1]["type"] == "INTEGER" + assert expected_schema in columns[1]["dflt_value"] + + row = await db.execute_write_fn( + lambda conn: conn.execute( + "insert into [{}] default values returning created, typeof(created)".format( + table + ) + ).fetchone() + ) + assert row[0] > minimum_value + assert row[1] == "integer" + + @pytest.mark.asyncio @pytest.mark.parametrize( "column,expected_error", @@ -2071,7 +2186,7 @@ async def test_create_table_with_column_constraints(ds_write): "type": "text", "default_expr": "datetime('now')", }, - "columns.0.default_expr: Input should be 'current_timestamp', 'current_date' or 'current_time'", + "columns.0.default_expr: Input should be 'current_timestamp', 'current_date', 'current_time', 'current_unixtime' or 'current_unixtime_ms'", ), ( { diff --git a/tests/test_playwright.py b/tests/test_playwright.py index fcf97a90..ee396de5 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -98,6 +98,10 @@ def write_playwright_database(db_path): notes text, score integer default 5 ); + create table defaults_demo ( + id integer primary key, + created_ms integer default (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)) + ); insert into projects (title, metadata, logo, notes, score) values ( 'Build Datasette', @@ -136,6 +140,11 @@ def write_playwright_config(config_path): "delete-row": True, }, }, + "defaults_demo": { + "permissions": { + "alter-table": True, + }, + }, }, }, }, @@ -394,6 +403,32 @@ def test_create_table_foreign_key_selection_updates_column_type(page, datasette_ assert foreign_key_select.input_value() == "projects\u001fid" +@pytest.mark.playwright +def test_create_table_unix_default_expression_updates_column_type( + page, datasette_server +): + page.goto(f"{datasette_server}data") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-database-action="create-table"]').click() + + dialog = page.locator("#table-create-dialog") + dialog.wait_for() + row = dialog.locator(".table-create-column-row").nth(1) + row.locator(".table-create-more-options").click() + row.locator(".table-create-default-options summary").click() + + type_select = row.locator(".table-create-column-type") + default_expr = row.locator(".table-create-default-expr") + assert type_select.input_value() == "text" + assert ( + "Current Unix time, integer milliseconds since the epoch" + in default_expr.locator("option").last.inner_text() + ) + + default_expr.select_option("current_unixtime_ms") + assert type_select.input_value() == "integer" + + @pytest.mark.playwright def test_alter_table_foreign_key_selection_updates_blank_column(page, datasette_server): page.goto(f"{datasette_server}data/projects") @@ -416,6 +451,52 @@ def test_alter_table_foreign_key_selection_updates_blank_column(page, datasette_ assert foreign_key_select.input_value() == "projects\u001fid" +@pytest.mark.playwright +def test_alter_table_unix_default_expression_updates_column_type( + page, datasette_server +): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + dialog.locator(".table-alter-add-column").click() + row = dialog.locator(".table-alter-column-row").last + row.locator(".table-alter-default-options summary").click() + + type_select = row.locator(".table-alter-column-type") + default_expr = row.locator(".table-alter-default-expr") + assert type_select.input_value() == "text" + assert ( + "Current Unix time, integer seconds since the epoch" + in default_expr.locator("option").all_inner_texts() + ) + + default_expr.select_option("current_unixtime") + assert type_select.input_value() == "integer" + + +@pytest.mark.playwright +def test_alter_table_existing_default_expression_populates_select( + page, datasette_server +): + page.goto(f"{datasette_server}data/defaults_demo") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + row = dialog.locator(".table-alter-column-row").nth(1) + row.locator(".table-alter-more-options").click() + row.locator(".table-alter-default-options summary").click() + + assert row.locator(".table-alter-default-expr").input_value() == ( + "current_unixtime_ms" + ) + assert row.locator(".table-alter-default").input_value() == "" + + @pytest.mark.playwright def test_alter_table_flow(page, datasette_server): page.goto(f"{datasette_server}data/projects") diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 9d1d1245..3af2bb08 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -40,6 +40,35 @@ def database_data_from_soup(soup): return json.loads(match.group(1)) +DEFAULT_EXPRESSION_OPTIONS = [ + { + "value": "current_timestamp", + "label": "Current timestamp in UTC, e.g. 2026-05-01 13:34:00", + "sqliteType": "text", + }, + { + "value": "current_date", + "label": "Current date in UTC, e.g. 2026-05-01", + "sqliteType": "text", + }, + { + "value": "current_time", + "label": "Current time in UTC, e.g. 13:34:00", + "sqliteType": "text", + }, + { + "value": "current_unixtime", + "label": "Current Unix time, integer seconds since the epoch", + "sqliteType": "integer", + }, + { + "value": "current_unixtime_ms", + "label": "Current Unix time, integer milliseconds since the epoch", + "sqliteType": "integer", + }, +] + + @pytest.mark.asyncio @pytest.mark.parametrize( "path,expected_definition_sql", @@ -997,11 +1026,7 @@ async def test_database_create_table_action_button_and_data(): "foreignKeyTargetsPath": "/data/-/foreign-key-targets", "databaseName": "data", "columnTypes": ["text", "integer", "float", "blob"], - "defaultExpressions": [ - "current_timestamp", - "current_date", - "current_time", - ], + "defaultExpressions": DEFAULT_EXPRESSION_OPTIONS, }, } assert "customColumnTypes" not in database_data_from_soup(soup)["createTable"] @@ -1113,7 +1138,9 @@ async def test_table_alter_action_button_and_data(): create table items ( id integer primary key, name text not null, - score integer default 5 + score integer default 5, + created text default current_timestamp, + created_ms integer default (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)) ); """) response = await ds.client.get("/data/items", actor={"id": "root"}) @@ -1145,11 +1172,7 @@ async def test_table_alter_action_button_and_data(): assert alter_data["foreignKeyTargetsPath"] == ( "/data/-/foreign-key-targets?table=items" ) - assert alter_data["defaultExpressions"] == [ - "current_timestamp", - "current_date", - "current_time", - ] + assert alter_data["defaultExpressions"] == DEFAULT_EXPRESSION_OPTIONS assert [option["name"] for option in alter_data["customColumnTypes"]] == [ "email", "json", @@ -1191,6 +1214,30 @@ async def test_table_alter_action_button_and_data(): "foreign_key": None, "column_type": None, }, + { + "name": "created", + "type": "text", + "sqlite_type": "TEXT", + "notnull": 0, + "default": None, + "default_expr": "current_timestamp", + "has_default": True, + "is_pk": False, + "foreign_key": None, + "column_type": None, + }, + { + "name": "created_ms", + "type": "integer", + "sqlite_type": "INTEGER", + "notnull": 0, + "default": None, + "default_expr": "current_unixtime_ms", + "has_default": True, + "is_pk": False, + "foreign_key": None, + "column_type": None, + }, ] response_without_permission = await ds.client.get( From b3b5c25df8c51fb0da39a1c8f19198cd5c0b4afb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 13:51:32 -0700 Subject: [PATCH 040/167] Draft changelog for create/alter table UI, refs #2787, #2788 --- docs/changelog.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1516604e..697df87d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _unreleased: + +Unreleased +---------- + +- New "Create table" interface in the database actions menu, backed by the ``//-/create`` :ref:`JSON API `. It can define columns, primary keys, custom column types, ``NOT NULL`` constraints, literal defaults, expression defaults and single-column foreign keys. (:issue:`2787`) +- New "Alter table" table action and ``//
/-/alter`` :ref:`JSON API ` for changing existing tables: add, rename, reorder and drop columns; change column types, defaults, ``NOT NULL`` constraints, primary keys and foreign keys; and rename the table. The alter table dialog also includes a "Drop table" button. (:issue:`2788`) +- New ``//-/foreign-key-targets`` and ``//
/-/foreign-key-suggestions`` JSON APIs for discovering valid single-column foreign key targets and suggested relationships. +- Create and alter table dialogs share their column-editing controls, including literal and expression defaults, custom column types, foreign keys and column ordering. + .. _v1_0_a34: 1.0a34 (2026-06-16) From f8313525517b774879482c597172da076b4ba59b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 14:24:34 -0700 Subject: [PATCH 041/167] Avoid SQLite RETURNING for compat with older SQLite Refs #2789 --- tests/test_api_write.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index b3a2f6a3..76797742 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1,6 +1,6 @@ from datasette.app import Datasette from datasette.events import RenameTableEvent -from datasette.utils import sqlite3 +from datasette.utils import escape_sqlite, sqlite3 from .utils import last_event import pytest import time @@ -40,6 +40,16 @@ def _headers(token): } +def _insert_and_fetch_created(conn, table, insert_sql): + cursor = conn.execute(insert_sql) + return conn.execute( + "select created, typeof(created) from {} where rowid = ?".format( + escape_sqlite(table) + ), + (cursor.lastrowid,), + ).fetchone() + + @pytest.mark.asyncio async def test_api_explorer_upsert_example_json(ds_write): response = await ds_write.client.get("/-/api", actor={"id": "root"}) @@ -948,10 +958,9 @@ async def test_alter_table_integer_default_expr( assert expected_schema in created_column["dflt_value"] row = await db.execute_write_fn( - lambda conn: conn.execute( - "insert into docs (title) values ('with default') " - "returning created, typeof(created)" - ).fetchone() + lambda conn: _insert_and_fetch_created( + conn, "docs", "insert into docs (title) values ('with default')" + ) ) assert row[0] > minimum_value assert row[1] == "integer" @@ -2158,11 +2167,9 @@ async def test_create_table_integer_default_expr( assert expected_schema in columns[1]["dflt_value"] row = await db.execute_write_fn( - lambda conn: conn.execute( - "insert into [{}] default values returning created, typeof(created)".format( - table - ) - ).fetchone() + lambda conn: _insert_and_fetch_created( + conn, table, "insert into {} default values".format(escape_sqlite(table)) + ) ) assert row[0] > minimum_value assert row[1] == "integer" From 86ea1d472275d353105cb2a8b2109c9ffc1a343e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 18:19:49 -0700 Subject: [PATCH 042/167] Visual improvements to table filter UI - Looks nicer - Add / remove buttons work properly Closes #2798 --- datasette/static/app.css | 300 +++++++++++++++++++++++++----- datasette/static/table.js | 169 ++++++++++++++--- datasette/templates/patterns.html | 14 +- datasette/templates/table.html | 14 +- 4 files changed, 411 insertions(+), 86 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index f7cd97b0..ce800f61 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -609,9 +609,6 @@ button.core[type=button] { border-color: #007bff; } -.filter-row { - margin-bottom: 0.6em; -} .search-row { margin-bottom: 1.8em; } @@ -623,72 +620,239 @@ button.core[type=button] { width: 80px; } -.select-wrapper { - border: 1px solid #ccc; - width: 120px; - border-radius: 3px; +.filters .search-row { + box-sizing: border-box; + display: grid; + grid-template-columns: max-content minmax(16rem, 1fr); + align-items: center; + gap: 0.6rem; + margin: 0 0 0.45rem; + padding-right: var(--filter-two-icon-space); +} + +.filters .search-row label { + width: auto; padding: 0; - background-color: #fafafa; - position: relative; + color: var(--filter-muted); + font-size: 0.875rem; + font-weight: 600; +} + +.filters { + --filter-ink: #0f0f0f; + --filter-paper: #eef6ff; + --filter-muted: #6b6b6b; + --filter-rule: #d8e6f5; + --filter-accent: #1a56db; + --filter-control-border: #bfccd9; + --filter-control-height: 2.125rem; + --filter-control-gap: 0.4rem; + --filter-row-icon-size: 2rem; + --filter-two-icon-space: calc( + (2 * var(--filter-row-icon-size)) + (2 * var(--filter-control-gap)) + ); + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 0.55rem; + max-width: 760px; + margin: 0 0 1rem; + padding: 0.75rem; + border: 1px solid var(--filter-rule); + border-radius: 8px; + background: var(--filter-paper); + color: var(--filter-ink); + font-size: 0.875rem; +} + +.filters .filter-row { + margin-bottom: 0; +} + +.filters .filter-controls-row { + display: grid; + min-width: 0; + width: 100%; + max-width: 100%; + box-sizing: border-box; + grid-template-columns: + minmax(8rem, 0.75fr) + minmax(6.5rem, 0.5fr) + minmax(12rem, 1.2fr) + var(--filter-row-icon-size) + var(--filter-row-icon-size); + gap: var(--filter-control-gap); + align-items: center; +} + +.filters .filter-actions-row { + display: flex; + align-items: center; + justify-content: flex-start; + flex-wrap: wrap; + gap: 0.6rem; +} + +.select-wrapper { display: inline-block; - margin-right: 0.3em; -} -.select-wrapper:focus-within { - border: 1px solid black; + width: 120px; + min-width: 0; } + .select-wrapper.filter-op { width: 80px; } -.select-wrapper::after { - content: "\25BE"; - position: absolute; - top: 0px; - right: 0.4em; - color: #bbb; - pointer-events: none; - font-size: 1.2em; - padding-top: 0.16em; + +.filters .select-wrapper { + width: auto; } .select-wrapper select { - padding: 9px 8px; + box-sizing: border-box; width: 100%; - border: none; - box-shadow: none; - background: transparent; - background-image: none; - -webkit-appearance: none; - -moz-appearance: none; + height: var(--filter-control-height); + border: 1px solid var(--filter-control-border, #ccc); + border-radius: 5px; + background-color: #fff; + color: inherit; cursor: pointer; + font-family: inherit; + font-size: 0.875rem; + line-height: 1.25; + padding: 7px 8px; } -.select-wrapper select { - font-size: 1em; - font-family: Helvetica, sans-serif; -} + .select-wrapper option { font-size: 1em; font-family: Helvetica, sans-serif; } .select-wrapper select:focus { + border-color: var(--filter-accent, #000); + box-shadow: 0 0 0 2px rgba(26, 86, 219, 0.14); outline: none; } -.filters { - font-size: 0.8em; -} .filters input.filter-value { - width: 200px; - border-radius: 3px; - -webkit-appearance: none; - padding: 9px 4px; - font-size: 16px; - font-family: Helvetica, sans-serif; + box-sizing: border-box; + width: 100%; + min-width: 0; + height: var(--filter-control-height); + border: 1px solid var(--filter-control-border); + border-radius: 5px; + background: #fff; + color: inherit; + font-family: inherit; + font-size: 0.875rem; + line-height: 1.25; + padding: 7px 9px; +} + +.filters input.filter-value:focus { + border-color: var(--filter-accent); + box-shadow: 0 0 0 2px rgba(26, 86, 219, 0.14); + outline: none; +} + +.filters input[type=submit] { + border-color: var(--filter-accent); + background: var(--filter-accent); + border-radius: 5px; + font-weight: 500; + padding: 0.55rem 0.85rem; +} + +.filters input[type=submit]:hover, +.filters input[type=submit]:focus { + background: #1949b8; + border-color: #1949b8; +} + +.filters button.filter-row-icon { + display: inline-flex; + align-items: center; + justify-content: center; + box-sizing: border-box; + width: var(--filter-row-icon-size); + height: var(--filter-row-icon-size); + min-width: var(--filter-row-icon-size); + padding: 0; + border: 1px solid var(--filter-rule); + border-radius: 5px; + background: #fff; + color: var(--filter-muted); + font-family: inherit; + font-size: 1.15rem; + font-weight: 600; + line-height: 1; +} + +.filters button.filter-row-icon[hidden] { + display: none; +} + +.filters button.filter-row-icon:focus-visible { + outline: 3px solid rgba(26, 86, 219, 0.14); + outline-offset: 2px; +} + +.filters .filter-row-remove-icon { + display: block; + height: 14px; + width: 14px; +} + +.filters button.filter-row-remove:hover, +.filters button.filter-row-remove:focus { + border-color: #c9d5e3; + background: #f8fbff; + color: var(--filter-ink); +} + +.filters button.filter-row-add { + border-color: var(--filter-accent); + background: var(--filter-accent); + color: #fff; +} + +.filters .filter-row-add-icon { + display: block; + height: 16px; + width: 16px; +} + +.filters button.filter-row-add:hover, +.filters button.filter-row-add:focus { + border-color: #1949b8; + background: #1949b8; + color: #fff; +} + +.filters button.filter-row-add:focus-visible svg { + color: #fff; + stroke: currentColor; } #_search { font-size: 16px; } +.filters #_search { + box-sizing: border-box; + width: 100%; + height: var(--filter-control-height); + border: 1px solid var(--filter-control-border); + border-radius: 5px; + background: #fff; + color: var(--filter-ink); + font: inherit; + padding: 7px 9px; +} + +.filters #_search:focus { + border-color: var(--filter-accent); + box-shadow: 0 0 0 2px rgba(26, 86, 219, 0.14); + outline: none; +} + @@ -3021,14 +3185,56 @@ select.table-alter-input { margin-bottom: 0.35rem; } - .select-wrapper { - width: 100px; + .filters { + max-width: none; + padding: 0.65rem; } - .select-wrapper.filter-op { - width: 60px; + .filters .search-row { + grid-template-columns: max-content minmax(0, 1fr); + padding-right: 0; + } + .filters .filter-controls-row { + --filter-value-button-space: 0px; + --filter-remove-button-offset: 0px; + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + .filters .filter-controls-row-one-button { + --filter-value-button-space: calc( + var(--filter-row-icon-size) + var(--filter-control-gap) + ); + } + .filters .filter-controls-row-two-buttons { + --filter-value-button-space: var(--filter-two-icon-space); + --filter-remove-button-offset: calc( + var(--filter-row-icon-size) + var(--filter-control-gap) + ); + } + .filters .filter-controls-row .select-wrapper { + grid-row: 1; + grid-column: 1 / 2; + } + .filters .filter-controls-row .select-wrapper.filter-op { + grid-column: 2 / 3; } .filters input.filter-value { - width: 140px; + grid-row: 2; + grid-column: 1 / 3; + justify-self: start; + width: calc(100% - var(--filter-value-button-space)); + } + .filters button.filter-row-icon { + grid-row: 2; + grid-column: 1 / 3; + justify-self: end; + } + .filters .filter-controls-row-two-buttons button.filter-row-remove:not([hidden]) { + margin-right: var(--filter-remove-button-offset); + } + .filters .filter-actions-row { + justify-content: flex-start; + } + .filters .filter-actions-row input[type=submit] { + flex: 0 1 auto; } button.choose-columns-mobile, button.table-insert-row, diff --git a/datasette/static/table.js b/datasette/static/table.js index f160f3f3..74a96d8e 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -633,32 +633,151 @@ const initDatasetteTable = function (manager) { }); }; -/* Add x buttons to the filter rows */ -function addButtonsToFilterRows(manager) { - var x = "✖"; - var rows = Array.from( - document.querySelectorAll(manager.selectors.filterRow), +function filterRowSelector(manager) { + return manager.selectors.filterRows || manager.selectors.filterRow; +} + +function filterRowsWithControls(manager) { + return Array.from( + document.querySelectorAll(filterRowSelector(manager)), ).filter((el) => el.querySelector(".filter-op")); - rows.forEach((row) => { - var a = document.createElement("a"); - a.setAttribute("href", "#"); - a.setAttribute("aria-label", "Remove this filter"); - a.style.textDecoration = "none"; - a.innerText = x; - a.addEventListener("click", (ev) => { - ev.preventDefault(); - let row = ev.target.closest("div"); - row.querySelector("select").value = ""; - row.querySelector(".filter-op select").value = "exact"; - row.querySelector("input.filter-value").value = ""; - ev.target.closest("a").style.display = "none"; - }); - row.appendChild(a); +} + +function filterRowNumberFromName(name) { + var match = name && name.match(/^_filter_column_(\d+)$/); + return match ? parseInt(match[1], 10) : 0; +} + +function nextFilterRowNumber(manager) { + return filterRowsWithControls(manager).reduce((max, row) => { var column = row.querySelector("select"); - if (!column.value) { - a.style.display = "none"; + return Math.max(max, filterRowNumberFromName(column && column.name)); + }, 0) + 1; +} + +function setFilterRowNumber(row, number) { + row.querySelector("select").name = `_filter_column_${number}`; + row.querySelector(".filter-op select").name = `_filter_op_${number}`; + row.querySelector("input.filter-value").name = `_filter_value_${number}`; +} + +function resetFilterRow(row) { + row.querySelector("select").value = ""; + row.querySelector(".filter-op select").value = "exact"; + row.querySelector("input.filter-value").value = ""; +} + +function updateFilterRowButtons(manager) { + var rows = filterRowsWithControls(manager); + rows.forEach((row, index) => { + var removeButton = row.querySelector(".filter-row-remove"); + var addButton = row.querySelector(".filter-row-add"); + var column = row.querySelector("select"); + if (removeButton) { + removeButton.hidden = index === 0; + } + if (addButton) { + addButton.hidden = index !== rows.length - 1 || !column.value; + } + var visibleButtonCount = [removeButton, addButton].filter(function (button) { + return button && !button.hidden; + }).length; + row.classList.toggle( + "filter-controls-row-has-buttons", + visibleButtonCount > 0, + ); + row.classList.toggle( + "filter-controls-row-one-button", + visibleButtonCount === 1, + ); + row.classList.toggle( + "filter-controls-row-two-buttons", + visibleButtonCount === 2, + ); + }); +} + +function cloneFilterRow(row) { + var clone = row.cloneNode(true); + clone.querySelector("select").name = "_filter_column"; + clone.querySelector(".filter-op select").name = "_filter_op"; + clone.querySelector("input.filter-value").name = "_filter_value"; + resetFilterRow(clone); + clone.querySelectorAll(".filter-row-icon").forEach((button) => button.remove()); + return clone; +} + +var FILTER_REMOVE_ICON_SVG = ``; + +var FILTER_ADD_ICON_SVG = ``; + +function addFilterRowButtons(row, manager) { + var removeButton = document.createElement("button"); + removeButton.type = "button"; + removeButton.className = "filter-row-icon filter-row-remove"; + removeButton.setAttribute("aria-label", "Remove this filter"); + removeButton.title = "Remove this filter"; + removeButton.tabIndex = 0; + removeButton.innerHTML = FILTER_REMOVE_ICON_SVG; + removeButton.addEventListener("click", (ev) => { + var row = ev.currentTarget.closest(filterRowSelector(manager)); + var rows = filterRowsWithControls(manager); + var rowIndex = rows.indexOf(row); + var focusRow = rows[rowIndex + 1] || rows[rowIndex - 1] || null; + row.remove(); + updateFilterRowButtons(manager); + if (focusRow) { + var focusTarget = + focusRow.querySelector(".filter-row-add:not([hidden])") || + focusRow.querySelector("select"); + if (focusTarget) { + focusTarget.focus(); + } } }); + row.appendChild(removeButton); + + var addButton = document.createElement("button"); + addButton.type = "button"; + addButton.className = "filter-row-icon filter-row-add"; + addButton.setAttribute("aria-label", "Add another filter"); + addButton.title = "Add another filter"; + addButton.tabIndex = 0; + addButton.innerHTML = FILTER_ADD_ICON_SVG; + addButton.addEventListener("click", (ev) => { + var row = ev.currentTarget.closest(filterRowSelector(manager)); + if (row.querySelector("select").name === "_filter_column") { + setFilterRowNumber(row, nextFilterRowNumber(manager)); + } + var clone = cloneFilterRow(row); + addFilterRowButtons(clone, manager); + row.parentNode.insertBefore(clone, row.nextSibling); + updateFilterRowButtons(manager); + clone.querySelector("select").focus(); + }); + row.appendChild(addButton); + + row.querySelector("select").addEventListener("change", () => { + updateFilterRowButtons(manager); + }); +} + +/* Add buttons to the filter rows */ +function addButtonsToFilterRows(manager) { + var rows = filterRowsWithControls(manager); + rows.forEach((row) => { + addFilterRowButtons(row, manager); + }); + updateFilterRowButtons(manager); } /* Set up datalist autocomplete for filter values */ @@ -687,11 +806,11 @@ function initAutocompleteForFilterValues(manager) { }); } createDataLists(); - // When any select with name=_filter_column changes, update the datalist + // When any filter column select changes, update the datalist document.body.addEventListener("change", function (event) { - if (event.target.name === "_filter_column") { + if (event.target.name && event.target.name.startsWith("_filter_column")) { event.target - .closest(manager.selectors.filterRow) + .closest(filterRowSelector(manager)) .querySelector(".filter-value") .setAttribute("list", "datalist-" + event.target.value); } diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index a46478a7..bff44341 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -202,9 +202,9 @@

3 rows where characteristic_id = 2

-
+
-
+
-
+
-
-
+
+
- - + +
diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 86b6a6ed..e06ef94e 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -55,12 +55,12 @@ {% endif %} -
+ {% if supports_search %}
{% endif %} {% for column, lookup, value in filters.selections() %} -
+
{% endfor %} -
+
-
+
{% if is_sortable %} -
+
- + {% endif %} {% for key, value in form_hidden_args %} {% endfor %} - +
From 4d031c85621ab7e63b1952d25e3e938fa72323b1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 19:47:21 -0700 Subject: [PATCH 043/167] Add count_truncated template context --- datasette/templates/database.html | 2 +- datasette/templates/table.html | 4 ++-- datasette/views/__init__.py | 2 ++ datasette/views/database.py | 29 ++++++++++++++++++++++---- datasette/views/table.py | 33 +++++++++++++++++++++++------- datasette/views/table_extras.py | 4 ++-- docs/template_context.rst | 13 +++++------- tests/test_template_context.py | 34 ++++++++++++++++++++++++++++++- 8 files changed, 96 insertions(+), 25 deletions(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 23eeb571..c09582f7 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -76,7 +76,7 @@

{{ table.name }}{% if table.private %} 🔒{% endif %}{% if table.hidden %} (hidden){% endif %}

{% for column in table.columns %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}

-

{% if table.count is none %}Many rows{% elif table.count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

+

{% if table.count is none %}Many rows{% elif table.count_truncated %}>{{ "{:,}".format(table.count - 1) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

{% endif %} {% endfor %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index e06ef94e..b7b776ab 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}{{ database }}: {{ table }}: {% if count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} +{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} {% block extra_head %} {{- super() -}} @@ -48,7 +48,7 @@ {% if count or human_description_en %}

- {% if count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows + {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows {% if allow_execute_sql and query.sql %} count all{% endif %} {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %} {% if human_description_en %}{{ human_description_en }}{% endif %} diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index 238b9bb3..e503ed35 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -32,6 +32,8 @@ class Context: "List of ContextField describing the documented fields of this context" documented = [] for f in dataclasses.fields(cls): + if f.name.startswith("_"): + continue from_extra = bool(f.metadata.get("from_extra")) if from_extra: help_text = cls._extra_description(f.name) diff --git a/datasette/views/database.py b/datasette/views/database.py index 0d35b87f..46a90c4c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -230,7 +230,6 @@ class DatabaseView(View): database_actions=database_actions, show_hidden=request.args.get("_show_hidden"), editable=True, - count_limit=db.count_limit, allow_download=datasette.setting("allow_download") and not db.is_mutable and not db.is_memory, @@ -267,7 +266,11 @@ class DatabaseContext(Context): ) path: str = field(metadata={"help": "The URL path to this database"}) size: int = field(metadata={"help": "The size of the database in bytes"}) - tables: list = field(metadata={"help": "List of table objects in the database"}) + tables: list = field( + metadata={ + "help": "List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total." + } + ) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) views: list = field(metadata={"help": "List of view objects in the database"}) queries: list = field(metadata={"help": "List of stored query objects"}) @@ -295,7 +298,6 @@ class DatabaseContext(Context): editable: bool = field( metadata={"help": "Boolean indicating if the database is editable"} ) - count_limit: int = field(metadata={"help": "The maximum number of rows to count"}) allow_download: bool = field( metadata={"help": "Boolean indicating if database download is allowed"} ) @@ -365,7 +367,11 @@ class QueryContext(Context): save_query_url: str = field( metadata={"help": "URL to save the current arbitrary SQL as a query"} ) - tables: list = field(metadata={"help": "List of table objects in the database"}) + tables: list = field( + metadata={ + "help": "List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total." + } + ) named_parameter_values: dict = field( metadata={"help": "Dictionary of parameter names/values"} ) @@ -430,6 +436,9 @@ async def get_tables(datasette, request, db, allowed_dict): "columns": table_columns, "primary_keys": await db.primary_keys(table), "count": table_counts[table], + "count_truncated": _table_count_truncated( + datasette, db, table, table_counts[table] + ), "hidden": table in hidden_table_names, "fts_table": await db.fts_table(table), "foreign_keys": all_foreign_keys[table], @@ -440,6 +449,18 @@ async def get_tables(datasette, request, db, allowed_dict): return tables +def _table_count_truncated(datasette, db, table, count): + if count != db.count_limit + 1: + return False + if not db.is_mutable and datasette.inspect_data: + try: + datasette.inspect_data[db.name]["tables"][table]["count"] + return False + except KeyError: + pass + return True + + async def database_download(request, datasette): from datasette.resources import DatabaseResource diff --git a/datasette/views/table.py b/datasette/views/table.py index a6769242..38a69f5f 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -113,6 +113,11 @@ class TableContext(Context): metadata={"help": "True if the data for this page was retrieved without errors"} ) next: str = field(metadata={"help": "Pagination token for the next page, or None"}) + count_truncated: bool = field( + metadata={ + "help": "True if ``count`` is a capped lower bound rather than an exact total, because Datasette stopped counting after its configured row-count limit." + } + ) rows: list = field( metadata={ "help": "The rows for this page, as a list of dictionaries mapping column name to value" @@ -185,11 +190,6 @@ class TableContext(Context): top_table: callable = field( metadata={"help": "Async function rendering the top_table plugin slot"} ) - count_limit: int = field( - metadata={ - "help": "The maximum number of rows Datasette will count before showing an approximation" - } - ) table_page_data: dict = field( metadata={"help": "JSON data used by JavaScript on the table page"} ) @@ -1391,7 +1391,6 @@ class TableFragmentView(BaseView): path_with_replaced_args=path_with_replaced_args, fix_path=self.ds.urls.path, settings=self.ds.settings_dict(), - count_limit=resolved.db.count_limit, ), request=request, view_name="table", @@ -1843,7 +1842,6 @@ async def table_view_traced(datasette, request): database=resolved.db.name, table=resolved.table, ), - count_limit=resolved.db.count_limit, ), request=request, view_name="table", @@ -2276,6 +2274,9 @@ async def table_view_data( data["rows"] = transformed_rows if context_for_html_hack: + data["count_truncated"] = _count_truncated_for_table_page( + datasette, db, database_name, table_name, count_sql, data.get("count") + ) data.update(extra_context_from_filters) # filter_columns combine the columns we know are available # in the table with any additional columns (such as rowid) @@ -2341,6 +2342,24 @@ async def table_view_data( return data, rows[:page_size], columns, expanded_columns, sql, next_url +def _count_truncated_for_table_page( + datasette, db, database_name, table_name, count_sql, count +): + if count != db.count_limit + 1: + return False + if ( + not db.is_mutable + and datasette.inspect_data + and count_sql == f"select count(*) from {table_name} " + ): + try: + datasette.inspect_data[database_name]["tables"][table_name]["count"] + return False + except KeyError: + pass + return True + + async def _next_value_and_url( datasette, db, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 7cb4d8f0..f688433d 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -127,8 +127,8 @@ class CountExtra(Extra): pass if context.count_sql and count is None and not context.nocount: - count_sql_limited = ( - f"select count(*) from (select * {context.from_sql} limit 10001)" + count_sql_limited = "select count(*) from (select * {} limit {})".format( + context.from_sql, context.db.count_limit + 1 ) try: count_rows = list( diff --git a/docs/template_context.rst b/docs/template_context.rst index f94ecd48..b27f2f79 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -100,9 +100,6 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``attached_databases`` - ``list`` List of names of attached databases -``count_limit`` - ``int`` - The maximum number of rows to count - ``database`` - ``str`` The name of the database @@ -152,7 +149,7 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re Dictionary mapping table names to their column lists ``tables`` - ``list`` - List of table objects in the database + List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total. ``top_database`` - ``callable`` Callable to render the top_database slot @@ -241,7 +238,7 @@ The page for arbitrary SQL queries (/database/-/query?sql=...) and stored querie Dictionary of table name to list of column names ``tables`` - ``list`` - List of table objects in the database + List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total. ``top_query`` - ``callable`` Callable to render the top_query slot @@ -280,12 +277,12 @@ Many of these keys are shared with the :ref:`JSON API ` for this page. ``count`` - ``int`` Total count of rows matching these filters -``count_limit`` - ``int`` - The maximum number of rows Datasette will count before showing an approximation - ``count_sql`` - ``str`` SQL query used to calculate the total count +``count_truncated`` - ``bool`` + True if ``count`` is a capped lower bound rather than an exact total, because Datasette stopped counting after its configured row-count limit. + ``custom_table_templates`` - ``list`` Custom template names considered for this table diff --git a/tests/test_template_context.py b/tests/test_template_context.py index b970cb8a..8d78bf77 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -21,6 +21,7 @@ def test_documented_fields(): @dataclass class DemoContext(Context): name: str = field(metadata={"help": "The name"}) + _internal: str = field() count: int = field(metadata={"help": "How many there are"}) fields = DemoContext.documented_fields() @@ -165,7 +166,11 @@ async def test_template_context_matches_documented_contract( # The full contract: every key in the rendered template context is # documented, and every documented key is present in the context documented = documented_context_keys(page_name) - actual = set(await get_template_context(context_ds, path)) + actual = { + key + for key in await get_template_context(context_ds, path) + if not key.startswith("_") + } undocumented = actual - documented no_longer_present = documented - actual assert not undocumented, ( @@ -178,6 +183,33 @@ async def test_template_context_matches_documented_contract( ) +@pytest.mark.asyncio +async def test_count_truncated_replaces_count_limit_context_key(context_ds): + db = context_ds.databases["fixtures"] + previous_count_limit = db.count_limit + previous_cached_table_counts = db._cached_table_counts + db.count_limit = 10 + db._cached_table_counts = None + try: + table_context = await get_template_context(context_ds, "/fixtures/facetable") + assert table_context["count"] == 11 + assert table_context["count_truncated"] is True + assert "count_limit" not in table_context + + database_context = await get_template_context(context_ds, "/fixtures") + facetable = next( + table + for table in database_context["tables"] + if table["name"] == "facetable" + ) + assert facetable["count"] == 11 + assert facetable["count_truncated"] is True + assert "count_limit" not in database_context + finally: + db.count_limit = previous_count_limit + db._cached_table_counts = previous_cached_table_counts + + def test_base_context_keys_all_have_docs(): for name, doc in TEMPLATE_BASE_CONTEXT.items(): assert doc, "Base context key {} is missing docs".format(name) From 29971d97293a78ca44f66a1129d8c8b922b09c38 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 07:29:14 -0700 Subject: [PATCH 044/167] Clarify base template context docs --- datasette/app.py | 42 +++++++++++++++++++++------------------ docs/template_context.rst | 29 ++++++++++++--------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 9f0c8397..90de60a9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -35,6 +35,7 @@ from jinja2 import ( ChoiceLoader, Environment, FileSystemLoader, + pass_context, PrefixLoader, ) from jinja2.environment import Template @@ -330,30 +331,37 @@ def _to_string(value): return json.dumps(value, default=str) +@pass_context +def _legacy_template_csrftoken(context): + request = context.get("request") + if request and "csrftoken" in request.scope: + return request.scope["csrftoken"]() + return "" + + # Documentation for the variables Datasette.render_template() adds to the # context for every page. This is part of the documented template contract: # keys added in render_template() must be documented here - the contract # tests in tests/test_template_context.py enforce this, and the docs in # docs/template_context.rst are generated from it. TEMPLATE_BASE_CONTEXT = { - "request": "The current Request object, or None", - "crumb_items": "Async function returning breadcrumb navigation items for the current page", - "urls": "Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation", - "actor": "The currently authenticated actor dictionary, or None", - "menu_links": "Async function returning links for the Datasette application menu, including those added by plugins", - "display_actor": "Function returning a display string for an actor dictionary", + "request": "The current :ref:`Request object `, or None. Common properties include ``request.path``, ``request.args``, ``request.actor``, ``request.url_vars`` and ``request.host``.", + "crumb_items": 'Async function returning breadcrumb navigation items for the current page. Call it with ``request=request`` plus optional ``database=`` and ``table=`` arguments; it returns a list of ``{"href": url, "label": label}`` dictionaries.', + "urls": "Object with methods for constructing URLs within Datasette. Common methods include ``urls.instance()``, ``urls.database(database)``, ``urls.table(database, table)``, ``urls.query(database, query)``, ``urls.row(database, table, row_path)`` and ``urls.static(path)`` - see :ref:`internals_datasette_urls`.", + "actor": "The currently authenticated actor dictionary, or None. Actors usually include an ``id`` key and may include any other keys supplied by authentication plugins.", + "menu_links": "Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`.", + "display_actor": "Function that accepts an actor dictionary and returns the display string used in the navigation menu.", "show_logout": "True if the logout link should be shown in the navigation menu", "app_css_hash": "Hash of Datasette's app.css contents, used for cache busting", "edit_tools_js_hash": "Hash of Datasette's edit-tools.js contents, used for cache busting", "table_js_hash": "Hash of Datasette's table.js contents, used for cache busting", - "zip": "Python's zip() builtin, made available to template logic", - "body_scripts": "List of script blocks for the page body contributed by plugins", - "format_bytes": "Function that formats a number of bytes as a human-readable size", - "show_messages": "Function returning any messages set for the current user, clearing them in the process", - "extra_css_urls": "List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration", - "extra_js_urls": "List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page", - "base_url": "The configured base_url setting", - "csrftoken": "Function returning the CSRF token for the current request", + "zip": "Python's ``zip()`` builtin, made available to template logic", + "body_scripts": 'List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in `` - """.format(escape(ex.sql))).strip(), - title="SQL Interrupted", - status=400, - message_is_html=True, - ) - except (sqlite3.OperationalError, InvalidSql) as e: - raise DatasetteError(str(e), title="Invalid SQL", status=400) - - except sqlite3.OperationalError as e: - raise DatasetteError(str(e)) - - except DatasetteError: - raise - - end = time.perf_counter() - data["query_ms"] = (end - start) * 1000 - - # Special case for .jsono extension - redirect to _shape=objects - if _format == "jsono": - return self.redirect( - request, - path_with_added_args( - request, - {"_shape": "objects"}, - path=request.path.rsplit(".jsono", 1)[0] + ".json", - ), - forward_querystring=False, - ) - - if _format in self.ds.renderers.keys(): - # Dispatch request to the correct output format renderer - # (CSV is not handled here due to streaming) - result = call_with_supported_arguments( - self.ds.renderers[_format][0], - datasette=self.ds, - columns=data.get("columns") or [], - rows=data.get("rows") or [], - sql=data.get("query", {}).get("sql", None), - query_name=data.get("query_name"), - database=database, - table=data.get("table"), - request=request, - view_name=self.name, - truncated=False, # TODO: support this - error=data.get("error"), - # These will be deprecated in Datasette 1.0: - args=request.args, - data=data, - ) - if asyncio.iscoroutine(result): - result = await result - if result is None: - raise NotFound("No data") - if isinstance(result, dict): - r = Response( - body=result.get("body"), - status=result.get("status_code", status_code or 200), - content_type=result.get("content_type", "text/plain"), - headers=result.get("headers"), - ) - elif isinstance(result, Response): - r = result - if status_code is not None: - # Over-ride the status code - r.status = status_code - else: - assert False, f"{result} should be dict or Response" - else: - extras = {} - if callable(extra_template_data): - extras = extra_template_data() - if asyncio.iscoroutine(extras): - extras = await extras - else: - extras = extra_template_data - url_labels_extra = {} - if data.get("expandable_columns"): - url_labels_extra = {"_labels": "on"} - - renderers = {} - for key, (_, can_render) in self.ds.renderers.items(): - it_can_render = call_with_supported_arguments( - can_render, - datasette=self.ds, - columns=data.get("columns") or [], - rows=data.get("rows") or [], - sql=data.get("query", {}).get("sql", None), - query_name=data.get("query_name"), - database=database, - table=data.get("table"), - request=request, - view_name=self.name, - ) - it_can_render = await await_me_maybe(it_can_render) - if it_can_render: - renderers[key] = self.ds.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format=key, - extra_qs={**url_labels_extra}, - ) - ) - - url_csv_args = {"_size": "max", **url_labels_extra} - url_csv = self.ds.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format="csv", - extra_qs=url_csv_args, - ) - ) - url_csv_path = url_csv.split("?")[0] - context = { - **data, - **extras, - **{ - "renderers": renderers, - "url_csv": url_csv, - "url_csv_path": url_csv_path, - "url_csv_hidden_args": [ - (key, value) - for key, value in urllib.parse.parse_qsl(request.query_string) - if key not in ("_labels", "_facet", "_size") - ] - + [("_size", "max")], - "settings": self.ds.settings_dict(), - }, - } - if "metadata" not in context: - context["metadata"] = await self.ds.get_instance_metadata() - r = await self.render(templates, request=request, context=context) - if status_code is not None: - r.status = status_code - - ttl = request.args.get("_ttl", None) - if ttl is None or not ttl.isdigit(): - ttl = self.ds.setting("default_cache_ttl") - - return self.set_response_headers(r, ttl) - - def set_response_headers(self, response, ttl): - # Set far-future cache expiry - if self.ds.cache_headers and response.status == 200: - ttl = int(ttl) - if ttl == 0: - ttl_header = "no-cache" - else: - ttl_header = f"max-age={ttl}" - response.headers["Cache-Control"] = ttl_header - response.headers["Referrer-Policy"] = "no-referrer" - if self.ds.cors: - add_cors_headers(response.headers) - return response - - def _error(messages, status=400): return Response.json({"ok": False, "errors": messages}, status=status) diff --git a/datasette/views/row.py b/datasette/views/row.py index 7802f45e..3e3e52a9 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -1,21 +1,34 @@ +import asyncio +import json +import textwrap +import time +import urllib.parse +from dataclasses import dataclass, field, fields + +import markupsafe +import sqlite_utils + from datasette.utils.asgi import NotFound, Forbidden, Response from datasette.database import QueryInterrupted from datasette.events import UpdateRowEvent, DeleteRowEvent from datasette.resources import TableResource -from .base import DataView, BaseView, _error +from .base import BaseView, DatasetteError, _error, stream_csv from datasette.utils import ( + add_cors_headers, await_me_maybe, + call_with_supported_arguments, CustomRow, + InvalidSql, make_slot_function, path_from_row_pks, + path_with_added_args, + path_with_format, + path_with_removed_args, to_css_class, escape_sqlite, + sqlite3, ) from datasette.plugins import pm -from dataclasses import dataclass, field -import json -import markupsafe -import sqlite_utils from datasette.extras import extra_names_from_request, ExtraScope from . import Context, extra_field from .table import ( @@ -121,9 +134,259 @@ class RowContext(Context): ) -class RowView(DataView): +class RowView(BaseView): name = "row" - context_class = RowContext + + def redirect(self, request, path, forward_querystring=True, remove_args=None): + if request.query_string and "?" not in path and forward_querystring: + path = f"{path}?{request.query_string}" + if remove_args: + path = path_with_removed_args(request, remove_args, path=path) + response = Response.redirect(path) + response.headers["Link"] = f"<{path}>; rel=preload" + if self.ds.cors: + add_cors_headers(response.headers) + return response + + async def as_csv(self, request, database): + return await stream_csv(self.ds, self.data, request, database) + + async def get(self, request): + db = await self.ds.resolve_database(request) + database = db.name + database_route = db.route + format_ = request.url_vars.get("format") or "html" + data_kwargs = {} + + if format_ == "csv": + return await self.as_csv(request, database_route) + + if format_ == "html": + # HTML views default to expanding all foreign key labels + data_kwargs["default_labels"] = True + + extra_template_data = {} + start = time.perf_counter() + status_code = None + templates = () + try: + response_or_template_contexts = await self.data(request, **data_kwargs) + if isinstance(response_or_template_contexts, Response): + return response_or_template_contexts + # If it has four items, it includes an HTTP status code + if len(response_or_template_contexts) == 4: + ( + data, + extra_template_data, + templates, + status_code, + ) = response_or_template_contexts + else: + data, extra_template_data, templates = response_or_template_contexts + except QueryInterrupted as ex: + raise DatasetteError( + textwrap.dedent(""" +

SQL query took too long. The time limit is controlled by the + sql_time_limit_ms + configuration option.

+ + + """.format(markupsafe.escape(ex.sql))).strip(), + title="SQL Interrupted", + status=400, + message_is_html=True, + ) + except (sqlite3.OperationalError, InvalidSql) as e: + raise DatasetteError(str(e), title="Invalid SQL", status=400) + except sqlite3.OperationalError as e: + raise DatasetteError(str(e)) + except DatasetteError: + raise + + end = time.perf_counter() + data["query_ms"] = (end - start) * 1000 + + # Special case for .jsono extension - redirect to _shape=objects + if format_ == "jsono": + return self.redirect( + request, + path_with_added_args( + request, + {"_shape": "objects"}, + path=request.path.rsplit(".jsono", 1)[0] + ".json", + ), + forward_querystring=False, + ) + + if format_ in self.ds.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + self.ds.renderers[format_][0], + datasette=self.ds, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=database, + table=data.get("table"), + request=request, + view_name=self.name, + truncated=False, # TODO: support this + error=data.get("error"), + # These will be deprecated in Datasette 1.0: + args=request.args, + data=data, + ) + if asyncio.iscoroutine(result): + result = await result + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + response = Response( + body=result.get("body"), + status=result.get("status_code", status_code or 200), + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + response = result + if status_code is not None: + # Over-ride the status code + response.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + response = await self.html(request, data, extra_template_data, templates) + if status_code is not None: + response.status = status_code + else: + raise NotFound("Invalid format: {}".format(format_)) + + ttl = request.args.get("_ttl", None) + if ttl is None or not ttl.isdigit(): + ttl = self.ds.setting("default_cache_ttl") + + return self.set_response_headers(response, ttl) + + async def html(self, request, data, extra_template_data, templates): + extras = {} + if callable(extra_template_data): + extras = extra_template_data() + if asyncio.iscoroutine(extras): + extras = await extras + else: + extras = extra_template_data + + url_labels_extra = {} + if data.get("expandable_columns"): + url_labels_extra = {"_labels": "on"} + + renderers = {} + for key, (_, can_render) in self.ds.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=self.ds, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=data.get("database"), + table=data.get("table"), + request=request, + view_name=self.name, + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = self.ds.urls.path( + path_with_format( + request=request, + path=request.scope.get("route_path"), + format=key, + extra_qs={**url_labels_extra}, + ) + ) + + url_csv_args = {"_size": "max", **url_labels_extra} + url_csv = self.ds.urls.path( + path_with_format( + request=request, + path=request.scope.get("route_path"), + format="csv", + extra_qs=url_csv_args, + ) + ) + url_csv_path = url_csv.split("?")[0] + context = {**data, **extras} + if "metadata" not in context: + context["metadata"] = await self.ds.get_instance_metadata() + + environment = self.ds.get_jinja_environment(request) + template = environment.select_template(templates) + alternate_url_json = self.ds.absolute_url( + request, + self.ds.urls.path( + path_with_format( + request=request, + path=request.scope.get("route_path"), + format="json", + ) + ), + ) + explicit_context = { + "renderers": renderers, + "url_csv": url_csv, + "url_csv_path": url_csv_path, + "url_csv_hidden_args": [ + (key, value) + for key, value in urllib.parse.parse_qsl(request.query_string) + if key not in ("_labels", "_facet", "_size") + ] + + [("_size", "max")], + "settings": self.ds.settings_dict(), + "alternate_url_json": alternate_url_json, + "select_templates": [ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + } + declared_fields = {f.name for f in fields(RowContext)} + context = { + key: value + for key, value in context.items() + if key in declared_fields and key not in explicit_context + } + + return Response.html( + await self.ds.render_template( + template, + RowContext(**context, **explicit_context), + request=request, + view_name=self.name, + ), + headers={ + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + }, + ) + + def set_response_headers(self, response, ttl): + # Set far-future cache expiry + if self.ds.cache_headers and response.status == 200: + ttl = int(ttl) + if ttl == 0: + ttl_header = "no-cache" + else: + ttl_header = f"max-age={ttl}" + response.headers["Cache-Control"] = ttl_header + response.headers["Referrer-Policy"] = "no-referrer" + if self.ds.cors: + add_cors_headers(response.headers) + return response async def data(self, request, default_labels=False): resolved = await self.ds.resolve_row(request) From a43e76c31a3c4b96ab7289b5f3202c7c3537bf6a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 09:08:40 -0700 Subject: [PATCH 049/167] Construct row template context explicitly --- datasette/views/row.py | 63 +++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index 3e3e52a9..ef39dce6 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -3,7 +3,7 @@ import json import textwrap import time import urllib.parse -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field import markupsafe import sqlite_utils @@ -336,34 +336,45 @@ class RowView(BaseView): ) ), ) - explicit_context = { - "renderers": renderers, - "url_csv": url_csv, - "url_csv_path": url_csv_path, - "url_csv_hidden_args": [ - (key, value) - for key, value in urllib.parse.parse_qsl(request.query_string) - if key not in ("_labels", "_facet", "_size") - ] - + [("_size", "max")], - "settings": self.ds.settings_dict(), - "alternate_url_json": alternate_url_json, - "select_templates": [ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], - } - declared_fields = {f.name for f in fields(RowContext)} - context = { - key: value - for key, value in context.items() - if key in declared_fields and key not in explicit_context - } - return Response.html( await self.ds.render_template( template, - RowContext(**context, **explicit_context), + RowContext( + columns=context["columns"], + database=context["database"], + database_color=context["database_color"], + foreign_key_tables=context["foreign_key_tables"], + metadata=context["metadata"], + primary_keys=context["primary_keys"], + private=context["private"], + table=context["table"], + ok=context["ok"], + rows=context["rows"], + primary_key_values=context["primary_key_values"], + query_ms=context["query_ms"], + display_columns=context["display_columns"], + display_rows=context["display_rows"], + custom_table_templates=context["custom_table_templates"], + row_actions=context["row_actions"], + row_mutation_ui=context["row_mutation_ui"], + table_page_data=context["table_page_data"], + top_row=context["top_row"], + renderers=renderers, + url_csv=url_csv, + url_csv_path=url_csv_path, + url_csv_hidden_args=[ + (key, value) + for key, value in urllib.parse.parse_qsl(request.query_string) + if key not in ("_labels", "_facet", "_size") + ] + + [("_size", "max")], + settings=self.ds.settings_dict(), + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + alternate_url_json=alternate_url_json, + ), request=request, view_name=self.name, ), From 59ab0c0ca0b2d86913102dace6813f42f947d772 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 11:30:30 -0700 Subject: [PATCH 050/167] Clarify template context metadata names --- datasette/template_contexts.py | 2 +- datasette/views/__init__.py | 14 ++++---- datasette/views/database.py | 4 +-- datasette/views/row.py | 20 +++++------ datasette/views/table.py | 62 +++++++++++++++++----------------- docs/template_context_doc.py | 2 +- tests/test_template_context.py | 24 +++++++------ 7 files changed, 65 insertions(+), 63 deletions(-) diff --git a/datasette/template_contexts.py b/datasette/template_contexts.py index 5fe6a80e..232eb051 100644 --- a/datasette/template_contexts.py +++ b/datasette/template_contexts.py @@ -7,7 +7,7 @@ the documentation lives next to the code it describes: - Every page renders a Context dataclass defined in its view module (DatabaseContext, QueryContext in views/database.py, TableContext in views/table.py, RowContext in views/row.py). Fields added by view code - carry ``help`` metadata; fields declared with extra_field() take their + carry ``help`` metadata; fields declared with from_extra() take their documentation from the description on the matching Extra class in views/table_extras.py. - The keys render_template() adds to every page are documented in diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index e503ed35..6fba1c90 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -10,7 +10,7 @@ class ContextField: from_extra: bool = False -def extra_field(): +def from_extra(): """ Declare a Context dataclass field whose value comes from a registered Extra of the same name - its documentation is the Extra description, @@ -23,7 +23,7 @@ def extra_field(): class Context: "Base class for all documented contexts" - # Set on subclasses whose extra_field() fields should be resolved + # Set on subclasses whose from_extra() fields should be resolved # against the extras registry for this scope extras_scope = None @@ -34,8 +34,8 @@ class Context: for f in dataclasses.fields(cls): if f.name.startswith("_"): continue - from_extra = bool(f.metadata.get("from_extra")) - if from_extra: + is_from_extra = bool(f.metadata.get("from_extra")) + if is_from_extra: help_text = cls._extra_description(f.name) else: help_text = f.metadata.get("help", "") @@ -44,7 +44,7 @@ class Context: name=f.name, type_name=getattr(f.type, "__name__", str(f.type)), help=help_text, - from_extra=from_extra, + from_extra=is_from_extra, ) ) return documented @@ -59,14 +59,14 @@ class Context: extra_class = table_extra_registry.classes_by_name[name] except KeyError: raise KeyError( - "{}.{} is declared with extra_field() but there is no " + "{}.{} is declared with from_extra() but there is no " "registered extra of that name".format(cls.__name__, name) ) if cls.extras_scope is not None and not extra_class.available_for( cls.extras_scope ): raise ValueError( - "{}.{} is declared with extra_field() but the {} extra is " + "{}.{} is declared with from_extra() but the {} extra is " "not available for scope {}".format( cls.__name__, name, name, cls.extras_scope ) diff --git a/datasette/views/database.py b/datasette/views/database.py index 7409e7d3..0b4ca647 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -258,7 +258,7 @@ class DatabaseView(View): class DatabaseContext(Context): "The page listing the tables, views and queries in a database, e.g. /fixtures." - template = "database.html" + documented_template = "database.html" database: str = field(metadata={"help": "The name of the database"}) private: bool = field( @@ -341,7 +341,7 @@ class DatabaseContext(Context): class QueryContext(Context): "The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)." - template = "query.html" + documented_template = "query.html" database: str = field(metadata={"help": "The name of the database being queried"}) database_color: str = field(metadata={"help": "The color of the database"}) diff --git a/datasette/views/row.py b/datasette/views/row.py index ef39dce6..129216b9 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -30,7 +30,7 @@ from datasette.utils import ( ) from datasette.plugins import pm from datasette.extras import extra_names_from_request, ExtraScope -from . import Context, extra_field +from . import Context, from_extra from .table import ( display_columns_and_rows, _table_page_data, @@ -43,19 +43,19 @@ from .table_extras import RowExtraContext, resolve_row_extras, table_extra_regis class RowContext(Context): "The page showing an individual row, e.g. /fixtures/facetable/1." - template = "row.html" + documented_template = "row.html" extras_scope = ExtraScope.ROW # Fields resolved by registered extras - their documentation comes # from the description on each Extra class in table_extras.py - columns: list = extra_field() - database: str = extra_field() - database_color: str = extra_field() - foreign_key_tables: list = extra_field() - metadata: dict = extra_field() - primary_keys: list = extra_field() - private: bool = extra_field() - table: str = extra_field() + columns: list = from_extra() + database: str = from_extra() + database_color: str = from_extra() + foreign_key_tables: list = from_extra() + metadata: dict = from_extra() + primary_keys: list = from_extra() + private: bool = from_extra() + table: str = from_extra() # Fields added by the view code ok: bool = field( diff --git a/datasette/views/table.py b/datasette/views/table.py index 1e182937..449c6216 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -49,7 +49,7 @@ import sqlite_utils from dataclasses import dataclass, field, fields from datasette.extras import ExtraScope -from . import Context, extra_field +from . import Context, from_extra from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView from .table_create_alter import ( @@ -73,40 +73,40 @@ from .table_extras import ( class TableContext(Context): "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable." - template = "table.html" + documented_template = "table.html" extras_scope = ExtraScope.TABLE # Fields resolved by registered extras - their documentation comes # from the description on each Extra class in table_extras.py - actions: callable = extra_field() - all_columns: list = extra_field() - columns: list = extra_field() - count: int = extra_field() - count_sql: str = extra_field() - custom_table_templates: list = extra_field() - database: str = extra_field() - database_color: str = extra_field() - display_columns: list = extra_field() - display_rows: list = extra_field() - expandable_columns: list = extra_field() - facet_results: dict = extra_field() - facets_timed_out: list = extra_field() - filters: Filters = extra_field() - form_hidden_args: list = extra_field() - human_description_en: str = extra_field() - is_view: bool = extra_field() - metadata: dict = extra_field() - next_url: str = extra_field() - primary_keys: list = extra_field() - private: bool = extra_field() - query: dict = extra_field() - renderers: dict = extra_field() - set_column_type_ui: dict = extra_field() - sorted_facet_results: list = extra_field() - suggested_facets: list = extra_field() - table: str = extra_field() - table_definition: str = extra_field() - view_definition: str = extra_field() + actions: callable = from_extra() + all_columns: list = from_extra() + columns: list = from_extra() + count: int = from_extra() + count_sql: str = from_extra() + custom_table_templates: list = from_extra() + database: str = from_extra() + database_color: str = from_extra() + display_columns: list = from_extra() + display_rows: list = from_extra() + expandable_columns: list = from_extra() + facet_results: dict = from_extra() + facets_timed_out: list = from_extra() + filters: Filters = from_extra() + form_hidden_args: list = from_extra() + human_description_en: str = from_extra() + is_view: bool = from_extra() + metadata: dict = from_extra() + next_url: str = from_extra() + primary_keys: list = from_extra() + private: bool = from_extra() + query: dict = from_extra() + renderers: dict = from_extra() + set_column_type_ui: dict = from_extra() + sorted_facet_results: list = from_extra() + suggested_facets: list = from_extra() + table: str = from_extra() + table_definition: str = from_extra() + view_definition: str = from_extra() # Fields added by the view code ok: bool = field( diff --git a/docs/template_context_doc.py b/docs/template_context_doc.py index c3ec77e2..a5f4fb6f 100644 --- a/docs/template_context_doc.py +++ b/docs/template_context_doc.py @@ -27,7 +27,7 @@ def template_context(cog): for klass in PAGES.values(): title = "{} page".format(klass.__name__.removesuffix("Context")) intro = "{} Rendered using the ``{}`` template.".format( - klass.__doc__, klass.template + klass.__doc__, klass.documented_template ) _section(cog, title, intro) if klass.extras_scope is not None: diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 8d78bf77..4c452774 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -40,20 +40,22 @@ def test_context_class_fields_all_have_help(klass): @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) -def test_context_class_has_docstring_and_template(klass): +def test_context_class_has_docstring_and_documented_template(klass): assert klass.__doc__, "{} is missing a docstring".format(klass.__name__) - assert klass.template, "{} is missing a template".format(klass.__name__) + assert klass.documented_template, "{} is missing a documented_template".format( + klass.__name__ + ) -def test_extra_field_documentation_comes_from_the_extra_class(): - from datasette.views import extra_field +def test_from_extra_documentation_comes_from_the_extra_class(): + from datasette.views import from_extra from datasette.views.table_extras import CountExtra @dataclass class DemoContext(Context): extras_scope = ExtraScope.TABLE - count: int = extra_field() + count: int = from_extra() name: str = field(metadata={"help": "The name"}) fields = {f.name: f for f in DemoContext.documented_fields()} @@ -63,28 +65,28 @@ def test_extra_field_documentation_comes_from_the_extra_class(): assert not fields["name"].from_extra -def test_extra_field_must_match_a_registered_extra(): - from datasette.views import extra_field +def test_from_extra_must_match_a_registered_extra(): + from datasette.views import from_extra @dataclass class BadContext(Context): extras_scope = ExtraScope.TABLE - not_a_real_extra: str = extra_field() + not_a_real_extra: str = from_extra() with pytest.raises(KeyError): BadContext.documented_fields() -def test_extra_field_must_be_available_for_the_scope(): - from datasette.views import extra_field +def test_from_extra_must_be_available_for_the_scope(): + from datasette.views import from_extra @dataclass class WrongScopeContext(Context): extras_scope = ExtraScope.ROW # count is a TABLE-scope extra, not available for ROW - count: int = extra_field() + count: int = from_extra() with pytest.raises(ValueError): WrongScopeContext.documented_fields() From 34d9a3bf33e3c4ff610f7d3a633f63c3d7272cc9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:11:26 -0700 Subject: [PATCH 051/167] Use dataclasses for database table context --- datasette/app.py | 17 +++++++++- datasette/views/__init__.py | 17 +++++++++- datasette/views/database.py | 57 +++++++++++++++++++++------------- docs/contributing.rst | 13 ++++++++ docs/template_context.rst | 8 ++--- tests/test_template_context.py | 10 ++++++ 6 files changed, 95 insertions(+), 27 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 90de60a9..57f893fe 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -331,6 +331,15 @@ def _to_string(value): return json.dumps(value, default=str) +def _template_context_json_default(value): + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + field.name: getattr(value, field.name) + for field in dataclasses.fields(value) + } + return repr(value) + + @pass_context def _legacy_template_csrftoken(context): request = context.get("request") @@ -2390,7 +2399,13 @@ class Datasette: } if request and request.args.get("_context") and self.setting("template_debug"): return "
{}
".format( - escape(json.dumps(template_context, default=repr, indent=4)) + escape( + json.dumps( + template_context, + default=_template_context_json_default, + indent=4, + ) + ) ) return await template.render_async(template_context) diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index 6fba1c90..ed7e175f 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -1,5 +1,7 @@ from dataclasses import dataclass import dataclasses +import types +import typing @dataclass(frozen=True) @@ -10,6 +12,19 @@ class ContextField: from_extra: bool = False +def _type_name(type_): + if type_ is type(None): + return "None" + origin = typing.get_origin(type_) + args = typing.get_args(type_) + if origin in (typing.Union, types.UnionType): + return " | ".join(_type_name(arg) for arg in args) + if origin is not None: + name = getattr(origin, "__name__", str(origin).removeprefix("typing.")) + return "{}[{}]".format(name, ", ".join(_type_name(arg) for arg in args)) + return getattr(type_, "__name__", str(type_).removeprefix("typing.")) + + def from_extra(): """ Declare a Context dataclass field whose value comes from a registered @@ -42,7 +57,7 @@ class Context: documented.append( ContextField( name=f.name, - type_name=getattr(f.type, "__name__", str(f.type)), + type_name=_type_name(f.type), help=help_text, from_extra=is_from_extra, ) diff --git a/datasette/views/database.py b/datasette/views/database.py index 0b4ca647..fd985b77 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from urllib.parse import parse_qsl, urlencode import asyncio import hashlib @@ -45,6 +45,21 @@ from .table_create_alter import _create_table_ui_context from . import Context +@dataclass +class DatabaseTable: + "Summary of a table or view shown on database and query pages." + + name: str + columns: list[str] + primary_keys: list[str] + count: int | None + count_truncated: bool + hidden: bool + fts_table: str | None + foreign_keys: dict[str, list[dict[str, str]]] + private: bool + + class DatabaseView(View): async def get(self, request, datasette): format_ = request.url_vars.get("format") or "html" @@ -169,8 +184,8 @@ class DatabaseView(View): "private": private, "path": datasette.urls.database(database), "size": db.size, - "tables": tables, - "hidden_count": len([t for t in tables if t["hidden"]]), + "tables": [asdict(table) for table in tables], + "hidden_count": len([table for table in tables if table.hidden]), "views": sql_views, "queries": [stored_query_to_dict(query) for query in stored_queries], "queries_more": queries_more, @@ -211,7 +226,7 @@ class DatabaseView(View): path=datasette.urls.database(database), size=db.size, tables=tables, - hidden_count=len([t for t in tables if t["hidden"]]), + hidden_count=len([table for table in tables if table.hidden]), views=sql_views, queries=stored_queries, queries_more=queries_more, @@ -266,9 +281,9 @@ class DatabaseContext(Context): ) path: str = field(metadata={"help": "The URL path to this database"}) size: int = field(metadata={"help": "The size of the database in bytes"}) - tables: list = field( + tables: list[DatabaseTable] = field( metadata={ - "help": "List of table dictionaries in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` keys. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." + "help": "List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." } ) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) @@ -391,9 +406,9 @@ class QueryContext(Context): save_query_url: str = field( metadata={"help": "URL to save the current arbitrary SQL as a query"} ) - tables: list = field( + tables: list[DatabaseTable] = field( metadata={ - "help": "List of table dictionaries in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` keys. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." + "help": "List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." } ) named_parameter_values: dict = field( @@ -456,7 +471,7 @@ class QueryContext(Context): ) -async def get_tables(datasette, request, db, allowed_dict): +async def get_tables(datasette, request, db, allowed_dict) -> list[DatabaseTable]: """ Get list of tables with metadata for the database view. @@ -477,21 +492,21 @@ async def get_tables(datasette, request, db, allowed_dict): table_columns = await db.table_columns(table) tables.append( - { - "name": table, - "columns": table_columns, - "primary_keys": await db.primary_keys(table), - "count": table_counts[table], - "count_truncated": _table_count_truncated( + DatabaseTable( + name=table, + columns=table_columns, + primary_keys=await db.primary_keys(table), + count=table_counts[table], + count_truncated=_table_count_truncated( datasette, db, table, table_counts[table] ), - "hidden": table in hidden_table_names, - "fts_table": await db.fts_table(table), - "foreign_keys": all_foreign_keys[table], - "private": allowed_dict[table].private, - } + hidden=table in hidden_table_names, + fts_table=await db.fts_table(table), + foreign_keys=all_foreign_keys[table], + private=allowed_dict[table].private, + ) ) - tables.sort(key=lambda t: (t["hidden"], t["name"])) + tables.sort(key=lambda table: (table.hidden, table.name)) return tables diff --git a/docs/contributing.rst b/docs/contributing.rst index 3b45834c..142c2f41 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -309,6 +309,19 @@ To update these pages, run the following command:: uv run cog -r docs/*.rst +.. _contributing_template_contexts: + +Documented template contexts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Datasette's documented template contexts are part of the public API for custom templates. They are defined as dataclasses next to the view code that renders them, for example ``DatabaseContext`` and ``QueryContext`` in ``datasette/views/database.py``. + +Every documented context class inherits from ``datasette.views.Context``. Fields that are added directly by view code should be declared as dataclass fields with ``help`` metadata, which is used to generate :ref:`template_context`. Fields resolved through the page extras system should use ``from_extra()`` so their documentation comes from the matching ``Extra`` class. + +Use ``documented_template`` on each context class to record the canonical template named in the generated documentation. This should be a string such as ``"database.html"``. Runtime template selection still happens in the view code, since most pages consider more specific template names before falling back to the canonical one. + +When a context field contains repeated structured data, prefer a small nested dataclass over an anonymous dictionary. For example, a field containing table summaries should be annotated as ``list[DatabaseTable]`` where ``DatabaseTable`` is a dataclass describing the keys and value types. This keeps the Python contract and generated documentation clear. JSON responses and ``?_context=1`` debug output will convert nested dataclasses back to JSON objects at the response boundary. + .. _contributing_continuous_deployment: Continuously deployed demo instances diff --git a/docs/template_context.rst b/docs/template_context.rst index 12f656d0..311de8bb 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -145,8 +145,8 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``table_columns`` - ``dict`` Dictionary mapping table names to lists of column names, used to power SQL autocomplete. -``tables`` - ``list`` - List of table dictionaries in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` keys. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total. +``tables`` - ``list[DatabaseTable]`` + List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total. ``top_database`` - ``callable`` Async callable that renders the ``top_database`` plugin slot for this database and returns HTML. @@ -234,8 +234,8 @@ The page for arbitrary SQL queries (/database/-/query?sql=...) and stored querie ``table_columns`` - ``dict`` Dictionary mapping table names to lists of column names, used to power SQL autocomplete. -``tables`` - ``list`` - List of table dictionaries in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` keys. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total. +``tables`` - ``list[DatabaseTable]`` + List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total. ``top_query`` - ``callable`` Async callable that renders the ``top_query`` plugin slot for this query and returns HTML. diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 4c452774..f1919891 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -18,16 +18,22 @@ from datasette.views import Context def test_documented_fields(): + @dataclass + class DemoNested: + name: str + @dataclass class DemoContext(Context): name: str = field(metadata={"help": "The name"}) _internal: str = field() count: int = field(metadata={"help": "How many there are"}) + items: list[DemoNested] = field(metadata={"help": "Nested items"}) fields = DemoContext.documented_fields() assert [(f.name, f.type_name, f.help) for f in fields] == [ ("name", "str", "The name"), ("count", "int", "How many there are"), + ("items", "list[DemoNested]", "Nested items"), ] @@ -230,3 +236,7 @@ def test_template_context_docs_cover_every_documented_key(): assert "``{}``".format(context_field.name) in docs, "{} ({} page)".format( context_field.name, page_name ) + assert ( + "``{}`` - ``{}``".format(context_field.name, context_field.type_name) + in docs + ), "{} type ({} page)".format(context_field.name, page_name) From 0d1c097396b8e39f4c5d20ea16282c295d2e4e0b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:16:42 -0700 Subject: [PATCH 052/167] Document database views and queries as dataclasses --- datasette/views/database.py | 22 +++++++++++++++------- docs/template_context.rst | 8 ++++---- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index fd985b77..e02de657 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -11,7 +11,7 @@ import textwrap from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource -from datasette.stored_queries import stored_query_to_dict +from datasette.stored_queries import StoredQuery, stored_query_to_dict from datasette.write_sql import QueryWriteRejected from datasette.utils import ( add_cors_headers, @@ -60,6 +60,14 @@ class DatabaseTable: private: bool +@dataclass +class DatabaseViewInfo: + "Summary of a SQLite view shown on the database page." + + name: str + private: bool + + class DatabaseView(View): async def get(self, request, datasette): format_ = request.url_vars.get("format") or "html" @@ -109,7 +117,7 @@ class DatabaseView(View): # Filter to just views view_names_set = set(await db.view_names()) sql_views = [ - {"name": name, "private": allowed_dict[name].private} + DatabaseViewInfo(name=name, private=allowed_dict[name].private) for name in allowed_dict if name in view_names_set ] @@ -186,7 +194,7 @@ class DatabaseView(View): "size": db.size, "tables": [asdict(table) for table in tables], "hidden_count": len([table for table in tables if table.hidden]), - "views": sql_views, + "views": [asdict(view) for view in sql_views], "queries": [stored_query_to_dict(query) for query in stored_queries], "queries_more": queries_more, "queries_count": queries_count, @@ -287,14 +295,14 @@ class DatabaseContext(Context): } ) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) - views: list = field( + views: list[DatabaseViewInfo] = field( metadata={ - "help": "List of SQLite view dictionaries. Each item has ``name`` and ``private`` keys." + "help": "List of ``DatabaseViewInfo`` objects describing SQLite views in the database. Each item has ``name`` and ``private`` attributes." } ) - queries: list = field( + queries: list[StoredQuery] = field( metadata={ - "help": "List of stored query objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``." + "help": "List of ``StoredQuery`` objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``." } ) queries_more: bool = field( diff --git a/docs/template_context.rst b/docs/template_context.rst index 311de8bb..52ccca49 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -124,8 +124,8 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``private`` - ``bool`` Boolean indicating if this is a private database -``queries`` - ``list`` - List of stored query objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``. +``queries`` - ``list[StoredQuery]`` + List of ``StoredQuery`` objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``. ``queries_count`` - ``int`` Count of visible stored queries @@ -151,8 +151,8 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``top_database`` - ``callable`` Async callable that renders the ``top_database`` plugin slot for this database and returns HTML. -``views`` - ``list`` - List of SQLite view dictionaries. Each item has ``name`` and ``private`` keys. +``views`` - ``list[DatabaseViewInfo]`` + List of ``DatabaseViewInfo`` objects describing SQLite views in the database. Each item has ``name`` and ``private`` attributes. Query page ---------- From 8276879997661ae618ffe984ab6a183d3b62788f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:24:42 -0700 Subject: [PATCH 053/167] Construct table context explicitly --- datasette/views/table.py | 110 ++++++++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 36 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 449c6216..4b923d20 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -46,7 +46,7 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response from datasette.filters import Filters import sqlite_utils -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field from datasette.extras import ExtraScope from . import Context, from_extra @@ -1825,44 +1825,82 @@ async def table_view_traced(datasette, request): ) } ) - # Only keys declared on TableContext are part of the documented - # template contract - anything else in data (e.g. extras requested - # with ?_extra= on the HTML page, or extra filter context added by - # filters_from_request plugins) is dropped here - declared_fields = {f.name for f in fields(TableContext)} + table_context = TableContext( + actions=data["actions"], + all_columns=data["all_columns"], + columns=data["columns"], + count=data["count"], + count_sql=data["count_sql"], + custom_table_templates=data["custom_table_templates"], + database=data["database"], + database_color=data["database_color"], + display_columns=data["display_columns"], + display_rows=data["display_rows"], + expandable_columns=data["expandable_columns"], + facet_results=data["facet_results"], + facets_timed_out=data["facets_timed_out"], + filters=data["filters"], + form_hidden_args=data["form_hidden_args"], + human_description_en=data["human_description_en"], + is_view=data["is_view"], + metadata=data["metadata"], + next_url=data["next_url"], + primary_keys=data["primary_keys"], + private=data["private"], + query=data["query"], + renderers=data["renderers"], + set_column_type_ui=data["set_column_type_ui"], + sorted_facet_results=data["sorted_facet_results"], + suggested_facets=data["suggested_facets"], + table=data["table"], + table_definition=data["table_definition"], + view_definition=data["view_definition"], + ok=data["ok"], + next=data["next"], + count_truncated=data["count_truncated"], + rows=data["rows"], + filter_columns=data["filter_columns"], + supports_search=data["supports_search"], + extra_wheres_for_ui=data["extra_wheres_for_ui"], + url_csv=data["url_csv"], + url_csv_path=data["url_csv_path"], + url_csv_hidden_args=data["url_csv_hidden_args"], + sort=data["sort"], + sort_desc=data["sort_desc"], + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=any(c["sortable"] for c in data["display_columns"]), + allow_execute_sql=await datasette.allowed( + action="execute-sql", + resource=DatabaseResource(database=resolved.db.name), + actor=request.actor, + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + top_table=make_slot_function( + "top_table", + datasette, + request, + database=resolved.db.name, + table=resolved.table, + ), + table_page_data=data["table_page_data"], + table_insert_ui=data["table_insert_ui"], + table_alter_ui=data["table_alter_ui"], + ) r = Response.html( await datasette.render_template( template, - TableContext( - **{k: v for k, v in data.items() if k in declared_fields}, - append_querystring=append_querystring, - path_with_replaced_args=path_with_replaced_args, - fix_path=datasette.urls.path, - settings=datasette.settings_dict(), - # TODO: review up all of these hacks: - alternate_url_json=alternate_url_json, - datasette_allow_facet=( - "true" if datasette.setting("allow_facet") else "false" - ), - is_sortable=any(c["sortable"] for c in data["display_columns"]), - allow_execute_sql=await datasette.allowed( - action="execute-sql", - resource=DatabaseResource(database=resolved.db.name), - actor=request.actor, - ), - query_ms=1.2, - select_templates=[ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], - top_table=make_slot_function( - "top_table", - datasette, - request, - database=resolved.db.name, - table=resolved.table, - ), - ), + table_context, request=request, view_name="table", ), From 0c523dda209e3b545c52cd3b113d788cba9a342b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:29:21 -0700 Subject: [PATCH 054/167] Remove count truncated context test --- tests/test_template_context.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index f1919891..7923d0e7 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -191,33 +191,6 @@ async def test_template_context_matches_documented_contract( ) -@pytest.mark.asyncio -async def test_count_truncated_replaces_count_limit_context_key(context_ds): - db = context_ds.databases["fixtures"] - previous_count_limit = db.count_limit - previous_cached_table_counts = db._cached_table_counts - db.count_limit = 10 - db._cached_table_counts = None - try: - table_context = await get_template_context(context_ds, "/fixtures/facetable") - assert table_context["count"] == 11 - assert table_context["count_truncated"] is True - assert "count_limit" not in table_context - - database_context = await get_template_context(context_ds, "/fixtures") - facetable = next( - table - for table in database_context["tables"] - if table["name"] == "facetable" - ) - assert facetable["count"] == 11 - assert facetable["count_truncated"] is True - assert "count_limit" not in database_context - finally: - db.count_limit = previous_count_limit - db._cached_table_counts = previous_cached_table_counts - - def test_base_context_keys_all_have_docs(): for name, doc in TEMPLATE_BASE_CONTEXT.items(): assert doc, "Base context key {} is missing docs".format(name) From e3ff63b0f93303a8dfe23d8d2d84bbc9551c3eea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:46:15 -0700 Subject: [PATCH 055/167] Warn plugin authors to avoid name conflicts with extra_template_vars Closes #1988 --- docs/plugin_hooks.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index d2f87e27..66419ec4 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -230,6 +230,10 @@ Function that returns an awaitable function that returns a dictionary Datasette runs Jinja2 in `async mode `__, which means you can add awaitable functions to the template scope and they will be automatically awaited when they are rendered by the template. +.. warning:: + + Be careful not to accidentally define a variable that conflicts with one that Datasette is already using for something else. Check :ref:`the template context documentation ` to see the variables defined by Datasette core. + Here's an example plugin that adds a ``"user_agent"`` variable to the template context containing the current request's User-Agent header: .. code-block:: python From a4f74d1d2bea4484f026c9adacae6ac961fd0f82 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:50:23 -0700 Subject: [PATCH 056/167] More unreleased changes in changelog --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 697df87d..01ef7f18 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,11 @@ Unreleased - New "Alter table" table action and ``//

/-/alter`` :ref:`JSON API ` for changing existing tables: add, rename, reorder and drop columns; change column types, defaults, ``NOT NULL`` constraints, primary keys and foreign keys; and rename the table. The alter table dialog also includes a "Drop table" button. (:issue:`2788`) - New ``//-/foreign-key-targets`` and ``//
/-/foreign-key-suggestions`` JSON APIs for discovering valid single-column foreign key targets and suggested relationships. - Create and alter table dialogs share their column-editing controls, including literal and expression defaults, custom column types, foreign keys and column ordering. +- The "Write to this database" page now includes a Create table starter template, alongside the existing Insert, Update and Delete templates. (:pr:`2794`) +- New :ref:`template_context` documentation listing the variables available to custom templates for Datasette's core pages. Variables documented there are treated as a stable API for custom templates until Datasette 2.0. The documentation is generated from dataclass definitions next to the view code, with tests that compare the documented fields against the actual contexts rendered by the database, table, query and row pages. (:issue:`1510`, :issue:`2127`, :issue:`1477`, :pr:`2803`) +- Database and table pages now use the ``count_truncated`` template context value to display capped row counts as ``>N rows`. +- Significant visual improvements to the table filter form UI, plus working add/remove filter buttons. (:issue:`2798`) +- Improved edit row icon on table pages. (:issue:`2796`) .. _v1_0_a34: From 5eca46a4bc5adebfb724a1c8e0e42852232152ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 13:44:58 -0700 Subject: [PATCH 057/167] Add cache-busted static asset helper (#2804) * Add cache-busted static asset helper Add a static() helper for Datasette, plugin, and mounted static assets that appends content-based hashes, caches hashes in production, and serves matching hashed asset URLs with immutable far-future cache headers. Closes #2800 --- datasette/app.py | 88 +++++++++++----- datasette/handle_exception.py | 1 - datasette/templates/_codemirror.html | 4 +- datasette/templates/api_explorer.html | 2 +- datasette/templates/base.html | 6 +- datasette/templates/database.html | 2 +- datasette/templates/debug_allowed.html | 2 +- datasette/templates/debug_autocomplete.html | 2 +- datasette/templates/debug_check.html | 2 +- datasette/templates/debug_rules.html | 2 +- datasette/templates/patterns.html | 2 +- datasette/templates/row.html | 4 +- datasette/templates/table.html | 10 +- datasette/utils/__init__.py | 11 ++ datasette/utils/asgi.py | 13 ++- docs/changelog.rst | 3 +- docs/contributing.rst | 3 + docs/custom_templates.rst | 39 +++++++ docs/internals.rst | 46 +++++++++ docs/plugin_hooks.rst | 5 +- docs/template_context.rst | 9 -- docs/writing_plugins.rst | 11 +- tests/test_html.py | 44 +++++++- tests/test_internals_datasette.py | 108 ++++++++++++++++++++ tests/test_utils.py | 7 ++ 25 files changed, 361 insertions(+), 65 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 57f893fe..9c9b7de4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -12,7 +12,6 @@ import dataclasses import datetime import functools import glob -import hashlib import httpx import importlib.metadata import inspect @@ -123,6 +122,7 @@ from .utils import ( parse_metadata, resolve_env_secrets, resolve_routes, + sha256_file, tilde_decode, tilde_encode, to_css_class, @@ -314,7 +314,7 @@ async def favicon(request, send): send, str(FAVICON_PATH), content_type="image/png", - headers={"Cache-Control": "max-age=3600, immutable, public"}, + headers={"Cache-Control": "max-age=3600, public"}, ) @@ -348,6 +348,16 @@ def _legacy_template_csrftoken(context): return "" +def _resolve_static_asset_path(root_path, path): + root = Path(root_path).resolve() + full_path = (root / path).resolve() + try: + full_path.relative_to(root) + except ValueError: + raise ValueError("Static asset path cannot escape static root") from None + return full_path + + # Documentation for the variables Datasette.render_template() adds to the # context for every page. This is part of the documented template contract: # keys added in render_template() must be documented here - the contract @@ -361,9 +371,6 @@ TEMPLATE_BASE_CONTEXT = { "menu_links": "Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`.", "display_actor": "Function that accepts an actor dictionary and returns the display string used in the navigation menu.", "show_logout": "True if the logout link should be shown in the navigation menu", - "app_css_hash": "Hash of Datasette's app.css contents, used for cache busting", - "edit_tools_js_hash": "Hash of Datasette's edit-tools.js contents, used for cache busting", - "table_js_hash": "Hash of Datasette's table.js contents, used for cache busting", "zip": "Python's ``zip()`` builtin, made available to template logic", "body_scripts": 'List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in `` - + + diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 5e483d34..aa1f0ecd 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -7,9 +7,9 @@ {% if row_mutation_ui %} {% if table_page_data.foreignKeys %} - + {% endif %} - + {% endif %} diff --git a/datasette/templates/_permissions_debug_tabs.html b/datasette/templates/_permissions_debug_tabs.html index d7203c1e..8e0f486e 100644 --- a/datasette/templates/_permissions_debug_tabs.html +++ b/datasette/templates/_permissions_debug_tabs.html @@ -44,10 +44,10 @@ diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 1ecc92df..fda4032c 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -3,29 +3,11 @@ {% block title %}Debug allow rules{% endblock %} {% block extra_head %} +{% include "_permission_ui_styles.html" %} {% endblock %} @@ -38,24 +20,28 @@ p.message-warning {

Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

-
-
-

- -
-
-

- -
-
- -
- +
+
+
+
+ + +
+
+ + +
+
+
+ +
+ -{% if error %}

{{ error }}

{% endif %} + {% if error %}

{{ error }}

{% endif %} -{% if result == "True" %}

Result: allow

{% endif %} + {% if result == "True" %}

Result: allow

{% endif %} -{% if result == "False" %}

Result: deny

{% endif %} + {% if result == "False" %}

Result: deny

{% endif %} +
{% endblock %} diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index 83cc1ae6..80249d9c 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -49,7 +49,7 @@
- + Number of results per page (max 200)
diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index 3b229a25..b9fc636a 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}Permission Check{% endblock %} +{% block title %}Explain a permission decision{% endblock %} {% block extra_head %} @@ -13,29 +13,35 @@ border-radius: 5px; } #output.allowed { - background-color: #e8f5e9; + background-color: #f3fbf4; border: 2px solid #4caf50; } #output.denied { - background-color: #ffebee; + background-color: #fff7f7; border: 2px solid #f44336; } #output h2 { margin-top: 0; } -#output .result-badge { +#output h3 { + margin-bottom: 0.5em; +} +#output .result-badge, +.effect-badge, +.rule-status { display: inline-block; - padding: 0.3em 0.8em; + padding: 0.2em 0.5em; border-radius: 3px; font-weight: bold; - font-size: 1.1em; } -#output .allowed-badge { - background-color: #4caf50; +#output .allowed-badge, +.effect-allow { + background-color: #2e7d32; color: white; } -#output .denied-badge { - background-color: #f44336; +#output .denied-badge, +.effect-deny { + background-color: #c62828; color: white; } .details-section { @@ -48,70 +54,130 @@ .details-section dd { margin-left: 1em; } +.explanation-section { + background: rgba(255, 255, 255, 0.75); + border: 1px solid #ddd; + border-radius: 4px; + margin-top: 1em; + padding: 0 1em 1em; +} +.rules-table { + border-collapse: collapse; + width: 100%; +} +.rules-table th, +.rules-table td { + border-bottom: 1px solid #ddd; + padding: 0.5em; + text-align: left; + vertical-align: top; +} +.rule-status { + background: #e8f5e9; + color: #1b5e20; +} +.rule-ignored { + background: #eee; + color: #555; + font-weight: normal; +} +.requirement-allowed { + color: #1b5e20; +} +.requirement-denied { + color: #b71c1c; +} +@media only screen and (max-width: 576px) { + .rules-table, + .rules-table tbody, + .rules-table tr, + .rules-table td { + display: block; + } + .rules-table thead { + display: none; + } + .rules-table td::before { + content: attr(data-label) ": "; + font-weight: bold; + } +} {% endblock %} {% block content %} -

Permission check

+

Explain a permission decision

{% set current_tab = "check" %} {% include "_permissions_debug_tabs.html" %} -

Use this tool to test permission checks for the current actor. It queries the /-/check.json API endpoint.

- -{% if request.actor %} -

Current actor: {{ request.actor.get("id", "anonymous") }}

-{% else %} -

Current actor: anonymous (not logged in)

-{% endif %} +

Test an actor, action and resource. The result explains which rules matched, which specificity level won, and whether actor restrictions or required actions changed the verdict.

-
+
- + + + Use null for an anonymous actor. This actor is simulated; it does not change who you are signed in as. +
+ +
+ - The permission action to check + The operation to evaluate
-
- +
+ - For database-level permissions, specify the database name + The database or other parent resource
-
- - - For table-level permissions, specify the table name (requires parent) +
+ + + The table, query or other child resource
- +
+actionSelect.addEventListener('change', updateResourceFields); +(function initializeFromUrl() { + const params = populateFormFromURL(); + updateResourceFields(); + if (params.get('action')) { + performCheck(); + } +})(); + {% endblock %} diff --git a/datasette/templates/debug_permissions_playground.html b/datasette/templates/debug_permissions_playground.html index 4410a677..8b0cbbcf 100644 --- a/datasette/templates/debug_permissions_playground.html +++ b/datasette/templates/debug_permissions_playground.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}Debug permissions{% endblock %} +{% block title %}Permission activity{% endblock %} {% block extra_head %} {% include "_permission_ui_styles.html" %} @@ -20,60 +20,45 @@ .check-action, .check-when, .check-result { font-size: 1.3em; } -textarea { - height: 10em; - width: 95%; - box-sizing: border-box; - padding: 0.5em; - border: 2px dotted black; -} -.two-col { - display: inline-block; - width: 48%; -} -.two-col label { - width: 48%; -} -@media only screen and (max-width: 576px) { - .two-col { - width: 100%; - } -} {% endblock %} {% block content %} -

Permission playground

+

Permission activity

{% set current_tab = "permissions" %} {% include "_permissions_debug_tabs.html" %} -

This tool lets you simulate an actor and a permission check for that actor.

+

Raw simulator

+ +

This form runs a hypothetical permission check and returns its raw explanation JSON. Use the Explain tool for a visual explanation of the same decision.

-
-
- - +
+
+
+ + +
-
-
-
- - -
-
- - -
-
- - +
+
+ + +
+
+ + +
+
+ + +
@@ -125,7 +110,7 @@ debugPost.addEventListener('submit', function(ev) { }); -

Recent permissions checks

+

Recent permission checks

{% if filter != "all" %}All{% else %}All{% endif %}, diff --git a/datasette/templates/debug_rules.html b/datasette/templates/debug_rules.html index d00ba9cc..233c0e94 100644 --- a/datasette/templates/debug_rules.html +++ b/datasette/templates/debug_rules.html @@ -37,7 +37,7 @@

- + Number of results per page (max 200)
diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index c7137e6b..67d3ce73 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -673,3 +673,239 @@ async def check_permission_for_resource( child=child, ) return results[action] + + +async def explain_permission_for_resource( + *, + datasette: "Datasette", + actor: dict | None, + action: str, + parent: str | None, + child: str | None, +) -> dict: + """Explain a permission decision for one action and resource. + + This is intended for Datasette's permission debugging tools. It uses the + same ``permission_resources_sql`` hook results and the same resolution + rules as :func:`check_permissions_for_actions`, but also returns the + matching rules, actor restriction results and ``also_requires`` chain. + + The returned dictionary is part of Datasette's unstable debugging API. + """ + + action_obj = datasette.actions.get(action) + if action_obj is None: + raise ValueError(f"Unknown action: {action}") + + explanation = await _explain_single_action( + datasette=datasette, + actor=actor, + action=action, + parent=parent, + child=child, + ) + + required_actions = [] + if action_obj.also_requires: + required = await explain_permission_for_resource( + datasette=datasette, + actor=actor, + action=action_obj.also_requires, + parent=parent, + child=child, + ) + required_actions.append(required) + + explanation["required_actions"] = required_actions + explanation["allowed"] = bool( + explanation["rule_allowed"] + and explanation["restriction_allowed"] + and all(required["allowed"] for required in required_actions) + ) + explanation["summary"] = _permission_explanation_summary(explanation) + return explanation + + +async def _explain_single_action( + *, + datasette: "Datasette", + actor: dict | None, + action: str, + parent: str | None, + child: str | None, +) -> dict: + """Return matching rules and restrictions for a single action.""" + from datasette.utils.permissions import SKIP_PERMISSION_CHECKS + + permission_sqls = await gather_permission_sql_from_hooks( + datasette=datasette, + actor=actor, + action=action, + ) + + if permission_sqls is SKIP_PERMISSION_CHECKS: + return { + "action": action, + "rule_allowed": True, + "restriction_allowed": True, + "winning_scope": "global", + "matched_rules": [ + { + "scope": "global", + "effect": "allow", + "source": "skip_permission_checks", + "reason": "Permission checks were explicitly skipped", + "decisive": True, + "ignored_because": None, + } + ], + "restrictions": [], + } + + db = datasette.get_internal_database() + matched_rules = [] + restrictions = [] + + for permission_sql in permission_sqls: + params = dict(permission_sql.params or {}) + parent_param = _unused_parameter_name(params, "_explain_parent") + params[parent_param] = parent + child_param = _unused_parameter_name(params, "_explain_child") + params[child_param] = child + + if permission_sql.sql: + rows = await db.execute( + f""" + SELECT parent, child, allow, reason + FROM ({permission_sql.sql}) AS permission_rules + WHERE (parent IS NULL OR parent = :{parent_param}) + AND (child IS NULL OR child = :{child_param}) + """, + params, + ) + for row in rows: + specificity = ( + 2 + if row["child"] is not None + else 1 if row["parent"] is not None else 0 + ) + matched_rules.append( + { + "scope": ("resource", "parent", "global")[2 - specificity], + "effect": "allow" if row["allow"] else "deny", + "source": permission_sql.source, + "reason": row["reason"], + "_specificity": specificity, + } + ) + + if permission_sql.restriction_sql: + restriction_row = ( + await db.execute( + f""" + SELECT EXISTS( + SELECT 1 FROM ({permission_sql.restriction_sql}) AS restriction_rules + WHERE (parent IS NULL OR parent = :{parent_param}) + AND (child IS NULL OR child = :{child_param}) + ) AS resource_is_in_allowlist + """, + params, + ) + ).first() + restriction_allowed = bool(restriction_row[0]) + restrictions.append( + { + "source": permission_sql.source, + "allowed": restriction_allowed, + "reason": params.get("deny") + or ( + "Resource is included in this restriction allowlist" + if restriction_allowed + else "Resource is not included in this restriction allowlist" + ), + } + ) + + matched_rules.sort( + key=lambda rule: ( + -rule["_specificity"], + 0 if rule["effect"] == "deny" else 1, + rule["source"] or "", + rule["reason"] or "", + ) + ) + + if matched_rules: + winning_specificity = matched_rules[0]["_specificity"] + winning_rules = [ + rule + for rule in matched_rules + if rule["_specificity"] == winning_specificity + ] + rule_allowed = not any(rule["effect"] == "deny" for rule in winning_rules) + winning_scope = winning_rules[0]["scope"] + else: + winning_specificity = None + rule_allowed = False + winning_scope = None + + for rule in matched_rules: + specificity = rule.pop("_specificity") + if specificity != winning_specificity: + rule["decisive"] = False + rule["ignored_because"] = "A more specific rule matched" + elif not rule_allowed and rule["effect"] == "allow": + rule["decisive"] = False + rule["ignored_because"] = "A deny rule matched at the same scope" + else: + rule["decisive"] = True + rule["ignored_because"] = None + + return { + "action": action, + "rule_allowed": rule_allowed, + "restriction_allowed": all( + restriction["allowed"] for restriction in restrictions + ), + "winning_scope": winning_scope, + "matched_rules": matched_rules, + "restrictions": restrictions, + } + + +def _unused_parameter_name(params: dict, preferred: str) -> str: + """Return a SQL parameter name that is not already in ``params``.""" + candidate = preferred + suffix = 2 + while candidate in params: + candidate = f"{preferred}_{suffix}" + suffix += 1 + return candidate + + +def _permission_explanation_summary(explanation: dict) -> str: + denied_requirement = next( + ( + required + for required in explanation["required_actions"] + if not required["allowed"] + ), + None, + ) + if denied_requirement: + return ( + f"Denied because {explanation['action']} also requires " + f"{denied_requirement['action']}, which was denied." + ) + if not explanation["matched_rules"]: + return "Denied because no permission rule matched this actor and resource." + if not explanation["rule_allowed"]: + return ( + f"Denied by a {explanation['winning_scope']}-level rule. " + "Deny rules take precedence over allow rules at the same scope." + ) + if not explanation["restriction_allowed"]: + return ( + "Denied because the resource is not included in the actor's restrictions." + ) + return f"Allowed by the matching {explanation['winning_scope']}-level rule." diff --git a/datasette/views/special.py b/datasette/views/special.py index c13191a1..28d34208 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -600,7 +600,7 @@ class PermissionRulesView(BaseView): async def _check_permission_for_actor(ds, action, parent, child, actor): - """Shared logic for checking permissions. Returns a dict with check results.""" + """Shared logic for checking and explaining a permission decision.""" if action not in ds.actions: return error_body(f"Unknown action: {action}", 404), 404 @@ -629,15 +629,28 @@ async def _check_permission_for_actor(ds, action, parent, child, actor): allowed = await ds.allowed(action=action, resource=resource_obj, actor=actor) + from datasette.utils.actions_sql import explain_permission_for_resource + + explanation = await explain_permission_for_resource( + datasette=ds, + actor=actor, + action=action, + parent=parent, + child=child, + ) + response = { "ok": True, + "unstable": UNSTABLE_API_MESSAGE, "action": action, "allowed": bool(allowed), + "actor": actor, "resource": { "parent": parent, "child": child, "path": _resource_path(parent, child), }, + "explanation": explanation, } if actor and "id" in actor: @@ -655,11 +668,25 @@ class PermissionCheckView(BaseView): as_format = request.url_vars.get("format") if not as_format: + actions = [ + { + "name": action.name, + "description": action.description, + "takes_parent": action.takes_parent, + "takes_child": action.takes_child, + "also_requires": action.also_requires, + } + for action in sorted( + self.ds.actions.values(), key=lambda action: action.name + ) + ] return await self.render( ["debug_check.html"], request, { - "sorted_actions": sorted(self.ds.actions.keys()), + "actions": actions, + "actor_json": request.args.get("actor") + or json.dumps(request.actor, indent=2), "has_debug_permission": True, }, ) @@ -671,9 +698,18 @@ class PermissionCheckView(BaseView): parent = request.args.get("parent") child = request.args.get("child") + actor = request.actor + actor_json = request.args.get("actor") + if actor_json is not None: + try: + actor = json.loads(actor_json) + except json.JSONDecodeError as ex: + return Response.error(f"Invalid actor JSON: {ex}", 400) + if actor is not None and not isinstance(actor, dict): + return Response.error("actor must be a JSON object or null", 400) response, status = await _check_permission_for_actor( - self.ds, action, parent, child, request.actor + self.ds, action, parent, child, actor ) return Response.json(response, status=status) diff --git a/docs/authentication.rst b/docs/authentication.rst index 51fa07d5..d720c4db 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -45,7 +45,7 @@ Using the "root" actor Datasette currently leaves almost all forms of authentication to plugins - `datasette-auth-github `__ for example. -The one exception is the "root" account, which you can sign into while using Datasette on your local machine. The root user has **all permissions** - they can perform any action regardless of other permission rules. +The one exception is the "root" account, which you can sign into while using Datasette on your local machine. The root user starts with **all permissions**: Datasette contributes a global allow rule for every action. More specific deny rules can still override that global rule. The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including: @@ -84,12 +84,12 @@ Click on that link and then visit ``http://127.0.0.1:8001/-/actor`` to confirm t Permissions =========== -Datasette's permissions system is built around SQL queries. Datasette and its plugins construct SQL queries to resolve the list of resources that an actor cas access. - The key question the permissions system answers is this: Is this **actor** allowed to perform this **action**, optionally against this particular **resource**? +Every permission decision can be understood in terms of those three values. Datasette implements the decisions using SQL, but you do not need to understand the generated SQL to configure or debug permissions. + **Actors** are :ref:`described above `. An **action** is a string describing the action the actor would like to perform. A full list is :ref:`provided below ` - examples include ``view-table`` and ``execute-sql``. @@ -138,7 +138,51 @@ This configuration will deny access to everyone except the user with ``id`` of ` How permissions are resolved ---------------------------- -Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. +Permission rules describe an effect (``allow`` or ``deny``) at one of three levels: + +``resource`` + A specific child resource, such as the ``analytics/sales`` table. + +``parent`` + A parent resource, such as the ``analytics`` database. A parent rule also applies to its child resources. + +``global`` + Every resource for that action. + +Datasette resolves matching rules from most specific to least specific: + +#. Resource rules take precedence over parent and global rules. +#. Parent rules take precedence over global rules. +#. If both allow and deny rules match at the same level, deny takes precedence. +#. If no rule matches, access is denied. + +This means a resource-level allow can provide an exception to a parent-level deny. It also means that two plugins which disagree at the same level resolve to deny. + +.. list-table:: Permission rule examples + :header-rows: 1 + + * - Matching rules + - Result + - Explanation + * - Global allow + - Allow + - The global rule is the most specific matching rule. + * - Global allow, parent deny + - Deny + - The parent rule is more specific. + * - Parent deny, resource allow + - Allow + - The resource rule is more specific. + * - Resource allow and resource deny + - Deny + - Deny takes precedence at the same level. + * - No matching rules + - Deny + - Permissions default to deny when no rule applies. + +The built-in public defaults are global allow rules for actions such as ``view-instance``, ``view-database`` and ``view-table``. They follow the same precedence rules as configuration and plugin rules. The ``--default-deny`` option prevents Datasette from contributing those default allow rules. + +Datasette performs checks using :ref:`datasette_allowed`, which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified. @@ -149,12 +193,12 @@ resources were allowed or denied. The combined sources are: * ``allow`` blocks configured in :ref:`datasette.yaml `. * :ref:`Actor restrictions ` encoded into the actor dictionary or API token. -* The "root" user shortcut when ``--root`` (or :attr:`Datasette.root_enabled `) is active, replying ``True`` to all permission chucks unless configuration rules deny them at a more specific level. +* The "root" user rule when ``--root`` (or :attr:`Datasette.root_enabled `) is active. This is a global allow rule, so a more specific configuration deny can override it. * Any additional SQL provided by plugins implementing :ref:`plugin_hook_permission_resources_sql`. -Datasette evaluates the SQL to determine if the requested ``resource`` is -included. Explicit deny rules returned by configuration or plugins will block -access even if other rules allowed it. +Actor restrictions are applied after the allow/deny rules. They act as an additional allowlist: a restriction can remove access but cannot grant access that the actor did not already have. See :ref:`authentication_cli_create_token_restrict`. + +Some actions have dependencies on other actions. These are evaluated as an ``AND`` condition. For example, ``execute-sql`` also requires ``view-database``: both decisions must be allowed for the final result to be allowed. .. _authentication_permissions_allow: @@ -1145,11 +1189,21 @@ The debug tool at ``/-/permissions`` is available to any actor with the ``permis datasette -s permissions.permissions-debug true data.db -The page shows the permission checks that have been carried out by the Datasette instance. +The permission debug tools answer four different questions: -It also provides an interface for running hypothetical permission checks against a hypothetical actor. This is a useful way of confirming that your configured permissions work in the way you expect. +Why was this decision allowed or denied? + Use :ref:`PermissionCheckView`. It shows every matching rule, identifies the winning specificity level, applies actor restrictions and evaluates any required actions. -This is designed to help administrators and plugin authors understand exactly how permission checks are being carried out, in order to effectively configure Datasette's permission system. +Which resources can the current actor access? + Use :ref:`AllowedResourcesView` to view an access map for a selected action. + +Which raw rules did Datasette and its plugins contribute? + Use :ref:`PermissionRulesView` to inspect the rules before they are resolved into decisions. + +Which checks has this Datasette instance performed recently? + Use ``/-/permissions`` to view recent permission activity. + +These tools are designed to help administrators and plugin authors understand and confirm the effective permissions configuration. These debug endpoints are exempt from the :ref:`JSON API stability promise ` - their JSON shapes may change in future releases. @@ -1184,11 +1238,20 @@ This endpoint requires the ``permissions-debug`` permission. Permission check view --------------------- -The ``/-/check`` endpoint evaluates a single action/resource pair and returns information indicating whether the access was allowed along with diagnostic information. +The ``/-/check`` endpoint evaluates and explains a single actor, action and resource decision. The explanation includes: + +* Every matching allow and deny rule, with its source and reason. +* The winning resource, parent or global scope. +* Rules ignored because a more specific rule matched, or because a deny won at the same scope. +* Actor restriction allowlists that included or excluded the resource. +* Additional actions required by the requested action. +* An explicit default-deny explanation when no rule matched. This endpoint provides an interactive HTML form interface. Add ``.json`` to the URL path (e.g. ``/-/check.json?action=view-instance``) to get the raw JSON response instead. -Pass ``?action=`` to specify the action to check, and optional ``?parent=`` and ``?child=`` parameters to specify the resource. +Pass ``?action=`` to specify the action to check, and optional ``?parent=`` and ``?child=`` parameters to specify the resource. The interactive form also accepts actor JSON, allowing a hypothetical actor to be tested without signing in as that actor. The JSON endpoint accepts the same value using the ``actor`` query string parameter. Use ``actor=null`` to represent an anonymous actor. + +This endpoint requires the ``permissions-debug`` permission. The hypothetical actor is used only for the decision being explained; access to the debug tool is checked against the actor who is actually signed in. .. _authentication_ds_actor: diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 88fe577f..cd1050d0 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -457,6 +457,20 @@ async def test_permissions_debug(ds_client, filter_): assert checks == expected_checks +@pytest.mark.asyncio +@pytest.mark.parametrize( + "permissions_debug,expected_status", + ( + (1, 200), + (0, 403), + ), +) +async def test_permissions_debug_numeric_boolean(permissions_debug, expected_status): + ds = Datasette(config={"permissions": {"permissions-debug": permissions_debug}}) + response = await ds.client.get("/-/permissions") + assert response.status_code == expected_status + + @pytest.mark.asyncio @pytest.mark.parametrize( "actor,allow,expected_fragment", @@ -748,7 +762,12 @@ async def test_actor_restricted_permissions( } if actor.get("id"): expected["actor_id"] = actor["id"] - assert response.json() == expected + data = response.json() + for key, value in expected.items(): + assert data[key] == value + assert data["actor"] == actor + assert data["explanation"]["allowed"] is expected_result + assert data["explanation"]["summary"] PermConfigTestCase = collections.namedtuple( @@ -1734,6 +1753,8 @@ async def test_permission_check_view_requires_debug_permission(): data = response.json() assert data["action"] == "view-instance" assert data["allowed"] is True + assert data["explanation"]["allowed"] is True + assert data["explanation"]["summary"] @pytest.mark.asyncio @@ -1759,6 +1780,211 @@ async def test_permission_check_view_query_actions(action): } +@pytest.mark.asyncio +async def test_permission_check_explains_specificity_for_hypothetical_actor(): + ds = Datasette( + config={ + "permissions": {"view-table": {"id": "alice"}}, + "databases": { + "analytics": { + "permissions": {"view-table": False}, + "tables": { + "public": {"permissions": {"view-table": {"id": "alice"}}} + }, + } + }, + } + ) + ds.root_enabled = True + await ds.invoke_startup() + + def path_for(child): + return "/-/check.json?" + urllib.parse.urlencode( + { + "action": "view-table", + "parent": "analytics", + "child": child, + "actor": json.dumps({"id": "alice"}), + } + ) + + public_response = await ds.client.get(path_for("public"), actor={"id": "root"}) + assert public_response.status_code == 200 + public = public_response.json() + assert public["actor"] == {"id": "alice"} + assert public["allowed"] is True + assert public["explanation"]["allowed"] is True + assert public["explanation"]["winning_scope"] == "resource" + public_rules = public["explanation"]["matched_rules"] + assert any( + rule["scope"] == "resource" and rule["effect"] == "allow" and rule["decisive"] + for rule in public_rules + ) + assert any( + rule["scope"] == "parent" + and rule["effect"] == "deny" + and rule["ignored_because"] == "A more specific rule matched" + for rule in public_rules + ) + + private_response = await ds.client.get(path_for("private"), actor={"id": "root"}) + assert private_response.status_code == 200 + private = private_response.json() + assert private["allowed"] is False + assert private["explanation"]["allowed"] is False + assert private["explanation"]["winning_scope"] == "parent" + assert private["explanation"]["summary"].startswith("Denied by a parent-level rule") + + +@pytest.mark.asyncio +async def test_permission_check_explains_deny_wins_at_same_scope(): + ds = Datasette(config={"permissions": {"view-table": {"id": "someone-else"}}}) + ds.root_enabled = True + await ds.invoke_startup() + path = "/-/check.json?" + urllib.parse.urlencode( + { + "action": "view-table", + "parent": "analytics", + "child": "users", + "actor": json.dumps({"id": "alice"}), + } + ) + response = await ds.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["allowed"] is False + assert data["explanation"]["winning_scope"] == "global" + rules = data["explanation"]["matched_rules"] + assert any(rule["effect"] == "deny" and rule["decisive"] for rule in rules) + assert any( + rule["effect"] == "allow" + and rule["ignored_because"] == "A deny rule matched at the same scope" + for rule in rules + ) + + +@pytest.mark.asyncio +async def test_permission_check_explains_default_deny(): + ds = Datasette() + ds.root_enabled = True + await ds.invoke_startup() + path = "/-/check.json?" + urllib.parse.urlencode( + { + "action": "insert-row", + "parent": "analytics", + "child": "users", + "actor": json.dumps({"id": "alice"}), + } + ) + response = await ds.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["allowed"] is False + explanation = data["explanation"] + assert explanation["allowed"] is False + assert explanation["matched_rules"] == [] + assert explanation["winning_scope"] is None + assert explanation["summary"] == ( + "Denied because no permission rule matched this actor and resource." + ) + + +@pytest.mark.asyncio +async def test_permission_check_explains_actor_restrictions(): + ds = Datasette() + ds.root_enabled = True + await ds.invoke_startup() + restricted_actor = { + "id": "alice", + "_r": {"r": {"analytics": {"public": ["vt"]}}}, + } + path = "/-/check.json?" + urllib.parse.urlencode( + { + "action": "view-table", + "parent": "analytics", + "child": "private", + "actor": json.dumps(restricted_actor), + } + ) + response = await ds.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["allowed"] is False + explanation = data["explanation"] + assert explanation["rule_allowed"] is True + assert explanation["restriction_allowed"] is False + assert explanation["allowed"] is False + assert explanation["restrictions"] + assert any( + restriction["allowed"] is False for restriction in explanation["restrictions"] + ) + assert "actor's restrictions" in explanation["summary"] + + +@pytest.mark.asyncio +async def test_permission_check_explains_required_actions(): + from datasette import hookimpl + from datasette.permissions import PermissionSQL + + class StoreQueryPermissions: + @hookimpl + def permission_resources_sql(self, actor, action): + if not actor or actor.get("id") != "alice": + return None + if action == "store-query": + return PermissionSQL( + sql="SELECT 'analytics' AS parent, NULL AS child, 1 AS allow, 'alice can store queries' AS reason" + ) + if action == "execute-sql": + return PermissionSQL( + sql="SELECT 'analytics' AS parent, NULL AS child, 0 AS allow, 'alice cannot execute SQL' AS reason" + ) + + ds = Datasette() + ds.root_enabled = True + await ds.invoke_startup() + ds.pm.register(StoreQueryPermissions(), name="store-query-test") + path = "/-/check.json?" + urllib.parse.urlencode( + { + "action": "store-query", + "parent": "analytics", + "actor": json.dumps({"id": "alice"}), + } + ) + response = await ds.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["allowed"] is False + explanation = data["explanation"] + assert explanation["rule_allowed"] is True + assert explanation["required_actions"][0]["action"] == "execute-sql" + assert explanation["required_actions"][0]["allowed"] is False + assert explanation["summary"] == ( + "Denied because store-query also requires execute-sql, which was denied." + ) + + +@pytest.mark.asyncio +async def test_permission_check_hypothetical_actor_validation(): + ds = Datasette() + ds.root_enabled = True + await ds.invoke_startup() + + response = await ds.client.get( + "/-/check.json?action=view-instance&actor=not-json", + actor={"id": "root"}, + ) + assert response.status_code == 400 + assert response.json()["error"].startswith("Invalid actor JSON:") + + response = await ds.client.get( + "/-/check.json?action=view-instance&actor=%5B%5D", + actor={"id": "root"}, + ) + assert response.status_code == 400 + assert response.json()["error"] == "actor must be a JSON object or null" + + @pytest.mark.asyncio async def test_root_allow_block_with_table_restricted_actor(): """ From 9cfc252394eb21d05f45818dac9c374e2d141a35 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jul 2026 08:41:27 -0700 Subject: [PATCH 163/167] Make internal catalog refresh atomic Refs #2831 --- datasette/app.py | 14 +--- datasette/utils/internal_db.py | 133 ++++++++++++++++++--------------- 2 files changed, 72 insertions(+), 75 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index c44f9095..0e31273d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -753,19 +753,7 @@ class Datasette: # Compare schema versions to see if we should skip it if schema_version == current_schema_versions.get(database_name): continue - placeholders = "(?, ?, ?, ?)" - values = [database_name, str(db.path), db.is_memory, schema_version] - if db.path is None: - placeholders = "(?, null, ?, ?)" - values = [database_name, db.is_memory, schema_version] - await internal_db.execute_write( - """ - INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version) - VALUES {} - """.format(placeholders), - values, - ) - await populate_schema_tables(internal_db, db) + await populate_schema_tables(internal_db, db, schema_version) @property def urls(self): diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 10ca32a5..e061d882 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -180,29 +180,9 @@ async def init_internal_db(db): await db.execute_write_fn(apply_migrations, transaction=False) -async def populate_schema_tables(internal_db, db): +async def populate_schema_tables(internal_db, db, schema_version): database_name = db.name - def delete_everything(conn): - conn.execute( - "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_views WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_foreign_keys WHERE database_name = ?", - [database_name], - ) - conn.execute( - "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] - ) - - await internal_db.execute_write_fn(delete_everything) - tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows views = (await db.execute("select * from sqlite_master WHERE type = 'view'")).rows @@ -266,47 +246,76 @@ async def populate_schema_tables(internal_db, db): indexes_to_insert, ) = await db.execute_fn(collect_info) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) - values (?, ?, ?, ?) - """, - tables_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_views (database_name, view_name, rootpage, sql) - values (?, ?, ?, ?) - """, - views_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_columns ( - database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden - ) VALUES ( - :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden + def replace_catalog(conn): + # Delete child rows before their catalog_tables parents so this also + # works if a prepare_connection plugin enables foreign key enforcement. + for table in ( + "catalog_columns", + "catalog_foreign_keys", + "catalog_indexes", + "catalog_views", + "catalog_tables", + ): + conn.execute( + "DELETE FROM {} WHERE database_name = ?".format(table), + [database_name], + ) + conn.execute( + """ + INSERT OR REPLACE INTO catalog_databases ( + database_name, path, is_memory, schema_version + ) VALUES (?, ?, ?, ?) + """, + [ + database_name, + str(db.path) if db.path is not None else None, + db.is_memory, + schema_version, + ], ) - """, - columns_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_foreign_keys ( - database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match - ) VALUES ( - :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match + conn.executemany( + """ + INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) + values (?, ?, ?, ?) + """, + tables_to_insert, ) - """, - foreign_keys_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_indexes ( - database_name, table_name, seq, name, "unique", origin, partial - ) VALUES ( - :database_name, :table_name, :seq, :name, :unique, :origin, :partial + conn.executemany( + """ + INSERT INTO catalog_views (database_name, view_name, rootpage, sql) + values (?, ?, ?, ?) + """, + views_to_insert, ) - """, - indexes_to_insert, - ) + conn.executemany( + """ + INSERT INTO catalog_columns ( + database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden + ) VALUES ( + :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden + ) + """, + columns_to_insert, + ) + conn.executemany( + """ + INSERT INTO catalog_foreign_keys ( + database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match + ) VALUES ( + :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match + ) + """, + foreign_keys_to_insert, + ) + conn.executemany( + """ + INSERT INTO catalog_indexes ( + database_name, table_name, seq, name, "unique", origin, partial + ) VALUES ( + :database_name, :table_name, :seq, :name, :unique, :origin, :partial + ) + """, + indexes_to_insert, + ) + + await internal_db.execute_write_fn(replace_catalog) From 591b909a4d216ed76d3c775484df52b54e89dc74 Mon Sep 17 00:00:00 2001 From: TowyTowy <85077986+TowyTowy@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:53:45 +0200 Subject: [PATCH 164/167] Escape table names with [square] brackets, refs #2431 (#2846) Several internal helpers quoted table names using SQLite [bracket] identifiers built with an f-string, e.g. PRAGMA foreign_key_list([{table}]). Bracket quoting cannot escape a "]" character, so any table whose name contains "]" (for example "[foo]" or "foo]") produced "sqlite3.OperationalError: unrecognized token" - crashing schema introspection at startup and 500-ing the table page. Switch these call sites to the existing escape_sqlite() helper, which uses "double quote" quoting with correct "" escaping (the same approach already used elsewhere in the codebase and in the test suite): - utils/internal_db.py: PRAGMA foreign_key_list / index_list - utils/__init__.py: get_outbound_foreign_keys - database.py: table_counts count query - facets.py: default "select * from" SQL Added a regression test covering table names with "]" characters. Co-authored-by: Claude --- datasette/database.py | 3 ++- datasette/facets.py | 2 +- datasette/utils/__init__.py | 2 +- datasette/utils/internal_db.py | 8 +++++--- tests/test_api.py | 33 ++++++++++++++++++++++++++++++++- 5 files changed, 41 insertions(+), 7 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index eb402b0c..bab3a378 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -17,6 +17,7 @@ from .utils import ( detect_fts, detect_primary_keys, detect_spatialite, + escape_sqlite, get_all_foreign_keys, get_outbound_foreign_keys, md5_not_usedforsecurity, @@ -608,7 +609,7 @@ class Database: try: table_count = ( await self.execute( - f"select count(*) from (select * from [{table}] limit {self.count_limit + 1})", + f"select count(*) from (select * from {escape_sqlite(table)} limit {self.count_limit + 1})", custom_time_limit=limit, ) ).rows[0][0] diff --git a/datasette/facets.py b/datasette/facets.py index abe0605e..5f757df3 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -85,7 +85,7 @@ class Facet: self.database = database # For foreign key expansion. Can be None for e.g. stored SQL queries: self.table = table - self.sql = sql or f"select * from [{table}]" + self.sql = sql or f"select * from {escape_sqlite(table)}" self.params = params or [] self.table_config = table_config # row_count can be None, in which case we calculate it ourselves: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 42574d3b..18d3ba52 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -636,7 +636,7 @@ def detect_primary_keys(conn, table): def get_outbound_foreign_keys(conn, table): - infos = conn.execute(f"PRAGMA foreign_key_list([{table}])").fetchall() + infos = conn.execute(f"PRAGMA foreign_key_list({escape_sqlite(table)})").fetchall() fks = [] for info in infos: if info is not None: diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index e061d882..702b53d8 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -3,7 +3,7 @@ import textwrap from sqlite_utils import Database as SQLiteUtilsDatabase from sqlite_utils import Migrations -from datasette.utils import table_column_details +from datasette.utils import escape_sqlite, table_column_details INTERNAL_DB_SCHEMA_TABLES = { "catalog_databases", @@ -213,7 +213,7 @@ async def populate_schema_tables(internal_db, db, schema_version): for column in columns ) foreign_keys = conn.execute( - f"PRAGMA foreign_key_list([{table_name}])" + f"PRAGMA foreign_key_list({escape_sqlite(table_name)})" ).fetchall() foreign_keys_to_insert.extend( { @@ -222,7 +222,9 @@ async def populate_schema_tables(internal_db, db, schema_version): } for foreign_key in foreign_keys ) - indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall() + indexes = conn.execute( + f"PRAGMA index_list({escape_sqlite(table_name)})" + ).fetchall() indexes_to_insert.extend( { **{"database_name": database_name, "table_name": table_name}, diff --git a/tests/test_api.py b/tests/test_api.py index d5f519b9..191d064a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,6 @@ from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS -from datasette.utils import UNSTABLE_API_MESSAGE +from datasette.utils import UNSTABLE_API_MESSAGE, escape_sqlite, tilde_encode from datasette.utils.sqlite import sqlite_version from datasette.version import __version__ from .fixtures import make_app_client, EXPECTED_PLUGINS @@ -930,6 +930,37 @@ async def test_tilde_encoded_database_names(db_name): assert response2.status_code == 200 +@pytest.mark.asyncio +@pytest.mark.parametrize("table_name", ("[foo]", "foo]", "[foo]/bar")) +async def test_table_with_reserved_characters_in_name(table_name): + # Table names containing characters such as "]" that cannot be escaped + # using SQLite [bracket] quoting used to break schema introspection and + # the table page - https://github.com/simonw/datasette/issues/2431 + ds = Datasette() + db = ds.add_memory_database("test_reserved_table_names") + await db.execute_write( + "create table {} (id integer primary key, name text)".format( + escape_sqlite(table_name) + ) + ) + await db.execute_write( + "insert into {} (id, name) values (1, 'one')".format(escape_sqlite(table_name)) + ) + # Schema introspection (populate_schema_tables) must not crash: + db_response = await ds.client.get("/test_reserved_table_names.json") + assert db_response.status_code == 200 + tables = {t["name"]: t for t in db_response.json()["tables"]} + assert tables[table_name]["count"] == 1 + # And the table page itself must load and return the row: + table_response = await ds.client.get( + "/test_reserved_table_names/{}.json?_shape=array".format( + tilde_encode(table_name) + ) + ) + assert table_response.status_code == 200 + assert table_response.json() == [{"id": 1, "name": "one"}] + + @pytest.mark.asyncio @pytest.mark.parametrize( "config,expected", From 8b7c942d5e5ada887aa89c4d58567af39d5a3e07 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jul 2026 09:18:51 -0700 Subject: [PATCH 165/167] Major performance boost for SQL permissions, closes #2832 --- datasette/utils/actions_sql.py | 217 +++++++++++++++++---------------- 1 file changed, 111 insertions(+), 106 deletions(-) diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index 67d3ce73..d767e391 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -252,88 +252,62 @@ async def _build_single_action_sql( ] ) - # Continue with the cascading logic - query_parts.extend( - [ - "child_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", - " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", - " FROM base b", - " LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child = b.child", - " GROUP BY b.parent, b.child", - "),", - "parent_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", - " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", - " FROM base b", - " LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", - "global_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", - " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", - " FROM base b", - " LEFT JOIN all_rules ar ON ar.parent IS NULL AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", + # Continue with the cascading logic. + # Aggregate the RULES by cascade level (small), rather than grouping + # base x rules (which scales with the number of resources). + def _agg(select_key, where, group_by): + parts = [ + f" SELECT {select_key}", + " MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow,", + " json_group_array(CASE WHEN allow = 0 THEN source_plugin || ': ' || reason END) AS deny_reasons,", + " json_group_array(CASE WHEN allow = 1 THEN source_plugin || ': ' || reason END) AS allow_reasons", + f" FROM all_rules WHERE {where}", ] + if group_by: + parts.append(f" GROUP BY {group_by}") + return parts + + query_parts.extend( + ["child_agg AS ("] + + _agg( + "parent, child,", + "parent IS NOT NULL AND child IS NOT NULL", + "parent, child", + ) + + ["),", "parent_agg AS ("] + + _agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent") + + ["),", "global_agg AS ("] + + _agg("", "parent IS NULL AND child IS NULL", None) + + ["),"] ) # Add anonymous decision logic if needed if include_is_private: - query_parts.extend( - [ - "anon_child_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", - " FROM base b", - " LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child = b.child", - " GROUP BY b.parent, b.child", - "),", - "anon_parent_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", - " FROM base b", - " LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", - "anon_global_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", - " FROM base b", - " LEFT JOIN anon_rules ar ON ar.parent IS NULL AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", - "anon_decisions AS (", - " SELECT", - " b.parent, b.child,", - " CASE", - " WHEN acl.any_deny = 1 THEN 0", - " WHEN acl.any_allow = 1 THEN 1", - " WHEN apl.any_deny = 1 THEN 0", - " WHEN apl.any_allow = 1 THEN 1", - " WHEN agl.any_deny = 1 THEN 0", - " WHEN agl.any_allow = 1 THEN 1", - " ELSE 0", - " END AS anon_is_allowed", - " FROM base b", - " JOIN anon_child_lvl acl ON b.parent = acl.parent AND (b.child = acl.child OR (b.child IS NULL AND acl.child IS NULL))", - " JOIN anon_parent_lvl apl ON b.parent = apl.parent AND (b.child = apl.child OR (b.child IS NULL AND apl.child IS NULL))", - " JOIN anon_global_lvl agl ON b.parent = agl.parent AND (b.child = agl.child OR (b.child IS NULL AND agl.child IS NULL))", - "),", + + def _anon_agg(select_key, where, group_by): + parts = [ + f" SELECT {select_key}", + " MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow", + f" FROM anon_rules WHERE {where}", ] + if group_by: + parts.append(f" GROUP BY {group_by}") + return parts + + query_parts.extend( + ["anon_child_agg AS ("] + + _anon_agg( + "parent, child,", + "parent IS NOT NULL AND child IS NOT NULL", + "parent, child", + ) + + ["),", "anon_parent_agg AS ("] + + _anon_agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent") + + ["),", "anon_global_agg AS ("] + + _anon_agg("", "parent IS NULL AND child IS NULL", None) + + ["),"] ) # Final decisions @@ -342,31 +316,28 @@ async def _build_single_action_sql( "decisions AS (", " SELECT", " b.parent, b.child,", - " -- Cascading permission logic: child → parent → global, DENY beats ALLOW at each level", + " -- Cascading permission logic: child -> parent -> global, DENY beats ALLOW at each level", " -- Priority order:", - " -- 1. Child-level deny (most specific, blocks access)", - " -- 2. Child-level allow (most specific, grants access)", - " -- 3. Parent-level deny (intermediate, blocks access)", - " -- 4. Parent-level allow (intermediate, grants access)", - " -- 5. Global-level deny (least specific, blocks access)", - " -- 6. Global-level allow (least specific, grants access)", + " -- 1. Child-level deny 2. Child-level allow", + " -- 3. Parent-level deny 4. Parent-level allow", + " -- 5. Global-level deny 6. Global-level allow", " -- 7. Default deny (no rules match)", " CASE", - " WHEN cl.any_deny = 1 THEN 0", - " WHEN cl.any_allow = 1 THEN 1", - " WHEN pl.any_deny = 1 THEN 0", - " WHEN pl.any_allow = 1 THEN 1", - " WHEN gl.any_deny = 1 THEN 0", - " WHEN gl.any_allow = 1 THEN 1", + " WHEN ca.any_deny = 1 THEN 0", + " WHEN ca.any_allow = 1 THEN 1", + " WHEN pa.any_deny = 1 THEN 0", + " WHEN pa.any_allow = 1 THEN 1", + " WHEN ga.any_deny = 1 THEN 0", + " WHEN ga.any_allow = 1 THEN 1", " ELSE 0", " END AS is_allowed,", " CASE", - " WHEN cl.any_deny = 1 THEN cl.deny_reasons", - " WHEN cl.any_allow = 1 THEN cl.allow_reasons", - " WHEN pl.any_deny = 1 THEN pl.deny_reasons", - " WHEN pl.any_allow = 1 THEN pl.allow_reasons", - " WHEN gl.any_deny = 1 THEN gl.deny_reasons", - " WHEN gl.any_allow = 1 THEN gl.allow_reasons", + " WHEN ca.any_deny = 1 THEN ca.deny_reasons", + " WHEN ca.any_allow = 1 THEN ca.allow_reasons", + " WHEN pa.any_deny = 1 THEN pa.deny_reasons", + " WHEN pa.any_allow = 1 THEN pa.allow_reasons", + " WHEN ga.any_deny = 1 THEN ga.deny_reasons", + " WHEN ga.any_allow = 1 THEN ga.allow_reasons", " ELSE '[]'", " END AS reason", ] @@ -374,21 +345,34 @@ async def _build_single_action_sql( if include_is_private: query_parts.append( - " , CASE WHEN ad.anon_is_allowed = 0 THEN 1 ELSE 0 END AS is_private" + " , CASE WHEN (" + "CASE" + " WHEN aca.any_deny = 1 THEN 0" + " WHEN aca.any_allow = 1 THEN 1" + " WHEN apa.any_deny = 1 THEN 0" + " WHEN apa.any_allow = 1 THEN 1" + " WHEN aga.any_deny = 1 THEN 0" + " WHEN aga.any_allow = 1 THEN 1" + " ELSE 0 END" + ") = 0 THEN 1 ELSE 0 END AS is_private" ) query_parts.extend( [ " FROM base b", - " JOIN child_lvl cl ON b.parent = cl.parent AND (b.child = cl.child OR (b.child IS NULL AND cl.child IS NULL))", - " JOIN parent_lvl pl ON b.parent = pl.parent AND (b.child = pl.child OR (b.child IS NULL AND pl.child IS NULL))", - " JOIN global_lvl gl ON b.parent = gl.parent AND (b.child = gl.child OR (b.child IS NULL AND gl.child IS NULL))", + " LEFT JOIN child_agg ca ON ca.parent = b.parent AND ca.child = b.child", + " LEFT JOIN parent_agg pa ON pa.parent = b.parent", + " CROSS JOIN global_agg ga", ] ) if include_is_private: - query_parts.append( - " JOIN anon_decisions ad ON b.parent = ad.parent AND (b.child = ad.child OR (b.child IS NULL AND ad.child IS NULL))" + query_parts.extend( + [ + " LEFT JOIN anon_child_agg aca ON aca.parent = b.parent AND aca.child = b.child", + " LEFT JOIN anon_parent_agg apa ON apa.parent = b.parent", + " CROSS JOIN anon_global_agg aga", + ] ) query_parts.append(")") @@ -400,8 +384,28 @@ async def _build_single_action_sql( restriction_intersect = "\nINTERSECT\n".join( f"SELECT * FROM ({sql})" for sql in restriction_sqls ) + # Decompose by NULL-pattern so the final filter can use pure-equality + # EXISTS lookups (satisfiable via automatic indexes) instead of a + # correlated OR-scan over the whole list. query_parts.extend( - [",", "restriction_list AS (", f" {restriction_intersect}", ")"] + [ + ",", + "restriction_list AS (", + f" {restriction_intersect}", + "),", + "restriction_exact AS (", + " SELECT parent, child FROM restriction_list WHERE parent IS NOT NULL AND child IS NOT NULL", + "),", + "restriction_parent_any AS (", + " SELECT DISTINCT parent FROM restriction_list WHERE parent IS NOT NULL AND child IS NULL", + "),", + "restriction_child_any AS (", + " SELECT DISTINCT child FROM restriction_list WHERE parent IS NULL AND child IS NOT NULL", + "),", + "restriction_all AS (", + " SELECT 1 AS matched FROM restriction_list WHERE parent IS NULL AND child IS NULL LIMIT 1", + ")", + ] ) # Final SELECT @@ -416,10 +420,11 @@ async def _build_single_action_sql( # Add restriction filter if there are restrictions if restriction_sqls: query_parts.append(""" - AND EXISTS ( - SELECT 1 FROM restriction_list r - WHERE (r.parent = decisions.parent OR r.parent IS NULL) - AND (r.child = decisions.child OR r.child IS NULL) + AND ( + EXISTS (SELECT 1 FROM restriction_all) + OR EXISTS (SELECT 1 FROM restriction_parent_any r WHERE r.parent = decisions.parent) + OR EXISTS (SELECT 1 FROM restriction_child_any r WHERE r.child = decisions.child) + OR EXISTS (SELECT 1 FROM restriction_exact r WHERE r.parent = decisions.parent AND r.child = decisions.child) )""") # Add parent filter if specified From 2ffd8a860e84ff58922e633c8e85e9a8e088ca93 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jul 2026 09:28:29 -0700 Subject: [PATCH 166/167] Release 1.0a37 Refs #2831, #2832, #2841, #2842, #2843, #2846 --- datasette/version.py | 2 +- docs/changelog.rst | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 387144e9..8e238ab5 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a36" +__version__ = "1.0a37" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index e3718543..1327baa6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,20 @@ Changelog ========= +.. _v1_0_a37: + +1.0a37 (2026-07-14) +------------------- + +Performance improvement for SQL-backed permission checks, plus an improved permission debugging interface. + +- SQL used to resolve permission checks now aggregates permission rules before joining them to resources, improving performance on instances with large schemas. (:issue:`2832`) +- The :ref:`PermissionCheckView` permission debugger now explains why a decision was allowed or denied, including the matching rules. The interactive form can also test a hypothetical actor supplied as JSON, and the :ref:`permissions documentation ` now describes resolution rules in more detail. (:issue:`2841`) +- :ref:`database_execute_write` has a new ``transaction=`` parameter, which can be set to ``False`` for statements such as ``VACUUM`` that cannot run inside a transaction. Write tasks now start their transactions using ``BEGIN IMMEDIATE``, which also ensures that writes are rolled back if the task fails. (:issue:`2831`) +- Refreshing a database's schema in Datasette's internal catalog is now performed as a single atomic operation. (:issue:`2831`) +- Fixed schema introspection, table pages, facets and table counts for tables with names containing a ``]`` character. Thanks, `TowyTowy `__. (:issue:`2431`, :pr:`2846`) +- ``/-/plugins.json`` once again returns a top-level JSON array of plugin objects, reverting the object envelope introduced in 1.0a36. This should fix a large number of trivial test failures in existing plugins. (:issue:`2842`, :pr:`2843`) + .. _v1_0_a36: 1.0a36 (2026-07-07) From 481df7ff6d78a8ccf919984d27f27201b081bd53 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jul 2026 09:31:28 -0700 Subject: [PATCH 167/167] Shorten link text in changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1327baa6..670166bb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,7 +13,7 @@ Performance improvement for SQL-backed permission checks, plus an improved permi - SQL used to resolve permission checks now aggregates permission rules before joining them to resources, improving performance on instances with large schemas. (:issue:`2832`) - The :ref:`PermissionCheckView` permission debugger now explains why a decision was allowed or denied, including the matching rules. The interactive form can also test a hypothetical actor supplied as JSON, and the :ref:`permissions documentation ` now describes resolution rules in more detail. (:issue:`2841`) -- :ref:`database_execute_write` has a new ``transaction=`` parameter, which can be set to ``False`` for statements such as ``VACUUM`` that cannot run inside a transaction. Write tasks now start their transactions using ``BEGIN IMMEDIATE``, which also ensures that writes are rolled back if the task fails. (:issue:`2831`) +- :ref:`db.execute_write(sql, ..., transaction=True) ` has a new ``transaction=`` parameter, which can be set to ``False`` for statements such as ``VACUUM`` that cannot run inside a transaction. Write tasks now start their transactions using ``BEGIN IMMEDIATE``, which also ensures that writes are rolled back if the task fails. (:issue:`2831`) - Refreshing a database's schema in Datasette's internal catalog is now performed as a single atomic operation. (:issue:`2831`) - Fixed schema introspection, table pages, facets and table counts for tables with names containing a ``]`` character. Thanks, `TowyTowy `__. (:issue:`2431`, :pr:`2846`) - ``/-/plugins.json`` once again returns a top-level JSON array of plugin objects, reverting the object envelope introduced in 1.0a36. This should fix a large number of trivial test failures in existing plugins. (:issue:`2842`, :pr:`2843`)