From 435ff7fa88fce25f5f2e4fa0aae8834deff7240c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:51:09 -0700 Subject: [PATCH 001/174] Context.documented_fields() and extras doc-metadata enforcement - Context dataclasses now expose documented_fields(), returning ContextField(name, type_name, help) for each field - ExtraRegistry.internal_classes_for_scope() returns the Extra classes that are available to HTML templates but excluded from JSON - Tests enforce that every registered Extra has a description and every DatabaseContext/QueryContext field has help metadata Refs #1510, #2127 Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 10 ++++++++++ datasette/views/__init__.py | 25 +++++++++++++++++++++++++ tests/test_extras.py | 32 ++++++++++++++++++++++++++++++++ tests/test_template_context.py | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 tests/test_template_context.py diff --git a/datasette/extras.py b/datasette/extras.py index 5cab52a4..36014185 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -81,6 +81,16 @@ class ExtraRegistry: def public_classes_for_scope(self, scope): return self.classes_for_scope(scope, include_internal=False) + def internal_classes_for_scope(self, scope): + # Extras that are available to HTML templates but excluded from + # JSON responses - plain Providers are dependency plumbing and + # never surface as keys, so they are not included + return [ + cls + for cls in self.classes_for_scope(scope) + if issubclass(cls, Extra) and not cls.public + ] + def _registry_for_scope(self, scope): registry = self._scope_registries.get(scope) if registry is None: diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index 88106737..de851708 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -1,2 +1,27 @@ +from dataclasses import dataclass +import dataclasses + + +@dataclass(frozen=True) +class ContextField: + name: str + type_name: str + help: str + + class Context: "Base class for all documented contexts" + + @classmethod + def documented_fields(cls): + "List of ContextField describing the documented fields of this context" + documented = [] + for f in dataclasses.fields(cls): + documented.append( + ContextField( + name=f.name, + type_name=getattr(f.type, "__name__", str(f.type)), + help=f.metadata.get("help", ""), + ) + ) + return documented diff --git a/tests/test_extras.py b/tests/test_extras.py index ad8a9f00..73b4965e 100644 --- a/tests/test_extras.py +++ b/tests/test_extras.py @@ -23,6 +23,38 @@ class DependentExtra(Extra): return slow_value + 1 +class InternalOnlyExtra(Extra): + description = "Internal extra for HTML templates only" + scopes = {ExtraScope.TABLE} + public = False + + async def resolve(self, context): + return "internal" + + +def test_internal_classes_for_scope(): + registry = ExtraRegistry([SlowValueExtra, DependentExtra, InternalOnlyExtra]) + assert registry.internal_classes_for_scope(ExtraScope.TABLE) == [InternalOnlyExtra] + assert registry.public_classes_for_scope(ExtraScope.TABLE) == [ + SlowValueExtra, + DependentExtra, + ] + + +def _registered_extra_classes(): + # Plain Providers are internal dependency plumbing, only Extra + # subclasses surface as documented JSON/template keys + from datasette.views.table_extras import table_extra_registry + + return [cls for cls in table_extra_registry.classes if issubclass(cls, Extra)] + + +@pytest.mark.parametrize("cls", _registered_extra_classes(), ids=lambda cls: cls.key()) +def test_registered_extras_have_descriptions(cls): + # Every registered extra is part of the documented template/JSON contract + assert cls.description, "{} is missing a description".format(cls.__name__) + + def test_registry_is_built_once_per_scope(): registry = ExtraRegistry([SlowValueExtra, DependentExtra]) first = registry._registry_for_scope(ExtraScope.TABLE) diff --git a/tests/test_template_context.py b/tests/test_template_context.py new file mode 100644 index 00000000..4ce00a55 --- /dev/null +++ b/tests/test_template_context.py @@ -0,0 +1,32 @@ +""" +Tests for the documented template context - the contract that custom +template authors can rely on for Datasette 1.0. +""" + +from dataclasses import dataclass, field + +import pytest + +from datasette.views import Context +from datasette.views.database import DatabaseContext, QueryContext + + +def test_documented_fields(): + @dataclass + class DemoContext(Context): + name: str = field(metadata={"help": "The name"}) + count: int = field(metadata={"help": "How many there are"}) + + fields = DemoContext.documented_fields() + assert [(f.name, f.type_name, f.help) for f in fields] == [ + ("name", "str", "The name"), + ("count", "int", "How many there are"), + ] + + +@pytest.mark.parametrize("klass", (DatabaseContext, QueryContext)) +def test_context_dataclass_fields_all_have_help(klass): + for context_field in klass.documented_fields(): + assert context_field.help, "{}.{} is missing help metadata".format( + klass.__name__, context_field.name + ) From 6a1b237b39ca11ca8a876f912b23e27f281eb0f4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:55:25 -0700 Subject: [PATCH 002/174] Documented template context manifest with contract tests datasette/template_contexts.py is the source of truth for the template context contract: the variables custom templates can rely on for the database, query, table and row pages, plus the base context that render_template() adds to every page. Documentation for each key comes from the Context dataclass field help (database, query), the Extra class description (table and row extras) or inline docs in the manifest (keys added by view code). Contract tests render each page with template_debug ?_context=1 and assert the real context keys exactly match the documented set, in both directions - an undocumented addition or a removed documented key both fail. Refs #1510, #2127 Co-Authored-By: Claude Fable 5 --- datasette/template_contexts.py | 240 +++++++++++++++++++++++++++++++++ tests/test_template_context.py | 103 ++++++++++++++ 2 files changed, 343 insertions(+) create mode 100644 datasette/template_contexts.py diff --git a/datasette/template_contexts.py b/datasette/template_contexts.py new file mode 100644 index 00000000..1ff1e2a8 --- /dev/null +++ b/datasette/template_contexts.py @@ -0,0 +1,240 @@ +""" +The documented template context for Datasette's core HTML pages. + +This module is the source of truth for the template context contract: +the set of variables that custom templates can rely on for each page. +It is consumed by the contract tests in tests/test_template_context.py, +which assert that the real rendered context for each page exactly +matches what is documented here, and by docs/template_context_doc.py +which generates the documentation in docs/template_context.rst. + +Documentation for each key comes from one of three places: + +- Pages that render a Context dataclass (database, query) use the + ``help`` metadata on each dataclass field +- Keys provided by registered extras use the ``description`` from the + Extra class +- Keys added inline by view code are documented in this module +""" + +from dataclasses import dataclass + +from datasette.extras import ExtraScope +from datasette.views.database import DatabaseContext, QueryContext +from datasette.views.table_extras import table_extra_registry + + +@dataclass(frozen=True) +class TemplateContextKey: + name: str + doc: str + + +def _keys(**docs): + return tuple(TemplateContextKey(name, doc) for name, doc in docs.items()) + + +# Added by Datasette.render_template() to the context for every page, +# including pages rendered by plugins +BASE_CONTEXT_KEYS = _keys( + request="The current Request object, or None", + crumb_items="Async function returning breadcrumb navigation items for the current page", + urls="Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation", + actor="The currently authenticated actor dictionary, or None", + menu_links="Async function returning links for the Datasette application menu, including those added by plugins", + display_actor="Function returning a display string for an actor dictionary", + show_logout="True if the logout link should be shown in the navigation menu", + app_css_hash="Hash of Datasette's app.css contents, used for cache busting", + zip="Python's zip() builtin, made available to template logic", + body_scripts="List of script blocks for the page body contributed by plugins", + format_bytes="Function that formats a number of bytes as a human-readable size", + show_messages="Function returning any messages set for the current user, clearing them in the process", + extra_css_urls="List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration", + extra_js_urls="List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page", + base_url="The configured base_url setting", + csrftoken="Function returning the CSRF token for the current request", + datasette_version="The version of Datasette that is running", +) + + +@dataclass(frozen=True) +class PageContext: + # Identifier used in tests and documentation, e.g. "table" + name: str + title: str + description: str + # The default template used to render this page + template: str + # For pages rendered from a Context dataclass + context_class: type = None + # For pages whose context includes resolved extras + extras_scope: ExtraScope = None + extra_keys: tuple = () + # Keys added inline by the view code, documented here + keys: tuple = () + + def documented_keys(self): + "Every page-specific documented key, excluding BASE_CONTEXT_KEYS" + documented = [] + if self.context_class is not None: + documented.extend( + TemplateContextKey(f.name, f.help) + for f in self.context_class.documented_fields() + ) + for name in self.extra_keys: + cls = table_extra_registry.classes_by_name[name] + documented.append(TemplateContextKey(name, cls.description or "")) + documented.extend(self.keys) + return sorted(documented, key=lambda key: key.name) + + +_SHARED_DOCS = dict( + ok="True if the data for this page was retrieved without errors", + rows="The rows for this page, as a list of dictionaries mapping column name to value", + query_ms="Time taken by the SQL queries for this page, in milliseconds", + select_templates="List of template names that were considered for this page, the one used marked with an asterisk", + settings="Dictionary of Datasette's current settings", + alternate_url_json="URL for the JSON version of this page", + url_csv="URL for the CSV export of this page", + url_csv_path="Path portion of the CSV export URL", + url_csv_hidden_args="(name, value) pairs for hidden form fields used by the CSV export form", + renderers="Dictionary mapping output format names (e.g. json) to their URLs for this page", + display_columns="Column objects formatted for the HTML table display", + display_rows="Row data formatted for the HTML table display", + custom_table_templates="Custom template names that were considered for displaying this table", +) + + +def _shared(*names): + return tuple(TemplateContextKey(name, _SHARED_DOCS[name]) for name in names) + + +PAGES = { + page.name: page + for page in ( + PageContext( + name="database", + title="Database", + description="The page listing the tables, views and queries in a database, e.g. /fixtures", + template="database.html", + context_class=DatabaseContext, + ), + PageContext( + name="query", + title="Query", + description="The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)", + template="query.html", + context_class=QueryContext, + ), + PageContext( + name="table", + title="Table", + description="The page showing the rows in a table or SQL view, e.g. /fixtures/facetable", + template="table.html", + extras_scope=ExtraScope.TABLE, + extra_keys=( + "actions", + "all_columns", + "columns", + "count", + "count_sql", + "custom_table_templates", + "database", + "database_color", + "display_columns", + "display_rows", + "expandable_columns", + "facet_results", + "facets_timed_out", + "filters", + "form_hidden_args", + "human_description_en", + "is_view", + "metadata", + "next_url", + "primary_keys", + "private", + "query", + "renderers", + "set_column_type_ui", + "sorted_facet_results", + "suggested_facets", + "table", + "table_definition", + "view_definition", + ), + keys=_shared( + "ok", + "rows", + "query_ms", + "select_templates", + "settings", + "alternate_url_json", + "url_csv", + "url_csv_path", + "url_csv_hidden_args", + ) + + _keys( + allow_execute_sql="True if the current actor can execute custom SQL against this database", + append_querystring="Function that appends additional querystring arguments to a URL", + count_limit="The maximum number of rows Datasette will count before showing an approximation", + datasette_allow_facet='The string "true" or "false" reflecting the allow_facet setting', + extra_wheres_for_ui="Extra where clauses from ?_where=, with links to remove them", + filter_columns="List of columns offered by the filter interface", + fix_path="Function that applies the base_url prefix to a path", + is_sortable="True if any of the displayed columns can be used to sort", + next="Pagination token for the next page, or None", + path_with_replaced_args="Function for building the current path with modified querystring arguments", + sort="Column the page is sorted by, or None", + sort_desc="Column the page is sorted by in descending order, or None", + supports_search="True if this table has full-text search configured", + top_table="Async function rendering the top_table plugin slot", + ), + ), + PageContext( + name="row", + title="Row", + description="The page showing an individual row, e.g. /fixtures/facetable/1", + template="row.html", + extras_scope=ExtraScope.ROW, + extra_keys=( + "columns", + "database", + "database_color", + "foreign_key_tables", + "metadata", + "primary_keys", + "private", + "table", + ), + keys=_shared( + "ok", + "rows", + "query_ms", + "select_templates", + "settings", + "alternate_url_json", + "url_csv", + "url_csv_path", + "url_csv_hidden_args", + "renderers", + "display_columns", + "display_rows", + "custom_table_templates", + ) + + _keys( + primary_key_values="Values of the primary keys for this row, from the URL", + row_actions="Row actions made available by plugin hooks", + top_row="Async function rendering the top_row plugin slot", + ), + ), + ) +} + + +def documented_context_keys(page_name): + "Set of every documented key for the named page, including base context keys" + page = PAGES[page_name] + return {key.name for key in BASE_CONTEXT_KEYS} | { + key.name for key in page.documented_keys() + } diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 4ce00a55..04002f31 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -3,12 +3,23 @@ Tests for the documented template context - the contract that custom template authors can rely on for Datasette 1.0. """ +import html +import json from dataclasses import dataclass, field import pytest +from datasette.app import Datasette +from datasette.extras import ExtraScope +from datasette.fixtures import write_fixture_database +from datasette.template_contexts import ( + BASE_CONTEXT_KEYS, + PAGES, + documented_context_keys, +) from datasette.views import Context from datasette.views.database import DatabaseContext, QueryContext +from datasette.views.table_extras import table_extra_registry def test_documented_fields(): @@ -30,3 +41,95 @@ def test_context_dataclass_fields_all_have_help(klass): assert context_field.help, "{}.{} is missing help metadata".format( klass.__name__, context_field.name ) + + +@pytest.fixture(scope="module") +def context_ds(tmp_path_factory): + db_path = tmp_path_factory.mktemp("template-context") / "fixtures.db" + write_fixture_database(db_path) + ds = Datasette( + [str(db_path)], + settings={"num_sql_threads": 1, "template_debug": True}, + config={ + "databases": { + "fixtures": { + "queries": { + "neighborhood_search": { + "sql": ( + "select _neighborhood from facetable " + "where _neighborhood like '%' || :text || '%'" + ), + "title": "Search neighborhoods", + } + } + } + } + }, + ) + yield ds + for db in ds.databases.values(): + if not db.is_memory: + db.close() + + +async def get_template_context(ds, path): + sep = "&" if "?" in path else "?" + response = await ds.client.get(path + sep + "_context=1") + assert response.status_code == 200, path + body = html.unescape( + response.text.removeprefix("
").removesuffix("
") + ) + return json.loads(body) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "page_name,path", + ( + ("database", "/fixtures"), + ("table", "/fixtures/facetable"), + ("table", "/fixtures/facetable?_city_id__exact=1"), + ("row", "/fixtures/facetable/1"), + ("query", "/fixtures/-/query?sql=select+*+from+facetable"), + ("query", "/fixtures/neighborhood_search?text=cork"), + ), +) +async def test_template_context_matches_documented_contract( + context_ds, page_name, path +): + # The full contract: every key in the rendered template context is + # documented, and every documented key is present in the context + documented = documented_context_keys(page_name) + actual = set(await get_template_context(context_ds, path)) + undocumented = actual - documented + no_longer_present = documented - actual + assert not undocumented, ( + "Undocumented keys in {} template context: {} - document them in " + "datasette/template_contexts.py".format(page_name, sorted(undocumented)) + ) + assert not no_longer_present, ( + "Documented keys missing from {} template context: {} - this would " + "break custom templates".format(page_name, sorted(no_longer_present)) + ) + + +def test_base_context_keys_all_have_docs(): + for key in BASE_CONTEXT_KEYS: + assert key.doc, "Base context key {} is missing docs".format(key.name) + + +@pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) +def test_page_documented_keys_all_have_docs(page): + for key in page.documented_keys(): + assert key.doc, "{} page key {} is missing docs".format(page.name, key.name) + + +@pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) +def test_page_extra_keys_are_registered_extras(page): + for name in page.extra_keys: + cls = table_extra_registry.classes_by_name.get(name) + assert cls is not None, "{} is not a registered extra".format(name) + assert page.extras_scope is not None + assert cls.available_for(page.extras_scope), ( + "{} extra is not available for scope {}".format(name, page.extras_scope) + ) From a55ae2adfc9500dfbb028ac167c379d8207f4676 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 00:00:42 -0700 Subject: [PATCH 003/174] Generated template context documentation, closes #1510 docs/template_context.rst is generated by cog from the manifest in datasette/template_contexts.py, following the json_api_doc.py pattern. It documents the base context available on every page plus the database, query, table and row pages, including the stability policy for custom template authors. Refs #2127 Co-Authored-By: Claude Fable 5 --- datasette/template_contexts.py | 8 +- docs/custom_templates.rst | 5 + docs/index.rst | 1 + docs/template_context.rst | 488 +++++++++++++++++++++++++++++++++ docs/template_context_doc.py | 54 ++++ tests/test_template_context.py | 25 +- 6 files changed, 571 insertions(+), 10 deletions(-) create mode 100644 docs/template_context.rst create mode 100644 docs/template_context_doc.py diff --git a/datasette/template_contexts.py b/datasette/template_contexts.py index 1ff1e2a8..e2854467 100644 --- a/datasette/template_contexts.py +++ b/datasette/template_contexts.py @@ -115,21 +115,21 @@ PAGES = { PageContext( name="database", title="Database", - description="The page listing the tables, views and queries in a database, e.g. /fixtures", + description="The page listing the tables, views and queries in a database, e.g. /fixtures.", template="database.html", context_class=DatabaseContext, ), PageContext( name="query", title="Query", - description="The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)", + description="The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name).", template="query.html", context_class=QueryContext, ), PageContext( name="table", title="Table", - description="The page showing the rows in a table or SQL view, e.g. /fixtures/facetable", + description="The page showing the rows in a table or SQL view, e.g. /fixtures/facetable.", template="table.html", extras_scope=ExtraScope.TABLE, extra_keys=( @@ -194,7 +194,7 @@ PAGES = { PageContext( name="row", title="Row", - description="The page showing an individual row, e.g. /fixtures/facetable/1", + description="The page showing an individual row, e.g. /fixtures/facetable/1.", template="row.html", extras_scope=ExtraScope.ROW, extra_keys=( diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index c324fb79..8066d28f 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -177,6 +177,11 @@ this:: Datasette will now first look for templates in that directory, and fall back on the defaults if no matches are found. +The variables made available to each template are documented on the +:ref:`template_context` page. Variables documented there are a stable API: +custom templates that use them will keep working in future Datasette +releases, up until the next major version. + It is also possible to over-ride templates on a per-database, per-row or per- table basis. diff --git a/docs/index.rst b/docs/index.rst index c76969bc..d494fd17 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -58,6 +58,7 @@ Contents settings introspection custom_templates + template_context plugins writing_plugins javascript_plugins diff --git a/docs/template_context.rst b/docs/template_context.rst new file mode 100644 index 00000000..a487a6e7 --- /dev/null +++ b/docs/template_context.rst @@ -0,0 +1,488 @@ +.. _template_context: + +Template context +================ + +This page documents the variables that are available to custom templates +for each of Datasette's core pages. See :ref:`customization_custom_templates` +for how to provide your own templates. + +The variables documented here are a stable contract: custom templates that +use them will continue to work across Datasette releases, up until the next +major version (Datasette 2.0). Anything present in the template context but +not documented on this page is not part of that contract and may change or +be removed in any release. + +You can inspect the full context for any page by starting Datasette with +``--setting template_debug 1`` and adding ``?_context=1`` to the page URL. + +.. [[[cog + from template_context_doc import template_context + template_context(cog) +.. ]]] + +Base context +------------ + +These variables are available on every page rendered by Datasette, including pages rendered by plugins that use :ref:`datasette.render_template() `. Plugins can add additional variables using the :ref:`plugin_hook_extra_template_vars` hook. + +``request`` + The current Request object, or None + +``crumb_items`` + Async function returning breadcrumb navigation items for the current page + +``urls`` + Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation + +``actor`` + The currently authenticated actor dictionary, or None + +``menu_links`` + Async function returning links for the Datasette application menu, including those added by plugins + +``display_actor`` + Function returning a display string for an actor dictionary + +``show_logout`` + True if the logout link should be shown in the navigation menu + +``app_css_hash`` + Hash of Datasette's app.css contents, used for cache busting + +``zip`` + Python's zip() builtin, made available to template logic + +``body_scripts`` + List of script blocks for the page body contributed by plugins + +``format_bytes`` + Function that formats a number of bytes as a human-readable size + +``show_messages`` + Function returning any messages set for the current user, clearing them in the process + +``extra_css_urls`` + List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration + +``extra_js_urls`` + List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page + +``base_url`` + The configured base_url setting + +``csrftoken`` + Function returning the CSRF token for the current request + +``datasette_version`` + The version of Datasette that is running + +Database page +------------- + +The page listing the tables, views and queries in a database, e.g. /fixtures. Rendered using the ``database.html`` template. + +``allow_download`` - ``bool`` + Boolean indicating if database download is allowed + +``allow_execute_sql`` - ``bool`` + Boolean indicating if custom SQL can be executed + +``alternate_url_json`` - ``str`` + URL for the alternate JSON version of this page + +``attached_databases`` - ``list`` + List of names of attached databases + +``count_limit`` - ``int`` + The maximum number of rows to count + +``database`` - ``str`` + The name of the database + +``database_actions`` - ``callable`` + Callable returning list of action links for the database menu + +``database_color`` - ``str`` + The color assigned to the database + +``editable`` - ``bool`` + Boolean indicating if the database is editable + +``hidden_count`` - ``int`` + Count of hidden tables + +``metadata`` - ``dict`` + Metadata for the database + +``path`` - ``str`` + The URL path to this database + +``private`` - ``bool`` + Boolean indicating if this is a private database + +``queries`` - ``list`` + List of stored query objects + +``queries_count`` - ``int`` + Count of visible stored queries + +``queries_more`` - ``bool`` + Boolean indicating if more stored queries are available + +``select_templates`` - ``list`` + List of templates that were considered for rendering this page + +``show_hidden`` - ``str`` + Value of _show_hidden query parameter + +``size`` - ``int`` + The size of the database in bytes + +``table_columns`` - ``dict`` + Dictionary mapping table names to their column lists + +``tables`` - ``list`` + List of table objects in the database + +``top_database`` - ``callable`` + Callable to render the top_database slot + +``views`` - ``list`` + List of view objects in the database + +Query page +---------- + +The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name). Rendered using the ``query.html`` template. + +``allow_execute_sql`` - ``bool`` + Boolean indicating if custom SQL can be executed + +``alternate_url_json`` - ``str`` + URL for alternate JSON version of this page + +``columns`` - ``list`` + List of column names + +``database`` - ``str`` + The name of the database being queried + +``database_color`` - ``str`` + The color of the database + +``db_is_immutable`` - ``bool`` + Boolean indicating if this database is immutable + +``display_rows`` - ``list`` + List of result rows to display + +``edit_sql_url`` - ``str`` + URL to edit the SQL for a stored query + +``editable`` - ``bool`` + Boolean indicating if the SQL can be edited + +``error`` - ``str`` + Any query error message + +``hide_sql`` - ``bool`` + Boolean indicating if the SQL should be hidden + +``metadata`` - ``dict`` + Metadata about the database or the stored query + +``named_parameter_values`` - ``dict`` + Dictionary of parameter names/values + +``private`` - ``bool`` + Boolean indicating if this is a private database + +``query`` - ``dict`` + The SQL query object containing the `sql` string + +``query_actions`` - ``callable`` + Callable returning a list of links for the query action menu + +``renderers`` - ``dict`` + Dictionary of renderer name to URL + +``save_query_url`` - ``str`` + URL to save the current arbitrary SQL as a query + +``select_templates`` - ``list`` + List of templates that were considered for rendering this page + +``show_hide_hidden`` - ``str`` + Hidden input field for the _show_sql parameter + +``show_hide_link`` - ``str`` + The URL to toggle showing/hiding the SQL + +``show_hide_text`` - ``str`` + The text for the show/hide SQL link + +``stored_query`` - ``str`` + The name of the stored query if this is a stored query + +``stored_query_write`` - ``bool`` + Boolean indicating if this is a stored query that allows writes + +``table_columns`` - ``dict`` + Dictionary of table name to list of column names + +``tables`` - ``list`` + List of table objects in the database + +``top_query`` - ``callable`` + Callable to render the top_query slot + +``top_stored_query`` - ``callable`` + Callable to render the top_stored_query slot + +``url_csv`` - ``str`` + URL for CSV export + +Table page +---------- + +The page showing the rows in a table or SQL view, e.g. /fixtures/facetable. Rendered using the ``table.html`` template. + +Many of these keys are shared with the :ref:`JSON API ` for this page. + +``actions`` + Table or view actions made available by plugin hooks + +``all_columns`` + All columns in the table, regardless of _col/_nocol filtering + +``allow_execute_sql`` + True if the current actor can execute custom SQL against this database + +``alternate_url_json`` + URL for the JSON version of this page + +``append_querystring`` + Function that appends additional querystring arguments to a URL + +``columns`` + Column names returned by this query + +``count`` + Total count of rows matching these filters + +``count_limit`` + The maximum number of rows Datasette will count before showing an approximation + +``count_sql`` + SQL query used to calculate the total count + +``custom_table_templates`` + Custom template names considered for this table + +``database`` + Database name + +``database_color`` + Color assigned to the database + +``datasette_allow_facet`` + The string "true" or "false" reflecting the allow_facet setting + +``display_columns`` + Column metadata used by the HTML table display + +``display_rows`` + Row data formatted for the HTML table display + +``expandable_columns`` + Foreign key columns that can be expanded with labels + +``extra_wheres_for_ui`` + Extra where clauses from ?_where=, with links to remove them + +``facet_results`` + Results of facets calculated against this data + +``facets_timed_out`` + Facet calculations that timed out + +``filter_columns`` + List of columns offered by the filter interface + +``filters`` + Filters object used by the HTML table interface + +``fix_path`` + Function that applies the base_url prefix to a path + +``form_hidden_args`` + Hidden form arguments used by the HTML table interface + +``human_description_en`` + Human-readable description of the filters + +``is_sortable`` + True if any of the displayed columns can be used to sort + +``is_view`` + Whether this resource is a view instead of a table + +``metadata`` + Metadata about the table, database or stored query + +``next`` + Pagination token for the next page, or None + +``next_url`` + Full URL for the next page of results + +``ok`` + True if the data for this page was retrieved without errors + +``path_with_replaced_args`` + Function for building the current path with modified querystring arguments + +``primary_keys`` + Primary keys for this table + +``private`` + Whether this resource is private to the current actor + +``query`` + Details of the underlying SQL query + +``query_ms`` + Time taken by the SQL queries for this page, in milliseconds + +``renderers`` + Alternative output renderers available for this table + +``rows`` + The rows for this page, as a list of dictionaries mapping column name to value + +``select_templates`` + List of template names that were considered for this page, the one used marked with an asterisk + +``set_column_type_ui`` + Column type UI metadata for this table + +``settings`` + Dictionary of Datasette's current settings + +``sort`` + Column the page is sorted by, or None + +``sort_desc`` + Column the page is sorted by in descending order, or None + +``sorted_facet_results`` + Facet results sorted for display + +``suggested_facets`` + Suggestions for facets that might return interesting results + +``supports_search`` + True if this table has full-text search configured + +``table`` + Table name + +``table_definition`` + SQL definition for this table + +``top_table`` + Async function rendering the top_table plugin slot + +``url_csv`` + URL for the CSV export of this page + +``url_csv_hidden_args`` + (name, value) pairs for hidden form fields used by the CSV export form + +``url_csv_path`` + Path portion of the CSV export URL + +``view_definition`` + SQL definition for this view + +Row page +-------- + +The page showing an individual row, e.g. /fixtures/facetable/1. Rendered using the ``row.html`` template. + +Many of these keys are shared with the :ref:`JSON API ` for this page. + +``alternate_url_json`` + URL for the JSON version of this page + +``columns`` + Column names returned by this query + +``custom_table_templates`` + Custom template names that were considered for displaying this table + +``database`` + Database name + +``database_color`` + Color assigned to the database + +``display_columns`` + Column objects formatted for the HTML table display + +``display_rows`` + Row data formatted for the HTML table display + +``foreign_key_tables`` + Tables that link to this row using foreign keys + +``metadata`` + Metadata about the table, database or stored query + +``ok`` + True if the data for this page was retrieved without errors + +``primary_key_values`` + Values of the primary keys for this row, from the URL + +``primary_keys`` + Primary keys for this table + +``private`` + Whether this resource is private to the current actor + +``query_ms`` + Time taken by the SQL queries for this page, in milliseconds + +``renderers`` + Dictionary mapping output format names (e.g. json) to their URLs for this page + +``row_actions`` + Row actions made available by plugin hooks + +``rows`` + The rows for this page, as a list of dictionaries mapping column name to value + +``select_templates`` + List of template names that were considered for this page, the one used marked with an asterisk + +``settings`` + Dictionary of Datasette's current settings + +``table`` + Table name + +``top_row`` + Async function rendering the top_row plugin slot + +``url_csv`` + URL for the CSV export of this page + +``url_csv_hidden_args`` + (name, value) pairs for hidden form fields used by the CSV export form + +``url_csv_path`` + Path portion of the CSV export URL + +.. [[[end]]] diff --git a/docs/template_context_doc.py b/docs/template_context_doc.py new file mode 100644 index 00000000..1539dc5e --- /dev/null +++ b/docs/template_context_doc.py @@ -0,0 +1,54 @@ +""" +Cog helpers for generating docs/template_context.rst from the manifest +in datasette/template_contexts.py - same pattern as json_api_doc.py. +""" + + +def template_context(cog): + from datasette.template_contexts import BASE_CONTEXT_KEYS, PAGES + + cog.out("\n") + _section( + cog, + "Base context", + ( + "These variables are available on every page rendered by " + "Datasette, including pages rendered by plugins that use " + ":ref:`datasette.render_template() `. " + "Plugins can add additional variables using the " + ":ref:`plugin_hook_extra_template_vars` hook." + ), + ) + _untyped_keys(cog, BASE_CONTEXT_KEYS) + + for page in PAGES.values(): + _section( + cog, + "{} page".format(page.title), + "{} Rendered using the ``{}`` template.".format( + page.description, page.template + ), + ) + if page.context_class is not None: + for f in sorted( + page.context_class.documented_fields(), key=lambda f: f.name + ): + cog.out("``{}`` - ``{}``\n".format(f.name, f.type_name)) + cog.out(" {}\n\n".format(f.help)) + else: + cog.out( + "Many of these keys are shared with the :ref:`JSON API " + "` for this page.\n\n" + ) + _untyped_keys(cog, page.documented_keys()) + + +def _section(cog, title, intro): + cog.out("{}\n{}\n\n".format(title, "-" * len(title))) + cog.out("{}\n\n".format(intro)) + + +def _untyped_keys(cog, keys): + for key in keys: + cog.out("``{}``\n".format(key.name)) + cog.out(" {}\n\n".format(key.doc)) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 04002f31..73e1d6c1 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -5,6 +5,7 @@ template authors can rely on for Datasette 1.0. import html import json +import pathlib from dataclasses import dataclass, field import pytest @@ -76,9 +77,7 @@ async def get_template_context(ds, path): sep = "&" if "?" in path else "?" response = await ds.client.get(path + sep + "_context=1") assert response.status_code == 200, path - body = html.unescape( - response.text.removeprefix("
").removesuffix("
") - ) + body = html.unescape(response.text.removeprefix("
").removesuffix("
")) return json.loads(body) @@ -124,12 +123,26 @@ def test_page_documented_keys_all_have_docs(page): assert key.doc, "{} page key {} is missing docs".format(page.name, key.name) +def test_template_context_docs_cover_every_documented_key(): + docs_path = pathlib.Path(__file__).parent.parent / "docs" / "template_context.rst" + assert docs_path.exists(), "docs/template_context.rst is missing" + docs = docs_path.read_text() + for key in BASE_CONTEXT_KEYS: + assert "``{}``".format(key.name) in docs, key.name + for page in PAGES.values(): + assert page.title in docs, page.title + for key in page.documented_keys(): + assert "``{}``".format(key.name) in docs, "{} ({} page)".format( + key.name, page.name + ) + + @pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) def test_page_extra_keys_are_registered_extras(page): for name in page.extra_keys: cls = table_extra_registry.classes_by_name.get(name) assert cls is not None, "{} is not a registered extra".format(name) assert page.extras_scope is not None - assert cls.available_for(page.extras_scope), ( - "{} extra is not available for scope {}".format(name, page.extras_scope) - ) + assert cls.available_for( + page.extras_scope + ), "{} extra is not available for scope {}".format(name, page.extras_scope) From 3ea7ed86065d3071fa76c755bdfa63e6776ea7cc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 00:06:56 -0700 Subject: [PATCH 004/174] Isolate test plugins from template context contract tests Datasette instances created with plugins_dir register their plugins on the global plugin manager for the rest of the process, so the contract tests could see extra_template_vars keys leaked from earlier test modules (e.g. the session-scoped ds_client fixture). A fixture now unregisters non-default plugins implementing extra_template_vars for the duration of each contract test and restores them afterwards. Co-Authored-By: Claude Fable 5 --- tests/test_template_context.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 73e1d6c1..3c6b13de 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -44,6 +44,27 @@ def test_context_dataclass_fields_all_have_help(klass): ) +@pytest.fixture +def isolate_extra_template_vars_plugins(): + # Datasette instances created with plugins_dir (e.g. the session-scoped + # ds_client fixture) register their plugins on the global plugin manager + # for the rest of the process. The contract documents plugin-free + # Datasette core, so unregister any non-default plugin that adds + # template variables via the extra_template_vars hook + from datasette.plugins import pm, DEFAULT_PLUGINS + + hook_plugins = {impl.plugin for impl in pm.hook.extra_template_vars.get_hookimpls()} + removed = [] + for plugin in list(pm.get_plugins()): + name = pm.get_name(plugin) + if name not in DEFAULT_PLUGINS and plugin in hook_plugins: + pm.unregister(plugin) + removed.append((plugin, name)) + yield + for plugin, name in removed: + pm.register(plugin, name) + + @pytest.fixture(scope="module") def context_ds(tmp_path_factory): db_path = tmp_path_factory.mktemp("template-context") / "fixtures.db" @@ -94,7 +115,7 @@ async def get_template_context(ds, path): ), ) async def test_template_context_matches_documented_contract( - context_ds, page_name, path + context_ds, isolate_extra_template_vars_plugins, page_name, path ): # The full contract: every key in the rendered template context is # documented, and every documented key is present in the context From 63995ce823124569968e9dfdcc94221c59a83810 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 06:51:54 -0700 Subject: [PATCH 005/174] extra_field() - Context fields documented by their Extra class A Context dataclass field declared with extra_field() takes its documentation from the description on the registered Extra of the same name, validated against the class's extras_scope. This keeps doc strings next to the resolve() code instead of duplicating them on the dataclass, ahead of introducing TableContext and RowContext. Co-Authored-By: Claude Fable 5 --- datasette/views/__init__.py | 47 +++++++++++++++++++++++++++++++++- tests/test_template_context.py | 45 ++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index de851708..238b9bb3 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -7,21 +7,66 @@ class ContextField: name: str type_name: str help: str + from_extra: bool = False + + +def extra_field(): + """ + Declare a Context dataclass field whose value comes from a registered + Extra of the same name - its documentation is the Extra description, + so the doc string lives next to the resolve() code rather than being + duplicated on the dataclass. + """ + return dataclasses.field(metadata={"from_extra": True}) class Context: "Base class for all documented contexts" + # Set on subclasses whose extra_field() fields should be resolved + # against the extras registry for this scope + extras_scope = None + @classmethod def documented_fields(cls): "List of ContextField describing the documented fields of this context" documented = [] for f in dataclasses.fields(cls): + from_extra = bool(f.metadata.get("from_extra")) + if from_extra: + help_text = cls._extra_description(f.name) + else: + help_text = f.metadata.get("help", "") documented.append( ContextField( name=f.name, type_name=getattr(f.type, "__name__", str(f.type)), - help=f.metadata.get("help", ""), + help=help_text, + from_extra=from_extra, ) ) return documented + + @classmethod + def _extra_description(cls, name): + # Imported lazily - table_extras is not needed just to define + # Context subclasses + from datasette.views.table_extras import table_extra_registry + + try: + extra_class = table_extra_registry.classes_by_name[name] + except KeyError: + raise KeyError( + "{}.{} is declared with extra_field() but there is no " + "registered extra of that name".format(cls.__name__, name) + ) + if cls.extras_scope is not None and not extra_class.available_for( + cls.extras_scope + ): + raise ValueError( + "{}.{} is declared with extra_field() but the {} extra is " + "not available for scope {}".format( + cls.__name__, name, name, cls.extras_scope + ) + ) + return extra_class.description or "" diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 3c6b13de..c385097a 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -44,6 +44,51 @@ def test_context_dataclass_fields_all_have_help(klass): ) +def test_extra_field_documentation_comes_from_the_extra_class(): + from datasette.views import extra_field + from datasette.views.table_extras import CountExtra + + @dataclass + class DemoContext(Context): + extras_scope = ExtraScope.TABLE + + count: int = extra_field() + name: str = field(metadata={"help": "The name"}) + + fields = {f.name: f for f in DemoContext.documented_fields()} + assert fields["count"].help == CountExtra.description + assert fields["count"].from_extra + assert fields["name"].help == "The name" + assert not fields["name"].from_extra + + +def test_extra_field_must_match_a_registered_extra(): + from datasette.views import extra_field + + @dataclass + class BadContext(Context): + extras_scope = ExtraScope.TABLE + + not_a_real_extra: str = extra_field() + + with pytest.raises(KeyError): + BadContext.documented_fields() + + +def test_extra_field_must_be_available_for_the_scope(): + from datasette.views import extra_field + + @dataclass + class WrongScopeContext(Context): + extras_scope = ExtraScope.ROW + + # count is a TABLE-scope extra, not available for ROW + count: int = extra_field() + + with pytest.raises(ValueError): + WrongScopeContext.documented_fields() + + @pytest.fixture def isolate_extra_template_vars_plugins(): # Datasette instances created with plugins_dir (e.g. the session-scoped From 8b89a3aca85bdf8eee93b9f4677dd85b2599dcb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 06:57:51 -0700 Subject: [PATCH 006/174] TableContext - table page now renders a documented Context dataclass The table HTML view constructs a TableContext instead of an ad-hoc dict, matching how the database and query pages already work. Fields resolved by registered extras are declared with extra_field() so their documentation lives on the Extra classes in table_extras.py; fields added by the view code carry help metadata next to the view. render_template() now converts Context dataclasses shallowly instead of via dataclasses.asdict(), which deep-copied every value and would fail on values like sqlite3.Row. Keys not declared on TableContext - extras requested with ?_extra= on the HTML page, or extra filter context from filters_from_request plugins - are now dropped from the HTML template context rather than passed through undocumented. Refs #2127 Co-Authored-By: Claude Fable 5 --- datasette/app.py | 6 +- datasette/views/table.py | 136 +++++++++++++++++++++++++++++- tests/test_internals_datasette.py | 25 ++++++ tests/test_template_context.py | 8 ++ 4 files changed, 172 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 81d23acb..275baae4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2162,7 +2162,11 @@ class Datasette: templates = [templates] template = self.get_jinja_environment(request).select_template(templates) if dataclasses.is_dataclass(context): - context = dataclasses.asdict(context) + # Shallow conversion - asdict() would deep-copy values, which + # is wasteful and fails on values like sqlite3.Row + context = { + f.name: getattr(context, f.name) for f in dataclasses.fields(context) + } body_scripts = [] # pylint: disable=no-member for extra_script in pm.hook.extra_body_script( diff --git a/datasette/views/table.py b/datasette/views/table.py index 65388c9c..356247ff 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -43,6 +43,10 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response from datasette.filters import Filters import sqlite_utils +from dataclasses import dataclass, field, fields + +from datasette.extras import ExtraScope +from . import Context, extra_field from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView from .table_extras import ( @@ -52,6 +56,129 @@ from .table_extras import ( table_extra_registry, ) + +@dataclass +class TableContext(Context): + "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable" + + extras_scope = ExtraScope.TABLE + + # Fields resolved by registered extras - their documentation comes + # from the description on each Extra class in table_extras.py + actions: callable = extra_field() + all_columns: list = extra_field() + columns: list = extra_field() + count: int = extra_field() + count_sql: str = extra_field() + custom_table_templates: list = extra_field() + database: str = extra_field() + database_color: str = extra_field() + display_columns: list = extra_field() + display_rows: list = extra_field() + expandable_columns: list = extra_field() + facet_results: dict = extra_field() + facets_timed_out: list = extra_field() + filters: Filters = extra_field() + form_hidden_args: list = extra_field() + human_description_en: str = extra_field() + is_view: bool = extra_field() + metadata: dict = extra_field() + next_url: str = extra_field() + primary_keys: list = extra_field() + private: bool = extra_field() + query: dict = extra_field() + renderers: dict = extra_field() + set_column_type_ui: dict = extra_field() + sorted_facet_results: list = extra_field() + suggested_facets: list = extra_field() + table: str = extra_field() + table_definition: str = extra_field() + view_definition: str = extra_field() + + # Fields added by the view code + ok: bool = field( + metadata={"help": "True if the data for this page was retrieved without errors"} + ) + next: str = field(metadata={"help": "Pagination token for the next page, or None"}) + rows: list = field( + metadata={ + "help": "The rows for this page, as a list of dictionaries mapping column name to value" + } + ) + filter_columns: list = field( + metadata={"help": "List of columns offered by the filter interface"} + ) + supports_search: bool = field( + metadata={"help": "True if this table has full-text search configured"} + ) + extra_wheres_for_ui: list = field( + metadata={ + "help": "Extra where clauses from ?_where=, with links to remove them" + } + ) + url_csv: str = field(metadata={"help": "URL for the CSV export of this page"}) + url_csv_path: str = field(metadata={"help": "Path portion of the CSV export URL"}) + url_csv_hidden_args: list = field( + metadata={ + "help": "(name, value) pairs for hidden form fields used by the CSV export form" + } + ) + sort: str = field(metadata={"help": "Column the page is sorted by, or None"}) + sort_desc: str = field( + metadata={"help": "Column the page is sorted by in descending order, or None"} + ) + append_querystring: callable = field( + metadata={ + "help": "Function that appends additional querystring arguments to a URL" + } + ) + path_with_replaced_args: callable = field( + metadata={ + "help": "Function for building the current path with modified querystring arguments" + } + ) + fix_path: callable = field( + metadata={"help": "Function that applies the base_url prefix to a path"} + ) + settings: dict = field( + metadata={"help": "Dictionary of Datasette's current settings"} + ) + alternate_url_json: str = field( + metadata={"help": "URL for the JSON version of this page"} + ) + datasette_allow_facet: str = field( + metadata={ + "help": 'The string "true" or "false" reflecting the allow_facet setting' + } + ) + is_sortable: bool = field( + metadata={"help": "True if any of the displayed columns can be used to sort"} + ) + allow_execute_sql: bool = field( + metadata={ + "help": "True if the current actor can execute custom SQL against this database" + } + ) + query_ms: float = field( + metadata={ + "help": "Time taken by the SQL queries for this page, in milliseconds" + } + ) + select_templates: list = field( + metadata={ + "help": "List of template names that were considered for this page, the one used marked with an asterisk" + } + ) + top_table: callable = field( + metadata={"help": "Async function rendering the top_table plugin slot"} + ) + count_limit: int = field( + metadata={ + "help": "The maximum number of rows Datasette will count before showing an approximation" + } + ) + + LINK_WITH_LABEL = ( '{label} {id}' ) @@ -1084,11 +1211,16 @@ async def table_view_traced(datasette, request): ) } ) + # Only keys declared on TableContext are part of the documented + # template contract - anything else in data (e.g. extras requested + # with ?_extra= on the HTML page, or extra filter context added by + # filters_from_request plugins) is dropped here + declared_fields = {f.name for f in fields(TableContext)} r = Response.html( await datasette.render_template( template, - dict( - data, + TableContext( + **{k: v for k, v in data.items() if k in declared_fields}, append_querystring=append_querystring, path_with_replaced_args=path_with_replaced_args, fix_path=datasette.urls.path, diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 3f867eb0..e9c78ecc 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -344,3 +344,28 @@ async def test_datasette_close_continues_past_db_error(): ds.close() assert good._closed assert ds._internal_database._closed + + +@pytest.mark.asyncio +async def test_datasette_render_template_dataclass_values_not_deep_copied(): + # display_rows can contain values like sqlite3.Row that cannot be + # deep-copied, so render_template must convert Context dataclasses + # shallowly - https://github.com/simonw/datasette/issues/2127 + class RefusesDeepCopy: + def __deepcopy__(self, memo): + raise RuntimeError("deepcopy not supported") + + def __str__(self): + return "shallow-copied-value" + + @dataclasses.dataclass + class ExampleContext(Context): + title: str + status: int + error: RefusesDeepCopy + + context = ExampleContext(title="Hello", status=200, error=RefusesDeepCopy()) + ds = Datasette(memory=True) + await ds.invoke_startup() + rendered = await ds.render_template("error.html", context) + assert "shallow-copied-value" in rendered diff --git a/tests/test_template_context.py b/tests/test_template_context.py index c385097a..4f08c476 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -178,6 +178,14 @@ async def test_template_context_matches_documented_contract( ) +def test_table_context_fields_match_documented_contract(): + from datasette.views.table import TableContext + + assert {f.name for f in TableContext.documented_fields()} == { + key.name for key in PAGES["table"].documented_keys() + } + + def test_base_context_keys_all_have_docs(): for key in BASE_CONTEXT_KEYS: assert key.doc, "Base context key {} is missing docs".format(key.name) From 3cc0fc07b49da3c9757ad33c73187f4fe49af557 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:01:57 -0700 Subject: [PATCH 007/174] RowContext - row page now renders a documented Context dataclass RowView declares context_class = RowContext; BaseView.render() constructs the dataclass from the assembled context, dropping any keys not declared on the class, after select_templates and alternate_url_json have been added. Extras-named fields use extra_field() so their documentation comes from the Extra classes; view-added fields carry help metadata next to the view code. Refs #2127 Co-Authored-By: Claude Fable 5 --- datasette/views/base.py | 9 ++++ datasette/views/row.py | 81 +++++++++++++++++++++++++++++++++- tests/test_template_context.py | 8 ++++ 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 2e2a5443..a3a207bd 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -1,5 +1,6 @@ import asyncio import csv +import dataclasses import hashlib import sys import textwrap @@ -88,6 +89,9 @@ class View: class BaseView: ds = None has_json_alternate = True + # Set to a Context subclass to render a documented template context - + # keys not declared on the class are dropped before rendering + context_class = None def __init__(self, datasette): self.ds = datasette @@ -169,6 +173,11 @@ class BaseView: ) } ) + if self.context_class is not None: + declared = {f.name for f in dataclasses.fields(self.context_class)} + template_context = self.context_class( + **{k: v for k, v in template_context.items() if k in declared} + ) return Response.html( await self.ds.render_template( template, diff --git a/datasette/views/row.py b/datasette/views/row.py index c300758b..f7475117 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -11,16 +11,95 @@ from datasette.utils import ( escape_sqlite, ) from datasette.plugins import pm +from dataclasses import dataclass, field import json import markupsafe import sqlite_utils -from datasette.extras import extra_names_from_request +from datasette.extras import extra_names_from_request, ExtraScope +from . import Context, extra_field from .table import display_columns_and_rows from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry +@dataclass +class RowContext(Context): + "The page showing an individual row, e.g. /fixtures/facetable/1" + + extras_scope = ExtraScope.ROW + + # Fields resolved by registered extras - their documentation comes + # from the description on each Extra class in table_extras.py + columns: list = extra_field() + database: str = extra_field() + database_color: str = extra_field() + foreign_key_tables: list = extra_field() + metadata: dict = extra_field() + primary_keys: list = extra_field() + private: bool = extra_field() + table: str = extra_field() + + # Fields added by the view code + ok: bool = field( + metadata={"help": "True if the data for this page was retrieved without errors"} + ) + rows: list = field( + metadata={ + "help": "The rows for this page, as a list of dictionaries mapping column name to value" + } + ) + primary_key_values: list = field( + metadata={"help": "Values of the primary keys for this row, from the URL"} + ) + query_ms: float = field( + metadata={ + "help": "Time taken by the SQL queries for this page, in milliseconds" + } + ) + display_columns: list = field( + metadata={"help": "Column objects formatted for the HTML table display"} + ) + display_rows: list = field( + metadata={"help": "Row data formatted for the HTML table display"} + ) + custom_table_templates: list = field( + metadata={ + "help": "Custom template names that were considered for displaying this table" + } + ) + row_actions: list = field( + metadata={"help": "Row actions made available by plugin hooks"} + ) + top_row: callable = field( + metadata={"help": "Async function rendering the top_row plugin slot"} + ) + renderers: dict = field( + metadata={ + "help": "Dictionary mapping output format names (e.g. json) to their URLs for this page" + } + ) + url_csv: str = field(metadata={"help": "URL for the CSV export of this page"}) + url_csv_path: str = field(metadata={"help": "Path portion of the CSV export URL"}) + url_csv_hidden_args: list = field( + metadata={ + "help": "(name, value) pairs for hidden form fields used by the CSV export form" + } + ) + settings: dict = field( + metadata={"help": "Dictionary of Datasette's current settings"} + ) + select_templates: list = field( + metadata={ + "help": "List of template names that were considered for this page, the one used marked with an asterisk" + } + ) + alternate_url_json: str = field( + metadata={"help": "URL for the JSON version of this page"} + ) + + class RowView(DataView): name = "row" + context_class = RowContext async def data(self, request, default_labels=False): resolved = await self.ds.resolve_row(request) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 4f08c476..0c694b2e 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -186,6 +186,14 @@ def test_table_context_fields_match_documented_contract(): } +def test_row_context_fields_match_documented_contract(): + from datasette.views.row import RowContext + + assert {f.name for f in RowContext.documented_fields()} == { + key.name for key in PAGES["row"].documented_keys() + } + + def test_base_context_keys_all_have_docs(): for key in BASE_CONTEXT_KEYS: assert key.doc, "Base context key {} is missing docs".format(key.name) From 8e01542fe98c237eff1390353c9c1287050f2edd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:08:21 -0700 Subject: [PATCH 008/174] One consistent pattern: every page context is a Context dataclass datasette/template_contexts.py is now a thin index with no documentation strings of its own - the docs live next to the code: - Each page's Context class (DatabaseContext, QueryContext, TableContext, RowContext) carries a docstring, its template name and help metadata on view-added fields, in the view module itself - extra_field() fields document themselves from the Extra classes - The keys render_template() adds to every page are documented in TEMPLATE_BASE_CONTEXT in app.py, next to the code that adds them, with the contract tests keeping the two in sync docs/template_context.rst is regenerated from the dataclasses, so the table and row pages now include field types like the others. Refs #2127 Co-Authored-By: Claude Fable 5 --- datasette/app.py | 28 ++++ datasette/template_contexts.py | 250 ++++----------------------------- datasette/views/database.py | 8 ++ datasette/views/row.py | 3 +- datasette/views/table.py | 3 +- docs/template_context.rst | 152 ++++++++++---------- docs/template_context_doc.py | 41 +++--- tests/test_template_context.py | 78 +++------- 8 files changed, 180 insertions(+), 383 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 275baae4..b683969a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -313,6 +313,32 @@ def _to_string(value): return json.dumps(value, default=str) +# Documentation for the variables Datasette.render_template() adds to the +# context for every page. This is part of the documented template contract: +# keys added in render_template() must be documented here - the contract +# tests in tests/test_template_context.py enforce this, and the docs in +# docs/template_context.rst are generated from it. +TEMPLATE_BASE_CONTEXT = { + "request": "The current Request object, or None", + "crumb_items": "Async function returning breadcrumb navigation items for the current page", + "urls": "Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation", + "actor": "The currently authenticated actor dictionary, or None", + "menu_links": "Async function returning links for the Datasette application menu, including those added by plugins", + "display_actor": "Function returning a display string for an actor dictionary", + "show_logout": "True if the logout link should be shown in the navigation menu", + "app_css_hash": "Hash of Datasette's app.css contents, used for cache busting", + "zip": "Python's zip() builtin, made available to template logic", + "body_scripts": "List of script blocks for the page body contributed by plugins", + "format_bytes": "Function that formats a number of bytes as a human-readable size", + "show_messages": "Function returning any messages set for the current user, clearing them in the process", + "extra_css_urls": "List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration", + "extra_js_urls": "List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page", + "base_url": "The configured base_url setting", + "csrftoken": "Function returning the CSRF token for the current request", + "datasette_version": "The version of Datasette that is running", +} + + class Datasette: # Message constants: INFO = 1 @@ -2216,6 +2242,8 @@ class Datasette: links.extend(extra_links) return links + # Keys added here must be documented in TEMPLATE_BASE_CONTEXT - + # the contract tests fail otherwise template_context = { **context, **{ diff --git a/datasette/template_contexts.py b/datasette/template_contexts.py index e2854467..5fe6a80e 100644 --- a/datasette/template_contexts.py +++ b/datasette/template_contexts.py @@ -1,240 +1,40 @@ """ -The documented template context for Datasette's core HTML pages. +Index of the documented template contexts for Datasette's core HTML pages. -This module is the source of truth for the template context contract: -the set of variables that custom templates can rely on for each page. -It is consumed by the contract tests in tests/test_template_context.py, -which assert that the real rendered context for each page exactly -matches what is documented here, and by docs/template_context_doc.py -which generates the documentation in docs/template_context.rst. +This module deliberately contains no documentation strings of its own - +the documentation lives next to the code it describes: -Documentation for each key comes from one of three places: +- Every page renders a Context dataclass defined in its view module + (DatabaseContext, QueryContext in views/database.py, TableContext in + views/table.py, RowContext in views/row.py). Fields added by view code + carry ``help`` metadata; fields declared with extra_field() take their + documentation from the description on the matching Extra class in + views/table_extras.py. +- The keys render_template() adds to every page are documented in + TEMPLATE_BASE_CONTEXT in datasette/app.py, next to the code that adds + them. -- Pages that render a Context dataclass (database, query) use the - ``help`` metadata on each dataclass field -- Keys provided by registered extras use the ``description`` from the - Extra class -- Keys added inline by view code are documented in this module +The contract tests in tests/test_template_context.py assert that the real +rendered context for each page exactly matches what is documented, and +docs/template_context_doc.py generates docs/template_context.rst from the +same classes. """ -from dataclasses import dataclass - -from datasette.extras import ExtraScope +from datasette.app import TEMPLATE_BASE_CONTEXT from datasette.views.database import DatabaseContext, QueryContext -from datasette.views.table_extras import table_extra_registry - - -@dataclass(frozen=True) -class TemplateContextKey: - name: str - doc: str - - -def _keys(**docs): - return tuple(TemplateContextKey(name, doc) for name, doc in docs.items()) - - -# Added by Datasette.render_template() to the context for every page, -# including pages rendered by plugins -BASE_CONTEXT_KEYS = _keys( - request="The current Request object, or None", - crumb_items="Async function returning breadcrumb navigation items for the current page", - urls="Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation", - actor="The currently authenticated actor dictionary, or None", - menu_links="Async function returning links for the Datasette application menu, including those added by plugins", - display_actor="Function returning a display string for an actor dictionary", - show_logout="True if the logout link should be shown in the navigation menu", - app_css_hash="Hash of Datasette's app.css contents, used for cache busting", - zip="Python's zip() builtin, made available to template logic", - body_scripts="List of script blocks for the page body contributed by plugins", - format_bytes="Function that formats a number of bytes as a human-readable size", - show_messages="Function returning any messages set for the current user, clearing them in the process", - extra_css_urls="List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration", - extra_js_urls="List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page", - base_url="The configured base_url setting", - csrftoken="Function returning the CSRF token for the current request", - datasette_version="The version of Datasette that is running", -) - - -@dataclass(frozen=True) -class PageContext: - # Identifier used in tests and documentation, e.g. "table" - name: str - title: str - description: str - # The default template used to render this page - template: str - # For pages rendered from a Context dataclass - context_class: type = None - # For pages whose context includes resolved extras - extras_scope: ExtraScope = None - extra_keys: tuple = () - # Keys added inline by the view code, documented here - keys: tuple = () - - def documented_keys(self): - "Every page-specific documented key, excluding BASE_CONTEXT_KEYS" - documented = [] - if self.context_class is not None: - documented.extend( - TemplateContextKey(f.name, f.help) - for f in self.context_class.documented_fields() - ) - for name in self.extra_keys: - cls = table_extra_registry.classes_by_name[name] - documented.append(TemplateContextKey(name, cls.description or "")) - documented.extend(self.keys) - return sorted(documented, key=lambda key: key.name) - - -_SHARED_DOCS = dict( - ok="True if the data for this page was retrieved without errors", - rows="The rows for this page, as a list of dictionaries mapping column name to value", - query_ms="Time taken by the SQL queries for this page, in milliseconds", - select_templates="List of template names that were considered for this page, the one used marked with an asterisk", - settings="Dictionary of Datasette's current settings", - alternate_url_json="URL for the JSON version of this page", - url_csv="URL for the CSV export of this page", - url_csv_path="Path portion of the CSV export URL", - url_csv_hidden_args="(name, value) pairs for hidden form fields used by the CSV export form", - renderers="Dictionary mapping output format names (e.g. json) to their URLs for this page", - display_columns="Column objects formatted for the HTML table display", - display_rows="Row data formatted for the HTML table display", - custom_table_templates="Custom template names that were considered for displaying this table", -) - - -def _shared(*names): - return tuple(TemplateContextKey(name, _SHARED_DOCS[name]) for name in names) - +from datasette.views.row import RowContext +from datasette.views.table import TableContext PAGES = { - page.name: page - for page in ( - PageContext( - name="database", - title="Database", - description="The page listing the tables, views and queries in a database, e.g. /fixtures.", - template="database.html", - context_class=DatabaseContext, - ), - PageContext( - name="query", - title="Query", - description="The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name).", - template="query.html", - context_class=QueryContext, - ), - PageContext( - name="table", - title="Table", - description="The page showing the rows in a table or SQL view, e.g. /fixtures/facetable.", - template="table.html", - extras_scope=ExtraScope.TABLE, - extra_keys=( - "actions", - "all_columns", - "columns", - "count", - "count_sql", - "custom_table_templates", - "database", - "database_color", - "display_columns", - "display_rows", - "expandable_columns", - "facet_results", - "facets_timed_out", - "filters", - "form_hidden_args", - "human_description_en", - "is_view", - "metadata", - "next_url", - "primary_keys", - "private", - "query", - "renderers", - "set_column_type_ui", - "sorted_facet_results", - "suggested_facets", - "table", - "table_definition", - "view_definition", - ), - keys=_shared( - "ok", - "rows", - "query_ms", - "select_templates", - "settings", - "alternate_url_json", - "url_csv", - "url_csv_path", - "url_csv_hidden_args", - ) - + _keys( - allow_execute_sql="True if the current actor can execute custom SQL against this database", - append_querystring="Function that appends additional querystring arguments to a URL", - count_limit="The maximum number of rows Datasette will count before showing an approximation", - datasette_allow_facet='The string "true" or "false" reflecting the allow_facet setting', - extra_wheres_for_ui="Extra where clauses from ?_where=, with links to remove them", - filter_columns="List of columns offered by the filter interface", - fix_path="Function that applies the base_url prefix to a path", - is_sortable="True if any of the displayed columns can be used to sort", - next="Pagination token for the next page, or None", - path_with_replaced_args="Function for building the current path with modified querystring arguments", - sort="Column the page is sorted by, or None", - sort_desc="Column the page is sorted by in descending order, or None", - supports_search="True if this table has full-text search configured", - top_table="Async function rendering the top_table plugin slot", - ), - ), - PageContext( - name="row", - title="Row", - description="The page showing an individual row, e.g. /fixtures/facetable/1.", - template="row.html", - extras_scope=ExtraScope.ROW, - extra_keys=( - "columns", - "database", - "database_color", - "foreign_key_tables", - "metadata", - "primary_keys", - "private", - "table", - ), - keys=_shared( - "ok", - "rows", - "query_ms", - "select_templates", - "settings", - "alternate_url_json", - "url_csv", - "url_csv_path", - "url_csv_hidden_args", - "renderers", - "display_columns", - "display_rows", - "custom_table_templates", - ) - + _keys( - primary_key_values="Values of the primary keys for this row, from the URL", - row_actions="Row actions made available by plugin hooks", - top_row="Async function rendering the top_row plugin slot", - ), - ), - ) + "database": DatabaseContext, + "query": QueryContext, + "table": TableContext, + "row": RowContext, } def documented_context_keys(page_name): "Set of every documented key for the named page, including base context keys" - page = PAGES[page_name] - return {key.name for key in BASE_CONTEXT_KEYS} | { - key.name for key in page.documented_keys() + return set(TEMPLATE_BASE_CONTEXT) | { + f.name for f in PAGES[page_name].documented_fields() } diff --git a/datasette/views/database.py b/datasette/views/database.py index f1756863..4f05c804 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -228,6 +228,10 @@ class DatabaseView(View): @dataclass class DatabaseContext(Context): + "The page listing the tables, views and queries in a database, e.g. /fixtures." + + template = "database.html" + database: str = field(metadata={"help": "The name of the database"}) private: bool = field( metadata={"help": "Boolean indicating if this is a private database"} @@ -281,6 +285,10 @@ class DatabaseContext(Context): @dataclass class QueryContext(Context): + "The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)." + + template = "query.html" + database: str = field(metadata={"help": "The name of the database being queried"}) database_color: str = field(metadata={"help": "The color of the database"}) query: dict = field( diff --git a/datasette/views/row.py b/datasette/views/row.py index f7475117..3ce02e21 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -23,8 +23,9 @@ from .table_extras import RowExtraContext, resolve_row_extras, table_extra_regis @dataclass class RowContext(Context): - "The page showing an individual row, e.g. /fixtures/facetable/1" + "The page showing an individual row, e.g. /fixtures/facetable/1." + template = "row.html" extras_scope = ExtraScope.ROW # Fields resolved by registered extras - their documentation comes diff --git a/datasette/views/table.py b/datasette/views/table.py index 356247ff..8ad7fa8c 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -59,8 +59,9 @@ from .table_extras import ( @dataclass class TableContext(Context): - "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable" + "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable." + template = "table.html" extras_scope = ExtraScope.TABLE # Fields resolved by registered extras - their documentation comes diff --git a/docs/template_context.rst b/docs/template_context.rst index a487a6e7..e0a34921 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -250,160 +250,160 @@ The page showing the rows in a table or SQL view, e.g. /fixtures/facetable. Rend Many of these keys are shared with the :ref:`JSON API ` for this page. -``actions`` +``actions`` - ``callable`` Table or view actions made available by plugin hooks -``all_columns`` +``all_columns`` - ``list`` All columns in the table, regardless of _col/_nocol filtering -``allow_execute_sql`` +``allow_execute_sql`` - ``bool`` True if the current actor can execute custom SQL against this database -``alternate_url_json`` +``alternate_url_json`` - ``str`` URL for the JSON version of this page -``append_querystring`` +``append_querystring`` - ``callable`` Function that appends additional querystring arguments to a URL -``columns`` +``columns`` - ``list`` Column names returned by this query -``count`` +``count`` - ``int`` Total count of rows matching these filters -``count_limit`` +``count_limit`` - ``int`` The maximum number of rows Datasette will count before showing an approximation -``count_sql`` +``count_sql`` - ``str`` SQL query used to calculate the total count -``custom_table_templates`` +``custom_table_templates`` - ``list`` Custom template names considered for this table -``database`` +``database`` - ``str`` Database name -``database_color`` +``database_color`` - ``str`` Color assigned to the database -``datasette_allow_facet`` +``datasette_allow_facet`` - ``str`` The string "true" or "false" reflecting the allow_facet setting -``display_columns`` +``display_columns`` - ``list`` Column metadata used by the HTML table display -``display_rows`` +``display_rows`` - ``list`` Row data formatted for the HTML table display -``expandable_columns`` +``expandable_columns`` - ``list`` Foreign key columns that can be expanded with labels -``extra_wheres_for_ui`` +``extra_wheres_for_ui`` - ``list`` Extra where clauses from ?_where=, with links to remove them -``facet_results`` +``facet_results`` - ``dict`` Results of facets calculated against this data -``facets_timed_out`` +``facets_timed_out`` - ``list`` Facet calculations that timed out -``filter_columns`` +``filter_columns`` - ``list`` List of columns offered by the filter interface -``filters`` +``filters`` - ``Filters`` Filters object used by the HTML table interface -``fix_path`` +``fix_path`` - ``callable`` Function that applies the base_url prefix to a path -``form_hidden_args`` +``form_hidden_args`` - ``list`` Hidden form arguments used by the HTML table interface -``human_description_en`` +``human_description_en`` - ``str`` Human-readable description of the filters -``is_sortable`` +``is_sortable`` - ``bool`` True if any of the displayed columns can be used to sort -``is_view`` +``is_view`` - ``bool`` Whether this resource is a view instead of a table -``metadata`` +``metadata`` - ``dict`` Metadata about the table, database or stored query -``next`` +``next`` - ``str`` Pagination token for the next page, or None -``next_url`` +``next_url`` - ``str`` Full URL for the next page of results -``ok`` +``ok`` - ``bool`` True if the data for this page was retrieved without errors -``path_with_replaced_args`` +``path_with_replaced_args`` - ``callable`` Function for building the current path with modified querystring arguments -``primary_keys`` +``primary_keys`` - ``list`` Primary keys for this table -``private`` +``private`` - ``bool`` Whether this resource is private to the current actor -``query`` +``query`` - ``dict`` Details of the underlying SQL query -``query_ms`` +``query_ms`` - ``float`` Time taken by the SQL queries for this page, in milliseconds -``renderers`` +``renderers`` - ``dict`` Alternative output renderers available for this table -``rows`` +``rows`` - ``list`` The rows for this page, as a list of dictionaries mapping column name to value -``select_templates`` +``select_templates`` - ``list`` List of template names that were considered for this page, the one used marked with an asterisk -``set_column_type_ui`` +``set_column_type_ui`` - ``dict`` Column type UI metadata for this table -``settings`` +``settings`` - ``dict`` Dictionary of Datasette's current settings -``sort`` +``sort`` - ``str`` Column the page is sorted by, or None -``sort_desc`` +``sort_desc`` - ``str`` Column the page is sorted by in descending order, or None -``sorted_facet_results`` +``sorted_facet_results`` - ``list`` Facet results sorted for display -``suggested_facets`` +``suggested_facets`` - ``list`` Suggestions for facets that might return interesting results -``supports_search`` +``supports_search`` - ``bool`` True if this table has full-text search configured -``table`` +``table`` - ``str`` Table name -``table_definition`` +``table_definition`` - ``str`` SQL definition for this table -``top_table`` +``top_table`` - ``callable`` Async function rendering the top_table plugin slot -``url_csv`` +``url_csv`` - ``str`` URL for the CSV export of this page -``url_csv_hidden_args`` +``url_csv_hidden_args`` - ``list`` (name, value) pairs for hidden form fields used by the CSV export form -``url_csv_path`` +``url_csv_path`` - ``str`` Path portion of the CSV export URL -``view_definition`` +``view_definition`` - ``str`` SQL definition for this view Row page @@ -413,76 +413,76 @@ The page showing an individual row, e.g. /fixtures/facetable/1. Rendered using t Many of these keys are shared with the :ref:`JSON API ` for this page. -``alternate_url_json`` +``alternate_url_json`` - ``str`` URL for the JSON version of this page -``columns`` +``columns`` - ``list`` Column names returned by this query -``custom_table_templates`` +``custom_table_templates`` - ``list`` Custom template names that were considered for displaying this table -``database`` +``database`` - ``str`` Database name -``database_color`` +``database_color`` - ``str`` Color assigned to the database -``display_columns`` +``display_columns`` - ``list`` Column objects formatted for the HTML table display -``display_rows`` +``display_rows`` - ``list`` Row data formatted for the HTML table display -``foreign_key_tables`` +``foreign_key_tables`` - ``list`` Tables that link to this row using foreign keys -``metadata`` +``metadata`` - ``dict`` Metadata about the table, database or stored query -``ok`` +``ok`` - ``bool`` True if the data for this page was retrieved without errors -``primary_key_values`` +``primary_key_values`` - ``list`` Values of the primary keys for this row, from the URL -``primary_keys`` +``primary_keys`` - ``list`` Primary keys for this table -``private`` +``private`` - ``bool`` Whether this resource is private to the current actor -``query_ms`` +``query_ms`` - ``float`` Time taken by the SQL queries for this page, in milliseconds -``renderers`` +``renderers`` - ``dict`` Dictionary mapping output format names (e.g. json) to their URLs for this page -``row_actions`` +``row_actions`` - ``list`` Row actions made available by plugin hooks -``rows`` +``rows`` - ``list`` The rows for this page, as a list of dictionaries mapping column name to value -``select_templates`` +``select_templates`` - ``list`` List of template names that were considered for this page, the one used marked with an asterisk -``settings`` +``settings`` - ``dict`` Dictionary of Datasette's current settings -``table`` +``table`` - ``str`` Table name -``top_row`` +``top_row`` - ``callable`` Async function rendering the top_row plugin slot -``url_csv`` +``url_csv`` - ``str`` URL for the CSV export of this page -``url_csv_hidden_args`` +``url_csv_hidden_args`` - ``list`` (name, value) pairs for hidden form fields used by the CSV export form -``url_csv_path`` +``url_csv_path`` - ``str`` Path portion of the CSV export URL .. [[[end]]] diff --git a/docs/template_context_doc.py b/docs/template_context_doc.py index 1539dc5e..c3ec77e2 100644 --- a/docs/template_context_doc.py +++ b/docs/template_context_doc.py @@ -1,11 +1,12 @@ """ -Cog helpers for generating docs/template_context.rst from the manifest -in datasette/template_contexts.py - same pattern as json_api_doc.py. +Cog helpers for generating docs/template_context.rst from the Context +dataclasses and TEMPLATE_BASE_CONTEXT - same pattern as json_api_doc.py. """ def template_context(cog): - from datasette.template_contexts import BASE_CONTEXT_KEYS, PAGES + from datasette.app import TEMPLATE_BASE_CONTEXT + from datasette.template_contexts import PAGES cog.out("\n") _section( @@ -19,36 +20,26 @@ def template_context(cog): ":ref:`plugin_hook_extra_template_vars` hook." ), ) - _untyped_keys(cog, BASE_CONTEXT_KEYS) + for name, doc in TEMPLATE_BASE_CONTEXT.items(): + cog.out("``{}``\n".format(name)) + cog.out(" {}\n\n".format(doc)) - for page in PAGES.values(): - _section( - cog, - "{} page".format(page.title), - "{} Rendered using the ``{}`` template.".format( - page.description, page.template - ), + for klass in PAGES.values(): + title = "{} page".format(klass.__name__.removesuffix("Context")) + intro = "{} Rendered using the ``{}`` template.".format( + klass.__doc__, klass.template ) - if page.context_class is not None: - for f in sorted( - page.context_class.documented_fields(), key=lambda f: f.name - ): - cog.out("``{}`` - ``{}``\n".format(f.name, f.type_name)) - cog.out(" {}\n\n".format(f.help)) - else: + _section(cog, title, intro) + if klass.extras_scope is not None: cog.out( "Many of these keys are shared with the :ref:`JSON API " "` for this page.\n\n" ) - _untyped_keys(cog, page.documented_keys()) + for f in sorted(klass.documented_fields(), key=lambda f: f.name): + cog.out("``{}`` - ``{}``\n".format(f.name, f.type_name)) + cog.out(" {}\n\n".format(f.help)) def _section(cog, title, intro): cog.out("{}\n{}\n\n".format(title, "-" * len(title))) cog.out("{}\n\n".format(intro)) - - -def _untyped_keys(cog, keys): - for key in keys: - cog.out("``{}``\n".format(key.name)) - cog.out(" {}\n\n".format(key.doc)) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 0c694b2e..b970cb8a 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -10,17 +10,11 @@ from dataclasses import dataclass, field import pytest -from datasette.app import Datasette +from datasette.app import Datasette, TEMPLATE_BASE_CONTEXT from datasette.extras import ExtraScope from datasette.fixtures import write_fixture_database -from datasette.template_contexts import ( - BASE_CONTEXT_KEYS, - PAGES, - documented_context_keys, -) +from datasette.template_contexts import PAGES, documented_context_keys from datasette.views import Context -from datasette.views.database import DatabaseContext, QueryContext -from datasette.views.table_extras import table_extra_registry def test_documented_fields(): @@ -36,14 +30,20 @@ def test_documented_fields(): ] -@pytest.mark.parametrize("klass", (DatabaseContext, QueryContext)) -def test_context_dataclass_fields_all_have_help(klass): +@pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) +def test_context_class_fields_all_have_help(klass): for context_field in klass.documented_fields(): - assert context_field.help, "{}.{} is missing help metadata".format( + assert context_field.help, "{}.{} is missing documentation".format( klass.__name__, context_field.name ) +@pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) +def test_context_class_has_docstring_and_template(klass): + assert klass.__doc__, "{} is missing a docstring".format(klass.__name__) + assert klass.template, "{} is missing a template".format(klass.__name__) + + def test_extra_field_documentation_comes_from_the_extra_class(): from datasette.views import extra_field from datasette.views.table_extras import CountExtra @@ -169,8 +169,8 @@ async def test_template_context_matches_documented_contract( undocumented = actual - documented no_longer_present = documented - actual assert not undocumented, ( - "Undocumented keys in {} template context: {} - document them in " - "datasette/template_contexts.py".format(page_name, sorted(undocumented)) + "Undocumented keys in {} template context: {} - add them to the " + "page's Context class".format(page_name, sorted(undocumented)) ) assert not no_longer_present, ( "Documented keys missing from {} template context: {} - this would " @@ -178,53 +178,21 @@ async def test_template_context_matches_documented_contract( ) -def test_table_context_fields_match_documented_contract(): - from datasette.views.table import TableContext - - assert {f.name for f in TableContext.documented_fields()} == { - key.name for key in PAGES["table"].documented_keys() - } - - -def test_row_context_fields_match_documented_contract(): - from datasette.views.row import RowContext - - assert {f.name for f in RowContext.documented_fields()} == { - key.name for key in PAGES["row"].documented_keys() - } - - def test_base_context_keys_all_have_docs(): - for key in BASE_CONTEXT_KEYS: - assert key.doc, "Base context key {} is missing docs".format(key.name) - - -@pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) -def test_page_documented_keys_all_have_docs(page): - for key in page.documented_keys(): - assert key.doc, "{} page key {} is missing docs".format(page.name, key.name) + for name, doc in TEMPLATE_BASE_CONTEXT.items(): + assert doc, "Base context key {} is missing docs".format(name) def test_template_context_docs_cover_every_documented_key(): docs_path = pathlib.Path(__file__).parent.parent / "docs" / "template_context.rst" assert docs_path.exists(), "docs/template_context.rst is missing" docs = docs_path.read_text() - for key in BASE_CONTEXT_KEYS: - assert "``{}``".format(key.name) in docs, key.name - for page in PAGES.values(): - assert page.title in docs, page.title - for key in page.documented_keys(): - assert "``{}``".format(key.name) in docs, "{} ({} page)".format( - key.name, page.name + for name in TEMPLATE_BASE_CONTEXT: + assert "``{}``".format(name) in docs, name + for page_name, klass in PAGES.items(): + title = "{} page".format(klass.__name__.removesuffix("Context")) + assert title in docs, title + for context_field in klass.documented_fields(): + assert "``{}``".format(context_field.name) in docs, "{} ({} page)".format( + context_field.name, page_name ) - - -@pytest.mark.parametrize("page", PAGES.values(), ids=lambda page: page.name) -def test_page_extra_keys_are_registered_extras(page): - for name in page.extra_keys: - cls = table_extra_registry.classes_by_name.get(name) - assert cls is not None, "{} is not a registered extra".format(name) - assert page.extras_scope is not None - assert cls.available_for( - page.extras_scope - ), "{} extra is not available for scope {}".format(name, page.extras_scope) From dfd5b95ec8adc425b683df22148cb1c14bb01128 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 09:52:02 -0700 Subject: [PATCH 009/174] Document --headed option, closes #2791 --- docs/contributing.rst | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 11bb7aa8..3b45834c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -113,12 +113,23 @@ You can pass extra ``pytest`` options after the browser name: just playwright chromium -k permissions just playwright-all -x -If you are not using ``just``, the equivalent Chromium commands are: +You can add the ``--headed`` option to have Playwright open a browser window that you can see while it runs the tests. This only works if you specify a browser, for example: + +.. code-block:: bash + + just playwright firefox --headed + +Combine this with ``-k`` to watch a specific test: + +.. code-block:: bash + + just playwright chromium --headed -k test_insert_row + +If you are not using ``just``, the equivalent ``uv run`` commands are: .. code-block:: bash uv run --group playwright playwright install chromium - uv run --group playwright pytest tests/test_playwright.py --playwright --browser chromium .. _contributing_using_fixtures: From f673e7416f3a18a1ad9471d696782206114532f7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 19 Jun 2026 17:14:33 -0700 Subject: [PATCH 010/174] Add create table execute-write template (#2794) Add a Create table starter template to the execute-write page, alongside the existing table-specific insert/update/delete templates. Keep template button clicks in-page by updating CodeMirror and the URL without collapsing the template controls. Refresh schema metadata after successful schema-changing execute-write statements so newly created tables appear immediately. --- datasette/app.py | 7 +- datasette/templates/execute_write.html | 71 ++++++++++---- datasette/views/execute_write.py | 33 +++++++ tests/test_queries.py | 127 +++++++++++++++++++++++++ 4 files changed, 220 insertions(+), 18 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 545a65c8..139e4c34 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -625,9 +625,12 @@ class Datasette: return action return None - async def refresh_schemas(self): + async def refresh_schemas(self, *, force=False): # Throttle schema refreshes to at most once per second - if time.monotonic() - getattr(self, "_last_schema_refresh", 0) < 1.0: + if ( + not force + and time.monotonic() - getattr(self, "_last_schema_refresh", 0) < 1.0 + ): return self._last_schema_refresh = time.monotonic() if self._refresh_schemas_lock.locked(): diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 949850ed..0d0a9931 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -93,20 +93,25 @@ form.sql.core input[data-execute-write-submit]:disabled { {% endif %}
- {% if write_template_tables %} + {% if write_create_table_template_sql or write_template_tables %}
Start with a template

- - + {% for table_name, table in write_template_tables|dictsort %} + + {% endfor %} + + {% for operation in write_template_operations %} + {% endfor %} - - {% for operation in write_template_operations %} - - {% endfor %} + {% endif %}

@@ -252,11 +257,12 @@ window.addEventListener("DOMContentLoaded", () => { }); -{% if write_template_tables %} +{% if write_create_table_template_sql or write_template_tables %} - {% include "_codemirror_foot.html" %} {% include "_sql_parameter_scripts.html" %} {% include "_execute_write_analysis_scripts.html" %} + +{% endif %} {% endblock %} {% block body_class %}db db-{{ database|to_css_class }}{% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index cd9565c6..db70b135 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -13,7 +13,8 @@ import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted -from datasette.resources import DatabaseResource, QueryResource +from datasette.column_types import SQLiteType +from datasette.resources import DatabaseResource, QueryResource, TableResource from datasette.stored_queries import stored_query_to_dict from datasette.write_sql import QueryWriteRejected from datasette.utils import ( @@ -46,6 +47,18 @@ from .table_extras import ( ) from . import Context +CREATE_TABLE_COLUMN_TYPES = ["text", "integer", "float", "blob"] +CREATE_TABLE_SQLITE_TYPES = { + "text": SQLiteType.TEXT, + "integer": SQLiteType.INTEGER, + "float": SQLiteType.REAL, + "blob": SQLiteType.BLOB, +} +CREATE_TABLE_TYPE_FOR_SQLITE_TYPE = { + sqlite_type: column_type + for column_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() +} + class DatabaseView(View): async def get(self, request, datasette): @@ -117,21 +130,36 @@ class DatabaseView(View): else len(stored_queries) ) + # Resolve the registered database-level actions for this database in + # one batched query, seeding the request permission cache so allowed() + # calls made inside plugin hooks below are served from the cache. + database_action_permissions = await datasette.allowed_many( + actions=[ + name + for name, action in datasette.actions.items() + if action.resource_class is DatabaseResource + ], + resource=DatabaseResource(database), + actor=request.actor, + ) + create_table_ui = await _database_create_table_ui( + datasette, request, db, database, database_action_permissions + ) + async def database_actions(): - # Resolve the registered database-level actions for this - # database in one batched query, seeding the request permission - # cache so that allowed() calls made inside the plugin hooks - # below are served from the cache - await datasette.allowed_many( - actions=[ - name - for name, action in datasette.actions.items() - if action.resource_class is DatabaseResource - ], - resource=DatabaseResource(database), - actor=request.actor, - ) links = [] + if create_table_ui: + links.append( + { + "type": "button", + "label": "Create table", + "description": "Create a new table in this database.", + "attrs": { + "aria-label": "Create table in {}".format(database), + "data-database-action": "create-table", + }, + } + ) for hook in pm.hook.database_actions( datasette=datasette, database=database, @@ -211,6 +239,9 @@ class DatabaseView(View): ), metadata=metadata, database_color=db.color, + database_page_data=( + {"createTable": create_table_ui} if create_table_ui else {} + ), database_actions=database_actions, show_hidden=request.args.get("_show_hidden"), editable=True, @@ -263,6 +294,9 @@ class DatabaseContext(Context): ) metadata: dict = field(metadata={"help": "Metadata for the database"}) database_color: str = field(metadata={"help": "The color assigned to the database"}) + database_page_data: dict = field( + metadata={"help": "JSON data used by JavaScript on the database page"} + ) database_actions: callable = field( metadata={ "help": "Callable returning list of action links for the database menu" @@ -292,6 +326,57 @@ class DatabaseContext(Context): ) +async def _database_create_table_ui( + datasette, request, db, database_name, database_action_permissions +): + if not db.is_mutable: + return None + if not database_action_permissions.get("create-table"): + return None + data = { + "path": "{}/-/create".format(datasette.urls.database(database_name)), + "databaseName": database_name, + "columnTypes": CREATE_TABLE_COLUMN_TYPES, + } + can_set_column_type = await datasette.allowed( + action="set-column-type", + resource=TableResource(database=database_name, table="__new_table__"), + actor=request.actor, + ) + if can_set_column_type: + data["customColumnTypes"] = _custom_column_type_options_for_create_table( + datasette + ) + return data + + +def _custom_column_type_options_for_create_table(datasette): + options = [] + for name, ct_cls in sorted(datasette._column_types.items()): + sqlite_types = getattr(ct_cls, "sqlite_types", None) + if sqlite_types is None: + option_sqlite_types = CREATE_TABLE_COLUMN_TYPES[:] + else: + option_sqlite_types = [ + create_table_type + for create_table_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() + if sqlite_type in sqlite_types + ] + if not option_sqlite_types: + continue + option = { + "name": name, + "description": ct_cls.description, + "sqliteTypes": option_sqlite_types, + } + if sqlite_types is not None and len(sqlite_types) == 1: + fixed_sqlite_type = CREATE_TABLE_TYPE_FOR_SQLITE_TYPE.get(sqlite_types[0]) + if fixed_sqlite_type is not None: + option["fixedSqliteType"] = fixed_sqlite_type + options.append(option) + return options + + @dataclass class QueryContext(Context): database: str = field(metadata={"help": "The name of the database being queried"}) @@ -1069,12 +1154,7 @@ class TableCreateView(BaseView): "replace", "alter", } - _supported_column_types = { - "text", - "integer", - "float", - "blob", - } + _supported_column_types = set(CREATE_TABLE_COLUMN_TYPES) # Any string that does not contain a newline or start with sqlite_ _table_name_re = re.compile(r"^(?!sqlite_)[^\n]+$") diff --git a/tests/test_playwright.py b/tests/test_playwright.py index a8c5aa4b..0c7042f0 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -117,6 +117,10 @@ def write_playwright_config(config_path): { "databases": { "data": { + "permissions": { + "create-table": True, + "set-column-type": True, + }, "tables": { "projects": { "label_column": "title", @@ -275,6 +279,55 @@ def test_datasette_homepage_contains_datasette(page, datasette_server): assert "Datasette" in page.locator("body").inner_text() +@pytest.mark.playwright +def test_create_table_flow(page, datasette_server): + page.goto(f"{datasette_server}data") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-database-action="create-table"]').click() + + dialog = page.locator("#table-create-dialog") + dialog.wait_for() + assert dialog.locator(".modal-title").inner_text() == "Create a table in data" + placeholder_select = dialog.locator(".table-create-custom-column-type").nth(0) + assert placeholder_select.input_value() == "" + assert ( + placeholder_select.locator("option:checked").inner_text() == "- custom type -" + ) + assert "table-create-input-placeholder" in placeholder_select.get_attribute("class") + dialog.locator('input[name="table"]').fill("playwright_created") + dialog.locator(".table-create-column-name").nth(1).fill("title") + dialog.locator(".table-create-add-column").click() + dialog.locator(".table-create-column-name").nth(2).fill("score") + dialog.locator(".table-create-column-type").nth(2).select_option("integer") + dialog.locator(".table-create-add-column").click() + dialog.locator(".table-create-column-name").nth(3).fill("metadata") + dialog.locator(".table-create-column-type").nth(3).select_option("integer") + dialog.locator(".table-create-custom-column-type").nth(3).select_option("json") + assert dialog.locator(".table-create-column-type").nth(3).input_value() == "text" + assert "table-create-input-placeholder" not in dialog.locator( + ".table-create-custom-column-type" + ).nth(3).get_attribute("class") + + dialog.locator(".table-create-save").click() + page.wait_for_url("**/data/playwright_created") + assert "playwright_created" in page.locator("h1").inner_text() + + response = httpx.get( + f"{datasette_server}data/playwright_created.json?_extra=columns,column_types" + ) + response.raise_for_status() + data = response.json() + assert data["columns"] == [ + "id", + "title", + "score", + "metadata", + ] + assert data["column_types"] == { + "metadata": {"type": "json", "config": None}, + } + + @pytest.mark.playwright def test_navigation_search_tracks_and_renders_recent_items(page, datasette_server): page.goto(datasette_server) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index aa67bb3f..887fdf50 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -23,6 +23,23 @@ def table_data_from_soup(soup): return json.loads(match.group(1)) +def database_data_from_soup(soup): + import json + import re + + database_script = [ + s + for s in soup.find_all("script") + if "_datasetteDatabaseData" in (s.string or "") + ][0] + match = re.search( + r"window\._datasetteDatabaseData\s*=\s*({.*?});", + database_script.string, + re.DOTALL, + ) + return json.loads(match.group(1)) + + @pytest.mark.asyncio @pytest.mark.parametrize( "path,expected_definition_sql", @@ -934,6 +951,133 @@ async def test_row_delete_action_data_attributes(): ds.close() +@pytest.mark.asyncio +async def test_database_create_table_action_button_and_data(): + ds = Datasette( + [], + config={ + "databases": { + "data": { + "permissions": { + "create-table": {"id": "root"}, + }, + }, + }, + }, + ) + try: + db = ds.add_database( + Database(ds, memory_name="test_database_create_table_action"), name="data" + ) + await db.execute_write_script(""" + create table items (id integer primary key, name text); + """) + + response = await ds.client.get("/data", actor={"id": "root"}) + assert response.status_code == 200 + soup = Soup(response.text, "html.parser") + + button = soup.select_one( + 'button.action-menu-button[data-database-action="create-table"]' + ) + assert button is not None + assert button["aria-label"] == "Create table in data" + assert button["role"] == "menuitem" + description = button.find("span", class_="dropdown-description") + assert description.text.strip() == "Create a new table in this database." + description.extract() + assert button.text.strip() == "Create table" + assert any( + "edit-tools.js" in script.get("src", "") + for script in soup.find_all("script") + ) + assert database_data_from_soup(soup) == { + "createTable": { + "path": "/data/-/create", + "databaseName": "data", + "columnTypes": ["text", "integer", "float", "blob"], + }, + } + assert "customColumnTypes" not in database_data_from_soup(soup)["createTable"] + + response_without_permission = await ds.client.get( + "/data", actor={"id": "someone-else"} + ) + assert response_without_permission.status_code == 200 + soup_without_permission = Soup(response_without_permission.text, "html.parser") + assert ( + soup_without_permission.select_one( + 'button[data-database-action="create-table"]' + ) + is None + ) + assert not any( + "_datasetteDatabaseData" in (script.string or "") + for script in soup_without_permission.find_all("script") + ) + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_database_create_table_data_includes_custom_column_types(): + ds = Datasette( + [], + config={ + "databases": { + "data": { + "permissions": { + "create-table": {"id": "root"}, + "set-column-type": {"id": "root"}, + }, + }, + }, + }, + ) + try: + db = ds.add_database( + Database(ds, memory_name="test_database_create_table_custom_types"), + name="data", + ) + await db.execute_write_script(""" + create table items (id integer primary key, name text); + """) + + response = await ds.client.get("/data", actor={"id": "root"}) + assert response.status_code == 200 + create_table_data = database_data_from_soup(Soup(response.text, "html.parser"))[ + "createTable" + ] + assert create_table_data["customColumnTypes"] == [ + { + "name": "email", + "description": "Email address", + "sqliteTypes": ["text"], + "fixedSqliteType": "text", + }, + { + "name": "json", + "description": "JSON data", + "sqliteTypes": ["text"], + "fixedSqliteType": "text", + }, + { + "name": "textarea", + "description": "Multiline text", + "sqliteTypes": ["text"], + "fixedSqliteType": "text", + }, + { + "name": "url", + "description": "URL", + "sqliteTypes": ["text"], + "fixedSqliteType": "text", + }, + ] + finally: + ds.close() + + @pytest.mark.asyncio async def test_table_insert_action_button_and_data(): ds = Datasette( From b40665dd143a81b5cea99ff039b3bc8a41c41d1f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 09:14:19 -0700 Subject: [PATCH 016/174] Add alter table JSON API - Add POST ///-/alter with Pydantic validation and dry-run support. - Support add, rename, alter, drop, primary-key and reorder operations, including allow-listed default expressions. - Document the endpoint and cover schema changes, validation, permissions, events and dry runs. Refs #2788 --- datasette/app.py | 5 + datasette/views/table.py | 353 +++++++++++++++++++++++++++++++++++++++ docs/json_api.rst | 103 ++++++++++++ pyproject.toml | 1 + tests/test_api_write.py | 205 +++++++++++++++++++++++ 5 files changed, 667 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 139e4c34..6ea3d5a4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -84,6 +84,7 @@ from .views.special import ( ) from .views.table import ( TableAutocompleteView, + TableAlterView, TableInsertView, TableUpsertView, TableSetColumnTypeView, @@ -2626,6 +2627,10 @@ class Datasette: TableUpsertView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/upsert$", ) + add_route( + TableAlterView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/alter$", + ) add_route( TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/set-column-type$", diff --git a/datasette/views/table.py b/datasette/views/table.py index c5448c85..11a28323 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,10 +1,12 @@ import asyncio import itertools import json +from typing import Annotated, Any, Literal, Union import urllib import urllib.parse import markupsafe +from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator from datasette.column_types import SQLiteType from datasette.extras import extra_names_from_request @@ -46,6 +48,7 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response from datasette.filters import Filters import sqlite_utils +from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView from .table_extras import ( @@ -649,6 +652,154 @@ async def display_columns_and_rows( return columns, cell_rows +SqliteApiType = Literal["text", "integer", "float", "blob"] +DefaultExpr = Literal["current_timestamp", "current_date", "current_time"] +DEFAULT_EXPR_SQL = { + "current_timestamp": "CURRENT_TIMESTAMP", + "current_date": "CURRENT_DATE", + "current_time": "CURRENT_TIME", +} + + +class _StrictPydanticModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class _DefaultArgsMixin(_StrictPydanticModel): + default: Any | None = None + default_expr: DefaultExpr | None = None + + @model_validator(mode="after") + def validate_default_fields(self): + has_default = "default" in self.model_fields_set + has_default_expr = "default_expr" in self.model_fields_set + if has_default and has_default_expr: + raise ValueError("default and default_expr cannot both be provided") + if has_default_expr and self.default_expr is None: + raise ValueError("default_expr cannot be null") + return self + + +class AddColumnArgs(_DefaultArgsMixin): + name: str + type: SqliteApiType = "text" + not_null: bool = False + + +class RenameColumnArgs(_StrictPydanticModel): + name: str + to: str + + +class AlterColumnArgs(_DefaultArgsMixin): + name: str + type: SqliteApiType | None = None + not_null: bool | None = None + + @model_validator(mode="after") + def require_change(self): + if not ( + {"type", "not_null", "default", "default_expr"} & self.model_fields_set + ): + raise ValueError( + "At least one of type, not_null, default or default_expr must be provided" + ) + return self + + +class DropColumnArgs(_StrictPydanticModel): + name: str + + +class SetPrimaryKeyArgs(_StrictPydanticModel): + columns: list[str] = Field(min_length=1) + + +class ReorderColumnsArgs(_StrictPydanticModel): + columns: list[str] = Field(min_length=1) + + +class AddColumnOperation(_StrictPydanticModel): + op: Literal["add_column"] + args: AddColumnArgs + + +class RenameColumnOperation(_StrictPydanticModel): + op: Literal["rename_column"] + args: RenameColumnArgs + + +class AlterColumnOperation(_StrictPydanticModel): + op: Literal["alter_column"] + args: AlterColumnArgs + + +class DropColumnOperation(_StrictPydanticModel): + op: Literal["drop_column"] + args: DropColumnArgs + + +class SetPrimaryKeyOperation(_StrictPydanticModel): + op: Literal["set_primary_key"] + args: SetPrimaryKeyArgs + + +class ReorderColumnsOperation(_StrictPydanticModel): + op: Literal["reorder_columns"] + args: ReorderColumnsArgs + + +AlterTableOperation = Annotated[ + Union[ + AddColumnOperation, + RenameColumnOperation, + AlterColumnOperation, + DropColumnOperation, + SetPrimaryKeyOperation, + ReorderColumnsOperation, + ], + Field(discriminator="op"), +] + + +class AlterTableRequest(_StrictPydanticModel): + operations: list[AlterTableOperation] = Field(min_length=1) + dry_run: bool = False + + +def _pydantic_errors(validation_error): + errors = [] + for error in validation_error.errors(): + location = ".".join(str(item) for item in error["loc"]) + message = error["msg"] + errors.append("{}: {}".format(location, message) if location else message) + return errors + + +def _table_schema_from_conn(conn, table_name): + row = conn.execute( + "select sql from sqlite_master where type = 'table' and name = ?", + [table_name], + ).fetchone() + return row[0] if row else None + + +def _primary_key_value(columns): + if len(columns) == 1: + return columns[0] + return tuple(columns) + + +def _default_expression_sql(default_expr): + return DEFAULT_EXPR_SQL[default_expr] + + +def _literal_default(db, value): + if isinstance(value, str): + return db.quote(value) + return value + + class TableInsertView(BaseView): name = "table-insert" @@ -946,6 +1097,208 @@ class TableUpsertView(TableInsertView): return await super().post(request, upsert=True) +class TableAlterView(BaseView): + name = "table-alter" + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + try: + resolved = await self.ds.resolve_table(request) + except NotFound as e: + return _error([e.args[0]], 404) + + db = resolved.db + database_name = db.name + table_name = resolved.table + + if not await self.ds.allowed( + action="alter-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return _error(["Permission denied: need alter-table"], 403) + + if not db.is_mutable: + return _error(["Database is immutable"], 403) + + content_type = request.headers.get("content-type") or "" + if not content_type.startswith("application/json"): + return _error(["Invalid content-type, must be application/json"], 400) + + try: + data = await request.json() + except json.JSONDecodeError as e: + return _error(["Invalid JSON: {}".format(e)], 400) + + if not isinstance(data, dict): + return _error(["JSON must be a dictionary"], 400) + + try: + alter_request = AlterTableRequest.model_validate(data) + except ValidationError as e: + return _error(_pydantic_errors(e), 400) + + def alter_table(conn): + before_schema = _table_schema_from_conn(conn, table_name) + + def apply_operations(operation_conn): + db_for_write = sqlite_utils.Database(operation_conn) + table = db_for_write[table_name] + + add_columns = [] + types = {} + rename = {} + drop = set() + not_null = {} + defaults = {} + column_order = None + pk = SQLITE_UTILS_DEFAULT + + for operation in alter_request.operations: + args = operation.args + if operation.op == "add_column": + if args.not_null and not ( + ( + "default" in args.model_fields_set + and args.default is not None + ) + or "default_expr" in args.model_fields_set + ): + raise ValueError( + "add_column args.default or args.default_expr is required when not_null is true" + ) + add_columns.append(args) + if "default" in args.model_fields_set and not args.not_null: + defaults[args.name] = _literal_default( + db_for_write, args.default + ) + if "default_expr" in args.model_fields_set and not args.not_null: + defaults[args.name] = _default_expression_sql( + args.default_expr + ) + elif operation.op == "rename_column": + rename[args.name] = args.to + elif operation.op == "alter_column": + if args.type is not None: + types[args.name] = args.type + if args.not_null is not None: + not_null[args.name] = args.not_null + if "default" in args.model_fields_set: + defaults[args.name] = ( + None + if args.default is None + else _literal_default(db_for_write, args.default) + ) + if "default_expr" in args.model_fields_set: + defaults[args.name] = _default_expression_sql( + args.default_expr + ) + elif operation.op == "drop_column": + drop.add(args.name) + elif operation.op == "set_primary_key": + pk = _primary_key_value(args.columns) + elif operation.op == "reorder_columns": + column_order = args.columns + + with operation_conn: + for column in add_columns: + not_null_default = None + if column.not_null: + if "default_expr" in column.model_fields_set: + not_null_default = _default_expression_sql( + column.default_expr + ) + else: + not_null_default = _literal_default( + db_for_write, column.default + ) + table.add_column( + column.name, + column.type, + not_null_default=not_null_default, + ) + + should_transform = any( + ( + types, + rename, + drop, + not_null, + defaults, + column_order is not None, + pk is not SQLITE_UTILS_DEFAULT, + ) + ) + if should_transform: + table.transform( + types=types or None, + rename=rename or None, + drop=drop or None, + pk=pk, + not_null=not_null or None, + defaults=defaults or None, + column_order=column_order, + ) + + return _table_schema_from_conn(operation_conn, table_name) + + if alter_request.dry_run: + memory_conn = sqlite3.connect(":memory:") + try: + conn.backup(memory_conn) + return before_schema, apply_operations(memory_conn) + finally: + memory_conn.close() + + after_schema = apply_operations(conn) + return before_schema, after_schema + + try: + before_schema, after_schema = await db.execute_write_fn( + alter_table, request=request + ) + except Exception as e: + return _error([str(e)], 400) + + altered = before_schema != after_schema + if altered and not alter_request.dry_run: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=before_schema, + after_schema=after_schema, + ) + ) + + table_url = self.ds.absolute_url( + request, self.ds.urls.table(database_name, table_name) + ) + table_api_url = self.ds.absolute_url( + request, self.ds.urls.table(database_name, table_name, format="json") + ) + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "table_url": table_url, + "table_api_url": table_api_url, + "altered": altered, + "schema": after_schema, + "before_schema": before_schema, + "operations_applied": 0 + if alter_request.dry_run + else len(alter_request.operations), + "dry_run": alter_request.dry_run, + }, + status=200, + ) + + class TableSetColumnTypeView(BaseView): name = "table-set-column-type" diff --git a/docs/json_api.rst b/docs/json_api.rst index f7a0caae..4074b479 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2072,6 +2072,109 @@ To use the ``"replace": true`` option you will also need the :ref:`actions_updat Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`actions_alter_table` permission. +.. _TableAlterView: + +Altering tables +~~~~~~~~~~~~~~~ + +To alter an existing table, make a ``POST`` to ``//
/-/alter``. This requires the :ref:`actions_alter_table` permission. + +:: + + POST //
/-/alter + Content-Type: application/json + Authorization: Bearer dstok_ + +The request body should include an ``operations`` array. Each operation has the same top-level shape: an ``op`` string and an ``args`` object. + +.. code-block:: json + + { + "operations": [ + { + "op": "add_column", + "args": { + "name": "slug", + "type": "text", + "not_null": true, + "default": "" + } + }, + { + "op": "add_column", + "args": { + "name": "created", + "type": "text", + "default_expr": "current_timestamp" + } + }, + { + "op": "rename_column", + "args": { + "name": "title", + "to": "headline" + } + }, + { + "op": "alter_column", + "args": { + "name": "score", + "type": "float" + } + }, + { + "op": "drop_column", + "args": { + "name": "draft_notes" + } + }, + { + "op": "set_primary_key", + "args": { + "columns": ["id"] + } + }, + { + "op": "reorder_columns", + "args": { + "columns": ["id", "headline", "slug", "created", "score"] + } + } + ] + } + +Set ``"dry_run": true`` to validate the operations and return the schema that would be created without modifying the table. + +Supported operations: + +* ``add_column`` adds a new column. ``args`` accepts ``name``, optional ``type`` of ``text``, ``integer``, ``float`` or ``blob``, optional ``not_null``, optional literal ``default`` and optional ``default_expr``. If ``not_null`` is ``true`` either a non-null ``default`` or ``default_expr`` is required. +* ``rename_column`` renames a column. ``args`` accepts ``name`` and ``to``. +* ``alter_column`` changes column properties. ``args`` accepts ``name`` and at least one of ``type``, ``not_null``, literal ``default`` or ``default_expr``. Passing ``"default": null`` removes an existing default. +* ``drop_column`` drops a column. ``args`` accepts ``name``. +* ``set_primary_key`` changes the table primary key. ``args`` accepts ``columns``, a list of one or more column names. +* ``reorder_columns`` reorders columns. ``args`` accepts ``columns``, a list of one or more column names. Columns omitted from this list will appear afterwards in their existing order. + +``default`` is always treated as a literal value. ``default_expr`` accepts one of ``current_timestamp``, ``current_date`` or ``current_time`` and is rendered as the corresponding SQLite default expression. + +A successful response returns the new schema and the previous schema: + +.. code-block:: json + + { + "ok": true, + "database": "data", + "table": "posts", + "table_url": "http://127.0.0.1:8001/data/posts", + "table_api_url": "http://127.0.0.1:8001/data/posts.json", + "altered": true, + "schema": "CREATE TABLE ...", + "before_schema": "CREATE TABLE ...", + "operations_applied": 7, + "dry_run": false + } + +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. + .. _TableSetColumnTypeView: Setting a column type diff --git a/pyproject.toml b/pyproject.toml index a19dc957..38776b2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "asyncinject>=0.7", "setuptools", "pip", + "pydantic>=2", ] [project.urls] diff --git a/tests/test_api_write.py b/tests/test_api_write.py index b7ceb6b2..f117c06e 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -794,6 +794,211 @@ async def test_update_row_alter(ds_write): assert response.json() == {"ok": True} +@pytest.mark.asyncio +async def test_alter_table_operations(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + before_schema = await db.execute_fn( + lambda conn: conn.execute( + "select sql from sqlite_master where type = 'table' and name = 'docs'" + ).fetchone()[0] + ) + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + { + "op": "add_column", + "args": { + "name": "slug", + "type": "text", + "not_null": True, + "default": "", + }, + }, + { + "op": "add_column", + "args": { + "name": "created", + "type": "text", + "default_expr": "current_timestamp", + }, + }, + { + "op": "add_column", + "args": { + "name": "literal_default", + "type": "text", + "default": "hello)", + }, + }, + {"op": "rename_column", "args": {"name": "title", "to": "headline"}}, + { + "op": "alter_column", + "args": {"name": "age", "type": "text", "default": "0"}, + }, + {"op": "drop_column", "args": {"name": "score"}}, + { + "op": "reorder_columns", + "args": { + "columns": [ + "id", + "headline", + "slug", + "created", + "literal_default", + "age", + ] + }, + }, + {"op": "set_primary_key", "args": {"columns": ["id"]}}, + ] + }, + headers=_headers(token), + ) + + assert response.status_code == 200, response.text + data = response.json() + assert data["ok"] is True + assert data["database"] == "data" + assert data["table"] == "docs" + assert data["altered"] is True + assert data["operations_applied"] == 8 + assert data["before_schema"] == before_schema + assert "headline" in data["schema"] + assert "score" not in data["schema"] + assert "DEFAULT CURRENT_TIMESTAMP" in data["schema"] + assert "DEFAULT 'hello)'" in data["schema"] + + columns = ( + await db.execute("select * from pragma_table_info('docs') order by cid") + ).dicts() + assert [column["name"] for column in columns] == [ + "id", + "headline", + "slug", + "created", + "literal_default", + "age", + ] + assert columns[0]["pk"] == 1 + assert columns[2]["notnull"] == 1 + assert columns[2]["dflt_value"] == "''" + assert columns[3]["dflt_value"] == "CURRENT_TIMESTAMP" + assert columns[4]["dflt_value"] == "'hello)'" + assert columns[5]["type"] == "TEXT" + assert columns[5]["dflt_value"] == "'0'" + + event = last_event(ds_write) + assert event.name == "alter-table" + assert event.database == "data" + assert event.table == "docs" + assert event.before_schema == before_schema + assert event.after_schema == data["schema"] + + +@pytest.mark.asyncio +async def test_alter_table_dry_run(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "dry_run": True, + "operations": [ + {"op": "add_column", "args": {"name": "slug", "type": "text"}} + ], + }, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["ok"] is True + assert data["dry_run"] is True + assert data["altered"] is True + assert data["operations_applied"] == 0 + assert "slug" in data["schema"] + columns = ( + await db.execute("select name from pragma_table_info('docs') order by cid") + ).dicts() + assert [column["name"] for column in columns] == ["id", "title", "score", "age"] + assert last_event(ds_write) is None + + +@pytest.mark.asyncio +async def test_alter_table_permission_denied(ds_write): + token = write_token(ds_write, permissions=["ir"]) + response = await ds_write.client.post( + "/data/docs/-/alter", + json={"operations": [{"op": "add_column", "args": {"name": "slug"}}]}, + headers=_headers(token), + ) + assert response.status_code == 403 + assert response.json() == { + "ok": False, + "errors": ["Permission denied: need alter-table"], + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body,expected_error", + ( + ( + {"operations": [{"op": "add_column", "args": {"type": "text"}}]}, + "operations.0.add_column.args.name: Field required", + ), + ( + { + "operations": [ + {"op": "add_column", "args": {"name": "x", "type": "bad"}} + ] + }, + "operations.0.add_column.args.type: Input should be 'text', 'integer', 'float' or 'blob'", + ), + ( + { + "operations": [ + { + "op": "add_column", + "args": { + "name": "x", + "default_expr": "datetime('now')", + }, + } + ] + }, + "operations.0.add_column.args.default_expr: Input should be 'current_timestamp', 'current_date' or 'current_time'", + ), + ( + { + "operations": [ + { + "op": "add_column", + "args": { + "name": "x", + "default": "x", + "default_expr": "current_timestamp", + }, + } + ] + }, + "operations.0.add_column.args: Value error, default and default_expr cannot both be provided", + ), + ), +) +async def test_alter_table_validation_errors(ds_write, body, expected_error): + response = await ds_write.client.post( + "/data/docs/-/alter", + json=body, + headers=_headers(write_token(ds_write, permissions=["at"])), + ) + assert response.status_code == 400 + assert response.json()["ok"] is False + assert response.json()["errors"] == [expected_error] + + @pytest.mark.asyncio async def test_execute_write_form_parameter_called_sql(): ds = Datasette(memory=True, default_deny=True) From fdd1b61a3e4bacba7e792554c5376856160f1dba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 09:21:09 -0700 Subject: [PATCH 017/174] Add alter table modal - Register a built-in table action and expose alter-table metadata to table pages. - Build the client-side modal for editing columns, defaults, ordering, primary keys, and custom column types. - Add a review/apply confirmation flow with HTML and Playwright coverage. Refs #2788 --- datasette/default_table_actions.py | 29 + datasette/plugins.py | 1 + datasette/static/app.css | 499 ++++++++++ datasette/static/edit-tools.js | 1478 +++++++++++++++++++++++++++- datasette/views/row.py | 1 + datasette/views/table.py | 82 +- tests/test_playwright.py | 210 ++++ tests/test_table_html.py | 117 +++ 8 files changed, 2414 insertions(+), 3 deletions(-) create mode 100644 datasette/default_table_actions.py diff --git a/datasette/default_table_actions.py b/datasette/default_table_actions.py new file mode 100644 index 00000000..e41434ef --- /dev/null +++ b/datasette/default_table_actions.py @@ -0,0 +1,29 @@ +from datasette import hookimpl +from datasette.resources import TableResource + + +@hookimpl +def table_actions(datasette, actor, database, table, request): + async def inner(): + db = datasette.get_database(database) + if not db.is_mutable: + return [] + if not await datasette.allowed( + action="alter-table", + resource=TableResource(database=database, table=table), + actor=actor, + ): + return [] + return [ + { + "type": "button", + "label": "Alter table", + "description": "Change columns and primary key for this table.", + "attrs": { + "aria-label": "Alter table {}".format(table), + "data-table-action": "alter-table", + }, + } + ] + + return inner diff --git a/datasette/plugins.py b/datasette/plugins.py index f0fbc7f8..ae2cb17d 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -31,6 +31,7 @@ DEFAULT_PLUGINS = ( "datasette.default_debug_menu", "datasette.default_jump_items", "datasette.default_database_actions", + "datasette.default_table_actions", "datasette.default_query_actions", "datasette.handle_exception", "datasette.forbidden", diff --git a/datasette/static/app.css b/datasette/static/app.css index ce0c46a8..f9ebe5ac 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -2032,6 +2032,505 @@ dialog.table-create-dialog::backdrop { cursor: not-allowed; } +dialog.table-alter-dialog { + --ink: #0f0f0f; + --paper: #eef6ff; + --muted: #6b6b6b; + --rule: #d8e6f5; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; + width: min(980px, calc(100vw - 32px)); + max-width: 95vw; + max-height: min(780px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.table-alter-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.table-alter-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +.table-alter-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 12px; + flex-shrink: 0; + min-width: 0; +} + +.table-alter-dialog .modal-title { + display: flex; + align-items: center; + min-width: 0; + max-width: 100%; + font-size: 1rem; + font-weight: 600; + color: var(--ink); +} + +.table-alter-form { + display: flex; + flex: 1 1 auto; + min-height: 0; + flex-direction: column; +} + +.table-alter-error { + border-left: 4px solid #b91c1c; + border-radius: 4px; + background: #fff1f1; + color: #7f1d1d; + font-size: 0.9rem; + margin: 12px 24px 0; + padding: 10px 12px; +} + +.table-alter-error:focus { + outline: 3px solid rgba(185, 28, 28, 0.18); + outline-offset: 2px; +} + +.table-alter-fields { + display: grid; + gap: 18px; + padding: 16px 24px 24px; + overflow-y: auto; +} + +.table-alter-fields[hidden], +.table-alter-dialog .modal-footer [hidden] { + display: none; +} + +.table-alter-review { + display: grid; + gap: 12px; + overflow-y: auto; + padding: 16px 24px 24px; +} + +.table-alter-review[hidden] { + display: none; +} + +.table-alter-review-title { + color: var(--ink); + font-size: 1rem; + line-height: 1.35; + margin: 0; +} + +.table-alter-review-title:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 2px; +} + +.table-alter-review-intro { + color: var(--muted); + font-size: 0.9rem; + margin: 0; +} + +.table-alter-review-warning { + border-left: 4px solid #b91c1c; + border-radius: 4px; + background: #fff1f1; + color: #7f1d1d; + font-size: 0.9rem; + margin: 0; + padding: 10px 12px; +} + +.table-alter-review-list { + display: grid; + gap: 8px; + margin: 0; + padding-left: 1.4rem; +} + +.table-alter-review-list li { + color: var(--ink); + line-height: 1.4; +} + +.table-alter-review-damaging { + font-weight: 600; +} + +.table-alter-review-name { + background: #eef6ff; + border: 1px solid #c9ddf2; + border-radius: 4px; + color: var(--ink); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.85em; + padding: 1px 4px; + white-space: nowrap; +} + +.table-alter-columns { + display: grid; + gap: 10px; +} + +.table-alter-column-list { + display: grid; + gap: 8px; +} + +.table-alter-column-headings, +.table-alter-column-main { + display: grid; + grid-template-columns: minmax(140px, 1.2fr) minmax(7.5rem, 0.7fr) minmax(12rem, 1fr) max-content 32px; + align-items: center; + gap: 8px; + min-width: 0; +} + +.table-alter-column-row { + display: grid; + gap: 8px; + min-width: 0; +} + +.table-alter-column-headings { + color: var(--muted); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.72rem; + padding: 0 1px; +} + +.table-alter-column-label { + border: 0; + clip: rect(0 0 0 0); + height: 1px; + margin: -1px; + overflow: hidden; + padding: 0; + position: absolute; + white-space: nowrap; + width: 1px; +} + +.table-alter-input { + box-sizing: border-box; + min-width: 0; + border: 1px solid var(--rule); + border-radius: 5px; + padding: 8px 10px; + color: var(--ink); + background: #fff; + font: inherit; +} + +.table-alter-input-placeholder { + color: var(--muted); +} + +.table-alter-default-expr option, +.table-alter-custom-column-type option { + color: var(--ink); +} + +.table-alter-default-expr option[value=""], +.table-alter-custom-column-type option[value=""] { + color: var(--muted); +} + +.table-alter-input:focus { + border-color: var(--accent); + outline: 3px solid rgba(26, 86, 219, 0.12); +} + +.table-alter-column-details { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + align-items: start; + gap: 12px 16px; + padding: 12px; + border-left: 3px solid var(--rule); + background: #f8fafc; +} + +.table-alter-column-details[hidden] { + display: none; +} + +.table-alter-detail-field { + display: grid; + gap: 4px; + min-width: 0; +} + +.table-alter-detail-label { + color: var(--muted); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.72rem; +} + +.table-alter-detail-check { + display: inline-flex; + align-items: flex-start; + gap: 8px; + color: var(--ink); + font-size: 0.85rem; + line-height: 1.35; + min-width: 0; + white-space: normal; +} + +.table-alter-not-null, +.table-alter-primary-key { + grid-column: 1 / -1; +} + +.table-alter-detail-check input { + flex: 0 0 auto; + margin: 0.15rem 0 0; +} + +.table-alter-detail-check span { + min-width: 0; + overflow-wrap: break-word; +} + +.table-alter-move-controls { + display: grid; + grid-template-columns: repeat(4, 32px); + gap: 4px; + justify-content: start; +} + +.table-alter-more-options { + appearance: none; + border: 0; + background: transparent; + color: var(--accent); + cursor: pointer; + font: inherit; + font-size: 0.85rem; + justify-self: start; + padding: 0; + grid-column: 1 / -1; + text-align: left; +} + +.table-alter-more-options:hover, +.table-alter-more-options:focus { + text-decoration: underline; +} + +.table-alter-more-options:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 2px; +} + +.table-alter-more-options:disabled { + color: var(--muted); + cursor: default; + text-decoration: none; +} + +.table-alter-icon-button { + appearance: none; + border: 1px solid rgba(74, 85, 104, 0.24); + background: transparent; + color: #4a5568; + border-radius: 4px; + cursor: pointer; + display: inline-grid; + place-items: center; + height: 32px; + width: 32px; + padding: 0; +} + +.table-alter-icon-button:hover, +.table-alter-icon-button:focus { + background: rgba(74, 85, 104, 0.07); +} + +.table-alter-icon-button:focus { + outline: 3px solid #b3d4ff; + outline-offset: 1px; +} + +.table-alter-icon-button svg { + display: block; +} + +.table-alter-add-column { + appearance: none; + justify-self: start; + border: 1px solid var(--rule); + border-radius: 5px; + background: #fff; + color: var(--accent); + cursor: pointer; + font: inherit; + font-size: 0.85rem; + padding: 7px 10px; +} + +.table-alter-add-column:hover, +.table-alter-add-column:focus { + background: #f8fafc; +} + +.table-alter-add-column:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 1px; +} + +.table-alter-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +.table-alter-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.table-alter-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.table-alter-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.table-alter-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.table-alter-dialog .btn-primary:hover { + background: #1949b8; +} + +.table-alter-dialog .btn-primary:disabled, +.table-alter-dialog .btn-primary:disabled:hover { + background: #a0aec0; + color: #fff; +} + +.table-alter-dialog .btn:disabled, +.table-alter-add-column:disabled, +.table-alter-icon-button:disabled { + opacity: 0.55; + cursor: not-allowed; +} + +@media (max-width: 900px) { + dialog.table-alter-dialog { + width: 95vw; + max-height: 85vh; + border-radius: 0.5rem; + } + + .table-alter-dialog .modal-header, + .table-alter-fields, + .table-alter-review { + padding-left: 18px; + padding-right: 18px; + } + + .table-alter-error { + margin-left: 18px; + margin-right: 18px; + } + + .table-alter-column-headings { + display: none; + } + + .table-alter-column-row { + padding-bottom: 8px; + border-bottom: 1px solid var(--rule); + } + + .table-alter-column-main { + grid-template-columns: minmax(0, 1fr) minmax(7.5rem, 0.8fr) 32px; + align-items: end; + } + + .table-alter-column-name { + grid-column: 1; + grid-row: 1; + } + + .table-alter-column-type { + grid-column: 2; + grid-row: 1; + } + + .table-alter-remove-column { + grid-column: 3; + grid-row: 1; + justify-self: end; + } + + .table-alter-custom-column-type { + grid-column: 1 / 3; + grid-row: 2; + } + + .table-alter-move-controls { + grid-column: 1; + grid-row: 3; + justify-self: start; + } + + .table-alter-more-options { + align-self: center; + grid-column: 2 / 4; + grid-row: 3; + } + + .table-alter-column-details { + grid-template-columns: 1fr; + } + + .table-alter-dialog .modal-footer { + padding-left: 18px; + padding-right: 18px; + } +} + .row-link-with-actions { display: inline-flex; align-items: center; diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 4b93c4fd..284d6bde 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -4,6 +4,8 @@ var ROW_EDIT_DIALOG_ID = "row-edit-dialog"; var rowEditDialogState = null; var TABLE_CREATE_DIALOG_ID = "table-create-dialog"; var tableCreateDialogState = null; +var TABLE_ALTER_DIALOG_ID = "table-alter-dialog"; +var tableAlterDialogState = null; function ensureRowMutationStatus(manager) { var status = document.querySelector(".row-mutation-status"); @@ -59,6 +61,16 @@ function tableCreateColumnTypes() { : ["text", "integer", "float", "blob"]; } +function sqliteColumnTypeLabel(type) { + if (type === "float") { + return "floating point number"; + } + if (type === "blob") { + return "blob - binary data"; + } + return type; +} + function tableCreateCustomColumnTypes() { var data = databaseCreateTableData() || {}; return data.customColumnTypes || []; @@ -147,7 +159,7 @@ function tableCreateSelectTypeValue(select, type) { options.forEach(function (option) { var optionElement = document.createElement("option"); optionElement.value = option; - optionElement.textContent = option; + optionElement.textContent = sqliteColumnTypeLabel(option); select.appendChild(optionElement); }); select.value = options.indexOf(type) === -1 ? options[0] : type; @@ -819,6 +831,1469 @@ function tableInsertData() { return tablePageData().insertRow; } +function tableAlterData() { + return tablePageData().alterTable; +} + +function tableAlterColumnTypes() { + var data = tableAlterData() || {}; + return data.columnTypes && data.columnTypes.length + ? data.columnTypes + : ["text", "integer", "float", "blob"]; +} + +function tableAlterDefaultExpressions() { + var data = tableAlterData() || {}; + return data.defaultExpressions || []; +} + +function tableAlterCustomColumnTypes() { + var data = tableAlterData() || {}; + return data.customColumnTypes || []; +} + +function tableAlterCustomColumnType(name) { + var options = tableAlterCustomColumnTypes(); + for (var i = 0; i < options.length; i += 1) { + if (options[i].name === name) { + return options[i]; + } + } + return null; +} + +function tableAlterCustomTypeAppliesToSqliteType(option, sqliteType) { + return ( + option && + option.sqliteTypes && + option.sqliteTypes.indexOf(sqliteType) !== -1 + ); +} + +function tableAlterDialogRows(state) { + return Array.prototype.slice.call( + state.columnList.querySelectorAll(".table-alter-column-row"), + ); +} + +function tableAlterRowSignature(row) { + return { + existing: row.dataset.existing === "1", + originalName: row.dataset.originalName || "", + name: row.querySelector(".table-alter-column-name").value, + type: row.querySelector(".table-alter-column-type").value, + customType: + ( + row.querySelector(".table-alter-custom-column-type") || { + value: "", + } + ).value || "", + notNull: row.querySelector(".table-alter-not-null-input").checked, + defaultValue: row.querySelector(".table-alter-default").value, + defaultExpr: row.querySelector(".table-alter-default-expr").value, + pk: row.querySelector(".table-alter-primary-key-input").checked, + }; +} + +function tableAlterDialogSignature(state) { + if (!state || !state.form) { + return ""; + } + return JSON.stringify({ + columns: tableAlterDialogRows(state).map(tableAlterRowSignature), + deletedColumns: state.deletedColumns.slice(), + }); +} + +function tableAlterDialogHasChanges(state) { + return ( + !!state && + !state.isSaving && + tableAlterDialogSignature(state) !== state.initialSignature + ); +} + +function updateTableAlterSaveButtonState(state) { + if (!state || !state.saveButton) { + return; + } + state.saveButton.disabled = + state.isSaving || + (state.mode !== "review" && + tableAlterDialogSignature(state) === state.initialSignature); +} + +function tableAlterRowIsPrimaryKey(row) { + var input = row && row.querySelector(".table-alter-primary-key-input"); + return !!(input && input.checked); +} + +function tableAlterFirstNonPrimaryRow(state) { + var rows = tableAlterDialogRows(state); + for (var i = 0; i < rows.length; i += 1) { + if (!tableAlterRowIsPrimaryKey(rows[i])) { + return rows[i]; + } + } + return null; +} + +function updateTableAlterMoveButtons(state) { + if (!state || !state.columnList) { + return; + } + var firstNonPrimary = tableAlterFirstNonPrimaryRow(state); + var rows = tableAlterDialogRows(state); + var hasPrimaryKeys = rows.some(function (row) { + return tableAlterRowIsPrimaryKey(row); + }); + var primaryKeyMoveTitle = "Primary key columns are always listed first"; + rows.forEach(function (row) { + var isPrimaryKey = tableAlterRowIsPrimaryKey(row); + var previous = row.previousElementSibling; + var next = row.nextElementSibling; + row.querySelectorAll(".table-alter-move-controls button").forEach( + function (button) { + button.title = button.dataset.defaultTitle || button.title; + button.disabled = state.isSaving || isPrimaryKey; + if (isPrimaryKey) { + button.title = primaryKeyMoveTitle; + } + }, + ); + if (!isPrimaryKey) { + var topButton = row.querySelector(".table-alter-move-top"); + var upButton = row.querySelector(".table-alter-move-up"); + var downButton = row.querySelector(".table-alter-move-down"); + var bottomButton = row.querySelector(".table-alter-move-bottom"); + topButton.disabled = + state.isSaving || !firstNonPrimary || row === firstNonPrimary; + upButton.disabled = + state.isSaving || !previous || tableAlterRowIsPrimaryKey(previous); + downButton.disabled = state.isSaving || !next; + bottomButton.disabled = state.isSaving || !next; + if (hasPrimaryKeys && row === firstNonPrimary) { + topButton.title = primaryKeyMoveTitle; + upButton.title = primaryKeyMoveTitle; + } + } + }); +} + +function normalizeTableAlterPrimaryKeyRows(state) { + var rows = tableAlterDialogRows(state); + rows + .filter(function (row) { + return tableAlterRowIsPrimaryKey(row); + }) + .concat( + rows.filter(function (row) { + return !tableAlterRowIsPrimaryKey(row); + }), + ) + .forEach(function (row) { + state.columnList.appendChild(row); + }); +} + +function clearTableAlterDialogError(state) { + state.error.hidden = true; + state.error.textContent = ""; + state.dialog.removeAttribute("aria-describedby"); +} + +function showTableAlterDialogError(state, message) { + state.error.hidden = false; + state.error.textContent = message; + state.dialog.setAttribute("aria-describedby", "table-alter-error"); + state.error.focus(); +} + +function setTableAlterDialogSaving(state, isSaving) { + state.isSaving = isSaving; + state.cancelButton.disabled = isSaving; + state.addColumnButton.disabled = isSaving; + state.backButton.disabled = isSaving; + state.saveButton.textContent = isSaving + ? state.mode === "review" + ? "Applying..." + : "Preparing..." + : tableAlterSaveButtonText(state); + state.columnList + .querySelectorAll("input, select, button") + .forEach(function (control) { + control.disabled = isSaving; + }); + if (!isSaving) { + state.columnList + .querySelectorAll(".table-alter-default-expr") + .forEach(function (select) { + syncTableAlterDefaultControls(select.closest(".table-alter-column-row")); + }); + } + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); +} + +function tableAlterSaveButtonText(state) { + return state && state.mode === "review" ? "Apply changes" : "Review changes"; +} + +function tableAlterSelectTypeValue(select, type) { + var options = tableAlterColumnTypes(); + options.forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option; + optionElement.textContent = sqliteColumnTypeLabel(option); + select.appendChild(optionElement); + }); + select.value = options.indexOf(type) === -1 ? options[0] : type; +} + +function updateTableAlterCustomColumnTypePlaceholder(select) { + select.classList.toggle( + "table-alter-input-placeholder", + !select.value, + ); +} + +function createTableAlterCustomColumnTypeSelect() { + var options = tableAlterCustomColumnTypes(); + var select = document.createElement("select"); + select.className = "table-alter-input table-alter-custom-column-type"; + select.setAttribute("aria-label", "Custom column type"); + var blankOption = document.createElement("option"); + blankOption.value = ""; + blankOption.textContent = "- custom type -"; + select.appendChild(blankOption); + options.forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option.name; + optionElement.textContent = option.description + ? option.description + " (" + option.name + ")" + : option.name; + select.appendChild(optionElement); + }); + updateTableAlterCustomColumnTypePlaceholder(select); + return select; +} + +function syncTableAlterCustomTypeForSqliteType(row) { + var typeSelect = row.querySelector(".table-alter-column-type"); + var customTypeSelect = row.querySelector(".table-alter-custom-column-type"); + if (!typeSelect || !customTypeSelect || !customTypeSelect.value) { + return; + } + var option = tableAlterCustomColumnType(customTypeSelect.value); + if (!tableAlterCustomTypeAppliesToSqliteType(option, typeSelect.value)) { + customTypeSelect.value = ""; + updateTableAlterCustomColumnTypePlaceholder(customTypeSelect); + } +} + +function tableAlterSelectDefaultExprValue(select, value) { + var blankOption = document.createElement("option"); + blankOption.value = ""; + blankOption.textContent = "- default expr -"; + select.appendChild(blankOption); + tableAlterDefaultExpressions().forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option; + optionElement.textContent = option.replace(/_/g, " "); + select.appendChild(optionElement); + }); + select.value = value || ""; + updateTableAlterDefaultExprPlaceholder(select); +} + +function updateTableAlterDefaultExprPlaceholder(select) { + select.classList.toggle("table-alter-input-placeholder", !select.value); +} + +function syncTableAlterDefaultControls(row) { + if (!row) { + return; + } + var defaultInput = row.querySelector(".table-alter-default"); + var defaultExprSelect = row.querySelector(".table-alter-default-expr"); + if (!defaultInput || !defaultExprSelect) { + return; + } + defaultInput.disabled = !!defaultExprSelect.value; + updateTableAlterDefaultExprPlaceholder(defaultExprSelect); +} + +function createTableAlterColumnRow(state, column) { + var index = state.nextColumnIndex; + state.nextColumnIndex += 1; + var existing = !!(column && column.existing); + var originalName = existing ? column.name || "" : ""; + var originalCustomType = + existing && column.column_type ? column.column_type.type || "" : ""; + var originalDefault = + existing && column.has_default && column.default !== null + ? String(column.default) + : ""; + + var row = document.createElement("div"); + row.className = "table-alter-column-row"; + row.dataset.existing = existing ? "1" : "0"; + row.dataset.originalName = originalName; + row.dataset.originalType = existing ? column.type || "text" : ""; + row.dataset.originalNotNull = existing && column.notnull ? "1" : "0"; + row.dataset.originalHasDefault = existing && column.has_default ? "1" : "0"; + row.dataset.originalDefault = originalDefault; + row.dataset.originalPk = existing && column.is_pk ? "1" : "0"; + row.dataset.originalCustomType = originalCustomType; + + var main = document.createElement("div"); + main.className = "table-alter-column-main"; + + var details = document.createElement("div"); + details.className = "table-alter-column-details"; + details.id = "table-alter-column-details-" + index; + details.hidden = !(column && column.expanded); + + var expandButton = document.createElement("button"); + expandButton.type = "button"; + expandButton.className = "table-alter-more-options"; + expandButton.setAttribute("aria-label", "Toggle column settings"); + expandButton.setAttribute("aria-controls", details.id); + expandButton.setAttribute( + "aria-expanded", + details.hidden ? "false" : "true", + ); + function updateExpandButton() { + var isExpanded = expandButton.getAttribute("aria-expanded") === "true"; + expandButton.textContent = isExpanded + ? "v Hide options" + : "> Advanced options"; + expandButton.title = isExpanded + ? "Hide column settings" + : "Show column settings"; + } + updateExpandButton(); + + var nameId = "table-alter-column-name-" + index; + var nameLabel = document.createElement("label"); + nameLabel.className = "table-alter-column-label"; + nameLabel.setAttribute("for", nameId); + nameLabel.textContent = "Column"; + + var nameInput = document.createElement("input"); + nameInput.id = nameId; + nameInput.className = "table-alter-input table-alter-column-name"; + nameInput.type = "text"; + nameInput.required = true; + nameInput.autocomplete = "off"; + nameInput.value = column && column.name ? column.name : ""; + + var typeSelect = document.createElement("select"); + typeSelect.className = "table-alter-input table-alter-column-type"; + typeSelect.setAttribute("aria-label", "Column type"); + tableAlterSelectTypeValue(typeSelect, column && column.type); + + var customTypeSelect = null; + if (tableAlterCustomColumnTypes().length) { + customTypeSelect = createTableAlterCustomColumnTypeSelect(); + customTypeSelect.value = originalCustomType; + updateTableAlterCustomColumnTypePlaceholder(customTypeSelect); + } + + var notNullLabel = document.createElement("label"); + notNullLabel.className = "table-alter-detail-check table-alter-not-null"; + var notNullInput = document.createElement("input"); + notNullInput.type = "checkbox"; + notNullInput.className = "table-alter-not-null-input"; + notNullInput.checked = !!(column && column.notnull); + var notNullText = document.createElement("span"); + var notNullStrong = document.createElement("strong"); + notNullStrong.textContent = "Not null"; + notNullText.appendChild(notNullStrong); + notNullText.appendChild( + document.createTextNode(" This value cannot be left unset"), + ); + notNullLabel.appendChild(notNullInput); + notNullLabel.appendChild(notNullText); + + var defaultId = "table-alter-column-default-" + index; + var defaultField = document.createElement("div"); + defaultField.className = "table-alter-detail-field"; + var defaultLabel = document.createElement("label"); + defaultLabel.className = "table-alter-detail-label"; + defaultLabel.setAttribute("for", defaultId); + defaultLabel.textContent = "Default value"; + var defaultInput = document.createElement("input"); + defaultInput.id = defaultId; + defaultInput.className = "table-alter-input table-alter-default"; + defaultInput.type = "text"; + defaultInput.autocomplete = "off"; + defaultInput.placeholder = "default"; + defaultInput.setAttribute("aria-label", "Default value"); + defaultInput.value = originalDefault; + defaultField.appendChild(defaultLabel); + defaultField.appendChild(defaultInput); + + var defaultExprId = "table-alter-column-default-expr-" + index; + var defaultExprField = document.createElement("div"); + defaultExprField.className = "table-alter-detail-field"; + var defaultExprLabel = document.createElement("label"); + defaultExprLabel.className = "table-alter-detail-label"; + defaultExprLabel.setAttribute("for", defaultExprId); + defaultExprLabel.textContent = "or default to a specific time"; + var defaultExprSelect = document.createElement("select"); + defaultExprSelect.id = defaultExprId; + defaultExprSelect.className = + "table-alter-input table-alter-default-expr"; + defaultExprSelect.setAttribute("aria-label", "or default to a specific time"); + tableAlterSelectDefaultExprValue(defaultExprSelect, ""); + defaultExprField.appendChild(defaultExprLabel); + defaultExprField.appendChild(defaultExprSelect); + + var pkLabel = document.createElement("label"); + pkLabel.className = "table-alter-detail-check table-alter-primary-key"; + var pkInput = document.createElement("input"); + pkInput.type = "checkbox"; + pkInput.className = "table-alter-primary-key-input"; + pkInput.checked = !!(column && column.is_pk); + var pkText = document.createElement("span"); + var pkStrong = document.createElement("strong"); + pkStrong.textContent = "Primary key"; + pkText.appendChild(pkStrong); + pkText.appendChild( + document.createTextNode(" An ID that uniquely identifies this record"), + ); + pkLabel.appendChild(pkInput); + pkLabel.appendChild(pkText); + + var moveControls = document.createElement("div"); + moveControls.className = "table-alter-move-controls"; + + var moveTopButton = document.createElement("button"); + moveTopButton.type = "button"; + moveTopButton.className = "table-alter-icon-button table-alter-move-top"; + moveTopButton.setAttribute("aria-label", "Move column to top"); + moveTopButton.title = "Move column to top"; + moveTopButton.dataset.defaultTitle = moveTopButton.title; + moveTopButton.innerHTML = + ''; + + var moveUpButton = document.createElement("button"); + moveUpButton.type = "button"; + moveUpButton.className = "table-alter-icon-button table-alter-move-up"; + moveUpButton.setAttribute("aria-label", "Move column up"); + moveUpButton.title = "Move column up"; + moveUpButton.dataset.defaultTitle = moveUpButton.title; + moveUpButton.innerHTML = + ''; + + var moveDownButton = document.createElement("button"); + moveDownButton.type = "button"; + moveDownButton.className = "table-alter-icon-button table-alter-move-down"; + moveDownButton.setAttribute("aria-label", "Move column down"); + moveDownButton.title = "Move column down"; + moveDownButton.dataset.defaultTitle = moveDownButton.title; + moveDownButton.innerHTML = + ''; + + var moveBottomButton = document.createElement("button"); + moveBottomButton.type = "button"; + moveBottomButton.className = "table-alter-icon-button table-alter-move-bottom"; + moveBottomButton.setAttribute("aria-label", "Move column to bottom"); + moveBottomButton.title = "Move column to bottom"; + moveBottomButton.dataset.defaultTitle = moveBottomButton.title; + moveBottomButton.innerHTML = + ''; + + var removeButton = document.createElement("button"); + removeButton.type = "button"; + removeButton.className = "table-alter-icon-button table-alter-remove-column"; + removeButton.setAttribute( + "aria-label", + existing ? "Drop column" : "Remove column", + ); + removeButton.title = existing ? "Drop column" : "Remove column"; + removeButton.innerHTML = + ''; + + moveControls.appendChild(moveTopButton); + moveControls.appendChild(moveUpButton); + moveControls.appendChild(moveDownButton); + moveControls.appendChild(moveBottomButton); + main.appendChild(nameLabel); + main.appendChild(nameInput); + main.appendChild(typeSelect); + if (customTypeSelect) { + main.appendChild(customTypeSelect); + } + main.appendChild(moveControls); + main.appendChild(removeButton); + main.appendChild(expandButton); + + details.appendChild(notNullLabel); + details.appendChild(defaultField); + details.appendChild(defaultExprField); + details.appendChild(pkLabel); + row.appendChild(main); + row.appendChild(details); + + var controls = [ + nameInput, + typeSelect, + notNullInput, + defaultInput, + defaultExprSelect, + pkInput, + ]; + if (customTypeSelect) { + controls.push(customTypeSelect); + } + controls.forEach(function (control) { + control.addEventListener("input", function () { + clearTableAlterDialogError(state); + updateTableAlterSaveButtonState(state); + }); + control.addEventListener("change", function () { + clearTableAlterDialogError(state); + updateTableAlterSaveButtonState(state); + }); + }); + + defaultInput.addEventListener("input", function () { + if (defaultInput.value) { + defaultExprSelect.value = ""; + syncTableAlterDefaultControls(row); + } + updateTableAlterSaveButtonState(state); + }); + defaultExprSelect.addEventListener("change", function () { + if (defaultExprSelect.value) { + defaultInput.value = ""; + } + syncTableAlterDefaultControls(row); + updateTableAlterSaveButtonState(state); + }); + pkInput.addEventListener("change", function () { + normalizeTableAlterPrimaryKeyRows(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); + }); + + expandButton.addEventListener("click", function () { + var isExpanded = expandButton.getAttribute("aria-expanded") === "true"; + details.hidden = isExpanded; + expandButton.setAttribute("aria-expanded", isExpanded ? "false" : "true"); + updateExpandButton(); + }); + + typeSelect.addEventListener("change", function () { + syncTableAlterCustomTypeForSqliteType(row); + updateTableAlterSaveButtonState(state); + }); + if (customTypeSelect) { + customTypeSelect.addEventListener("change", function () { + updateTableAlterCustomColumnTypePlaceholder(customTypeSelect); + var option = tableAlterCustomColumnType(customTypeSelect.value); + if ( + option && + option.fixedSqliteType && + tableAlterColumnTypes().indexOf(option.fixedSqliteType) !== -1 + ) { + typeSelect.value = option.fixedSqliteType; + } + updateTableAlterSaveButtonState(state); + }); + } + + moveTopButton.addEventListener("click", function () { + var first = tableAlterFirstNonPrimaryRow(state); + if (state.isSaving || tableAlterRowIsPrimaryKey(row) || !first || first === row) { + return; + } + state.columnList.insertBefore(row, first); + clearTableAlterDialogError(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); + row.querySelector(".table-alter-column-name").focus(); + }); + + moveUpButton.addEventListener("click", function () { + var previous = row.previousElementSibling; + if ( + state.isSaving || + tableAlterRowIsPrimaryKey(row) || + !previous || + tableAlterRowIsPrimaryKey(previous) + ) { + return; + } + state.columnList.insertBefore(row, previous); + clearTableAlterDialogError(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); + row.querySelector(".table-alter-column-name").focus(); + }); + + moveDownButton.addEventListener("click", function () { + var next = row.nextElementSibling; + if (state.isSaving || tableAlterRowIsPrimaryKey(row) || !next) { + return; + } + state.columnList.insertBefore(next, row); + clearTableAlterDialogError(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); + row.querySelector(".table-alter-column-name").focus(); + }); + + moveBottomButton.addEventListener("click", function () { + var last = state.columnList.lastElementChild; + if (state.isSaving || tableAlterRowIsPrimaryKey(row) || !last || last === row) { + return; + } + state.columnList.appendChild(row); + clearTableAlterDialogError(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); + row.querySelector(".table-alter-column-name").focus(); + }); + + removeButton.addEventListener("click", function () { + if (state.isSaving) { + return; + } + if (row.dataset.existing === "1") { + state.deletedColumns.push(row.dataset.originalName); + } + row.remove(); + clearTableAlterDialogError(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); + var nextInput = state.columnList.querySelector(".table-alter-column-name"); + if (nextInput) { + nextInput.focus(); + } else { + state.addColumnButton.focus(); + } + }); + + syncTableAlterDefaultControls(row); + return row; +} + +function addTableAlterColumn(state, column) { + var row = createTableAlterColumnRow(state, column || { type: "text" }); + state.columnList.appendChild(row); + return row; +} + +function resetTableAlterDialog(state, data) { + state.nextColumnIndex = 0; + state.deletedColumns = []; + state.originalPrimaryKeys = (data.primaryKeys || []).slice(); + state.originalColumnNames = (data.columns || []).map(function (column) { + return column.name; + }); + state.columnList.textContent = ""; + (data.columns || []).forEach(function (column) { + addTableAlterColumn( + state, + Object.assign({}, column, { + existing: true, + }), + ); + }); + normalizeTableAlterPrimaryKeyRows(state); + state.initialSignature = tableAlterDialogSignature(state); + showTableAlterEditor(state); +} + +function collectTableAlterRows(state) { + return tableAlterDialogRows(state).map(function (row) { + var signature = tableAlterRowSignature(row); + signature.originalType = row.dataset.originalType || ""; + signature.originalNotNull = row.dataset.originalNotNull === "1"; + signature.originalHasDefault = row.dataset.originalHasDefault === "1"; + signature.originalDefault = row.dataset.originalDefault || ""; + signature.originalPk = row.dataset.originalPk === "1"; + signature.originalCustomType = row.dataset.originalCustomType || ""; + return signature; + }); +} + +function validateTableAlterRows(state, rows) { + if (!rows.length) { + return "At least one column is required."; + } + var seen = {}; + var supportedTypes = tableAlterColumnTypes(); + for (var i = 0; i < rows.length; i += 1) { + var row = rows[i]; + var name = row.name.trim(); + if (!name) { + return "Column name is required."; + } + if (name.indexOf("\n") !== -1) { + return "Column names cannot contain newlines."; + } + var columnKey = name.toLowerCase(); + if (seen[columnKey]) { + return "Duplicate column name: " + name; + } + seen[columnKey] = true; + if (supportedTypes.indexOf(row.type) === -1) { + return "Unsupported column type: " + row.type; + } + if (row.customType) { + var option = tableAlterCustomColumnType(row.customType); + if (!option) { + return "Unknown custom column type: " + row.customType; + } + if (!tableAlterCustomTypeAppliesToSqliteType(option, row.type)) { + return ( + "Custom type " + + row.customType + + " cannot be used with SQLite type " + + row.type + + "." + ); + } + } + if (row.defaultValue && row.defaultExpr) { + return "Use either a default value or a default expression."; + } + if (!row.existing && row.notNull && !row.defaultValue && !row.defaultExpr) { + return "New NOT NULL columns need a default or default expression."; + } + } + var pkColumns = rows.filter(function (row) { + return row.pk; + }); + if (state.originalPrimaryKeys.length && !pkColumns.length) { + return "At least one primary key column is required."; + } + return null; +} + +function collectTableAlterColumnTypeAssignments(rows) { + var assignments = []; + if (!tableAlterCustomColumnTypes().length) { + return assignments; + } + rows.forEach(function (row) { + var renamed = row.existing && row.name.trim() !== row.originalName; + if (row.customType === row.originalCustomType && !renamed) { + return; + } + if (!row.customType && !row.originalCustomType) { + return; + } + assignments.push({ + column: row.name.trim(), + columnType: row.customType || null, + sqliteType: row.type, + }); + }); + return assignments; +} + +function tableAlterPkIdentityColumns(rows) { + return rows + .filter(function (row) { + return row.pk; + }) + .map(function (row) { + return row.existing ? row.originalName : row.name.trim(); + }); +} + +function tableAlterPkChanged(state, rows) { + return ( + JSON.stringify(tableAlterPkIdentityColumns(rows)) !== + JSON.stringify(state.originalPrimaryKeys) + ); +} + +function tableAlterNaturalColumnOrder(state, rows) { + var existingRowsByOriginalName = {}; + var newRows = []; + rows.forEach(function (row) { + if (row.existing) { + existingRowsByOriginalName[row.originalName] = row; + } else { + newRows.push(row); + } + }); + var naturalOrder = []; + state.originalColumnNames.forEach(function (originalName) { + var row = existingRowsByOriginalName[originalName]; + if (row) { + naturalOrder.push(row.name.trim()); + } + }); + newRows.forEach(function (row) { + naturalOrder.push(row.name.trim()); + }); + return naturalOrder; +} + +function tableAlterColumnsReordered(state, rows) { + var finalOrder = rows.map(function (row) { + return row.name.trim(); + }); + return ( + JSON.stringify(finalOrder) !== + JSON.stringify(tableAlterNaturalColumnOrder(state, rows)) + ); +} + +function collectTableAlterPayload(state) { + var rows = collectTableAlterRows(state); + var validationError = validateTableAlterRows(state, rows); + if (validationError) { + return { error: validationError }; + } + + var operations = []; + var columnTypeAssignments = collectTableAlterColumnTypeAssignments(rows); + rows.forEach(function (row) { + var name = row.name.trim(); + if (!row.existing) { + var addArgs = { + name: name, + type: row.type, + not_null: row.notNull, + }; + if (row.defaultExpr) { + addArgs.default_expr = row.defaultExpr; + } else if (row.defaultValue || row.notNull) { + addArgs.default = row.defaultValue; + } + operations.push({ op: "add_column", args: addArgs }); + return; + } + + var originalName = row.originalName; + if (name !== originalName) { + operations.push({ + op: "rename_column", + args: { name: originalName, to: name }, + }); + } + + var alterArgs = { name: originalName }; + if (row.type !== row.originalType) { + alterArgs.type = row.type; + } + if (row.notNull !== row.originalNotNull) { + alterArgs.not_null = row.notNull; + } + if (row.defaultExpr) { + alterArgs.default_expr = row.defaultExpr; + } else if (row.originalHasDefault) { + if (row.defaultValue !== row.originalDefault) { + alterArgs.default = row.defaultValue === "" ? null : row.defaultValue; + } + } else if (row.defaultValue) { + alterArgs.default = row.defaultValue; + } + if (Object.keys(alterArgs).length > 1) { + operations.push({ op: "alter_column", args: alterArgs }); + } + }); + + state.deletedColumns.forEach(function (name) { + operations.push({ op: "drop_column", args: { name: name } }); + }); + + var pkColumns = rows + .filter(function (row) { + return row.pk; + }) + .map(function (row) { + return row.name.trim(); + }); + if (tableAlterPkChanged(state, rows)) { + operations.push({ op: "set_primary_key", args: { columns: pkColumns } }); + } + + if (tableAlterColumnsReordered(state, rows)) { + operations.push({ + op: "reorder_columns", + args: { + columns: rows.map(function (row) { + return row.name.trim(); + }), + }, + }); + } + + if (!operations.length && !columnTypeAssignments.length) { + return { error: "No changes to apply." }; + } + return { + payload: operations.length ? { operations: operations } : null, + columnTypeAssignments: columnTypeAssignments, + }; +} + +function tableAlterQuotedName(name) { + return '"' + name + '"'; +} + +function tableAlterReadableDefaultExpression(value) { + return value ? value.replace(/_/g, " ") : ""; +} + +function tableAlterReadableValue(value) { + if (value === null) { + return "NULL"; + } + return '"' + String(value) + '"'; +} + +function tableAlterOperationSummary(operation) { + var args = operation.args || {}; + if (operation.op === "add_column") { + var addDetails = ["as " + args.type]; + if (args.not_null) { + addDetails.push("with values required"); + } + if (args.default_expr) { + addDetails.push( + "defaulting to " + + tableAlterReadableDefaultExpression(args.default_expr), + ); + } else if (Object.prototype.hasOwnProperty.call(args, "default")) { + addDetails.push( + "with default value " + tableAlterReadableValue(args.default), + ); + } + return { + text: + "Add column " + + tableAlterQuotedName(args.name) + + " " + + addDetails.join(", ") + + ".", + damaging: false, + }; + } + if (operation.op === "rename_column") { + return { + text: + "Rename column " + + tableAlterQuotedName(args.name) + + " to " + + tableAlterQuotedName(args.to) + + ".", + damaging: false, + }; + } + if (operation.op === "alter_column") { + var changes = []; + if (args.type) { + changes.push("set type to " + args.type); + } + if (Object.prototype.hasOwnProperty.call(args, "not_null")) { + changes.push( + args.not_null + ? "not null (require values)" + : "allow unset values", + ); + } + if (args.default_expr) { + changes.push( + "default to " + tableAlterReadableDefaultExpression(args.default_expr), + ); + } else if (Object.prototype.hasOwnProperty.call(args, "default")) { + changes.push( + args.default === null + ? "remove the default value" + : "set default value to " + tableAlterReadableValue(args.default), + ); + } + return { + text: + "Change column " + + tableAlterQuotedName(args.name) + + ": " + + changes.join(", ") + + ".", + damaging: false, + }; + } + if (operation.op === "drop_column") { + return { + text: "Drop column " + tableAlterQuotedName(args.name) + ".", + damaging: true, + }; + } + if (operation.op === "set_primary_key") { + return { + text: + "Set primary key to " + + (args.columns || []).map(tableAlterQuotedName).join(", ") + + ".", + damaging: false, + }; + } + if (operation.op === "reorder_columns") { + return { + text: + "Set column order to " + + (args.columns || []).map(tableAlterQuotedName).join(", ") + + ".", + damaging: false, + }; + } + return { + text: "Run " + operation.op + ".", + damaging: false, + }; +} + +function tableAlterColumnTypeAssignmentSummary(assignment) { + return { + text: assignment.columnType + ? "Set custom type for column " + + tableAlterQuotedName(assignment.column) + + " to " + + assignment.columnType + + "." + : "Remove custom type from column " + + tableAlterQuotedName(assignment.column) + + ".", + damaging: false, + }; +} + +function tableAlterReviewItems(result) { + var items = []; + var operations = result.payload ? result.payload.operations || [] : []; + operations.forEach(function (operation) { + items.push(tableAlterOperationSummary(operation)); + }); + (result.columnTypeAssignments || []).forEach(function (assignment) { + items.push(tableAlterColumnTypeAssignmentSummary(assignment)); + }); + return items; +} + +function tableAlterReviewHasDamagingItems(items) { + return items.some(function (item) { + return item.damaging; + }); +} + +function appendTableAlterReviewText(element, text) { + text.split(/("[^"]+")/g).forEach(function (part) { + if (!part) { + return; + } + if (part.charAt(0) === '"' && part.charAt(part.length - 1) === '"') { + var name = document.createElement("code"); + name.className = "table-alter-review-name"; + name.textContent = part.slice(1, -1); + element.appendChild(name); + } else { + element.appendChild(document.createTextNode(part)); + } + }); +} + +function tableAlterSetColumnTypeUrl() { + var data = tableAlterData(); + if (!data || !data.path) { + return null; + } + var url = new URL(data.path, location.href); + url.pathname = url.pathname.replace(/\/-\/alter\/?$/, "/-/set-column-type"); + return url.toString(); +} + +async function assignTableAlterColumnTypes(assignments) { + if (!assignments.length) { + return; + } + var url = tableAlterSetColumnTypeUrl(); + if (!url) { + throw new Error("Could not find the set column type URL."); + } + for (var i = 0; i < assignments.length; i += 1) { + var assignment = assignments[i]; + var response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ + column: assignment.column, + column_type: assignment.columnType + ? { + type: assignment.columnType, + } + : null, + }), + }); + var data = null; + try { + data = await response.json(); + } catch (_error) { + data = null; + } + if (!response.ok || (data && data.ok === false)) { + var error = rowMutationRequestError(response, data); + throw new Error( + "Saved schema changes, but could not set custom type for " + + assignment.column + + ": " + + error.message, + ); + } + } +} + +function showTableAlterEditor(state) { + state.mode = "edit"; + state.reviewResult = null; + state.dialog.classList.remove("table-alter-reviewing"); + state.fields.hidden = false; + state.review.hidden = true; + state.review.textContent = ""; + state.backButton.hidden = true; + state.saveButton.textContent = tableAlterSaveButtonText(state); + updateTableAlterMoveButtons(state); + updateTableAlterSaveButtonState(state); +} + +function showTableAlterReview(state, result) { + var items = tableAlterReviewItems(result); + state.mode = "review"; + state.reviewResult = result; + state.dialog.classList.add("table-alter-reviewing"); + state.fields.hidden = true; + state.review.hidden = false; + state.review.textContent = ""; + state.backButton.hidden = false; + state.saveButton.textContent = tableAlterSaveButtonText(state); + updateTableAlterSaveButtonState(state); + + var heading = document.createElement("h3"); + heading.className = "table-alter-review-title"; + heading.tabIndex = -1; + heading.textContent = "Review changes"; + state.review.appendChild(heading); + + var intro = document.createElement("p"); + intro.className = "table-alter-review-intro"; + intro.textContent = "These changes will be applied to the table."; + state.review.appendChild(intro); + + if (tableAlterReviewHasDamagingItems(items)) { + var warning = document.createElement("p"); + warning.className = "table-alter-review-warning"; + warning.setAttribute("role", "alert"); + warning.textContent = + "Warning: data in dropped columns will be permanently lost."; + state.review.appendChild(warning); + } + + var list = document.createElement("ol"); + list.className = "table-alter-review-list"; + items.forEach(function (item) { + var listItem = document.createElement("li"); + appendTableAlterReviewText(listItem, item.text); + if (item.damaging) { + listItem.className = "table-alter-review-damaging"; + } + list.appendChild(listItem); + }); + state.review.appendChild(list); + heading.focus(); +} + +async function applyTableAlterChanges(state, result) { + if (state.isSaving) { + return; + } + if (!result) { + showTableAlterDialogError(state, "Could not find the reviewed changes."); + return; + } + var data = tableAlterData(); + if (!data || !data.path) { + showTableAlterDialogError(state, "Could not find the alter table URL."); + return; + } + clearTableAlterDialogError(state); + if (result.error) { + showTableAlterDialogError(state, result.error); + return; + } + setTableAlterDialogSaving(state, true); + try { + if (result.payload) { + var response = await fetch(data.path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(result.payload), + }); + var responseData = null; + try { + responseData = await response.json(); + } catch (_error) { + responseData = null; + } + if (!response.ok || (responseData && responseData.ok === false)) { + throw rowMutationRequestError(response, responseData); + } + } + await assignTableAlterColumnTypes(result.columnTypeAssignments || []); + state.shouldRestoreFocus = false; + state.dialog.close(); + location.reload(); + } catch (error) { + setTableAlterDialogSaving(state, false); + showTableAlterDialogError(state, error.message || "Could not alter table"); + } +} + +async function saveTableAlterDialog(state) { + if (state.isSaving) { + return; + } + if (state.mode === "review") { + if (!state.reviewResult) { + showTableAlterDialogError(state, "Could not find the reviewed changes."); + return; + } + await applyTableAlterChanges(state, state.reviewResult); + return; + } + clearTableAlterDialogError(state); + var result = collectTableAlterPayload(state); + if (result.error) { + showTableAlterDialogError(state, result.error); + return; + } + showTableAlterReview(state, result); +} + +function confirmDiscardTableAlterChanges(state) { + if (!tableAlterDialogHasChanges(state)) { + return true; + } + return window.confirm("Discard table changes?"); +} + +function closeTableAlterDialogIfConfirmed(state) { + if (!state || state.isSaving) { + return false; + } + if (!confirmDiscardTableAlterChanges(state)) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + +function closeTableAlterDialog(state) { + if (!state || state.isSaving) { + return false; + } + state.shouldRestoreFocus = true; + state.dialog.close(); + return true; +} + +function ensureTableAlterDialog(manager) { + if (tableAlterDialogState) { + return tableAlterDialogState; + } + if (!window.HTMLDialogElement) { + return null; + } + + var dialog = document.createElement("dialog"); + dialog.id = TABLE_ALTER_DIALOG_ID; + dialog.className = "table-alter-dialog"; + dialog.setAttribute("aria-labelledby", "table-alter-title"); + dialog.innerHTML = ` +
+ Alter table +
+ + +
+
+ +
+ +
+
+ +
+ + + +
+ + `; + document.body.appendChild(dialog); + + tableAlterDialogState = { + dialog: dialog, + form: dialog.querySelector(".table-alter-form"), + title: dialog.querySelector(".modal-title"), + error: dialog.querySelector(".table-alter-error"), + fields: dialog.querySelector(".table-alter-fields"), + review: dialog.querySelector(".table-alter-review"), + columnList: dialog.querySelector(".table-alter-column-list"), + addColumnButton: dialog.querySelector(".table-alter-add-column"), + backButton: dialog.querySelector(".table-alter-back"), + cancelButton: dialog.querySelector(".table-alter-cancel"), + saveButton: dialog.querySelector(".table-alter-save"), + currentButton: null, + shouldRestoreFocus: true, + isSaving: false, + initialSignature: "", + nextColumnIndex: 0, + deletedColumns: [], + originalColumnNames: [], + originalPrimaryKeys: [], + mode: "edit", + reviewResult: null, + manager: manager, + }; + + tableAlterDialogState.form.addEventListener("submit", function (ev) { + ev.preventDefault(); + saveTableAlterDialog(tableAlterDialogState); + }); + + tableAlterDialogState.addColumnButton.addEventListener("click", function () { + if (tableAlterDialogState.isSaving) { + return; + } + var row = addTableAlterColumn(tableAlterDialogState, { + type: "text", + existing: false, + expanded: true, + }); + clearTableAlterDialogError(tableAlterDialogState); + updateTableAlterMoveButtons(tableAlterDialogState); + updateTableAlterSaveButtonState(tableAlterDialogState); + row.querySelector(".table-alter-column-name").focus(); + }); + + tableAlterDialogState.cancelButton.addEventListener("click", function () { + closeTableAlterDialog(tableAlterDialogState); + }); + + tableAlterDialogState.backButton.addEventListener("click", function () { + if (tableAlterDialogState.isSaving) { + return; + } + clearTableAlterDialogError(tableAlterDialogState); + showTableAlterEditor(tableAlterDialogState); + var firstName = tableAlterDialogState.columnList.querySelector( + ".table-alter-column-name", + ); + if (firstName) { + firstName.focus(); + } + }); + + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog) { + closeTableAlterDialogIfConfirmed(tableAlterDialogState); + } + }); + + dialog.addEventListener("keydown", function (ev) { + if (ev.key !== "Escape") { + return; + } + ev.preventDefault(); + closeTableAlterDialogIfConfirmed(tableAlterDialogState); + }); + + dialog.addEventListener("cancel", function (ev) { + ev.preventDefault(); + closeTableAlterDialogIfConfirmed(tableAlterDialogState); + }); + + dialog.addEventListener("close", function () { + var state = tableAlterDialogState; + clearTableAlterDialogError(state); + setTableAlterDialogSaving(state, false); + if ( + state.shouldRestoreFocus && + state.currentButton && + document.contains(state.currentButton) + ) { + state.currentButton.focus(); + } + }); + + return tableAlterDialogState; +} + +function openTableAlterDialog(button, manager) { + var data = tableAlterData(); + if (!data) { + return; + } + var state = ensureTableAlterDialog(manager); + if (!state) { + return; + } + + var menu = button.closest("details"); + if (menu) { + menu.open = false; + } + state.manager = manager; + state.currentButton = button; + state.shouldRestoreFocus = true; + state.title.textContent = "Alter table " + data.tableName; + clearTableAlterDialogError(state); + resetTableAlterDialog(state, data); + if (!state.dialog.open) { + state.dialog.showModal(); + } + var firstName = state.columnList.querySelector(".table-alter-column-name"); + if (firstName) { + firstName.focus(); + } +} + +function initTableAlterActions(manager) { + if (!window.fetch || !window.HTMLDialogElement || !tableAlterData()) { + return; + } + document.addEventListener("click", function (ev) { + var button = ev.target.closest('button[data-table-action="alter-table"]'); + if (!button) { + return; + } + ev.preventDefault(); + openTableAlterDialog(button, manager); + }); +} + function tableForeignKeys() { return tablePageData().foreignKeys || {}; } @@ -2696,6 +4171,7 @@ document.addEventListener("datasette_init", function (evt) { registerBuiltinColumnFieldPlugins(manager); initTableCreateActions(manager); + initTableAlterActions(manager); initRowInsertActions(manager); initRowEditActions(manager); initRowDeleteActions(manager); diff --git a/datasette/views/row.py b/datasette/views/row.py index 4d61eb91..34773d48 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -212,6 +212,7 @@ class RowView(DataView): table, not is_table, None, + None, ), "row_actions": row_actions, "top_row": make_slot_function( diff --git a/datasette/views/table.py b/datasette/views/table.py index 11a28323..5175c7d9 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -50,7 +50,7 @@ from datasette.filters import Filters import sqlite_utils from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT from .base import BaseView, DatasetteError, _error, stream_csv -from .database import QueryView +from .database import QueryView, _custom_column_type_options_for_create_table from .table_extras import ( TABLE_EXTRA_BUNDLES, TableExtraContext, @@ -62,6 +62,13 @@ LINK_WITH_LABEL = ( '{label} {id}' ) LINK_WITH_VALUE = '{id}' +ALTER_TABLE_COLUMN_TYPES = ["text", "integer", "float", "blob"] +ALTER_TABLE_TYPE_FOR_SQLITE_TYPE = { + SQLiteType.TEXT: "text", + SQLiteType.INTEGER: "integer", + SQLiteType.REAL: "float", + SQLiteType.BLOB: "blob", +} class Row: @@ -283,7 +290,14 @@ async def _foreign_key_autocomplete_urls( async def _table_page_data( - datasette, request, db, database_name, table_name, is_view, table_insert_ui + datasette, + request, + db, + database_name, + table_name, + is_view, + table_insert_ui, + table_alter_ui, ): data = { "database": database_name, @@ -292,6 +306,8 @@ async def _table_page_data( } if table_insert_ui: data["insertRow"] = table_insert_ui + if table_alter_ui: + data["alterTable"] = table_alter_ui if not is_view: foreign_keys = await _foreign_key_autocomplete_urls( datasette, request, db, database_name, table_name @@ -354,6 +370,63 @@ async def _table_insert_ui( } +async def _table_alter_ui( + datasette, request, db, database_name, table_name, is_view, pks +): + if is_view or not db.is_mutable: + return None + + if not await datasette.allowed( + action="alter-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return None + + column_types_map = await datasette.get_column_types(database_name, table_name) + columns = [] + for column in await db.table_column_details(table_name): + if column.hidden: + continue + sqlite_type = SQLiteType.from_declared_type(column.type) + column_type = column_types_map.get(column.name) + columns.append( + { + "name": column.name, + "type": ALTER_TABLE_TYPE_FOR_SQLITE_TYPE.get(sqlite_type, "text"), + "sqlite_type": sqlite_type.value, + "notnull": column.notnull, + "default": column.default_value, + "has_default": column.default_value is not None, + "is_pk": column.name in pks, + "column_type": ( + {"type": column_type.name, "config": column_type.config} + if column_type is not None + else None + ), + } + ) + + data = { + "path": "{}/-/alter".format(datasette.urls.table(database_name, table_name)), + "tableName": table_name, + "columns": columns, + "primaryKeys": pks, + "columnTypes": ALTER_TABLE_COLUMN_TYPES, + "defaultExpressions": list(DEFAULT_EXPR_SQL), + } + can_set_column_type = await datasette.allowed( + action="set-column-type", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ) + if can_set_column_type: + data["customColumnTypes"] = _custom_column_type_options_for_create_table( + datasette + ) + return data + + async def display_columns_and_rows( datasette, database_name, @@ -2421,7 +2494,11 @@ async def table_view_data( table_insert_ui = await _table_insert_ui( datasette, request, db, database_name, table_name, is_view, pks ) + table_alter_ui = await _table_alter_ui( + datasette, request, db, database_name, table_name, is_view, pks + ) data["table_insert_ui"] = table_insert_ui + data["table_alter_ui"] = table_alter_ui data["table_page_data"] = await _table_page_data( datasette, request, @@ -2430,6 +2507,7 @@ async def table_view_data( table_name, is_view, table_insert_ui, + table_alter_ui, ) return data, rows[:page_size], columns, expanded_columns, sql, next_url diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 0c7042f0..b7990009 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -130,6 +130,7 @@ def write_playwright_config(config_path): "notes": "textarea", }, "permissions": { + "alter-table": True, "insert-row": True, "update-row": True, "delete-row": True, @@ -328,6 +329,215 @@ def test_create_table_flow(page, datasette_server): } +@pytest.mark.playwright +def test_alter_table_flow(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + assert dialog.locator(".modal-title").inner_text() == "Alter table projects" + assert dialog.locator(".table-alter-save").is_disabled() + type_options = dialog.locator(".table-alter-column-type").first.locator("option") + assert type_options.all_inner_texts() == [ + "text", + "integer", + "floating point number", + "blob - binary data", + ] + first_more_options = dialog.locator(".table-alter-more-options").first + assert first_more_options.inner_text() == "> Advanced options" + first_more_options.click() + assert first_more_options.inner_text() == "v Hide options" + expanded_options_text = dialog.locator(".table-alter-column-details").first.inner_text() + assert dialog.locator(".table-alter-fields").evaluate( + "node => node.scrollWidth <= node.clientWidth + 1" + ) + assert "Not null" in expanded_options_text + assert "This value cannot be left unset" in expanded_options_text + assert "Default value" in expanded_options_text + assert "or default to a specific time" in expanded_options_text + assert "Primary key" in expanded_options_text + assert "An ID that uniquely identifies this record" in expanded_options_text + + dialog.locator(".table-alter-add-column").click() + assert dialog.locator(".table-alter-save").is_enabled() + dialog.locator(".table-alter-column-name").last.fill("status") + dialog.locator(".table-alter-column-type").last.select_option("text") + dialog.locator(".table-alter-default").last.fill("planned") + dialog.locator(".table-alter-save").click() + review = dialog.locator(".table-alter-review") + review.wait_for() + assert not dialog.locator(".table-alter-column-list").is_visible() + review_text = review.inner_text() + assert "Add column status as text, with default value planned." in review_text + assert "Set column order to" not in review_text + assert dialog.locator(".table-alter-back").is_visible() + assert dialog.locator(".table-alter-save").inner_text() == "Apply changes" + dialog.locator(".table-alter-save").click() + + columns = [] + for _ in range(20): + response = httpx.get(f"{datasette_server}data/projects.json?_extra=columns") + response.raise_for_status() + columns = response.json()["columns"] + if "status" in columns: + break + time.sleep(0.1) + assert "status" in columns + + +@pytest.mark.playwright +def test_alter_table_primary_key_columns_stay_at_top(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + rows = dialog.locator(".table-alter-column-row") + assert rows.nth(0).locator(".table-alter-column-name").input_value() == "id" + first_row_move_buttons = rows.nth(0).locator(".table-alter-move-controls button") + for i in range(first_row_move_buttons.count()): + assert first_row_move_buttons.nth(i).is_disabled() + assert ( + first_row_move_buttons.nth(i).get_attribute("title") + == "Primary key columns are always listed first" + ) + + assert rows.nth(1).locator(".table-alter-move-up").is_disabled() + assert rows.nth(1).locator(".table-alter-move-top").get_attribute("title") == ( + "Primary key columns are always listed first" + ) + assert rows.nth(1).locator(".table-alter-move-up").get_attribute("title") == ( + "Primary key columns are always listed first" + ) + last_row = rows.nth(rows.count() - 1) + assert last_row.locator(".table-alter-column-name").input_value() == "score" + last_row.locator(".table-alter-move-top").click() + assert rows.nth(0).locator(".table-alter-column-name").input_value() == "id" + assert rows.nth(1).locator(".table-alter-column-name").input_value() == "score" + + +@pytest.mark.playwright +def test_alter_table_review_rename_primary_key_column(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + save = dialog.locator(".table-alter-save") + assert save.is_disabled() + dialog.locator(".table-alter-column-name").first.fill("id3") + assert save.is_enabled() + save.click() + + review = dialog.locator(".table-alter-review") + review.wait_for() + review_text = review.inner_text() + assert "Rename column id to id3." in review_text + assert "Set primary key to" not in review_text + assert dialog.locator(".table-alter-review-name").all_inner_texts() == [ + "id", + "id3", + ] + + +@pytest.mark.playwright +def test_alter_table_review_not_null_wording(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + dialog.locator(".table-alter-more-options").first.click() + dialog.locator(".table-alter-not-null-input").first.check() + dialog.locator(".table-alter-save").click() + + review = dialog.locator(".table-alter-review") + review.wait_for() + assert "Change column id: not null (require values)." in review.inner_text() + + +@pytest.mark.playwright +def test_alter_table_review_warns_when_dropping_column(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + remove_buttons = dialog.locator(".table-alter-remove-column") + remove_buttons.nth(remove_buttons.count() - 1).click() + dialog.locator(".table-alter-save").click() + + review = dialog.locator(".table-alter-review") + review.wait_for() + assert not dialog.locator(".table-alter-column-list").is_visible() + review_text = review.inner_text() + assert "Warning: data in dropped columns will be permanently lost." in review_text + assert "Drop column score." in review_text + assert "Set column order to" not in review_text + assert dialog.locator(".table-alter-review-damaging").inner_text() == ( + "Drop column score." + ) + + dialog.locator(".table-alter-back").click() + assert dialog.locator(".table-alter-column-list").is_visible() + assert dialog.locator(".table-alter-save").inner_text() == "Review changes" + + +@pytest.mark.playwright +def test_alter_table_cancel_skips_discard_prompt(page, datasette_server): + def open_alter_dialog(): + page.locator("details.actions-menu-links").evaluate("node => node.open = true") + page.locator('button[data-table-action="alter-table"]').click() + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + return dialog + + page.goto(f"{datasette_server}data/projects") + page.evaluate( + """ + () => { + window.__discardConfirmMessages = []; + window.confirm = (message) => { + window.__discardConfirmMessages.push(message); + return false; + }; + } + """ + ) + + dialog = open_alter_dialog() + dialog.locator(".table-alter-add-column").click() + dialog.locator(".table-alter-column-name").last.fill("cancel_me") + dialog.locator(".table-alter-cancel").click() + assert dialog.evaluate("node => node.open") is False + assert page.evaluate("() => window.__discardConfirmMessages") == [] + + dialog = open_alter_dialog() + dialog.locator(".table-alter-add-column").click() + dialog.locator(".table-alter-column-name").last.fill("escape_me") + page.keyboard.press("Escape") + assert page.evaluate("() => window.__discardConfirmMessages") == [ + "Discard table changes?" + ] + assert dialog.evaluate("node => node.open") is True + + page.evaluate("() => window.__discardConfirmMessages = []") + dialog.evaluate( + """node => node.dispatchEvent(new MouseEvent("click", {bubbles: true}))""" + ) + assert page.evaluate("() => window.__discardConfirmMessages") == [ + "Discard table changes?" + ] + assert dialog.evaluate("node => node.open") is True + + @pytest.mark.playwright def test_navigation_search_tracks_and_renders_recent_items(page, datasette_server): page.goto(datasette_server) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 887fdf50..374cc08d 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1078,6 +1078,123 @@ async def test_database_create_table_data_includes_custom_column_types(): ds.close() +@pytest.mark.asyncio +async def test_table_alter_action_button_and_data(): + ds = Datasette( + [], + config={ + "databases": { + "data": { + "tables": { + "items": { + "permissions": { + "alter-table": {"id": "root"}, + "set-column-type": {"id": "root"}, + }, + "column_types": {"name": "textarea"}, + }, + }, + }, + }, + }, + ) + try: + db = ds.add_database( + Database(ds, memory_name="test_table_alter_action"), name="data" + ) + await db.execute_write_script(""" + create table items ( + id integer primary key, + name text not null, + score integer default 5 + ); + """) + response = await ds.client.get("/data/items", actor={"id": "root"}) + assert response.status_code == 200 + soup = Soup(response.text, "html.parser") + + button = soup.select_one( + 'button.action-menu-button[data-table-action="alter-table"]' + ) + assert button is not None + assert button["aria-label"] == "Alter table items" + assert button["role"] == "menuitem" + description = button.find("span", class_="dropdown-description") + assert description.text.strip() == ( + "Change columns and primary key for this table." + ) + description.extract() + assert button.text.strip() == "Alter table" + assert any( + "edit-tools.js" in script.get("src", "") + for script in soup.find_all("script") + ) + + alter_data = table_data_from_soup(soup)["alterTable"] + assert alter_data["path"] == "/data/items/-/alter" + assert alter_data["tableName"] == "items" + assert alter_data["primaryKeys"] == ["id"] + assert alter_data["columnTypes"] == ["text", "integer", "float", "blob"] + assert alter_data["defaultExpressions"] == [ + "current_timestamp", + "current_date", + "current_time", + ] + assert [option["name"] for option in alter_data["customColumnTypes"]] == [ + "email", + "json", + "textarea", + "url", + ] + assert alter_data["columns"] == [ + { + "name": "id", + "type": "integer", + "sqlite_type": "INTEGER", + "notnull": 0, + "default": None, + "has_default": False, + "is_pk": True, + "column_type": None, + }, + { + "name": "name", + "type": "text", + "sqlite_type": "TEXT", + "notnull": 1, + "default": None, + "has_default": False, + "is_pk": False, + "column_type": {"type": "textarea", "config": None}, + }, + { + "name": "score", + "type": "integer", + "sqlite_type": "INTEGER", + "notnull": 0, + "default": "5", + "has_default": True, + "is_pk": False, + "column_type": None, + }, + ] + + response_without_permission = await ds.client.get( + "/data/items", actor={"id": "someone-else"} + ) + assert response_without_permission.status_code == 200 + soup_without_permission = Soup(response_without_permission.text, "html.parser") + assert ( + soup_without_permission.select_one( + 'button[data-table-action="alter-table"]' + ) + is None + ) + assert "alterTable" not in table_data_from_soup(soup_without_permission) + finally: + ds.close() + + @pytest.mark.asyncio async def test_table_insert_action_button_and_data(): ds = Datasette( From 15a3ac58cc10a4004e21e67845e53b2ead26a3b3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 09:24:28 -0700 Subject: [PATCH 018/174] Ran Prettier --- datasette/static/edit-tools.js | 96 +++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 43 deletions(-) diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 284d6bde..cb964365 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -49,8 +49,7 @@ function hideRowMutationStatus() { function databaseCreateTableData() { return ( - window._datasetteDatabaseData && - window._datasetteDatabaseData.createTable + window._datasetteDatabaseData && window._datasetteDatabaseData.createTable ); } @@ -166,10 +165,7 @@ function tableCreateSelectTypeValue(select, type) { } function updateTableCreateCustomColumnTypePlaceholder(select) { - select.classList.toggle( - "table-create-input-placeholder", - !select.value, - ); + select.classList.toggle("table-create-input-placeholder", !select.value); } function createTableCustomColumnTypeSelect() { @@ -273,9 +269,7 @@ function createTableColumnRow(state, column) { } row.remove(); clearTableCreateDialogError(state); - var nextInput = state.columnList.querySelector( - ".table-create-column-name", - ); + var nextInput = state.columnList.querySelector(".table-create-column-name"); if (nextInput) { nextInput.focus(); } else { @@ -418,7 +412,9 @@ function validateTableCreateColumnTypeAssignments(assignments) { if (!option) { return "Unknown custom column type: " + assignment.columnType; } - if (!tableCreateCustomTypeAppliesToSqliteType(option, assignment.sqliteType)) { + if ( + !tableCreateCustomTypeAppliesToSqliteType(option, assignment.sqliteType) + ) { return ( "Custom type " + assignment.columnType + @@ -441,7 +437,8 @@ function fallbackTableUrl(tableName) { function tableCreateSetColumnTypeUrl(responseData, payload) { var tableUrl = - responseData.table_url || fallbackTableUrl(responseData.table || payload.table); + responseData.table_url || + fallbackTableUrl(responseData.table || payload.table); if (!tableUrl) { return null; } @@ -452,7 +449,11 @@ function tableCreateSetColumnTypeUrl(responseData, payload) { return url.toString(); } -async function assignTableCreateColumnTypes(responseData, payload, assignments) { +async function assignTableCreateColumnTypes( + responseData, + payload, + assignments, +) { if (!assignments.length) { return; } @@ -542,7 +543,8 @@ async function saveTableCreateDialog(state) { columnTypeAssignments, ); var tableUrl = - responseData.table_url || fallbackTableUrl(responseData.table || payload.table); + responseData.table_url || + fallbackTableUrl(responseData.table || payload.table); state.shouldRestoreFocus = false; state.dialog.close(); if (tableUrl) { @@ -718,7 +720,11 @@ function openTableCreateDialog(button, manager) { } function initTableCreateActions(manager) { - if (!window.fetch || !window.HTMLDialogElement || !databaseCreateTableData()) { + if ( + !window.fetch || + !window.HTMLDialogElement || + !databaseCreateTableData() + ) { return; } document.addEventListener("click", function (ev) { @@ -952,15 +958,15 @@ function updateTableAlterMoveButtons(state) { var isPrimaryKey = tableAlterRowIsPrimaryKey(row); var previous = row.previousElementSibling; var next = row.nextElementSibling; - row.querySelectorAll(".table-alter-move-controls button").forEach( - function (button) { + row + .querySelectorAll(".table-alter-move-controls button") + .forEach(function (button) { button.title = button.dataset.defaultTitle || button.title; button.disabled = state.isSaving || isPrimaryKey; if (isPrimaryKey) { button.title = primaryKeyMoveTitle; } - }, - ); + }); if (!isPrimaryKey) { var topButton = row.querySelector(".table-alter-move-top"); var upButton = row.querySelector(".table-alter-move-up"); @@ -1028,7 +1034,9 @@ function setTableAlterDialogSaving(state, isSaving) { state.columnList .querySelectorAll(".table-alter-default-expr") .forEach(function (select) { - syncTableAlterDefaultControls(select.closest(".table-alter-column-row")); + syncTableAlterDefaultControls( + select.closest(".table-alter-column-row"), + ); }); } updateTableAlterMoveButtons(state); @@ -1051,10 +1059,7 @@ function tableAlterSelectTypeValue(select, type) { } function updateTableAlterCustomColumnTypePlaceholder(select) { - select.classList.toggle( - "table-alter-input-placeholder", - !select.value, - ); + select.classList.toggle("table-alter-input-placeholder", !select.value); } function createTableAlterCustomColumnTypeSelect() { @@ -1159,10 +1164,7 @@ function createTableAlterColumnRow(state, column) { expandButton.className = "table-alter-more-options"; expandButton.setAttribute("aria-label", "Toggle column settings"); expandButton.setAttribute("aria-controls", details.id); - expandButton.setAttribute( - "aria-expanded", - details.hidden ? "false" : "true", - ); + expandButton.setAttribute("aria-expanded", details.hidden ? "false" : "true"); function updateExpandButton() { var isExpanded = expandButton.getAttribute("aria-expanded") === "true"; expandButton.textContent = isExpanded @@ -1243,8 +1245,7 @@ function createTableAlterColumnRow(state, column) { defaultExprLabel.textContent = "or default to a specific time"; var defaultExprSelect = document.createElement("select"); defaultExprSelect.id = defaultExprId; - defaultExprSelect.className = - "table-alter-input table-alter-default-expr"; + defaultExprSelect.className = "table-alter-input table-alter-default-expr"; defaultExprSelect.setAttribute("aria-label", "or default to a specific time"); tableAlterSelectDefaultExprValue(defaultExprSelect, ""); defaultExprField.appendChild(defaultExprLabel); @@ -1298,7 +1299,8 @@ function createTableAlterColumnRow(state, column) { var moveBottomButton = document.createElement("button"); moveBottomButton.type = "button"; - moveBottomButton.className = "table-alter-icon-button table-alter-move-bottom"; + moveBottomButton.className = + "table-alter-icon-button table-alter-move-bottom"; moveBottomButton.setAttribute("aria-label", "Move column to bottom"); moveBottomButton.title = "Move column to bottom"; moveBottomButton.dataset.defaultTitle = moveBottomButton.title; @@ -1349,15 +1351,15 @@ function createTableAlterColumnRow(state, column) { controls.push(customTypeSelect); } controls.forEach(function (control) { - control.addEventListener("input", function () { - clearTableAlterDialogError(state); - updateTableAlterSaveButtonState(state); - }); - control.addEventListener("change", function () { - clearTableAlterDialogError(state); - updateTableAlterSaveButtonState(state); - }); + control.addEventListener("input", function () { + clearTableAlterDialogError(state); + updateTableAlterSaveButtonState(state); }); + control.addEventListener("change", function () { + clearTableAlterDialogError(state); + updateTableAlterSaveButtonState(state); + }); + }); defaultInput.addEventListener("input", function () { if (defaultInput.value) { @@ -1407,7 +1409,12 @@ function createTableAlterColumnRow(state, column) { moveTopButton.addEventListener("click", function () { var first = tableAlterFirstNonPrimaryRow(state); - if (state.isSaving || tableAlterRowIsPrimaryKey(row) || !first || first === row) { + if ( + state.isSaving || + tableAlterRowIsPrimaryKey(row) || + !first || + first === row + ) { return; } state.columnList.insertBefore(row, first); @@ -1448,7 +1455,12 @@ function createTableAlterColumnRow(state, column) { moveBottomButton.addEventListener("click", function () { var last = state.columnList.lastElementChild; - if (state.isSaving || tableAlterRowIsPrimaryKey(row) || !last || last === row) { + if ( + state.isSaving || + tableAlterRowIsPrimaryKey(row) || + !last || + last === row + ) { return; } state.columnList.appendChild(row); @@ -1797,9 +1809,7 @@ function tableAlterOperationSummary(operation) { } if (Object.prototype.hasOwnProperty.call(args, "not_null")) { changes.push( - args.not_null - ? "not null (require values)" - : "allow unset values", + args.not_null ? "not null (require values)" : "allow unset values", ); } if (args.default_expr) { From c9c79fdfc8395fe49d0e9d262548f2b337bec3bc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 09:58:39 -0700 Subject: [PATCH 019/174] Isolate Unix domain socket test server paths - Use a per-process socket path for the UDS test fixture. - Clean up stale socket files before and after the fixture runs. - Close the HTTP client and wait for the Datasette subprocess to exit. --- tests/conftest.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8860d54c..7ec03146 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -260,8 +260,12 @@ def ds_unix_domain_socket_server(tmp_path_factory): # This used to use tmp_path_factory.mktemp("uds") but that turned out to # produce paths that were too long to use as UDS on macOS, see # https://github.com/simonw/datasette/issues/1407 - so I switched to - # using tempfile.gettempdir() - uds = str(pathlib.Path(tempfile.gettempdir()) / "datasette.sock") + # using tempfile.gettempdir() with a per-process filename. + uds = str(pathlib.Path(tempfile.gettempdir()) / f"datasette-{os.getpid()}.sock") + try: + os.unlink(uds) + except FileNotFoundError: + pass ds_proc = subprocess.Popen( [sys.executable, "-m", "datasette", "--memory", "--uds", uds], stdout=subprocess.PIPE, @@ -271,12 +275,26 @@ def ds_unix_domain_socket_server(tmp_path_factory): # Poll until available transport = httpx.HTTPTransport(uds=uds) client = httpx.Client(transport=transport) - wait_until_responds("http://localhost/_memory.json", client=client) - # Check it started successfully - assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") - yield ds_proc, uds - # Shut it down at the end of the pytest session - ds_proc.terminate() + try: + wait_until_responds( + "http://localhost/_memory.json", timeout=30.0, client=client + ) + # Check it started successfully + assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") + yield ds_proc, uds + finally: + client.close() + # Shut it down at the end of the pytest session + ds_proc.terminate() + try: + ds_proc.wait(timeout=5) + except subprocess.TimeoutExpired: + ds_proc.kill() + ds_proc.wait() + try: + os.unlink(uds) + except FileNotFoundError: + pass # Import fixtures from fixtures.py to make them available From 4115213e17d23cdf3286bc4583e7005991892568 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 10:22:42 -0700 Subject: [PATCH 020/174] Precompute action permissions for table pages - Extract reusable helpers for database and table action permission preloading. - Precompute those permissions before building table-page HTML data. - Document the default table actions plugin. --- datasette/views/table.py | 22 ++++++++++++--- datasette/views/table_extras.py | 49 ++++++++++++++++++++++----------- docs/plugins.rst | 9 ++++++ tests/test_playwright.py | 10 +++---- 4 files changed, 65 insertions(+), 25 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 5175c7d9..82c7e03d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -54,6 +54,8 @@ from .database import QueryView, _custom_column_type_options_for_create_table from .table_extras import ( TABLE_EXTRA_BUNDLES, TableExtraContext, + precompute_database_action_permissions, + precompute_table_action_permissions, resolve_table_extras, table_extra_registry, ) @@ -1247,7 +1249,10 @@ class TableAlterView(BaseView): defaults[args.name] = _literal_default( db_for_write, args.default ) - if "default_expr" in args.model_fields_set and not args.not_null: + if ( + "default_expr" in args.model_fields_set + and not args.not_null + ): defaults[args.name] = _default_expression_sql( args.default_expr ) @@ -1363,9 +1368,9 @@ class TableAlterView(BaseView): "altered": altered, "schema": after_schema, "before_schema": before_schema, - "operations_applied": 0 - if alter_request.dry_run - else len(alter_request.operations), + "operations_applied": ( + 0 if alter_request.dry_run else len(alter_request.operations) + ), "dry_run": alter_request.dry_run, }, status=200, @@ -2068,6 +2073,15 @@ async def table_view_data( if redirect_response: return redirect_response + if context_for_html_hack: + await precompute_database_action_permissions( + datasette, request.actor, database_name + ) + if not is_view: + await precompute_table_action_permissions( + datasette, request.actor, database_name, table_name + ) + # Introspect columns and primary keys for table pks = await db.primary_keys(table_name) table_columns = await db.table_columns(table_name) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index a0308e49..7cb4d8f0 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -367,23 +367,14 @@ class ActionsExtra(Extra): # that allowed() calls made inside the plugin hooks below # are served from the cache datasette = context.datasette - await datasette.allowed_many( - actions=[ - name - for name, action in datasette.actions.items() - if action.resource_class is TableResource - ], - resource=TableResource(context.database_name, context.table_name), - actor=context.request.actor, + await precompute_table_action_permissions( + datasette, + context.request.actor, + context.database_name, + context.table_name, ) - await datasette.allowed_many( - actions=[ - name - for name, action in datasette.actions.items() - if action.resource_class is DatabaseResource - ], - resource=DatabaseResource(context.database_name), - actor=context.request.actor, + await precompute_database_action_permissions( + datasette, context.request.actor, context.database_name ) for hook in method(**kwargs): extra_links = await await_me_maybe(hook) @@ -394,6 +385,32 @@ class ActionsExtra(Extra): return actions +async def precompute_table_action_permissions( + datasette, actor, database_name, table_name +): + await datasette.allowed_many( + actions=[ + name + for name, action in datasette.actions.items() + if action.resource_class is TableResource + ], + resource=TableResource(database_name, table_name), + actor=actor, + ) + + +async def precompute_database_action_permissions(datasette, actor, database_name): + await datasette.allowed_many( + actions=[ + name + for name, action in datasette.actions.items() + if action.resource_class is DatabaseResource + ], + resource=DatabaseResource(database_name), + actor=actor, + ) + + class IsViewExtra(Extra): description = "Whether this resource is a view instead of a table" example = ExtraExample("/fixtures/simple_view.json?_extra=is_view") diff --git a/docs/plugins.rst b/docs/plugins.rst index c2eb282a..d2b5c20a 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -280,6 +280,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "query_actions" ] }, + { + "name": "datasette.default_table_actions", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "table_actions" + ] + }, { "name": "datasette.events", "static": false, diff --git a/tests/test_playwright.py b/tests/test_playwright.py index b7990009..70b5a8c1 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -350,7 +350,9 @@ def test_alter_table_flow(page, datasette_server): assert first_more_options.inner_text() == "> Advanced options" first_more_options.click() assert first_more_options.inner_text() == "v Hide options" - expanded_options_text = dialog.locator(".table-alter-column-details").first.inner_text() + expanded_options_text = dialog.locator( + ".table-alter-column-details" + ).first.inner_text() assert dialog.locator(".table-alter-fields").evaluate( "node => node.scrollWidth <= node.clientWidth + 1" ) @@ -500,8 +502,7 @@ def test_alter_table_cancel_skips_discard_prompt(page, datasette_server): return dialog page.goto(f"{datasette_server}data/projects") - page.evaluate( - """ + page.evaluate(""" () => { window.__discardConfirmMessages = []; window.confirm = (message) => { @@ -509,8 +510,7 @@ def test_alter_table_cancel_skips_discard_prompt(page, datasette_server): return false; }; } - """ - ) + """) dialog = open_alter_dialog() dialog.locator(".table-alter-add-column").click() From 8cec528eebd253994fbb41c85d571ab649d723e9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 10:32:21 -0700 Subject: [PATCH 021/174] Test against pyodide/v314.0.0 Now that we depend on pydantic we need a more recent pyodide in order to load the emscripten build of pydantic-core. Refs https://github.com/simonw/datasette/pull/2789#issuecomment-4733412763 --- test-in-pyodide-with-shot-scraper.sh | 29 +++++++++++++--------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh index 4d1c4968..8d7fa08e 100755 --- a/test-in-pyodide-with-shot-scraper.sh +++ b/test-in-pyodide-with-shot-scraper.sh @@ -1,40 +1,37 @@ #!/bin/bash -set -e -# So the script fails if there are any errors +set -euo pipefail + +read -r -a PYTHON_CMD <<< "${PYTHON:-python3}" +read -r -a SHOT_SCRAPER_CMD <<< "${SHOT_SCRAPER:-shot-scraper}" # Build the wheel -python3 -m build +"${PYTHON_CMD[@]}" -m build -# Find name of wheel, strip off the dist/ -wheel=$(basename $(ls dist/*.whl) | head -n 1) +# Find name of most recently built wheel, strip off the dist/ +wheel=$(basename "$(ls -t dist/*.whl | head -n 1)") # Create a blank index page echo ' - + ' > dist/index.html # Run a server for that dist/ folder -cd dist -python3 -m http.server 8529 & -cd .. +"${PYTHON_CMD[@]}" -m http.server 8529 --directory dist & +server_pid=$! # Register the kill_server function to be called on script exit kill_server() { - pkill -f 'http.server 8529' + kill "$server_pid" 2>/dev/null || true } trap kill_server EXIT -shot-scraper javascript http://localhost:8529/ " +"${SHOT_SCRAPER_CMD[@]}" javascript http://localhost:8529/ " async () => { let pyodide = await loadPyodide(); - await pyodide.loadPackage(['micropip', 'ssl', 'setuptools']); + await pyodide.loadPackage(['micropip', 'setuptools']); let output = await pyodide.runPythonAsync(\` import micropip - await micropip.install('h11==0.12.0') - await micropip.install('httpx==0.23') - # To avoid 'from typing_extensions import deprecated' error: - await micropip.install('typing-extensions>=4.12.2') await micropip.install('http://localhost:8529/$wheel') import ssl import setuptools From 1972ba8952f0e10fc13054c4214df2473c448daf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 10:54:25 -0700 Subject: [PATCH 022/174] Split table create and alter views - Move create-table and alter-table API views into table_create_alter.py. - Keep create and alter schema-editing constants and helpers together. - Rename the create table modal context helper. --- datasette/app.py | 3 +- datasette/views/database.py | 323 +----------- datasette/views/table.py | 371 +------------- datasette/views/table_create_alter.py | 687 ++++++++++++++++++++++++++ 4 files changed, 699 insertions(+), 685 deletions(-) create mode 100644 datasette/views/table_create_alter.py diff --git a/datasette/app.py b/datasette/app.py index 6ea3d5a4..13e25d4e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -47,9 +47,9 @@ from .views import Context from .views.database import ( database_download, DatabaseView, - TableCreateView, QueryView, ) +from .views.table_create_alter import TableAlterView, TableCreateView from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView from .views.stored_queries import ( QueryCreateAnalyzeView, @@ -84,7 +84,6 @@ from .views.special import ( ) from .views.table import ( TableAutocompleteView, - TableAlterView, TableInsertView, TableUpsertView, TableSetColumnTypeView, diff --git a/datasette/views/database.py b/datasette/views/database.py index db70b135..8e9c1361 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -6,15 +6,11 @@ import itertools import json import markupsafe import os -import re -import sqlite_utils import textwrap -from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted -from datasette.column_types import SQLiteType -from datasette.resources import DatabaseResource, QueryResource, TableResource +from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import stored_query_to_dict from datasette.write_sql import QueryWriteRejected from datasette.utils import ( @@ -38,27 +34,16 @@ from datasette.utils import ( from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden from datasette.plugins import pm -from .base import BaseView, DatasetteError, View, _error, stream_csv +from .base import DatasetteError, View, stream_csv from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns from .table_extras import ( QueryExtraContext, resolve_query_extras, table_extra_registry, ) +from .table_create_alter import _create_table_ui_context from . import Context -CREATE_TABLE_COLUMN_TYPES = ["text", "integer", "float", "blob"] -CREATE_TABLE_SQLITE_TYPES = { - "text": SQLiteType.TEXT, - "integer": SQLiteType.INTEGER, - "float": SQLiteType.REAL, - "blob": SQLiteType.BLOB, -} -CREATE_TABLE_TYPE_FOR_SQLITE_TYPE = { - sqlite_type: column_type - for column_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() -} - class DatabaseView(View): async def get(self, request, datasette): @@ -142,7 +127,7 @@ class DatabaseView(View): resource=DatabaseResource(database), actor=request.actor, ) - create_table_ui = await _database_create_table_ui( + create_table_ui = await _create_table_ui_context( datasette, request, db, database, database_action_permissions ) @@ -326,57 +311,6 @@ class DatabaseContext(Context): ) -async def _database_create_table_ui( - datasette, request, db, database_name, database_action_permissions -): - if not db.is_mutable: - return None - if not database_action_permissions.get("create-table"): - return None - data = { - "path": "{}/-/create".format(datasette.urls.database(database_name)), - "databaseName": database_name, - "columnTypes": CREATE_TABLE_COLUMN_TYPES, - } - can_set_column_type = await datasette.allowed( - action="set-column-type", - resource=TableResource(database=database_name, table="__new_table__"), - actor=request.actor, - ) - if can_set_column_type: - data["customColumnTypes"] = _custom_column_type_options_for_create_table( - datasette - ) - return data - - -def _custom_column_type_options_for_create_table(datasette): - options = [] - for name, ct_cls in sorted(datasette._column_types.items()): - sqlite_types = getattr(ct_cls, "sqlite_types", None) - if sqlite_types is None: - option_sqlite_types = CREATE_TABLE_COLUMN_TYPES[:] - else: - option_sqlite_types = [ - create_table_type - for create_table_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() - if sqlite_type in sqlite_types - ] - if not option_sqlite_types: - continue - option = { - "name": name, - "description": ct_cls.description, - "sqliteTypes": option_sqlite_types, - } - if sqlite_types is not None and len(sqlite_types) == 1: - fixed_sqlite_type = CREATE_TABLE_TYPE_FOR_SQLITE_TYPE.get(sqlite_types[0]) - if fixed_sqlite_type is not None: - option["fixedSqliteType"] = fixed_sqlite_type - options.append(option) - return options - - @dataclass class QueryContext(Context): database: str = field(metadata={"help": "The name of the database being queried"}) @@ -1140,255 +1074,6 @@ class MagicParameters(dict): return super().__getitem__(key) -class TableCreateView(BaseView): - name = "table-create" - - _valid_keys = { - "table", - "rows", - "row", - "columns", - "pk", - "pks", - "ignore", - "replace", - "alter", - } - _supported_column_types = set(CREATE_TABLE_COLUMN_TYPES) - # Any string that does not contain a newline or start with sqlite_ - _table_name_re = re.compile(r"^(?!sqlite_)[^\n]+$") - - def __init__(self, datasette): - self.ds = datasette - - async def post(self, request): - db = await self.ds.resolve_database(request) - database_name = db.name - - # Must have create-table permission - if not await self.ds.allowed( - action="create-table", - resource=DatabaseResource(database=database_name), - actor=request.actor, - ): - return _error(["Permission denied"], 403) - - try: - data = await request.json() - except json.JSONDecodeError as e: - return _error(["Invalid JSON: {}".format(e)]) - - if not isinstance(data, dict): - return _error(["JSON must be an object"]) - - invalid_keys = set(data.keys()) - self._valid_keys - if invalid_keys: - return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) - - # ignore and replace are mutually exclusive - if data.get("ignore") and data.get("replace"): - return _error(["ignore and replace are mutually exclusive"]) - - # ignore and replace only allowed with row or rows - if "ignore" in data or "replace" in data: - if not data.get("row") and not data.get("rows"): - return _error(["ignore and replace require row or rows"]) - - # ignore and replace require pk or pks - if "ignore" in data or "replace" in data: - if not data.get("pk") and not data.get("pks"): - return _error(["ignore and replace require pk or pks"]) - - ignore = data.get("ignore") - replace = data.get("replace") - - if replace: - # Must have update-row permission - if not await self.ds.allowed( - action="update-row", - resource=DatabaseResource(database=database_name), - actor=request.actor, - ): - return _error(["Permission denied: need update-row"], 403) - - table_name = data.get("table") - if not table_name: - return _error(["Table is required"]) - - if not self._table_name_re.match(table_name): - return _error(["Invalid table name"]) - - table_exists = await db.table_exists(data["table"]) - columns = data.get("columns") - rows = data.get("rows") - row = data.get("row") - if not columns and not rows and not row: - return _error(["columns, rows or row is required"]) - - if rows and row: - return _error(["Cannot specify both rows and row"]) - - if rows or row: - # Must have insert-row permission - if not await self.ds.allowed( - action="insert-row", - resource=DatabaseResource(database=database_name), - actor=request.actor, - ): - return _error(["Permission denied: need insert-row"], 403) - - alter = False - if rows or row: - if not table_exists: - # if table is being created for the first time, alter=True - alter = True - else: - # alter=True only if they request it AND they have permission - if data.get("alter"): - if not await self.ds.allowed( - action="alter-table", - resource=DatabaseResource(database=database_name), - actor=request.actor, - ): - return _error(["Permission denied: need alter-table"], 403) - alter = True - - if columns: - if rows or row: - return _error(["Cannot specify columns with rows or row"]) - if not isinstance(columns, list): - return _error(["columns must be a list"]) - for column in columns: - if not isinstance(column, dict): - return _error(["columns must be a list of objects"]) - if not column.get("name") or not isinstance(column.get("name"), str): - return _error(["Column name is required"]) - if not column.get("type"): - column["type"] = "text" - if column["type"] not in self._supported_column_types: - return _error( - ["Unsupported column type: {}".format(column["type"])] - ) - # No duplicate column names - dupes = {c["name"] for c in columns if columns.count(c) > 1} - if dupes: - return _error(["Duplicate column name: {}".format(", ".join(dupes))]) - - if row: - rows = [row] - - if rows: - if not isinstance(rows, list): - return _error(["rows must be a list"]) - for row in rows: - if not isinstance(row, dict): - return _error(["rows must be a list of objects"]) - - pk = data.get("pk") - pks = data.get("pks") - - if pk and pks: - return _error(["Cannot specify both pk and pks"]) - if pk: - if not isinstance(pk, str): - return _error(["pk must be a string"]) - if pks: - if not isinstance(pks, list): - return _error(["pks must be a list"]) - for pk in pks: - if not isinstance(pk, str): - return _error(["pks must be a list of strings"]) - - # If table exists already, read pks from that instead - if table_exists: - actual_pks = await db.primary_keys(table_name) - # if pk passed and table already exists check it does not change - bad_pks = False - if len(actual_pks) == 1 and data.get("pk") and data["pk"] != actual_pks[0]: - bad_pks = True - elif ( - len(actual_pks) > 1 - and data.get("pks") - and set(data["pks"]) != set(actual_pks) - ): - bad_pks = True - if bad_pks: - return _error(["pk cannot be changed for existing table"]) - pks = actual_pks - - initial_schema = None - if table_exists: - initial_schema = await db.execute_fn( - lambda conn: sqlite_utils.Database(conn)[table_name].schema - ) - - def create_table(conn): - table = sqlite_utils.Database(conn)[table_name] - if rows: - table.insert_all( - rows, pk=pks or pk, ignore=ignore, replace=replace, alter=alter - ) - else: - table.create( - {c["name"]: c["type"] for c in columns}, - pk=pks or pk, - ) - return table.schema - - try: - schema = await db.execute_write_fn(create_table, request=request) - except Exception as e: - return _error([str(e)]) - - if initial_schema is not None and initial_schema != schema: - await self.ds.track_event( - AlterTableEvent( - request.actor, - database=database_name, - table=table_name, - before_schema=initial_schema, - after_schema=schema, - ) - ) - - table_url = self.ds.absolute_url( - request, self.ds.urls.table(db.name, table_name) - ) - table_api_url = self.ds.absolute_url( - request, self.ds.urls.table(db.name, table_name, format="json") - ) - details = { - "ok": True, - "database": db.name, - "table": table_name, - "table_url": table_url, - "table_api_url": table_api_url, - "schema": schema, - } - if rows: - details["row_count"] = len(rows) - - if not table_exists: - # Only log creation if we created a table - await self.ds.track_event( - CreateTableEvent( - request.actor, database=db.name, table=table_name, schema=schema - ) - ) - if rows: - await self.ds.track_event( - InsertRowsEvent( - request.actor, - database=db.name, - table=table_name, - num_rows=len(rows), - ignore=ignore, - replace=replace, - ) - ) - return Response.json(details, status=201) - - async def display_rows(datasette, database, request, rows, columns): display_rows = [] truncate_cells = datasette.setting("truncate_cells_html") diff --git a/datasette/views/table.py b/datasette/views/table.py index 82c7e03d..6c0d2914 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,12 +1,10 @@ import asyncio import itertools import json -from typing import Annotated, Any, Literal, Union import urllib import urllib.parse import markupsafe -from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator from datasette.column_types import SQLiteType from datasette.extras import extra_names_from_request @@ -48,9 +46,14 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response from datasette.filters import Filters import sqlite_utils -from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT from .base import BaseView, DatasetteError, _error, stream_csv -from .database import QueryView, _custom_column_type_options_for_create_table +from .database import QueryView +from .table_create_alter import ( + ALTER_TABLE_COLUMN_TYPES, + ALTER_TABLE_TYPE_FOR_SQLITE_TYPE, + DEFAULT_EXPR_SQL, + _custom_column_type_options_for_create_table, +) from .table_extras import ( TABLE_EXTRA_BUNDLES, TableExtraContext, @@ -64,13 +67,6 @@ LINK_WITH_LABEL = ( '{label} {id}' ) LINK_WITH_VALUE = '{id}' -ALTER_TABLE_COLUMN_TYPES = ["text", "integer", "float", "blob"] -ALTER_TABLE_TYPE_FOR_SQLITE_TYPE = { - SQLiteType.TEXT: "text", - SQLiteType.INTEGER: "integer", - SQLiteType.REAL: "float", - SQLiteType.BLOB: "blob", -} class Row: @@ -727,154 +723,6 @@ async def display_columns_and_rows( return columns, cell_rows -SqliteApiType = Literal["text", "integer", "float", "blob"] -DefaultExpr = Literal["current_timestamp", "current_date", "current_time"] -DEFAULT_EXPR_SQL = { - "current_timestamp": "CURRENT_TIMESTAMP", - "current_date": "CURRENT_DATE", - "current_time": "CURRENT_TIME", -} - - -class _StrictPydanticModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - -class _DefaultArgsMixin(_StrictPydanticModel): - default: Any | None = None - default_expr: DefaultExpr | None = None - - @model_validator(mode="after") - def validate_default_fields(self): - has_default = "default" in self.model_fields_set - has_default_expr = "default_expr" in self.model_fields_set - if has_default and has_default_expr: - raise ValueError("default and default_expr cannot both be provided") - if has_default_expr and self.default_expr is None: - raise ValueError("default_expr cannot be null") - return self - - -class AddColumnArgs(_DefaultArgsMixin): - name: str - type: SqliteApiType = "text" - not_null: bool = False - - -class RenameColumnArgs(_StrictPydanticModel): - name: str - to: str - - -class AlterColumnArgs(_DefaultArgsMixin): - name: str - type: SqliteApiType | None = None - not_null: bool | None = None - - @model_validator(mode="after") - def require_change(self): - if not ( - {"type", "not_null", "default", "default_expr"} & self.model_fields_set - ): - raise ValueError( - "At least one of type, not_null, default or default_expr must be provided" - ) - return self - - -class DropColumnArgs(_StrictPydanticModel): - name: str - - -class SetPrimaryKeyArgs(_StrictPydanticModel): - columns: list[str] = Field(min_length=1) - - -class ReorderColumnsArgs(_StrictPydanticModel): - columns: list[str] = Field(min_length=1) - - -class AddColumnOperation(_StrictPydanticModel): - op: Literal["add_column"] - args: AddColumnArgs - - -class RenameColumnOperation(_StrictPydanticModel): - op: Literal["rename_column"] - args: RenameColumnArgs - - -class AlterColumnOperation(_StrictPydanticModel): - op: Literal["alter_column"] - args: AlterColumnArgs - - -class DropColumnOperation(_StrictPydanticModel): - op: Literal["drop_column"] - args: DropColumnArgs - - -class SetPrimaryKeyOperation(_StrictPydanticModel): - op: Literal["set_primary_key"] - args: SetPrimaryKeyArgs - - -class ReorderColumnsOperation(_StrictPydanticModel): - op: Literal["reorder_columns"] - args: ReorderColumnsArgs - - -AlterTableOperation = Annotated[ - Union[ - AddColumnOperation, - RenameColumnOperation, - AlterColumnOperation, - DropColumnOperation, - SetPrimaryKeyOperation, - ReorderColumnsOperation, - ], - Field(discriminator="op"), -] - - -class AlterTableRequest(_StrictPydanticModel): - operations: list[AlterTableOperation] = Field(min_length=1) - dry_run: bool = False - - -def _pydantic_errors(validation_error): - errors = [] - for error in validation_error.errors(): - location = ".".join(str(item) for item in error["loc"]) - message = error["msg"] - errors.append("{}: {}".format(location, message) if location else message) - return errors - - -def _table_schema_from_conn(conn, table_name): - row = conn.execute( - "select sql from sqlite_master where type = 'table' and name = ?", - [table_name], - ).fetchone() - return row[0] if row else None - - -def _primary_key_value(columns): - if len(columns) == 1: - return columns[0] - return tuple(columns) - - -def _default_expression_sql(default_expr): - return DEFAULT_EXPR_SQL[default_expr] - - -def _literal_default(db, value): - if isinstance(value, str): - return db.quote(value) - return value - - class TableInsertView(BaseView): name = "table-insert" @@ -1172,211 +1020,6 @@ class TableUpsertView(TableInsertView): return await super().post(request, upsert=True) -class TableAlterView(BaseView): - name = "table-alter" - - def __init__(self, datasette): - self.ds = datasette - - async def post(self, request): - try: - resolved = await self.ds.resolve_table(request) - except NotFound as e: - return _error([e.args[0]], 404) - - db = resolved.db - database_name = db.name - table_name = resolved.table - - if not await self.ds.allowed( - action="alter-table", - resource=TableResource(database=database_name, table=table_name), - actor=request.actor, - ): - return _error(["Permission denied: need alter-table"], 403) - - if not db.is_mutable: - return _error(["Database is immutable"], 403) - - content_type = request.headers.get("content-type") or "" - if not content_type.startswith("application/json"): - return _error(["Invalid content-type, must be application/json"], 400) - - try: - data = await request.json() - except json.JSONDecodeError as e: - return _error(["Invalid JSON: {}".format(e)], 400) - - if not isinstance(data, dict): - return _error(["JSON must be a dictionary"], 400) - - try: - alter_request = AlterTableRequest.model_validate(data) - except ValidationError as e: - return _error(_pydantic_errors(e), 400) - - def alter_table(conn): - before_schema = _table_schema_from_conn(conn, table_name) - - def apply_operations(operation_conn): - db_for_write = sqlite_utils.Database(operation_conn) - table = db_for_write[table_name] - - add_columns = [] - types = {} - rename = {} - drop = set() - not_null = {} - defaults = {} - column_order = None - pk = SQLITE_UTILS_DEFAULT - - for operation in alter_request.operations: - args = operation.args - if operation.op == "add_column": - if args.not_null and not ( - ( - "default" in args.model_fields_set - and args.default is not None - ) - or "default_expr" in args.model_fields_set - ): - raise ValueError( - "add_column args.default or args.default_expr is required when not_null is true" - ) - add_columns.append(args) - if "default" in args.model_fields_set and not args.not_null: - defaults[args.name] = _literal_default( - db_for_write, args.default - ) - if ( - "default_expr" in args.model_fields_set - and not args.not_null - ): - defaults[args.name] = _default_expression_sql( - args.default_expr - ) - elif operation.op == "rename_column": - rename[args.name] = args.to - elif operation.op == "alter_column": - if args.type is not None: - types[args.name] = args.type - if args.not_null is not None: - not_null[args.name] = args.not_null - if "default" in args.model_fields_set: - defaults[args.name] = ( - None - if args.default is None - else _literal_default(db_for_write, args.default) - ) - if "default_expr" in args.model_fields_set: - defaults[args.name] = _default_expression_sql( - args.default_expr - ) - elif operation.op == "drop_column": - drop.add(args.name) - elif operation.op == "set_primary_key": - pk = _primary_key_value(args.columns) - elif operation.op == "reorder_columns": - column_order = args.columns - - with operation_conn: - for column in add_columns: - not_null_default = None - if column.not_null: - if "default_expr" in column.model_fields_set: - not_null_default = _default_expression_sql( - column.default_expr - ) - else: - not_null_default = _literal_default( - db_for_write, column.default - ) - table.add_column( - column.name, - column.type, - not_null_default=not_null_default, - ) - - should_transform = any( - ( - types, - rename, - drop, - not_null, - defaults, - column_order is not None, - pk is not SQLITE_UTILS_DEFAULT, - ) - ) - if should_transform: - table.transform( - types=types or None, - rename=rename or None, - drop=drop or None, - pk=pk, - not_null=not_null or None, - defaults=defaults or None, - column_order=column_order, - ) - - return _table_schema_from_conn(operation_conn, table_name) - - if alter_request.dry_run: - memory_conn = sqlite3.connect(":memory:") - try: - conn.backup(memory_conn) - return before_schema, apply_operations(memory_conn) - finally: - memory_conn.close() - - after_schema = apply_operations(conn) - return before_schema, after_schema - - try: - before_schema, after_schema = await db.execute_write_fn( - alter_table, request=request - ) - except Exception as e: - return _error([str(e)], 400) - - altered = before_schema != after_schema - if altered and not alter_request.dry_run: - await self.ds.track_event( - AlterTableEvent( - request.actor, - database=database_name, - table=table_name, - before_schema=before_schema, - after_schema=after_schema, - ) - ) - - table_url = self.ds.absolute_url( - request, self.ds.urls.table(database_name, table_name) - ) - table_api_url = self.ds.absolute_url( - request, self.ds.urls.table(database_name, table_name, format="json") - ) - return Response.json( - { - "ok": True, - "database": database_name, - "table": table_name, - "table_url": table_url, - "table_api_url": table_api_url, - "altered": altered, - "schema": after_schema, - "before_schema": before_schema, - "operations_applied": ( - 0 if alter_request.dry_run else len(alter_request.operations) - ), - "dry_run": alter_request.dry_run, - }, - status=200, - ) - - class TableSetColumnTypeView(BaseView): name = "table-set-column-type" diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py new file mode 100644 index 00000000..7decfad2 --- /dev/null +++ b/datasette/views/table_create_alter.py @@ -0,0 +1,687 @@ +import json +import re +from typing import Annotated, Any, Literal, Union + +from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator +import sqlite_utils +from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT + +from datasette.column_types import SQLiteType +from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent +from datasette.resources import DatabaseResource, TableResource +from datasette.utils import sqlite3 +from datasette.utils.asgi import NotFound, Response + +from .base import BaseView, _error + +CREATE_TABLE_COLUMN_TYPES = ["text", "integer", "float", "blob"] +CREATE_TABLE_SQLITE_TYPES = { + "text": SQLiteType.TEXT, + "integer": SQLiteType.INTEGER, + "float": SQLiteType.REAL, + "blob": SQLiteType.BLOB, +} +CREATE_TABLE_TYPE_FOR_SQLITE_TYPE = { + sqlite_type: column_type + for column_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() +} +ALTER_TABLE_COLUMN_TYPES = CREATE_TABLE_COLUMN_TYPES +ALTER_TABLE_TYPE_FOR_SQLITE_TYPE = { + SQLiteType.TEXT: "text", + SQLiteType.INTEGER: "integer", + SQLiteType.REAL: "float", + SQLiteType.BLOB: "blob", +} + + +async def _create_table_ui_context( + datasette, request, db, database_name, database_action_permissions +): + if not db.is_mutable: + return None + if not database_action_permissions.get("create-table"): + return None + data = { + "path": "{}/-/create".format(datasette.urls.database(database_name)), + "databaseName": database_name, + "columnTypes": CREATE_TABLE_COLUMN_TYPES, + } + can_set_column_type = await datasette.allowed( + action="set-column-type", + resource=TableResource(database=database_name, table="__new_table__"), + actor=request.actor, + ) + if can_set_column_type: + data["customColumnTypes"] = _custom_column_type_options_for_create_table( + datasette + ) + return data + + +def _custom_column_type_options_for_create_table(datasette): + options = [] + for name, ct_cls in sorted(datasette._column_types.items()): + sqlite_types = getattr(ct_cls, "sqlite_types", None) + if sqlite_types is None: + option_sqlite_types = CREATE_TABLE_COLUMN_TYPES[:] + else: + option_sqlite_types = [ + create_table_type + for create_table_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() + if sqlite_type in sqlite_types + ] + if not option_sqlite_types: + continue + option = { + "name": name, + "description": ct_cls.description, + "sqliteTypes": option_sqlite_types, + } + if sqlite_types is not None and len(sqlite_types) == 1: + fixed_sqlite_type = CREATE_TABLE_TYPE_FOR_SQLITE_TYPE.get(sqlite_types[0]) + if fixed_sqlite_type is not None: + option["fixedSqliteType"] = fixed_sqlite_type + options.append(option) + return options + + +SqliteApiType = Literal["text", "integer", "float", "blob"] +DefaultExpr = Literal["current_timestamp", "current_date", "current_time"] +DEFAULT_EXPR_SQL = { + "current_timestamp": "CURRENT_TIMESTAMP", + "current_date": "CURRENT_DATE", + "current_time": "CURRENT_TIME", +} + + +class _StrictPydanticModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class _DefaultArgsMixin(_StrictPydanticModel): + default: Any | None = None + default_expr: DefaultExpr | None = None + + @model_validator(mode="after") + def validate_default_fields(self): + has_default = "default" in self.model_fields_set + has_default_expr = "default_expr" in self.model_fields_set + if has_default and has_default_expr: + raise ValueError("default and default_expr cannot both be provided") + if has_default_expr and self.default_expr is None: + raise ValueError("default_expr cannot be null") + return self + + +class AddColumnArgs(_DefaultArgsMixin): + name: str + type: SqliteApiType = "text" + not_null: bool = False + + +class RenameColumnArgs(_StrictPydanticModel): + name: str + to: str + + +class AlterColumnArgs(_DefaultArgsMixin): + name: str + type: SqliteApiType | None = None + not_null: bool | None = None + + @model_validator(mode="after") + def require_change(self): + if not ( + {"type", "not_null", "default", "default_expr"} & self.model_fields_set + ): + raise ValueError( + "At least one of type, not_null, default or default_expr must be provided" + ) + return self + + +class DropColumnArgs(_StrictPydanticModel): + name: str + + +class SetPrimaryKeyArgs(_StrictPydanticModel): + columns: list[str] = Field(min_length=1) + + +class ReorderColumnsArgs(_StrictPydanticModel): + columns: list[str] = Field(min_length=1) + + +class AddColumnOperation(_StrictPydanticModel): + op: Literal["add_column"] + args: AddColumnArgs + + +class RenameColumnOperation(_StrictPydanticModel): + op: Literal["rename_column"] + args: RenameColumnArgs + + +class AlterColumnOperation(_StrictPydanticModel): + op: Literal["alter_column"] + args: AlterColumnArgs + + +class DropColumnOperation(_StrictPydanticModel): + op: Literal["drop_column"] + args: DropColumnArgs + + +class SetPrimaryKeyOperation(_StrictPydanticModel): + op: Literal["set_primary_key"] + args: SetPrimaryKeyArgs + + +class ReorderColumnsOperation(_StrictPydanticModel): + op: Literal["reorder_columns"] + args: ReorderColumnsArgs + + +AlterTableOperation = Annotated[ + Union[ + AddColumnOperation, + RenameColumnOperation, + AlterColumnOperation, + DropColumnOperation, + SetPrimaryKeyOperation, + ReorderColumnsOperation, + ], + Field(discriminator="op"), +] + + +class AlterTableRequest(_StrictPydanticModel): + operations: list[AlterTableOperation] = Field(min_length=1) + dry_run: bool = False + + +def _pydantic_errors(validation_error): + errors = [] + for error in validation_error.errors(): + location = ".".join(str(item) for item in error["loc"]) + message = error["msg"] + errors.append("{}: {}".format(location, message) if location else message) + return errors + + +def _table_schema_from_conn(conn, table_name): + row = conn.execute( + "select sql from sqlite_master where type = 'table' and name = ?", + [table_name], + ).fetchone() + return row[0] if row else None + + +def _primary_key_value(columns): + if len(columns) == 1: + return columns[0] + return tuple(columns) + + +def _default_expression_sql(default_expr): + return DEFAULT_EXPR_SQL[default_expr] + + +def _literal_default(db, value): + if isinstance(value, str): + return db.quote(value) + return value + + +class TableCreateView(BaseView): + name = "table-create" + + _valid_keys = { + "table", + "rows", + "row", + "columns", + "pk", + "pks", + "ignore", + "replace", + "alter", + } + _supported_column_types = set(CREATE_TABLE_COLUMN_TYPES) + # Any string that does not contain a newline or start with sqlite_ + _table_name_re = re.compile(r"^(?!sqlite_)[^\n]+$") + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + db = await self.ds.resolve_database(request) + database_name = db.name + + # Must have create-table permission + if not await self.ds.allowed( + action="create-table", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ): + return _error(["Permission denied"], 403) + + try: + data = await request.json() + except json.JSONDecodeError as e: + return _error(["Invalid JSON: {}".format(e)]) + + if not isinstance(data, dict): + return _error(["JSON must be an object"]) + + invalid_keys = set(data.keys()) - self._valid_keys + if invalid_keys: + return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) + + # ignore and replace are mutually exclusive + if data.get("ignore") and data.get("replace"): + return _error(["ignore and replace are mutually exclusive"]) + + # ignore and replace only allowed with row or rows + if "ignore" in data or "replace" in data: + if not data.get("row") and not data.get("rows"): + return _error(["ignore and replace require row or rows"]) + + # ignore and replace require pk or pks + if "ignore" in data or "replace" in data: + if not data.get("pk") and not data.get("pks"): + return _error(["ignore and replace require pk or pks"]) + + ignore = data.get("ignore") + replace = data.get("replace") + + if replace: + # Must have update-row permission + if not await self.ds.allowed( + action="update-row", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ): + return _error(["Permission denied: need update-row"], 403) + + table_name = data.get("table") + if not table_name: + return _error(["Table is required"]) + + if not self._table_name_re.match(table_name): + return _error(["Invalid table name"]) + + table_exists = await db.table_exists(data["table"]) + columns = data.get("columns") + rows = data.get("rows") + row = data.get("row") + if not columns and not rows and not row: + return _error(["columns, rows or row is required"]) + + if rows and row: + return _error(["Cannot specify both rows and row"]) + + if rows or row: + # Must have insert-row permission + if not await self.ds.allowed( + action="insert-row", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ): + return _error(["Permission denied: need insert-row"], 403) + + alter = False + if rows or row: + if not table_exists: + # if table is being created for the first time, alter=True + alter = True + else: + # alter=True only if they request it AND they have permission + if data.get("alter"): + if not await self.ds.allowed( + action="alter-table", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ): + return _error(["Permission denied: need alter-table"], 403) + alter = True + + if columns: + if rows or row: + return _error(["Cannot specify columns with rows or row"]) + if not isinstance(columns, list): + return _error(["columns must be a list"]) + for column in columns: + if not isinstance(column, dict): + return _error(["columns must be a list of objects"]) + if not column.get("name") or not isinstance(column.get("name"), str): + return _error(["Column name is required"]) + if not column.get("type"): + column["type"] = "text" + if column["type"] not in self._supported_column_types: + return _error( + ["Unsupported column type: {}".format(column["type"])] + ) + # No duplicate column names + dupes = {c["name"] for c in columns if columns.count(c) > 1} + if dupes: + return _error(["Duplicate column name: {}".format(", ".join(dupes))]) + + if row: + rows = [row] + + if rows: + if not isinstance(rows, list): + return _error(["rows must be a list"]) + for row in rows: + if not isinstance(row, dict): + return _error(["rows must be a list of objects"]) + + pk = data.get("pk") + pks = data.get("pks") + + if pk and pks: + return _error(["Cannot specify both pk and pks"]) + if pk: + if not isinstance(pk, str): + return _error(["pk must be a string"]) + if pks: + if not isinstance(pks, list): + return _error(["pks must be a list"]) + for pk in pks: + if not isinstance(pk, str): + return _error(["pks must be a list of strings"]) + + # If table exists already, read pks from that instead + if table_exists: + actual_pks = await db.primary_keys(table_name) + # if pk passed and table already exists check it does not change + bad_pks = False + if len(actual_pks) == 1 and data.get("pk") and data["pk"] != actual_pks[0]: + bad_pks = True + elif ( + len(actual_pks) > 1 + and data.get("pks") + and set(data["pks"]) != set(actual_pks) + ): + bad_pks = True + if bad_pks: + return _error(["pk cannot be changed for existing table"]) + pks = actual_pks + + initial_schema = None + if table_exists: + initial_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + + def create_table(conn): + table = sqlite_utils.Database(conn)[table_name] + if rows: + table.insert_all( + rows, pk=pks or pk, ignore=ignore, replace=replace, alter=alter + ) + else: + table.create( + {c["name"]: c["type"] for c in columns}, + pk=pks or pk, + ) + return table.schema + + try: + schema = await db.execute_write_fn(create_table, request=request) + except Exception as e: + return _error([str(e)]) + + if initial_schema is not None and initial_schema != schema: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=initial_schema, + after_schema=schema, + ) + ) + + table_url = self.ds.absolute_url( + request, self.ds.urls.table(db.name, table_name) + ) + table_api_url = self.ds.absolute_url( + request, self.ds.urls.table(db.name, table_name, format="json") + ) + details = { + "ok": True, + "database": db.name, + "table": table_name, + "table_url": table_url, + "table_api_url": table_api_url, + "schema": schema, + } + if rows: + details["row_count"] = len(rows) + + if not table_exists: + # Only log creation if we created a table + await self.ds.track_event( + CreateTableEvent( + request.actor, database=db.name, table=table_name, schema=schema + ) + ) + if rows: + await self.ds.track_event( + InsertRowsEvent( + request.actor, + database=db.name, + table=table_name, + num_rows=len(rows), + ignore=ignore, + replace=replace, + ) + ) + return Response.json(details, status=201) + + +class TableAlterView(BaseView): + name = "table-alter" + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + try: + resolved = await self.ds.resolve_table(request) + except NotFound as e: + return _error([e.args[0]], 404) + + db = resolved.db + database_name = db.name + table_name = resolved.table + + if not await self.ds.allowed( + action="alter-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return _error(["Permission denied: need alter-table"], 403) + + if not db.is_mutable: + return _error(["Database is immutable"], 403) + + content_type = request.headers.get("content-type") or "" + if not content_type.startswith("application/json"): + return _error(["Invalid content-type, must be application/json"], 400) + + try: + data = await request.json() + except json.JSONDecodeError as e: + return _error(["Invalid JSON: {}".format(e)], 400) + + if not isinstance(data, dict): + return _error(["JSON must be a dictionary"], 400) + + try: + alter_request = AlterTableRequest.model_validate(data) + except ValidationError as e: + return _error(_pydantic_errors(e), 400) + + def alter_table(conn): + before_schema = _table_schema_from_conn(conn, table_name) + + def apply_operations(operation_conn): + db_for_write = sqlite_utils.Database(operation_conn) + table = db_for_write[table_name] + + add_columns = [] + types = {} + rename = {} + drop = set() + not_null = {} + defaults = {} + column_order = None + pk = SQLITE_UTILS_DEFAULT + + for operation in alter_request.operations: + args = operation.args + if operation.op == "add_column": + if args.not_null and not ( + ( + "default" in args.model_fields_set + and args.default is not None + ) + or "default_expr" in args.model_fields_set + ): + raise ValueError( + "add_column args.default or args.default_expr is required when not_null is true" + ) + add_columns.append(args) + if "default" in args.model_fields_set and not args.not_null: + defaults[args.name] = _literal_default( + db_for_write, args.default + ) + if ( + "default_expr" in args.model_fields_set + and not args.not_null + ): + defaults[args.name] = _default_expression_sql( + args.default_expr + ) + elif operation.op == "rename_column": + rename[args.name] = args.to + elif operation.op == "alter_column": + if args.type is not None: + types[args.name] = args.type + if args.not_null is not None: + not_null[args.name] = args.not_null + if "default" in args.model_fields_set: + defaults[args.name] = ( + None + if args.default is None + else _literal_default(db_for_write, args.default) + ) + if "default_expr" in args.model_fields_set: + defaults[args.name] = _default_expression_sql( + args.default_expr + ) + elif operation.op == "drop_column": + drop.add(args.name) + elif operation.op == "set_primary_key": + pk = _primary_key_value(args.columns) + elif operation.op == "reorder_columns": + column_order = args.columns + + with operation_conn: + for column in add_columns: + not_null_default = None + if column.not_null: + if "default_expr" in column.model_fields_set: + not_null_default = _default_expression_sql( + column.default_expr + ) + else: + not_null_default = _literal_default( + db_for_write, column.default + ) + table.add_column( + column.name, + column.type, + not_null_default=not_null_default, + ) + + should_transform = any( + ( + types, + rename, + drop, + not_null, + defaults, + column_order is not None, + pk is not SQLITE_UTILS_DEFAULT, + ) + ) + if should_transform: + table.transform( + types=types or None, + rename=rename or None, + drop=drop or None, + pk=pk, + not_null=not_null or None, + defaults=defaults or None, + column_order=column_order, + ) + + return _table_schema_from_conn(operation_conn, table_name) + + if alter_request.dry_run: + memory_conn = sqlite3.connect(":memory:") + try: + conn.backup(memory_conn) + return before_schema, apply_operations(memory_conn) + finally: + memory_conn.close() + + after_schema = apply_operations(conn) + return before_schema, after_schema + + try: + before_schema, after_schema = await db.execute_write_fn( + alter_table, request=request + ) + except Exception as e: + return _error([str(e)], 400) + + altered = before_schema != after_schema + if altered and not alter_request.dry_run: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=before_schema, + after_schema=after_schema, + ) + ) + + table_url = self.ds.absolute_url( + request, self.ds.urls.table(database_name, table_name) + ) + table_api_url = self.ds.absolute_url( + request, self.ds.urls.table(database_name, table_name, format="json") + ) + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "table_url": table_url, + "table_api_url": table_api_url, + "altered": altered, + "schema": after_schema, + "before_schema": before_schema, + "operations_applied": ( + 0 if alter_request.dry_run else len(alter_request.operations) + ), + "dry_run": alter_request.dry_run, + }, + status=200, + ) From 9766a9c0876351a5c60430c4582a7686cb24ad79 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 12:38:51 -0700 Subject: [PATCH 023/174] Add foreign keys to create table API - Add fk_table and optional fk_column support to create-table columns. - Validate create-table requests with Pydantic while preserving existing errors. - Document the API and cover inferred primary-key and validation cases. Refs https://github.com/simonw/datasette/pull/2789#issuecomment-4733544452 --- datasette/views/table_create_alter.py | 287 ++++++++++++++++---------- docs/json_api.rst | 25 +++ tests/test_api_write.py | 115 +++++++++++ 3 files changed, 321 insertions(+), 106 deletions(-) diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 7decfad2..e8264a6f 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -2,7 +2,15 @@ import json import re from typing import Annotated, Any, Literal, Union -from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationError, + field_validator, + model_validator, +) +from pydantic_core import PydanticCustomError import sqlite_utils from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT @@ -25,6 +33,7 @@ CREATE_TABLE_TYPE_FOR_SQLITE_TYPE = { sqlite_type: column_type for column_type, sqlite_type in CREATE_TABLE_SQLITE_TYPES.items() } +TABLE_NAME_RE = re.compile(r"^(?!sqlite_)[^\n]+$") ALTER_TABLE_COLUMN_TYPES = CREATE_TABLE_COLUMN_TYPES ALTER_TABLE_TYPE_FOR_SQLITE_TYPE = { SQLiteType.TEXT: "text", @@ -98,6 +107,137 @@ class _StrictPydanticModel(BaseModel): model_config = ConfigDict(extra="forbid") +class CreateTableColumn(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: Any = None + type: Any = "text" + fk_table: str | None = None + fk_column: str | None = None + + @model_validator(mode="after") + def validate_column(self): + if not self.name or not isinstance(self.name, str): + raise PydanticCustomError("create_table", "Column name is required") + if not self.type: + self.type = "text" + elif self.type not in CREATE_TABLE_COLUMN_TYPES: + raise PydanticCustomError( + "create_table", "Unsupported column type: {type}", {"type": self.type} + ) + if self.fk_column and not self.fk_table: + raise PydanticCustomError( + "create_table_with_location", + "fk_column requires fk_table", + ) + return self + + +class CreateTableRequest(_StrictPydanticModel): + table: Any = None + rows: Any = None + row: Any = None + columns: list[CreateTableColumn] | None = None + pk: Any = None + pks: Any = None + ignore: bool | None = None + replace: bool | None = None + alter: bool | None = None + + @field_validator("columns", mode="before") + @classmethod + def validate_columns_list(cls, value): + if value is None: + return value + if not isinstance(value, list): + raise PydanticCustomError("create_table", "columns must be a list") + if not all(isinstance(column, dict) for column in value): + raise PydanticCustomError( + "create_table", "columns must be a list of objects" + ) + return value + + @model_validator(mode="after") + def validate_request(self): + if not self.table: + raise PydanticCustomError("create_table", "Table is required") + if not isinstance(self.table, str) or not TABLE_NAME_RE.match(self.table): + raise PydanticCustomError("create_table", "Invalid table name") + if not self.columns and not self.rows and not self.row: + raise PydanticCustomError( + "create_table", "columns, rows or row is required" + ) + if self.rows and self.row: + raise PydanticCustomError( + "create_table", "Cannot specify both rows and row" + ) + if self.columns and (self.rows or self.row): + raise PydanticCustomError( + "create_table", "Cannot specify columns with rows or row" + ) + if self.columns is not None: + seen = set() + duplicates = [] + for column in self.columns: + if column.name in seen and column.name not in duplicates: + duplicates.append(column.name) + seen.add(column.name) + if duplicates: + raise PydanticCustomError( + "create_table", + "Duplicate column name: {names}", + {"names": ", ".join(duplicates)}, + ) + if self.rows is not None: + if not isinstance(self.rows, list): + raise PydanticCustomError("create_table", "rows must be a list") + if not all(isinstance(row, dict) for row in self.rows): + raise PydanticCustomError( + "create_table", "rows must be a list of objects" + ) + if self.pk is not None and not isinstance(self.pk, str): + raise PydanticCustomError("create_table", "pk must be a string") + if self.pk and self.pks: + raise PydanticCustomError("create_table", "Cannot specify both pk and pks") + if self.pks is not None: + if not isinstance(self.pks, list): + raise PydanticCustomError("create_table", "pks must be a list") + if not all(isinstance(pk, str) for pk in self.pks): + raise PydanticCustomError( + "create_table", "pks must be a list of strings" + ) + if self.ignore and self.replace: + raise PydanticCustomError( + "create_table", "ignore and replace are mutually exclusive" + ) + if {"ignore", "replace"} & self.model_fields_set: + if not self.row and not self.rows: + raise PydanticCustomError( + "create_table", "ignore and replace require row or rows" + ) + if not self.pk and not self.pks: + raise PydanticCustomError( + "create_table", "ignore and replace require pk or pks" + ) + return self + + @property + def rows_list(self): + return [self.row] if self.row else self.rows + + @property + def foreign_keys(self): + if not self.columns: + return None + foreign_keys = [] + for column in self.columns: + if column.fk_table and column.fk_column: + foreign_keys.append((column.name, column.fk_table, column.fk_column)) + elif column.fk_table: + foreign_keys.append((column.name, column.fk_table)) + return foreign_keys or None + + class _DefaultArgsMixin(_StrictPydanticModel): default: Any | None = None default_expr: DefaultExpr | None = None @@ -209,6 +349,27 @@ def _pydantic_errors(validation_error): return errors +def _create_table_pydantic_errors(validation_error): + errors = validation_error.errors() + invalid_keys = sorted( + str(error["loc"][0]) + for error in errors + if error["type"] == "extra_forbidden" and len(error["loc"]) == 1 + ) + if invalid_keys: + return ["Invalid keys: {}".format(", ".join(invalid_keys))] + + output = [] + for error in errors: + message = error["msg"] + if error["type"] == "create_table": + output.append(message) + continue + location = ".".join(str(item) for item in error["loc"]) + output.append("{}: {}".format(location, message) if location else message) + return output + + def _table_schema_from_conn(conn, table_name): row = conn.execute( "select sql from sqlite_master where type = 'table' and name = ?", @@ -236,21 +397,6 @@ def _literal_default(db, value): class TableCreateView(BaseView): name = "table-create" - _valid_keys = { - "table", - "rows", - "row", - "columns", - "pk", - "pks", - "ignore", - "replace", - "alter", - } - _supported_column_types = set(CREATE_TABLE_COLUMN_TYPES) - # Any string that does not contain a newline or start with sqlite_ - _table_name_re = re.compile(r"^(?!sqlite_)[^\n]+$") - def __init__(self, datasette): self.ds = datasette @@ -274,26 +420,13 @@ class TableCreateView(BaseView): if not isinstance(data, dict): return _error(["JSON must be an object"]) - invalid_keys = set(data.keys()) - self._valid_keys - if invalid_keys: - return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) + try: + create_request = CreateTableRequest.model_validate(data) + except ValidationError as e: + return _error(_create_table_pydantic_errors(e)) - # ignore and replace are mutually exclusive - if data.get("ignore") and data.get("replace"): - return _error(["ignore and replace are mutually exclusive"]) - - # ignore and replace only allowed with row or rows - if "ignore" in data or "replace" in data: - if not data.get("row") and not data.get("rows"): - return _error(["ignore and replace require row or rows"]) - - # ignore and replace require pk or pks - if "ignore" in data or "replace" in data: - if not data.get("pk") and not data.get("pks"): - return _error(["ignore and replace require pk or pks"]) - - ignore = data.get("ignore") - replace = data.get("replace") + ignore = create_request.ignore + replace = create_request.replace if replace: # Must have update-row permission @@ -304,24 +437,12 @@ class TableCreateView(BaseView): ): return _error(["Permission denied: need update-row"], 403) - table_name = data.get("table") - if not table_name: - return _error(["Table is required"]) + table_name = create_request.table + table_exists = await db.table_exists(table_name) + columns = create_request.columns + rows = create_request.rows_list - if not self._table_name_re.match(table_name): - return _error(["Invalid table name"]) - - table_exists = await db.table_exists(data["table"]) - columns = data.get("columns") - rows = data.get("rows") - row = data.get("row") - if not columns and not rows and not row: - return _error(["columns, rows or row is required"]) - - if rows and row: - return _error(["Cannot specify both rows and row"]) - - if rows or row: + if rows: # Must have insert-row permission if not await self.ds.allowed( action="insert-row", @@ -331,13 +452,13 @@ class TableCreateView(BaseView): return _error(["Permission denied: need insert-row"], 403) alter = False - if rows or row: + if rows: if not table_exists: # if table is being created for the first time, alter=True alter = True else: # alter=True only if they request it AND they have permission - if data.get("alter"): + if create_request.alter: if not await self.ds.allowed( action="alter-table", resource=DatabaseResource(database=database_name), @@ -346,64 +467,17 @@ class TableCreateView(BaseView): return _error(["Permission denied: need alter-table"], 403) alter = True - if columns: - if rows or row: - return _error(["Cannot specify columns with rows or row"]) - if not isinstance(columns, list): - return _error(["columns must be a list"]) - for column in columns: - if not isinstance(column, dict): - return _error(["columns must be a list of objects"]) - if not column.get("name") or not isinstance(column.get("name"), str): - return _error(["Column name is required"]) - if not column.get("type"): - column["type"] = "text" - if column["type"] not in self._supported_column_types: - return _error( - ["Unsupported column type: {}".format(column["type"])] - ) - # No duplicate column names - dupes = {c["name"] for c in columns if columns.count(c) > 1} - if dupes: - return _error(["Duplicate column name: {}".format(", ".join(dupes))]) - - if row: - rows = [row] - - if rows: - if not isinstance(rows, list): - return _error(["rows must be a list"]) - for row in rows: - if not isinstance(row, dict): - return _error(["rows must be a list of objects"]) - - pk = data.get("pk") - pks = data.get("pks") - - if pk and pks: - return _error(["Cannot specify both pk and pks"]) - if pk: - if not isinstance(pk, str): - return _error(["pk must be a string"]) - if pks: - if not isinstance(pks, list): - return _error(["pks must be a list"]) - for pk in pks: - if not isinstance(pk, str): - return _error(["pks must be a list of strings"]) + pk = create_request.pk + pks = create_request.pks # If table exists already, read pks from that instead if table_exists: actual_pks = await db.primary_keys(table_name) # if pk passed and table already exists check it does not change bad_pks = False - if len(actual_pks) == 1 and data.get("pk") and data["pk"] != actual_pks[0]: + if len(actual_pks) == 1 and pk and pk != actual_pks[0]: bad_pks = True - elif ( - len(actual_pks) > 1 - and data.get("pks") - and set(data["pks"]) != set(actual_pks) - ): + elif len(actual_pks) > 1 and pks and set(pks) != set(actual_pks): bad_pks = True if bad_pks: return _error(["pk cannot be changed for existing table"]) @@ -423,8 +497,9 @@ class TableCreateView(BaseView): ) else: table.create( - {c["name"]: c["type"] for c in columns}, + {column.name: column.type for column in columns}, pk=pks or pk, + foreign_keys=create_request.foreign_keys, ) return table.schema diff --git a/docs/json_api.rst b/docs/json_api.rst index 4074b479..1b4a196e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -1981,6 +1981,7 @@ The JSON here describes the table that will be created: - ``name`` is the name of the column. This is required. - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. + - ``fk_table`` can be used to create a single-column foreign key constraint referencing another table. ``fk_column`` is optional and can be used to specify the referenced column - if omitted, Datasette will use the single primary key of ``fk_table``. * ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. @@ -1993,6 +1994,30 @@ The JSON here describes the table that will be created: * ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. This requires the :ref:`actions_update_row` permission. * ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`actions_alter_table` permission. +This example creates a foreign key from ``projects.owner_id`` to the single primary key of ``owners``: + +.. code-block:: json + + { + "table": "projects", + "columns": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "owner_id", + "type": "integer", + "fk_table": "owners" + }, + { + "name": "title", + "type": "text" + } + ], + "pk": "id" + } + If the table is successfully created this will return a ``201`` status code and the following response: .. code-block:: json diff --git a/tests/test_api_write.py b/tests/test_api_write.py index f117c06e..627b1ac1 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1614,6 +1614,121 @@ async def test_create_table( assert [e.name for e in events] == expected_events +@pytest.mark.asyncio +async def test_create_table_with_foreign_key(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "owners", + "columns": [ + {"name": "id", "type": "integer"}, + {"name": "name", "type": "text"}, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "projects", + "columns": [ + {"name": "id", "type": "integer"}, + { + "name": "owner_id", + "type": "integer", + "fk_table": "owners", + }, + {"name": "title", "type": "text"}, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + data = response.json() + assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "column,expected_error", + ( + ( + {"name": "owner_id", "type": "integer", "fk_table": "owners"}, + None, + ), + ( + {"name": "owner_id", "type": "integer", "fk_column": "id"}, + "columns.0: fk_column requires fk_table", + ), + ), +) +async def test_create_table_foreign_key_validation(ds_write, column, expected_error): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "projects", + "columns": [column], + }, + headers=_headers(token), + ) + if expected_error: + assert response.status_code == 400 + assert response.json() == {"ok": False, "errors": [expected_error]} + else: + assert response.status_code == 400 + assert response.json() == { + "ok": False, + "errors": ["Could not detect single primary key for table 'owners'"], + } + + +@pytest.mark.asyncio +async def test_create_table_foreign_key_without_fk_column_requires_single_pk(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "accounts", + "columns": [ + {"name": "tenant_id", "type": "integer"}, + {"name": "id", "type": "integer"}, + {"name": "name", "type": "text"}, + ], + "pks": ["tenant_id", "id"], + }, + headers=_headers(token), + ) + assert response.status_code == 201 + + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "projects", + "columns": [ + {"name": "id", "type": "integer"}, + { + "name": "account_id", + "type": "integer", + "fk_table": "accounts", + }, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 400 + assert response.json() == { + "ok": False, + "errors": ["Could not detect single primary key for table 'accounts'"], + } + + @pytest.mark.asyncio @pytest.mark.parametrize( "permissions,body,expected_status,expected_errors", From 9d9a2d3ff379b1f1ee3db4d24b53310ef2a71023 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 12:51:19 -0700 Subject: [PATCH 024/174] Add foreign keys to alter table API - Add add_foreign_key, drop_foreign_key, and set_foreign_keys operations. - Validate flat fk_table and fk_column arguments with Pydantic. - Document the API and cover inferred primary-key and validation cases. --- datasette/views/table_create_alter.py | 67 +++++++++++++++ docs/json_api.rst | 30 +++++++ tests/test_api_write.py | 118 ++++++++++++++++++++++++++ 3 files changed, 215 insertions(+) diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index e8264a6f..20dcc03e 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -292,6 +292,40 @@ class ReorderColumnsArgs(_StrictPydanticModel): columns: list[str] = Field(min_length=1) +class ForeignKeyArgs(_StrictPydanticModel): + column: str + fk_table: str | None = None + fk_column: str | None = None + + @model_validator(mode="after") + def validate_foreign_key(self): + if self.fk_column and not self.fk_table: + raise PydanticCustomError( + "alter_table_foreign_key", + "fk_column requires fk_table", + ) + if not self.fk_table: + raise PydanticCustomError( + "alter_table_foreign_key", + "fk_table is required", + ) + return self + + @property + def tuple(self): + if self.fk_column: + return (self.column, self.fk_table, self.fk_column) + return (self.column, self.fk_table) + + +class DropForeignKeyArgs(_StrictPydanticModel): + column: str + + +class SetForeignKeysArgs(_StrictPydanticModel): + foreign_keys: list[ForeignKeyArgs] + + class AddColumnOperation(_StrictPydanticModel): op: Literal["add_column"] args: AddColumnArgs @@ -322,6 +356,21 @@ class ReorderColumnsOperation(_StrictPydanticModel): args: ReorderColumnsArgs +class AddForeignKeyOperation(_StrictPydanticModel): + op: Literal["add_foreign_key"] + args: ForeignKeyArgs + + +class DropForeignKeyOperation(_StrictPydanticModel): + op: Literal["drop_foreign_key"] + args: DropForeignKeyArgs + + +class SetForeignKeysOperation(_StrictPydanticModel): + op: Literal["set_foreign_keys"] + args: SetForeignKeysArgs + + AlterTableOperation = Annotated[ Union[ AddColumnOperation, @@ -330,6 +379,9 @@ AlterTableOperation = Annotated[ DropColumnOperation, SetPrimaryKeyOperation, ReorderColumnsOperation, + AddForeignKeyOperation, + DropForeignKeyOperation, + SetForeignKeysOperation, ], Field(discriminator="op"), ] @@ -615,6 +667,9 @@ class TableAlterView(BaseView): defaults = {} column_order = None pk = SQLITE_UTILS_DEFAULT + add_foreign_keys = [] + drop_foreign_keys = [] + foreign_keys = None for operation in alter_request.operations: args = operation.args @@ -664,6 +719,12 @@ class TableAlterView(BaseView): pk = _primary_key_value(args.columns) elif operation.op == "reorder_columns": column_order = args.columns + elif operation.op == "add_foreign_key": + add_foreign_keys.append(args.tuple) + elif operation.op == "drop_foreign_key": + drop_foreign_keys.append(args.column) + elif operation.op == "set_foreign_keys": + foreign_keys = [fk.tuple for fk in args.foreign_keys] with operation_conn: for column in add_columns: @@ -692,6 +753,9 @@ class TableAlterView(BaseView): defaults, column_order is not None, pk is not SQLITE_UTILS_DEFAULT, + add_foreign_keys, + drop_foreign_keys, + foreign_keys is not None, ) ) if should_transform: @@ -703,6 +767,9 @@ class TableAlterView(BaseView): not_null=not_null or None, defaults=defaults or None, column_order=column_order, + add_foreign_keys=add_foreign_keys or None, + drop_foreign_keys=drop_foreign_keys or None, + foreign_keys=foreign_keys, ) return _table_schema_from_conn(operation_conn, table_name) diff --git a/docs/json_api.rst b/docs/json_api.rst index 1b4a196e..af16626f 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2159,6 +2159,31 @@ The request body should include an ``operations`` array. Each operation has the "columns": ["id"] } }, + { + "op": "add_foreign_key", + "args": { + "column": "owner_id", + "fk_table": "owners" + } + }, + { + "op": "drop_foreign_key", + "args": { + "column": "old_owner_id" + } + }, + { + "op": "set_foreign_keys", + "args": { + "foreign_keys": [ + { + "column": "owner_id", + "fk_table": "owners", + "fk_column": "id" + } + ] + } + }, { "op": "reorder_columns", "args": { @@ -2177,10 +2202,15 @@ Supported operations: * ``alter_column`` changes column properties. ``args`` accepts ``name`` and at least one of ``type``, ``not_null``, literal ``default`` or ``default_expr``. Passing ``"default": null`` removes an existing default. * ``drop_column`` drops a column. ``args`` accepts ``name``. * ``set_primary_key`` changes the table primary key. ``args`` accepts ``columns``, a list of one or more column names. +* ``add_foreign_key`` adds a single-column foreign key constraint. ``args`` accepts ``column``, ``fk_table`` and optional ``fk_column``. If ``fk_column`` is omitted, Datasette will use the single primary key of ``fk_table``. +* ``drop_foreign_key`` removes the foreign key constraint for a column. ``args`` accepts ``column``. +* ``set_foreign_keys`` replaces all foreign key constraints on the table. ``args`` accepts ``foreign_keys``, a list of objects that each have ``column``, ``fk_table`` and optional ``fk_column``. An empty list removes all foreign key constraints. * ``reorder_columns`` reorders columns. ``args`` accepts ``columns``, a list of one or more column names. Columns omitted from this list will appear afterwards in their existing order. ``default`` is always treated as a literal value. ``default_expr`` accepts one of ``current_timestamp``, ``current_date`` or ``current_time`` and is rendered as the corresponding SQLite default expression. +For foreign key operations that omit ``fk_column``, the referenced ``fk_table`` must have a single-column primary key. Datasette will return an error if it cannot identify a single primary key column for that table. + A successful response returns the new schema and the previous schema: .. code-block:: json diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 627b1ac1..046cf695 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -926,6 +926,124 @@ async def test_alter_table_dry_run(ds_write): assert last_event(ds_write) is None +@pytest.mark.asyncio +async def test_alter_table_foreign_key_operations(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + await db.execute_write("create table owners (id integer primary key)") + await db.execute_write("create table categories (id integer primary key)") + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + {"op": "add_column", "args": {"name": "owner_id", "type": "integer"}}, + { + "op": "add_foreign_key", + "args": {"column": "owner_id", "fk_table": "owners"}, + }, + ] + }, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["operations_applied"] == 2 + assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"] + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [{"op": "drop_foreign_key", "args": {"column": "owner_id"}}] + }, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert "[owner_id] INTEGER REFERENCES" not in data["schema"] + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + { + "op": "set_foreign_keys", + "args": { + "foreign_keys": [ + { + "column": "owner_id", + "fk_table": "categories", + "fk_column": "id", + } + ] + }, + } + ] + }, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert "[owner_id] INTEGER REFERENCES [categories]([id])" in data["schema"] + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={"operations": [{"op": "set_foreign_keys", "args": {"foreign_keys": []}}]}, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert "[owner_id] INTEGER REFERENCES" not in data["schema"] + + +@pytest.mark.asyncio +async def test_alter_table_foreign_key_requires_fk_table_for_fk_column(ds_write): + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + { + "op": "add_foreign_key", + "args": {"column": "age", "fk_column": "id"}, + } + ] + }, + headers=_headers(write_token(ds_write, permissions=["at"])), + ) + assert response.status_code == 400 + assert response.json() == { + "ok": False, + "errors": ["operations.0.add_foreign_key.args: fk_column requires fk_table"], + } + + +@pytest.mark.asyncio +async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + await db.execute_write( + "create table accounts (tenant_id integer, id integer, primary key (tenant_id, id))" + ) + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + { + "op": "add_foreign_key", + "args": {"column": "age", "fk_table": "accounts"}, + } + ] + }, + headers=_headers(token), + ) + assert response.status_code == 400 + assert response.json() == { + "ok": False, + "errors": ["Could not detect single primary key for table 'accounts'"], + } + + @pytest.mark.asyncio async def test_alter_table_permission_denied(ds_write): token = write_token(ds_write, permissions=["ir"]) From 2900efb32d5348fdd4b3ecb33e5f8b1ee836f5c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 14:47:25 -0700 Subject: [PATCH 025/174] /db/table/-/foreign-key-suggestions API Improved version of the implementation datasette-edit-schema --- datasette/app.py | 10 +- datasette/views/table_create_alter.py | 303 +++++++++++++++++++++++++- docs/json_api.rst | 58 +++++ tests/test_api_write.py | 100 +++++++++ 4 files changed, 469 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 13e25d4e..6b9f47ba 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -49,7 +49,11 @@ from .views.database import ( DatabaseView, QueryView, ) -from .views.table_create_alter import TableAlterView, TableCreateView +from .views.table_create_alter import ( + TableAlterView, + TableCreateView, + TableForeignKeySuggestionsView, +) from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView from .views.stored_queries import ( QueryCreateAnalyzeView, @@ -2630,6 +2634,10 @@ class Datasette: TableAlterView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/alter$", ) + add_route( + TableForeignKeySuggestionsView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/foreign-key-suggestions$", + ) add_route( TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/set-column-type$", diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 20dcc03e..2cb59ac1 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -1,7 +1,9 @@ import json import re +import time from typing import Annotated, Any, Literal, Union +from datasette.database import QueryInterrupted from pydantic import ( BaseModel, ConfigDict, @@ -17,8 +19,14 @@ from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT from datasette.column_types import SQLiteType from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.resources import DatabaseResource, TableResource -from datasette.utils import sqlite3 +from datasette.utils import ( + escape_sqlite, + get_outbound_foreign_keys, + sqlite3, + table_column_details, +) from datasette.utils.asgi import NotFound, Response +from datasette.utils.sqlite import sqlite_hidden_table_names from .base import BaseView, _error @@ -41,6 +49,177 @@ ALTER_TABLE_TYPE_FOR_SQLITE_TYPE = { SQLiteType.REAL: "float", SQLiteType.BLOB: "blob", } +FOREIGN_KEY_SUGGESTION_ROW_LIMIT = 500 +FOREIGN_KEY_SUGGESTION_TIME_LIMIT_MS = 50 +FOREIGN_KEY_SUGGESTION_TOTAL_TIME_LIMIT_MS = 200 + + +class ForeignKeySuggestionTimedOut(Exception): + pass + + +def _sqlite_type_affinity(type_name): + type_name = (type_name or "").upper() + if "INT" in type_name: + return "integer" + if any(token in type_name for token in ("CHAR", "CLOB", "TEXT")): + return "text" + if "BLOB" in type_name or not type_name: + return "blob" + if any(token in type_name for token in ("REAL", "FLOA", "DOUB")): + return "real" + return "numeric" + + +def _foreign_key_type_compatible(source_affinity, target_affinity): + if source_affinity == target_affinity: + return True + numeric_affinities = {"integer", "real", "numeric"} + if source_affinity == "numeric": + return target_affinity in numeric_affinities + if target_affinity == "numeric": + return source_affinity in numeric_affinities + return False + + +def _public_foreign_key_target(target): + return { + "fk_table": target["fk_table"], + "fk_column": target["fk_column"], + "type": target["type"], + } + + +def _singular(name): + if name.endswith("ies") and len(name) > 3: + return name[:-3] + "y" + if name.endswith("s") and len(name) > 1: + return name[:-1] + return name + + +def _foreign_key_name_reasons(source_column, target): + source = source_column.lower() + table = target["fk_table"].lower() + singular_table = _singular(table) + column = target["fk_column"].lower() + possible_names = { + "{}_{}".format(table, column), + "{}_{}".format(singular_table, column), + } + if column == "id": + possible_names.update( + { + "{}_id".format(table), + "{}_id".format(singular_table), + } + ) + return ["name_match"] if source in possible_names else [] + + +def _foreign_key_option_sort_key(source_column, target): + has_name_match = bool(_foreign_key_name_reasons(source_column, target)) + return ( + 0 if has_name_match else 1, + target["fk_table"], + target["fk_column"], + ) + + +def _foreign_key_suggestion_metadata(conn, table_name): + hidden_tables = set(sqlite_hidden_table_names(conn)) + source_columns = [ + { + "column": column.name, + "type": (column.type or "").upper(), + "affinity": _sqlite_type_affinity(column.type), + } + for column in table_column_details(conn, table_name) + if not column.hidden + ] + current_by_column = { + fk["column"]: { + "fk_table": fk["other_table"], + "fk_column": fk["other_column"], + } + for fk in get_outbound_foreign_keys(conn, table_name) + } + table_names = [ + row[0] + for row in conn.execute( + "select name from sqlite_master where type = 'table' order by name" + ).fetchall() + if not row[0].startswith("sqlite_") + ] + targets = [] + for candidate_table in table_names: + if candidate_table == table_name or candidate_table in hidden_tables: + continue + columns = [column for column in table_column_details(conn, candidate_table)] + pks = [column for column in columns if column.is_pk and not column.hidden] + pks.sort(key=lambda column: column.is_pk) + if len(pks) != 1: + continue + pk = pks[0] + targets.append( + { + "fk_table": candidate_table, + "fk_column": pk.name, + "type": (pk.type or "").upper(), + "affinity": _sqlite_type_affinity(pk.type), + } + ) + return source_columns, targets, current_by_column + + +async def _foreign_key_suggestion_samples(db, table_name, columns): + if not columns: + return 0, {} + sql = "select {} from {} limit {}".format( + ", ".join(escape_sqlite(column) for column in columns), + escape_sqlite(table_name), + FOREIGN_KEY_SUGGESTION_ROW_LIMIT, + ) + try: + results = await db.execute( + sql, + custom_time_limit=FOREIGN_KEY_SUGGESTION_TIME_LIMIT_MS, + log_sql_errors=False, + ) + except QueryInterrupted as e: + raise ForeignKeySuggestionTimedOut from e + values_by_column = {column: [] for column in columns} + seen_by_column = {column: set() for column in columns} + for row in results.rows: + for column in columns: + value = row[column] + if value is None or value in seen_by_column[column]: + continue + seen_by_column[column].add(value) + values_by_column[column].append(value) + return len(results.rows), values_by_column + + +async def _foreign_key_suggestion_values_exist(db, target, values, time_limit_ms): + if not values: + return False + sql = "select {} from {} where {} in ({})".format( + escape_sqlite(target["fk_column"]), + escape_sqlite(target["fk_table"]), + escape_sqlite(target["fk_column"]), + ", ".join("?" for _ in values), + ) + try: + results = await db.execute( + sql, + params=values, + custom_time_limit=time_limit_ms, + log_sql_errors=False, + ) + except QueryInterrupted as e: + raise ForeignKeySuggestionTimedOut from e + found = {row[0] for row in results.rows} + return all(value in found for value in values) async def _create_table_ui_context( @@ -609,6 +788,128 @@ class TableCreateView(BaseView): return Response.json(details, status=201) +class TableForeignKeySuggestionsView(BaseView): + name = "table-foreign-key-suggestions" + + def __init__(self, datasette): + self.ds = datasette + + async def get(self, request): + try: + resolved = await self.ds.resolve_table(request) + except NotFound as e: + return _error([e.args[0]], 404) + + db = resolved.db + database_name = db.name + table_name = resolved.table + + if resolved.is_view: + return _error(["Cannot suggest foreign keys for a view"], 400) + + if not await self.ds.allowed( + action="alter-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return _error(["Permission denied: need alter-table"], 403) + + source_columns, targets, current_by_column = await db.execute_fn( + lambda conn: _foreign_key_suggestion_metadata(conn, table_name) + ) + + columns = [] + options_by_column = {} + for source_column in source_columns: + options = sorted( + [ + target + for target in targets + if _foreign_key_type_compatible( + source_column["affinity"], target["affinity"] + ) + ], + key=lambda target: _foreign_key_option_sort_key( + source_column["column"], target + ), + ) + options_by_column[source_column["column"]] = options + columns.append( + { + "column": source_column["column"], + "type": source_column["type"], + "affinity": source_column["affinity"], + "current": current_by_column.get(source_column["column"]), + "suggestions": [], + "options": [ + _public_foreign_key_target(option) for option in options + ], + } + ) + + columns_to_sample = [ + column["column"] + for column in columns + if options_by_column[column["column"]] + ] + row_check = { + "attempted": bool(columns_to_sample), + "status": "completed" if columns_to_sample else "skipped", + "row_limit": FOREIGN_KEY_SUGGESTION_ROW_LIMIT, + "sampled_rows": 0, + "checked_options": 0, + } + + try: + sampled_rows, values_by_column = await _foreign_key_suggestion_samples( + db, table_name, columns_to_sample + ) + row_check["sampled_rows"] = sampled_rows + deadline = time.perf_counter() + ( + FOREIGN_KEY_SUGGESTION_TOTAL_TIME_LIMIT_MS / 1000 + ) + for column_info in columns: + values = values_by_column.get(column_info["column"]) or [] + if not values: + continue + for option in options_by_column[column_info["column"]]: + remaining_ms = int((deadline - time.perf_counter()) * 1000) + if remaining_ms <= 0: + raise ForeignKeySuggestionTimedOut + if await _foreign_key_suggestion_values_exist( + db, + option, + values, + min(FOREIGN_KEY_SUGGESTION_TIME_LIMIT_MS, remaining_ms), + ): + reasons = [ + "type_match", + "sample_values_exist", + ] + _foreign_key_name_reasons(column_info["column"], option) + column_info["suggestions"].append( + { + "fk_table": option["fk_table"], + "fk_column": option["fk_column"], + "confidence": "sampled", + "sampled_values": len(values), + "reasons": reasons, + } + ) + row_check["checked_options"] += 1 + except ForeignKeySuggestionTimedOut: + row_check["status"] = "timed_out" + + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "row_check": row_check, + "columns": columns, + } + ) + + class TableAlterView(BaseView): name = "table-alter" diff --git a/docs/json_api.rst b/docs/json_api.rst index af16626f..5b05e920 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2097,6 +2097,64 @@ To use the ``"replace": true`` option you will also need the :ref:`actions_updat Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`actions_alter_table` permission. +.. _TableForeignKeySuggestionsView: + +Table foreign key suggestions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``//
/-/foreign-key-suggestions`` endpoint suggests possible single-column foreign key relationships for a table. This requires the :ref:`actions_alter_table` permission. + +:: + + GET //
/-/foreign-key-suggestions + +The response includes every type-compatible single-column primary key target for each column in ``options``. Datasette also performs a bounded data check against up to 500 rows in the table: if the sampled non-null values for a column all exist in a target primary key, that target is included in ``suggestions``. + +If the bounded check takes too long, the endpoint fails open. It still returns the type-compatible ``options`` for each column, but ``row_check.status`` will be ``"timed_out"`` and there may be no ``suggestions``. + +.. code-block:: json + + { + "ok": true, + "database": "data", + "table": "projects", + "row_check": { + "attempted": true, + "status": "completed", + "row_limit": 500, + "sampled_rows": 3, + "checked_options": 4 + }, + "columns": [ + { + "column": "owner_id", + "type": "INTEGER", + "affinity": "integer", + "current": null, + "suggestions": [ + { + "fk_table": "owners", + "fk_column": "id", + "confidence": "sampled", + "sampled_values": 3, + "reasons": [ + "type_match", + "sample_values_exist", + "name_match" + ] + } + ], + "options": [ + { + "fk_table": "owners", + "fk_column": "id", + "type": "INTEGER" + } + ] + } + ] + } + .. _TableAlterView: Altering tables diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 046cf695..36fe40e9 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1044,6 +1044,106 @@ async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_w } +@pytest.mark.asyncio +async def test_foreign_key_suggestions(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + await db.execute_write("create table owners (id integer primary key)") + await db.execute_write("insert into owners (id) values (1), (2), (3)") + await db.execute_write("create table categories (slug text primary key)") + await db.execute_write("insert into categories (slug) values ('one'), ('two')") + await db.execute_write("create table numbers (id integer primary key)") + await db.execute_write("insert into numbers (id) values (10), (20)") + await db.execute_write("create table weights (id real primary key)") + await db.execute_write("insert into weights (id) values (1.5), (2.5)") + await db.execute_write( + "insert into docs (id, title, score, age) values " + "(1, 'one', 1.5, 1), (2, 'two', 999.5, 2), (3, null, null, null)" + ) + + response = await ds_write.client.get( + "/data/docs/-/foreign-key-suggestions", + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["ok"] is True + assert data["database"] == "data" + assert data["table"] == "docs" + assert data["row_check"]["attempted"] is True + assert data["row_check"]["status"] == "completed" + assert data["row_check"]["row_limit"] == 500 + assert data["row_check"]["sampled_rows"] == 3 + + columns = {column["column"]: column for column in data["columns"]} + assert columns["age"]["options"] == [ + {"fk_table": "numbers", "fk_column": "id", "type": "INTEGER"}, + {"fk_table": "owners", "fk_column": "id", "type": "INTEGER"}, + ] + assert columns["age"]["suggestions"] == [ + { + "fk_table": "owners", + "fk_column": "id", + "confidence": "sampled", + "sampled_values": 2, + "reasons": ["type_match", "sample_values_exist"], + } + ] + assert columns["title"]["options"] == [ + {"fk_table": "categories", "fk_column": "slug", "type": "TEXT"} + ] + assert columns["title"]["suggestions"][0]["fk_table"] == "categories" + assert columns["score"]["options"] == [ + {"fk_table": "weights", "fk_column": "id", "type": "REAL"} + ] + assert columns["score"]["suggestions"] == [] + + +@pytest.mark.asyncio +async def test_foreign_key_suggestions_permission_denied(ds_write): + token = write_token(ds_write, permissions=["ir"]) + response = await ds_write.client.get( + "/data/docs/-/foreign-key-suggestions", + headers=_headers(token), + ) + assert response.status_code == 403 + assert response.json() == { + "ok": False, + "errors": ["Permission denied: need alter-table"], + } + + +@pytest.mark.asyncio +async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + await db.execute_write("create table owners (id integer primary key)") + + async def raise_timeout(*args, **kwargs): + raise table_create_alter.ForeignKeySuggestionTimedOut + + from datasette.views import table_create_alter + + monkeypatch.setattr( + table_create_alter, + "_foreign_key_suggestion_samples", + raise_timeout, + ) + + response = await ds_write.client.get( + "/data/docs/-/foreign-key-suggestions", + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["row_check"]["status"] == "timed_out" + columns = {column["column"]: column for column in data["columns"]} + assert columns["age"]["options"] == [ + {"fk_table": "owners", "fk_column": "id", "type": "INTEGER"} + ] + assert columns["age"]["suggestions"] == [] + + @pytest.mark.asyncio async def test_alter_table_permission_denied(ds_write): token = write_token(ds_write, permissions=["ir"]) From a6ef65f90da8b7789a41972fbe89aba37e1b6a30 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 17 Jun 2026 16:29:59 -0700 Subject: [PATCH 026/174] //-/foreign-key-targets API endpoint Returns a list of tables with a single primary key, and for each one the name of that primary key column and its SQLite type affinity. This will be used by the create table UI to suggest foreign keys. --- datasette/app.py | 5 ++ datasette/views/table_create_alter.py | 60 ++++++++++++++++++++- docs/json_api.rst | 34 ++++++++++++ tests/test_api_write.py | 75 +++++++++++++++++++++++++++ 4 files changed, 173 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index 6b9f47ba..79dffb66 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -50,6 +50,7 @@ from .views.database import ( QueryView, ) from .views.table_create_alter import ( + DatabaseForeignKeyTargetsView, TableAlterView, TableCreateView, TableForeignKeySuggestionsView, @@ -2566,6 +2567,10 @@ class Datasette: r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") + add_route( + DatabaseForeignKeyTargetsView.as_view(self), + r"/(?P[^\/\.]+)/-/foreign-key-targets$", + ) add_route( QueryListView.as_view(self), r"/(?P[^\/\.]+)/-/queries(\.(?Pjson))?$", diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 2cb59ac1..ce43199c 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -52,6 +52,34 @@ ALTER_TABLE_TYPE_FOR_SQLITE_TYPE = { FOREIGN_KEY_SUGGESTION_ROW_LIMIT = 500 FOREIGN_KEY_SUGGESTION_TIME_LIMIT_MS = 50 FOREIGN_KEY_SUGGESTION_TOTAL_TIME_LIMIT_MS = 200 +FOREIGN_KEY_TARGETS_SQL = """ +select + m.name as fk_table, + p.name as fk_column, + case + when upper(coalesce(p.type, '')) like '%INT%' then 'integer' + when upper(coalesce(p.type, '')) like '%CHAR%' + or upper(coalesce(p.type, '')) like '%CLOB%' + or upper(coalesce(p.type, '')) like '%TEXT%' then 'text' + when upper(coalesce(p.type, '')) like '%BLOB%' + or coalesce(p.type, '') = '' then 'blob' + when upper(coalesce(p.type, '')) like '%REAL%' + or upper(coalesce(p.type, '')) like '%FLOA%' + or upper(coalesce(p.type, '')) like '%' || 'DOU' || 'B' || '%' then 'real' + else 'numeric' + end as type +from sqlite_master as m +cross join pragma_table_info(m.name) as p +where m.type = 'table' + and m.name not like 'sqlite_%' + and p.pk > 0 + and ( + select count(*) + from pragma_table_info(m.name) as p2 + where p2.pk > 0 + ) = 1 +order by m.name +""" class ForeignKeySuggestionTimedOut(Exception): @@ -66,7 +94,10 @@ def _sqlite_type_affinity(type_name): return "text" if "BLOB" in type_name or not type_name: return "blob" - if any(token in type_name for token in ("REAL", "FLOA", "DOUB")): + if any( + token in type_name + for token in ("REAL", "FLOA", "DOUB") # codespell:ignore doub + ): return "real" return "numeric" @@ -788,6 +819,33 @@ class TableCreateView(BaseView): return Response.json(details, status=201) +class DatabaseForeignKeyTargetsView(BaseView): + name = "database-foreign-key-targets" + + def __init__(self, datasette): + self.ds = datasette + + async def get(self, request): + db = await self.ds.resolve_database(request) + database_name = db.name + + if not await self.ds.allowed( + action="create-table", + resource=DatabaseResource(database=database_name), + actor=request.actor, + ): + return _error(["Permission denied: need create-table"], 403) + + targets = (await db.execute(FOREIGN_KEY_TARGETS_SQL)).dicts() + return Response.json( + { + "ok": True, + "database": database_name, + "targets": targets, + } + ) + + class TableForeignKeySuggestionsView(BaseView): name = "table-foreign-key-suggestions" diff --git a/docs/json_api.rst b/docs/json_api.rst index 5b05e920..dee98ef2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2097,6 +2097,40 @@ To use the ``"replace": true`` option you will also need the :ref:`actions_updat Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`actions_alter_table` permission. +.. _DatabaseForeignKeyTargetsView: + +Database foreign key targets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``//-/foreign-key-targets`` endpoint returns the list of tables in a database that can be referenced by a single-column foreign key. This requires the :ref:`actions_create_table` permission. + +:: + + GET //-/foreign-key-targets + +The response includes only tables with exactly one primary key column. Tables with compound primary keys and tables with no explicit primary key are omitted. + +Each target includes the normalized SQLite type affinity for the primary key column in ``type``. The type is calculated using SQLite's documented affinity rules: ``INT`` maps to ``integer``; ``CHAR``, ``CLOB`` or ``TEXT`` maps to ``text``; ``BLOB`` or no type maps to ``blob``; ``REAL`` and floating-point declared types map to ``real``; everything else maps to ``numeric``. + +.. code-block:: json + + { + "ok": true, + "database": "data", + "targets": [ + { + "fk_table": "owners", + "fk_column": "id", + "type": "integer" + }, + { + "fk_table": "categories", + "fk_column": "slug", + "type": "text" + } + ] + } + .. _TableForeignKeySuggestionsView: Table foreign key suggestions diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 36fe40e9..18ffe43d 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1144,6 +1144,81 @@ async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch): assert columns["age"]["suggestions"] == [] +@pytest.mark.asyncio +async def test_foreign_key_targets(ds_write): + token = write_token(ds_write, permissions=["ct"]) + db = ds_write.get_database("data") + await db.execute_write("create table owners (id integer primary key)") + await db.execute_write("create table categories (slug varchar(30) primary key)") + await db.execute_write("create table blob_things (hash blob primary key)") + await db.execute_write( + "create table numeric_codes (code decimal(10,5) primary key)" + ) + await db.execute_write( + 'create table floating_point (value "FLOATING POINT" primary key)' + ) + await db.execute_write( + "create table compound (a integer, b integer, primary key (a, b))" + ) + await db.execute_write("create table no_pk (name text)") + + response = await ds_write.client.get( + "/data/-/foreign-key-targets", + headers=_headers(token), + ) + assert response.status_code == 200, response.text + assert response.json() == { + "ok": True, + "database": "data", + "targets": [ + { + "fk_table": "blob_things", + "fk_column": "hash", + "type": "blob", + }, + { + "fk_table": "categories", + "fk_column": "slug", + "type": "text", + }, + { + "fk_table": "docs", + "fk_column": "id", + "type": "integer", + }, + { + "fk_table": "floating_point", + "fk_column": "value", + "type": "integer", + }, + { + "fk_table": "numeric_codes", + "fk_column": "code", + "type": "numeric", + }, + { + "fk_table": "owners", + "fk_column": "id", + "type": "integer", + }, + ], + } + + +@pytest.mark.asyncio +async def test_foreign_key_targets_permission_denied(ds_write): + token = write_token(ds_write, permissions=["ir"]) + response = await ds_write.client.get( + "/data/-/foreign-key-targets", + headers=_headers(token), + ) + assert response.status_code == 403 + assert response.json() == { + "ok": False, + "errors": ["Permission denied: need create-table"], + } + + @pytest.mark.asyncio async def test_alter_table_permission_denied(ds_write): token = write_token(ds_write, permissions=["ir"]) From 21c156dfb1ca76d8e8c2e04c044766a155e22edc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Jun 2026 08:13:28 -0700 Subject: [PATCH 027/174] Expose foreign key targets to create table UI - Add foreignKeyTargetsPath to create table page data - Filter hidden tables from database-level foreign key target results - Update JSON API docs and tests for filtered targets --- datasette/views/table_create_alter.py | 12 +++++++++++- docs/json_api.rst | 2 +- tests/test_api_write.py | 8 ++++++++ tests/test_table_html.py | 1 + 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index ce43199c..f7775479 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -262,6 +262,9 @@ async def _create_table_ui_context( return None data = { "path": "{}/-/create".format(datasette.urls.database(database_name)), + "foreignKeyTargetsPath": "{}/-/foreign-key-targets".format( + datasette.urls.database(database_name) + ), "databaseName": database_name, "columnTypes": CREATE_TABLE_COLUMN_TYPES, } @@ -836,7 +839,14 @@ class DatabaseForeignKeyTargetsView(BaseView): ): return _error(["Permission denied: need create-table"], 403) - targets = (await db.execute(FOREIGN_KEY_TARGETS_SQL)).dicts() + hidden_tables = await db.execute_fn( + lambda conn: set(sqlite_hidden_table_names(conn)) + ) + targets = [ + target + for target in (await db.execute(FOREIGN_KEY_TARGETS_SQL)).dicts() + if target["fk_table"] not in hidden_tables + ] return Response.json( { "ok": True, diff --git a/docs/json_api.rst b/docs/json_api.rst index dee98ef2..1db46dd2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2108,7 +2108,7 @@ The ``//-/foreign-key-targets`` endpoint returns the list of tables in GET //-/foreign-key-targets -The response includes only tables with exactly one primary key column. Tables with compound primary keys and tables with no explicit primary key are omitted. +The response includes only tables with exactly one primary key column. Hidden tables, tables with compound primary keys and tables with no explicit primary key are omitted. Each target includes the normalized SQLite type affinity for the primary key column in ``type``. The type is calculated using SQLite's documented affinity rules: ``INT`` maps to ``integer``; ``CHAR``, ``CLOB`` or ``TEXT`` maps to ``text``; ``BLOB`` or no type maps to ``blob``; ``REAL`` and floating-point declared types map to ``real``; everything else maps to ``numeric``. diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 18ffe43d..1d16ad26 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1161,6 +1161,10 @@ async def test_foreign_key_targets(ds_write): "create table compound (a integer, b integer, primary key (a, b))" ) await db.execute_write("create table no_pk (name text)") + try: + await db.execute_write("create virtual table search_docs using fts5(body)") + except Exception: + pass response = await ds_write.client.get( "/data/-/foreign-key-targets", @@ -1203,6 +1207,10 @@ async def test_foreign_key_targets(ds_write): }, ], } + assert not any( + target["fk_table"].startswith("search_docs_") + for target in response.json()["targets"] + ) @pytest.mark.asyncio diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 374cc08d..fa01c8ec 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -994,6 +994,7 @@ async def test_database_create_table_action_button_and_data(): assert database_data_from_soup(soup) == { "createTable": { "path": "/data/-/create", + "foreignKeyTargetsPath": "/data/-/foreign-key-targets", "databaseName": "data", "columnTypes": ["text", "integer", "float", "blob"], }, From 1f863def5e410cc3f836f1e07404aa68b578db9b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Jun 2026 08:13:37 -0700 Subject: [PATCH 028/174] Add foreign key controls to create table dialog - Add create table advanced controls for foreign keys and first-column primary keys - Share schema dialog row helpers between create and alter dialogs - Move custom type into advanced options and add Add column icons --- datasette/static/app.css | 210 ++++++-- datasette/static/edit-tools.js | 954 ++++++++++++++++++++++++--------- 2 files changed, 862 insertions(+), 302 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index f9ebe5ac..49f070b1 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1760,7 +1760,7 @@ dialog.table-create-dialog { border-radius: var(--modal-border-radius, 0.75rem); padding: 0; margin: auto; - width: min(760px, calc(100vw - 32px)); + width: min(980px, calc(100vw - 32px)); max-width: 95vw; max-height: min(780px, calc(100vh - 32px)); box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); @@ -1839,7 +1839,7 @@ dialog.table-create-dialog::backdrop { } .table-create-label, -.table-create-columns-heading { +.table-create-column-headings { color: var(--ink); font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 0.82rem; @@ -1876,10 +1876,12 @@ dialog.table-create-dialog::backdrop { color: var(--muted); } +.table-create-foreign-key-target option, .table-create-custom-column-type option { color: var(--ink); } +.table-create-foreign-key-target option[value=""], .table-create-custom-column-type option[value=""] { color: var(--muted); } @@ -1898,40 +1900,130 @@ dialog.table-create-dialog::backdrop { gap: 10px; } -.table-create-columns-heading { - font-weight: 600; -} - .table-create-column-list { display: grid; gap: 8px; } -.table-create-column-row { +.table-create-column-headings, +.table-create-column-main { display: grid; - grid-template-columns: minmax(140px, 1.2fr) minmax(7.5rem, 0.7fr) minmax(12rem, 1fr) minmax(3.5rem, max-content) 32px; + grid-template-columns: minmax(140px, 1.2fr) minmax(7.5rem, 0.7fr) max-content 32px; align-items: center; gap: 8px; min-width: 0; - position: relative; } -.table-create-primary-key { - display: inline-flex; - align-items: center; - gap: 6px; - color: var(--ink); - font-size: 0.85rem; +.table-create-column-row { + display: grid; + gap: 8px; min-width: 0; - white-space: nowrap; - justify-self: center; } -.table-create-primary-key-input { +.table-create-column-headings { + color: var(--muted); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.72rem; + padding: 0 1px; +} + +.table-create-column-details { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + align-items: start; + gap: 12px 16px; + padding: 12px; + border-left: 3px solid var(--rule); + background: #f8fafc; +} + +.table-create-column-details[hidden] { + display: none; +} + +.table-create-detail-field { + display: grid; + gap: 4px; + min-width: 0; +} + +.table-create-detail-label { + color: var(--muted); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.72rem; +} + +.table-create-detail-help { + color: var(--muted); + font-size: 0.82rem; + line-height: 1.35; margin: 0; } -.table-create-remove-column { +.table-create-detail-check { + display: inline-flex; + align-items: flex-start; + gap: 8px; + color: var(--ink); + font-size: 0.85rem; + line-height: 1.35; + min-width: 0; + white-space: normal; +} + +.table-create-primary-key, +.table-create-foreign-key-field { + grid-column: 1 / -1; +} + +.table-create-detail-check input { + flex: 0 0 auto; + margin: 0.15rem 0 0; +} + +.table-create-detail-check span { + min-width: 0; + overflow-wrap: break-word; +} + +.table-create-move-controls { + display: grid; + grid-template-columns: repeat(4, 32px); + gap: 4px; + justify-content: start; +} + +.table-create-more-options { + appearance: none; + border: 0; + background: transparent; + color: var(--accent); + cursor: pointer; + font: inherit; + font-size: 0.85rem; + justify-self: start; + padding: 0; + grid-column: 1 / -1; + text-align: left; +} + +.table-create-more-options:hover, +.table-create-more-options:focus { + text-decoration: underline; +} + +.table-create-more-options:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 2px; +} + +.table-create-more-options:disabled { + color: var(--muted); + cursor: default; + text-decoration: none; +} + +.table-create-icon-button { appearance: none; border: 1px solid rgba(74, 85, 104, 0.24); background: transparent; @@ -1945,17 +2037,17 @@ dialog.table-create-dialog::backdrop { padding: 0; } -.table-create-remove-column:hover, -.table-create-remove-column:focus { +.table-create-icon-button:hover, +.table-create-icon-button:focus { background: rgba(74, 85, 104, 0.07); } -.table-create-remove-column:focus { +.table-create-icon-button:focus { outline: 3px solid #b3d4ff; outline-offset: 1px; } -.table-create-remove-column svg { +.table-create-icon-button svg { display: block; } @@ -1967,11 +2059,19 @@ dialog.table-create-dialog::backdrop { background: #fff; color: var(--accent); cursor: pointer; + display: inline-flex; + align-items: center; + gap: 6px; font: inherit; font-size: 0.85rem; padding: 7px 10px; } +.table-create-add-column svg { + display: block; + flex: 0 0 auto; +} + .table-create-add-column:hover, .table-create-add-column:focus { background: #f8fafc; @@ -2027,7 +2127,7 @@ dialog.table-create-dialog::backdrop { .table-create-dialog .btn:disabled, .table-create-add-column:disabled, -.table-create-remove-column:disabled { +.table-create-icon-button:disabled { opacity: 0.55; cursor: not-allowed; } @@ -2198,7 +2298,7 @@ dialog.table-alter-dialog::backdrop { .table-alter-column-headings, .table-alter-column-main { display: grid; - grid-template-columns: minmax(140px, 1.2fr) minmax(7.5rem, 0.7fr) minmax(12rem, 1fr) max-content 32px; + grid-template-columns: minmax(140px, 1.2fr) minmax(7.5rem, 0.7fr) max-content 32px; align-items: center; gap: 8px; min-width: 0; @@ -2384,11 +2484,19 @@ dialog.table-alter-dialog::backdrop { background: #fff; color: var(--accent); cursor: pointer; + display: inline-flex; + align-items: center; + gap: 6px; font: inherit; font-size: 0.85rem; padding: 7px 10px; } +.table-alter-add-column svg { + display: block; + flex: 0 0 auto; +} + .table-alter-add-column:hover, .table-alter-add-column:focus { background: #f8fafc; @@ -2504,21 +2612,16 @@ dialog.table-alter-dialog::backdrop { justify-self: end; } - .table-alter-custom-column-type { - grid-column: 1 / 3; - grid-row: 2; - } - .table-alter-move-controls { grid-column: 1; - grid-row: 3; + grid-row: 2; justify-self: start; } .table-alter-more-options { align-self: center; grid-column: 2 / 4; - grid-row: 3; + grid-row: 2; } .table-alter-column-details { @@ -2700,13 +2803,50 @@ dialog.table-alter-dialog::backdrop { padding-top: 0; } + .table-create-column-headings { + display: none; + } + .table-create-column-row { - grid-template-columns: minmax(0, 1fr) 8.5rem 3.5rem 32px; + padding-bottom: 8px; + border-bottom: 1px solid var(--rule); + } + + .table-create-column-main { + grid-template-columns: minmax(0, 1fr) minmax(7.5rem, 0.8fr) 32px; align-items: end; } - .table-create-custom-column-type { - grid-column: 1 / -1; + .table-create-column-name { + grid-column: 1; + grid-row: 1; + } + + .table-create-column-type { + grid-column: 2; + grid-row: 1; + } + + .table-create-remove-column { + grid-column: 3; + grid-row: 1; + justify-self: end; + } + + .table-create-move-controls { + grid-column: 1; + grid-row: 2; + justify-self: start; + } + + .table-create-more-options { + align-self: center; + grid-column: 2 / 4; + grid-row: 2; + } + + .table-create-column-details { + grid-template-columns: 1fr; } .table-create-dialog .modal-footer { diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index cb964365..5a9d0962 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -64,12 +64,220 @@ function sqliteColumnTypeLabel(type) { if (type === "float") { return "floating point number"; } + if (type === "real") { + return "floating point number"; + } if (type === "blob") { return "blob - binary data"; } return type; } +function populateSqliteColumnTypeSelect(select, type, options) { + options.forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option; + optionElement.textContent = sqliteColumnTypeLabel(option); + select.appendChild(optionElement); + }); + select.value = options.indexOf(type) === -1 ? options[0] : type; +} + +function updateSelectPlaceholder(select, placeholderClass) { + select.classList.toggle(placeholderClass, !select.value); +} + +function createCustomColumnTypeSelect(options, className, placeholderClass) { + var select = document.createElement("select"); + select.className = className; + select.setAttribute("aria-label", "Custom column type"); + var blankOption = document.createElement("option"); + blankOption.value = ""; + blankOption.textContent = "- custom type -"; + select.appendChild(blankOption); + options.forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option.name; + optionElement.textContent = option.description + ? option.description + " (" + option.name + ")" + : option.name; + select.appendChild(optionElement); + }); + updateSelectPlaceholder(select, placeholderClass); + return select; +} + +var COLUMN_MOVE_ICONS = { + top: '', + up: '', + down: '', + bottom: + '', + remove: + '', +}; + +function createSchemaDialogIconButton(prefix, modifier, ariaLabel, title, svg) { + var button = document.createElement("button"); + button.type = "button"; + button.className = prefix + "-icon-button " + prefix + "-" + modifier; + button.setAttribute("aria-label", ariaLabel); + button.title = title; + button.dataset.defaultTitle = title; + button.innerHTML = svg; + return button; +} + +function createSchemaDialogMoveControls(prefix) { + var moveControls = document.createElement("div"); + moveControls.className = prefix + "-move-controls"; + + var moveTopButton = createSchemaDialogIconButton( + prefix, + "move-top", + "Move column to top", + "Move column to top", + COLUMN_MOVE_ICONS.top, + ); + var moveUpButton = createSchemaDialogIconButton( + prefix, + "move-up", + "Move column up", + "Move column up", + COLUMN_MOVE_ICONS.up, + ); + var moveDownButton = createSchemaDialogIconButton( + prefix, + "move-down", + "Move column down", + "Move column down", + COLUMN_MOVE_ICONS.down, + ); + var moveBottomButton = createSchemaDialogIconButton( + prefix, + "move-bottom", + "Move column to bottom", + "Move column to bottom", + COLUMN_MOVE_ICONS.bottom, + ); + + moveControls.appendChild(moveTopButton); + moveControls.appendChild(moveUpButton); + moveControls.appendChild(moveDownButton); + moveControls.appendChild(moveBottomButton); + + return { + controls: moveControls, + topButton: moveTopButton, + upButton: moveUpButton, + downButton: moveDownButton, + bottomButton: moveBottomButton, + }; +} + +function createSchemaDialogMoreOptionsButton(prefix, details) { + var expandButton = document.createElement("button"); + expandButton.type = "button"; + expandButton.className = prefix + "-more-options"; + expandButton.setAttribute("aria-label", "Toggle column settings"); + expandButton.setAttribute("aria-controls", details.id); + expandButton.setAttribute("aria-expanded", details.hidden ? "false" : "true"); + updateSchemaDialogMoreOptionsButton(expandButton); + return expandButton; +} + +function updateSchemaDialogMoreOptionsButton(button) { + var isExpanded = button.getAttribute("aria-expanded") === "true"; + button.textContent = isExpanded ? "v Hide options" : "> Advanced options"; + button.title = isExpanded ? "Hide column settings" : "Show column settings"; +} + +function toggleSchemaDialogMoreOptions(button, details) { + var isExpanded = button.getAttribute("aria-expanded") === "true"; + details.hidden = isExpanded; + button.setAttribute("aria-expanded", isExpanded ? "false" : "true"); + updateSchemaDialogMoreOptionsButton(button); +} + +function schemaDialogRows(state, prefix) { + return Array.prototype.slice.call( + state.columnList.querySelectorAll("." + prefix + "-column-row"), + ); +} + +function schemaDialogRowIsPrimaryKey(row, prefix) { + var input = row && row.querySelector("." + prefix + "-primary-key-input"); + return !!(input && input.checked); +} + +function schemaDialogFirstNonPrimaryRow(state, prefix) { + var rows = schemaDialogRows(state, prefix); + for (var i = 0; i < rows.length; i += 1) { + if (!schemaDialogRowIsPrimaryKey(rows[i], prefix)) { + return rows[i]; + } + } + return null; +} + +function updateSchemaDialogMoveButtons(state, prefix) { + if (!state || !state.columnList) { + return; + } + var firstNonPrimary = schemaDialogFirstNonPrimaryRow(state, prefix); + var rows = schemaDialogRows(state, prefix); + var hasPrimaryKeys = rows.some(function (row) { + return schemaDialogRowIsPrimaryKey(row, prefix); + }); + var primaryKeyMoveTitle = "Primary key columns are always listed first"; + rows.forEach(function (row) { + var isPrimaryKey = schemaDialogRowIsPrimaryKey(row, prefix); + var previous = row.previousElementSibling; + var next = row.nextElementSibling; + row + .querySelectorAll("." + prefix + "-move-controls button") + .forEach(function (button) { + button.title = button.dataset.defaultTitle || button.title; + button.disabled = state.isSaving || isPrimaryKey; + if (isPrimaryKey) { + button.title = primaryKeyMoveTitle; + } + }); + if (!isPrimaryKey) { + var topButton = row.querySelector("." + prefix + "-move-top"); + var upButton = row.querySelector("." + prefix + "-move-up"); + var downButton = row.querySelector("." + prefix + "-move-down"); + var bottomButton = row.querySelector("." + prefix + "-move-bottom"); + topButton.disabled = + state.isSaving || !firstNonPrimary || row === firstNonPrimary; + upButton.disabled = + state.isSaving || !previous || schemaDialogRowIsPrimaryKey(previous, prefix); + downButton.disabled = state.isSaving || !next; + bottomButton.disabled = state.isSaving || !next; + if (hasPrimaryKeys && row === firstNonPrimary) { + topButton.title = primaryKeyMoveTitle; + upButton.title = primaryKeyMoveTitle; + } + } + }); +} + +function normalizeSchemaDialogPrimaryKeyRows(state, prefix) { + var rows = schemaDialogRows(state, prefix); + rows + .filter(function (row) { + return schemaDialogRowIsPrimaryKey(row, prefix); + }) + .concat( + rows.filter(function (row) { + return !schemaDialogRowIsPrimaryKey(row, prefix); + }), + ) + .forEach(function (row) { + state.columnList.appendChild(row); + }); +} + function tableCreateCustomColumnTypes() { var data = databaseCreateTableData() || {}; return data.customColumnTypes || []; @@ -93,15 +301,205 @@ function tableCreateCustomTypeAppliesToSqliteType(option, sqliteType) { ); } +function tableCreateDialogRows(state) { + return schemaDialogRows(state, "table-create"); +} + +function tableCreateRowIsPrimaryKey(row) { + return schemaDialogRowIsPrimaryKey(row, "table-create"); +} + +function tableCreateFirstNonPrimaryRow(state) { + return schemaDialogFirstNonPrimaryRow(state, "table-create"); +} + +function updateTableCreateMoveButtons(state) { + updateSchemaDialogMoveButtons(state, "table-create"); +} + +function tableCreateTypeAffinity(type) { + if (type === "float") { + return "real"; + } + return type; +} + +function foreignKeyTypesCompatible(sourceAffinity, targetAffinity) { + if (sourceAffinity === targetAffinity) { + return true; + } + var numericAffinities = ["integer", "real", "numeric"]; + if (sourceAffinity === "numeric") { + return numericAffinities.indexOf(targetAffinity) !== -1; + } + if (targetAffinity === "numeric") { + return numericAffinities.indexOf(sourceAffinity) !== -1; + } + return false; +} + +function tableCreateForeignKeyTargetKey(target) { + return target.fk_table + "\u001f" + target.fk_column; +} + +function tableCreateForeignKeyTargetLabel(target) { + return ( + target.fk_table + + "." + + target.fk_column + + " (" + + sqliteColumnTypeLabel(target.type) + + ")" + ); +} + +function tableCreateForeignKeyTargetsUrl() { + var data = databaseCreateTableData() || {}; + if (data.foreignKeyTargetsPath) { + return data.foreignKeyTargetsPath; + } + if (!data.path) { + return null; + } + return data.path.replace(/\/-\/create$/, "/-/foreign-key-targets"); +} + +function populateTableCreateForeignKeySelect(select, state, sourceType) { + var previousKey = select.value || select.dataset.selectedKey || ""; + select.textContent = ""; + + var blankOption = document.createElement("option"); + blankOption.value = ""; + blankOption.textContent = "- no foreign key -"; + select.appendChild(blankOption); + + if (state.foreignKeyTargetsLoading) { + var loadingOption = document.createElement("option"); + loadingOption.value = ""; + loadingOption.disabled = true; + loadingOption.textContent = "Loading foreign keys..."; + select.appendChild(loadingOption); + } else if (state.foreignKeyTargetsError) { + var errorOption = document.createElement("option"); + errorOption.value = ""; + errorOption.disabled = true; + errorOption.textContent = "Could not load foreign keys"; + select.appendChild(errorOption); + } else { + var sourceAffinity = tableCreateTypeAffinity(sourceType); + (state.foreignKeyTargets || []).forEach(function (target) { + if (!foreignKeyTypesCompatible(sourceAffinity, target.type)) { + return; + } + var optionElement = document.createElement("option"); + optionElement.value = tableCreateForeignKeyTargetKey(target); + optionElement.dataset.fkTable = target.fk_table; + optionElement.dataset.fkColumn = target.fk_column; + optionElement.textContent = tableCreateForeignKeyTargetLabel(target); + select.appendChild(optionElement); + }); + } + + select.value = previousKey; + if (select.value !== previousKey) { + select.value = ""; + } + select.dataset.selectedKey = select.value; + select.disabled = state.isSaving || select.options.length <= 1; + updateSelectPlaceholder(select, "table-create-input-placeholder"); +} + +function syncTableCreateForeignKeyOptions(row, state) { + var typeSelect = row.querySelector(".table-create-column-type"); + var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); + if (!typeSelect || !foreignKeySelect) { + return; + } + populateTableCreateForeignKeySelect( + foreignKeySelect, + state, + typeSelect.value, + ); +} + +function refreshTableCreateForeignKeyControls(state) { + tableCreateDialogRows(state).forEach(function (row, index) { + if (index > 0) { + syncTableCreateForeignKeyOptions(row, state); + } + }); +} + +function updateTableCreateColumnRules(state) { + tableCreateDialogRows(state).forEach(function (row, index) { + var isFirstColumn = index === 0; + var pkLabel = row.querySelector(".table-create-primary-key"); + var pkInput = row.querySelector(".table-create-primary-key-input"); + var foreignKeyField = row.querySelector(".table-create-foreign-key-field"); + var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); + + if (pkLabel && pkInput) { + pkLabel.hidden = !isFirstColumn; + if (!isFirstColumn) { + pkInput.checked = false; + } + } + + if (foreignKeyField && foreignKeySelect) { + foreignKeyField.hidden = isFirstColumn; + if (isFirstColumn) { + foreignKeySelect.value = ""; + foreignKeySelect.dataset.selectedKey = ""; + foreignKeySelect.disabled = true; + updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); + } else { + syncTableCreateForeignKeyOptions(row, state); + } + } + }); + updateTableCreateMoveButtons(state); +} + +async function loadTableCreateForeignKeyTargets(state) { + var url = tableCreateForeignKeyTargetsUrl(); + if (!url || !window.fetch) { + state.foreignKeyTargets = []; + state.foreignKeyTargetsLoading = false; + refreshTableCreateForeignKeyControls(state); + return; + } + state.foreignKeyTargets = []; + state.foreignKeyTargetsError = null; + state.foreignKeyTargetsLoading = true; + refreshTableCreateForeignKeyControls(state); + try { + var response = await fetch(url, { + headers: { + Accept: "application/json", + }, + }); + var data = await response.json(); + if (!response.ok || data.ok === false) { + throw rowMutationRequestError(response, data); + } + state.foreignKeyTargets = data.targets || []; + } catch (error) { + state.foreignKeyTargets = []; + state.foreignKeyTargetsError = error; + } finally { + state.foreignKeyTargetsLoading = false; + refreshTableCreateForeignKeyControls(state); + } +} + function tableCreateDialogSignature(state) { if (!state || !state.form) { return ""; } - var columns = []; - state.columnList - .querySelectorAll(".table-create-column-row") - .forEach(function (row) { - columns.push({ + return JSON.stringify({ + table: state.tableName.value, + columns: tableCreateDialogRows(state).map(function (row) { + return { name: row.querySelector(".table-create-column-name").value, type: row.querySelector(".table-create-column-type").value, customType: @@ -111,11 +509,14 @@ function tableCreateDialogSignature(state) { } ).value || "", pk: row.querySelector(".table-create-primary-key-input").checked, - }); - }); - return JSON.stringify({ - table: state.tableName.value, - columns: columns, + foreignKey: + ( + row.querySelector(".table-create-foreign-key-target") || { + value: "", + } + ).value || "", + }; + }), }); } @@ -151,42 +552,28 @@ function setTableCreateDialogSaving(state, isSaving) { .forEach(function (control) { control.disabled = isSaving; }); + if (!isSaving) { + updateTableCreateColumnRules(state); + } + updateTableCreateMoveButtons(state); } function tableCreateSelectTypeValue(select, type) { var options = tableCreateColumnTypes(); - options.forEach(function (option) { - var optionElement = document.createElement("option"); - optionElement.value = option; - optionElement.textContent = sqliteColumnTypeLabel(option); - select.appendChild(optionElement); - }); - select.value = options.indexOf(type) === -1 ? options[0] : type; + populateSqliteColumnTypeSelect(select, type, options); } function updateTableCreateCustomColumnTypePlaceholder(select) { - select.classList.toggle("table-create-input-placeholder", !select.value); + updateSelectPlaceholder(select, "table-create-input-placeholder"); } function createTableCustomColumnTypeSelect() { var options = tableCreateCustomColumnTypes(); - var select = document.createElement("select"); - select.className = "table-create-input table-create-custom-column-type"; - select.setAttribute("aria-label", "Custom column type"); - var blankOption = document.createElement("option"); - blankOption.value = ""; - blankOption.textContent = "- custom type -"; - select.appendChild(blankOption); - options.forEach(function (option) { - var optionElement = document.createElement("option"); - optionElement.value = option.name; - optionElement.textContent = option.description - ? option.description + " (" + option.name + ")" - : option.name; - select.appendChild(optionElement); - }); - updateTableCreateCustomColumnTypePlaceholder(select); - return select; + return createCustomColumnTypeSelect( + options, + "table-create-input table-create-custom-column-type", + "table-create-input-placeholder", + ); } function syncTableCreateCustomTypeForSqliteType(row) { @@ -209,6 +596,19 @@ function createTableColumnRow(state, column) { var row = document.createElement("div"); row.className = "table-create-column-row"; + var main = document.createElement("div"); + main.className = "table-create-column-main"; + + var details = document.createElement("div"); + details.className = "table-create-column-details"; + details.id = "table-create-column-details-" + index; + details.hidden = !(column && column.expanded); + + var expandButton = createSchemaDialogMoreOptionsButton( + "table-create", + details, + ); + var nameId = "table-create-column-name-" + index; var nameLabel = document.createElement("label"); nameLabel.className = "table-create-column-label"; @@ -228,40 +628,88 @@ function createTableColumnRow(state, column) { typeSelect.setAttribute("aria-label", "Column type"); tableCreateSelectTypeValue(typeSelect, column && column.type); - var customTypeSelect = createTableCustomColumnTypeSelect(); - if (column && column.customType) { - customTypeSelect.value = column.customType; + var customTypeSelect = null; + var customTypeField = null; + if (tableCreateCustomColumnTypes().length) { + var customTypeId = "table-create-column-custom-type-" + index; + customTypeField = document.createElement("div"); + customTypeField.className = + "table-create-detail-field table-create-custom-type-field"; + var customTypeLabel = document.createElement("label"); + customTypeLabel.className = "table-create-detail-label"; + customTypeLabel.setAttribute("for", customTypeId); + customTypeLabel.textContent = "Custom type"; + customTypeSelect = createTableCustomColumnTypeSelect(); + customTypeSelect.id = customTypeId; + customTypeSelect.value = column && column.customType ? column.customType : ""; + updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + customTypeField.appendChild(customTypeLabel); + customTypeField.appendChild(customTypeSelect); } - updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); var pkLabel = document.createElement("label"); - pkLabel.className = "table-create-primary-key"; + pkLabel.className = "table-create-detail-check table-create-primary-key"; var pkInput = document.createElement("input"); pkInput.type = "checkbox"; pkInput.className = "table-create-primary-key-input"; pkInput.checked = !!(column && column.primaryKey); var pkText = document.createElement("span"); - pkText.textContent = "PK"; - pkText.title = "Primary key"; + var pkStrong = document.createElement("strong"); + pkStrong.textContent = "Primary key"; + pkText.appendChild(pkStrong); + pkText.appendChild( + document.createTextNode(" This ID uniquely identifies the record"), + ); pkLabel.appendChild(pkInput); pkLabel.appendChild(pkText); - var removeButton = document.createElement("button"); - removeButton.type = "button"; - removeButton.className = "table-create-remove-column"; - removeButton.setAttribute("aria-label", "Remove column"); - removeButton.title = "Remove column"; - removeButton.innerHTML = - ''; + var foreignKeyId = "table-create-column-foreign-key-" + index; + var foreignKeyHelpId = "table-create-column-foreign-key-help-" + index; + var foreignKeyField = document.createElement("div"); + foreignKeyField.className = + "table-create-detail-field table-create-foreign-key-field"; + var foreignKeyLabel = document.createElement("label"); + foreignKeyLabel.className = "table-create-detail-label"; + foreignKeyLabel.setAttribute("for", foreignKeyId); + foreignKeyLabel.textContent = "Foreign key"; + var foreignKeyHelp = document.createElement("p"); + foreignKeyHelp.id = foreignKeyHelpId; + foreignKeyHelp.className = "table-create-detail-help"; + foreignKeyHelp.textContent = "Link this column to another table."; + var foreignKeySelect = document.createElement("select"); + foreignKeySelect.id = foreignKeyId; + foreignKeySelect.className = + "table-create-input table-create-foreign-key-target"; + foreignKeySelect.setAttribute("aria-label", "Foreign key target"); + foreignKeySelect.setAttribute("aria-describedby", foreignKeyHelpId); + foreignKeyField.appendChild(foreignKeyLabel); + foreignKeyField.appendChild(foreignKeyHelp); + foreignKeyField.appendChild(foreignKeySelect); - row.appendChild(nameLabel); - row.appendChild(nameInput); - row.appendChild(typeSelect); - if (tableCreateCustomColumnTypes().length) { - row.appendChild(customTypeSelect); + var moveControls = createSchemaDialogMoveControls("table-create"); + + var removeButton = createSchemaDialogIconButton( + "table-create", + "remove-column", + "Remove column", + "Remove column", + COLUMN_MOVE_ICONS.remove, + ); + + main.appendChild(nameLabel); + main.appendChild(nameInput); + main.appendChild(typeSelect); + main.appendChild(moveControls.controls); + main.appendChild(removeButton); + main.appendChild(expandButton); + + if (customTypeField) { + details.appendChild(customTypeField); } - row.appendChild(pkLabel); - row.appendChild(removeButton); + details.appendChild(foreignKeyField); + details.appendChild(pkLabel); + row.appendChild(main); + row.appendChild(details); removeButton.addEventListener("click", function () { if (state.isSaving) { @@ -269,6 +717,7 @@ function createTableColumnRow(state, column) { } row.remove(); clearTableCreateDialogError(state); + updateTableCreateColumnRules(state); var nextInput = state.columnList.querySelector(".table-create-column-name"); if (nextInput) { nextInput.focus(); @@ -283,21 +732,94 @@ function createTableColumnRow(state, column) { typeSelect.addEventListener("change", function () { clearTableCreateDialogError(state); syncTableCreateCustomTypeForSqliteType(row); + syncTableCreateForeignKeyOptions(row, state); }); - customTypeSelect.addEventListener("change", function () { - clearTableCreateDialogError(state); - updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); - var option = tableCreateCustomColumnType(customTypeSelect.value); - if ( - option && - option.fixedSqliteType && - tableCreateColumnTypes().indexOf(option.fixedSqliteType) !== -1 - ) { - typeSelect.value = option.fixedSqliteType; - } - }); + if (customTypeSelect) { + customTypeSelect.addEventListener("change", function () { + clearTableCreateDialogError(state); + updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + var option = tableCreateCustomColumnType(customTypeSelect.value); + if ( + option && + option.fixedSqliteType && + tableCreateColumnTypes().indexOf(option.fixedSqliteType) !== -1 + ) { + typeSelect.value = option.fixedSqliteType; + syncTableCreateForeignKeyOptions(row, state); + } + }); + } pkInput.addEventListener("change", function () { clearTableCreateDialogError(state); + updateTableCreateColumnRules(state); + }); + foreignKeySelect.addEventListener("change", function () { + foreignKeySelect.dataset.selectedKey = foreignKeySelect.value; + clearTableCreateDialogError(state); + updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); + }); + + expandButton.addEventListener("click", function () { + toggleSchemaDialogMoreOptions(expandButton, details); + }); + + moveControls.topButton.addEventListener("click", function () { + var first = tableCreateFirstNonPrimaryRow(state); + if ( + state.isSaving || + tableCreateRowIsPrimaryKey(row) || + !first || + first === row + ) { + return; + } + state.columnList.insertBefore(row, first); + clearTableCreateDialogError(state); + updateTableCreateColumnRules(state); + row.querySelector(".table-create-column-name").focus(); + }); + + moveControls.upButton.addEventListener("click", function () { + var previous = row.previousElementSibling; + if ( + state.isSaving || + tableCreateRowIsPrimaryKey(row) || + !previous || + tableCreateRowIsPrimaryKey(previous) + ) { + return; + } + state.columnList.insertBefore(row, previous); + clearTableCreateDialogError(state); + updateTableCreateColumnRules(state); + row.querySelector(".table-create-column-name").focus(); + }); + + moveControls.downButton.addEventListener("click", function () { + var next = row.nextElementSibling; + if (state.isSaving || tableCreateRowIsPrimaryKey(row) || !next) { + return; + } + state.columnList.insertBefore(next, row); + clearTableCreateDialogError(state); + updateTableCreateColumnRules(state); + row.querySelector(".table-create-column-name").focus(); + }); + + moveControls.bottomButton.addEventListener("click", function () { + var last = state.columnList.lastElementChild; + if ( + state.isSaving || + tableCreateRowIsPrimaryKey(row) || + !last || + last === row + ) { + return; + } + state.columnList.appendChild(row); + clearTableCreateDialogError(state); + updateTableCreateColumnRules(state); + row.querySelector(".table-create-column-name").focus(); }); return row; @@ -306,6 +828,7 @@ function createTableColumnRow(state, column) { function addTableCreateColumn(state, column) { var row = createTableColumnRow(state, column || { type: "text" }); state.columnList.appendChild(row); + updateTableCreateColumnRules(state); return row; } @@ -323,6 +846,7 @@ function resetTableCreateDialog(state) { type: "text", primaryKey: false, }); + updateTableCreateColumnRules(state); state.initialSignature = tableCreateDialogSignature(state); } @@ -332,16 +856,32 @@ function collectTableCreatePayload(state) { columns: [], }; var primaryKeys = []; - state.columnList - .querySelectorAll(".table-create-column-row") - .forEach(function (row) { - var name = row.querySelector(".table-create-column-name").value.trim(); - var type = row.querySelector(".table-create-column-type").value; - payload.columns.push({ name: name, type: type }); - if (row.querySelector(".table-create-primary-key-input").checked) { - primaryKeys.push(name); - } - }); + tableCreateDialogRows(state).forEach(function (row, index) { + var name = row.querySelector(".table-create-column-name").value.trim(); + var type = row.querySelector(".table-create-column-type").value; + var column = { name: name, type: type }; + var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); + var foreignKeyOption = + foreignKeySelect && foreignKeySelect.selectedOptions + ? foreignKeySelect.selectedOptions[0] + : null; + if ( + index > 0 && + foreignKeyOption && + foreignKeyOption.dataset.fkTable && + foreignKeyOption.dataset.fkColumn + ) { + column.fk_table = foreignKeyOption.dataset.fkTable; + column.fk_column = foreignKeyOption.dataset.fkColumn; + } + payload.columns.push(column); + if ( + index === 0 && + row.querySelector(".table-create-primary-key-input").checked + ) { + primaryKeys.push(name); + } + }); if (primaryKeys.length === 1) { payload.pk = primaryKeys[0]; } else if (primaryKeys.length > 1) { @@ -352,21 +892,19 @@ function collectTableCreatePayload(state) { function collectTableCreateColumnTypeAssignments(state) { var assignments = []; - state.columnList - .querySelectorAll(".table-create-column-row") - .forEach(function (row) { - var customTypeSelect = row.querySelector( - ".table-create-custom-column-type", - ); - if (!customTypeSelect || !customTypeSelect.value) { - return; - } - assignments.push({ - column: row.querySelector(".table-create-column-name").value.trim(), - columnType: customTypeSelect.value, - sqliteType: row.querySelector(".table-create-column-type").value, - }); + tableCreateDialogRows(state).forEach(function (row) { + var customTypeSelect = row.querySelector( + ".table-create-custom-column-type", + ); + if (!customTypeSelect || !customTypeSelect.value) { + return; + } + assignments.push({ + column: row.querySelector(".table-create-column-name").value.trim(), + columnType: customTypeSelect.value, + sqliteType: row.querySelector(".table-create-column-type").value, }); + }); return assignments; } @@ -604,9 +1142,14 @@ function ensureTableCreateDialog(manager) {
-
Columns
+
- +
@@ -633,6 +1176,9 @@ function ensureTableCreateDialog(manager) { isSaving: false, initialSignature: "", nextColumnIndex: 0, + foreignKeyTargets: [], + foreignKeyTargetsError: null, + foreignKeyTargetsLoading: false, manager: manager, }; @@ -713,6 +1259,7 @@ function openTableCreateDialog(button, manager) { state.title.textContent = "Create a table in " + data.databaseName; clearTableCreateDialogError(state); resetTableCreateDialog(state); + loadTableCreateForeignKeyTargets(state); if (!state.dialog.open) { state.dialog.showModal(); } @@ -877,9 +1424,7 @@ function tableAlterCustomTypeAppliesToSqliteType(option, sqliteType) { } function tableAlterDialogRows(state) { - return Array.prototype.slice.call( - state.columnList.querySelectorAll(".table-alter-column-row"), - ); + return schemaDialogRows(state, "table-alter"); } function tableAlterRowSignature(row) { @@ -930,76 +1475,19 @@ function updateTableAlterSaveButtonState(state) { } function tableAlterRowIsPrimaryKey(row) { - var input = row && row.querySelector(".table-alter-primary-key-input"); - return !!(input && input.checked); + return schemaDialogRowIsPrimaryKey(row, "table-alter"); } function tableAlterFirstNonPrimaryRow(state) { - var rows = tableAlterDialogRows(state); - for (var i = 0; i < rows.length; i += 1) { - if (!tableAlterRowIsPrimaryKey(rows[i])) { - return rows[i]; - } - } - return null; + return schemaDialogFirstNonPrimaryRow(state, "table-alter"); } function updateTableAlterMoveButtons(state) { - if (!state || !state.columnList) { - return; - } - var firstNonPrimary = tableAlterFirstNonPrimaryRow(state); - var rows = tableAlterDialogRows(state); - var hasPrimaryKeys = rows.some(function (row) { - return tableAlterRowIsPrimaryKey(row); - }); - var primaryKeyMoveTitle = "Primary key columns are always listed first"; - rows.forEach(function (row) { - var isPrimaryKey = tableAlterRowIsPrimaryKey(row); - var previous = row.previousElementSibling; - var next = row.nextElementSibling; - row - .querySelectorAll(".table-alter-move-controls button") - .forEach(function (button) { - button.title = button.dataset.defaultTitle || button.title; - button.disabled = state.isSaving || isPrimaryKey; - if (isPrimaryKey) { - button.title = primaryKeyMoveTitle; - } - }); - if (!isPrimaryKey) { - var topButton = row.querySelector(".table-alter-move-top"); - var upButton = row.querySelector(".table-alter-move-up"); - var downButton = row.querySelector(".table-alter-move-down"); - var bottomButton = row.querySelector(".table-alter-move-bottom"); - topButton.disabled = - state.isSaving || !firstNonPrimary || row === firstNonPrimary; - upButton.disabled = - state.isSaving || !previous || tableAlterRowIsPrimaryKey(previous); - downButton.disabled = state.isSaving || !next; - bottomButton.disabled = state.isSaving || !next; - if (hasPrimaryKeys && row === firstNonPrimary) { - topButton.title = primaryKeyMoveTitle; - upButton.title = primaryKeyMoveTitle; - } - } - }); + updateSchemaDialogMoveButtons(state, "table-alter"); } function normalizeTableAlterPrimaryKeyRows(state) { - var rows = tableAlterDialogRows(state); - rows - .filter(function (row) { - return tableAlterRowIsPrimaryKey(row); - }) - .concat( - rows.filter(function (row) { - return !tableAlterRowIsPrimaryKey(row); - }), - ) - .forEach(function (row) { - state.columnList.appendChild(row); - }); + normalizeSchemaDialogPrimaryKeyRows(state, "table-alter"); } function clearTableAlterDialogError(state) { @@ -1049,38 +1537,20 @@ function tableAlterSaveButtonText(state) { function tableAlterSelectTypeValue(select, type) { var options = tableAlterColumnTypes(); - options.forEach(function (option) { - var optionElement = document.createElement("option"); - optionElement.value = option; - optionElement.textContent = sqliteColumnTypeLabel(option); - select.appendChild(optionElement); - }); - select.value = options.indexOf(type) === -1 ? options[0] : type; + populateSqliteColumnTypeSelect(select, type, options); } function updateTableAlterCustomColumnTypePlaceholder(select) { - select.classList.toggle("table-alter-input-placeholder", !select.value); + updateSelectPlaceholder(select, "table-alter-input-placeholder"); } function createTableAlterCustomColumnTypeSelect() { var options = tableAlterCustomColumnTypes(); - var select = document.createElement("select"); - select.className = "table-alter-input table-alter-custom-column-type"; - select.setAttribute("aria-label", "Custom column type"); - var blankOption = document.createElement("option"); - blankOption.value = ""; - blankOption.textContent = "- custom type -"; - select.appendChild(blankOption); - options.forEach(function (option) { - var optionElement = document.createElement("option"); - optionElement.value = option.name; - optionElement.textContent = option.description - ? option.description + " (" + option.name + ")" - : option.name; - select.appendChild(optionElement); - }); - updateTableAlterCustomColumnTypePlaceholder(select); - return select; + return createCustomColumnTypeSelect( + options, + "table-alter-input table-alter-custom-column-type", + "table-alter-input-placeholder", + ); } function syncTableAlterCustomTypeForSqliteType(row) { @@ -1112,7 +1582,7 @@ function tableAlterSelectDefaultExprValue(select, value) { } function updateTableAlterDefaultExprPlaceholder(select) { - select.classList.toggle("table-alter-input-placeholder", !select.value); + updateSelectPlaceholder(select, "table-alter-input-placeholder"); } function syncTableAlterDefaultControls(row) { @@ -1159,22 +1629,10 @@ function createTableAlterColumnRow(state, column) { details.id = "table-alter-column-details-" + index; details.hidden = !(column && column.expanded); - var expandButton = document.createElement("button"); - expandButton.type = "button"; - expandButton.className = "table-alter-more-options"; - expandButton.setAttribute("aria-label", "Toggle column settings"); - expandButton.setAttribute("aria-controls", details.id); - expandButton.setAttribute("aria-expanded", details.hidden ? "false" : "true"); - function updateExpandButton() { - var isExpanded = expandButton.getAttribute("aria-expanded") === "true"; - expandButton.textContent = isExpanded - ? "v Hide options" - : "> Advanced options"; - expandButton.title = isExpanded - ? "Hide column settings" - : "Show column settings"; - } - updateExpandButton(); + var expandButton = createSchemaDialogMoreOptionsButton( + "table-alter", + details, + ); var nameId = "table-alter-column-name-" + index; var nameLabel = document.createElement("label"); @@ -1196,10 +1654,22 @@ function createTableAlterColumnRow(state, column) { tableAlterSelectTypeValue(typeSelect, column && column.type); var customTypeSelect = null; + var customTypeField = null; if (tableAlterCustomColumnTypes().length) { + var customTypeId = "table-alter-column-custom-type-" + index; + customTypeField = document.createElement("div"); + customTypeField.className = + "table-alter-detail-field table-alter-custom-type-field"; + var customTypeLabel = document.createElement("label"); + customTypeLabel.className = "table-alter-detail-label"; + customTypeLabel.setAttribute("for", customTypeId); + customTypeLabel.textContent = "Custom type"; customTypeSelect = createTableAlterCustomColumnTypeSelect(); + customTypeSelect.id = customTypeId; customTypeSelect.value = originalCustomType; updateTableAlterCustomColumnTypePlaceholder(customTypeSelect); + customTypeField.appendChild(customTypeLabel); + customTypeField.appendChild(customTypeSelect); } var notNullLabel = document.createElement("label"); @@ -1262,76 +1732,30 @@ function createTableAlterColumnRow(state, column) { pkStrong.textContent = "Primary key"; pkText.appendChild(pkStrong); pkText.appendChild( - document.createTextNode(" An ID that uniquely identifies this record"), + document.createTextNode(" This ID uniquely identifies the record"), ); pkLabel.appendChild(pkInput); pkLabel.appendChild(pkText); - var moveControls = document.createElement("div"); - moveControls.className = "table-alter-move-controls"; + var moveControls = createSchemaDialogMoveControls("table-alter"); - var moveTopButton = document.createElement("button"); - moveTopButton.type = "button"; - moveTopButton.className = "table-alter-icon-button table-alter-move-top"; - moveTopButton.setAttribute("aria-label", "Move column to top"); - moveTopButton.title = "Move column to top"; - moveTopButton.dataset.defaultTitle = moveTopButton.title; - moveTopButton.innerHTML = - ''; - - var moveUpButton = document.createElement("button"); - moveUpButton.type = "button"; - moveUpButton.className = "table-alter-icon-button table-alter-move-up"; - moveUpButton.setAttribute("aria-label", "Move column up"); - moveUpButton.title = "Move column up"; - moveUpButton.dataset.defaultTitle = moveUpButton.title; - moveUpButton.innerHTML = - ''; - - var moveDownButton = document.createElement("button"); - moveDownButton.type = "button"; - moveDownButton.className = "table-alter-icon-button table-alter-move-down"; - moveDownButton.setAttribute("aria-label", "Move column down"); - moveDownButton.title = "Move column down"; - moveDownButton.dataset.defaultTitle = moveDownButton.title; - moveDownButton.innerHTML = - ''; - - var moveBottomButton = document.createElement("button"); - moveBottomButton.type = "button"; - moveBottomButton.className = - "table-alter-icon-button table-alter-move-bottom"; - moveBottomButton.setAttribute("aria-label", "Move column to bottom"); - moveBottomButton.title = "Move column to bottom"; - moveBottomButton.dataset.defaultTitle = moveBottomButton.title; - moveBottomButton.innerHTML = - ''; - - var removeButton = document.createElement("button"); - removeButton.type = "button"; - removeButton.className = "table-alter-icon-button table-alter-remove-column"; - removeButton.setAttribute( - "aria-label", + var removeButton = createSchemaDialogIconButton( + "table-alter", + "remove-column", existing ? "Drop column" : "Remove column", + existing ? "Drop column" : "Remove column", + COLUMN_MOVE_ICONS.remove, ); - removeButton.title = existing ? "Drop column" : "Remove column"; - removeButton.innerHTML = - ''; - - moveControls.appendChild(moveTopButton); - moveControls.appendChild(moveUpButton); - moveControls.appendChild(moveDownButton); - moveControls.appendChild(moveBottomButton); main.appendChild(nameLabel); main.appendChild(nameInput); main.appendChild(typeSelect); - if (customTypeSelect) { - main.appendChild(customTypeSelect); - } - main.appendChild(moveControls); + main.appendChild(moveControls.controls); main.appendChild(removeButton); main.appendChild(expandButton); + if (customTypeField) { + details.appendChild(customTypeField); + } details.appendChild(notNullLabel); details.appendChild(defaultField); details.appendChild(defaultExprField); @@ -1382,10 +1806,7 @@ function createTableAlterColumnRow(state, column) { }); expandButton.addEventListener("click", function () { - var isExpanded = expandButton.getAttribute("aria-expanded") === "true"; - details.hidden = isExpanded; - expandButton.setAttribute("aria-expanded", isExpanded ? "false" : "true"); - updateExpandButton(); + toggleSchemaDialogMoreOptions(expandButton, details); }); typeSelect.addEventListener("change", function () { @@ -1407,7 +1828,7 @@ function createTableAlterColumnRow(state, column) { }); } - moveTopButton.addEventListener("click", function () { + moveControls.topButton.addEventListener("click", function () { var first = tableAlterFirstNonPrimaryRow(state); if ( state.isSaving || @@ -1424,7 +1845,7 @@ function createTableAlterColumnRow(state, column) { row.querySelector(".table-alter-column-name").focus(); }); - moveUpButton.addEventListener("click", function () { + moveControls.upButton.addEventListener("click", function () { var previous = row.previousElementSibling; if ( state.isSaving || @@ -1441,7 +1862,7 @@ function createTableAlterColumnRow(state, column) { row.querySelector(".table-alter-column-name").focus(); }); - moveDownButton.addEventListener("click", function () { + moveControls.downButton.addEventListener("click", function () { var next = row.nextElementSibling; if (state.isSaving || tableAlterRowIsPrimaryKey(row) || !next) { return; @@ -1453,7 +1874,7 @@ function createTableAlterColumnRow(state, column) { row.querySelector(".table-alter-column-name").focus(); }); - moveBottomButton.addEventListener("click", function () { + moveControls.bottomButton.addEventListener("click", function () { var last = state.columnList.lastElementChild; if ( state.isSaving || @@ -2145,12 +2566,11 @@ function ensureTableAlterDialog(manager) {
- +
From c77dad910b47c370c2b4c2cf16bebf3d74b51069 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Jun 2026 21:16:56 -0700 Subject: [PATCH 029/174] More robust test_datasette_https_server.sh test --- tests/test_datasette_https_server.sh | 31 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/test_datasette_https_server.sh b/tests/test_datasette_https_server.sh index aee262cc..a544b9a8 100755 --- a/tests/test_datasette_https_server.sh +++ b/tests/test_datasette_https_server.sh @@ -40,22 +40,23 @@ curl -f --cacert client.pem $test_url curl_exit_code=$? # Shut down the server -kill $server_pid -waiting=0 -# show all pids -# | find just the $server_pid -# | | don’t match on the previous grep -# | | | we don’t need the output -# | | | | -until ( ! ps ax | grep $server_pid | grep -v grep > /dev/null ); do - if [ $waiting -eq 4 ]; then - echo "$server_pid does still exist, server failed to stop" - cleanup - exit 1 +kill $server_pid 2>/dev/null || true +( + sleep 5 + if kill -0 $server_pid 2>/dev/null; then + kill -9 $server_pid 2>/dev/null || true fi - let waiting=waiting+1 - sleep 1 -done +) & +killer_pid=$! +wait_status=0 +wait $server_pid 2>/dev/null || wait_status=$? +kill $killer_pid 2>/dev/null || true +wait $killer_pid 2>/dev/null || true +if [ $wait_status -eq 137 ]; then + echo "$server_pid did not stop after SIGTERM, server failed to stop" + cleanup + exit 1 +fi # Clean up the certificates cleanup From e834008075eead04c142a57275dcaa3f08d44d38 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 19 Jun 2026 22:32:37 -0700 Subject: [PATCH 030/174] Make custom type and foreign key mutually exclusive In the create table dialog a column can now have either a custom display type or a foreign key target, but not both - a foreign key column's type is determined by the referenced primary key, so a custom type doesn't apply. Setting one clears and disables the other, and the foreign key select stays disabled on the primary key column and when no targets exist. Also add "Controls how Datasette displays and edits this column" help text (with aria-describedby) under the custom type selector in both the create and alter dialogs, and style the alter dialog help text. --- datasette/static/app.css | 3 +- datasette/static/edit-tools.js | 71 ++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 49f070b1..06919444 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1953,7 +1953,8 @@ dialog.table-create-dialog::backdrop { font-size: 0.72rem; } -.table-create-detail-help { +.table-create-detail-help, +.table-alter-detail-help { color: var(--muted); font-size: 0.82rem; line-height: 1.35; diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 5a9d0962..2a0a29b2 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -422,11 +422,42 @@ function syncTableCreateForeignKeyOptions(row, state) { ); } +function syncTableCreateCustomTypeAndForeignKey(row, state, isFirstColumn) { + var customTypeSelect = row.querySelector(".table-create-custom-column-type"); + var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); + if (!foreignKeySelect) { + return; + } + + var hasCustomType = customTypeSelect && !!customTypeSelect.value; + var hasForeignKey = !!foreignKeySelect.value; + + if (customTypeSelect && hasForeignKey) { + customTypeSelect.value = ""; + updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + hasCustomType = false; + } + + if (isFirstColumn || hasCustomType) { + foreignKeySelect.value = ""; + foreignKeySelect.dataset.selectedKey = ""; + updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); + hasForeignKey = false; + } + + if (customTypeSelect) { + customTypeSelect.disabled = state.isSaving; + } + foreignKeySelect.disabled = + state.isSaving || isFirstColumn || foreignKeySelect.options.length <= 1; +} + function refreshTableCreateForeignKeyControls(state) { tableCreateDialogRows(state).forEach(function (row, index) { if (index > 0) { syncTableCreateForeignKeyOptions(row, state); } + syncTableCreateCustomTypeAndForeignKey(row, state, index === 0); }); } @@ -455,6 +486,7 @@ function updateTableCreateColumnRules(state) { } else { syncTableCreateForeignKeyOptions(row, state); } + syncTableCreateCustomTypeAndForeignKey(row, state, isFirstColumn); } }); updateTableCreateMoveButtons(state); @@ -639,11 +671,19 @@ function createTableColumnRow(state, column) { customTypeLabel.className = "table-create-detail-label"; customTypeLabel.setAttribute("for", customTypeId); customTypeLabel.textContent = "Custom type"; + var customTypeHelpId = "table-create-column-custom-type-help-" + index; + var customTypeHelp = document.createElement("p"); + customTypeHelp.id = customTypeHelpId; + customTypeHelp.className = "table-create-detail-help"; + customTypeHelp.textContent = + "Controls how Datasette displays and edits this column"; customTypeSelect = createTableCustomColumnTypeSelect(); customTypeSelect.id = customTypeId; + customTypeSelect.setAttribute("aria-describedby", customTypeHelpId); customTypeSelect.value = column && column.customType ? column.customType : ""; updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); customTypeField.appendChild(customTypeLabel); + customTypeField.appendChild(customTypeHelp); customTypeField.appendChild(customTypeSelect); } @@ -733,11 +773,20 @@ function createTableColumnRow(state, column) { clearTableCreateDialogError(state); syncTableCreateCustomTypeForSqliteType(row); syncTableCreateForeignKeyOptions(row, state); + syncTableCreateCustomTypeAndForeignKey( + row, + state, + row === state.columnList.firstElementChild, + ); }); if (customTypeSelect) { customTypeSelect.addEventListener("change", function () { clearTableCreateDialogError(state); updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + if (customTypeSelect.value) { + foreignKeySelect.value = ""; + foreignKeySelect.dataset.selectedKey = ""; + } var option = tableCreateCustomColumnType(customTypeSelect.value); if ( option && @@ -747,6 +796,11 @@ function createTableColumnRow(state, column) { typeSelect.value = option.fixedSqliteType; syncTableCreateForeignKeyOptions(row, state); } + syncTableCreateCustomTypeAndForeignKey( + row, + state, + row === state.columnList.firstElementChild, + ); }); } pkInput.addEventListener("change", function () { @@ -757,6 +811,15 @@ function createTableColumnRow(state, column) { foreignKeySelect.dataset.selectedKey = foreignKeySelect.value; clearTableCreateDialogError(state); updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); + if (customTypeSelect && foreignKeySelect.value) { + customTypeSelect.value = ""; + updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + } + syncTableCreateCustomTypeAndForeignKey( + row, + state, + row === state.columnList.firstElementChild, + ); }); expandButton.addEventListener("click", function () { @@ -1664,11 +1727,19 @@ function createTableAlterColumnRow(state, column) { customTypeLabel.className = "table-alter-detail-label"; customTypeLabel.setAttribute("for", customTypeId); customTypeLabel.textContent = "Custom type"; + var customTypeHelpId = "table-alter-column-custom-type-help-" + index; + var customTypeHelp = document.createElement("p"); + customTypeHelp.id = customTypeHelpId; + customTypeHelp.className = "table-alter-detail-help"; + customTypeHelp.textContent = + "Controls how Datasette displays and edits this column"; customTypeSelect = createTableAlterCustomColumnTypeSelect(); customTypeSelect.id = customTypeId; + customTypeSelect.setAttribute("aria-describedby", customTypeHelpId); customTypeSelect.value = originalCustomType; updateTableAlterCustomColumnTypePlaceholder(customTypeSelect); customTypeField.appendChild(customTypeLabel); + customTypeField.appendChild(customTypeHelp); customTypeField.appendChild(customTypeSelect); } From 354780a1366bbfa6c5c9c34fdec3ccf24163e099 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 19 Jun 2026 22:40:14 -0700 Subject: [PATCH 031/174] Drop table button in alter dialog --- datasette/static/app.css | 16 ++++++++ datasette/static/edit-tools.js | 68 ++++++++++++++++++++++++++++++++++ datasette/views/table.py | 14 +++++++ tests/test_table_html.py | 14 ++++++- 4 files changed, 111 insertions(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 06919444..1cca8b80 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -2542,6 +2542,22 @@ dialog.table-alter-dialog::backdrop { color: var(--ink); } +.table-alter-dialog .btn-danger { + background: #b91c1c; + color: #fff; + margin-right: auto; +} + +.table-alter-dialog .btn-danger:hover { + background: #991b1b; +} + +.table-alter-dialog .btn-danger:disabled, +.table-alter-dialog .btn-danger:disabled:hover { + background: #d98c8c; + color: #fff; +} + .table-alter-dialog .btn-primary { background: var(--accent); color: #fff; diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 2a0a29b2..779ec702 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -1571,6 +1571,7 @@ function setTableAlterDialogSaving(state, isSaving) { state.cancelButton.disabled = isSaving; state.addColumnButton.disabled = isSaving; state.backButton.disabled = isSaving; + state.dropButton.disabled = isSaving; state.saveButton.textContent = isSaving ? state.mode === "review" ? "Applying..." @@ -2465,6 +2466,8 @@ function showTableAlterEditor(state) { state.review.hidden = true; state.review.textContent = ""; state.backButton.hidden = true; + var data = tableAlterData(); + state.dropButton.hidden = !(data && data.dropPath); state.saveButton.textContent = tableAlterSaveButtonText(state); updateTableAlterMoveButtons(state); updateTableAlterSaveButtonState(state); @@ -2479,6 +2482,7 @@ function showTableAlterReview(state, result) { state.review.hidden = false; state.review.textContent = ""; state.backButton.hidden = false; + state.dropButton.hidden = true; state.saveButton.textContent = tableAlterSaveButtonText(state); updateTableAlterSaveButtonState(state); @@ -2565,6 +2569,64 @@ async function applyTableAlterChanges(state, result) { } } +function tableAlterDatabaseUrl() { + var data = tableAlterData(); + if (!data || !data.path) { + return null; + } + var url = new URL(data.path, location.href); + url.pathname = url.pathname.replace(/\/[^/]+\/-\/alter\/?$/, ""); + url.search = ""; + url.hash = ""; + return url.toString(); +} + +async function dropTableFromAlterDialog(state) { + if (state.isSaving) { + return; + } + var data = tableAlterData(); + if (!data || !data.dropPath) { + return; + } + if ( + !window.confirm( + 'Permanently delete the table "' + + data.tableName + + '"? This will delete all of its data and cannot be undone.', + ) + ) { + return; + } + clearTableAlterDialogError(state); + setTableAlterDialogSaving(state, true); + try { + var response = await fetch(data.dropPath, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ confirm: true }), + }); + var responseData = null; + try { + responseData = await response.json(); + } catch (_error) { + responseData = null; + } + if (!response.ok || (responseData && responseData.ok === false)) { + throw rowMutationRequestError(response, responseData); + } + state.shouldRestoreFocus = false; + state.dialog.close(); + window.location.href = tableAlterDatabaseUrl() || "/"; + } catch (error) { + setTableAlterDialogSaving(state, false); + showTableAlterDialogError(state, error.message || "Could not drop table"); + } +} + async function saveTableAlterDialog(state) { if (state.isSaving) { return; @@ -2646,6 +2708,7 @@ function ensureTableAlterDialog(manager) {
+ @@ -2664,6 +2727,7 @@ function ensureTableAlterDialog(manager) { columnList: dialog.querySelector(".table-alter-column-list"), addColumnButton: dialog.querySelector(".table-alter-add-column"), backButton: dialog.querySelector(".table-alter-back"), + dropButton: dialog.querySelector(".table-alter-drop"), cancelButton: dialog.querySelector(".table-alter-cancel"), saveButton: dialog.querySelector(".table-alter-save"), currentButton: null, @@ -2703,6 +2767,10 @@ function ensureTableAlterDialog(manager) { closeTableAlterDialog(tableAlterDialogState); }); + tableAlterDialogState.dropButton.addEventListener("click", function () { + dropTableFromAlterDialog(tableAlterDialogState); + }); + tableAlterDialogState.backButton.addEventListener("click", function () { if (tableAlterDialogState.isSaving) { return; diff --git a/datasette/views/table.py b/datasette/views/table.py index 6c0d2914..4adcd181 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -422,6 +422,15 @@ async def _table_alter_ui( data["customColumnTypes"] = _custom_column_type_options_for_create_table( datasette ) + can_drop_table = await datasette.allowed( + action="drop-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ) + if can_drop_table: + data["dropPath"] = "{}/-/drop".format( + datasette.urls.table(database_name, table_name) + ) return data @@ -1193,6 +1202,11 @@ class TableDropView(BaseView): actor=request.actor, database=database_name, table=table_name ) ) + self.ds.add_message( + request, + "Table {} dropped".format(table_name), + self.ds.WARNING, + ) return Response.json({"ok": True}, status=200) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index fa01c8ec..4c80691c 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1089,8 +1089,9 @@ async def test_table_alter_action_button_and_data(): "tables": { "items": { "permissions": { - "alter-table": {"id": "root"}, + "alter-table": {"id": ["root", "alter-only"]}, "set-column-type": {"id": "root"}, + "drop-table": {"id": "root"}, }, "column_types": {"name": "textarea"}, }, @@ -1147,6 +1148,7 @@ async def test_table_alter_action_button_and_data(): "textarea", "url", ] + assert alter_data["dropPath"] == "/data/items/-/drop" assert alter_data["columns"] == [ { "name": "id", @@ -1192,6 +1194,16 @@ async def test_table_alter_action_button_and_data(): is None ) assert "alterTable" not in table_data_from_soup(soup_without_permission) + + # An actor that can alter but not drop should not get a dropPath + response_alter_only = await ds.client.get( + "/data/items", actor={"id": "alter-only"} + ) + assert response_alter_only.status_code == 200 + alter_only_data = table_data_from_soup( + Soup(response_alter_only.text, "html.parser") + )["alterTable"] + assert "dropPath" not in alter_only_data finally: ds.close() From a87c4ac555ed763b4e4e48716a7d4402dd439b99 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 09:47:39 -0700 Subject: [PATCH 032/174] Object not chain of ifs Refs https://github.com/simonw/datasette/pull/2789/changes#r3453964430 --- datasette/static/edit-tools.js | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 779ec702..4c6d7d10 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -60,17 +60,14 @@ function tableCreateColumnTypes() { : ["text", "integer", "float", "blob"]; } +var SQLITE_COLUMN_TYPE_LABELS = { + float: "floating point number", + real: "floating point number", + blob: "blob - binary data", +}; + function sqliteColumnTypeLabel(type) { - if (type === "float") { - return "floating point number"; - } - if (type === "real") { - return "floating point number"; - } - if (type === "blob") { - return "blob - binary data"; - } - return type; + return SQLITE_COLUMN_TYPE_LABELS[type] || type; } function populateSqliteColumnTypeSelect(select, type, options) { From b02999bca6a4cf9aa3712b3ba424f42b0cbee35b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 09:59:29 -0700 Subject: [PATCH 033/174] Fix broken Playwright tests --- tests/test_playwright.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 70b5a8c1..94dcbd64 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -303,6 +303,7 @@ def test_create_table_flow(page, datasette_server): dialog.locator(".table-create-add-column").click() dialog.locator(".table-create-column-name").nth(3).fill("metadata") dialog.locator(".table-create-column-type").nth(3).select_option("integer") + dialog.locator(".table-create-more-options").nth(3).click() dialog.locator(".table-create-custom-column-type").nth(3).select_option("json") assert dialog.locator(".table-create-column-type").nth(3).input_value() == "text" assert "table-create-input-placeholder" not in dialog.locator( @@ -361,7 +362,7 @@ def test_alter_table_flow(page, datasette_server): assert "Default value" in expanded_options_text assert "or default to a specific time" in expanded_options_text assert "Primary key" in expanded_options_text - assert "An ID that uniquely identifies this record" in expanded_options_text + assert "This ID uniquely identifies the record" in expanded_options_text dialog.locator(".table-alter-add-column").click() assert dialog.locator(".table-alter-save").is_enabled() From 17876ccf45b35a025c5fa0186427685629b66492 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 10:01:39 -0700 Subject: [PATCH 034/174] Keyword arguments for readability --- datasette/views/row.py | 16 ++++++++-------- datasette/views/table.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index 34773d48..e3575b6c 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -205,14 +205,14 @@ class RowView(DataView): ], "row_mutation_ui": any(row_action_permissions.values()), "table_page_data": await _table_page_data( - self.ds, - request, - db, - database, - table, - not is_table, - None, - None, + datasette=self.ds, + request=request, + db=db, + database_name=database, + table_name=table, + is_view=not is_table, + table_insert_ui=None, + table_alter_ui=None, ), "row_actions": row_actions, "top_row": make_slot_function( diff --git a/datasette/views/table.py b/datasette/views/table.py index 4adcd181..988d3808 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -2171,14 +2171,14 @@ async def table_view_data( data["table_insert_ui"] = table_insert_ui data["table_alter_ui"] = table_alter_ui data["table_page_data"] = await _table_page_data( - datasette, - request, - db, - database_name, - table_name, - is_view, - table_insert_ui, - table_alter_ui, + datasette=datasette, + request=request, + db=db, + database_name=database_name, + table_name=table_name, + is_view=is_view, + table_insert_ui=table_insert_ui, + table_alter_ui=table_alter_ui, ) return data, rows[:page_size], columns, expanded_columns, sql, next_url From c4aead65ee5a37a40f9fa8abb9fc45a03979e535 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 10:02:42 -0700 Subject: [PATCH 035/174] Ran prettier --- datasette/static/edit-tools.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 4c6d7d10..c4982e54 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -248,7 +248,9 @@ function updateSchemaDialogMoveButtons(state, prefix) { topButton.disabled = state.isSaving || !firstNonPrimary || row === firstNonPrimary; upButton.disabled = - state.isSaving || !previous || schemaDialogRowIsPrimaryKey(previous, prefix); + state.isSaving || + !previous || + schemaDialogRowIsPrimaryKey(previous, prefix); downButton.disabled = state.isSaving || !next; bottomButton.disabled = state.isSaving || !next; if (hasPrimaryKeys && row === firstNonPrimary) { @@ -464,7 +466,9 @@ function updateTableCreateColumnRules(state) { var pkLabel = row.querySelector(".table-create-primary-key"); var pkInput = row.querySelector(".table-create-primary-key-input"); var foreignKeyField = row.querySelector(".table-create-foreign-key-field"); - var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); + var foreignKeySelect = row.querySelector( + ".table-create-foreign-key-target", + ); if (pkLabel && pkInput) { pkLabel.hidden = !isFirstColumn; @@ -677,7 +681,8 @@ function createTableColumnRow(state, column) { customTypeSelect = createTableCustomColumnTypeSelect(); customTypeSelect.id = customTypeId; customTypeSelect.setAttribute("aria-describedby", customTypeHelpId); - customTypeSelect.value = column && column.customType ? column.customType : ""; + customTypeSelect.value = + column && column.customType ? column.customType : ""; updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); customTypeField.appendChild(customTypeLabel); customTypeField.appendChild(customTypeHelp); @@ -920,7 +925,9 @@ function collectTableCreatePayload(state) { var name = row.querySelector(".table-create-column-name").value.trim(); var type = row.querySelector(".table-create-column-type").value; var column = { name: name, type: type }; - var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); + var foreignKeySelect = row.querySelector( + ".table-create-foreign-key-target", + ); var foreignKeyOption = foreignKeySelect && foreignKeySelect.selectedOptions ? foreignKeySelect.selectedOptions[0] From 084df1fba2076eec2124309360e05bfa2a497a28 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 10:18:01 -0700 Subject: [PATCH 036/174] Removed the alter table dry run feature It works by doing conn.backup(memory_conn) which could use a lot of memory for a large database. --- datasette/views/table_create_alter.py | 17 ++---------- docs/json_api.rst | 5 +--- tests/test_api_write.py | 37 +++++++-------------------- 3 files changed, 12 insertions(+), 47 deletions(-) diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index f7775479..88bdafc4 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -22,7 +22,6 @@ from datasette.resources import DatabaseResource, TableResource from datasette.utils import ( escape_sqlite, get_outbound_foreign_keys, - sqlite3, table_column_details, ) from datasette.utils.asgi import NotFound, Response @@ -602,7 +601,6 @@ AlterTableOperation = Annotated[ class AlterTableRequest(_StrictPydanticModel): operations: list[AlterTableOperation] = Field(min_length=1) - dry_run: bool = False def _pydantic_errors(validation_error): @@ -1143,14 +1141,6 @@ class TableAlterView(BaseView): return _table_schema_from_conn(operation_conn, table_name) - if alter_request.dry_run: - memory_conn = sqlite3.connect(":memory:") - try: - conn.backup(memory_conn) - return before_schema, apply_operations(memory_conn) - finally: - memory_conn.close() - after_schema = apply_operations(conn) return before_schema, after_schema @@ -1162,7 +1152,7 @@ class TableAlterView(BaseView): return _error([str(e)], 400) altered = before_schema != after_schema - if altered and not alter_request.dry_run: + if altered: await self.ds.track_event( AlterTableEvent( request.actor, @@ -1189,10 +1179,7 @@ class TableAlterView(BaseView): "altered": altered, "schema": after_schema, "before_schema": before_schema, - "operations_applied": ( - 0 if alter_request.dry_run else len(alter_request.operations) - ), - "dry_run": alter_request.dry_run, + "operations_applied": len(alter_request.operations), }, status=200, ) diff --git a/docs/json_api.rst b/docs/json_api.rst index 1db46dd2..a42175f7 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2285,8 +2285,6 @@ The request body should include an ``operations`` array. Each operation has the ] } -Set ``"dry_run": true`` to validate the operations and return the schema that would be created without modifying the table. - Supported operations: * ``add_column`` adds a new column. ``args`` accepts ``name``, optional ``type`` of ``text``, ``integer``, ``float`` or ``blob``, optional ``not_null``, optional literal ``default`` and optional ``default_expr``. If ``not_null`` is ``true`` either a non-null ``default`` or ``default_expr`` is required. @@ -2316,8 +2314,7 @@ A successful response returns the new schema and the previous schema: "altered": true, "schema": "CREATE TABLE ...", "before_schema": "CREATE TABLE ...", - "operations_applied": 7, - "dry_run": false + "operations_applied": 7 } Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 1d16ad26..49f38cde 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -898,34 +898,6 @@ async def test_alter_table_operations(ds_write): assert event.after_schema == data["schema"] -@pytest.mark.asyncio -async def test_alter_table_dry_run(ds_write): - token = write_token(ds_write, permissions=["at"]) - db = ds_write.get_database("data") - response = await ds_write.client.post( - "/data/docs/-/alter", - json={ - "dry_run": True, - "operations": [ - {"op": "add_column", "args": {"name": "slug", "type": "text"}} - ], - }, - headers=_headers(token), - ) - assert response.status_code == 200, response.text - data = response.json() - assert data["ok"] is True - assert data["dry_run"] is True - assert data["altered"] is True - assert data["operations_applied"] == 0 - assert "slug" in data["schema"] - columns = ( - await db.execute("select name from pragma_table_info('docs') order by cid") - ).dicts() - assert [column["name"] for column in columns] == ["id", "title", "score", "age"] - assert last_event(ds_write) is None - - @pytest.mark.asyncio async def test_alter_table_foreign_key_operations(ds_write): token = write_token(ds_write, permissions=["at"]) @@ -1246,6 +1218,15 @@ async def test_alter_table_permission_denied(ds_write): @pytest.mark.parametrize( "body,expected_error", ( + ( + { + "dry_run": True, + "operations": [ + {"op": "add_column", "args": {"name": "slug", "type": "text"}} + ], + }, + "dry_run: Extra inputs are not permitted", + ), ( {"operations": [{"op": "add_column", "args": {"type": "text"}}]}, "operations.0.add_column.args.name: Field required", From a2e75967ce19445d2862ac40cd9306447f2b4524 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 11:03:49 -0700 Subject: [PATCH 037/174] sqlite-utils>=3.30,<4.0 So we don't get test failures from reformatted SQL. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 38776b2c..215b2cca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "PyYAML>=5.3", "mergedeep>=1.1.1", "itsdangerous>=1.1", - "sqlite-utils>=3.30", + "sqlite-utils>=3.30,<4.0", "asyncinject>=0.7", "setuptools", "pip", From 87354cf94e9921262d1fb50dfb9b25afd0cc028f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 11:04:19 -0700 Subject: [PATCH 038/174] not_null, default and default_exr support for create table API columns --- datasette/views/table_create_alter.py | 53 ++++++++++++------- docs/json_api.rst | 14 ++++- tests/test_api_write.py | 73 ++++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 21 deletions(-) diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 88bdafc4..d4259287 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -319,13 +319,28 @@ class _StrictPydanticModel(BaseModel): model_config = ConfigDict(extra="forbid") -class CreateTableColumn(BaseModel): - model_config = ConfigDict(extra="forbid") +class _DefaultArgsMixin(_StrictPydanticModel): + default: Any | None = None + default_expr: DefaultExpr | None = None + + @model_validator(mode="after") + def validate_default_fields(self): + has_default = "default" in self.model_fields_set + has_default_expr = "default_expr" in self.model_fields_set + if has_default and has_default_expr: + raise ValueError("default and default_expr cannot both be provided") + if has_default_expr and self.default_expr is None: + raise ValueError("default_expr cannot be null") + return self + + +class CreateTableColumn(_DefaultArgsMixin): name: Any = None type: Any = "text" fk_table: str | None = None fk_column: str | None = None + not_null: bool = False @model_validator(mode="after") def validate_column(self): @@ -450,21 +465,6 @@ class CreateTableRequest(_StrictPydanticModel): return foreign_keys or None -class _DefaultArgsMixin(_StrictPydanticModel): - default: Any | None = None - default_expr: DefaultExpr | None = None - - @model_validator(mode="after") - def validate_default_fields(self): - has_default = "default" in self.model_fields_set - has_default_expr = "default_expr" in self.model_fields_set - if has_default and has_default_expr: - raise ValueError("default and default_expr cannot both be provided") - if has_default_expr and self.default_expr is None: - raise ValueError("default_expr cannot be null") - return self - - class AddColumnArgs(_DefaultArgsMixin): name: str type: SqliteApiType = "text" @@ -753,16 +753,33 @@ class TableCreateView(BaseView): ) def create_table(conn): - table = sqlite_utils.Database(conn)[table_name] + db_for_write = sqlite_utils.Database(conn) + table = db_for_write[table_name] if rows: table.insert_all( rows, pk=pks or pk, ignore=ignore, replace=replace, alter=alter ) else: + not_null = [column.name for column in columns if column.not_null] + defaults = {} + for column in columns: + if "default_expr" in column.model_fields_set: + defaults[column.name] = _default_expression_sql( + column.default_expr + ) + elif ( + "default" in column.model_fields_set + and column.default is not None + ): + defaults[column.name] = _literal_default( + db_for_write, column.default + ) table.create( {column.name: column.type for column in columns}, pk=pks or pk, foreign_keys=create_request.foreign_keys, + not_null=not_null or None, + defaults=defaults or None, ) return table.schema diff --git a/docs/json_api.rst b/docs/json_api.rst index a42175f7..b2927e38 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -1968,7 +1968,14 @@ To create a table, make a ``POST`` to ``//-/create``. This requires th }, { "name": "title", - "type": "text" + "type": "text", + "not_null": true, + "default": "Untitled" + }, + { + "name": "created", + "type": "text", + "default_expr": "current_timestamp" } ], "pk": "id" @@ -1981,6 +1988,9 @@ The JSON here describes the table that will be created: - ``name`` is the name of the column. This is required. - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. + - ``not_null`` can be set to ``true`` to create this column with a ``NOT NULL`` constraint. + - ``default`` can be used to set a literal default value for this column. + - ``default_expr`` can be used instead of ``default`` to set a SQLite default expression. The supported values are ``current_timestamp``, ``current_date`` and ``current_time``. - ``fk_table`` can be used to create a single-column foreign key constraint referencing another table. ``fk_column`` is optional and can be used to specify the referenced column - if omitted, Datasette will use the single primary key of ``fk_table``. * ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. @@ -2028,7 +2038,7 @@ If the table is successfully created this will return a ``201`` status code and "table": "name_of_new_table", "table_url": "http://127.0.0.1:8001/data/name_of_new_table", "table_api_url": "http://127.0.0.1:8001/data/name_of_new_table.json", - "schema": "CREATE TABLE [name_of_new_table] (\n [id] INTEGER PRIMARY KEY,\n [title] TEXT\n)" + "schema": "CREATE TABLE [name_of_new_table] (\n [id] INTEGER PRIMARY KEY,\n [title] TEXT NOT NULL DEFAULT 'Untitled',\n [created] TEXT DEFAULT CURRENT_TIMESTAMP\n)" } .. _TableCreateView_example: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 49f38cde..32be8905 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1935,6 +1935,60 @@ async def test_create_table_with_foreign_key(ds_write): assert "[owner_id] INTEGER REFERENCES [owners]([id])" in data["schema"] +@pytest.mark.asyncio +async def test_create_table_with_column_constraints(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "constrained", + "columns": [ + {"name": "id", "type": "integer"}, + { + "name": "title", + "type": "text", + "not_null": True, + "default": "Untitled", + }, + { + "name": "created", + "type": "text", + "default_expr": "current_timestamp", + }, + {"name": "score", "type": "integer", "default": 0}, + {"name": "literal_default", "type": "text", "default": "hello)"}, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201, response.text + data = response.json() + assert data["ok"] is True + assert "NOT NULL DEFAULT 'Untitled'" in data["schema"] + assert "DEFAULT CURRENT_TIMESTAMP" in data["schema"] + assert "DEFAULT 0" in data["schema"] + assert "DEFAULT 'hello)'" in data["schema"] + + db = ds_write.get_database("data") + columns = ( + await db.execute("select * from pragma_table_info('constrained') order by cid") + ).dicts() + assert [column["name"] for column in columns] == [ + "id", + "title", + "created", + "score", + "literal_default", + ] + assert columns[0]["pk"] == 1 + assert columns[1]["notnull"] == 1 + assert columns[1]["dflt_value"] == "'Untitled'" + assert columns[2]["dflt_value"] == "CURRENT_TIMESTAMP" + assert columns[3]["dflt_value"] == "0" + assert columns[4]["dflt_value"] == "'hello)'" + + @pytest.mark.asyncio @pytest.mark.parametrize( "column,expected_error", @@ -1947,9 +2001,26 @@ async def test_create_table_with_foreign_key(ds_write): {"name": "owner_id", "type": "integer", "fk_column": "id"}, "columns.0: fk_column requires fk_table", ), + ( + { + "name": "created", + "type": "text", + "default_expr": "datetime('now')", + }, + "columns.0.default_expr: Input should be 'current_timestamp', 'current_date' or 'current_time'", + ), + ( + { + "name": "created", + "type": "text", + "default": "x", + "default_expr": "current_timestamp", + }, + "columns.0: Value error, default and default_expr cannot both be provided", + ), ), ) -async def test_create_table_foreign_key_validation(ds_write, column, expected_error): +async def test_create_table_column_validation(ds_write, column, expected_error): token = write_token(ds_write) response = await ds_write.client.post( "/data/-/create", From 4b219be8bd14b445b1a48206050d2c34a6943185 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 12:02:51 -0700 Subject: [PATCH 039/174] Alter table API can now rename tables, refs #2788 Refs https://github.com/simonw/datasette/pull/2789#issuecomment-4771774289 --- datasette/views/table_create_alter.py | 53 +++++++++++++++++++++++---- docs/json_api.rst | 17 ++++++--- tests/test_api_write.py | 53 +++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 13 deletions(-) diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index d4259287..07f13e9d 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -476,6 +476,20 @@ class RenameColumnArgs(_StrictPydanticModel): to: str +class RenameTableArgs(_StrictPydanticModel): + to: str + + @field_validator("to") + @classmethod + def validate_table_name(cls, v): + if not TABLE_NAME_RE.match(v): + raise PydanticCustomError( + "alter_table_rename_table", + "Invalid table name", + ) + return v + + class AlterColumnArgs(_DefaultArgsMixin): name: str type: SqliteApiType | None = None @@ -548,6 +562,11 @@ class RenameColumnOperation(_StrictPydanticModel): args: RenameColumnArgs +class RenameTableOperation(_StrictPydanticModel): + op: Literal["rename_table"] + args: RenameTableArgs + + class AlterColumnOperation(_StrictPydanticModel): op: Literal["alter_column"] args: AlterColumnArgs @@ -587,6 +606,7 @@ AlterTableOperation = Annotated[ Union[ AddColumnOperation, RenameColumnOperation, + RenameTableOperation, AlterColumnOperation, DropColumnOperation, SetPrimaryKeyOperation, @@ -1042,10 +1062,12 @@ class TableAlterView(BaseView): def apply_operations(operation_conn): db_for_write = sqlite_utils.Database(operation_conn) table = db_for_write[table_name] + current_table_name = table_name add_columns = [] types = {} rename = {} + rename_table_to = None drop = set() not_null = {} defaults = {} @@ -1080,6 +1102,8 @@ class TableAlterView(BaseView): defaults[args.name] = _default_expression_sql( args.default_expr ) + elif operation.op == "rename_table": + rename_table_to = args.to elif operation.op == "rename_column": rename[args.name] = args.to elif operation.op == "alter_column": @@ -1155,14 +1179,27 @@ class TableAlterView(BaseView): drop_foreign_keys=drop_foreign_keys or None, foreign_keys=foreign_keys, ) + if ( + rename_table_to is not None + and rename_table_to != current_table_name + ): + operation_conn.execute( + "alter table {} rename to {}".format( + escape_sqlite(current_table_name), + escape_sqlite(rename_table_to), + ) + ) + current_table_name = rename_table_to - return _table_schema_from_conn(operation_conn, table_name) + return current_table_name, _table_schema_from_conn( + operation_conn, current_table_name + ) - after_schema = apply_operations(conn) - return before_schema, after_schema + after_table_name, after_schema = apply_operations(conn) + return before_schema, after_schema, after_table_name try: - before_schema, after_schema = await db.execute_write_fn( + before_schema, after_schema, after_table_name = await db.execute_write_fn( alter_table, request=request ) except Exception as e: @@ -1174,23 +1211,23 @@ class TableAlterView(BaseView): AlterTableEvent( request.actor, database=database_name, - table=table_name, + table=after_table_name, before_schema=before_schema, after_schema=after_schema, ) ) table_url = self.ds.absolute_url( - request, self.ds.urls.table(database_name, table_name) + request, self.ds.urls.table(database_name, after_table_name) ) table_api_url = self.ds.absolute_url( - request, self.ds.urls.table(database_name, table_name, format="json") + request, self.ds.urls.table(database_name, after_table_name, format="json") ) return Response.json( { "ok": True, "database": database_name, - "table": table_name, + "table": after_table_name, "table_url": table_url, "table_api_url": table_api_url, "altered": altered, diff --git a/docs/json_api.rst b/docs/json_api.rst index b2927e38..849a8acf 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2242,6 +2242,12 @@ The request body should include an ``operations`` array. Each operation has the "to": "headline" } }, + { + "op": "rename_table", + "args": { + "to": "published_posts" + } + }, { "op": "alter_column", "args": { @@ -2299,6 +2305,7 @@ Supported operations: * ``add_column`` adds a new column. ``args`` accepts ``name``, optional ``type`` of ``text``, ``integer``, ``float`` or ``blob``, optional ``not_null``, optional literal ``default`` and optional ``default_expr``. If ``not_null`` is ``true`` either a non-null ``default`` or ``default_expr`` is required. * ``rename_column`` renames a column. ``args`` accepts ``name`` and ``to``. +* ``rename_table`` renames the table. ``args`` accepts ``to``, the new table name. If combined with other operations, Datasette applies the column, primary key, foreign key and column order changes before renaming the table. * ``alter_column`` changes column properties. ``args`` accepts ``name`` and at least one of ``type``, ``not_null``, literal ``default`` or ``default_expr``. Passing ``"default": null`` removes an existing default. * ``drop_column`` drops a column. ``args`` accepts ``name``. * ``set_primary_key`` changes the table primary key. ``args`` accepts ``columns``, a list of one or more column names. @@ -2311,20 +2318,20 @@ Supported operations: For foreign key operations that omit ``fk_column``, the referenced ``fk_table`` must have a single-column primary key. Datasette will return an error if it cannot identify a single primary key column for that table. -A successful response returns the new schema and the previous schema: +A successful response returns the new schema and the previous schema. If the request used ``rename_table``, ``table``, ``table_url`` and ``table_api_url`` will use the new table name. Renaming a table through this endpoint triggers the :class:`~datasette.events.RenameTableEvent` event. .. code-block:: json { "ok": true, "database": "data", - "table": "posts", - "table_url": "http://127.0.0.1:8001/data/posts", - "table_api_url": "http://127.0.0.1:8001/data/posts.json", + "table": "published_posts", + "table_url": "http://127.0.0.1:8001/data/published_posts", + "table_api_url": "http://127.0.0.1:8001/data/published_posts.json", "altered": true, "schema": "CREATE TABLE ...", "before_schema": "CREATE TABLE ...", - "operations_applied": 7 + "operations_applied": 11 } Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 32be8905..ca8a46f5 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1,4 +1,5 @@ from datasette.app import Datasette +from datasette.events import RenameTableEvent from datasette.utils import sqlite3 from .utils import last_event import pytest @@ -898,6 +899,58 @@ async def test_alter_table_operations(ds_write): assert event.after_schema == data["schema"] +@pytest.mark.asyncio +async def test_alter_table_rename_table(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + before_schema = await db.execute_fn( + lambda conn: conn.execute( + "select sql from sqlite_master where type = 'table' and name = 'docs'" + ).fetchone()[0] + ) + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + {"op": "rename_table", "args": {"to": "documents"}}, + ] + }, + headers=_headers(token), + ) + + assert response.status_code == 200, response.text + data = response.json() + assert data["ok"] is True + assert data["database"] == "data" + assert data["table"] == "documents" + assert data["table_url"].endswith("/data/documents") + assert data["table_api_url"].endswith("/data/documents.json") + assert data["altered"] is True + assert data["operations_applied"] == 1 + assert data["before_schema"] == before_schema + assert 'CREATE TABLE "documents"' in data["schema"] + + tables = ( + await db.execute( + "select name from sqlite_master where type = 'table' order by name" + ) + ).dicts() + table_names = [table["name"] for table in tables] + assert "docs" not in table_names + assert "documents" in table_names + + rename_events = [ + event + for event in ds_write._tracked_events + if isinstance(event, RenameTableEvent) + ] + assert len(rename_events) == 1 + assert rename_events[0].database == "data" + assert rename_events[0].old_table == "docs" + assert rename_events[0].new_table == "documents" + + @pytest.mark.asyncio async def test_alter_table_foreign_key_operations(ds_write): token = write_token(ds_write, permissions=["at"]) From dada4de172c974f677c51855e8234ffb22c6259a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 12:35:34 -0700 Subject: [PATCH 040/174] Fix for Safari select box heights Refs https://github.com/simonw/datasette/pull/2789#issuecomment-4772241681 --- datasette/static/app.css | 12 ++++++++++++ tests/test_playwright.py | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index 1cca8b80..90d7e7e9 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1864,12 +1864,18 @@ dialog.table-create-dialog::backdrop { .table-create-input { box-sizing: border-box; min-width: 0; + min-height: 46px; border: 1px solid var(--rule); border-radius: 5px; padding: 8px 10px; color: var(--ink); background: #fff; font: inherit; + line-height: 1.35; +} + +select.table-create-input { + height: 46px; } .table-create-input-placeholder { @@ -2333,12 +2339,18 @@ dialog.table-alter-dialog::backdrop { .table-alter-input { box-sizing: border-box; min-width: 0; + min-height: 46px; border: 1px solid var(--rule); border-radius: 5px; padding: 8px 10px; color: var(--ink); background: #fff; font: inherit; + line-height: 1.35; +} + +select.table-alter-input { + height: 46px; } .table-alter-input-placeholder { diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 94dcbd64..eefec118 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -295,6 +295,15 @@ def test_create_table_flow(page, datasette_server): placeholder_select.locator("option:checked").inner_text() == "- custom type -" ) assert "table-create-input-placeholder" in placeholder_select.get_attribute("class") + assert dialog.locator(".table-create-column-main").first.evaluate("""node => { + const inputHeight = node.querySelector( + ".table-create-column-name" + ).getBoundingClientRect().height; + const selectHeight = node.querySelector( + ".table-create-column-type" + ).getBoundingClientRect().height; + return Math.abs(inputHeight - selectHeight) <= 1; + }""") dialog.locator('input[name="table"]').fill("playwright_created") dialog.locator(".table-create-column-name").nth(1).fill("title") dialog.locator(".table-create-add-column").click() @@ -340,6 +349,15 @@ def test_alter_table_flow(page, datasette_server): dialog.wait_for() assert dialog.locator(".modal-title").inner_text() == "Alter table projects" assert dialog.locator(".table-alter-save").is_disabled() + assert dialog.locator(".table-alter-column-main").first.evaluate("""node => { + const inputHeight = node.querySelector( + ".table-alter-column-name" + ).getBoundingClientRect().height; + const selectHeight = node.querySelector( + ".table-alter-column-type" + ).getBoundingClientRect().height; + return Math.abs(inputHeight - selectHeight) <= 1; + }""") type_options = dialog.locator(".table-alter-column-type").first.locator("option") assert type_options.all_inner_texts() == [ "text", From 063b04ad83f88f642053bd88c5368e9ad19f13af Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 12:47:02 -0700 Subject: [PATCH 041/174] Expose foreign key data to alter table UI Include current foreign key metadata in the alter table page data and allow the foreign-key-targets endpoint to be read by actors with alter-table permission for a specific table. Add API and HTML data tests for the new alter-table foreign key support. --- datasette/views/table.py | 18 ++++++++++++++++++ datasette/views/table_create_alter.py | 13 +++++++++++-- tests/test_api_write.py | 11 +++++++++++ tests/test_table_html.py | 6 ++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 988d3808..dbca3a18 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -382,6 +382,19 @@ async def _table_alter_ui( return None column_types_map = await datasette.get_column_types(database_name, table_name) + foreign_keys_by_column = {} + for fk in await db.foreign_keys_for_table(table_name): + other_column = fk["other_column"] + if other_column is None and await db.table_exists(fk["other_table"]): + other_pks = await db.primary_keys(fk["other_table"]) + if len(other_pks) == 1: + other_column = other_pks[0] + if other_column is None: + continue + foreign_keys_by_column[fk["column"]] = { + "fk_table": fk["other_table"], + "fk_column": other_column, + } columns = [] for column in await db.table_column_details(table_name): if column.hidden: @@ -397,6 +410,7 @@ async def _table_alter_ui( "default": column.default_value, "has_default": column.default_value is not None, "is_pk": column.name in pks, + "foreign_key": foreign_keys_by_column.get(column.name), "column_type": ( {"type": column_type.name, "config": column_type.config} if column_type is not None @@ -412,6 +426,10 @@ async def _table_alter_ui( "primaryKeys": pks, "columnTypes": ALTER_TABLE_COLUMN_TYPES, "defaultExpressions": list(DEFAULT_EXPR_SQL), + "foreignKeyTargetsPath": "{}/-/foreign-key-targets?table={}".format( + datasette.urls.database(database_name), + urllib.parse.quote(table_name, safe=""), + ), } can_set_column_type = await datasette.allowed( action="set-column-type", diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 07f13e9d..fa89aae4 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -867,11 +867,20 @@ class DatabaseForeignKeyTargetsView(BaseView): db = await self.ds.resolve_database(request) database_name = db.name - if not await self.ds.allowed( + table_name = request.args.get("table") + can_create_table = await self.ds.allowed( action="create-table", resource=DatabaseResource(database=database_name), actor=request.actor, - ): + ) + can_alter_table = False + if table_name and await db.table_exists(table_name): + can_alter_table = await self.ds.allowed( + action="alter-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ) + if not (can_create_table or can_alter_table): return _error(["Permission denied: need create-table"], 403) hidden_tables = await db.execute_fn( diff --git a/tests/test_api_write.py b/tests/test_api_write.py index ca8a46f5..6e8c4d4b 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1252,6 +1252,17 @@ async def test_foreign_key_targets_permission_denied(ds_write): } +@pytest.mark.asyncio +async def test_foreign_key_targets_allowed_for_alter_table(ds_write): + token = write_token(ds_write, permissions=["at"]) + response = await ds_write.client.get( + "/data/-/foreign-key-targets?table=docs", + headers=_headers(token), + ) + assert response.status_code == 200, response.text + assert response.json()["ok"] is True + + @pytest.mark.asyncio async def test_alter_table_permission_denied(ds_write): token = write_token(ds_write, permissions=["ir"]) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 4c80691c..ba9c03a5 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1137,6 +1137,9 @@ async def test_table_alter_action_button_and_data(): assert alter_data["tableName"] == "items" assert alter_data["primaryKeys"] == ["id"] assert alter_data["columnTypes"] == ["text", "integer", "float", "blob"] + assert alter_data["foreignKeyTargetsPath"] == ( + "/data/-/foreign-key-targets?table=items" + ) assert alter_data["defaultExpressions"] == [ "current_timestamp", "current_date", @@ -1158,6 +1161,7 @@ async def test_table_alter_action_button_and_data(): "default": None, "has_default": False, "is_pk": True, + "foreign_key": None, "column_type": None, }, { @@ -1168,6 +1172,7 @@ async def test_table_alter_action_button_and_data(): "default": None, "has_default": False, "is_pk": False, + "foreign_key": None, "column_type": {"type": "textarea", "config": None}, }, { @@ -1178,6 +1183,7 @@ async def test_table_alter_action_button_and_data(): "default": "5", "has_default": True, "is_pk": False, + "foreign_key": None, "column_type": None, }, ] From fa43aba30920eb268557897b2eb1d7d6a41acade Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 12:50:57 -0700 Subject: [PATCH 042/174] Unify create and alter table modal controls Share default value controls between the create and alter table dialogs and expose create-table default expressions to the frontend. Add create-table not-null/default handling and align the shared foreign key picker behavior across both dialogs. --- datasette/static/app.css | 60 ++- datasette/static/edit-tools.js | 716 ++++++++++++++++++++------ datasette/views/table_create_alter.py | 1 + tests/test_playwright.py | 99 +++- tests/test_table_html.py | 5 + 5 files changed, 709 insertions(+), 172 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 90d7e7e9..32b7a4c2 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1883,12 +1883,14 @@ select.table-create-input { } .table-create-foreign-key-target option, -.table-create-custom-column-type option { +.table-create-custom-column-type option, +.table-create-default-expr option { color: var(--ink); } .table-create-foreign-key-target option[value=""], -.table-create-custom-column-type option[value=""] { +.table-create-custom-column-type option[value=""], +.table-create-default-expr option[value=""] { color: var(--muted); } @@ -1978,11 +1980,44 @@ select.table-create-input { white-space: normal; } +.table-create-not-null, .table-create-primary-key, -.table-create-foreign-key-field { +.table-create-foreign-key-field, +.table-create-default-options { grid-column: 1 / -1; } +.table-create-default-options, +.table-alter-default-options { + border: 1px solid var(--rule); + border-radius: 5px; + background: #fff; + color: var(--ink); + min-width: 0; +} + +.table-create-default-options > summary, +.table-alter-default-options > summary { + cursor: pointer; + color: var(--accent); + font-size: 0.85rem; + padding: 8px 10px; +} + +.table-create-default-options > summary:focus, +.table-alter-default-options > summary:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 1px; +} + +.table-create-default-grid, +.table-alter-default-grid { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + gap: 12px 16px; + padding: 0 10px 10px; +} + .table-create-detail-check input { flex: 0 0 auto; margin: 0.15rem 0 0; @@ -2221,6 +2256,7 @@ dialog.table-alter-dialog::backdrop { overflow-y: auto; } + .table-alter-fields[hidden], .table-alter-dialog .modal-footer [hidden] { display: none; @@ -2358,12 +2394,14 @@ select.table-alter-input { } .table-alter-default-expr option, -.table-alter-custom-column-type option { +.table-alter-custom-column-type option, +.table-alter-foreign-key-target option { color: var(--ink); } .table-alter-default-expr option[value=""], -.table-alter-custom-column-type option[value=""] { +.table-alter-custom-column-type option[value=""], +.table-alter-foreign-key-target option[value=""] { color: var(--muted); } @@ -2410,7 +2448,9 @@ select.table-alter-input { } .table-alter-not-null, -.table-alter-primary-key { +.table-alter-primary-key, +.table-alter-foreign-key-field, +.table-alter-default-options { grid-column: 1 / -1; } @@ -2657,6 +2697,10 @@ select.table-alter-input { grid-template-columns: 1fr; } + .table-alter-default-grid { + grid-template-columns: 1fr; + } + .table-alter-dialog .modal-footer { padding-left: 18px; padding-right: 18px; @@ -2878,6 +2922,10 @@ select.table-alter-input { grid-template-columns: 1fr; } + .table-create-default-grid { + grid-template-columns: 1fr; + } + .table-create-dialog .modal-footer { padding-left: 18px; padding-right: 18px; diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index c4982e54..2e4331e6 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -60,6 +60,11 @@ function tableCreateColumnTypes() { : ["text", "integer", "float", "blob"]; } +function tableCreateDefaultExpressions() { + var data = databaseCreateTableData() || {}; + return data.defaultExpressions || []; +} + var SQLITE_COLUMN_TYPE_LABELS = { float: "floating point number", real: "floating point number", @@ -104,6 +109,112 @@ function createCustomColumnTypeSelect(options, className, placeholderClass) { return select; } +var DEFAULT_EXPRESSION_LABELS = { + current_timestamp: "Current timestamp in UTC, e.g. 2026-05-01 13:34:00", + current_date: "Current date in UTC, e.g. 2026-05-01", + current_time: "Current time in UTC, e.g. 13:34:00", +}; + +function defaultExpressionLabel(value) { + return DEFAULT_EXPRESSION_LABELS[value] || value.replace(/_/g, " "); +} + +function createDefaultExpressionSelect( + options, + className, + placeholderClass, + value, +) { + var select = document.createElement("select"); + select.className = className; + var blankOption = document.createElement("option"); + blankOption.value = ""; + blankOption.textContent = "- default expr -"; + select.appendChild(blankOption); + options.forEach(function (option) { + var optionElement = document.createElement("option"); + optionElement.value = option; + optionElement.textContent = defaultExpressionLabel(option); + select.appendChild(optionElement); + }); + select.value = value || ""; + updateSelectPlaceholder(select, placeholderClass); + return select; +} + +function createSchemaDialogDefaultControls(prefix, index, expressions, column) { + var defaultDetails = document.createElement("details"); + defaultDetails.className = prefix + "-default-options"; + defaultDetails.open = !!( + column && + (column.defaultValue || column.defaultExpr) + ); + var summary = document.createElement("summary"); + summary.textContent = "Set a default value"; + defaultDetails.appendChild(summary); + + var defaultGrid = document.createElement("div"); + defaultGrid.className = prefix + "-default-grid"; + + var defaultExprId = prefix + "-column-default-expr-" + index; + var defaultExprField = document.createElement("div"); + defaultExprField.className = prefix + "-detail-field"; + var defaultExprLabel = document.createElement("label"); + defaultExprLabel.className = prefix + "-detail-label"; + defaultExprLabel.setAttribute("for", defaultExprId); + defaultExprLabel.textContent = "Default expression"; + var defaultExprSelect = createDefaultExpressionSelect( + expressions, + prefix + "-input " + prefix + "-default-expr", + prefix + "-input-placeholder", + column && column.defaultExpr, + ); + defaultExprSelect.id = defaultExprId; + defaultExprSelect.setAttribute("aria-label", "Default expression"); + defaultExprField.appendChild(defaultExprLabel); + defaultExprField.appendChild(defaultExprSelect); + + var defaultId = prefix + "-column-default-" + index; + var defaultField = document.createElement("div"); + defaultField.className = prefix + "-detail-field"; + var defaultLabel = document.createElement("label"); + defaultLabel.className = prefix + "-detail-label"; + defaultLabel.setAttribute("for", defaultId); + defaultLabel.textContent = "or default to a specific value"; + var defaultInput = document.createElement("input"); + defaultInput.id = defaultId; + defaultInput.className = prefix + "-input " + prefix + "-default"; + defaultInput.type = "text"; + defaultInput.autocomplete = "off"; + defaultInput.placeholder = "default"; + defaultInput.setAttribute("aria-label", "or default to a specific value"); + defaultInput.value = column && column.defaultValue ? column.defaultValue : ""; + defaultField.appendChild(defaultLabel); + defaultField.appendChild(defaultInput); + + defaultGrid.appendChild(defaultExprField); + defaultGrid.appendChild(defaultField); + defaultDetails.appendChild(defaultGrid); + + return { + controls: defaultDetails, + defaultInput: defaultInput, + defaultExprSelect: defaultExprSelect, + }; +} + +function syncSchemaDialogDefaultControls(row, prefix) { + if (!row) { + return; + } + var defaultInput = row.querySelector("." + prefix + "-default"); + var defaultExprSelect = row.querySelector("." + prefix + "-default-expr"); + if (!defaultInput || !defaultExprSelect) { + return; + } + updateSelectPlaceholder(defaultExprSelect, prefix + "-input-placeholder"); +} + var COLUMN_MOVE_ICONS = { top: '', up: '', @@ -316,7 +427,7 @@ function updateTableCreateMoveButtons(state) { updateSchemaDialogMoveButtons(state, "table-create"); } -function tableCreateTypeAffinity(type) { +function schemaDialogTypeAffinity(type) { if (type === "float") { return "real"; } @@ -337,11 +448,11 @@ function foreignKeyTypesCompatible(sourceAffinity, targetAffinity) { return false; } -function tableCreateForeignKeyTargetKey(target) { +function foreignKeyTargetKey(target) { return target.fk_table + "\u001f" + target.fk_column; } -function tableCreateForeignKeyTargetLabel(target) { +function foreignKeyTargetLabel(target) { return ( target.fk_table + "." + @@ -352,6 +463,54 @@ function tableCreateForeignKeyTargetLabel(target) { ); } +function appendForeignKeyTargetOption(select, target) { + var optionElement = document.createElement("option"); + optionElement.value = foreignKeyTargetKey(target); + optionElement.dataset.fkTable = target.fk_table; + optionElement.dataset.fkColumn = target.fk_column; + optionElement.dataset.fkType = target.type; + optionElement.textContent = foreignKeyTargetLabel(target); + select.appendChild(optionElement); + return optionElement; +} + +function sqliteColumnTypeForForeignKeyTarget(type) { + var affinity = schemaDialogTypeAffinity(type); + if (affinity === "real" || affinity === "numeric") { + return "float"; + } + if (["text", "integer", "blob"].indexOf(affinity) !== -1) { + return affinity; + } + return ""; +} + +function selectedSchemaDialogForeignKeyOption(foreignKeySelect) { + return foreignKeySelect && foreignKeySelect.selectedOptions + ? foreignKeySelect.selectedOptions[0] + : null; +} + +function setBlankSchemaDialogColumnNameFromForeignKey( + row, + prefix, + foreignKeyOption, +) { + var nameInput = row.querySelector("." + prefix + "-column-name"); + if ( + nameInput && + !nameInput.value.trim() && + foreignKeyOption && + foreignKeyOption.dataset.fkTable && + foreignKeyOption.dataset.fkColumn + ) { + nameInput.value = + foreignKeyOption.dataset.fkTable + + "_" + + foreignKeyOption.dataset.fkColumn; + } +} + function tableCreateForeignKeyTargetsUrl() { var data = databaseCreateTableData() || {}; if (data.foreignKeyTargetsPath) { @@ -363,7 +522,14 @@ function tableCreateForeignKeyTargetsUrl() { return data.path.replace(/\/-\/create$/, "/-/foreign-key-targets"); } -function populateTableCreateForeignKeySelect(select, state, sourceType) { +function populateSchemaDialogForeignKeySelect( + select, + state, + prefix, + sourceType, + options, +) { + options = options || {}; var previousKey = select.value || select.dataset.selectedKey || ""; select.textContent = ""; @@ -385,45 +551,64 @@ function populateTableCreateForeignKeySelect(select, state, sourceType) { errorOption.textContent = "Could not load foreign keys"; select.appendChild(errorOption); } else { - var sourceAffinity = tableCreateTypeAffinity(sourceType); + var sourceAffinity = schemaDialogTypeAffinity(sourceType); (state.foreignKeyTargets || []).forEach(function (target) { - if (!foreignKeyTypesCompatible(sourceAffinity, target.type)) { + if ( + options.filterByType !== false && + !foreignKeyTypesCompatible(sourceAffinity, target.type) + ) { return; } - var optionElement = document.createElement("option"); - optionElement.value = tableCreateForeignKeyTargetKey(target); - optionElement.dataset.fkTable = target.fk_table; - optionElement.dataset.fkColumn = target.fk_column; - optionElement.textContent = tableCreateForeignKeyTargetLabel(target); - select.appendChild(optionElement); + appendForeignKeyTargetOption(select, target); }); } select.value = previousKey; + if ( + previousKey && + select.value !== previousKey && + select.dataset.currentFkTable && + select.dataset.currentFkColumn + ) { + appendForeignKeyTargetOption(select, { + fk_table: select.dataset.currentFkTable, + fk_column: select.dataset.currentFkColumn, + type: select.dataset.currentFkType || sourceType, + }); + select.value = previousKey; + } if (select.value !== previousKey) { select.value = ""; } select.dataset.selectedKey = select.value; select.disabled = state.isSaving || select.options.length <= 1; - updateSelectPlaceholder(select, "table-create-input-placeholder"); + updateSelectPlaceholder(select, prefix + "-input-placeholder"); } -function syncTableCreateForeignKeyOptions(row, state) { - var typeSelect = row.querySelector(".table-create-column-type"); - var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); +function syncSchemaDialogForeignKeyOptions(row, state, prefix, options) { + var typeSelect = row.querySelector("." + prefix + "-column-type"); + var foreignKeySelect = row.querySelector( + "." + prefix + "-foreign-key-target", + ); if (!typeSelect || !foreignKeySelect) { return; } - populateTableCreateForeignKeySelect( + populateSchemaDialogForeignKeySelect( foreignKeySelect, state, + prefix, typeSelect.value, + options, ); } -function syncTableCreateCustomTypeAndForeignKey(row, state, isFirstColumn) { - var customTypeSelect = row.querySelector(".table-create-custom-column-type"); - var foreignKeySelect = row.querySelector(".table-create-foreign-key-target"); +function syncSchemaDialogCustomTypeAndForeignKey(row, state, prefix) { + var customTypeSelect = row.querySelector( + "." + prefix + "-custom-column-type", + ); + var foreignKeySelect = row.querySelector( + "." + prefix + "-foreign-key-target", + ); if (!foreignKeySelect) { return; } @@ -433,14 +618,14 @@ function syncTableCreateCustomTypeAndForeignKey(row, state, isFirstColumn) { if (customTypeSelect && hasForeignKey) { customTypeSelect.value = ""; - updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + updateSelectPlaceholder(customTypeSelect, prefix + "-input-placeholder"); hasCustomType = false; } - if (isFirstColumn || hasCustomType) { + if (hasCustomType) { foreignKeySelect.value = ""; foreignKeySelect.dataset.selectedKey = ""; - updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); + updateSelectPlaceholder(foreignKeySelect, prefix + "-input-placeholder"); hasForeignKey = false; } @@ -448,63 +633,73 @@ function syncTableCreateCustomTypeAndForeignKey(row, state, isFirstColumn) { customTypeSelect.disabled = state.isSaving; } foreignKeySelect.disabled = - state.isSaving || isFirstColumn || foreignKeySelect.options.length <= 1; + state.isSaving || foreignKeySelect.options.length <= 1; } -function refreshTableCreateForeignKeyControls(state) { - tableCreateDialogRows(state).forEach(function (row, index) { - if (index > 0) { - syncTableCreateForeignKeyOptions(row, state); - } - syncTableCreateCustomTypeAndForeignKey(row, state, index === 0); - }); -} +function handleSchemaDialogForeignKeyChange(row, state, prefix, options) { + options = options || {}; + var foreignKeySelect = row.querySelector( + "." + prefix + "-foreign-key-target", + ); + var typeSelect = row.querySelector("." + prefix + "-column-type"); + var customTypeSelect = row.querySelector( + "." + prefix + "-custom-column-type", + ); + if (!foreignKeySelect) { + return; + } + foreignKeySelect.dataset.selectedKey = foreignKeySelect.value; + updateSelectPlaceholder(foreignKeySelect, prefix + "-input-placeholder"); -function updateTableCreateColumnRules(state) { - tableCreateDialogRows(state).forEach(function (row, index) { - var isFirstColumn = index === 0; - var pkLabel = row.querySelector(".table-create-primary-key"); - var pkInput = row.querySelector(".table-create-primary-key-input"); - var foreignKeyField = row.querySelector(".table-create-foreign-key-field"); - var foreignKeySelect = row.querySelector( - ".table-create-foreign-key-target", + var foreignKeyOption = selectedSchemaDialogForeignKeyOption(foreignKeySelect); + setBlankSchemaDialogColumnNameFromForeignKey(row, prefix, foreignKeyOption); + + var columnTypes = options.columnTypes || []; + var foreignKeyColumnType = + foreignKeyOption && foreignKeyOption.dataset.fkType + ? sqliteColumnTypeForForeignKeyTarget(foreignKeyOption.dataset.fkType) + : ""; + if ( + options.matchType && + typeSelect && + foreignKeyColumnType && + columnTypes.indexOf(foreignKeyColumnType) !== -1 && + typeSelect.value !== foreignKeyColumnType + ) { + typeSelect.value = foreignKeyColumnType; + syncSchemaDialogForeignKeyOptions( + row, + state, + prefix, + options.foreignKeyOptions, ); + } - if (pkLabel && pkInput) { - pkLabel.hidden = !isFirstColumn; - if (!isFirstColumn) { - pkInput.checked = false; - } - } - - if (foreignKeyField && foreignKeySelect) { - foreignKeyField.hidden = isFirstColumn; - if (isFirstColumn) { - foreignKeySelect.value = ""; - foreignKeySelect.dataset.selectedKey = ""; - foreignKeySelect.disabled = true; - updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); - } else { - syncTableCreateForeignKeyOptions(row, state); - } - syncTableCreateCustomTypeAndForeignKey(row, state, isFirstColumn); - } - }); - updateTableCreateMoveButtons(state); + if (customTypeSelect && foreignKeySelect.value) { + customTypeSelect.value = ""; + updateSelectPlaceholder(customTypeSelect, prefix + "-input-placeholder"); + } + syncSchemaDialogCustomTypeAndForeignKey(row, state, prefix); } -async function loadTableCreateForeignKeyTargets(state) { - var url = tableCreateForeignKeyTargetsUrl(); +function refreshSchemaDialogForeignKeyControls(state, prefix, options) { + schemaDialogRows(state, prefix).forEach(function (row) { + syncSchemaDialogForeignKeyOptions(row, state, prefix, options); + syncSchemaDialogCustomTypeAndForeignKey(row, state, prefix); + }); +} + +async function loadSchemaDialogForeignKeyTargets(state, prefix, url, options) { if (!url || !window.fetch) { state.foreignKeyTargets = []; state.foreignKeyTargetsLoading = false; - refreshTableCreateForeignKeyControls(state); + refreshSchemaDialogForeignKeyControls(state, prefix, options); return; } state.foreignKeyTargets = []; state.foreignKeyTargetsError = null; state.foreignKeyTargetsLoading = true; - refreshTableCreateForeignKeyControls(state); + refreshSchemaDialogForeignKeyControls(state, prefix, options); try { var response = await fetch(url, { headers: { @@ -521,10 +716,46 @@ async function loadTableCreateForeignKeyTargets(state) { state.foreignKeyTargetsError = error; } finally { state.foreignKeyTargetsLoading = false; - refreshTableCreateForeignKeyControls(state); + refreshSchemaDialogForeignKeyControls(state, prefix, options); } } +function syncTableCreateForeignKeyOptions(row, state) { + syncSchemaDialogForeignKeyOptions(row, state, "table-create", { + filterByType: false, + }); +} + +function syncTableCreateCustomTypeAndForeignKey(row, state) { + syncSchemaDialogCustomTypeAndForeignKey(row, state, "table-create"); +} + +function refreshTableCreateForeignKeyControls(state) { + tableCreateDialogRows(state).forEach(function (row) { + syncTableCreateForeignKeyOptions(row, state); + syncTableCreateCustomTypeAndForeignKey(row, state); + }); +} + +function updateTableCreateColumnRules(state) { + normalizeSchemaDialogPrimaryKeyRows(state, "table-create"); + tableCreateDialogRows(state).forEach(function (row) { + syncTableCreateForeignKeyOptions(row, state); + syncTableCreateCustomTypeAndForeignKey(row, state); + syncSchemaDialogDefaultControls(row, "table-create"); + }); + updateTableCreateMoveButtons(state); +} + +async function loadTableCreateForeignKeyTargets(state) { + return loadSchemaDialogForeignKeyTargets( + state, + "table-create", + tableCreateForeignKeyTargetsUrl(), + { filterByType: false }, + ); +} + function tableCreateDialogSignature(state) { if (!state || !state.form) { return ""; @@ -542,6 +773,9 @@ function tableCreateDialogSignature(state) { } ).value || "", pk: row.querySelector(".table-create-primary-key-input").checked, + notNull: row.querySelector(".table-create-not-null-input").checked, + defaultValue: row.querySelector(".table-create-default").value, + defaultExpr: row.querySelector(".table-create-default-expr").value, foreignKey: ( row.querySelector(".table-create-foreign-key-target") || { @@ -654,6 +888,7 @@ function createTableColumnRow(state, column) { nameInput.type = "text"; nameInput.required = true; nameInput.autocomplete = "off"; + nameInput.placeholder = "column name"; nameInput.value = column && column.name ? column.name : ""; var typeSelect = document.createElement("select"); @@ -728,6 +963,32 @@ function createTableColumnRow(state, column) { foreignKeyField.appendChild(foreignKeyHelp); foreignKeyField.appendChild(foreignKeySelect); + var notNullLabel = document.createElement("label"); + notNullLabel.className = "table-create-detail-check table-create-not-null"; + var notNullInput = document.createElement("input"); + notNullInput.type = "checkbox"; + notNullInput.className = "table-create-not-null-input"; + notNullInput.checked = !!(column && column.notNull); + var notNullText = document.createElement("span"); + var notNullStrong = document.createElement("strong"); + notNullStrong.textContent = "Not null"; + notNullText.appendChild(notNullStrong); + notNullText.appendChild( + document.createTextNode(" This value cannot be left unset"), + ); + notNullLabel.appendChild(notNullInput); + notNullLabel.appendChild(notNullText); + + var defaultControls = createSchemaDialogDefaultControls( + "table-create", + index, + tableCreateDefaultExpressions(), + { + defaultValue: column && column.defaultValue, + defaultExpr: column && column.defaultExpr, + }, + ); + var moveControls = createSchemaDialogMoveControls("table-create"); var removeButton = createSchemaDialogIconButton( @@ -748,8 +1009,10 @@ function createTableColumnRow(state, column) { if (customTypeField) { details.appendChild(customTypeField); } - details.appendChild(foreignKeyField); + details.appendChild(defaultControls.controls); + details.appendChild(notNullLabel); details.appendChild(pkLabel); + details.appendChild(foreignKeyField); row.appendChild(main); row.appendChild(details); @@ -775,11 +1038,7 @@ function createTableColumnRow(state, column) { clearTableCreateDialogError(state); syncTableCreateCustomTypeForSqliteType(row); syncTableCreateForeignKeyOptions(row, state); - syncTableCreateCustomTypeAndForeignKey( - row, - state, - row === state.columnList.firstElementChild, - ); + syncTableCreateCustomTypeAndForeignKey(row, state); }); if (customTypeSelect) { customTypeSelect.addEventListener("change", function () { @@ -798,30 +1057,37 @@ function createTableColumnRow(state, column) { typeSelect.value = option.fixedSqliteType; syncTableCreateForeignKeyOptions(row, state); } - syncTableCreateCustomTypeAndForeignKey( - row, - state, - row === state.columnList.firstElementChild, - ); + syncTableCreateCustomTypeAndForeignKey(row, state); }); } pkInput.addEventListener("change", function () { clearTableCreateDialogError(state); updateTableCreateColumnRules(state); }); - foreignKeySelect.addEventListener("change", function () { - foreignKeySelect.dataset.selectedKey = foreignKeySelect.value; + notNullInput.addEventListener("change", function () { clearTableCreateDialogError(state); - updateTableCreateCustomColumnTypePlaceholder(foreignKeySelect); - if (customTypeSelect && foreignKeySelect.value) { - customTypeSelect.value = ""; - updateTableCreateCustomColumnTypePlaceholder(customTypeSelect); + }); + defaultControls.defaultInput.addEventListener("input", function () { + if (defaultControls.defaultInput.value) { + defaultControls.defaultExprSelect.value = ""; + syncSchemaDialogDefaultControls(row, "table-create"); } - syncTableCreateCustomTypeAndForeignKey( - row, - state, - row === state.columnList.firstElementChild, - ); + clearTableCreateDialogError(state); + }); + defaultControls.defaultExprSelect.addEventListener("change", function () { + if (defaultControls.defaultExprSelect.value) { + defaultControls.defaultInput.value = ""; + } + syncSchemaDialogDefaultControls(row, "table-create"); + clearTableCreateDialogError(state); + }); + foreignKeySelect.addEventListener("change", function () { + clearTableCreateDialogError(state); + handleSchemaDialogForeignKeyChange(row, state, "table-create", { + columnTypes: tableCreateColumnTypes(), + foreignKeyOptions: { filterByType: false }, + matchType: true, + }); }); expandButton.addEventListener("click", function () { @@ -887,6 +1153,7 @@ function createTableColumnRow(state, column) { row.querySelector(".table-create-column-name").focus(); }); + syncSchemaDialogDefaultControls(row, "table-create"); return row; } @@ -921,7 +1188,7 @@ function collectTableCreatePayload(state) { columns: [], }; var primaryKeys = []; - tableCreateDialogRows(state).forEach(function (row, index) { + tableCreateDialogRows(state).forEach(function (row) { var name = row.querySelector(".table-create-column-name").value.trim(); var type = row.querySelector(".table-create-column-type").value; var column = { name: name, type: type }; @@ -933,7 +1200,6 @@ function collectTableCreatePayload(state) { ? foreignKeySelect.selectedOptions[0] : null; if ( - index > 0 && foreignKeyOption && foreignKeyOption.dataset.fkTable && foreignKeyOption.dataset.fkColumn @@ -941,11 +1207,18 @@ function collectTableCreatePayload(state) { column.fk_table = foreignKeyOption.dataset.fkTable; column.fk_column = foreignKeyOption.dataset.fkColumn; } + if (row.querySelector(".table-create-not-null-input").checked) { + column.not_null = true; + } + var defaultExpr = row.querySelector(".table-create-default-expr").value; + var defaultValue = row.querySelector(".table-create-default").value; + if (defaultExpr) { + column.default_expr = defaultExpr; + } else if (defaultValue) { + column.default = defaultValue; + } payload.columns.push(column); - if ( - index === 0 && - row.querySelector(".table-create-primary-key-input").checked - ) { + if (row.querySelector(".table-create-primary-key-input").checked) { primaryKeys.push(name); } }); @@ -1006,6 +1279,9 @@ function validateTableCreatePayload(payload) { if (supportedTypes.indexOf(column.type) === -1) { return "Unsupported column type: " + column.type; } + if (column.default && column.default_expr) { + return "Use either a default value or a default expression."; + } } return null; } @@ -1467,6 +1743,11 @@ function tableAlterDefaultExpressions() { return data.defaultExpressions || []; } +function tableAlterForeignKeyTargetsUrl() { + var data = tableAlterData() || {}; + return data.foreignKeyTargetsPath || null; +} + function tableAlterCustomColumnTypes() { var data = tableAlterData() || {}; return data.customColumnTypes || []; @@ -1494,7 +1775,21 @@ function tableAlterDialogRows(state) { return schemaDialogRows(state, "table-alter"); } +function syncTableAlterForeignKeyOptions(row, state) { + syncSchemaDialogForeignKeyOptions(row, state, "table-alter", { + filterByType: false, + }); +} + function tableAlterRowSignature(row) { + var foreignKeySelect = row.querySelector(".table-alter-foreign-key-target"); + var foreignKeyValue = foreignKeySelect + ? foreignKeySelect.value || foreignKeySelect.dataset.selectedKey || "" + : ""; + var foreignKeyOption = + foreignKeySelect && foreignKeySelect.selectedOptions + ? foreignKeySelect.selectedOptions[0] + : null; return { existing: row.dataset.existing === "1", originalName: row.dataset.originalName || "", @@ -1510,6 +1805,15 @@ function tableAlterRowSignature(row) { defaultValue: row.querySelector(".table-alter-default").value, defaultExpr: row.querySelector(".table-alter-default-expr").value, pk: row.querySelector(".table-alter-primary-key-input").checked, + foreignKey: foreignKeyValue, + foreignKeyTable: + foreignKeyOption && foreignKeyOption.dataset.fkTable + ? foreignKeyOption.dataset.fkTable + : "", + foreignKeyColumn: + foreignKeyOption && foreignKeyOption.dataset.fkColumn + ? foreignKeyOption.dataset.fkColumn + : "", }; } @@ -1581,7 +1885,7 @@ function setTableAlterDialogSaving(state, isSaving) { ? "Applying..." : "Preparing..." : tableAlterSaveButtonText(state); - state.columnList + state.fields .querySelectorAll("input, select, button") .forEach(function (control) { control.disabled = isSaving; @@ -1590,10 +1894,12 @@ function setTableAlterDialogSaving(state, isSaving) { state.columnList .querySelectorAll(".table-alter-default-expr") .forEach(function (select) { - syncTableAlterDefaultControls( + syncSchemaDialogDefaultControls( select.closest(".table-alter-column-row"), + "table-alter", ); }); + refreshSchemaDialogForeignKeyControls(state, "table-alter"); } updateTableAlterMoveButtons(state); updateTableAlterSaveButtonState(state); @@ -1634,38 +1940,6 @@ function syncTableAlterCustomTypeForSqliteType(row) { } } -function tableAlterSelectDefaultExprValue(select, value) { - var blankOption = document.createElement("option"); - blankOption.value = ""; - blankOption.textContent = "- default expr -"; - select.appendChild(blankOption); - tableAlterDefaultExpressions().forEach(function (option) { - var optionElement = document.createElement("option"); - optionElement.value = option; - optionElement.textContent = option.replace(/_/g, " "); - select.appendChild(optionElement); - }); - select.value = value || ""; - updateTableAlterDefaultExprPlaceholder(select); -} - -function updateTableAlterDefaultExprPlaceholder(select) { - updateSelectPlaceholder(select, "table-alter-input-placeholder"); -} - -function syncTableAlterDefaultControls(row) { - if (!row) { - return; - } - var defaultInput = row.querySelector(".table-alter-default"); - var defaultExprSelect = row.querySelector(".table-alter-default-expr"); - if (!defaultInput || !defaultExprSelect) { - return; - } - defaultInput.disabled = !!defaultExprSelect.value; - updateTableAlterDefaultExprPlaceholder(defaultExprSelect); -} - function createTableAlterColumnRow(state, column) { var index = state.nextColumnIndex; state.nextColumnIndex += 1; @@ -1677,6 +1951,10 @@ function createTableAlterColumnRow(state, column) { existing && column.has_default && column.default !== null ? String(column.default) : ""; + var originalForeignKey = + existing && column.foreign_key + ? foreignKeyTargetKey(column.foreign_key) + : ""; var row = document.createElement("div"); row.className = "table-alter-column-row"; @@ -1688,6 +1966,7 @@ function createTableAlterColumnRow(state, column) { row.dataset.originalDefault = originalDefault; row.dataset.originalPk = existing && column.is_pk ? "1" : "0"; row.dataset.originalCustomType = originalCustomType; + row.dataset.originalForeignKey = originalForeignKey; var main = document.createElement("div"); main.className = "table-alter-column-main"; @@ -1714,6 +1993,7 @@ function createTableAlterColumnRow(state, column) { nameInput.type = "text"; nameInput.required = true; nameInput.autocomplete = "off"; + nameInput.placeholder = "column name"; nameInput.value = column && column.name ? column.name : ""; var typeSelect = document.createElement("select"); @@ -1764,38 +2044,51 @@ function createTableAlterColumnRow(state, column) { notNullLabel.appendChild(notNullInput); notNullLabel.appendChild(notNullText); - var defaultId = "table-alter-column-default-" + index; - var defaultField = document.createElement("div"); - defaultField.className = "table-alter-detail-field"; - var defaultLabel = document.createElement("label"); - defaultLabel.className = "table-alter-detail-label"; - defaultLabel.setAttribute("for", defaultId); - defaultLabel.textContent = "Default value"; - var defaultInput = document.createElement("input"); - defaultInput.id = defaultId; - defaultInput.className = "table-alter-input table-alter-default"; - defaultInput.type = "text"; - defaultInput.autocomplete = "off"; - defaultInput.placeholder = "default"; - defaultInput.setAttribute("aria-label", "Default value"); - defaultInput.value = originalDefault; - defaultField.appendChild(defaultLabel); - defaultField.appendChild(defaultInput); + var defaultControls = createSchemaDialogDefaultControls( + "table-alter", + index, + tableAlterDefaultExpressions(), + { + defaultValue: originalDefault, + defaultExpr: "", + }, + ); + var defaultInput = defaultControls.defaultInput; + var defaultExprSelect = defaultControls.defaultExprSelect; - var defaultExprId = "table-alter-column-default-expr-" + index; - var defaultExprField = document.createElement("div"); - defaultExprField.className = "table-alter-detail-field"; - var defaultExprLabel = document.createElement("label"); - defaultExprLabel.className = "table-alter-detail-label"; - defaultExprLabel.setAttribute("for", defaultExprId); - defaultExprLabel.textContent = "or default to a specific time"; - var defaultExprSelect = document.createElement("select"); - defaultExprSelect.id = defaultExprId; - defaultExprSelect.className = "table-alter-input table-alter-default-expr"; - defaultExprSelect.setAttribute("aria-label", "or default to a specific time"); - tableAlterSelectDefaultExprValue(defaultExprSelect, ""); - defaultExprField.appendChild(defaultExprLabel); - defaultExprField.appendChild(defaultExprSelect); + var foreignKeyId = "table-alter-column-foreign-key-" + index; + var foreignKeyHelpId = "table-alter-column-foreign-key-help-" + index; + var foreignKeyField = document.createElement("div"); + foreignKeyField.className = + "table-alter-detail-field table-alter-foreign-key-field"; + var foreignKeyLabel = document.createElement("label"); + foreignKeyLabel.className = "table-alter-detail-label"; + foreignKeyLabel.setAttribute("for", foreignKeyId); + foreignKeyLabel.textContent = "Foreign key"; + var foreignKeyHelp = document.createElement("p"); + foreignKeyHelp.id = foreignKeyHelpId; + foreignKeyHelp.className = "table-alter-detail-help"; + foreignKeyHelp.textContent = "Link this column to another table."; + var foreignKeySelect = document.createElement("select"); + foreignKeySelect.id = foreignKeyId; + foreignKeySelect.className = + "table-alter-input table-alter-foreign-key-target"; + foreignKeySelect.setAttribute("aria-label", "Foreign key target"); + foreignKeySelect.setAttribute("aria-describedby", foreignKeyHelpId); + foreignKeySelect.dataset.selectedKey = originalForeignKey; + if (column && column.foreign_key) { + foreignKeySelect.dataset.currentFkTable = column.foreign_key.fk_table; + foreignKeySelect.dataset.currentFkColumn = column.foreign_key.fk_column; + appendForeignKeyTargetOption(foreignKeySelect, { + fk_table: column.foreign_key.fk_table, + fk_column: column.foreign_key.fk_column, + type: column.type || "text", + }); + foreignKeySelect.value = originalForeignKey; + } + foreignKeyField.appendChild(foreignKeyLabel); + foreignKeyField.appendChild(foreignKeyHelp); + foreignKeyField.appendChild(foreignKeySelect); var pkLabel = document.createElement("label"); pkLabel.className = "table-alter-detail-check table-alter-primary-key"; @@ -1832,10 +2125,10 @@ function createTableAlterColumnRow(state, column) { if (customTypeField) { details.appendChild(customTypeField); } + details.appendChild(defaultControls.controls); details.appendChild(notNullLabel); - details.appendChild(defaultField); - details.appendChild(defaultExprField); details.appendChild(pkLabel); + details.appendChild(foreignKeyField); row.appendChild(main); row.appendChild(details); @@ -1846,6 +2139,7 @@ function createTableAlterColumnRow(state, column) { defaultInput, defaultExprSelect, pkInput, + foreignKeySelect, ]; if (customTypeSelect) { controls.push(customTypeSelect); @@ -1864,7 +2158,7 @@ function createTableAlterColumnRow(state, column) { defaultInput.addEventListener("input", function () { if (defaultInput.value) { defaultExprSelect.value = ""; - syncTableAlterDefaultControls(row); + syncSchemaDialogDefaultControls(row, "table-alter"); } updateTableAlterSaveButtonState(state); }); @@ -1872,7 +2166,7 @@ function createTableAlterColumnRow(state, column) { if (defaultExprSelect.value) { defaultInput.value = ""; } - syncTableAlterDefaultControls(row); + syncSchemaDialogDefaultControls(row, "table-alter"); updateTableAlterSaveButtonState(state); }); pkInput.addEventListener("change", function () { @@ -1887,6 +2181,8 @@ function createTableAlterColumnRow(state, column) { typeSelect.addEventListener("change", function () { syncTableAlterCustomTypeForSqliteType(row); + syncTableAlterForeignKeyOptions(row, state); + syncSchemaDialogCustomTypeAndForeignKey(row, state, "table-alter"); updateTableAlterSaveButtonState(state); }); if (customTypeSelect) { @@ -1899,10 +2195,20 @@ function createTableAlterColumnRow(state, column) { tableAlterColumnTypes().indexOf(option.fixedSqliteType) !== -1 ) { typeSelect.value = option.fixedSqliteType; + syncTableAlterForeignKeyOptions(row, state); } + syncSchemaDialogCustomTypeAndForeignKey(row, state, "table-alter"); updateTableAlterSaveButtonState(state); }); } + foreignKeySelect.addEventListener("change", function () { + handleSchemaDialogForeignKeyChange(row, state, "table-alter", { + columnTypes: tableAlterColumnTypes(), + foreignKeyOptions: { filterByType: false }, + matchType: true, + }); + updateTableAlterSaveButtonState(state); + }); moveControls.topButton.addEventListener("click", function () { var first = tableAlterFirstNonPrimaryRow(state); @@ -1986,7 +2292,9 @@ function createTableAlterColumnRow(state, column) { } }); - syncTableAlterDefaultControls(row); + syncSchemaDialogDefaultControls(row, "table-alter"); + syncTableAlterForeignKeyOptions(row, state); + syncSchemaDialogCustomTypeAndForeignKey(row, state, "table-alter"); return row; } @@ -2026,6 +2334,7 @@ function collectTableAlterRows(state) { signature.originalDefault = row.dataset.originalDefault || ""; signature.originalPk = row.dataset.originalPk === "1"; signature.originalCustomType = row.dataset.originalCustomType || ""; + signature.originalForeignKey = row.dataset.originalForeignKey || ""; return signature; }); } @@ -2156,6 +2465,46 @@ function tableAlterColumnsReordered(state, rows) { ); } +function tableAlterForeignKeyIdentity(row) { + return [ + row.name.trim(), + row.foreignKeyTable || "", + row.foreignKeyColumn || "", + ].join("\u001f"); +} + +function tableAlterOriginalForeignKeyIdentity(row) { + return [row.originalName || "", row.originalForeignKey].join("\u001f"); +} + +function tableAlterForeignKeyRows(rows) { + return rows + .filter(function (row) { + return row.foreignKey && row.foreignKeyTable && row.foreignKeyColumn; + }) + .map(function (row) { + return { + column: row.name.trim(), + fk_table: row.foreignKeyTable, + fk_column: row.foreignKeyColumn, + }; + }); +} + +function tableAlterForeignKeysChanged(rows) { + var original = rows + .filter(function (row) { + return row.existing && row.originalForeignKey; + }) + .map(tableAlterOriginalForeignKeyIdentity); + var final = rows + .filter(function (row) { + return row.foreignKey && row.foreignKeyTable && row.foreignKeyColumn; + }) + .map(tableAlterForeignKeyIdentity); + return JSON.stringify(original) !== JSON.stringify(final); +} + function collectTableAlterPayload(state) { var rows = collectTableAlterRows(state); var validationError = validateTableAlterRows(state, rows); @@ -2237,6 +2586,15 @@ function collectTableAlterPayload(state) { }); } + if (tableAlterForeignKeysChanged(rows)) { + operations.push({ + op: "set_foreign_keys", + args: { + foreign_keys: tableAlterForeignKeyRows(rows), + }, + }); + } + if (!operations.length && !columnTypeAssignments.length) { return { error: "No changes to apply." }; } @@ -2354,6 +2712,27 @@ function tableAlterOperationSummary(operation) { damaging: false, }; } + if (operation.op === "set_foreign_keys") { + var foreignKeys = args.foreign_keys || []; + return { + text: foreignKeys.length + ? "Set foreign keys to " + + foreignKeys + .map(function (foreignKey) { + return ( + tableAlterQuotedName(foreignKey.column) + + " -> " + + foreignKey.fk_table + + "." + + foreignKey.fk_column + ); + }) + .join(", ") + + "." + : "Remove all foreign keys.", + damaging: false, + }; + } return { text: "Run " + operation.op + ".", damaging: false, @@ -2742,6 +3121,9 @@ function ensureTableAlterDialog(manager) { deletedColumns: [], originalColumnNames: [], originalPrimaryKeys: [], + foreignKeyTargets: [], + foreignKeyTargetsError: null, + foreignKeyTargetsLoading: false, mode: "edit", reviewResult: null, manager: manager, @@ -2844,6 +3226,12 @@ function openTableAlterDialog(button, manager) { state.title.textContent = "Alter table " + data.tableName; clearTableAlterDialogError(state); resetTableAlterDialog(state, data); + loadSchemaDialogForeignKeyTargets( + state, + "table-alter", + tableAlterForeignKeyTargetsUrl(), + { filterByType: false }, + ); if (!state.dialog.open) { state.dialog.showModal(); } diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index fa89aae4..111dfab1 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -266,6 +266,7 @@ async def _create_table_ui_context( ), "databaseName": database_name, "columnTypes": CREATE_TABLE_COLUMN_TYPES, + "defaultExpressions": list(DEFAULT_EXPR_SQL), } can_set_column_type = await datasette.allowed( action="set-column-type", diff --git a/tests/test_playwright.py b/tests/test_playwright.py index eefec118..5719039a 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -295,6 +295,10 @@ def test_create_table_flow(page, datasette_server): placeholder_select.locator("option:checked").inner_text() == "- custom type -" ) assert "table-create-input-placeholder" in placeholder_select.get_attribute("class") + assert ( + dialog.locator(".table-create-column-name").nth(0).get_attribute("placeholder") + == "column name" + ) assert dialog.locator(".table-create-column-main").first.evaluate("""node => { const inputHeight = node.querySelector( ".table-create-column-name" @@ -306,6 +310,22 @@ def test_create_table_flow(page, datasette_server): }""") dialog.locator('input[name="table"]').fill("playwright_created") dialog.locator(".table-create-column-name").nth(1).fill("title") + dialog.locator(".table-create-more-options").nth(1).click() + dialog.locator(".table-create-not-null-input").nth(1).check() + title_defaults = dialog.locator(".table-create-default-options").nth(1) + assert title_defaults.locator("summary").inner_text() == "Set a default value" + title_defaults.locator("summary").click() + assert "or default to a specific value" in title_defaults.inner_text() + title_default_expr = title_defaults.locator(".table-create-default-expr") + title_default_input = title_defaults.locator(".table-create-default") + assert ( + "Current timestamp in UTC, e.g. 2026-05-01 13:34:00" + in title_default_expr.locator("option").nth(1).inner_text() + ) + title_default_expr.select_option("current_timestamp") + assert title_default_input.is_enabled() + title_default_input.fill("Untitled") + assert title_default_expr.input_value() == "" dialog.locator(".table-create-add-column").click() dialog.locator(".table-create-column-name").nth(2).fill("score") dialog.locator(".table-create-column-type").nth(2).select_option("integer") @@ -337,6 +357,63 @@ def test_create_table_flow(page, datasette_server): assert data["column_types"] == { "metadata": {"type": "json", "config": None}, } + schema_response = httpx.get( + f"{datasette_server}data/-/query.json", + params={ + "sql": ( + "select sql from sqlite_master where type = 'table' " + "and name = 'playwright_created'" + ) + }, + ) + schema_response.raise_for_status() + schema = schema_response.json()["rows"][0]["sql"] + assert "title" in schema + assert "NOT NULL DEFAULT 'Untitled'" in schema + + +@pytest.mark.playwright +def test_create_table_foreign_key_selection_updates_column_type(page, datasette_server): + page.goto(f"{datasette_server}data") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-database-action="create-table"]').click() + + dialog = page.locator("#table-create-dialog") + dialog.wait_for() + dialog.locator(".table-create-more-options").nth(1).click() + + column_name = dialog.locator(".table-create-column-name").nth(1) + type_select = dialog.locator(".table-create-column-type").nth(1) + foreign_key_select = dialog.locator(".table-create-foreign-key-target").nth(1) + assert column_name.input_value() == "" + assert type_select.input_value() == "text" + + foreign_key_select.select_option("projects\u001fid") + assert column_name.input_value() == "projects_id" + assert type_select.input_value() == "integer" + assert foreign_key_select.input_value() == "projects\u001fid" + + +@pytest.mark.playwright +def test_alter_table_foreign_key_selection_updates_blank_column(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + dialog.locator(".table-alter-add-column").click() + + column_name = dialog.locator(".table-alter-column-name").last + type_select = dialog.locator(".table-alter-column-type").last + foreign_key_select = dialog.locator(".table-alter-foreign-key-target").last + assert column_name.input_value() == "" + assert type_select.input_value() == "text" + + foreign_key_select.select_option("projects\u001fid") + assert column_name.input_value() == "projects_id" + assert type_select.input_value() == "integer" + assert foreign_key_select.input_value() == "projects\u001fid" @pytest.mark.playwright @@ -349,6 +426,10 @@ def test_alter_table_flow(page, datasette_server): dialog.wait_for() assert dialog.locator(".modal-title").inner_text() == "Alter table projects" assert dialog.locator(".table-alter-save").is_disabled() + assert ( + dialog.locator(".table-alter-column-name").first.get_attribute("placeholder") + == "column name" + ) assert dialog.locator(".table-alter-column-main").first.evaluate("""node => { const inputHeight = node.querySelector( ".table-alter-column-name" @@ -377,15 +458,29 @@ def test_alter_table_flow(page, datasette_server): ) assert "Not null" in expanded_options_text assert "This value cannot be left unset" in expanded_options_text - assert "Default value" in expanded_options_text - assert "or default to a specific time" in expanded_options_text + assert "Set a default value" in expanded_options_text assert "Primary key" in expanded_options_text assert "This ID uniquely identifies the record" in expanded_options_text + assert "Foreign key" in expanded_options_text + first_defaults = dialog.locator(".table-alter-default-options").first + first_defaults.locator("summary").click() + assert "or default to a specific value" in first_defaults.inner_text() + first_default_expr = first_defaults.locator(".table-alter-default-expr") + first_default_input = first_defaults.locator(".table-alter-default") + assert ( + "Current timestamp in UTC, e.g. 2026-05-01 13:34:00" + in first_default_expr.locator("option").nth(1).inner_text() + ) + first_default_expr.select_option("current_timestamp") + assert first_default_input.is_enabled() + first_default_input.fill("manual") + assert first_default_expr.input_value() == "" dialog.locator(".table-alter-add-column").click() assert dialog.locator(".table-alter-save").is_enabled() dialog.locator(".table-alter-column-name").last.fill("status") dialog.locator(".table-alter-column-type").last.select_option("text") + dialog.locator(".table-alter-default-options").last.locator("summary").click() dialog.locator(".table-alter-default").last.fill("planned") dialog.locator(".table-alter-save").click() review = dialog.locator(".table-alter-review") diff --git a/tests/test_table_html.py b/tests/test_table_html.py index ba9c03a5..9d1d1245 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -997,6 +997,11 @@ async def test_database_create_table_action_button_and_data(): "foreignKeyTargetsPath": "/data/-/foreign-key-targets", "databaseName": "data", "columnTypes": ["text", "integer", "float", "blob"], + "defaultExpressions": [ + "current_timestamp", + "current_date", + "current_time", + ], }, } assert "customColumnTypes" not in database_data_from_soup(soup)["createTable"] From 2ebae5ed71887cfd65af03aa88514170bdf4af17 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 12:51:23 -0700 Subject: [PATCH 043/174] Add rename table controls to alter table dialog Add a collapsed rename-table section to the alter table modal and include rename_table operations in the review/apply flow. Redirect to the renamed table URL after applying changes and cover the review text in Playwright. --- datasette/static/app.css | 22 ++++++++++ datasette/static/edit-tools.js | 75 +++++++++++++++++++++++++++++++--- tests/test_playwright.py | 30 ++++++++++++++ 3 files changed, 121 insertions(+), 6 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 32b7a4c2..f7cd97b0 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -2256,6 +2256,28 @@ dialog.table-alter-dialog::backdrop { overflow-y: auto; } +.table-alter-table-options { + border-top: 1px solid var(--rule); + padding-top: 12px; +} + +.table-alter-table-options > summary { + color: var(--ink); + cursor: pointer; + font-weight: 600; +} + +.table-alter-table-options > summary:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 3px; +} + +.table-alter-table-name-field { + display: grid; + gap: 4px; + margin-top: 10px; + max-width: 24rem; +} .table-alter-fields[hidden], .table-alter-dialog .modal-footer [hidden] { diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 2e4331e6..fd8d9a0c 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -1822,6 +1822,7 @@ function tableAlterDialogSignature(state) { return ""; } return JSON.stringify({ + tableName: state.tableNameInput ? state.tableNameInput.value.trim() : "", columns: tableAlterDialogRows(state).map(tableAlterRowSignature), deletedColumns: state.deletedColumns.slice(), }); @@ -2307,6 +2308,9 @@ function addTableAlterColumn(state, column) { function resetTableAlterDialog(state, data) { state.nextColumnIndex = 0; state.deletedColumns = []; + state.originalTableName = data.tableName || ""; + state.tableNameInput.value = state.originalTableName; + state.tableOptions.open = false; state.originalPrimaryKeys = (data.primaryKeys || []).slice(); state.originalColumnNames = (data.columns || []).map(function (column) { return column.name; @@ -2340,6 +2344,16 @@ function collectTableAlterRows(state) { } function validateTableAlterRows(state, rows) { + var tableName = state.tableNameInput.value.trim(); + if (!tableName) { + return "Table name is required."; + } + if (tableName.indexOf("\n") !== -1) { + return "Table names cannot contain newlines."; + } + if (/^sqlite_/.test(tableName)) { + return "Table names cannot start with sqlite_."; + } if (!rows.length) { return "At least one column is required."; } @@ -2513,6 +2527,13 @@ function collectTableAlterPayload(state) { } var operations = []; + var tableName = state.tableNameInput.value.trim(); + if (tableName !== state.originalTableName) { + operations.push({ + op: "rename_table", + args: { to: tableName }, + }); + } var columnTypeAssignments = collectTableAlterColumnTypeAssignments(rows); rows.forEach(function (row) { var name = row.name.trim(); @@ -2621,6 +2642,12 @@ function tableAlterReadableValue(value) { function tableAlterOperationSummary(operation) { var args = operation.args || {}; + if (operation.op === "rename_table") { + return { + text: "Rename table to " + tableAlterQuotedName(args.to) + ".", + damaging: false, + }; + } if (operation.op === "add_column") { var addDetails = ["as " + args.type]; if (args.not_null) { @@ -2788,7 +2815,10 @@ function appendTableAlterReviewText(element, text) { }); } -function tableAlterSetColumnTypeUrl() { +function tableAlterSetColumnTypeUrl(tableUrl) { + if (tableUrl) { + return tableUrl.replace(/\/$/, "") + "/-/set-column-type"; + } var data = tableAlterData(); if (!data || !data.path) { return null; @@ -2798,11 +2828,11 @@ function tableAlterSetColumnTypeUrl() { return url.toString(); } -async function assignTableAlterColumnTypes(assignments) { +async function assignTableAlterColumnTypes(assignments, tableUrl) { if (!assignments.length) { return; } - var url = tableAlterSetColumnTypeUrl(); + var url = tableAlterSetColumnTypeUrl(tableUrl); if (!url) { throw new Error("Could not find the set column type URL."); } @@ -2841,6 +2871,16 @@ async function assignTableAlterColumnTypes(assignments) { } } +function tableAlterResultRenamesTable(result) { + return !!( + result && + result.payload && + (result.payload.operations || []).some(function (operation) { + return operation.op === "rename_table"; + }) + ); +} + function showTableAlterEditor(state) { state.mode = "edit"; state.reviewResult = null; @@ -2923,6 +2963,7 @@ async function applyTableAlterChanges(state, result) { } setTableAlterDialogSaving(state, true); try { + var responseData = null; if (result.payload) { var response = await fetch(data.path, { method: "POST", @@ -2932,7 +2973,6 @@ async function applyTableAlterChanges(state, result) { }, body: JSON.stringify(result.payload), }); - var responseData = null; try { responseData = await response.json(); } catch (_error) { @@ -2942,10 +2982,18 @@ async function applyTableAlterChanges(state, result) { throw rowMutationRequestError(response, responseData); } } - await assignTableAlterColumnTypes(result.columnTypeAssignments || []); + var tableUrl = responseData && responseData.table_url; + await assignTableAlterColumnTypes( + result.columnTypeAssignments || [], + tableUrl, + ); state.shouldRestoreFocus = false; state.dialog.close(); - location.reload(); + if (tableAlterResultRenamesTable(result) && tableUrl) { + window.location.href = tableUrl; + } else { + location.reload(); + } } catch (error) { setTableAlterDialogSaving(state, false); showTableAlterDialogError(state, error.message || "Could not alter table"); @@ -3088,6 +3136,13 @@ function ensureTableAlterDialog(manager) {
+
+ Rename table +
+ + +
+
@@ -3106,6 +3161,8 @@ function ensureTableAlterDialog(manager) { title: dialog.querySelector(".modal-title"), error: dialog.querySelector(".table-alter-error"), fields: dialog.querySelector(".table-alter-fields"), + tableOptions: dialog.querySelector(".table-alter-table-options"), + tableNameInput: dialog.querySelector(".table-alter-table-name"), review: dialog.querySelector(".table-alter-review"), columnList: dialog.querySelector(".table-alter-column-list"), addColumnButton: dialog.querySelector(".table-alter-add-column"), @@ -3117,6 +3174,7 @@ function ensureTableAlterDialog(manager) { shouldRestoreFocus: true, isSaving: false, initialSignature: "", + originalTableName: "", nextColumnIndex: 0, deletedColumns: [], originalColumnNames: [], @@ -3134,6 +3192,11 @@ function ensureTableAlterDialog(manager) { saveTableAlterDialog(tableAlterDialogState); }); + tableAlterDialogState.tableNameInput.addEventListener("input", function () { + clearTableAlterDialogError(tableAlterDialogState); + updateTableAlterSaveButtonState(tableAlterDialogState); + }); + tableAlterDialogState.addColumnButton.addEventListener("click", function () { if (tableAlterDialogState.isSaving) { return; diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 5719039a..fcf97a90 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -561,6 +561,36 @@ def test_alter_table_review_rename_primary_key_column(page, datasette_server): ] +@pytest.mark.playwright +def test_alter_table_review_rename_table(page, datasette_server): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + save = dialog.locator(".table-alter-save") + rename_details = dialog.locator(".table-alter-table-options") + assert rename_details.locator("summary").inner_text() == "Rename table" + assert not dialog.locator(".table-alter-table-name").is_visible() + assert save.is_disabled() + + rename_details.locator("summary").click() + table_name = dialog.locator(".table-alter-table-name") + assert table_name.input_value() == "projects" + assert table_name.get_attribute("placeholder") == "table name" + table_name.fill("projects_archive") + assert save.is_enabled() + save.click() + + review = dialog.locator(".table-alter-review") + review.wait_for() + assert "Rename table to projects_archive." in review.inner_text() + assert dialog.locator(".table-alter-review-name").all_inner_texts() == [ + "projects_archive", + ] + + @pytest.mark.playwright def test_alter_table_review_not_null_wording(page, datasette_server): page.goto(f"{datasette_server}data/projects") From 1717ab02f720286aea8e4634a1047e6a6f2a1d45 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 22 Jun 2026 13:22:56 -0700 Subject: [PATCH 044/174] Replace row edit icon SVG in table.py Closes #2796 --- datasette/views/table.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index c5448c85..0e291524 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -455,8 +455,8 @@ async def display_columns_and_rows( 'xmlns="http://www.w3.org/2000/svg" width="14" height="14" ' 'viewBox="0 0 24 24" fill="none" stroke="currentColor" ' 'stroke-width="2" stroke-linecap="round" stroke-linejoin="round">' - '' - '' + '' + '' "" ) delete_icon = ( From b932d0dc78a3ddc466409a6d21a31124a20d8c17 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 13:42:35 -0700 Subject: [PATCH 045/174] current_unixtime and current_unixtime_ms default_expr options Plus tweaked how alter table changing those works a bit. --- datasette/static/edit-tools.js | 119 +++++++++++++++++++++++--- datasette/views/table.py | 41 +++++---- datasette/views/table_create_alter.py | 116 +++++++++++++++++++++++-- docs/json_api.rst | 32 ++++++- tests/test_api_write.py | 119 +++++++++++++++++++++++++- tests/test_playwright.py | 81 ++++++++++++++++++ tests/test_table_html.py | 69 ++++++++++++--- 7 files changed, 522 insertions(+), 55 deletions(-) diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index fd8d9a0c..9f4f89b9 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -109,14 +109,64 @@ function createCustomColumnTypeSelect(options, className, placeholderClass) { return select; } -var DEFAULT_EXPRESSION_LABELS = { - current_timestamp: "Current timestamp in UTC, e.g. 2026-05-01 13:34:00", - current_date: "Current date in UTC, e.g. 2026-05-01", - current_time: "Current time in UTC, e.g. 13:34:00", -}; +function normalizeDefaultExpressionOption(option) { + if (typeof option === "string") { + return { + value: option, + label: option.replace(/_/g, " "), + sqliteType: "", + }; + } + option = option || {}; + return { + value: option.value || "", + label: option.label || option.value || "", + sqliteType: option.sqliteType || "", + }; +} -function defaultExpressionLabel(value) { - return DEFAULT_EXPRESSION_LABELS[value] || value.replace(/_/g, " "); +function defaultExpressionOptionForValue(options, value) { + var match = null; + (options || []).some(function (option) { + var normalized = normalizeDefaultExpressionOption(option); + if (normalized.value === value) { + match = normalized; + return true; + } + return false; + }); + return match; +} + +function defaultExpressionLabelForValue(options, value) { + var option = defaultExpressionOptionForValue(options, value); + return option && option.label + ? option.label + : value + ? value.replace(/_/g, " ") + : ""; +} + +function applyDefaultExpressionColumnType(row, prefix, options, columnTypes) { + var defaultExprSelect = row.querySelector("." + prefix + "-default-expr"); + var typeSelect = row.querySelector("." + prefix + "-column-type"); + if (!defaultExprSelect || !typeSelect || !defaultExprSelect.value) { + return false; + } + var option = defaultExpressionOptionForValue( + options, + defaultExprSelect.value, + ); + if ( + option && + option.sqliteType && + (columnTypes || []).indexOf(option.sqliteType) !== -1 && + typeSelect.value !== option.sqliteType + ) { + typeSelect.value = option.sqliteType; + return true; + } + return false; } function createDefaultExpressionSelect( @@ -132,9 +182,13 @@ function createDefaultExpressionSelect( blankOption.textContent = "- default expr -"; select.appendChild(blankOption); options.forEach(function (option) { + var normalized = normalizeDefaultExpressionOption(option); + if (!normalized.value) { + return; + } var optionElement = document.createElement("option"); - optionElement.value = option; - optionElement.textContent = defaultExpressionLabel(option); + optionElement.value = normalized.value; + optionElement.textContent = normalized.label; select.appendChild(optionElement); }); select.value = value || ""; @@ -1078,6 +1132,18 @@ function createTableColumnRow(state, column) { if (defaultControls.defaultExprSelect.value) { defaultControls.defaultInput.value = ""; } + if ( + applyDefaultExpressionColumnType( + row, + "table-create", + tableCreateDefaultExpressions(), + tableCreateColumnTypes(), + ) + ) { + syncTableCreateCustomTypeForSqliteType(row); + syncTableCreateForeignKeyOptions(row, state); + syncTableCreateCustomTypeAndForeignKey(row, state); + } syncSchemaDialogDefaultControls(row, "table-create"); clearTableCreateDialogError(state); }); @@ -1948,8 +2014,15 @@ function createTableAlterColumnRow(state, column) { var originalName = existing ? column.name || "" : ""; var originalCustomType = existing && column.column_type ? column.column_type.type || "" : ""; + var originalDefaultExpr = + existing && column.has_default && column.default_expr + ? column.default_expr + : ""; var originalDefault = - existing && column.has_default && column.default !== null + existing && + column.has_default && + !originalDefaultExpr && + column.default !== null ? String(column.default) : ""; var originalForeignKey = @@ -1965,6 +2038,7 @@ function createTableAlterColumnRow(state, column) { row.dataset.originalNotNull = existing && column.notnull ? "1" : "0"; row.dataset.originalHasDefault = existing && column.has_default ? "1" : "0"; row.dataset.originalDefault = originalDefault; + row.dataset.originalDefaultExpr = originalDefaultExpr; row.dataset.originalPk = existing && column.is_pk ? "1" : "0"; row.dataset.originalCustomType = originalCustomType; row.dataset.originalForeignKey = originalForeignKey; @@ -2051,7 +2125,7 @@ function createTableAlterColumnRow(state, column) { tableAlterDefaultExpressions(), { defaultValue: originalDefault, - defaultExpr: "", + defaultExpr: originalDefaultExpr, }, ); var defaultInput = defaultControls.defaultInput; @@ -2167,6 +2241,18 @@ function createTableAlterColumnRow(state, column) { if (defaultExprSelect.value) { defaultInput.value = ""; } + if ( + applyDefaultExpressionColumnType( + row, + "table-alter", + tableAlterDefaultExpressions(), + tableAlterColumnTypes(), + ) + ) { + syncTableAlterCustomTypeForSqliteType(row); + syncTableAlterForeignKeyOptions(row, state); + syncSchemaDialogCustomTypeAndForeignKey(row, state, "table-alter"); + } syncSchemaDialogDefaultControls(row, "table-alter"); updateTableAlterSaveButtonState(state); }); @@ -2336,6 +2422,7 @@ function collectTableAlterRows(state) { signature.originalNotNull = row.dataset.originalNotNull === "1"; signature.originalHasDefault = row.dataset.originalHasDefault === "1"; signature.originalDefault = row.dataset.originalDefault || ""; + signature.originalDefaultExpr = row.dataset.originalDefaultExpr || ""; signature.originalPk = row.dataset.originalPk === "1"; signature.originalCustomType = row.dataset.originalCustomType || ""; signature.originalForeignKey = row.dataset.originalForeignKey || ""; @@ -2567,8 +2654,12 @@ function collectTableAlterPayload(state) { if (row.notNull !== row.originalNotNull) { alterArgs.not_null = row.notNull; } - if (row.defaultExpr) { - alterArgs.default_expr = row.defaultExpr; + if (row.defaultExpr !== row.originalDefaultExpr) { + if (row.defaultExpr) { + alterArgs.default_expr = row.defaultExpr; + } else { + alterArgs.default = row.defaultValue === "" ? null : row.defaultValue; + } } else if (row.originalHasDefault) { if (row.defaultValue !== row.originalDefault) { alterArgs.default = row.defaultValue === "" ? null : row.defaultValue; @@ -2630,7 +2721,7 @@ function tableAlterQuotedName(name) { } function tableAlterReadableDefaultExpression(value) { - return value ? value.replace(/_/g, " ") : ""; + return defaultExpressionLabelForValue(tableAlterDefaultExpressions(), value); } function tableAlterReadableValue(value) { diff --git a/datasette/views/table.py b/datasette/views/table.py index dbca3a18..80e1c514 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -51,8 +51,9 @@ from .database import QueryView from .table_create_alter import ( ALTER_TABLE_COLUMN_TYPES, ALTER_TABLE_TYPE_FOR_SQLITE_TYPE, - DEFAULT_EXPR_SQL, _custom_column_type_options_for_create_table, + default_expr_for_sql, + default_expression_options, ) from .table_extras import ( TABLE_EXTRA_BUNDLES, @@ -401,23 +402,25 @@ async def _table_alter_ui( continue sqlite_type = SQLiteType.from_declared_type(column.type) column_type = column_types_map.get(column.name) - columns.append( - { - "name": column.name, - "type": ALTER_TABLE_TYPE_FOR_SQLITE_TYPE.get(sqlite_type, "text"), - "sqlite_type": sqlite_type.value, - "notnull": column.notnull, - "default": column.default_value, - "has_default": column.default_value is not None, - "is_pk": column.name in pks, - "foreign_key": foreign_keys_by_column.get(column.name), - "column_type": ( - {"type": column_type.name, "config": column_type.config} - if column_type is not None - else None - ), - } - ) + default_expr = default_expr_for_sql(column.default_value) + column_data = { + "name": column.name, + "type": ALTER_TABLE_TYPE_FOR_SQLITE_TYPE.get(sqlite_type, "text"), + "sqlite_type": sqlite_type.value, + "notnull": column.notnull, + "default": None if default_expr else column.default_value, + "has_default": column.default_value is not None, + "is_pk": column.name in pks, + "foreign_key": foreign_keys_by_column.get(column.name), + "column_type": ( + {"type": column_type.name, "config": column_type.config} + if column_type is not None + else None + ), + } + if default_expr: + column_data["default_expr"] = default_expr + columns.append(column_data) data = { "path": "{}/-/alter".format(datasette.urls.table(database_name, table_name)), @@ -425,7 +428,7 @@ async def _table_alter_ui( "columns": columns, "primaryKeys": pks, "columnTypes": ALTER_TABLE_COLUMN_TYPES, - "defaultExpressions": list(DEFAULT_EXPR_SQL), + "defaultExpressions": default_expression_options(), "foreignKeyTargetsPath": "{}/-/foreign-key-targets?table={}".format( datasette.urls.database(database_name), urllib.parse.quote(table_name, safe=""), diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 111dfab1..ffeb7f14 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -266,7 +266,7 @@ async def _create_table_ui_context( ), "databaseName": database_name, "columnTypes": CREATE_TABLE_COLUMN_TYPES, - "defaultExpressions": list(DEFAULT_EXPR_SQL), + "defaultExpressions": default_expression_options(), } can_set_column_type = await datasette.allowed( action="set-column-type", @@ -308,12 +308,109 @@ def _custom_column_type_options_for_create_table(datasette): SqliteApiType = Literal["text", "integer", "float", "blob"] -DefaultExpr = Literal["current_timestamp", "current_date", "current_time"] -DEFAULT_EXPR_SQL = { - "current_timestamp": "CURRENT_TIMESTAMP", - "current_date": "CURRENT_DATE", - "current_time": "CURRENT_TIME", +DEFAULT_EXPRESSIONS = { + "current_timestamp": { + "sql": "CURRENT_TIMESTAMP", + "label": "Current timestamp in UTC, e.g. 2026-05-01 13:34:00", + "sqliteType": "text", + }, + "current_date": { + "sql": "CURRENT_DATE", + "label": "Current date in UTC, e.g. 2026-05-01", + "sqliteType": "text", + }, + "current_time": { + "sql": "CURRENT_TIME", + "label": "Current time in UTC, e.g. 13:34:00", + "sqliteType": "text", + }, + "current_unixtime": { + "sql": "(CAST(strftime('%s', 'now') AS INTEGER))", + "label": "Current Unix time, integer seconds since the epoch", + "sqliteType": "integer", + }, + "current_unixtime_ms": { + "sql": "(CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER))", + "label": "Current Unix time, integer milliseconds since the epoch", + "sqliteType": "integer", + }, } +DefaultExpr = str +DEFAULT_EXPR_SQL = { + name: metadata["sql"] for name, metadata in DEFAULT_EXPRESSIONS.items() +} + + +def _strip_wrapping_parentheses(expression): + expression = expression.strip() + while expression.startswith("(") and expression.endswith(")"): + depth = 0 + in_single_quote = False + wraps_whole_expression = True + i = 0 + while i < len(expression): + char = expression[i] + if char == "'": + if ( + in_single_quote + and i + 1 < len(expression) + and expression[i + 1] == "'" + ): + i += 2 + continue + in_single_quote = not in_single_quote + elif not in_single_quote: + if char == "(": + depth += 1 + elif char == ")": + depth -= 1 + if depth == 0 and i != len(expression) - 1: + wraps_whole_expression = False + break + i += 1 + if not wraps_whole_expression or depth != 0 or in_single_quote: + break + expression = expression[1:-1].strip() + return expression + + +def _default_expression_lookup_key(expression): + return re.sub(r"\s+", " ", _strip_wrapping_parentheses(expression)).lower() + + +DEFAULT_EXPR_BY_SQL = { + _default_expression_lookup_key(sql): name for name, sql in DEFAULT_EXPR_SQL.items() +} + + +def default_expr_for_sql(expression): + if expression is None: + return None + return DEFAULT_EXPR_BY_SQL.get(_default_expression_lookup_key(expression)) + + +def _quoted_options(options): + if len(options) == 1: + return "'{}'".format(options[0]) + return "{} or '{}'".format( + ", ".join("'{}'".format(option) for option in options[:-1]), + options[-1], + ) + + +def _default_expr_error_message(): + return "Input should be {}".format(_quoted_options(list(DEFAULT_EXPRESSIONS))) + + +def default_expression_options(): + return [ + { + "value": value, + "label": metadata["label"], + "sqliteType": metadata["sqliteType"], + } + for value, metadata in DEFAULT_EXPRESSIONS.items() + ] class _StrictPydanticModel(BaseModel): @@ -324,6 +421,13 @@ class _DefaultArgsMixin(_StrictPydanticModel): default: Any | None = None default_expr: DefaultExpr | None = None + @field_validator("default_expr") + @classmethod + def validate_default_expr_value(cls, value): + if value is not None and value not in DEFAULT_EXPRESSIONS: + raise PydanticCustomError("default_expr", _default_expr_error_message()) + return value + @model_validator(mode="after") def validate_default_fields(self): has_default = "default" in self.model_fields_set diff --git a/docs/json_api.rst b/docs/json_api.rst index 849a8acf..b8632bc3 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -44,7 +44,7 @@ looks like this: ``"ok"`` is always ``true`` if an error did not occur. -The ``"rows"`` key is a list of objects, each one representing a row. +The ``"rows"`` key is a list of objects, each one representing a row. The ``"truncated"`` key lets you know if the query was truncated. This can happen if a SQL query returns more than 1,000 results (or the :ref:`setting_max_returned_rows` setting). @@ -1990,7 +1990,7 @@ The JSON here describes the table that will be created: - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. - ``not_null`` can be set to ``true`` to create this column with a ``NOT NULL`` constraint. - ``default`` can be used to set a literal default value for this column. - - ``default_expr`` can be used instead of ``default`` to set a SQLite default expression. The supported values are ``current_timestamp``, ``current_date`` and ``current_time``. + - ``default_expr`` can be used instead of ``default`` to set a SQLite default expression. See :ref:`default_expr values `. - ``fk_table`` can be used to create a single-column foreign key constraint referencing another table. ``fk_column`` is optional and can be used to specify the referenced column - if omitted, Datasette will use the single primary key of ``fk_table``. * ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. @@ -2004,6 +2004,32 @@ The JSON here describes the table that will be created: * ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. This requires the :ref:`actions_update_row` permission. * ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`actions_alter_table` permission. +.. _json_api_default_expr_values: + +``default_expr`` accepts these values: + +.. list-table:: + :header-rows: 1 + + * - Value + - Recommended column type + - Example inserted value + * - ``current_timestamp`` + - ``text`` + - ``2026-05-01 13:34:00`` + * - ``current_date`` + - ``text`` + - ``2026-05-01`` + * - ``current_time`` + - ``text`` + - ``13:34:00`` + * - ``current_unixtime`` + - ``integer`` + - ``1777642440`` + * - ``current_unixtime_ms`` + - ``integer`` + - ``1777642440000`` + This example creates a foreign key from ``projects.owner_id`` to the single primary key of ``owners``: .. code-block:: json @@ -2314,7 +2340,7 @@ Supported operations: * ``set_foreign_keys`` replaces all foreign key constraints on the table. ``args`` accepts ``foreign_keys``, a list of objects that each have ``column``, ``fk_table`` and optional ``fk_column``. An empty list removes all foreign key constraints. * ``reorder_columns`` reorders columns. ``args`` accepts ``columns``, a list of one or more column names. Columns omitted from this list will appear afterwards in their existing order. -``default`` is always treated as a literal value. ``default_expr`` accepts one of ``current_timestamp``, ``current_date`` or ``current_time`` and is rendered as the corresponding SQLite default expression. +``default`` is always treated as a literal value. ``default_expr`` accepts the values shown in :ref:`default_expr values ` and is rendered as the corresponding SQLite default expression. For foreign key operations that omit ``fk_column``, the referenced ``fk_table`` must have a single-column primary key. Datasette will return an error if it cannot identify a single primary key column for that table. diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 6e8c4d4b..b3a2f6a3 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -899,6 +899,64 @@ async def test_alter_table_operations(ds_write): assert event.after_schema == data["schema"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "default_expr,minimum_value,expected_schema", + ( + ( + "current_unixtime", + 1_600_000_000, + "strftime('%s', 'now')", + ), + ( + "current_unixtime_ms", + 1_600_000_000_000, + "julianday('now')", + ), + ), +) +async def test_alter_table_integer_default_expr( + ds_write, default_expr, minimum_value, expected_schema +): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + { + "op": "add_column", + "args": { + "name": "created", + "type": "integer", + "default_expr": default_expr, + }, + } + ] + }, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert expected_schema in data["schema"] + + columns = await db.execute("select * from pragma_table_info('docs')") + created_column = [ + column for column in columns.dicts() if column["name"] == "created" + ][0] + assert created_column["type"] == "INTEGER" + assert expected_schema in created_column["dflt_value"] + + row = await db.execute_write_fn( + lambda conn: conn.execute( + "insert into docs (title) values ('with default') " + "returning created, typeof(created)" + ).fetchone() + ) + assert row[0] > minimum_value + assert row[1] == "integer" + + @pytest.mark.asyncio async def test_alter_table_rename_table(ds_write): token = write_token(ds_write, permissions=["at"]) @@ -1315,7 +1373,7 @@ async def test_alter_table_permission_denied(ds_write): } ] }, - "operations.0.add_column.args.default_expr: Input should be 'current_timestamp', 'current_date' or 'current_time'", + "operations.0.add_column.args.default_expr: Input should be 'current_timestamp', 'current_date', 'current_time', 'current_unixtime' or 'current_unixtime_ms'", ), ( { @@ -2053,6 +2111,63 @@ async def test_create_table_with_column_constraints(ds_write): assert columns[4]["dflt_value"] == "'hello)'" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "default_expr,minimum_value,expected_schema", + ( + ( + "current_unixtime", + 1_600_000_000, + "strftime('%s', 'now')", + ), + ( + "current_unixtime_ms", + 1_600_000_000_000, + "julianday('now')", + ), + ), +) +async def test_create_table_integer_default_expr( + ds_write, default_expr, minimum_value, expected_schema +): + token = write_token(ds_write) + table = "default_{}".format(default_expr) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": table, + "columns": [ + {"name": "id", "type": "integer"}, + { + "name": "created", + "type": "integer", + "default_expr": default_expr, + }, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201, response.text + data = response.json() + assert expected_schema in data["schema"] + + db = ds_write.get_database("data") + columns = (await db.execute("select * from pragma_table_info(?)", [table])).dicts() + assert columns[1]["type"] == "INTEGER" + assert expected_schema in columns[1]["dflt_value"] + + row = await db.execute_write_fn( + lambda conn: conn.execute( + "insert into [{}] default values returning created, typeof(created)".format( + table + ) + ).fetchone() + ) + assert row[0] > minimum_value + assert row[1] == "integer" + + @pytest.mark.asyncio @pytest.mark.parametrize( "column,expected_error", @@ -2071,7 +2186,7 @@ async def test_create_table_with_column_constraints(ds_write): "type": "text", "default_expr": "datetime('now')", }, - "columns.0.default_expr: Input should be 'current_timestamp', 'current_date' or 'current_time'", + "columns.0.default_expr: Input should be 'current_timestamp', 'current_date', 'current_time', 'current_unixtime' or 'current_unixtime_ms'", ), ( { diff --git a/tests/test_playwright.py b/tests/test_playwright.py index fcf97a90..ee396de5 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -98,6 +98,10 @@ def write_playwright_database(db_path): notes text, score integer default 5 ); + create table defaults_demo ( + id integer primary key, + created_ms integer default (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)) + ); insert into projects (title, metadata, logo, notes, score) values ( 'Build Datasette', @@ -136,6 +140,11 @@ def write_playwright_config(config_path): "delete-row": True, }, }, + "defaults_demo": { + "permissions": { + "alter-table": True, + }, + }, }, }, }, @@ -394,6 +403,32 @@ def test_create_table_foreign_key_selection_updates_column_type(page, datasette_ assert foreign_key_select.input_value() == "projects\u001fid" +@pytest.mark.playwright +def test_create_table_unix_default_expression_updates_column_type( + page, datasette_server +): + page.goto(f"{datasette_server}data") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-database-action="create-table"]').click() + + dialog = page.locator("#table-create-dialog") + dialog.wait_for() + row = dialog.locator(".table-create-column-row").nth(1) + row.locator(".table-create-more-options").click() + row.locator(".table-create-default-options summary").click() + + type_select = row.locator(".table-create-column-type") + default_expr = row.locator(".table-create-default-expr") + assert type_select.input_value() == "text" + assert ( + "Current Unix time, integer milliseconds since the epoch" + in default_expr.locator("option").last.inner_text() + ) + + default_expr.select_option("current_unixtime_ms") + assert type_select.input_value() == "integer" + + @pytest.mark.playwright def test_alter_table_foreign_key_selection_updates_blank_column(page, datasette_server): page.goto(f"{datasette_server}data/projects") @@ -416,6 +451,52 @@ def test_alter_table_foreign_key_selection_updates_blank_column(page, datasette_ assert foreign_key_select.input_value() == "projects\u001fid" +@pytest.mark.playwright +def test_alter_table_unix_default_expression_updates_column_type( + page, datasette_server +): + page.goto(f"{datasette_server}data/projects") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + dialog.locator(".table-alter-add-column").click() + row = dialog.locator(".table-alter-column-row").last + row.locator(".table-alter-default-options summary").click() + + type_select = row.locator(".table-alter-column-type") + default_expr = row.locator(".table-alter-default-expr") + assert type_select.input_value() == "text" + assert ( + "Current Unix time, integer seconds since the epoch" + in default_expr.locator("option").all_inner_texts() + ) + + default_expr.select_option("current_unixtime") + assert type_select.input_value() == "integer" + + +@pytest.mark.playwright +def test_alter_table_existing_default_expression_populates_select( + page, datasette_server +): + page.goto(f"{datasette_server}data/defaults_demo") + page.locator("details.actions-menu-links summary").click() + page.locator('button[data-table-action="alter-table"]').click() + + dialog = page.locator("#table-alter-dialog") + dialog.wait_for() + row = dialog.locator(".table-alter-column-row").nth(1) + row.locator(".table-alter-more-options").click() + row.locator(".table-alter-default-options summary").click() + + assert row.locator(".table-alter-default-expr").input_value() == ( + "current_unixtime_ms" + ) + assert row.locator(".table-alter-default").input_value() == "" + + @pytest.mark.playwright def test_alter_table_flow(page, datasette_server): page.goto(f"{datasette_server}data/projects") diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 9d1d1245..3af2bb08 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -40,6 +40,35 @@ def database_data_from_soup(soup): return json.loads(match.group(1)) +DEFAULT_EXPRESSION_OPTIONS = [ + { + "value": "current_timestamp", + "label": "Current timestamp in UTC, e.g. 2026-05-01 13:34:00", + "sqliteType": "text", + }, + { + "value": "current_date", + "label": "Current date in UTC, e.g. 2026-05-01", + "sqliteType": "text", + }, + { + "value": "current_time", + "label": "Current time in UTC, e.g. 13:34:00", + "sqliteType": "text", + }, + { + "value": "current_unixtime", + "label": "Current Unix time, integer seconds since the epoch", + "sqliteType": "integer", + }, + { + "value": "current_unixtime_ms", + "label": "Current Unix time, integer milliseconds since the epoch", + "sqliteType": "integer", + }, +] + + @pytest.mark.asyncio @pytest.mark.parametrize( "path,expected_definition_sql", @@ -997,11 +1026,7 @@ async def test_database_create_table_action_button_and_data(): "foreignKeyTargetsPath": "/data/-/foreign-key-targets", "databaseName": "data", "columnTypes": ["text", "integer", "float", "blob"], - "defaultExpressions": [ - "current_timestamp", - "current_date", - "current_time", - ], + "defaultExpressions": DEFAULT_EXPRESSION_OPTIONS, }, } assert "customColumnTypes" not in database_data_from_soup(soup)["createTable"] @@ -1113,7 +1138,9 @@ async def test_table_alter_action_button_and_data(): create table items ( id integer primary key, name text not null, - score integer default 5 + score integer default 5, + created text default current_timestamp, + created_ms integer default (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)) ); """) response = await ds.client.get("/data/items", actor={"id": "root"}) @@ -1145,11 +1172,7 @@ async def test_table_alter_action_button_and_data(): assert alter_data["foreignKeyTargetsPath"] == ( "/data/-/foreign-key-targets?table=items" ) - assert alter_data["defaultExpressions"] == [ - "current_timestamp", - "current_date", - "current_time", - ] + assert alter_data["defaultExpressions"] == DEFAULT_EXPRESSION_OPTIONS assert [option["name"] for option in alter_data["customColumnTypes"]] == [ "email", "json", @@ -1191,6 +1214,30 @@ async def test_table_alter_action_button_and_data(): "foreign_key": None, "column_type": None, }, + { + "name": "created", + "type": "text", + "sqlite_type": "TEXT", + "notnull": 0, + "default": None, + "default_expr": "current_timestamp", + "has_default": True, + "is_pk": False, + "foreign_key": None, + "column_type": None, + }, + { + "name": "created_ms", + "type": "integer", + "sqlite_type": "INTEGER", + "notnull": 0, + "default": None, + "default_expr": "current_unixtime_ms", + "has_default": True, + "is_pk": False, + "foreign_key": None, + "column_type": None, + }, ] response_without_permission = await ds.client.get( From b3b5c25df8c51fb0da39a1c8f19198cd5c0b4afb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 13:51:32 -0700 Subject: [PATCH 046/174] Draft changelog for create/alter table UI, refs #2787, #2788 --- docs/changelog.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1516604e..697df87d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _unreleased: + +Unreleased +---------- + +- New "Create table" interface in the database actions menu, backed by the ``//-/create`` :ref:`JSON API `. It can define columns, primary keys, custom column types, ``NOT NULL`` constraints, literal defaults, expression defaults and single-column foreign keys. (:issue:`2787`) +- New "Alter table" table action and ``//
/-/alter`` :ref:`JSON API ` for changing existing tables: add, rename, reorder and drop columns; change column types, defaults, ``NOT NULL`` constraints, primary keys and foreign keys; and rename the table. The alter table dialog also includes a "Drop table" button. (:issue:`2788`) +- New ``//-/foreign-key-targets`` and ``//
/-/foreign-key-suggestions`` JSON APIs for discovering valid single-column foreign key targets and suggested relationships. +- Create and alter table dialogs share their column-editing controls, including literal and expression defaults, custom column types, foreign keys and column ordering. + .. _v1_0_a34: 1.0a34 (2026-06-16) From f8313525517b774879482c597172da076b4ba59b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 14:24:34 -0700 Subject: [PATCH 047/174] Avoid SQLite RETURNING for compat with older SQLite Refs #2789 --- tests/test_api_write.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index b3a2f6a3..76797742 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1,6 +1,6 @@ from datasette.app import Datasette from datasette.events import RenameTableEvent -from datasette.utils import sqlite3 +from datasette.utils import escape_sqlite, sqlite3 from .utils import last_event import pytest import time @@ -40,6 +40,16 @@ def _headers(token): } +def _insert_and_fetch_created(conn, table, insert_sql): + cursor = conn.execute(insert_sql) + return conn.execute( + "select created, typeof(created) from {} where rowid = ?".format( + escape_sqlite(table) + ), + (cursor.lastrowid,), + ).fetchone() + + @pytest.mark.asyncio async def test_api_explorer_upsert_example_json(ds_write): response = await ds_write.client.get("/-/api", actor={"id": "root"}) @@ -948,10 +958,9 @@ async def test_alter_table_integer_default_expr( assert expected_schema in created_column["dflt_value"] row = await db.execute_write_fn( - lambda conn: conn.execute( - "insert into docs (title) values ('with default') " - "returning created, typeof(created)" - ).fetchone() + lambda conn: _insert_and_fetch_created( + conn, "docs", "insert into docs (title) values ('with default')" + ) ) assert row[0] > minimum_value assert row[1] == "integer" @@ -2158,11 +2167,9 @@ async def test_create_table_integer_default_expr( assert expected_schema in columns[1]["dflt_value"] row = await db.execute_write_fn( - lambda conn: conn.execute( - "insert into [{}] default values returning created, typeof(created)".format( - table - ) - ).fetchone() + lambda conn: _insert_and_fetch_created( + conn, table, "insert into {} default values".format(escape_sqlite(table)) + ) ) assert row[0] > minimum_value assert row[1] == "integer" From 86ea1d472275d353105cb2a8b2109c9ffc1a343e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 18:19:49 -0700 Subject: [PATCH 048/174] Visual improvements to table filter UI - Looks nicer - Add / remove buttons work properly Closes #2798 --- datasette/static/app.css | 300 +++++++++++++++++++++++++----- datasette/static/table.js | 169 ++++++++++++++--- datasette/templates/patterns.html | 14 +- datasette/templates/table.html | 14 +- 4 files changed, 411 insertions(+), 86 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index f7cd97b0..ce800f61 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -609,9 +609,6 @@ button.core[type=button] { border-color: #007bff; } -.filter-row { - margin-bottom: 0.6em; -} .search-row { margin-bottom: 1.8em; } @@ -623,72 +620,239 @@ button.core[type=button] { width: 80px; } -.select-wrapper { - border: 1px solid #ccc; - width: 120px; - border-radius: 3px; +.filters .search-row { + box-sizing: border-box; + display: grid; + grid-template-columns: max-content minmax(16rem, 1fr); + align-items: center; + gap: 0.6rem; + margin: 0 0 0.45rem; + padding-right: var(--filter-two-icon-space); +} + +.filters .search-row label { + width: auto; padding: 0; - background-color: #fafafa; - position: relative; + color: var(--filter-muted); + font-size: 0.875rem; + font-weight: 600; +} + +.filters { + --filter-ink: #0f0f0f; + --filter-paper: #eef6ff; + --filter-muted: #6b6b6b; + --filter-rule: #d8e6f5; + --filter-accent: #1a56db; + --filter-control-border: #bfccd9; + --filter-control-height: 2.125rem; + --filter-control-gap: 0.4rem; + --filter-row-icon-size: 2rem; + --filter-two-icon-space: calc( + (2 * var(--filter-row-icon-size)) + (2 * var(--filter-control-gap)) + ); + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 0.55rem; + max-width: 760px; + margin: 0 0 1rem; + padding: 0.75rem; + border: 1px solid var(--filter-rule); + border-radius: 8px; + background: var(--filter-paper); + color: var(--filter-ink); + font-size: 0.875rem; +} + +.filters .filter-row { + margin-bottom: 0; +} + +.filters .filter-controls-row { + display: grid; + min-width: 0; + width: 100%; + max-width: 100%; + box-sizing: border-box; + grid-template-columns: + minmax(8rem, 0.75fr) + minmax(6.5rem, 0.5fr) + minmax(12rem, 1.2fr) + var(--filter-row-icon-size) + var(--filter-row-icon-size); + gap: var(--filter-control-gap); + align-items: center; +} + +.filters .filter-actions-row { + display: flex; + align-items: center; + justify-content: flex-start; + flex-wrap: wrap; + gap: 0.6rem; +} + +.select-wrapper { display: inline-block; - margin-right: 0.3em; -} -.select-wrapper:focus-within { - border: 1px solid black; + width: 120px; + min-width: 0; } + .select-wrapper.filter-op { width: 80px; } -.select-wrapper::after { - content: "\25BE"; - position: absolute; - top: 0px; - right: 0.4em; - color: #bbb; - pointer-events: none; - font-size: 1.2em; - padding-top: 0.16em; + +.filters .select-wrapper { + width: auto; } .select-wrapper select { - padding: 9px 8px; + box-sizing: border-box; width: 100%; - border: none; - box-shadow: none; - background: transparent; - background-image: none; - -webkit-appearance: none; - -moz-appearance: none; + height: var(--filter-control-height); + border: 1px solid var(--filter-control-border, #ccc); + border-radius: 5px; + background-color: #fff; + color: inherit; cursor: pointer; + font-family: inherit; + font-size: 0.875rem; + line-height: 1.25; + padding: 7px 8px; } -.select-wrapper select { - font-size: 1em; - font-family: Helvetica, sans-serif; -} + .select-wrapper option { font-size: 1em; font-family: Helvetica, sans-serif; } .select-wrapper select:focus { + border-color: var(--filter-accent, #000); + box-shadow: 0 0 0 2px rgba(26, 86, 219, 0.14); outline: none; } -.filters { - font-size: 0.8em; -} .filters input.filter-value { - width: 200px; - border-radius: 3px; - -webkit-appearance: none; - padding: 9px 4px; - font-size: 16px; - font-family: Helvetica, sans-serif; + box-sizing: border-box; + width: 100%; + min-width: 0; + height: var(--filter-control-height); + border: 1px solid var(--filter-control-border); + border-radius: 5px; + background: #fff; + color: inherit; + font-family: inherit; + font-size: 0.875rem; + line-height: 1.25; + padding: 7px 9px; +} + +.filters input.filter-value:focus { + border-color: var(--filter-accent); + box-shadow: 0 0 0 2px rgba(26, 86, 219, 0.14); + outline: none; +} + +.filters input[type=submit] { + border-color: var(--filter-accent); + background: var(--filter-accent); + border-radius: 5px; + font-weight: 500; + padding: 0.55rem 0.85rem; +} + +.filters input[type=submit]:hover, +.filters input[type=submit]:focus { + background: #1949b8; + border-color: #1949b8; +} + +.filters button.filter-row-icon { + display: inline-flex; + align-items: center; + justify-content: center; + box-sizing: border-box; + width: var(--filter-row-icon-size); + height: var(--filter-row-icon-size); + min-width: var(--filter-row-icon-size); + padding: 0; + border: 1px solid var(--filter-rule); + border-radius: 5px; + background: #fff; + color: var(--filter-muted); + font-family: inherit; + font-size: 1.15rem; + font-weight: 600; + line-height: 1; +} + +.filters button.filter-row-icon[hidden] { + display: none; +} + +.filters button.filter-row-icon:focus-visible { + outline: 3px solid rgba(26, 86, 219, 0.14); + outline-offset: 2px; +} + +.filters .filter-row-remove-icon { + display: block; + height: 14px; + width: 14px; +} + +.filters button.filter-row-remove:hover, +.filters button.filter-row-remove:focus { + border-color: #c9d5e3; + background: #f8fbff; + color: var(--filter-ink); +} + +.filters button.filter-row-add { + border-color: var(--filter-accent); + background: var(--filter-accent); + color: #fff; +} + +.filters .filter-row-add-icon { + display: block; + height: 16px; + width: 16px; +} + +.filters button.filter-row-add:hover, +.filters button.filter-row-add:focus { + border-color: #1949b8; + background: #1949b8; + color: #fff; +} + +.filters button.filter-row-add:focus-visible svg { + color: #fff; + stroke: currentColor; } #_search { font-size: 16px; } +.filters #_search { + box-sizing: border-box; + width: 100%; + height: var(--filter-control-height); + border: 1px solid var(--filter-control-border); + border-radius: 5px; + background: #fff; + color: var(--filter-ink); + font: inherit; + padding: 7px 9px; +} + +.filters #_search:focus { + border-color: var(--filter-accent); + box-shadow: 0 0 0 2px rgba(26, 86, 219, 0.14); + outline: none; +} + @@ -3021,14 +3185,56 @@ select.table-alter-input { margin-bottom: 0.35rem; } - .select-wrapper { - width: 100px; + .filters { + max-width: none; + padding: 0.65rem; } - .select-wrapper.filter-op { - width: 60px; + .filters .search-row { + grid-template-columns: max-content minmax(0, 1fr); + padding-right: 0; + } + .filters .filter-controls-row { + --filter-value-button-space: 0px; + --filter-remove-button-offset: 0px; + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + .filters .filter-controls-row-one-button { + --filter-value-button-space: calc( + var(--filter-row-icon-size) + var(--filter-control-gap) + ); + } + .filters .filter-controls-row-two-buttons { + --filter-value-button-space: var(--filter-two-icon-space); + --filter-remove-button-offset: calc( + var(--filter-row-icon-size) + var(--filter-control-gap) + ); + } + .filters .filter-controls-row .select-wrapper { + grid-row: 1; + grid-column: 1 / 2; + } + .filters .filter-controls-row .select-wrapper.filter-op { + grid-column: 2 / 3; } .filters input.filter-value { - width: 140px; + grid-row: 2; + grid-column: 1 / 3; + justify-self: start; + width: calc(100% - var(--filter-value-button-space)); + } + .filters button.filter-row-icon { + grid-row: 2; + grid-column: 1 / 3; + justify-self: end; + } + .filters .filter-controls-row-two-buttons button.filter-row-remove:not([hidden]) { + margin-right: var(--filter-remove-button-offset); + } + .filters .filter-actions-row { + justify-content: flex-start; + } + .filters .filter-actions-row input[type=submit] { + flex: 0 1 auto; } button.choose-columns-mobile, button.table-insert-row, diff --git a/datasette/static/table.js b/datasette/static/table.js index f160f3f3..74a96d8e 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -633,32 +633,151 @@ const initDatasetteTable = function (manager) { }); }; -/* Add x buttons to the filter rows */ -function addButtonsToFilterRows(manager) { - var x = "✖"; - var rows = Array.from( - document.querySelectorAll(manager.selectors.filterRow), +function filterRowSelector(manager) { + return manager.selectors.filterRows || manager.selectors.filterRow; +} + +function filterRowsWithControls(manager) { + return Array.from( + document.querySelectorAll(filterRowSelector(manager)), ).filter((el) => el.querySelector(".filter-op")); - rows.forEach((row) => { - var a = document.createElement("a"); - a.setAttribute("href", "#"); - a.setAttribute("aria-label", "Remove this filter"); - a.style.textDecoration = "none"; - a.innerText = x; - a.addEventListener("click", (ev) => { - ev.preventDefault(); - let row = ev.target.closest("div"); - row.querySelector("select").value = ""; - row.querySelector(".filter-op select").value = "exact"; - row.querySelector("input.filter-value").value = ""; - ev.target.closest("a").style.display = "none"; - }); - row.appendChild(a); +} + +function filterRowNumberFromName(name) { + var match = name && name.match(/^_filter_column_(\d+)$/); + return match ? parseInt(match[1], 10) : 0; +} + +function nextFilterRowNumber(manager) { + return filterRowsWithControls(manager).reduce((max, row) => { var column = row.querySelector("select"); - if (!column.value) { - a.style.display = "none"; + return Math.max(max, filterRowNumberFromName(column && column.name)); + }, 0) + 1; +} + +function setFilterRowNumber(row, number) { + row.querySelector("select").name = `_filter_column_${number}`; + row.querySelector(".filter-op select").name = `_filter_op_${number}`; + row.querySelector("input.filter-value").name = `_filter_value_${number}`; +} + +function resetFilterRow(row) { + row.querySelector("select").value = ""; + row.querySelector(".filter-op select").value = "exact"; + row.querySelector("input.filter-value").value = ""; +} + +function updateFilterRowButtons(manager) { + var rows = filterRowsWithControls(manager); + rows.forEach((row, index) => { + var removeButton = row.querySelector(".filter-row-remove"); + var addButton = row.querySelector(".filter-row-add"); + var column = row.querySelector("select"); + if (removeButton) { + removeButton.hidden = index === 0; + } + if (addButton) { + addButton.hidden = index !== rows.length - 1 || !column.value; + } + var visibleButtonCount = [removeButton, addButton].filter(function (button) { + return button && !button.hidden; + }).length; + row.classList.toggle( + "filter-controls-row-has-buttons", + visibleButtonCount > 0, + ); + row.classList.toggle( + "filter-controls-row-one-button", + visibleButtonCount === 1, + ); + row.classList.toggle( + "filter-controls-row-two-buttons", + visibleButtonCount === 2, + ); + }); +} + +function cloneFilterRow(row) { + var clone = row.cloneNode(true); + clone.querySelector("select").name = "_filter_column"; + clone.querySelector(".filter-op select").name = "_filter_op"; + clone.querySelector("input.filter-value").name = "_filter_value"; + resetFilterRow(clone); + clone.querySelectorAll(".filter-row-icon").forEach((button) => button.remove()); + return clone; +} + +var FILTER_REMOVE_ICON_SVG = ``; + +var FILTER_ADD_ICON_SVG = ``; + +function addFilterRowButtons(row, manager) { + var removeButton = document.createElement("button"); + removeButton.type = "button"; + removeButton.className = "filter-row-icon filter-row-remove"; + removeButton.setAttribute("aria-label", "Remove this filter"); + removeButton.title = "Remove this filter"; + removeButton.tabIndex = 0; + removeButton.innerHTML = FILTER_REMOVE_ICON_SVG; + removeButton.addEventListener("click", (ev) => { + var row = ev.currentTarget.closest(filterRowSelector(manager)); + var rows = filterRowsWithControls(manager); + var rowIndex = rows.indexOf(row); + var focusRow = rows[rowIndex + 1] || rows[rowIndex - 1] || null; + row.remove(); + updateFilterRowButtons(manager); + if (focusRow) { + var focusTarget = + focusRow.querySelector(".filter-row-add:not([hidden])") || + focusRow.querySelector("select"); + if (focusTarget) { + focusTarget.focus(); + } } }); + row.appendChild(removeButton); + + var addButton = document.createElement("button"); + addButton.type = "button"; + addButton.className = "filter-row-icon filter-row-add"; + addButton.setAttribute("aria-label", "Add another filter"); + addButton.title = "Add another filter"; + addButton.tabIndex = 0; + addButton.innerHTML = FILTER_ADD_ICON_SVG; + addButton.addEventListener("click", (ev) => { + var row = ev.currentTarget.closest(filterRowSelector(manager)); + if (row.querySelector("select").name === "_filter_column") { + setFilterRowNumber(row, nextFilterRowNumber(manager)); + } + var clone = cloneFilterRow(row); + addFilterRowButtons(clone, manager); + row.parentNode.insertBefore(clone, row.nextSibling); + updateFilterRowButtons(manager); + clone.querySelector("select").focus(); + }); + row.appendChild(addButton); + + row.querySelector("select").addEventListener("change", () => { + updateFilterRowButtons(manager); + }); +} + +/* Add buttons to the filter rows */ +function addButtonsToFilterRows(manager) { + var rows = filterRowsWithControls(manager); + rows.forEach((row) => { + addFilterRowButtons(row, manager); + }); + updateFilterRowButtons(manager); } /* Set up datalist autocomplete for filter values */ @@ -687,11 +806,11 @@ function initAutocompleteForFilterValues(manager) { }); } createDataLists(); - // When any select with name=_filter_column changes, update the datalist + // When any filter column select changes, update the datalist document.body.addEventListener("change", function (event) { - if (event.target.name === "_filter_column") { + if (event.target.name && event.target.name.startsWith("_filter_column")) { event.target - .closest(manager.selectors.filterRow) + .closest(filterRowSelector(manager)) .querySelector(".filter-value") .setAttribute("list", "datalist-" + event.target.value); } diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index a46478a7..bff44341 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -202,9 +202,9 @@

3 rows where characteristic_id = 2

-
+
-
+
-
+
-
-
+
+
- - + +
diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 86b6a6ed..e06ef94e 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -55,12 +55,12 @@ {% endif %} -
+ {% if supports_search %}
{% endif %} {% for column, lookup, value in filters.selections() %} -
+
{% endfor %} -
+
-
+
{% if is_sortable %} -
+
- + {% endif %} {% for key, value in form_hidden_args %} {% endfor %} - +
From 4d031c85621ab7e63b1952d25e3e938fa72323b1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 19:47:21 -0700 Subject: [PATCH 049/174] Add count_truncated template context --- datasette/templates/database.html | 2 +- datasette/templates/table.html | 4 ++-- datasette/views/__init__.py | 2 ++ datasette/views/database.py | 29 ++++++++++++++++++++++---- datasette/views/table.py | 33 +++++++++++++++++++++++------- datasette/views/table_extras.py | 4 ++-- docs/template_context.rst | 13 +++++------- tests/test_template_context.py | 34 ++++++++++++++++++++++++++++++- 8 files changed, 96 insertions(+), 25 deletions(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 23eeb571..c09582f7 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -76,7 +76,7 @@

{{ table.name }}{% if table.private %} 🔒{% endif %}{% if table.hidden %} (hidden){% endif %}

{% for column in table.columns %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}

-

{% if table.count is none %}Many rows{% elif table.count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

+

{% if table.count is none %}Many rows{% elif table.count_truncated %}>{{ "{:,}".format(table.count - 1) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

{% endif %} {% endfor %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index e06ef94e..b7b776ab 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}{{ database }}: {{ table }}: {% if count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} +{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} {% block extra_head %} {{- super() -}} @@ -48,7 +48,7 @@ {% if count or human_description_en %}

- {% if count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows + {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows {% if allow_execute_sql and query.sql %} count all{% endif %} {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %} {% if human_description_en %}{{ human_description_en }}{% endif %} diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index 238b9bb3..e503ed35 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -32,6 +32,8 @@ class Context: "List of ContextField describing the documented fields of this context" documented = [] for f in dataclasses.fields(cls): + if f.name.startswith("_"): + continue from_extra = bool(f.metadata.get("from_extra")) if from_extra: help_text = cls._extra_description(f.name) diff --git a/datasette/views/database.py b/datasette/views/database.py index 0d35b87f..46a90c4c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -230,7 +230,6 @@ class DatabaseView(View): database_actions=database_actions, show_hidden=request.args.get("_show_hidden"), editable=True, - count_limit=db.count_limit, allow_download=datasette.setting("allow_download") and not db.is_mutable and not db.is_memory, @@ -267,7 +266,11 @@ class DatabaseContext(Context): ) path: str = field(metadata={"help": "The URL path to this database"}) size: int = field(metadata={"help": "The size of the database in bytes"}) - tables: list = field(metadata={"help": "List of table objects in the database"}) + tables: list = field( + metadata={ + "help": "List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total." + } + ) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) views: list = field(metadata={"help": "List of view objects in the database"}) queries: list = field(metadata={"help": "List of stored query objects"}) @@ -295,7 +298,6 @@ class DatabaseContext(Context): editable: bool = field( metadata={"help": "Boolean indicating if the database is editable"} ) - count_limit: int = field(metadata={"help": "The maximum number of rows to count"}) allow_download: bool = field( metadata={"help": "Boolean indicating if database download is allowed"} ) @@ -365,7 +367,11 @@ class QueryContext(Context): save_query_url: str = field( metadata={"help": "URL to save the current arbitrary SQL as a query"} ) - tables: list = field(metadata={"help": "List of table objects in the database"}) + tables: list = field( + metadata={ + "help": "List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total." + } + ) named_parameter_values: dict = field( metadata={"help": "Dictionary of parameter names/values"} ) @@ -430,6 +436,9 @@ async def get_tables(datasette, request, db, allowed_dict): "columns": table_columns, "primary_keys": await db.primary_keys(table), "count": table_counts[table], + "count_truncated": _table_count_truncated( + datasette, db, table, table_counts[table] + ), "hidden": table in hidden_table_names, "fts_table": await db.fts_table(table), "foreign_keys": all_foreign_keys[table], @@ -440,6 +449,18 @@ async def get_tables(datasette, request, db, allowed_dict): return tables +def _table_count_truncated(datasette, db, table, count): + if count != db.count_limit + 1: + return False + if not db.is_mutable and datasette.inspect_data: + try: + datasette.inspect_data[db.name]["tables"][table]["count"] + return False + except KeyError: + pass + return True + + async def database_download(request, datasette): from datasette.resources import DatabaseResource diff --git a/datasette/views/table.py b/datasette/views/table.py index a6769242..38a69f5f 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -113,6 +113,11 @@ class TableContext(Context): metadata={"help": "True if the data for this page was retrieved without errors"} ) next: str = field(metadata={"help": "Pagination token for the next page, or None"}) + count_truncated: bool = field( + metadata={ + "help": "True if ``count`` is a capped lower bound rather than an exact total, because Datasette stopped counting after its configured row-count limit." + } + ) rows: list = field( metadata={ "help": "The rows for this page, as a list of dictionaries mapping column name to value" @@ -185,11 +190,6 @@ class TableContext(Context): top_table: callable = field( metadata={"help": "Async function rendering the top_table plugin slot"} ) - count_limit: int = field( - metadata={ - "help": "The maximum number of rows Datasette will count before showing an approximation" - } - ) table_page_data: dict = field( metadata={"help": "JSON data used by JavaScript on the table page"} ) @@ -1391,7 +1391,6 @@ class TableFragmentView(BaseView): path_with_replaced_args=path_with_replaced_args, fix_path=self.ds.urls.path, settings=self.ds.settings_dict(), - count_limit=resolved.db.count_limit, ), request=request, view_name="table", @@ -1843,7 +1842,6 @@ async def table_view_traced(datasette, request): database=resolved.db.name, table=resolved.table, ), - count_limit=resolved.db.count_limit, ), request=request, view_name="table", @@ -2276,6 +2274,9 @@ async def table_view_data( data["rows"] = transformed_rows if context_for_html_hack: + data["count_truncated"] = _count_truncated_for_table_page( + datasette, db, database_name, table_name, count_sql, data.get("count") + ) data.update(extra_context_from_filters) # filter_columns combine the columns we know are available # in the table with any additional columns (such as rowid) @@ -2341,6 +2342,24 @@ async def table_view_data( return data, rows[:page_size], columns, expanded_columns, sql, next_url +def _count_truncated_for_table_page( + datasette, db, database_name, table_name, count_sql, count +): + if count != db.count_limit + 1: + return False + if ( + not db.is_mutable + and datasette.inspect_data + and count_sql == f"select count(*) from {table_name} " + ): + try: + datasette.inspect_data[database_name]["tables"][table_name]["count"] + return False + except KeyError: + pass + return True + + async def _next_value_and_url( datasette, db, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 7cb4d8f0..f688433d 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -127,8 +127,8 @@ class CountExtra(Extra): pass if context.count_sql and count is None and not context.nocount: - count_sql_limited = ( - f"select count(*) from (select * {context.from_sql} limit 10001)" + count_sql_limited = "select count(*) from (select * {} limit {})".format( + context.from_sql, context.db.count_limit + 1 ) try: count_rows = list( diff --git a/docs/template_context.rst b/docs/template_context.rst index f94ecd48..b27f2f79 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -100,9 +100,6 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``attached_databases`` - ``list`` List of names of attached databases -``count_limit`` - ``int`` - The maximum number of rows to count - ``database`` - ``str`` The name of the database @@ -152,7 +149,7 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re Dictionary mapping table names to their column lists ``tables`` - ``list`` - List of table objects in the database + List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total. ``top_database`` - ``callable`` Callable to render the top_database slot @@ -241,7 +238,7 @@ The page for arbitrary SQL queries (/database/-/query?sql=...) and stored querie Dictionary of table name to list of column names ``tables`` - ``list`` - List of table objects in the database + List of table objects in the database. Each item includes a ``count_truncated`` key that is true if ``count`` is a capped lower bound rather than an exact total. ``top_query`` - ``callable`` Callable to render the top_query slot @@ -280,12 +277,12 @@ Many of these keys are shared with the :ref:`JSON API ` for this page. ``count`` - ``int`` Total count of rows matching these filters -``count_limit`` - ``int`` - The maximum number of rows Datasette will count before showing an approximation - ``count_sql`` - ``str`` SQL query used to calculate the total count +``count_truncated`` - ``bool`` + True if ``count`` is a capped lower bound rather than an exact total, because Datasette stopped counting after its configured row-count limit. + ``custom_table_templates`` - ``list`` Custom template names considered for this table diff --git a/tests/test_template_context.py b/tests/test_template_context.py index b970cb8a..8d78bf77 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -21,6 +21,7 @@ def test_documented_fields(): @dataclass class DemoContext(Context): name: str = field(metadata={"help": "The name"}) + _internal: str = field() count: int = field(metadata={"help": "How many there are"}) fields = DemoContext.documented_fields() @@ -165,7 +166,11 @@ async def test_template_context_matches_documented_contract( # The full contract: every key in the rendered template context is # documented, and every documented key is present in the context documented = documented_context_keys(page_name) - actual = set(await get_template_context(context_ds, path)) + actual = { + key + for key in await get_template_context(context_ds, path) + if not key.startswith("_") + } undocumented = actual - documented no_longer_present = documented - actual assert not undocumented, ( @@ -178,6 +183,33 @@ async def test_template_context_matches_documented_contract( ) +@pytest.mark.asyncio +async def test_count_truncated_replaces_count_limit_context_key(context_ds): + db = context_ds.databases["fixtures"] + previous_count_limit = db.count_limit + previous_cached_table_counts = db._cached_table_counts + db.count_limit = 10 + db._cached_table_counts = None + try: + table_context = await get_template_context(context_ds, "/fixtures/facetable") + assert table_context["count"] == 11 + assert table_context["count_truncated"] is True + assert "count_limit" not in table_context + + database_context = await get_template_context(context_ds, "/fixtures") + facetable = next( + table + for table in database_context["tables"] + if table["name"] == "facetable" + ) + assert facetable["count"] == 11 + assert facetable["count_truncated"] is True + assert "count_limit" not in database_context + finally: + db.count_limit = previous_count_limit + db._cached_table_counts = previous_cached_table_counts + + def test_base_context_keys_all_have_docs(): for name, doc in TEMPLATE_BASE_CONTEXT.items(): assert doc, "Base context key {} is missing docs".format(name) From 29971d97293a78ca44f66a1129d8c8b922b09c38 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 07:29:14 -0700 Subject: [PATCH 050/174] Clarify base template context docs --- datasette/app.py | 42 +++++++++++++++++++++------------------ docs/template_context.rst | 29 ++++++++++++--------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 9f0c8397..90de60a9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -35,6 +35,7 @@ from jinja2 import ( ChoiceLoader, Environment, FileSystemLoader, + pass_context, PrefixLoader, ) from jinja2.environment import Template @@ -330,30 +331,37 @@ def _to_string(value): return json.dumps(value, default=str) +@pass_context +def _legacy_template_csrftoken(context): + request = context.get("request") + if request and "csrftoken" in request.scope: + return request.scope["csrftoken"]() + return "" + + # Documentation for the variables Datasette.render_template() adds to the # context for every page. This is part of the documented template contract: # keys added in render_template() must be documented here - the contract # tests in tests/test_template_context.py enforce this, and the docs in # docs/template_context.rst are generated from it. TEMPLATE_BASE_CONTEXT = { - "request": "The current Request object, or None", - "crumb_items": "Async function returning breadcrumb navigation items for the current page", - "urls": "Object with methods for constructing URLs to pages within Datasette - see datasette.urls in the internals documentation", - "actor": "The currently authenticated actor dictionary, or None", - "menu_links": "Async function returning links for the Datasette application menu, including those added by plugins", - "display_actor": "Function returning a display string for an actor dictionary", + "request": "The current :ref:`Request object `, or None. Common properties include ``request.path``, ``request.args``, ``request.actor``, ``request.url_vars`` and ``request.host``.", + "crumb_items": 'Async function returning breadcrumb navigation items for the current page. Call it with ``request=request`` plus optional ``database=`` and ``table=`` arguments; it returns a list of ``{"href": url, "label": label}`` dictionaries.', + "urls": "Object with methods for constructing URLs within Datasette. Common methods include ``urls.instance()``, ``urls.database(database)``, ``urls.table(database, table)``, ``urls.query(database, query)``, ``urls.row(database, table, row_path)`` and ``urls.static(path)`` - see :ref:`internals_datasette_urls`.", + "actor": "The currently authenticated actor dictionary, or None. Actors usually include an ``id`` key and may include any other keys supplied by authentication plugins.", + "menu_links": "Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`.", + "display_actor": "Function that accepts an actor dictionary and returns the display string used in the navigation menu.", "show_logout": "True if the logout link should be shown in the navigation menu", "app_css_hash": "Hash of Datasette's app.css contents, used for cache busting", "edit_tools_js_hash": "Hash of Datasette's edit-tools.js contents, used for cache busting", "table_js_hash": "Hash of Datasette's table.js contents, used for cache busting", - "zip": "Python's zip() builtin, made available to template logic", - "body_scripts": "List of script blocks for the page body contributed by plugins", - "format_bytes": "Function that formats a number of bytes as a human-readable size", - "show_messages": "Function returning any messages set for the current user, clearing them in the process", - "extra_css_urls": "List of {url, sri} dictionaries of extra CSS stylesheets to include on the page, from plugins and configuration", - "extra_js_urls": "List of {url, sri, module} dictionaries of extra JavaScript URLs to include on the page", - "base_url": "The configured base_url setting", - "csrftoken": "Function returning the CSRF token for the current request", + "zip": "Python's ``zip()`` builtin, made available to template logic", + "body_scripts": 'List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in `` - """.format(escape(ex.sql))).strip(), - title="SQL Interrupted", - status=400, - message_is_html=True, - ) - except (sqlite3.OperationalError, InvalidSql) as e: - raise DatasetteError(str(e), title="Invalid SQL", status=400) - - except sqlite3.OperationalError as e: - raise DatasetteError(str(e)) - - except DatasetteError: - raise - - end = time.perf_counter() - data["query_ms"] = (end - start) * 1000 - - # Special case for .jsono extension - redirect to _shape=objects - if _format == "jsono": - return self.redirect( - request, - path_with_added_args( - request, - {"_shape": "objects"}, - path=request.path.rsplit(".jsono", 1)[0] + ".json", - ), - forward_querystring=False, - ) - - if _format in self.ds.renderers.keys(): - # Dispatch request to the correct output format renderer - # (CSV is not handled here due to streaming) - result = call_with_supported_arguments( - self.ds.renderers[_format][0], - datasette=self.ds, - columns=data.get("columns") or [], - rows=data.get("rows") or [], - sql=data.get("query", {}).get("sql", None), - query_name=data.get("query_name"), - database=database, - table=data.get("table"), - request=request, - view_name=self.name, - truncated=False, # TODO: support this - error=data.get("error"), - # These will be deprecated in Datasette 1.0: - args=request.args, - data=data, - ) - if asyncio.iscoroutine(result): - result = await result - if result is None: - raise NotFound("No data") - if isinstance(result, dict): - r = Response( - body=result.get("body"), - status=result.get("status_code", status_code or 200), - content_type=result.get("content_type", "text/plain"), - headers=result.get("headers"), - ) - elif isinstance(result, Response): - r = result - if status_code is not None: - # Over-ride the status code - r.status = status_code - else: - assert False, f"{result} should be dict or Response" - else: - extras = {} - if callable(extra_template_data): - extras = extra_template_data() - if asyncio.iscoroutine(extras): - extras = await extras - else: - extras = extra_template_data - url_labels_extra = {} - if data.get("expandable_columns"): - url_labels_extra = {"_labels": "on"} - - renderers = {} - for key, (_, can_render) in self.ds.renderers.items(): - it_can_render = call_with_supported_arguments( - can_render, - datasette=self.ds, - columns=data.get("columns") or [], - rows=data.get("rows") or [], - sql=data.get("query", {}).get("sql", None), - query_name=data.get("query_name"), - database=database, - table=data.get("table"), - request=request, - view_name=self.name, - ) - it_can_render = await await_me_maybe(it_can_render) - if it_can_render: - renderers[key] = self.ds.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format=key, - extra_qs={**url_labels_extra}, - ) - ) - - url_csv_args = {"_size": "max", **url_labels_extra} - url_csv = self.ds.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format="csv", - extra_qs=url_csv_args, - ) - ) - url_csv_path = url_csv.split("?")[0] - context = { - **data, - **extras, - **{ - "renderers": renderers, - "url_csv": url_csv, - "url_csv_path": url_csv_path, - "url_csv_hidden_args": [ - (key, value) - for key, value in urllib.parse.parse_qsl(request.query_string) - if key not in ("_labels", "_facet", "_size") - ] - + [("_size", "max")], - "settings": self.ds.settings_dict(), - }, - } - if "metadata" not in context: - context["metadata"] = await self.ds.get_instance_metadata() - r = await self.render(templates, request=request, context=context) - if status_code is not None: - r.status = status_code - - ttl = request.args.get("_ttl", None) - if ttl is None or not ttl.isdigit(): - ttl = self.ds.setting("default_cache_ttl") - - return self.set_response_headers(r, ttl) - - def set_response_headers(self, response, ttl): - # Set far-future cache expiry - if self.ds.cache_headers and response.status == 200: - ttl = int(ttl) - if ttl == 0: - ttl_header = "no-cache" - else: - ttl_header = f"max-age={ttl}" - response.headers["Cache-Control"] = ttl_header - response.headers["Referrer-Policy"] = "no-referrer" - if self.ds.cors: - add_cors_headers(response.headers) - return response - - def _error(messages, status=400): return Response.json({"ok": False, "errors": messages}, status=status) diff --git a/datasette/views/row.py b/datasette/views/row.py index 7802f45e..3e3e52a9 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -1,21 +1,34 @@ +import asyncio +import json +import textwrap +import time +import urllib.parse +from dataclasses import dataclass, field, fields + +import markupsafe +import sqlite_utils + from datasette.utils.asgi import NotFound, Forbidden, Response from datasette.database import QueryInterrupted from datasette.events import UpdateRowEvent, DeleteRowEvent from datasette.resources import TableResource -from .base import DataView, BaseView, _error +from .base import BaseView, DatasetteError, _error, stream_csv from datasette.utils import ( + add_cors_headers, await_me_maybe, + call_with_supported_arguments, CustomRow, + InvalidSql, make_slot_function, path_from_row_pks, + path_with_added_args, + path_with_format, + path_with_removed_args, to_css_class, escape_sqlite, + sqlite3, ) from datasette.plugins import pm -from dataclasses import dataclass, field -import json -import markupsafe -import sqlite_utils from datasette.extras import extra_names_from_request, ExtraScope from . import Context, extra_field from .table import ( @@ -121,9 +134,259 @@ class RowContext(Context): ) -class RowView(DataView): +class RowView(BaseView): name = "row" - context_class = RowContext + + def redirect(self, request, path, forward_querystring=True, remove_args=None): + if request.query_string and "?" not in path and forward_querystring: + path = f"{path}?{request.query_string}" + if remove_args: + path = path_with_removed_args(request, remove_args, path=path) + response = Response.redirect(path) + response.headers["Link"] = f"<{path}>; rel=preload" + if self.ds.cors: + add_cors_headers(response.headers) + return response + + async def as_csv(self, request, database): + return await stream_csv(self.ds, self.data, request, database) + + async def get(self, request): + db = await self.ds.resolve_database(request) + database = db.name + database_route = db.route + format_ = request.url_vars.get("format") or "html" + data_kwargs = {} + + if format_ == "csv": + return await self.as_csv(request, database_route) + + if format_ == "html": + # HTML views default to expanding all foreign key labels + data_kwargs["default_labels"] = True + + extra_template_data = {} + start = time.perf_counter() + status_code = None + templates = () + try: + response_or_template_contexts = await self.data(request, **data_kwargs) + if isinstance(response_or_template_contexts, Response): + return response_or_template_contexts + # If it has four items, it includes an HTTP status code + if len(response_or_template_contexts) == 4: + ( + data, + extra_template_data, + templates, + status_code, + ) = response_or_template_contexts + else: + data, extra_template_data, templates = response_or_template_contexts + except QueryInterrupted as ex: + raise DatasetteError( + textwrap.dedent(""" +

SQL query took too long. The time limit is controlled by the + sql_time_limit_ms + configuration option.

+ + + """.format(markupsafe.escape(ex.sql))).strip(), + title="SQL Interrupted", + status=400, + message_is_html=True, + ) + except (sqlite3.OperationalError, InvalidSql) as e: + raise DatasetteError(str(e), title="Invalid SQL", status=400) + except sqlite3.OperationalError as e: + raise DatasetteError(str(e)) + except DatasetteError: + raise + + end = time.perf_counter() + data["query_ms"] = (end - start) * 1000 + + # Special case for .jsono extension - redirect to _shape=objects + if format_ == "jsono": + return self.redirect( + request, + path_with_added_args( + request, + {"_shape": "objects"}, + path=request.path.rsplit(".jsono", 1)[0] + ".json", + ), + forward_querystring=False, + ) + + if format_ in self.ds.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + self.ds.renderers[format_][0], + datasette=self.ds, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=database, + table=data.get("table"), + request=request, + view_name=self.name, + truncated=False, # TODO: support this + error=data.get("error"), + # These will be deprecated in Datasette 1.0: + args=request.args, + data=data, + ) + if asyncio.iscoroutine(result): + result = await result + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + response = Response( + body=result.get("body"), + status=result.get("status_code", status_code or 200), + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + response = result + if status_code is not None: + # Over-ride the status code + response.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + response = await self.html(request, data, extra_template_data, templates) + if status_code is not None: + response.status = status_code + else: + raise NotFound("Invalid format: {}".format(format_)) + + ttl = request.args.get("_ttl", None) + if ttl is None or not ttl.isdigit(): + ttl = self.ds.setting("default_cache_ttl") + + return self.set_response_headers(response, ttl) + + async def html(self, request, data, extra_template_data, templates): + extras = {} + if callable(extra_template_data): + extras = extra_template_data() + if asyncio.iscoroutine(extras): + extras = await extras + else: + extras = extra_template_data + + url_labels_extra = {} + if data.get("expandable_columns"): + url_labels_extra = {"_labels": "on"} + + renderers = {} + for key, (_, can_render) in self.ds.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=self.ds, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=data.get("database"), + table=data.get("table"), + request=request, + view_name=self.name, + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = self.ds.urls.path( + path_with_format( + request=request, + path=request.scope.get("route_path"), + format=key, + extra_qs={**url_labels_extra}, + ) + ) + + url_csv_args = {"_size": "max", **url_labels_extra} + url_csv = self.ds.urls.path( + path_with_format( + request=request, + path=request.scope.get("route_path"), + format="csv", + extra_qs=url_csv_args, + ) + ) + url_csv_path = url_csv.split("?")[0] + context = {**data, **extras} + if "metadata" not in context: + context["metadata"] = await self.ds.get_instance_metadata() + + environment = self.ds.get_jinja_environment(request) + template = environment.select_template(templates) + alternate_url_json = self.ds.absolute_url( + request, + self.ds.urls.path( + path_with_format( + request=request, + path=request.scope.get("route_path"), + format="json", + ) + ), + ) + explicit_context = { + "renderers": renderers, + "url_csv": url_csv, + "url_csv_path": url_csv_path, + "url_csv_hidden_args": [ + (key, value) + for key, value in urllib.parse.parse_qsl(request.query_string) + if key not in ("_labels", "_facet", "_size") + ] + + [("_size", "max")], + "settings": self.ds.settings_dict(), + "alternate_url_json": alternate_url_json, + "select_templates": [ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + } + declared_fields = {f.name for f in fields(RowContext)} + context = { + key: value + for key, value in context.items() + if key in declared_fields and key not in explicit_context + } + + return Response.html( + await self.ds.render_template( + template, + RowContext(**context, **explicit_context), + request=request, + view_name=self.name, + ), + headers={ + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + }, + ) + + def set_response_headers(self, response, ttl): + # Set far-future cache expiry + if self.ds.cache_headers and response.status == 200: + ttl = int(ttl) + if ttl == 0: + ttl_header = "no-cache" + else: + ttl_header = f"max-age={ttl}" + response.headers["Cache-Control"] = ttl_header + response.headers["Referrer-Policy"] = "no-referrer" + if self.ds.cors: + add_cors_headers(response.headers) + return response async def data(self, request, default_labels=False): resolved = await self.ds.resolve_row(request) From a43e76c31a3c4b96ab7289b5f3202c7c3537bf6a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 09:08:40 -0700 Subject: [PATCH 055/174] Construct row template context explicitly --- datasette/views/row.py | 63 +++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index 3e3e52a9..ef39dce6 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -3,7 +3,7 @@ import json import textwrap import time import urllib.parse -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field import markupsafe import sqlite_utils @@ -336,34 +336,45 @@ class RowView(BaseView): ) ), ) - explicit_context = { - "renderers": renderers, - "url_csv": url_csv, - "url_csv_path": url_csv_path, - "url_csv_hidden_args": [ - (key, value) - for key, value in urllib.parse.parse_qsl(request.query_string) - if key not in ("_labels", "_facet", "_size") - ] - + [("_size", "max")], - "settings": self.ds.settings_dict(), - "alternate_url_json": alternate_url_json, - "select_templates": [ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], - } - declared_fields = {f.name for f in fields(RowContext)} - context = { - key: value - for key, value in context.items() - if key in declared_fields and key not in explicit_context - } - return Response.html( await self.ds.render_template( template, - RowContext(**context, **explicit_context), + RowContext( + columns=context["columns"], + database=context["database"], + database_color=context["database_color"], + foreign_key_tables=context["foreign_key_tables"], + metadata=context["metadata"], + primary_keys=context["primary_keys"], + private=context["private"], + table=context["table"], + ok=context["ok"], + rows=context["rows"], + primary_key_values=context["primary_key_values"], + query_ms=context["query_ms"], + display_columns=context["display_columns"], + display_rows=context["display_rows"], + custom_table_templates=context["custom_table_templates"], + row_actions=context["row_actions"], + row_mutation_ui=context["row_mutation_ui"], + table_page_data=context["table_page_data"], + top_row=context["top_row"], + renderers=renderers, + url_csv=url_csv, + url_csv_path=url_csv_path, + url_csv_hidden_args=[ + (key, value) + for key, value in urllib.parse.parse_qsl(request.query_string) + if key not in ("_labels", "_facet", "_size") + ] + + [("_size", "max")], + settings=self.ds.settings_dict(), + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + alternate_url_json=alternate_url_json, + ), request=request, view_name=self.name, ), From 59ab0c0ca0b2d86913102dace6813f42f947d772 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 11:30:30 -0700 Subject: [PATCH 056/174] Clarify template context metadata names --- datasette/template_contexts.py | 2 +- datasette/views/__init__.py | 14 ++++---- datasette/views/database.py | 4 +-- datasette/views/row.py | 20 +++++------ datasette/views/table.py | 62 +++++++++++++++++----------------- docs/template_context_doc.py | 2 +- tests/test_template_context.py | 24 +++++++------ 7 files changed, 65 insertions(+), 63 deletions(-) diff --git a/datasette/template_contexts.py b/datasette/template_contexts.py index 5fe6a80e..232eb051 100644 --- a/datasette/template_contexts.py +++ b/datasette/template_contexts.py @@ -7,7 +7,7 @@ the documentation lives next to the code it describes: - Every page renders a Context dataclass defined in its view module (DatabaseContext, QueryContext in views/database.py, TableContext in views/table.py, RowContext in views/row.py). Fields added by view code - carry ``help`` metadata; fields declared with extra_field() take their + carry ``help`` metadata; fields declared with from_extra() take their documentation from the description on the matching Extra class in views/table_extras.py. - The keys render_template() adds to every page are documented in diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index e503ed35..6fba1c90 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -10,7 +10,7 @@ class ContextField: from_extra: bool = False -def extra_field(): +def from_extra(): """ Declare a Context dataclass field whose value comes from a registered Extra of the same name - its documentation is the Extra description, @@ -23,7 +23,7 @@ def extra_field(): class Context: "Base class for all documented contexts" - # Set on subclasses whose extra_field() fields should be resolved + # Set on subclasses whose from_extra() fields should be resolved # against the extras registry for this scope extras_scope = None @@ -34,8 +34,8 @@ class Context: for f in dataclasses.fields(cls): if f.name.startswith("_"): continue - from_extra = bool(f.metadata.get("from_extra")) - if from_extra: + is_from_extra = bool(f.metadata.get("from_extra")) + if is_from_extra: help_text = cls._extra_description(f.name) else: help_text = f.metadata.get("help", "") @@ -44,7 +44,7 @@ class Context: name=f.name, type_name=getattr(f.type, "__name__", str(f.type)), help=help_text, - from_extra=from_extra, + from_extra=is_from_extra, ) ) return documented @@ -59,14 +59,14 @@ class Context: extra_class = table_extra_registry.classes_by_name[name] except KeyError: raise KeyError( - "{}.{} is declared with extra_field() but there is no " + "{}.{} is declared with from_extra() but there is no " "registered extra of that name".format(cls.__name__, name) ) if cls.extras_scope is not None and not extra_class.available_for( cls.extras_scope ): raise ValueError( - "{}.{} is declared with extra_field() but the {} extra is " + "{}.{} is declared with from_extra() but the {} extra is " "not available for scope {}".format( cls.__name__, name, name, cls.extras_scope ) diff --git a/datasette/views/database.py b/datasette/views/database.py index 7409e7d3..0b4ca647 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -258,7 +258,7 @@ class DatabaseView(View): class DatabaseContext(Context): "The page listing the tables, views and queries in a database, e.g. /fixtures." - template = "database.html" + documented_template = "database.html" database: str = field(metadata={"help": "The name of the database"}) private: bool = field( @@ -341,7 +341,7 @@ class DatabaseContext(Context): class QueryContext(Context): "The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)." - template = "query.html" + documented_template = "query.html" database: str = field(metadata={"help": "The name of the database being queried"}) database_color: str = field(metadata={"help": "The color of the database"}) diff --git a/datasette/views/row.py b/datasette/views/row.py index ef39dce6..129216b9 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -30,7 +30,7 @@ from datasette.utils import ( ) from datasette.plugins import pm from datasette.extras import extra_names_from_request, ExtraScope -from . import Context, extra_field +from . import Context, from_extra from .table import ( display_columns_and_rows, _table_page_data, @@ -43,19 +43,19 @@ from .table_extras import RowExtraContext, resolve_row_extras, table_extra_regis class RowContext(Context): "The page showing an individual row, e.g. /fixtures/facetable/1." - template = "row.html" + documented_template = "row.html" extras_scope = ExtraScope.ROW # Fields resolved by registered extras - their documentation comes # from the description on each Extra class in table_extras.py - columns: list = extra_field() - database: str = extra_field() - database_color: str = extra_field() - foreign_key_tables: list = extra_field() - metadata: dict = extra_field() - primary_keys: list = extra_field() - private: bool = extra_field() - table: str = extra_field() + columns: list = from_extra() + database: str = from_extra() + database_color: str = from_extra() + foreign_key_tables: list = from_extra() + metadata: dict = from_extra() + primary_keys: list = from_extra() + private: bool = from_extra() + table: str = from_extra() # Fields added by the view code ok: bool = field( diff --git a/datasette/views/table.py b/datasette/views/table.py index 1e182937..449c6216 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -49,7 +49,7 @@ import sqlite_utils from dataclasses import dataclass, field, fields from datasette.extras import ExtraScope -from . import Context, extra_field +from . import Context, from_extra from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView from .table_create_alter import ( @@ -73,40 +73,40 @@ from .table_extras import ( class TableContext(Context): "The page showing the rows in a table or SQL view, e.g. /fixtures/facetable." - template = "table.html" + documented_template = "table.html" extras_scope = ExtraScope.TABLE # Fields resolved by registered extras - their documentation comes # from the description on each Extra class in table_extras.py - actions: callable = extra_field() - all_columns: list = extra_field() - columns: list = extra_field() - count: int = extra_field() - count_sql: str = extra_field() - custom_table_templates: list = extra_field() - database: str = extra_field() - database_color: str = extra_field() - display_columns: list = extra_field() - display_rows: list = extra_field() - expandable_columns: list = extra_field() - facet_results: dict = extra_field() - facets_timed_out: list = extra_field() - filters: Filters = extra_field() - form_hidden_args: list = extra_field() - human_description_en: str = extra_field() - is_view: bool = extra_field() - metadata: dict = extra_field() - next_url: str = extra_field() - primary_keys: list = extra_field() - private: bool = extra_field() - query: dict = extra_field() - renderers: dict = extra_field() - set_column_type_ui: dict = extra_field() - sorted_facet_results: list = extra_field() - suggested_facets: list = extra_field() - table: str = extra_field() - table_definition: str = extra_field() - view_definition: str = extra_field() + actions: callable = from_extra() + all_columns: list = from_extra() + columns: list = from_extra() + count: int = from_extra() + count_sql: str = from_extra() + custom_table_templates: list = from_extra() + database: str = from_extra() + database_color: str = from_extra() + display_columns: list = from_extra() + display_rows: list = from_extra() + expandable_columns: list = from_extra() + facet_results: dict = from_extra() + facets_timed_out: list = from_extra() + filters: Filters = from_extra() + form_hidden_args: list = from_extra() + human_description_en: str = from_extra() + is_view: bool = from_extra() + metadata: dict = from_extra() + next_url: str = from_extra() + primary_keys: list = from_extra() + private: bool = from_extra() + query: dict = from_extra() + renderers: dict = from_extra() + set_column_type_ui: dict = from_extra() + sorted_facet_results: list = from_extra() + suggested_facets: list = from_extra() + table: str = from_extra() + table_definition: str = from_extra() + view_definition: str = from_extra() # Fields added by the view code ok: bool = field( diff --git a/docs/template_context_doc.py b/docs/template_context_doc.py index c3ec77e2..a5f4fb6f 100644 --- a/docs/template_context_doc.py +++ b/docs/template_context_doc.py @@ -27,7 +27,7 @@ def template_context(cog): for klass in PAGES.values(): title = "{} page".format(klass.__name__.removesuffix("Context")) intro = "{} Rendered using the ``{}`` template.".format( - klass.__doc__, klass.template + klass.__doc__, klass.documented_template ) _section(cog, title, intro) if klass.extras_scope is not None: diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 8d78bf77..4c452774 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -40,20 +40,22 @@ def test_context_class_fields_all_have_help(klass): @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) -def test_context_class_has_docstring_and_template(klass): +def test_context_class_has_docstring_and_documented_template(klass): assert klass.__doc__, "{} is missing a docstring".format(klass.__name__) - assert klass.template, "{} is missing a template".format(klass.__name__) + assert klass.documented_template, "{} is missing a documented_template".format( + klass.__name__ + ) -def test_extra_field_documentation_comes_from_the_extra_class(): - from datasette.views import extra_field +def test_from_extra_documentation_comes_from_the_extra_class(): + from datasette.views import from_extra from datasette.views.table_extras import CountExtra @dataclass class DemoContext(Context): extras_scope = ExtraScope.TABLE - count: int = extra_field() + count: int = from_extra() name: str = field(metadata={"help": "The name"}) fields = {f.name: f for f in DemoContext.documented_fields()} @@ -63,28 +65,28 @@ def test_extra_field_documentation_comes_from_the_extra_class(): assert not fields["name"].from_extra -def test_extra_field_must_match_a_registered_extra(): - from datasette.views import extra_field +def test_from_extra_must_match_a_registered_extra(): + from datasette.views import from_extra @dataclass class BadContext(Context): extras_scope = ExtraScope.TABLE - not_a_real_extra: str = extra_field() + not_a_real_extra: str = from_extra() with pytest.raises(KeyError): BadContext.documented_fields() -def test_extra_field_must_be_available_for_the_scope(): - from datasette.views import extra_field +def test_from_extra_must_be_available_for_the_scope(): + from datasette.views import from_extra @dataclass class WrongScopeContext(Context): extras_scope = ExtraScope.ROW # count is a TABLE-scope extra, not available for ROW - count: int = extra_field() + count: int = from_extra() with pytest.raises(ValueError): WrongScopeContext.documented_fields() From 34d9a3bf33e3c4ff610f7d3a633f63c3d7272cc9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:11:26 -0700 Subject: [PATCH 057/174] Use dataclasses for database table context --- datasette/app.py | 17 +++++++++- datasette/views/__init__.py | 17 +++++++++- datasette/views/database.py | 57 +++++++++++++++++++++------------- docs/contributing.rst | 13 ++++++++ docs/template_context.rst | 8 ++--- tests/test_template_context.py | 10 ++++++ 6 files changed, 95 insertions(+), 27 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 90de60a9..57f893fe 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -331,6 +331,15 @@ def _to_string(value): return json.dumps(value, default=str) +def _template_context_json_default(value): + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + field.name: getattr(value, field.name) + for field in dataclasses.fields(value) + } + return repr(value) + + @pass_context def _legacy_template_csrftoken(context): request = context.get("request") @@ -2390,7 +2399,13 @@ class Datasette: } if request and request.args.get("_context") and self.setting("template_debug"): return "
{}
".format( - escape(json.dumps(template_context, default=repr, indent=4)) + escape( + json.dumps( + template_context, + default=_template_context_json_default, + indent=4, + ) + ) ) return await template.render_async(template_context) diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index 6fba1c90..ed7e175f 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -1,5 +1,7 @@ from dataclasses import dataclass import dataclasses +import types +import typing @dataclass(frozen=True) @@ -10,6 +12,19 @@ class ContextField: from_extra: bool = False +def _type_name(type_): + if type_ is type(None): + return "None" + origin = typing.get_origin(type_) + args = typing.get_args(type_) + if origin in (typing.Union, types.UnionType): + return " | ".join(_type_name(arg) for arg in args) + if origin is not None: + name = getattr(origin, "__name__", str(origin).removeprefix("typing.")) + return "{}[{}]".format(name, ", ".join(_type_name(arg) for arg in args)) + return getattr(type_, "__name__", str(type_).removeprefix("typing.")) + + def from_extra(): """ Declare a Context dataclass field whose value comes from a registered @@ -42,7 +57,7 @@ class Context: documented.append( ContextField( name=f.name, - type_name=getattr(f.type, "__name__", str(f.type)), + type_name=_type_name(f.type), help=help_text, from_extra=is_from_extra, ) diff --git a/datasette/views/database.py b/datasette/views/database.py index 0b4ca647..fd985b77 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from urllib.parse import parse_qsl, urlencode import asyncio import hashlib @@ -45,6 +45,21 @@ from .table_create_alter import _create_table_ui_context from . import Context +@dataclass +class DatabaseTable: + "Summary of a table or view shown on database and query pages." + + name: str + columns: list[str] + primary_keys: list[str] + count: int | None + count_truncated: bool + hidden: bool + fts_table: str | None + foreign_keys: dict[str, list[dict[str, str]]] + private: bool + + class DatabaseView(View): async def get(self, request, datasette): format_ = request.url_vars.get("format") or "html" @@ -169,8 +184,8 @@ class DatabaseView(View): "private": private, "path": datasette.urls.database(database), "size": db.size, - "tables": tables, - "hidden_count": len([t for t in tables if t["hidden"]]), + "tables": [asdict(table) for table in tables], + "hidden_count": len([table for table in tables if table.hidden]), "views": sql_views, "queries": [stored_query_to_dict(query) for query in stored_queries], "queries_more": queries_more, @@ -211,7 +226,7 @@ class DatabaseView(View): path=datasette.urls.database(database), size=db.size, tables=tables, - hidden_count=len([t for t in tables if t["hidden"]]), + hidden_count=len([table for table in tables if table.hidden]), views=sql_views, queries=stored_queries, queries_more=queries_more, @@ -266,9 +281,9 @@ class DatabaseContext(Context): ) path: str = field(metadata={"help": "The URL path to this database"}) size: int = field(metadata={"help": "The size of the database in bytes"}) - tables: list = field( + tables: list[DatabaseTable] = field( metadata={ - "help": "List of table dictionaries in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` keys. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." + "help": "List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." } ) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) @@ -391,9 +406,9 @@ class QueryContext(Context): save_query_url: str = field( metadata={"help": "URL to save the current arbitrary SQL as a query"} ) - tables: list = field( + tables: list[DatabaseTable] = field( metadata={ - "help": "List of table dictionaries in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` keys. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." + "help": "List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total." } ) named_parameter_values: dict = field( @@ -456,7 +471,7 @@ class QueryContext(Context): ) -async def get_tables(datasette, request, db, allowed_dict): +async def get_tables(datasette, request, db, allowed_dict) -> list[DatabaseTable]: """ Get list of tables with metadata for the database view. @@ -477,21 +492,21 @@ async def get_tables(datasette, request, db, allowed_dict): table_columns = await db.table_columns(table) tables.append( - { - "name": table, - "columns": table_columns, - "primary_keys": await db.primary_keys(table), - "count": table_counts[table], - "count_truncated": _table_count_truncated( + DatabaseTable( + name=table, + columns=table_columns, + primary_keys=await db.primary_keys(table), + count=table_counts[table], + count_truncated=_table_count_truncated( datasette, db, table, table_counts[table] ), - "hidden": table in hidden_table_names, - "fts_table": await db.fts_table(table), - "foreign_keys": all_foreign_keys[table], - "private": allowed_dict[table].private, - } + hidden=table in hidden_table_names, + fts_table=await db.fts_table(table), + foreign_keys=all_foreign_keys[table], + private=allowed_dict[table].private, + ) ) - tables.sort(key=lambda t: (t["hidden"], t["name"])) + tables.sort(key=lambda table: (table.hidden, table.name)) return tables diff --git a/docs/contributing.rst b/docs/contributing.rst index 3b45834c..142c2f41 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -309,6 +309,19 @@ To update these pages, run the following command:: uv run cog -r docs/*.rst +.. _contributing_template_contexts: + +Documented template contexts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Datasette's documented template contexts are part of the public API for custom templates. They are defined as dataclasses next to the view code that renders them, for example ``DatabaseContext`` and ``QueryContext`` in ``datasette/views/database.py``. + +Every documented context class inherits from ``datasette.views.Context``. Fields that are added directly by view code should be declared as dataclass fields with ``help`` metadata, which is used to generate :ref:`template_context`. Fields resolved through the page extras system should use ``from_extra()`` so their documentation comes from the matching ``Extra`` class. + +Use ``documented_template`` on each context class to record the canonical template named in the generated documentation. This should be a string such as ``"database.html"``. Runtime template selection still happens in the view code, since most pages consider more specific template names before falling back to the canonical one. + +When a context field contains repeated structured data, prefer a small nested dataclass over an anonymous dictionary. For example, a field containing table summaries should be annotated as ``list[DatabaseTable]`` where ``DatabaseTable`` is a dataclass describing the keys and value types. This keeps the Python contract and generated documentation clear. JSON responses and ``?_context=1`` debug output will convert nested dataclasses back to JSON objects at the response boundary. + .. _contributing_continuous_deployment: Continuously deployed demo instances diff --git a/docs/template_context.rst b/docs/template_context.rst index 12f656d0..311de8bb 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -145,8 +145,8 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``table_columns`` - ``dict`` Dictionary mapping table names to lists of column names, used to power SQL autocomplete. -``tables`` - ``list`` - List of table dictionaries in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` keys. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total. +``tables`` - ``list[DatabaseTable]`` + List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total. ``top_database`` - ``callable`` Async callable that renders the ``top_database`` plugin slot for this database and returns HTML. @@ -234,8 +234,8 @@ The page for arbitrary SQL queries (/database/-/query?sql=...) and stored querie ``table_columns`` - ``dict`` Dictionary mapping table names to lists of column names, used to power SQL autocomplete. -``tables`` - ``list`` - List of table dictionaries in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` keys. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total. +``tables`` - ``list[DatabaseTable]`` + List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total. ``top_query`` - ``callable`` Async callable that renders the ``top_query`` plugin slot for this query and returns HTML. diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 4c452774..f1919891 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -18,16 +18,22 @@ from datasette.views import Context def test_documented_fields(): + @dataclass + class DemoNested: + name: str + @dataclass class DemoContext(Context): name: str = field(metadata={"help": "The name"}) _internal: str = field() count: int = field(metadata={"help": "How many there are"}) + items: list[DemoNested] = field(metadata={"help": "Nested items"}) fields = DemoContext.documented_fields() assert [(f.name, f.type_name, f.help) for f in fields] == [ ("name", "str", "The name"), ("count", "int", "How many there are"), + ("items", "list[DemoNested]", "Nested items"), ] @@ -230,3 +236,7 @@ def test_template_context_docs_cover_every_documented_key(): assert "``{}``".format(context_field.name) in docs, "{} ({} page)".format( context_field.name, page_name ) + assert ( + "``{}`` - ``{}``".format(context_field.name, context_field.type_name) + in docs + ), "{} type ({} page)".format(context_field.name, page_name) From 0d1c097396b8e39f4c5d20ea16282c295d2e4e0b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:16:42 -0700 Subject: [PATCH 058/174] Document database views and queries as dataclasses --- datasette/views/database.py | 22 +++++++++++++++------- docs/template_context.rst | 8 ++++---- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index fd985b77..e02de657 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -11,7 +11,7 @@ import textwrap from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource -from datasette.stored_queries import stored_query_to_dict +from datasette.stored_queries import StoredQuery, stored_query_to_dict from datasette.write_sql import QueryWriteRejected from datasette.utils import ( add_cors_headers, @@ -60,6 +60,14 @@ class DatabaseTable: private: bool +@dataclass +class DatabaseViewInfo: + "Summary of a SQLite view shown on the database page." + + name: str + private: bool + + class DatabaseView(View): async def get(self, request, datasette): format_ = request.url_vars.get("format") or "html" @@ -109,7 +117,7 @@ class DatabaseView(View): # Filter to just views view_names_set = set(await db.view_names()) sql_views = [ - {"name": name, "private": allowed_dict[name].private} + DatabaseViewInfo(name=name, private=allowed_dict[name].private) for name in allowed_dict if name in view_names_set ] @@ -186,7 +194,7 @@ class DatabaseView(View): "size": db.size, "tables": [asdict(table) for table in tables], "hidden_count": len([table for table in tables if table.hidden]), - "views": sql_views, + "views": [asdict(view) for view in sql_views], "queries": [stored_query_to_dict(query) for query in stored_queries], "queries_more": queries_more, "queries_count": queries_count, @@ -287,14 +295,14 @@ class DatabaseContext(Context): } ) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) - views: list = field( + views: list[DatabaseViewInfo] = field( metadata={ - "help": "List of SQLite view dictionaries. Each item has ``name`` and ``private`` keys." + "help": "List of ``DatabaseViewInfo`` objects describing SQLite views in the database. Each item has ``name`` and ``private`` attributes." } ) - queries: list = field( + queries: list[StoredQuery] = field( metadata={ - "help": "List of stored query objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``." + "help": "List of ``StoredQuery`` objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``." } ) queries_more: bool = field( diff --git a/docs/template_context.rst b/docs/template_context.rst index 311de8bb..52ccca49 100644 --- a/docs/template_context.rst +++ b/docs/template_context.rst @@ -124,8 +124,8 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``private`` - ``bool`` Boolean indicating if this is a private database -``queries`` - ``list`` - List of stored query objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``. +``queries`` - ``list[StoredQuery]`` + List of ``StoredQuery`` objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``. ``queries_count`` - ``int`` Count of visible stored queries @@ -151,8 +151,8 @@ The page listing the tables, views and queries in a database, e.g. /fixtures. Re ``top_database`` - ``callable`` Async callable that renders the ``top_database`` plugin slot for this database and returns HTML. -``views`` - ``list`` - List of SQLite view dictionaries. Each item has ``name`` and ``private`` keys. +``views`` - ``list[DatabaseViewInfo]`` + List of ``DatabaseViewInfo`` objects describing SQLite views in the database. Each item has ``name`` and ``private`` attributes. Query page ---------- From 8276879997661ae618ffe984ab6a183d3b62788f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:24:42 -0700 Subject: [PATCH 059/174] Construct table context explicitly --- datasette/views/table.py | 110 ++++++++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 36 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 449c6216..4b923d20 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -46,7 +46,7 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response from datasette.filters import Filters import sqlite_utils -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field from datasette.extras import ExtraScope from . import Context, from_extra @@ -1825,44 +1825,82 @@ async def table_view_traced(datasette, request): ) } ) - # Only keys declared on TableContext are part of the documented - # template contract - anything else in data (e.g. extras requested - # with ?_extra= on the HTML page, or extra filter context added by - # filters_from_request plugins) is dropped here - declared_fields = {f.name for f in fields(TableContext)} + table_context = TableContext( + actions=data["actions"], + all_columns=data["all_columns"], + columns=data["columns"], + count=data["count"], + count_sql=data["count_sql"], + custom_table_templates=data["custom_table_templates"], + database=data["database"], + database_color=data["database_color"], + display_columns=data["display_columns"], + display_rows=data["display_rows"], + expandable_columns=data["expandable_columns"], + facet_results=data["facet_results"], + facets_timed_out=data["facets_timed_out"], + filters=data["filters"], + form_hidden_args=data["form_hidden_args"], + human_description_en=data["human_description_en"], + is_view=data["is_view"], + metadata=data["metadata"], + next_url=data["next_url"], + primary_keys=data["primary_keys"], + private=data["private"], + query=data["query"], + renderers=data["renderers"], + set_column_type_ui=data["set_column_type_ui"], + sorted_facet_results=data["sorted_facet_results"], + suggested_facets=data["suggested_facets"], + table=data["table"], + table_definition=data["table_definition"], + view_definition=data["view_definition"], + ok=data["ok"], + next=data["next"], + count_truncated=data["count_truncated"], + rows=data["rows"], + filter_columns=data["filter_columns"], + supports_search=data["supports_search"], + extra_wheres_for_ui=data["extra_wheres_for_ui"], + url_csv=data["url_csv"], + url_csv_path=data["url_csv_path"], + url_csv_hidden_args=data["url_csv_hidden_args"], + sort=data["sort"], + sort_desc=data["sort_desc"], + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=any(c["sortable"] for c in data["display_columns"]), + allow_execute_sql=await datasette.allowed( + action="execute-sql", + resource=DatabaseResource(database=resolved.db.name), + actor=request.actor, + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + top_table=make_slot_function( + "top_table", + datasette, + request, + database=resolved.db.name, + table=resolved.table, + ), + table_page_data=data["table_page_data"], + table_insert_ui=data["table_insert_ui"], + table_alter_ui=data["table_alter_ui"], + ) r = Response.html( await datasette.render_template( template, - TableContext( - **{k: v for k, v in data.items() if k in declared_fields}, - append_querystring=append_querystring, - path_with_replaced_args=path_with_replaced_args, - fix_path=datasette.urls.path, - settings=datasette.settings_dict(), - # TODO: review up all of these hacks: - alternate_url_json=alternate_url_json, - datasette_allow_facet=( - "true" if datasette.setting("allow_facet") else "false" - ), - is_sortable=any(c["sortable"] for c in data["display_columns"]), - allow_execute_sql=await datasette.allowed( - action="execute-sql", - resource=DatabaseResource(database=resolved.db.name), - actor=request.actor, - ), - query_ms=1.2, - select_templates=[ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], - top_table=make_slot_function( - "top_table", - datasette, - request, - database=resolved.db.name, - table=resolved.table, - ), - ), + table_context, request=request, view_name="table", ), From 0c523dda209e3b545c52cd3b113d788cba9a342b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:29:21 -0700 Subject: [PATCH 060/174] Remove count truncated context test --- tests/test_template_context.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/tests/test_template_context.py b/tests/test_template_context.py index f1919891..7923d0e7 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -191,33 +191,6 @@ async def test_template_context_matches_documented_contract( ) -@pytest.mark.asyncio -async def test_count_truncated_replaces_count_limit_context_key(context_ds): - db = context_ds.databases["fixtures"] - previous_count_limit = db.count_limit - previous_cached_table_counts = db._cached_table_counts - db.count_limit = 10 - db._cached_table_counts = None - try: - table_context = await get_template_context(context_ds, "/fixtures/facetable") - assert table_context["count"] == 11 - assert table_context["count_truncated"] is True - assert "count_limit" not in table_context - - database_context = await get_template_context(context_ds, "/fixtures") - facetable = next( - table - for table in database_context["tables"] - if table["name"] == "facetable" - ) - assert facetable["count"] == 11 - assert facetable["count_truncated"] is True - assert "count_limit" not in database_context - finally: - db.count_limit = previous_count_limit - db._cached_table_counts = previous_cached_table_counts - - def test_base_context_keys_all_have_docs(): for name, doc in TEMPLATE_BASE_CONTEXT.items(): assert doc, "Base context key {} is missing docs".format(name) From e3ff63b0f93303a8dfe23d8d2d84bbc9551c3eea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:46:15 -0700 Subject: [PATCH 061/174] Warn plugin authors to avoid name conflicts with extra_template_vars Closes #1988 --- docs/plugin_hooks.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index d2f87e27..66419ec4 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -230,6 +230,10 @@ Function that returns an awaitable function that returns a dictionary Datasette runs Jinja2 in `async mode `__, which means you can add awaitable functions to the template scope and they will be automatically awaited when they are rendered by the template. +.. warning:: + + Be careful not to accidentally define a variable that conflicts with one that Datasette is already using for something else. Check :ref:`the template context documentation ` to see the variables defined by Datasette core. + Here's an example plugin that adds a ``"user_agent"`` variable to the template context containing the current request's User-Agent header: .. code-block:: python From a4f74d1d2bea4484f026c9adacae6ac961fd0f82 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 12:50:23 -0700 Subject: [PATCH 062/174] More unreleased changes in changelog --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 697df87d..01ef7f18 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,11 @@ Unreleased - New "Alter table" table action and ``//

/-/alter`` :ref:`JSON API ` for changing existing tables: add, rename, reorder and drop columns; change column types, defaults, ``NOT NULL`` constraints, primary keys and foreign keys; and rename the table. The alter table dialog also includes a "Drop table" button. (:issue:`2788`) - New ``//-/foreign-key-targets`` and ``//
/-/foreign-key-suggestions`` JSON APIs for discovering valid single-column foreign key targets and suggested relationships. - Create and alter table dialogs share their column-editing controls, including literal and expression defaults, custom column types, foreign keys and column ordering. +- The "Write to this database" page now includes a Create table starter template, alongside the existing Insert, Update and Delete templates. (:pr:`2794`) +- New :ref:`template_context` documentation listing the variables available to custom templates for Datasette's core pages. Variables documented there are treated as a stable API for custom templates until Datasette 2.0. The documentation is generated from dataclass definitions next to the view code, with tests that compare the documented fields against the actual contexts rendered by the database, table, query and row pages. (:issue:`1510`, :issue:`2127`, :issue:`1477`, :pr:`2803`) +- Database and table pages now use the ``count_truncated`` template context value to display capped row counts as ``>N rows`. +- Significant visual improvements to the table filter form UI, plus working add/remove filter buttons. (:issue:`2798`) +- Improved edit row icon on table pages. (:issue:`2796`) .. _v1_0_a34: From 5eca46a4bc5adebfb724a1c8e0e42852232152ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Jun 2026 13:44:58 -0700 Subject: [PATCH 063/174] Add cache-busted static asset helper (#2804) * Add cache-busted static asset helper Add a static() helper for Datasette, plugin, and mounted static assets that appends content-based hashes, caches hashes in production, and serves matching hashed asset URLs with immutable far-future cache headers. Closes #2800 --- datasette/app.py | 88 +++++++++++----- datasette/handle_exception.py | 1 - datasette/templates/_codemirror.html | 4 +- datasette/templates/api_explorer.html | 2 +- datasette/templates/base.html | 6 +- datasette/templates/database.html | 2 +- datasette/templates/debug_allowed.html | 2 +- datasette/templates/debug_autocomplete.html | 2 +- datasette/templates/debug_check.html | 2 +- datasette/templates/debug_rules.html | 2 +- datasette/templates/patterns.html | 2 +- datasette/templates/row.html | 4 +- datasette/templates/table.html | 10 +- datasette/utils/__init__.py | 11 ++ datasette/utils/asgi.py | 13 ++- docs/changelog.rst | 3 +- docs/contributing.rst | 3 + docs/custom_templates.rst | 39 +++++++ docs/internals.rst | 46 +++++++++ docs/plugin_hooks.rst | 5 +- docs/template_context.rst | 9 -- docs/writing_plugins.rst | 11 +- tests/test_html.py | 44 +++++++- tests/test_internals_datasette.py | 108 ++++++++++++++++++++ tests/test_utils.py | 7 ++ 25 files changed, 361 insertions(+), 65 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 57f893fe..9c9b7de4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -12,7 +12,6 @@ import dataclasses import datetime import functools import glob -import hashlib import httpx import importlib.metadata import inspect @@ -123,6 +122,7 @@ from .utils import ( parse_metadata, resolve_env_secrets, resolve_routes, + sha256_file, tilde_decode, tilde_encode, to_css_class, @@ -314,7 +314,7 @@ async def favicon(request, send): send, str(FAVICON_PATH), content_type="image/png", - headers={"Cache-Control": "max-age=3600, immutable, public"}, + headers={"Cache-Control": "max-age=3600, public"}, ) @@ -348,6 +348,16 @@ def _legacy_template_csrftoken(context): return "" +def _resolve_static_asset_path(root_path, path): + root = Path(root_path).resolve() + full_path = (root / path).resolve() + try: + full_path.relative_to(root) + except ValueError: + raise ValueError("Static asset path cannot escape static root") from None + return full_path + + # Documentation for the variables Datasette.render_template() adds to the # context for every page. This is part of the documented template contract: # keys added in render_template() must be documented here - the contract @@ -361,9 +371,6 @@ TEMPLATE_BASE_CONTEXT = { "menu_links": "Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`.", "display_actor": "Function that accepts an actor dictionary and returns the display string used in the navigation menu.", "show_logout": "True if the logout link should be shown in the navigation menu", - "app_css_hash": "Hash of Datasette's app.css contents, used for cache busting", - "edit_tools_js_hash": "Hash of Datasette's edit-tools.js contents, used for cache busting", - "table_js_hash": "Hash of Datasette's table.js contents, used for cache busting", "zip": "Python's ``zip()`` builtin, made available to template logic", "body_scripts": 'List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in `` - + + diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 5e483d34..aa1f0ecd 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -7,9 +7,9 @@ {% if row_mutation_ui %} {% if table_page_data.foreignKeys %} - + {% endif %} - + {% endif %} diff --git a/datasette/templates/_permissions_debug_tabs.html b/datasette/templates/_permissions_debug_tabs.html index d7203c1e..8e0f486e 100644 --- a/datasette/templates/_permissions_debug_tabs.html +++ b/datasette/templates/_permissions_debug_tabs.html @@ -44,10 +44,10 @@ diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 1ecc92df..fda4032c 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -3,29 +3,11 @@ {% block title %}Debug allow rules{% endblock %} {% block extra_head %} +{% include "_permission_ui_styles.html" %} {% endblock %} @@ -38,24 +20,28 @@ p.message-warning {

Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

-
-
-

- -
-
-

- -
-
- -
- +
+
+
+
+ + +
+
+ + +
+
+
+ +
+ -{% if error %}

{{ error }}

{% endif %} + {% if error %}

{{ error }}

{% endif %} -{% if result == "True" %}

Result: allow

{% endif %} + {% if result == "True" %}

Result: allow

{% endif %} -{% if result == "False" %}

Result: deny

{% endif %} + {% if result == "False" %}

Result: deny

{% endif %} +
{% endblock %} diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index 83cc1ae6..80249d9c 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -49,7 +49,7 @@
- + Number of results per page (max 200)
diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index 3b229a25..b9fc636a 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}Permission Check{% endblock %} +{% block title %}Explain a permission decision{% endblock %} {% block extra_head %} @@ -13,29 +13,35 @@ border-radius: 5px; } #output.allowed { - background-color: #e8f5e9; + background-color: #f3fbf4; border: 2px solid #4caf50; } #output.denied { - background-color: #ffebee; + background-color: #fff7f7; border: 2px solid #f44336; } #output h2 { margin-top: 0; } -#output .result-badge { +#output h3 { + margin-bottom: 0.5em; +} +#output .result-badge, +.effect-badge, +.rule-status { display: inline-block; - padding: 0.3em 0.8em; + padding: 0.2em 0.5em; border-radius: 3px; font-weight: bold; - font-size: 1.1em; } -#output .allowed-badge { - background-color: #4caf50; +#output .allowed-badge, +.effect-allow { + background-color: #2e7d32; color: white; } -#output .denied-badge { - background-color: #f44336; +#output .denied-badge, +.effect-deny { + background-color: #c62828; color: white; } .details-section { @@ -48,70 +54,130 @@ .details-section dd { margin-left: 1em; } +.explanation-section { + background: rgba(255, 255, 255, 0.75); + border: 1px solid #ddd; + border-radius: 4px; + margin-top: 1em; + padding: 0 1em 1em; +} +.rules-table { + border-collapse: collapse; + width: 100%; +} +.rules-table th, +.rules-table td { + border-bottom: 1px solid #ddd; + padding: 0.5em; + text-align: left; + vertical-align: top; +} +.rule-status { + background: #e8f5e9; + color: #1b5e20; +} +.rule-ignored { + background: #eee; + color: #555; + font-weight: normal; +} +.requirement-allowed { + color: #1b5e20; +} +.requirement-denied { + color: #b71c1c; +} +@media only screen and (max-width: 576px) { + .rules-table, + .rules-table tbody, + .rules-table tr, + .rules-table td { + display: block; + } + .rules-table thead { + display: none; + } + .rules-table td::before { + content: attr(data-label) ": "; + font-weight: bold; + } +} {% endblock %} {% block content %} -

Permission check

+

Explain a permission decision

{% set current_tab = "check" %} {% include "_permissions_debug_tabs.html" %} -

Use this tool to test permission checks for the current actor. It queries the /-/check.json API endpoint.

- -{% if request.actor %} -

Current actor: {{ request.actor.get("id", "anonymous") }}

-{% else %} -

Current actor: anonymous (not logged in)

-{% endif %} +

Test an actor, action and resource. The result explains which rules matched, which specificity level won, and whether actor restrictions or required actions changed the verdict.

-
+
- + + + Use null for an anonymous actor. This actor is simulated; it does not change who you are signed in as. +
+ +
+ - The permission action to check + The operation to evaluate
-
- +
+ - For database-level permissions, specify the database name + The database or other parent resource
-
- - - For table-level permissions, specify the table name (requires parent) +
+ + + The table, query or other child resource
- +
+actionSelect.addEventListener('change', updateResourceFields); +(function initializeFromUrl() { + const params = populateFormFromURL(); + updateResourceFields(); + if (params.get('action')) { + performCheck(); + } +})(); + {% endblock %} diff --git a/datasette/templates/debug_permissions_playground.html b/datasette/templates/debug_permissions_playground.html index 4410a677..8b0cbbcf 100644 --- a/datasette/templates/debug_permissions_playground.html +++ b/datasette/templates/debug_permissions_playground.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}Debug permissions{% endblock %} +{% block title %}Permission activity{% endblock %} {% block extra_head %} {% include "_permission_ui_styles.html" %} @@ -20,60 +20,45 @@ .check-action, .check-when, .check-result { font-size: 1.3em; } -textarea { - height: 10em; - width: 95%; - box-sizing: border-box; - padding: 0.5em; - border: 2px dotted black; -} -.two-col { - display: inline-block; - width: 48%; -} -.two-col label { - width: 48%; -} -@media only screen and (max-width: 576px) { - .two-col { - width: 100%; - } -} {% endblock %} {% block content %} -

Permission playground

+

Permission activity

{% set current_tab = "permissions" %} {% include "_permissions_debug_tabs.html" %} -

This tool lets you simulate an actor and a permission check for that actor.

+

Raw simulator

+ +

This form runs a hypothetical permission check and returns its raw explanation JSON. Use the Explain tool for a visual explanation of the same decision.

-
-
- - +
+
+
+ + +
-
-
-
- - -
-
- - -
-
- - +
+
+ + +
+
+ + +
+
+ + +
@@ -125,7 +110,7 @@ debugPost.addEventListener('submit', function(ev) { }); -

Recent permissions checks

+

Recent permission checks

{% if filter != "all" %}All{% else %}All{% endif %}, diff --git a/datasette/templates/debug_rules.html b/datasette/templates/debug_rules.html index d00ba9cc..233c0e94 100644 --- a/datasette/templates/debug_rules.html +++ b/datasette/templates/debug_rules.html @@ -37,7 +37,7 @@

- + Number of results per page (max 200)
diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index c7137e6b..67d3ce73 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -673,3 +673,239 @@ async def check_permission_for_resource( child=child, ) return results[action] + + +async def explain_permission_for_resource( + *, + datasette: "Datasette", + actor: dict | None, + action: str, + parent: str | None, + child: str | None, +) -> dict: + """Explain a permission decision for one action and resource. + + This is intended for Datasette's permission debugging tools. It uses the + same ``permission_resources_sql`` hook results and the same resolution + rules as :func:`check_permissions_for_actions`, but also returns the + matching rules, actor restriction results and ``also_requires`` chain. + + The returned dictionary is part of Datasette's unstable debugging API. + """ + + action_obj = datasette.actions.get(action) + if action_obj is None: + raise ValueError(f"Unknown action: {action}") + + explanation = await _explain_single_action( + datasette=datasette, + actor=actor, + action=action, + parent=parent, + child=child, + ) + + required_actions = [] + if action_obj.also_requires: + required = await explain_permission_for_resource( + datasette=datasette, + actor=actor, + action=action_obj.also_requires, + parent=parent, + child=child, + ) + required_actions.append(required) + + explanation["required_actions"] = required_actions + explanation["allowed"] = bool( + explanation["rule_allowed"] + and explanation["restriction_allowed"] + and all(required["allowed"] for required in required_actions) + ) + explanation["summary"] = _permission_explanation_summary(explanation) + return explanation + + +async def _explain_single_action( + *, + datasette: "Datasette", + actor: dict | None, + action: str, + parent: str | None, + child: str | None, +) -> dict: + """Return matching rules and restrictions for a single action.""" + from datasette.utils.permissions import SKIP_PERMISSION_CHECKS + + permission_sqls = await gather_permission_sql_from_hooks( + datasette=datasette, + actor=actor, + action=action, + ) + + if permission_sqls is SKIP_PERMISSION_CHECKS: + return { + "action": action, + "rule_allowed": True, + "restriction_allowed": True, + "winning_scope": "global", + "matched_rules": [ + { + "scope": "global", + "effect": "allow", + "source": "skip_permission_checks", + "reason": "Permission checks were explicitly skipped", + "decisive": True, + "ignored_because": None, + } + ], + "restrictions": [], + } + + db = datasette.get_internal_database() + matched_rules = [] + restrictions = [] + + for permission_sql in permission_sqls: + params = dict(permission_sql.params or {}) + parent_param = _unused_parameter_name(params, "_explain_parent") + params[parent_param] = parent + child_param = _unused_parameter_name(params, "_explain_child") + params[child_param] = child + + if permission_sql.sql: + rows = await db.execute( + f""" + SELECT parent, child, allow, reason + FROM ({permission_sql.sql}) AS permission_rules + WHERE (parent IS NULL OR parent = :{parent_param}) + AND (child IS NULL OR child = :{child_param}) + """, + params, + ) + for row in rows: + specificity = ( + 2 + if row["child"] is not None + else 1 if row["parent"] is not None else 0 + ) + matched_rules.append( + { + "scope": ("resource", "parent", "global")[2 - specificity], + "effect": "allow" if row["allow"] else "deny", + "source": permission_sql.source, + "reason": row["reason"], + "_specificity": specificity, + } + ) + + if permission_sql.restriction_sql: + restriction_row = ( + await db.execute( + f""" + SELECT EXISTS( + SELECT 1 FROM ({permission_sql.restriction_sql}) AS restriction_rules + WHERE (parent IS NULL OR parent = :{parent_param}) + AND (child IS NULL OR child = :{child_param}) + ) AS resource_is_in_allowlist + """, + params, + ) + ).first() + restriction_allowed = bool(restriction_row[0]) + restrictions.append( + { + "source": permission_sql.source, + "allowed": restriction_allowed, + "reason": params.get("deny") + or ( + "Resource is included in this restriction allowlist" + if restriction_allowed + else "Resource is not included in this restriction allowlist" + ), + } + ) + + matched_rules.sort( + key=lambda rule: ( + -rule["_specificity"], + 0 if rule["effect"] == "deny" else 1, + rule["source"] or "", + rule["reason"] or "", + ) + ) + + if matched_rules: + winning_specificity = matched_rules[0]["_specificity"] + winning_rules = [ + rule + for rule in matched_rules + if rule["_specificity"] == winning_specificity + ] + rule_allowed = not any(rule["effect"] == "deny" for rule in winning_rules) + winning_scope = winning_rules[0]["scope"] + else: + winning_specificity = None + rule_allowed = False + winning_scope = None + + for rule in matched_rules: + specificity = rule.pop("_specificity") + if specificity != winning_specificity: + rule["decisive"] = False + rule["ignored_because"] = "A more specific rule matched" + elif not rule_allowed and rule["effect"] == "allow": + rule["decisive"] = False + rule["ignored_because"] = "A deny rule matched at the same scope" + else: + rule["decisive"] = True + rule["ignored_because"] = None + + return { + "action": action, + "rule_allowed": rule_allowed, + "restriction_allowed": all( + restriction["allowed"] for restriction in restrictions + ), + "winning_scope": winning_scope, + "matched_rules": matched_rules, + "restrictions": restrictions, + } + + +def _unused_parameter_name(params: dict, preferred: str) -> str: + """Return a SQL parameter name that is not already in ``params``.""" + candidate = preferred + suffix = 2 + while candidate in params: + candidate = f"{preferred}_{suffix}" + suffix += 1 + return candidate + + +def _permission_explanation_summary(explanation: dict) -> str: + denied_requirement = next( + ( + required + for required in explanation["required_actions"] + if not required["allowed"] + ), + None, + ) + if denied_requirement: + return ( + f"Denied because {explanation['action']} also requires " + f"{denied_requirement['action']}, which was denied." + ) + if not explanation["matched_rules"]: + return "Denied because no permission rule matched this actor and resource." + if not explanation["rule_allowed"]: + return ( + f"Denied by a {explanation['winning_scope']}-level rule. " + "Deny rules take precedence over allow rules at the same scope." + ) + if not explanation["restriction_allowed"]: + return ( + "Denied because the resource is not included in the actor's restrictions." + ) + return f"Allowed by the matching {explanation['winning_scope']}-level rule." diff --git a/datasette/views/special.py b/datasette/views/special.py index c13191a1..28d34208 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -600,7 +600,7 @@ class PermissionRulesView(BaseView): async def _check_permission_for_actor(ds, action, parent, child, actor): - """Shared logic for checking permissions. Returns a dict with check results.""" + """Shared logic for checking and explaining a permission decision.""" if action not in ds.actions: return error_body(f"Unknown action: {action}", 404), 404 @@ -629,15 +629,28 @@ async def _check_permission_for_actor(ds, action, parent, child, actor): allowed = await ds.allowed(action=action, resource=resource_obj, actor=actor) + from datasette.utils.actions_sql import explain_permission_for_resource + + explanation = await explain_permission_for_resource( + datasette=ds, + actor=actor, + action=action, + parent=parent, + child=child, + ) + response = { "ok": True, + "unstable": UNSTABLE_API_MESSAGE, "action": action, "allowed": bool(allowed), + "actor": actor, "resource": { "parent": parent, "child": child, "path": _resource_path(parent, child), }, + "explanation": explanation, } if actor and "id" in actor: @@ -655,11 +668,25 @@ class PermissionCheckView(BaseView): as_format = request.url_vars.get("format") if not as_format: + actions = [ + { + "name": action.name, + "description": action.description, + "takes_parent": action.takes_parent, + "takes_child": action.takes_child, + "also_requires": action.also_requires, + } + for action in sorted( + self.ds.actions.values(), key=lambda action: action.name + ) + ] return await self.render( ["debug_check.html"], request, { - "sorted_actions": sorted(self.ds.actions.keys()), + "actions": actions, + "actor_json": request.args.get("actor") + or json.dumps(request.actor, indent=2), "has_debug_permission": True, }, ) @@ -671,9 +698,18 @@ class PermissionCheckView(BaseView): parent = request.args.get("parent") child = request.args.get("child") + actor = request.actor + actor_json = request.args.get("actor") + if actor_json is not None: + try: + actor = json.loads(actor_json) + except json.JSONDecodeError as ex: + return Response.error(f"Invalid actor JSON: {ex}", 400) + if actor is not None and not isinstance(actor, dict): + return Response.error("actor must be a JSON object or null", 400) response, status = await _check_permission_for_actor( - self.ds, action, parent, child, request.actor + self.ds, action, parent, child, actor ) return Response.json(response, status=status) diff --git a/docs/authentication.rst b/docs/authentication.rst index 51fa07d5..d720c4db 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -45,7 +45,7 @@ Using the "root" actor Datasette currently leaves almost all forms of authentication to plugins - `datasette-auth-github `__ for example. -The one exception is the "root" account, which you can sign into while using Datasette on your local machine. The root user has **all permissions** - they can perform any action regardless of other permission rules. +The one exception is the "root" account, which you can sign into while using Datasette on your local machine. The root user starts with **all permissions**: Datasette contributes a global allow rule for every action. More specific deny rules can still override that global rule. The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including: @@ -84,12 +84,12 @@ Click on that link and then visit ``http://127.0.0.1:8001/-/actor`` to confirm t Permissions =========== -Datasette's permissions system is built around SQL queries. Datasette and its plugins construct SQL queries to resolve the list of resources that an actor cas access. - The key question the permissions system answers is this: Is this **actor** allowed to perform this **action**, optionally against this particular **resource**? +Every permission decision can be understood in terms of those three values. Datasette implements the decisions using SQL, but you do not need to understand the generated SQL to configure or debug permissions. + **Actors** are :ref:`described above `. An **action** is a string describing the action the actor would like to perform. A full list is :ref:`provided below ` - examples include ``view-table`` and ``execute-sql``. @@ -138,7 +138,51 @@ This configuration will deny access to everyone except the user with ``id`` of ` How permissions are resolved ---------------------------- -Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. +Permission rules describe an effect (``allow`` or ``deny``) at one of three levels: + +``resource`` + A specific child resource, such as the ``analytics/sales`` table. + +``parent`` + A parent resource, such as the ``analytics`` database. A parent rule also applies to its child resources. + +``global`` + Every resource for that action. + +Datasette resolves matching rules from most specific to least specific: + +#. Resource rules take precedence over parent and global rules. +#. Parent rules take precedence over global rules. +#. If both allow and deny rules match at the same level, deny takes precedence. +#. If no rule matches, access is denied. + +This means a resource-level allow can provide an exception to a parent-level deny. It also means that two plugins which disagree at the same level resolve to deny. + +.. list-table:: Permission rule examples + :header-rows: 1 + + * - Matching rules + - Result + - Explanation + * - Global allow + - Allow + - The global rule is the most specific matching rule. + * - Global allow, parent deny + - Deny + - The parent rule is more specific. + * - Parent deny, resource allow + - Allow + - The resource rule is more specific. + * - Resource allow and resource deny + - Deny + - Deny takes precedence at the same level. + * - No matching rules + - Deny + - Permissions default to deny when no rule applies. + +The built-in public defaults are global allow rules for actions such as ``view-instance``, ``view-database`` and ``view-table``. They follow the same precedence rules as configuration and plugin rules. The ``--default-deny`` option prevents Datasette from contributing those default allow rules. + +Datasette performs checks using :ref:`datasette_allowed`, which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified. @@ -149,12 +193,12 @@ resources were allowed or denied. The combined sources are: * ``allow`` blocks configured in :ref:`datasette.yaml `. * :ref:`Actor restrictions ` encoded into the actor dictionary or API token. -* The "root" user shortcut when ``--root`` (or :attr:`Datasette.root_enabled `) is active, replying ``True`` to all permission chucks unless configuration rules deny them at a more specific level. +* The "root" user rule when ``--root`` (or :attr:`Datasette.root_enabled `) is active. This is a global allow rule, so a more specific configuration deny can override it. * Any additional SQL provided by plugins implementing :ref:`plugin_hook_permission_resources_sql`. -Datasette evaluates the SQL to determine if the requested ``resource`` is -included. Explicit deny rules returned by configuration or plugins will block -access even if other rules allowed it. +Actor restrictions are applied after the allow/deny rules. They act as an additional allowlist: a restriction can remove access but cannot grant access that the actor did not already have. See :ref:`authentication_cli_create_token_restrict`. + +Some actions have dependencies on other actions. These are evaluated as an ``AND`` condition. For example, ``execute-sql`` also requires ``view-database``: both decisions must be allowed for the final result to be allowed. .. _authentication_permissions_allow: @@ -1145,11 +1189,21 @@ The debug tool at ``/-/permissions`` is available to any actor with the ``permis datasette -s permissions.permissions-debug true data.db -The page shows the permission checks that have been carried out by the Datasette instance. +The permission debug tools answer four different questions: -It also provides an interface for running hypothetical permission checks against a hypothetical actor. This is a useful way of confirming that your configured permissions work in the way you expect. +Why was this decision allowed or denied? + Use :ref:`PermissionCheckView`. It shows every matching rule, identifies the winning specificity level, applies actor restrictions and evaluates any required actions. -This is designed to help administrators and plugin authors understand exactly how permission checks are being carried out, in order to effectively configure Datasette's permission system. +Which resources can the current actor access? + Use :ref:`AllowedResourcesView` to view an access map for a selected action. + +Which raw rules did Datasette and its plugins contribute? + Use :ref:`PermissionRulesView` to inspect the rules before they are resolved into decisions. + +Which checks has this Datasette instance performed recently? + Use ``/-/permissions`` to view recent permission activity. + +These tools are designed to help administrators and plugin authors understand and confirm the effective permissions configuration. These debug endpoints are exempt from the :ref:`JSON API stability promise ` - their JSON shapes may change in future releases. @@ -1184,11 +1238,20 @@ This endpoint requires the ``permissions-debug`` permission. Permission check view --------------------- -The ``/-/check`` endpoint evaluates a single action/resource pair and returns information indicating whether the access was allowed along with diagnostic information. +The ``/-/check`` endpoint evaluates and explains a single actor, action and resource decision. The explanation includes: + +* Every matching allow and deny rule, with its source and reason. +* The winning resource, parent or global scope. +* Rules ignored because a more specific rule matched, or because a deny won at the same scope. +* Actor restriction allowlists that included or excluded the resource. +* Additional actions required by the requested action. +* An explicit default-deny explanation when no rule matched. This endpoint provides an interactive HTML form interface. Add ``.json`` to the URL path (e.g. ``/-/check.json?action=view-instance``) to get the raw JSON response instead. -Pass ``?action=`` to specify the action to check, and optional ``?parent=`` and ``?child=`` parameters to specify the resource. +Pass ``?action=`` to specify the action to check, and optional ``?parent=`` and ``?child=`` parameters to specify the resource. The interactive form also accepts actor JSON, allowing a hypothetical actor to be tested without signing in as that actor. The JSON endpoint accepts the same value using the ``actor`` query string parameter. Use ``actor=null`` to represent an anonymous actor. + +This endpoint requires the ``permissions-debug`` permission. The hypothetical actor is used only for the decision being explained; access to the debug tool is checked against the actor who is actually signed in. .. _authentication_ds_actor: diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 88fe577f..cd1050d0 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -457,6 +457,20 @@ async def test_permissions_debug(ds_client, filter_): assert checks == expected_checks +@pytest.mark.asyncio +@pytest.mark.parametrize( + "permissions_debug,expected_status", + ( + (1, 200), + (0, 403), + ), +) +async def test_permissions_debug_numeric_boolean(permissions_debug, expected_status): + ds = Datasette(config={"permissions": {"permissions-debug": permissions_debug}}) + response = await ds.client.get("/-/permissions") + assert response.status_code == expected_status + + @pytest.mark.asyncio @pytest.mark.parametrize( "actor,allow,expected_fragment", @@ -748,7 +762,12 @@ async def test_actor_restricted_permissions( } if actor.get("id"): expected["actor_id"] = actor["id"] - assert response.json() == expected + data = response.json() + for key, value in expected.items(): + assert data[key] == value + assert data["actor"] == actor + assert data["explanation"]["allowed"] is expected_result + assert data["explanation"]["summary"] PermConfigTestCase = collections.namedtuple( @@ -1734,6 +1753,8 @@ async def test_permission_check_view_requires_debug_permission(): data = response.json() assert data["action"] == "view-instance" assert data["allowed"] is True + assert data["explanation"]["allowed"] is True + assert data["explanation"]["summary"] @pytest.mark.asyncio @@ -1759,6 +1780,211 @@ async def test_permission_check_view_query_actions(action): } +@pytest.mark.asyncio +async def test_permission_check_explains_specificity_for_hypothetical_actor(): + ds = Datasette( + config={ + "permissions": {"view-table": {"id": "alice"}}, + "databases": { + "analytics": { + "permissions": {"view-table": False}, + "tables": { + "public": {"permissions": {"view-table": {"id": "alice"}}} + }, + } + }, + } + ) + ds.root_enabled = True + await ds.invoke_startup() + + def path_for(child): + return "/-/check.json?" + urllib.parse.urlencode( + { + "action": "view-table", + "parent": "analytics", + "child": child, + "actor": json.dumps({"id": "alice"}), + } + ) + + public_response = await ds.client.get(path_for("public"), actor={"id": "root"}) + assert public_response.status_code == 200 + public = public_response.json() + assert public["actor"] == {"id": "alice"} + assert public["allowed"] is True + assert public["explanation"]["allowed"] is True + assert public["explanation"]["winning_scope"] == "resource" + public_rules = public["explanation"]["matched_rules"] + assert any( + rule["scope"] == "resource" and rule["effect"] == "allow" and rule["decisive"] + for rule in public_rules + ) + assert any( + rule["scope"] == "parent" + and rule["effect"] == "deny" + and rule["ignored_because"] == "A more specific rule matched" + for rule in public_rules + ) + + private_response = await ds.client.get(path_for("private"), actor={"id": "root"}) + assert private_response.status_code == 200 + private = private_response.json() + assert private["allowed"] is False + assert private["explanation"]["allowed"] is False + assert private["explanation"]["winning_scope"] == "parent" + assert private["explanation"]["summary"].startswith("Denied by a parent-level rule") + + +@pytest.mark.asyncio +async def test_permission_check_explains_deny_wins_at_same_scope(): + ds = Datasette(config={"permissions": {"view-table": {"id": "someone-else"}}}) + ds.root_enabled = True + await ds.invoke_startup() + path = "/-/check.json?" + urllib.parse.urlencode( + { + "action": "view-table", + "parent": "analytics", + "child": "users", + "actor": json.dumps({"id": "alice"}), + } + ) + response = await ds.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["allowed"] is False + assert data["explanation"]["winning_scope"] == "global" + rules = data["explanation"]["matched_rules"] + assert any(rule["effect"] == "deny" and rule["decisive"] for rule in rules) + assert any( + rule["effect"] == "allow" + and rule["ignored_because"] == "A deny rule matched at the same scope" + for rule in rules + ) + + +@pytest.mark.asyncio +async def test_permission_check_explains_default_deny(): + ds = Datasette() + ds.root_enabled = True + await ds.invoke_startup() + path = "/-/check.json?" + urllib.parse.urlencode( + { + "action": "insert-row", + "parent": "analytics", + "child": "users", + "actor": json.dumps({"id": "alice"}), + } + ) + response = await ds.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["allowed"] is False + explanation = data["explanation"] + assert explanation["allowed"] is False + assert explanation["matched_rules"] == [] + assert explanation["winning_scope"] is None + assert explanation["summary"] == ( + "Denied because no permission rule matched this actor and resource." + ) + + +@pytest.mark.asyncio +async def test_permission_check_explains_actor_restrictions(): + ds = Datasette() + ds.root_enabled = True + await ds.invoke_startup() + restricted_actor = { + "id": "alice", + "_r": {"r": {"analytics": {"public": ["vt"]}}}, + } + path = "/-/check.json?" + urllib.parse.urlencode( + { + "action": "view-table", + "parent": "analytics", + "child": "private", + "actor": json.dumps(restricted_actor), + } + ) + response = await ds.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["allowed"] is False + explanation = data["explanation"] + assert explanation["rule_allowed"] is True + assert explanation["restriction_allowed"] is False + assert explanation["allowed"] is False + assert explanation["restrictions"] + assert any( + restriction["allowed"] is False for restriction in explanation["restrictions"] + ) + assert "actor's restrictions" in explanation["summary"] + + +@pytest.mark.asyncio +async def test_permission_check_explains_required_actions(): + from datasette import hookimpl + from datasette.permissions import PermissionSQL + + class StoreQueryPermissions: + @hookimpl + def permission_resources_sql(self, actor, action): + if not actor or actor.get("id") != "alice": + return None + if action == "store-query": + return PermissionSQL( + sql="SELECT 'analytics' AS parent, NULL AS child, 1 AS allow, 'alice can store queries' AS reason" + ) + if action == "execute-sql": + return PermissionSQL( + sql="SELECT 'analytics' AS parent, NULL AS child, 0 AS allow, 'alice cannot execute SQL' AS reason" + ) + + ds = Datasette() + ds.root_enabled = True + await ds.invoke_startup() + ds.pm.register(StoreQueryPermissions(), name="store-query-test") + path = "/-/check.json?" + urllib.parse.urlencode( + { + "action": "store-query", + "parent": "analytics", + "actor": json.dumps({"id": "alice"}), + } + ) + response = await ds.client.get(path, actor={"id": "root"}) + assert response.status_code == 200 + data = response.json() + assert data["allowed"] is False + explanation = data["explanation"] + assert explanation["rule_allowed"] is True + assert explanation["required_actions"][0]["action"] == "execute-sql" + assert explanation["required_actions"][0]["allowed"] is False + assert explanation["summary"] == ( + "Denied because store-query also requires execute-sql, which was denied." + ) + + +@pytest.mark.asyncio +async def test_permission_check_hypothetical_actor_validation(): + ds = Datasette() + ds.root_enabled = True + await ds.invoke_startup() + + response = await ds.client.get( + "/-/check.json?action=view-instance&actor=not-json", + actor={"id": "root"}, + ) + assert response.status_code == 400 + assert response.json()["error"].startswith("Invalid actor JSON:") + + response = await ds.client.get( + "/-/check.json?action=view-instance&actor=%5B%5D", + actor={"id": "root"}, + ) + assert response.status_code == 400 + assert response.json()["error"] == "actor must be a JSON object or null" + + @pytest.mark.asyncio async def test_root_allow_block_with_table_restricted_actor(): """ From 9cfc252394eb21d05f45818dac9c374e2d141a35 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jul 2026 08:41:27 -0700 Subject: [PATCH 169/174] Make internal catalog refresh atomic Refs #2831 --- datasette/app.py | 14 +--- datasette/utils/internal_db.py | 133 ++++++++++++++++++--------------- 2 files changed, 72 insertions(+), 75 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index c44f9095..0e31273d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -753,19 +753,7 @@ class Datasette: # Compare schema versions to see if we should skip it if schema_version == current_schema_versions.get(database_name): continue - placeholders = "(?, ?, ?, ?)" - values = [database_name, str(db.path), db.is_memory, schema_version] - if db.path is None: - placeholders = "(?, null, ?, ?)" - values = [database_name, db.is_memory, schema_version] - await internal_db.execute_write( - """ - INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version) - VALUES {} - """.format(placeholders), - values, - ) - await populate_schema_tables(internal_db, db) + await populate_schema_tables(internal_db, db, schema_version) @property def urls(self): diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 10ca32a5..e061d882 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -180,29 +180,9 @@ async def init_internal_db(db): await db.execute_write_fn(apply_migrations, transaction=False) -async def populate_schema_tables(internal_db, db): +async def populate_schema_tables(internal_db, db, schema_version): database_name = db.name - def delete_everything(conn): - conn.execute( - "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_views WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_foreign_keys WHERE database_name = ?", - [database_name], - ) - conn.execute( - "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] - ) - - await internal_db.execute_write_fn(delete_everything) - tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows views = (await db.execute("select * from sqlite_master WHERE type = 'view'")).rows @@ -266,47 +246,76 @@ async def populate_schema_tables(internal_db, db): indexes_to_insert, ) = await db.execute_fn(collect_info) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) - values (?, ?, ?, ?) - """, - tables_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_views (database_name, view_name, rootpage, sql) - values (?, ?, ?, ?) - """, - views_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_columns ( - database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden - ) VALUES ( - :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden + def replace_catalog(conn): + # Delete child rows before their catalog_tables parents so this also + # works if a prepare_connection plugin enables foreign key enforcement. + for table in ( + "catalog_columns", + "catalog_foreign_keys", + "catalog_indexes", + "catalog_views", + "catalog_tables", + ): + conn.execute( + "DELETE FROM {} WHERE database_name = ?".format(table), + [database_name], + ) + conn.execute( + """ + INSERT OR REPLACE INTO catalog_databases ( + database_name, path, is_memory, schema_version + ) VALUES (?, ?, ?, ?) + """, + [ + database_name, + str(db.path) if db.path is not None else None, + db.is_memory, + schema_version, + ], ) - """, - columns_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_foreign_keys ( - database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match - ) VALUES ( - :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match + conn.executemany( + """ + INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) + values (?, ?, ?, ?) + """, + tables_to_insert, ) - """, - foreign_keys_to_insert, - ) - await internal_db.execute_write_many( - """ - INSERT INTO catalog_indexes ( - database_name, table_name, seq, name, "unique", origin, partial - ) VALUES ( - :database_name, :table_name, :seq, :name, :unique, :origin, :partial + conn.executemany( + """ + INSERT INTO catalog_views (database_name, view_name, rootpage, sql) + values (?, ?, ?, ?) + """, + views_to_insert, ) - """, - indexes_to_insert, - ) + conn.executemany( + """ + INSERT INTO catalog_columns ( + database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden + ) VALUES ( + :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden + ) + """, + columns_to_insert, + ) + conn.executemany( + """ + INSERT INTO catalog_foreign_keys ( + database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match + ) VALUES ( + :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match + ) + """, + foreign_keys_to_insert, + ) + conn.executemany( + """ + INSERT INTO catalog_indexes ( + database_name, table_name, seq, name, "unique", origin, partial + ) VALUES ( + :database_name, :table_name, :seq, :name, :unique, :origin, :partial + ) + """, + indexes_to_insert, + ) + + await internal_db.execute_write_fn(replace_catalog) From 591b909a4d216ed76d3c775484df52b54e89dc74 Mon Sep 17 00:00:00 2001 From: TowyTowy <85077986+TowyTowy@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:53:45 +0200 Subject: [PATCH 170/174] Escape table names with [square] brackets, refs #2431 (#2846) Several internal helpers quoted table names using SQLite [bracket] identifiers built with an f-string, e.g. PRAGMA foreign_key_list([{table}]). Bracket quoting cannot escape a "]" character, so any table whose name contains "]" (for example "[foo]" or "foo]") produced "sqlite3.OperationalError: unrecognized token" - crashing schema introspection at startup and 500-ing the table page. Switch these call sites to the existing escape_sqlite() helper, which uses "double quote" quoting with correct "" escaping (the same approach already used elsewhere in the codebase and in the test suite): - utils/internal_db.py: PRAGMA foreign_key_list / index_list - utils/__init__.py: get_outbound_foreign_keys - database.py: table_counts count query - facets.py: default "select * from" SQL Added a regression test covering table names with "]" characters. Co-authored-by: Claude --- datasette/database.py | 3 ++- datasette/facets.py | 2 +- datasette/utils/__init__.py | 2 +- datasette/utils/internal_db.py | 8 +++++--- tests/test_api.py | 33 ++++++++++++++++++++++++++++++++- 5 files changed, 41 insertions(+), 7 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index eb402b0c..bab3a378 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -17,6 +17,7 @@ from .utils import ( detect_fts, detect_primary_keys, detect_spatialite, + escape_sqlite, get_all_foreign_keys, get_outbound_foreign_keys, md5_not_usedforsecurity, @@ -608,7 +609,7 @@ class Database: try: table_count = ( await self.execute( - f"select count(*) from (select * from [{table}] limit {self.count_limit + 1})", + f"select count(*) from (select * from {escape_sqlite(table)} limit {self.count_limit + 1})", custom_time_limit=limit, ) ).rows[0][0] diff --git a/datasette/facets.py b/datasette/facets.py index abe0605e..5f757df3 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -85,7 +85,7 @@ class Facet: self.database = database # For foreign key expansion. Can be None for e.g. stored SQL queries: self.table = table - self.sql = sql or f"select * from [{table}]" + self.sql = sql or f"select * from {escape_sqlite(table)}" self.params = params or [] self.table_config = table_config # row_count can be None, in which case we calculate it ourselves: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 42574d3b..18d3ba52 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -636,7 +636,7 @@ def detect_primary_keys(conn, table): def get_outbound_foreign_keys(conn, table): - infos = conn.execute(f"PRAGMA foreign_key_list([{table}])").fetchall() + infos = conn.execute(f"PRAGMA foreign_key_list({escape_sqlite(table)})").fetchall() fks = [] for info in infos: if info is not None: diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index e061d882..702b53d8 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -3,7 +3,7 @@ import textwrap from sqlite_utils import Database as SQLiteUtilsDatabase from sqlite_utils import Migrations -from datasette.utils import table_column_details +from datasette.utils import escape_sqlite, table_column_details INTERNAL_DB_SCHEMA_TABLES = { "catalog_databases", @@ -213,7 +213,7 @@ async def populate_schema_tables(internal_db, db, schema_version): for column in columns ) foreign_keys = conn.execute( - f"PRAGMA foreign_key_list([{table_name}])" + f"PRAGMA foreign_key_list({escape_sqlite(table_name)})" ).fetchall() foreign_keys_to_insert.extend( { @@ -222,7 +222,9 @@ async def populate_schema_tables(internal_db, db, schema_version): } for foreign_key in foreign_keys ) - indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall() + indexes = conn.execute( + f"PRAGMA index_list({escape_sqlite(table_name)})" + ).fetchall() indexes_to_insert.extend( { **{"database_name": database_name, "table_name": table_name}, diff --git a/tests/test_api.py b/tests/test_api.py index d5f519b9..191d064a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,6 @@ from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS -from datasette.utils import UNSTABLE_API_MESSAGE +from datasette.utils import UNSTABLE_API_MESSAGE, escape_sqlite, tilde_encode from datasette.utils.sqlite import sqlite_version from datasette.version import __version__ from .fixtures import make_app_client, EXPECTED_PLUGINS @@ -930,6 +930,37 @@ async def test_tilde_encoded_database_names(db_name): assert response2.status_code == 200 +@pytest.mark.asyncio +@pytest.mark.parametrize("table_name", ("[foo]", "foo]", "[foo]/bar")) +async def test_table_with_reserved_characters_in_name(table_name): + # Table names containing characters such as "]" that cannot be escaped + # using SQLite [bracket] quoting used to break schema introspection and + # the table page - https://github.com/simonw/datasette/issues/2431 + ds = Datasette() + db = ds.add_memory_database("test_reserved_table_names") + await db.execute_write( + "create table {} (id integer primary key, name text)".format( + escape_sqlite(table_name) + ) + ) + await db.execute_write( + "insert into {} (id, name) values (1, 'one')".format(escape_sqlite(table_name)) + ) + # Schema introspection (populate_schema_tables) must not crash: + db_response = await ds.client.get("/test_reserved_table_names.json") + assert db_response.status_code == 200 + tables = {t["name"]: t for t in db_response.json()["tables"]} + assert tables[table_name]["count"] == 1 + # And the table page itself must load and return the row: + table_response = await ds.client.get( + "/test_reserved_table_names/{}.json?_shape=array".format( + tilde_encode(table_name) + ) + ) + assert table_response.status_code == 200 + assert table_response.json() == [{"id": 1, "name": "one"}] + + @pytest.mark.asyncio @pytest.mark.parametrize( "config,expected", From 8b7c942d5e5ada887aa89c4d58567af39d5a3e07 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jul 2026 09:18:51 -0700 Subject: [PATCH 171/174] Major performance boost for SQL permissions, closes #2832 --- datasette/utils/actions_sql.py | 217 +++++++++++++++++---------------- 1 file changed, 111 insertions(+), 106 deletions(-) diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index 67d3ce73..d767e391 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -252,88 +252,62 @@ async def _build_single_action_sql( ] ) - # Continue with the cascading logic - query_parts.extend( - [ - "child_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", - " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", - " FROM base b", - " LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child = b.child", - " GROUP BY b.parent, b.child", - "),", - "parent_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", - " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", - " FROM base b", - " LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", - "global_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,", - " json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,", - " json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons", - " FROM base b", - " LEFT JOIN all_rules ar ON ar.parent IS NULL AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", + # Continue with the cascading logic. + # Aggregate the RULES by cascade level (small), rather than grouping + # base x rules (which scales with the number of resources). + def _agg(select_key, where, group_by): + parts = [ + f" SELECT {select_key}", + " MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow,", + " json_group_array(CASE WHEN allow = 0 THEN source_plugin || ': ' || reason END) AS deny_reasons,", + " json_group_array(CASE WHEN allow = 1 THEN source_plugin || ': ' || reason END) AS allow_reasons", + f" FROM all_rules WHERE {where}", ] + if group_by: + parts.append(f" GROUP BY {group_by}") + return parts + + query_parts.extend( + ["child_agg AS ("] + + _agg( + "parent, child,", + "parent IS NOT NULL AND child IS NOT NULL", + "parent, child", + ) + + ["),", "parent_agg AS ("] + + _agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent") + + ["),", "global_agg AS ("] + + _agg("", "parent IS NULL AND child IS NULL", None) + + ["),"] ) # Add anonymous decision logic if needed if include_is_private: - query_parts.extend( - [ - "anon_child_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", - " FROM base b", - " LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child = b.child", - " GROUP BY b.parent, b.child", - "),", - "anon_parent_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", - " FROM base b", - " LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", - "anon_global_lvl AS (", - " SELECT b.parent, b.child,", - " MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,", - " MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow", - " FROM base b", - " LEFT JOIN anon_rules ar ON ar.parent IS NULL AND ar.child IS NULL", - " GROUP BY b.parent, b.child", - "),", - "anon_decisions AS (", - " SELECT", - " b.parent, b.child,", - " CASE", - " WHEN acl.any_deny = 1 THEN 0", - " WHEN acl.any_allow = 1 THEN 1", - " WHEN apl.any_deny = 1 THEN 0", - " WHEN apl.any_allow = 1 THEN 1", - " WHEN agl.any_deny = 1 THEN 0", - " WHEN agl.any_allow = 1 THEN 1", - " ELSE 0", - " END AS anon_is_allowed", - " FROM base b", - " JOIN anon_child_lvl acl ON b.parent = acl.parent AND (b.child = acl.child OR (b.child IS NULL AND acl.child IS NULL))", - " JOIN anon_parent_lvl apl ON b.parent = apl.parent AND (b.child = apl.child OR (b.child IS NULL AND apl.child IS NULL))", - " JOIN anon_global_lvl agl ON b.parent = agl.parent AND (b.child = agl.child OR (b.child IS NULL AND agl.child IS NULL))", - "),", + + def _anon_agg(select_key, where, group_by): + parts = [ + f" SELECT {select_key}", + " MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,", + " MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow", + f" FROM anon_rules WHERE {where}", ] + if group_by: + parts.append(f" GROUP BY {group_by}") + return parts + + query_parts.extend( + ["anon_child_agg AS ("] + + _anon_agg( + "parent, child,", + "parent IS NOT NULL AND child IS NOT NULL", + "parent, child", + ) + + ["),", "anon_parent_agg AS ("] + + _anon_agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent") + + ["),", "anon_global_agg AS ("] + + _anon_agg("", "parent IS NULL AND child IS NULL", None) + + ["),"] ) # Final decisions @@ -342,31 +316,28 @@ async def _build_single_action_sql( "decisions AS (", " SELECT", " b.parent, b.child,", - " -- Cascading permission logic: child → parent → global, DENY beats ALLOW at each level", + " -- Cascading permission logic: child -> parent -> global, DENY beats ALLOW at each level", " -- Priority order:", - " -- 1. Child-level deny (most specific, blocks access)", - " -- 2. Child-level allow (most specific, grants access)", - " -- 3. Parent-level deny (intermediate, blocks access)", - " -- 4. Parent-level allow (intermediate, grants access)", - " -- 5. Global-level deny (least specific, blocks access)", - " -- 6. Global-level allow (least specific, grants access)", + " -- 1. Child-level deny 2. Child-level allow", + " -- 3. Parent-level deny 4. Parent-level allow", + " -- 5. Global-level deny 6. Global-level allow", " -- 7. Default deny (no rules match)", " CASE", - " WHEN cl.any_deny = 1 THEN 0", - " WHEN cl.any_allow = 1 THEN 1", - " WHEN pl.any_deny = 1 THEN 0", - " WHEN pl.any_allow = 1 THEN 1", - " WHEN gl.any_deny = 1 THEN 0", - " WHEN gl.any_allow = 1 THEN 1", + " WHEN ca.any_deny = 1 THEN 0", + " WHEN ca.any_allow = 1 THEN 1", + " WHEN pa.any_deny = 1 THEN 0", + " WHEN pa.any_allow = 1 THEN 1", + " WHEN ga.any_deny = 1 THEN 0", + " WHEN ga.any_allow = 1 THEN 1", " ELSE 0", " END AS is_allowed,", " CASE", - " WHEN cl.any_deny = 1 THEN cl.deny_reasons", - " WHEN cl.any_allow = 1 THEN cl.allow_reasons", - " WHEN pl.any_deny = 1 THEN pl.deny_reasons", - " WHEN pl.any_allow = 1 THEN pl.allow_reasons", - " WHEN gl.any_deny = 1 THEN gl.deny_reasons", - " WHEN gl.any_allow = 1 THEN gl.allow_reasons", + " WHEN ca.any_deny = 1 THEN ca.deny_reasons", + " WHEN ca.any_allow = 1 THEN ca.allow_reasons", + " WHEN pa.any_deny = 1 THEN pa.deny_reasons", + " WHEN pa.any_allow = 1 THEN pa.allow_reasons", + " WHEN ga.any_deny = 1 THEN ga.deny_reasons", + " WHEN ga.any_allow = 1 THEN ga.allow_reasons", " ELSE '[]'", " END AS reason", ] @@ -374,21 +345,34 @@ async def _build_single_action_sql( if include_is_private: query_parts.append( - " , CASE WHEN ad.anon_is_allowed = 0 THEN 1 ELSE 0 END AS is_private" + " , CASE WHEN (" + "CASE" + " WHEN aca.any_deny = 1 THEN 0" + " WHEN aca.any_allow = 1 THEN 1" + " WHEN apa.any_deny = 1 THEN 0" + " WHEN apa.any_allow = 1 THEN 1" + " WHEN aga.any_deny = 1 THEN 0" + " WHEN aga.any_allow = 1 THEN 1" + " ELSE 0 END" + ") = 0 THEN 1 ELSE 0 END AS is_private" ) query_parts.extend( [ " FROM base b", - " JOIN child_lvl cl ON b.parent = cl.parent AND (b.child = cl.child OR (b.child IS NULL AND cl.child IS NULL))", - " JOIN parent_lvl pl ON b.parent = pl.parent AND (b.child = pl.child OR (b.child IS NULL AND pl.child IS NULL))", - " JOIN global_lvl gl ON b.parent = gl.parent AND (b.child = gl.child OR (b.child IS NULL AND gl.child IS NULL))", + " LEFT JOIN child_agg ca ON ca.parent = b.parent AND ca.child = b.child", + " LEFT JOIN parent_agg pa ON pa.parent = b.parent", + " CROSS JOIN global_agg ga", ] ) if include_is_private: - query_parts.append( - " JOIN anon_decisions ad ON b.parent = ad.parent AND (b.child = ad.child OR (b.child IS NULL AND ad.child IS NULL))" + query_parts.extend( + [ + " LEFT JOIN anon_child_agg aca ON aca.parent = b.parent AND aca.child = b.child", + " LEFT JOIN anon_parent_agg apa ON apa.parent = b.parent", + " CROSS JOIN anon_global_agg aga", + ] ) query_parts.append(")") @@ -400,8 +384,28 @@ async def _build_single_action_sql( restriction_intersect = "\nINTERSECT\n".join( f"SELECT * FROM ({sql})" for sql in restriction_sqls ) + # Decompose by NULL-pattern so the final filter can use pure-equality + # EXISTS lookups (satisfiable via automatic indexes) instead of a + # correlated OR-scan over the whole list. query_parts.extend( - [",", "restriction_list AS (", f" {restriction_intersect}", ")"] + [ + ",", + "restriction_list AS (", + f" {restriction_intersect}", + "),", + "restriction_exact AS (", + " SELECT parent, child FROM restriction_list WHERE parent IS NOT NULL AND child IS NOT NULL", + "),", + "restriction_parent_any AS (", + " SELECT DISTINCT parent FROM restriction_list WHERE parent IS NOT NULL AND child IS NULL", + "),", + "restriction_child_any AS (", + " SELECT DISTINCT child FROM restriction_list WHERE parent IS NULL AND child IS NOT NULL", + "),", + "restriction_all AS (", + " SELECT 1 AS matched FROM restriction_list WHERE parent IS NULL AND child IS NULL LIMIT 1", + ")", + ] ) # Final SELECT @@ -416,10 +420,11 @@ async def _build_single_action_sql( # Add restriction filter if there are restrictions if restriction_sqls: query_parts.append(""" - AND EXISTS ( - SELECT 1 FROM restriction_list r - WHERE (r.parent = decisions.parent OR r.parent IS NULL) - AND (r.child = decisions.child OR r.child IS NULL) + AND ( + EXISTS (SELECT 1 FROM restriction_all) + OR EXISTS (SELECT 1 FROM restriction_parent_any r WHERE r.parent = decisions.parent) + OR EXISTS (SELECT 1 FROM restriction_child_any r WHERE r.child = decisions.child) + OR EXISTS (SELECT 1 FROM restriction_exact r WHERE r.parent = decisions.parent AND r.child = decisions.child) )""") # Add parent filter if specified From 2ffd8a860e84ff58922e633c8e85e9a8e088ca93 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jul 2026 09:28:29 -0700 Subject: [PATCH 172/174] Release 1.0a37 Refs #2831, #2832, #2841, #2842, #2843, #2846 --- datasette/version.py | 2 +- docs/changelog.rst | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 387144e9..8e238ab5 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a36" +__version__ = "1.0a37" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index e3718543..1327baa6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,20 @@ Changelog ========= +.. _v1_0_a37: + +1.0a37 (2026-07-14) +------------------- + +Performance improvement for SQL-backed permission checks, plus an improved permission debugging interface. + +- SQL used to resolve permission checks now aggregates permission rules before joining them to resources, improving performance on instances with large schemas. (:issue:`2832`) +- The :ref:`PermissionCheckView` permission debugger now explains why a decision was allowed or denied, including the matching rules. The interactive form can also test a hypothetical actor supplied as JSON, and the :ref:`permissions documentation ` now describes resolution rules in more detail. (:issue:`2841`) +- :ref:`database_execute_write` has a new ``transaction=`` parameter, which can be set to ``False`` for statements such as ``VACUUM`` that cannot run inside a transaction. Write tasks now start their transactions using ``BEGIN IMMEDIATE``, which also ensures that writes are rolled back if the task fails. (:issue:`2831`) +- Refreshing a database's schema in Datasette's internal catalog is now performed as a single atomic operation. (:issue:`2831`) +- Fixed schema introspection, table pages, facets and table counts for tables with names containing a ``]`` character. Thanks, `TowyTowy `__. (:issue:`2431`, :pr:`2846`) +- ``/-/plugins.json`` once again returns a top-level JSON array of plugin objects, reverting the object envelope introduced in 1.0a36. This should fix a large number of trivial test failures in existing plugins. (:issue:`2842`, :pr:`2843`) + .. _v1_0_a36: 1.0a36 (2026-07-07) From 481df7ff6d78a8ccf919984d27f27201b081bd53 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Jul 2026 09:31:28 -0700 Subject: [PATCH 173/174] Shorten link text in changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1327baa6..670166bb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,7 +13,7 @@ Performance improvement for SQL-backed permission checks, plus an improved permi - SQL used to resolve permission checks now aggregates permission rules before joining them to resources, improving performance on instances with large schemas. (:issue:`2832`) - The :ref:`PermissionCheckView` permission debugger now explains why a decision was allowed or denied, including the matching rules. The interactive form can also test a hypothetical actor supplied as JSON, and the :ref:`permissions documentation ` now describes resolution rules in more detail. (:issue:`2841`) -- :ref:`database_execute_write` has a new ``transaction=`` parameter, which can be set to ``False`` for statements such as ``VACUUM`` that cannot run inside a transaction. Write tasks now start their transactions using ``BEGIN IMMEDIATE``, which also ensures that writes are rolled back if the task fails. (:issue:`2831`) +- :ref:`db.execute_write(sql, ..., transaction=True) ` has a new ``transaction=`` parameter, which can be set to ``False`` for statements such as ``VACUUM`` that cannot run inside a transaction. Write tasks now start their transactions using ``BEGIN IMMEDIATE``, which also ensures that writes are rolled back if the task fails. (:issue:`2831`) - Refreshing a database's schema in Datasette's internal catalog is now performed as a single atomic operation. (:issue:`2831`) - Fixed schema introspection, table pages, facets and table counts for tables with names containing a ``]`` character. Thanks, `TowyTowy `__. (:issue:`2431`, :pr:`2846`) - ``/-/plugins.json`` once again returns a top-level JSON array of plugin objects, reverting the object envelope introduced in 1.0a36. This should fix a large number of trivial test failures in existing plugins. (:issue:`2842`, :pr:`2843`) From e889403d3bbe143854262682161c98a57bdb6594 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 15:47:08 -0700 Subject: [PATCH 174/174] Upgrade to ruff>=0.16.0 (#2857) * ruff>=0.16.0 See https://astral.sh/blog/ruff-v0.16.0 * uv run ruff check . --fix --unsafe-fixes * Ruff fixes by Claude Code Opus 5 --- datasette/_pytest_plugin.py | 3 +- datasette/actor_auth_cookie.py | 8 +- datasette/app.py | 354 +++++++++--------- datasette/blob_renderer.py | 7 +- datasette/cli.py | 99 +++-- datasette/column_types.py | 4 +- datasette/csrf.py | 17 +- datasette/database.py | 68 ++-- datasette/default_actions.py | 2 +- datasette/default_magic_parameters.py | 3 +- datasette/default_permissions/__init__.py | 31 +- datasette/default_permissions/config.py | 38 +- datasette/default_permissions/defaults.py | 25 +- datasette/default_permissions/helpers.py | 20 +- datasette/default_permissions/restrictions.py | 24 +- datasette/default_permissions/root.py | 8 +- datasette/default_permissions/tokens.py | 8 +- datasette/default_table_actions.py | 2 +- datasette/events.py | 3 +- datasette/facets.py | 46 +-- datasette/filters.py | 23 +- datasette/fixtures.py | 9 +- datasette/forbidden.py | 3 +- datasette/handle_exception.py | 17 +- datasette/hookspecs.py | 3 +- datasette/inspect.py | 8 +- datasette/jump.py | 2 +- datasette/permissions.py | 7 +- datasette/plugins.py | 20 +- datasette/publish/cloudrun.py | 28 +- datasette/publish/common.py | 10 +- datasette/publish/heroku.py | 14 +- datasette/renderer.py | 7 +- datasette/stored_queries.py | 17 +- datasette/tokens.py | 31 +- datasette/tracer.py | 19 +- datasette/url_builder.py | 11 +- datasette/utils/__init__.py | 219 ++++++++--- datasette/utils/asgi.py | 45 +-- datasette/utils/baseconv.py | 2 +- datasette/utils/check_callable.py | 6 +- datasette/utils/internal_db.py | 11 +- datasette/utils/multipart.py | 82 ++-- datasette/utils/permissions.py | 31 +- datasette/utils/shutil_backport.py | 2 +- datasette/utils/sql_analysis.py | 8 +- datasette/utils/sqlite.py | 8 +- datasette/utils/testing.py | 5 +- datasette/views/__init__.py | 12 +- datasette/views/base.py | 32 +- datasette/views/database.py | 75 ++-- datasette/views/execute_write.py | 50 +-- datasette/views/index.py | 15 +- datasette/views/query_helpers.py | 37 +- datasette/views/row.py | 71 ++-- datasette/views/special.py | 37 +- datasette/views/stored_queries.py | 18 +- datasette/views/table.py | 204 ++++------ datasette/views/table_create_alter.py | 95 +++-- datasette/views/table_extras.py | 153 +++++--- docs/conf.py | 2 - docs/json_api_doc.py | 12 +- docs/metadata_doc.py | 11 +- docs/template_context_doc.py | 14 +- pyproject.toml | 6 +- ruff.toml | 7 +- tests/conftest.py | 26 +- tests/fixtures.py | 18 +- tests/plugins/my_plugin.py | 24 +- tests/plugins/my_plugin_2.py | 20 +- tests/plugins/register_output_renderer.py | 7 +- tests/plugins/sleep_sql_function.py | 3 +- tests/test_actions_sql.py | 5 +- tests/test_actor_restriction_bug.py | 1 + tests/test_allowed_many.py | 7 +- tests/test_allowed_resources.py | 3 +- tests/test_api.py | 31 +- tests/test_api_write.py | 77 ++-- tests/test_auth.py | 23 +- tests/test_base_view.py | 8 +- tests/test_cli.py | 39 +- tests/test_cli_serve_get.py | 12 +- tests/test_cli_serve_server.py | 3 +- tests/test_column_types.py | 12 +- tests/test_config_dir.py | 4 +- tests/test_crossdb.py | 10 +- tests/test_csrf_middleware.py | 2 +- tests/test_csv.py | 8 +- tests/test_custom_pages.py | 2 + tests/test_default_deny.py | 1 + tests/test_docs.py | 12 +- tests/test_docs_plugins.py | 5 +- tests/test_error_shape.py | 14 +- tests/test_extras.py | 9 +- tests/test_facets.py | 13 +- tests/test_filters.py | 5 +- tests/test_html.py | 46 +-- tests/test_internal_db.py | 32 +- tests/test_internals_database.py | 39 +- tests/test_internals_datasette.py | 22 +- tests/test_internals_datasette_client.py | 3 +- tests/test_internals_request.py | 4 +- tests/test_internals_response.py | 4 +- tests/test_internals_urls.py | 3 +- tests/test_label_column_for_table.py | 3 +- tests/test_load_extensions.py | 6 +- tests/test_messages.py | 3 +- tests/test_multipart.py | 4 +- tests/test_package.py | 8 +- tests/test_permission_endpoints.py | 3 +- tests/test_permissions.py | 29 +- tests/test_plugins.py | 62 +-- tests/test_publish_cloudrun.py | 24 +- tests/test_publish_heroku.py | 8 +- tests/test_pytest_autoclose_plugin.py | 1 + tests/test_queries.py | 48 +-- tests/test_restriction_sql.py | 1 + tests/test_routes.py | 5 +- tests/test_schema_endpoints.py | 1 + tests/test_search_tables.py | 1 + tests/test_spatialite.py | 8 +- tests/test_stored_queries.py | 8 +- tests/test_success_envelope.py | 3 +- tests/test_table_api.py | 11 +- tests/test_table_html.py | 50 +-- tests/test_template_context.py | 41 +- tests/test_token_handler.py | 5 +- tests/test_tracer.py | 8 +- tests/test_utils.py | 35 +- tests/test_utils_check_callable.py | 3 +- tests/test_utils_permissions.py | 11 +- tests/test_utils_sql_analysis.py | 2 +- tests/test_write_wrapper.py | 16 +- 133 files changed, 1656 insertions(+), 1578 deletions(-) diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 103c616d..587380ed 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -89,7 +89,8 @@ def pytest_runtest_protocol(item, nextitem): continue try: ds.close() - except Exception as e: + except Exception as e: # noqa: BLE001 + # Surfaced as a pytest warning; teardown must not fail the run item.warn( pytest.PytestUnraisableExceptionWarning( f"Error closing Datasette instance: {e!r}" diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py index 368213af..7503f1d5 100644 --- a/datasette/actor_auth_cookie.py +++ b/datasette/actor_auth_cookie.py @@ -1,8 +1,10 @@ -from datasette import hookimpl -from itsdangerous import BadSignature -from datasette.utils import baseconv import time +from itsdangerous import BadSignature + +from datasette import hookimpl +from datasette.utils import baseconv + @hookimpl def actor_from_request(datasette, request): diff --git a/datasette/app.py b/datasette/app.py index 0e31273d..c82ea075 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2,7 +2,8 @@ from __future__ import annotations import asyncio import contextvars -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence +from collections.abc import Iterable, Sequence +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from datasette.permissions import Resource @@ -12,11 +13,10 @@ import dataclasses import datetime import functools import glob -import httpx import importlib.metadata import inspect -from itsdangerous import BadSignature import json +import logging import os import re import secrets @@ -28,90 +28,41 @@ import urllib.parse from concurrent import futures from pathlib import Path -from markupsafe import Markup, escape -from itsdangerous import URLSafeSerializer +import httpx +from itsdangerous import BadSignature, URLSafeSerializer from jinja2 import ( ChoiceLoader, Environment, FileSystemLoader, - pass_context, PrefixLoader, + pass_context, ) from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound +from markupsafe import Markup, escape -from .events import Event -from .column_types import SQLiteType from . import stored_queries, write_sql -from .views import Context -from .views.database import ( - database_download, - DatabaseView, - QueryView, -) -from .views.table_create_alter import ( - DatabaseForeignKeyTargetsView, - TableAlterView, - TableCreateView, - TableForeignKeySuggestionsView, -) -from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView -from .views.stored_queries import ( - QueryCreateAnalyzeView, - QueryDeleteView, - QueryDefinitionView, - QueryEditView, - GlobalQueryListView, - QueryListView, - QueryParametersView, - QueryStoreView, - QueryUpdateView, -) -from .views.index import IndexView -from .views.special import ( - JsonDataView, - PatternPortfolioView, - AutocompleteDebugView, - AuthTokenView, - ApiExplorerView, - CreateTokenView, - LogoutView, - AllowDebugView, - PermissionsDebugView, - MessagesDebugView, - AllowedResourcesView, - PermissionRulesView, - PermissionCheckView, - JumpView, - InstanceSchemaView, - DatabaseSchemaView, - TableSchemaView, -) -from .views.table import ( - TableAutocompleteView, - TableInsertView, - TableUpsertView, - TableSetColumnTypeView, - TableDropView, - TableFragmentView, - table_view, -) -from .views.row import RowView, RowDeleteView, RowUpdateView -from .renderer import json_renderer -from .url_builder import Urls +from .column_types import SQLiteType +from .csrf import CrossOriginProtectionMiddleware from .database import Database, QueryInterrupted - +from .events import Event +from .plugins import DEFAULT_PLUGINS, get_plugins, pm +from .renderer import json_renderer +from .resources import DatabaseResource, TableResource +from .tokens import TokenInvalid +from .tracer import AsgiTracer +from .url_builder import Urls from .utils import ( + SPATIALITE_FUNCTIONS, PaginatedResources, PrefixedUrlString, - SPATIALITE_FUNCTIONS, StartupError, + add_cors_headers, async_call_with_supported_arguments, await_me_maybe, baseconv, call_with_supported_arguments, detect_json1, - add_cors_headers, display_actor, escape_css_string, escape_sqlite, @@ -121,47 +72,97 @@ from .utils import ( move_plugins_and_allow, move_table_config, parse_metadata, + redact_keys, resolve_env_secrets, resolve_routes, + row_sql_params_pks, sha256_file, tilde_decode, tilde_encode, to_css_class, urlsafe_components, - redact_keys, - row_sql_params_pks, ) -from .tokens import TokenInvalid from .utils.asgi import ( AsgiLifespan, + AsgiRunOnFirstRequest, BadRequest, + DatabaseNotFound, Forbidden, NotFound, - DatabaseNotFound, - TableNotFound, - RowNotFound, Request, Response, - AsgiRunOnFirstRequest, - asgi_static, + RowNotFound, + TableNotFound, asgi_send, asgi_send_file, asgi_send_redirect, + asgi_static, ) -from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, using_pysqlite3, ) -from .tracer import AsgiTracer -from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ - -from .resources import DatabaseResource, TableResource +from .views import Context +from .views.database import ( + DatabaseView, + QueryView, + database_download, +) +from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView +from .views.index import IndexView +from .views.row import RowDeleteView, RowUpdateView, RowView +from .views.special import ( + AllowDebugView, + AllowedResourcesView, + ApiExplorerView, + AuthTokenView, + AutocompleteDebugView, + CreateTokenView, + DatabaseSchemaView, + InstanceSchemaView, + JsonDataView, + JumpView, + LogoutView, + MessagesDebugView, + PatternPortfolioView, + PermissionCheckView, + PermissionRulesView, + PermissionsDebugView, + TableSchemaView, +) +from .views.stored_queries import ( + GlobalQueryListView, + QueryCreateAnalyzeView, + QueryDefinitionView, + QueryDeleteView, + QueryEditView, + QueryListView, + QueryParametersView, + QueryStoreView, + QueryUpdateView, +) +from .views.table import ( + TableAutocompleteView, + TableDropView, + TableFragmentView, + TableInsertView, + TableSetColumnTypeView, + TableUpsertView, + table_view, +) +from .views.table_create_alter import ( + DatabaseForeignKeyTargetsView, + TableAlterView, + TableCreateView, + TableForeignKeySuggestionsView, +) app_root = Path(__file__).parent.parent +logger = logging.getLogger(__name__) + # Context variable to track when code is executing within a datasette.client request _in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) @@ -184,7 +185,7 @@ class PermissionCheck: """Represents a logged permission check for debugging purposes.""" when: str - actor: Dict[str, Any] | None + actor: dict[str, Any] | None action: str parent: str | None child: str | None @@ -434,7 +435,7 @@ class Datasette: if config_dir: db_files = [] for ext in ("db", "sqlite", "sqlite3"): - db_files.extend(config_dir.glob("*.{}".format(ext))) + db_files.extend(config_dir.glob(f"*.{ext}")) self.files += tuple(str(f) for f in db_files) if ( config_dir @@ -675,10 +676,10 @@ class Datasette: def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env if request: - for environment in pm.hook.jinja2_environment_from_request( + for hook_environment in pm.hook.jinja2_environment_from_request( datasette=self, request=request, env=environment ): - pass + environment = hook_environment return environment def get_action(self, name_or_abbr: str): @@ -732,7 +733,7 @@ class Datasette: catalog_database_names.update( row["database_name"] for row in await internal_db.execute( - "select distinct database_name from {}".format(table) + f"select distinct database_name from {table}" ) if row["database_name"] is not None ) @@ -743,7 +744,7 @@ class Datasette: for stale_db_name in stale_databases: for table in catalog_table_names: conn.execute( - "DELETE FROM {} WHERE database_name = ?".format(table), + f"DELETE FROM {table} WHERE database_name = ?", [stale_db_name], ) @@ -792,17 +793,13 @@ class Datasette: action.name in action_names and action != action_names[action.name] ): - raise StartupError( - "Duplicate action name: {}".format(action.name) - ) + raise StartupError(f"Duplicate action name: {action.name}") if ( action.abbr and action.abbr in action_abbrs and action != action_abbrs[action.abbr] ): - raise StartupError( - "Duplicate action abbr: {}".format(action.abbr) - ) + raise StartupError(f"Duplicate action abbr: {action.abbr}") action_names[action.name] = action if action.abbr: action_abbrs[action.abbr] = action @@ -861,7 +858,7 @@ class Datasette: actor_id: str, *, expires_after: int | None = None, - restrictions: "TokenRestrictions | None" = None, + restrictions: TokenRestrictions | None = None, handler: str | None = None, ) -> str: """ @@ -918,7 +915,7 @@ class Datasette: raise KeyError return matches[0] if name is None: - name = [key for key in self.databases.keys()][0] + name = next(iter(self.databases.keys())) return self.databases[name] def add_database(self, db, name=None, route=None): @@ -931,7 +928,7 @@ class Datasette: suggestion = name i = 2 while name in self.databases: - name = "{}_{}".format(suggestion, i) + name = f"{suggestion}_{i}" i += 1 db.name = name db.route = route or name @@ -966,13 +963,14 @@ class Datasette: for db in dbs: try: db.close() - except Exception as e: + except Exception as e: # noqa: BLE001 + # Collect the first failure and re-raise after every close() has run if first_exception is None: first_exception = e if self.executor is not None: try: self.executor.shutdown(wait=True, cancel_futures=True) - except Exception as e: + except Exception as e: # noqa: BLE001 if first_exception is None: first_exception = e if first_exception is not None: @@ -1321,24 +1319,15 @@ class Datasette: actual = ( actual_sqlite_type.value if actual_sqlite_type is not None - else "unrecognized {!r}".format(column_detail.type) + else f"unrecognized {column_detail.type!r}" ) raise ValueError( - "Column type {!r} is only applicable to SQLite types {} but {}.{}.{} " - "has SQLite type {}".format( - ct_cls.name, - allowed, - database, - resource, - column, - actual, - ) + f"Column type {ct_cls.name!r} is only applicable to SQLite types {allowed} but {database}.{resource}.{column} " + f"has SQLite type {actual}" ) async def _apply_column_types_config(self): """Load column_types from datasette.json config into the internal DB.""" - import logging - for db_name, db_conf in (self.config or {}).get("databases", {}).items(): for table_name, table_conf in db_conf.get("tables", {}).items(): for col_name, ct in table_conf.get("column_types", {}).items(): @@ -1348,7 +1337,7 @@ class Datasette: col_type = ct["type"] config = ct.get("config") if col_type not in self._column_types: - logging.warning( + logger.warning( "column_types config references unknown type %r " "for %s.%s.%s", col_type, @@ -1361,7 +1350,7 @@ class Datasette: db_name, table_name, col_name, col_type, config ) except ValueError as ex: - logging.warning(str(ex)) + logger.warning(str(ex)) async def get_column_type(self, database: str, resource: str, column: str): """ @@ -1414,7 +1403,7 @@ class Datasette: resource: str, column: str, column_type: str, - config: dict = None, + config: dict | None = None, ) -> None: """Assign a column type. Overwrites any existing assignment.""" ct_cls = self._column_types.get(column_type) @@ -1498,9 +1487,7 @@ class Datasette: possible_names = {plugin["name"], plugin["name"].replace("-", "_")} if plugin_name in possible_names: return _resolve_static_asset_path(plugin["static_path"], path) - raise FileNotFoundError( - "No static assets found for plugin {}".format(plugin_name) - ) + raise FileNotFoundError(f"No static assets found for plugin {plugin_name}") def _static_mounted_asset(self, mount_name, path): mount_name = mount_name.strip("/") @@ -1510,7 +1497,7 @@ class Datasette: _resolve_static_asset_path(dirname, path), self.urls.path("/{}/{}".format(mount_name, path.lstrip("/"))), ) - raise FileNotFoundError("No static mount found for {}".format(mount_name)) + raise FileNotFoundError(f"No static mount found for {mount_name}") def _static_asset_hash(self, filepath): filepath = Path(filepath) @@ -1601,18 +1588,17 @@ class Datasette: if await self.allowed(action="view-instance", actor=actor): crumbs.append({"href": self.urls.instance(), "label": "home"}) # Database link - if database: - if await self.allowed( - action="view-database", - resource=DatabaseResource(database=database), - actor=actor, - ): - crumbs.append( - { - "href": self.urls.database(database), - "label": database, - } - ) + if database and await self.allowed( + action="view-database", + resource=DatabaseResource(database=database), + actor=actor, + ): + crumbs.append( + { + "href": self.urls.database(database), + "label": database, + } + ) # Table link if table: assert database, "table= requires database=" @@ -1631,7 +1617,7 @@ class Datasette: async def actors_from_ids( self, actor_ids: Iterable[str | int] - ) -> Dict[int | str, Dict]: + ) -> dict[int | str, dict]: result = pm.hook.actors_from_ids(datasette=self, actor_ids=actor_ids) if result is None: # Do the default thing @@ -1640,9 +1626,9 @@ class Datasette: return result async def track_event(self, event: Event): - assert isinstance(event, self.event_classes), "Invalid event type: {}".format( - type(event) - ) + assert isinstance( + event, self.event_classes + ), f"Invalid event type: {type(event)}" for hook in pm.hook.track_event(datasette=self, event=event): await await_me_maybe(hook) @@ -1679,7 +1665,7 @@ class Datasette: self, actor: dict, action: str, - resource: "Resource" | None = None, + resource: Resource | None = None, ): """ Check if actor can see a resource and if it's private. @@ -1878,10 +1864,7 @@ class Datasette: if truncated and resources: last_resource = resources[-1] # Use tilde-encoding like table pagination - next_token = "{},{}".format( - tilde_encode(str(last_resource.parent)), - tilde_encode(str(last_resource.child)), - ) + next_token = f"{tilde_encode(str(last_resource.parent))},{tilde_encode(str(last_resource.child))}" return PaginatedResources( resources=resources, @@ -1899,7 +1882,7 @@ class Datasette: self, *, action: str, - resource: "Resource" = None, + resource: Resource = None, actor: dict | None = None, ) -> bool: """ @@ -1930,7 +1913,7 @@ class Datasette: self, *, actions: Sequence[str], - resource: "Resource" = None, + resource: Resource = None, actor: dict | None = None, ) -> dict[str, bool]: """ @@ -1951,11 +1934,11 @@ class Datasette: ) # {"edit-schema": True, "drop-table": True, "insert-row": False} """ - from datasette.utils.actions_sql import check_permissions_for_actions from datasette.permissions import ( _permission_check_cache, _skip_permission_checks, ) + from datasette.utils.actions_sql import check_permissions_for_actions # For global actions, resource is None parent = resource.parent if resource else None @@ -2044,7 +2027,7 @@ class Datasette: self, *, action: str, - resource: "Resource" = None, + resource: Resource = None, actor: dict | None = None, ): """ @@ -2098,13 +2081,15 @@ class Datasette: db = self.databases[database] foreign_keys = await db.foreign_keys_for_table(table) # Find the foreign_key for this column - try: - fk = [ + fk = next( + ( foreign_key for foreign_key in foreign_keys if foreign_key["column"] == column - ][0] - except IndexError: + ), + None, + ) + if fk is None: return {} # Ensure user has permission to view the referenced table from datasette.resources import TableResource @@ -2192,16 +2177,17 @@ class Datasette: sqlite_extensions[extension] = result.fetchone()[0] else: sqlite_extensions[extension] = None - except Exception: + except Exception: # noqa: BLE001, S110 + # Probing for optional SQLite extensions - absence is the normal case pass # More details on SpatiaLite if "spatialite" in sqlite_extensions: spatialite_details = {} for fn in SPATIALITE_FUNCTIONS: try: - result = conn.execute("select {}()".format(fn)) + result = conn.execute(f"select {fn}()") spatialite_details[fn] = result.fetchone()[0] - except Exception as e: + except sqlite3.Error as e: spatialite_details[fn] = {"error": str(e)} sqlite_extensions["spatialite"] = spatialite_details @@ -2209,9 +2195,7 @@ class Datasette: fts_versions = [] for fts in ("FTS5", "FTS4", "FTS3"): try: - conn.execute( - "CREATE VIRTUAL TABLE v{fts} USING {fts} (data)".format(fts=fts) - ) + conn.execute(f"CREATE VIRTUAL TABLE v{fts} USING {fts} (data)") fts_versions.append(fts) except sqlite3.OperationalError: continue @@ -2270,7 +2254,7 @@ class Datasette: "static": p["static_path"] is not None, "templates": p["templates_path"] is not None, "version": p.get("version"), - "hooks": list(sorted(set(p["hooks"]))), + "hooks": sorted(set(p["hooks"])), } for p in ps ] @@ -2346,13 +2330,15 @@ class Datasette: async def render_template( self, - templates: List[str] | str | Template, - context: Dict[str, Any] | Context | None = None, + templates: list[str] | str | Template, + context: dict[str, Any] | Context | None = None, request: Request | None = None, view_name: str | None = None, ): if not self._startup_invoked: - raise Exception("render_template() called before await ds.invoke_startup()") + raise RuntimeError( + "render_template() called before await ds.invoke_startup()" + ) context = context or {} if isinstance(templates, Template): template = templates @@ -2398,9 +2384,9 @@ class Datasette: datasette=self, ): extra_vars = await await_me_maybe(extra_vars) - assert isinstance(extra_vars, dict), "extra_vars is of type {}".format( - type(extra_vars) - ) + assert isinstance( + extra_vars, dict + ), f"extra_vars is of type {type(extra_vars)}" extra_template_vars.update(extra_vars) async def menu_links(): @@ -2419,29 +2405,27 @@ class Datasette: # the contract tests fail otherwise template_context = { **context, - **{ - "request": request, - "crumb_items": self._crumb_items, - "urls": self.urls, - "actor": request.actor if request else None, - "menu_links": menu_links, - "display_actor": display_actor, - "show_logout": request is not None - and "ds_actor" in request.cookies - and request.actor, - "zip": zip, - "body_scripts": body_scripts, - "format_bytes": format_bytes, - "show_messages": lambda: self._show_messages(request), - "extra_css_urls": await self._asset_urls( - "extra_css_urls", template, context, request, view_name - ), - "extra_js_urls": await self._asset_urls( - "extra_js_urls", template, context, request, view_name - ), - "base_url": self.setting("base_url"), - "datasette_version": __version__, - }, + "request": request, + "crumb_items": self._crumb_items, + "urls": self.urls, + "actor": request.actor if request else None, + "menu_links": menu_links, + "display_actor": display_actor, + "show_logout": request is not None + and "ds_actor" in request.cookies + and request.actor, + "zip": zip, + "body_scripts": body_scripts, + "format_bytes": format_bytes, + "show_messages": lambda: self._show_messages(request), + "extra_css_urls": await self._asset_urls( + "extra_css_urls", template, context, request, view_name + ), + "extra_js_urls": await self._asset_urls( + "extra_js_urls", template, context, request, view_name + ), + "base_url": self.setting("base_url"), + "datasette_version": __version__, **extra_template_vars, } if request and request.args.get("_context") and self.setting("template_debug"): @@ -2941,7 +2925,8 @@ class DatasetteRouter: custom_response ), "Default forbidden() hook should have been called" return await custom_response.asgi_send(send) - except Exception as exception: + except Exception as exception: # noqa: BLE001 + # This IS the top-level error handler - it must catch everything return await self.handle_exception(request, send, exception) async def handle_401(self, request, send, exception): @@ -2963,7 +2948,7 @@ class DatasetteRouter: request.path.replace("~", "~7E").replace("%", "~").replace(".", "~2E") ) if request.query_string: - new_path += "?{}".format(request.query_string) + new_path += f"?{request.query_string}" await asgi_send_redirect(send, new_path) return # If URL has a trailing slash, redirect to URL without it @@ -3173,8 +3158,7 @@ _curly_re = re.compile(r"({.*?})") def route_pattern_from_filepath(filepath): # Drop the ".html" suffix - if filepath.endswith(".html"): - filepath = filepath[: -len(".html")] + filepath = filepath.removesuffix(".html") re_bits = ["/"] for bit in _curly_re.split(filepath): if _curly_re.match(bit): diff --git a/datasette/blob_renderer.py b/datasette/blob_renderer.py index 4d8c6bea..b6c8b77f 100644 --- a/datasette/blob_renderer.py +++ b/datasette/blob_renderer.py @@ -1,8 +1,9 @@ -from datasette import hookimpl -from datasette.utils.asgi import Response, BadRequest -from datasette.utils import to_css_class import hashlib +from datasette import hookimpl +from datasette.utils import to_css_class +from datasette.utils.asgi import BadRequest, Response + _BLOB_COLUMN = "_blob_column" _BLOB_HASH = "_blob_hash" diff --git a/datasette/cli.py b/datasette/cli.py index 90a33e80..57db83b6 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -1,43 +1,45 @@ import asyncio -import uvicorn -import click -from click import formatting -from click.types import CompositeParamType -from click_default_group import DefaultGroup import functools import json import os import pathlib -from runpy import run_module import shutil -from subprocess import call import sys import textwrap import webbrowser +from runpy import run_module +from subprocess import call + +import click +import uvicorn +from click import formatting +from click.types import CompositeParamType +from click_default_group import DefaultGroup + from .app import ( - Datasette, DEFAULT_SETTINGS, SETTINGS, SQLITE_LIMIT_ATTACHED, + Datasette, pm, ) from .inspect import inspect_tables from .utils import ( + ConnectionProblem, LoadExtension, + SpatialiteConnectionProblem, + SpatialiteNotFound, StartupError, + StaticMount, + ValueAsBooleanError, check_connection, deep_dict_update, find_spatialite, - parse_metadata, - ConnectionProblem, - SpatialiteConnectionProblem, initial_path_for_datasette, pairs_to_nested_config, + parse_metadata, temporary_docker_directory, value_as_boolean, - SpatialiteNotFound, - StaticMount, - ValueAsBooleanError, ) from .utils.sqlite import sqlite3 from .utils.testing import TestClient @@ -75,7 +77,7 @@ class Setting(CompositeParamType): # Datasette 1.0, we turn bare setting names into setting.name # Type checking for those older settings default = DEFAULT_SETTINGS[name] - name = "settings.{}".format(name) + name = f"settings.{name}" if isinstance(default, bool): try: return name, "true" if value_as_boolean(value) else "false" @@ -171,7 +173,6 @@ async def inspect_(files, sqlite_extensions): @cli.group() def publish(): """Publish specified SQLite database files to the internet along with a Datasette-powered interface and API""" - pass # Register publish plugins @@ -578,27 +579,27 @@ def serve( # https://github.com/simonw/datasette/issues/2389 deep_dict_update(config_data, settings_updates) - kwargs = dict( - immutables=immutable, - cache_headers=not reload, - cors=cors, - inspect_data=inspect_data, - config=config_data, - metadata=metadata_data, - sqlite_extensions=sqlite_extensions, - template_dir=template_dir, - plugins_dir=plugins_dir, - static_mounts=static, - settings=None, # These are passed in config= now - memory=memory, - secret=secret, - version_note=version_note, - pdb=pdb, - crossdb=crossdb, - nolock=nolock, - internal=internal, - default_deny=default_deny, - ) + kwargs = { + "immutables": immutable, + "cache_headers": not reload, + "cors": cors, + "inspect_data": inspect_data, + "config": config_data, + "metadata": metadata_data, + "sqlite_extensions": sqlite_extensions, + "template_dir": template_dir, + "plugins_dir": plugins_dir, + "static_mounts": static, + "settings": None, # These are passed in config= now + "memory": memory, + "secret": secret, + "version_note": version_note, + "pdb": pdb, + "crossdb": crossdb, + "nolock": nolock, + "internal": internal, + "default_deny": default_deny, + } # Separate directories from files directories = [f for f in files if os.path.isdir(f)] @@ -621,9 +622,7 @@ def serve( conn.close() else: raise click.ClickException( - "Invalid value for '[FILES]...': Path '{}' does not exist.".format( - file - ) + f"Invalid value for '[FILES]...': Path '{file}' does not exist." ) # Check for duplicate files by resolving all paths to their absolute forms @@ -684,7 +683,7 @@ def serve( client = TestClient(ds) request_headers = {} if token: - request_headers["Authorization"] = "Bearer {}".format(token) + request_headers["Authorization"] = f"Bearer {token}" cookies = {} if actor: cookies["ds_actor"] = client.actor_cookie(json.loads(actor)) @@ -719,9 +718,13 @@ def serve( path = run_sync(lambda: initial_path_for_datasette(ds)) url = f"http://{host}:{port}{path}" webbrowser.open(url) - uvicorn_kwargs = dict( - host=host, port=port, log_level="info", lifespan="on", workers=1 - ) + uvicorn_kwargs = { + "host": host, + "port": port, + "log_level": "info", + "lifespan": "on", + "workers": 1, + } if uds: uvicorn_kwargs["uds"] = uds if ssl_keyfile: @@ -885,7 +888,7 @@ async def check_databases(ds): ) except ConnectionProblem as e: raise click.UsageError( - f"Connection to {database.path} failed check: {str(e.args[0])}" + f"Connection to {database.path} failed check: {e.args[0]!s}" ) # If --crossdb and more than SQLITE_LIMIT_ATTACHED show warning if ( @@ -893,9 +896,5 @@ async def check_databases(ds): and len([db for db in ds.databases.values() if not db.is_memory]) > SQLITE_LIMIT_ATTACHED ): - msg = ( - "Warning: --crossdb only works with the first {} attached databases".format( - SQLITE_LIMIT_ATTACHED - ) - ) + msg = f"Warning: --crossdb only works with the first {SQLITE_LIMIT_ATTACHED} attached databases" click.echo(click.style(msg, bold=True, fg="yellow"), err=True) diff --git a/datasette/column_types.py b/datasette/column_types.py index 11a14ec0..92fdd969 100644 --- a/datasette/column_types.py +++ b/datasette/column_types.py @@ -64,14 +64,14 @@ class ColumnType: Return an HTML string to render this cell value, or None to fall through to the default render_cell plugin hook chain. """ - return None + return async def validate(self, value, datasette): """ Validate a value before it is written. Return None if valid, or a string error message if invalid. """ - return None + return async def transform_value(self, value, datasette): """ diff --git a/datasette/csrf.py b/datasette/csrf.py index df239aee..a62f9473 100644 --- a/datasette/csrf.py +++ b/datasette/csrf.py @@ -40,12 +40,12 @@ def _origin_tuple(value): scheme = (parsed.scheme or "").lower() host = (parsed.hostname or "").lower() if not scheme or not host: - raise ValueError("missing scheme or host in {!r}".format(value)) + raise ValueError(f"missing scheme or host in {value!r}") port = parsed.port # may raise ValueError on bad ports if port is None: port = DEFAULT_PORTS.get(scheme) if port is None: - raise ValueError("unknown default port for scheme {!r}".format(scheme)) + raise ValueError(f"unknown default port for scheme {scheme!r}") return scheme, host, port @@ -125,9 +125,7 @@ class CrossOriginProtectionMiddleware: return await self._forbid( send, - "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format( - sec_fetch_site - ), + f"Sec-Fetch-Site was {sec_fetch_site!r}, expected 'same-origin' or 'none'", ) return @@ -141,11 +139,11 @@ class CrossOriginProtectionMiddleware: request_scheme = self._request_scheme(scope) try: origin_tuple = _origin_tuple(origin) - expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host)) + expected_tuple = _origin_tuple(f"{request_scheme}://{host}") except ValueError: await self._forbid( send, - "Malformed Origin {!r} or Host {!r}".format(origin, host), + f"Malformed Origin {origin!r} or Host {host!r}", ) return @@ -155,7 +153,7 @@ class CrossOriginProtectionMiddleware: await self._forbid( send, - "Origin {!r} does not match Host {!r}".format(origin, host), + f"Origin {origin!r} does not match Host {host!r}", ) def _request_scheme(self, scope): @@ -163,7 +161,8 @@ class CrossOriginProtectionMiddleware: try: if self.datasette.setting("force_https_urls"): return "https" - except Exception: + except Exception: # noqa: BLE001, S110 + # Settings may not be readable this early; fall back to the ASGI scheme pass return scope.get("scheme") or "http" diff --git a/datasette/database.py b/datasette/database.py index bab3a378..e162d34e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,16 +1,18 @@ import asyncio import atexit -from collections import namedtuple import inspect import os -from pathlib import Path import queue -import sqlite_utils import sys import tempfile import threading import uuid +from collections import namedtuple +from pathlib import Path +import sqlite_utils + +from .inspect import inspect_hash from .tracer import trace from .utils import ( call_with_supported_arguments, @@ -21,14 +23,13 @@ from .utils import ( get_all_foreign_keys, get_outbound_foreign_keys, md5_not_usedforsecurity, - sqlite_timelimit, sqlite3, - table_columns, + sqlite_timelimit, table_column_details, + table_columns, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables from .utils.sqlite import sqlite_hidden_table_names -from .inspect import inspect_hash connections = threading.local() @@ -99,9 +100,7 @@ class Database: def _check_not_closed(self): if self._closed: - raise DatasetteClosedError( - "Database {!r} has been closed".format(self.name) - ) + raise DatasetteClosedError(f"Database {self.name!r} has been closed") def _remove_pending_execute_future(self, future): with self._pending_execute_futures_lock: @@ -140,7 +139,7 @@ class Database: if write: extra_kwargs["isolation_level"] = "IMMEDIATE" if self.memory_name: - uri = "file:{}?mode=memory&cache=shared".format(self.memory_name) + uri = f"file:{self.memory_name}?mode=memory&cache=shared" conn = sqlite3.connect( uri, uri=True, check_same_thread=False, **extra_kwargs ) @@ -193,21 +192,20 @@ class Database: write_thread.join(timeout=10) if write_thread.is_alive(): sys.stderr.write( - "Datasette: write thread for {!r} did not exit within 10s\n".format( - self.name - ) + f"Datasette: write thread for {self.name!r} did not exit within 10s\n" ) sys.stderr.flush() for future in pending_execute_futures: try: future.result() - except Exception: + except Exception: # noqa: BLE001, S110 + # Shutdown teardown - a failed pending write must not block close() pass # Close anything still tracked in _all_file_connections for connection in self._all_file_connections: try: connection.close() - except Exception: + except Exception: # noqa: BLE001, S110 pass self._all_file_connections = [] # Drop per-thread cached read connections we can reach @@ -219,13 +217,13 @@ class Database: if self._read_connection is not None: try: self._read_connection.close() - except Exception: + except Exception: # noqa: BLE001, S110 pass self._read_connection = None if self._write_connection is not None: try: self._write_connection.close() - except Exception: + except Exception: # noqa: BLE001, S110 pass self._write_connection = None if self.is_temp_disk: @@ -371,7 +369,8 @@ class Database: async def _dispatch_events_after_write(): try: await reply_future - except Exception: + except Exception: # noqa: BLE001 + # The write failed; skip success events regardless of why # if the write failed, don't emit success events return for event in pending_events: @@ -424,9 +423,7 @@ class Database: self._write_thread = threading.Thread( target=self._execute_writes, daemon=True ) - self._write_thread.name = "_execute_writes for database {}".format( - self.name - ) + self._write_thread.name = f"_execute_writes for database {self.name}" self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") loop = asyncio.get_running_loop() @@ -447,7 +444,8 @@ class Database: try: conn = self.connect(write=True) self.ds._prepare_connection(conn, self.name) - except Exception as e: + except Exception as e: # noqa: BLE001 + # Stored and re-raised to whoever queues the next write conn_exception = e while True: task = self._write_queue.get() @@ -455,7 +453,8 @@ class Database: if conn is not None: try: conn.close() - except Exception: + except Exception: # noqa: BLE001, S110 + # Best-effort close as the write thread exits pass return exception = None @@ -474,8 +473,9 @@ class Database: except ValueError: # Was probably a memory connection pass - except Exception as e: - sys.stderr.write("{}\n".format(e)) + except Exception as e: # noqa: BLE001 + # Write thread must survive any task failure or the database wedges + sys.stderr.write(f"{e}\n") sys.stderr.flush() exception = e else: @@ -486,8 +486,8 @@ class Database: result = task.fn(conn) else: result = task.fn(conn) - except Exception as e: - sys.stderr.write("{}\n".format(e)) + except Exception as e: # noqa: BLE001 + sys.stderr.write(f"{e}\n") sys.stderr.flush() exception = e _deliver_write_result(task, result, exception) @@ -554,9 +554,7 @@ class Database: raise QueryInterrupted(e, sql, params) if log_sql_errors: sys.stderr.write( - "ERROR: conn={}, sql = {}, params = {}: {}\n".format( - conn, repr(sql), params, e - ) + f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" ) sys.stderr.flush() raise @@ -713,9 +711,9 @@ class Database: column_names and len(column_names) == 2 and ("id" in column_names or "pk" in column_names) - and not set(column_names) == {"id", "pk"} + and set(column_names) != {"id", "pk"} ): - return [c for c in column_names if c not in ("id", "pk")][0] + return next(c for c in column_names if c not in ("id", "pk")) # Couldn't find a label: return None @@ -857,10 +855,10 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: __slots__ = ( "fn", - "task_id", + "isolated_connection", "loop", "reply_future", - "isolated_connection", + "task_id", "transaction", ) @@ -901,7 +899,7 @@ class QueryInterrupted(Exception): self.params = params def __str__(self): - return "QueryInterrupted: {}".format(self.e) + return f"QueryInterrupted: {self.e}" class MultipleValues(Exception): diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 602e0df4..ee165ae5 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -2,8 +2,8 @@ from datasette import hookimpl from datasette.permissions import Action from datasette.resources import ( DatabaseResource, - TableResource, QueryResource, + TableResource, ) diff --git a/datasette/default_magic_parameters.py b/datasette/default_magic_parameters.py index 91c1c5aa..bff5f0a7 100644 --- a/datasette/default_magic_parameters.py +++ b/datasette/default_magic_parameters.py @@ -1,8 +1,9 @@ -from datasette import hookimpl import datetime import os import time +from datasette import hookimpl + def header(key, request): key = key.replace("_", "-").encode("utf-8") diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index 6cd46f04..dee5df42 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -17,18 +17,29 @@ UNION/INTERSECT operations. The order of evaluation is: from __future__ import annotations -# Re-export all hooks and public utilities -from .restrictions import ( - actor_restrictions_sql as actor_restrictions_sql, - restrictions_allow_action as restrictions_allow_action, - ActorRestrictions as ActorRestrictions, -) -from .root import root_user_permissions_sql as root_user_permissions_sql from .config import config_permissions_sql as config_permissions_sql +from .defaults import ( + DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, +) +from .defaults import ( + default_action_permissions_sql as default_action_permissions_sql, +) from .defaults import ( # Avoid "datasette.default_permissions" does not explicitly export attribute default_allow_sql_check as default_allow_sql_check, - default_action_permissions_sql as default_action_permissions_sql, - default_query_permissions_sql as default_query_permissions_sql, - DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, ) +from .defaults import ( + default_query_permissions_sql as default_query_permissions_sql, +) +from .restrictions import ( + ActorRestrictions as ActorRestrictions, +) + +# Re-export all hooks and public utilities +from .restrictions import ( + actor_restrictions_sql as actor_restrictions_sql, +) +from .restrictions import ( + restrictions_allow_action as restrictions_allow_action, +) +from .root import root_user_permissions_sql as root_user_permissions_sql diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py index 8edc976e..4494f07f 100644 --- a/datasette/default_permissions/config.py +++ b/datasette/default_permissions/config.py @@ -6,7 +6,7 @@ Applies permission rules from datasette.yaml configuration. from __future__ import annotations -from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from datasette.app import Datasette @@ -55,8 +55,8 @@ class ConfigPermissionProcessor: def __init__( self, - datasette: "Datasette", - actor: Optional[dict], + datasette: Datasette, + actor: dict | None, action: str, ): self.datasette = datasette @@ -74,8 +74,8 @@ class ConfigPermissionProcessor: self.restrictions = actor.get("_r", {}) if actor else {} # Pre-compute restriction info for efficiency - self.restricted_databases: Set[str] = set() - self.restricted_tables: Set[Tuple[str, str]] = set() + self.restricted_databases: set[str] = set() + self.restricted_tables: set[tuple[str, str]] = set() if self.has_restrictions: self.restricted_databases = { @@ -92,7 +92,7 @@ class ConfigPermissionProcessor: # Tables implicitly reference their parent databases self.restricted_databases.update(db for db, _ in self.restricted_tables) - def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]: + def evaluate_allow_block(self, allow_block: Any) -> bool | None: """Evaluate an allow block against the current actor.""" if allow_block is None: return None @@ -104,8 +104,8 @@ class ConfigPermissionProcessor: def is_in_restriction_allowlist( self, - parent: Optional[str], - child: Optional[str], + parent: str | None, + child: str | None, ) -> bool: """Check if resource is allowed by actor restrictions.""" if not self.has_restrictions: @@ -147,9 +147,9 @@ class ConfigPermissionProcessor: def add_permissions_rule( self, - parent: Optional[str], - child: Optional[str], - permissions_block: Optional[dict], + parent: str | None, + child: str | None, + permissions_block: dict | None, scope_desc: str, ) -> None: """Add a rule from a permissions:{action} block.""" @@ -169,8 +169,8 @@ class ConfigPermissionProcessor: def add_allow_block_rule( self, - parent: Optional[str], - child: Optional[str], + parent: str | None, + child: str | None, allow_block: Any, scope_desc: str, ) -> None: @@ -202,8 +202,8 @@ class ConfigPermissionProcessor: def _add_restriction_gate_denies( self, - parent: Optional[str], - child: Optional[str], + parent: str | None, + child: str | None, is_allowed: bool, scope_desc: str, ) -> None: @@ -235,7 +235,7 @@ class ConfigPermissionProcessor: if db_name == parent: self.collector.add(db_name, table_name, False, reason) - def process(self) -> Optional[PermissionSQL]: + def process(self) -> PermissionSQL | None: """Process all config rules and return combined PermissionSQL.""" self._process_root_permissions() self._process_databases() @@ -425,10 +425,10 @@ class ConfigPermissionProcessor: @hookimpl(specname="permission_resources_sql") async def config_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], + datasette: Datasette, + actor: dict | None, action: str, -) -> Optional[List[PermissionSQL]]: +) -> list[PermissionSQL] | None: """ Apply permission rules from datasette.yaml configuration. diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 5bc74425..6f97812b 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -6,7 +6,7 @@ Provides default allow rules for standard view/execute actions. from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING if TYPE_CHECKING: from datasette.app import Datasette @@ -29,29 +29,28 @@ DEFAULT_ALLOW_ACTIONS = frozenset( @hookimpl(specname="permission_resources_sql") async def default_allow_sql_check( - datasette: "Datasette", - actor: Optional[dict], + datasette: Datasette, + actor: dict | None, action: str, -) -> Optional[PermissionSQL]: +) -> PermissionSQL | None: """ Enforce the default_allow_sql setting. When default_allow_sql is false (the default), execute-sql is denied unless explicitly allowed by config or other rules. """ - if action == "execute-sql": - if not datasette.setting("default_allow_sql"): - return PermissionSQL.deny(reason="default_allow_sql is false") + if action == "execute-sql" and not datasette.setting("default_allow_sql"): + return PermissionSQL.deny(reason="default_allow_sql is false") return None @hookimpl(specname="permission_resources_sql") async def default_action_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], + datasette: Datasette, + actor: dict | None, action: str, -) -> Optional[PermissionSQL]: +) -> PermissionSQL | None: """ Provide default allow rules for standard view/execute actions. @@ -71,10 +70,10 @@ async def default_action_permissions_sql( @hookimpl(specname="permission_resources_sql") async def default_query_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], + datasette: Datasette, + actor: dict | None, action: str, -) -> Optional[PermissionSQL]: +) -> PermissionSQL | None: actor_id = actor.get("id") if isinstance(actor, dict) else None if action not in {"view-query", "update-query", "delete-query"}: diff --git a/datasette/default_permissions/helpers.py b/datasette/default_permissions/helpers.py index 47e03569..5e59b7b4 100644 --- a/datasette/default_permissions/helpers.py +++ b/datasette/default_permissions/helpers.py @@ -5,7 +5,7 @@ Shared helper utilities for default permission implementations. from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Set +from typing import TYPE_CHECKING if TYPE_CHECKING: from datasette.app import Datasette @@ -13,7 +13,7 @@ if TYPE_CHECKING: from datasette.permissions import PermissionSQL -def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]: +def get_action_name_variants(datasette: Datasette, action: str) -> set[str]: """ Get all name variants for an action (full name and abbreviation). @@ -27,7 +27,7 @@ def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]: return variants -def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bool: +def action_in_list(datasette: Datasette, action: str, action_list: list) -> bool: """Check if an action (or its abbreviation) is in a list.""" return bool(get_action_name_variants(datasette, action).intersection(action_list)) @@ -36,8 +36,8 @@ def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bo class PermissionRow: """A single permission rule row.""" - parent: Optional[str] - child: Optional[str] + parent: str | None + child: str | None allow: bool reason: str @@ -46,14 +46,14 @@ class PermissionRowCollector: """Collects permission rows and converts them to PermissionSQL.""" def __init__(self, prefix: str = "row"): - self.rows: List[PermissionRow] = [] + self.rows: list[PermissionRow] = [] self.prefix = prefix def add( self, - parent: Optional[str], - child: Optional[str], - allow: Optional[bool], + parent: str | None, + child: str | None, + allow: bool | None, reason: str, if_not_none: bool = False, ) -> None: @@ -62,7 +62,7 @@ class PermissionRowCollector: return self.rows.append(PermissionRow(parent, child, allow, reason)) - def to_permission_sql(self) -> Optional[PermissionSQL]: + def to_permission_sql(self) -> PermissionSQL | None: """Convert collected rows to a PermissionSQL object.""" if not self.rows: return None diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py index a22cd7e5..88e1d274 100644 --- a/datasette/default_permissions/restrictions.py +++ b/datasette/default_permissions/restrictions.py @@ -8,7 +8,7 @@ contains allowlists of resources the actor can access. from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Set, Tuple +from typing import TYPE_CHECKING if TYPE_CHECKING: from datasette.app import Datasette @@ -23,12 +23,12 @@ from .helpers import action_in_list, get_action_name_variants class ActorRestrictions: """Parsed actor restrictions from the _r key.""" - global_actions: List[str] # _r.a - globally allowed actions + global_actions: list[str] # _r.a - globally allowed actions database_actions: dict # _r.d - {db_name: [actions]} table_actions: dict # _r.r - {db_name: {table: [actions]}} @classmethod - def from_actor(cls, actor: Optional[dict]) -> Optional["ActorRestrictions"]: + def from_actor(cls, actor: dict | None) -> ActorRestrictions | None: """Parse restrictions from actor dict. Returns None if no restrictions.""" if not actor: return None @@ -44,11 +44,11 @@ class ActorRestrictions: table_actions=restrictions.get("r", {}), ) - def is_action_globally_allowed(self, datasette: "Datasette", action: str) -> bool: + def is_action_globally_allowed(self, datasette: Datasette, action: str) -> bool: """Check if action is in the global allowlist.""" return action_in_list(datasette, action, self.global_actions) - def get_allowed_databases(self, datasette: "Datasette", action: str) -> Set[str]: + def get_allowed_databases(self, datasette: Datasette, action: str) -> set[str]: """Get database names where this action is allowed.""" allowed = set() for db_name, db_actions in self.database_actions.items(): @@ -57,8 +57,8 @@ class ActorRestrictions: return allowed def get_allowed_tables( - self, datasette: "Datasette", action: str - ) -> Set[Tuple[str, str]]: + self, datasette: Datasette, action: str + ) -> set[tuple[str, str]]: """Get (database, table) pairs where this action is allowed.""" allowed = set() for db_name, tables in self.table_actions.items(): @@ -70,10 +70,10 @@ class ActorRestrictions: @hookimpl(specname="permission_resources_sql") async def actor_restrictions_sql( - datasette: "Datasette", - actor: Optional[dict], + datasette: Datasette, + actor: dict | None, action: str, -) -> Optional[List[PermissionSQL]]: +) -> list[PermissionSQL] | None: """ Handle actor restriction-based permission rules. @@ -140,10 +140,10 @@ async def actor_restrictions_sql( def restrictions_allow_action( - datasette: "Datasette", + datasette: Datasette, restrictions: dict, action: str, - resource: Optional[str | Tuple[str, str]], + resource: str | tuple[str, str] | None, ) -> bool: """ Check if restrictions allow the requested action on the requested resource. diff --git a/datasette/default_permissions/root.py b/datasette/default_permissions/root.py index 4931f7ff..22d13f65 100644 --- a/datasette/default_permissions/root.py +++ b/datasette/default_permissions/root.py @@ -6,7 +6,7 @@ Grants full permissions to the root user when --root flag is used. from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING if TYPE_CHECKING: from datasette.app import Datasette @@ -17,9 +17,9 @@ from datasette.permissions import PermissionSQL @hookimpl(specname="permission_resources_sql") async def root_user_permissions_sql( - datasette: "Datasette", - actor: Optional[dict], -) -> Optional[PermissionSQL]: + datasette: Datasette, + actor: dict | None, +) -> PermissionSQL | None: """ Grant root user full permissions when --root flag is used. """ diff --git a/datasette/default_permissions/tokens.py b/datasette/default_permissions/tokens.py index 7a359dc6..52daf8a2 100644 --- a/datasette/default_permissions/tokens.py +++ b/datasette/default_permissions/tokens.py @@ -7,7 +7,7 @@ to datasette.verify_token() so all registered handlers are tried. from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING if TYPE_CHECKING: from datasette.app import Datasette @@ -17,15 +17,13 @@ from datasette.tokens import SignedTokenHandler @hookimpl -def register_token_handler(datasette: "Datasette"): +def register_token_handler(datasette: Datasette): """Register the default signed token handler.""" return SignedTokenHandler() @hookimpl(specname="actor_from_request") -async def actor_from_signed_api_token( - datasette: "Datasette", request -) -> Optional[dict]: +async def actor_from_signed_api_token(datasette: Datasette, request) -> dict | None: """ Authenticate requests using API tokens by delegating to all registered token handlers via datasette.verify_token(). diff --git a/datasette/default_table_actions.py b/datasette/default_table_actions.py index e41434ef..0f2f32ef 100644 --- a/datasette/default_table_actions.py +++ b/datasette/default_table_actions.py @@ -20,7 +20,7 @@ def table_actions(datasette, actor, database, table, request): "label": "Alter table", "description": "Change columns and primary key for this table.", "attrs": { - "aria-label": "Alter table {}".format(table), + "aria-label": f"Alter table {table}", "data-table-action": "alter-table", }, } diff --git a/datasette/events.py b/datasette/events.py index e8786da9..5f3fd06e 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -1,8 +1,9 @@ from abc import ABC, abstractproperty from dataclasses import asdict, dataclass, field -from datasette.hookspecs import hookimpl from datetime import datetime, timezone +from datasette.hookspecs import hookimpl + @dataclass class Event(ABC): diff --git a/datasette/facets.py b/datasette/facets.py index 5f757df3..8c09e1dc 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -1,12 +1,13 @@ import json import urllib + from datasette import hookimpl from datasette.database import QueryInterrupted from datasette.utils import ( + detect_json1, escape_sqlite, path_with_added_args, path_with_removed_args, - detect_json1, sqlite3, ) @@ -30,7 +31,7 @@ def load_facet_configs(request, table_config): assert ( len(facet_config.values()) == 1 ), "Metadata config dicts should be {type: config}" - type, facet_config = list(facet_config.items())[0] + type, facet_config = next(iter(facet_config.items())) if isinstance(facet_config, str): facet_config = {"simple": facet_config} facet_configs.setdefault(type, []).append( @@ -160,18 +161,13 @@ class ColumnFacet(Facet): for column in columns: if column in already_enabled: continue - suggested_facet_sql = """ - with limited as (select * from ({sql}) limit {suggest_consider}) - select {column} as value, count(*) as n from limited + suggested_facet_sql = f""" + with limited as (select * from ({self.sql}) limit {self.suggest_consider}) + select {escape_sqlite(column)} as value, count(*) as n from limited where value is not null group by value - limit {limit} - """.format( - column=escape_sqlite(column), - sql=self.sql, - limit=facet_size + 1, - suggest_consider=self.suggest_consider, - ) + limit {facet_size + 1} + """ distinct_values = None try: distinct_values = await self.ds.execute( @@ -267,7 +263,7 @@ class ColumnFacet(Facet): for row in facet_rows: column_qs = column if column.startswith("_"): - column_qs = "{}__exact".format(column) + column_qs = f"{column}__exact" selected = (column_qs, str(row["value"])) in qs_pairs if selected: toggle_path = path_with_removed_args( @@ -342,12 +338,12 @@ class ArrayFacet(Facet): for v in await self.ds.execute( self.database, ( - "select {column} from ({sql}) " - "where {column} is not null " - "and {column} != '' " - "and json_array_length({column}) > 0 " + f"select {escape_sqlite(column)} from ({self.sql}) " + f"where {escape_sqlite(column)} is not null " + f"and {escape_sqlite(column)} != '' " + f"and json_array_length({escape_sqlite(column)}) > 0 " "limit 100" - ).format(column=escape_sqlite(column), sql=self.sql), + ), self.params, truncate=False, custom_time_limit=self.ds.setting( @@ -388,14 +384,14 @@ class ArrayFacet(Facet): source = source_and_config["source"] column = config.get("column") or config["simple"] # https://github.com/simonw/datasette/issues/448 - facet_sql = """ - with inner as ({sql}), + facet_sql = f""" + with inner as ({self.sql}), deduped_array_items as ( select distinct j.value, inner.* from - json_each([inner].{col}) j + json_each([inner].{escape_sqlite(column)}) j join inner ) select @@ -406,12 +402,8 @@ class ArrayFacet(Facet): group by value order by - count(*) desc, value limit {limit} - """.format( - col=escape_sqlite(column), - sql=self.sql, - limit=facet_size + 1, - ) + count(*) desc, value limit {facet_size + 1} + """ try: facet_rows_results = await self.ds.execute( self.database, diff --git a/datasette/filters.py b/datasette/filters.py index 95cc5f37..1d4e32c2 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,8 +1,11 @@ +import json +from typing import ClassVar + from datasette import hookimpl from datasette.resources import DatabaseResource -from datasette.views.base import DatasetteError from datasette.utils.asgi import BadRequest -import json +from datasette.views.base import DatasetteError + from .utils import detect_json1, escape_sqlite, path_with_removed_args @@ -99,9 +102,9 @@ def search_filters(request, database, table, datasette): fts_table=escape_sqlite(fts_table), search_col=escape_sqlite(search_col), match_clause=( - ":search_{}".format(i) + f":search_{i}" if search_mode_raw - else "escape_fts(:search_{})".format(i) + else f"escape_fts(:search_{i})" ), ) ) @@ -134,11 +137,11 @@ def through_filters(request, database, table, datasette): value = through_data["value"] db = datasette.get_database(database) outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) - try: - fk_to_us = [ - fk for fk in outgoing_foreign_keys if fk["other_table"] == table - ][0] - except IndexError: + fk_to_us = next( + (fk for fk in outgoing_foreign_keys if fk["other_table"] == table), + None, + ) + if fk_to_us is None: raise DatasetteError( "Invalid _through - could not find corresponding foreign key" ) @@ -365,7 +368,7 @@ class Filters: ), ] ) - _filters_by_key = {f.key: f for f in _filters} + _filters_by_key: ClassVar[dict[str, Filter]] = {f.key: f for f in _filters} def __init__(self, pairs): self.pairs = pairs diff --git a/datasette/fixtures.py b/datasette/fixtures.py index 7c85e16a..049e35ed 100644 --- a/datasette/fixtures.py +++ b/datasette/fixtures.py @@ -1,9 +1,10 @@ -from datasette.utils.sqlite import sqlite3 -from datasette.utils import documented import itertools import random import string +from datasette.utils import documented +from datasette.utils.sqlite import sqlite3 + __all__ = [ "EXTRA_DATABASE_SQL", "TABLES", @@ -346,9 +347,7 @@ CREATE VIEW searchable_view_configured_by_metadata AS + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' + "\n".join( [ - 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( - a=a, b=b, c=c, content=content - ) + f'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");' for a, b, c, content in generate_compound_rows(1001) ] ) diff --git a/datasette/forbidden.py b/datasette/forbidden.py index 91b1ff96..67bf0d8b 100644 --- a/datasette/forbidden.py +++ b/datasette/forbidden.py @@ -1,4 +1,5 @@ -from datasette import hookimpl, Response +from datasette import Response, hookimpl + from .utils import add_cors_headers diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index e255ddf2..c36d5dbe 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -1,16 +1,21 @@ -from datasette import hookimpl, Response +import traceback + +from markupsafe import Markup + +from datasette import Response, hookimpl + from .utils import add_cors_headers, error_body from .utils.asgi import ( Base400, ) from .views.base import DatasetteError -from markupsafe import Markup -import traceback +# Debugger imports are deliberate - they back the "pdb" setting, which drops +# into a debugger on unhandled exceptions try: - import ipdb as pdb + import ipdb as pdb # noqa: T100 except ImportError: - import pdb + import pdb # noqa: T100 try: import rich @@ -69,7 +74,7 @@ def handle_exception(datasette, request, exception): dict( info, urls=datasette.urls, - menu_links=lambda: [], + menu_links=list, ) ), status=status, diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 7c56f882..f89f2f36 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -1,5 +1,4 @@ -from pluggy import HookimplMarker -from pluggy import HookspecMarker +from pluggy import HookimplMarker, HookspecMarker hookspec = HookspecMarker("datasette") hookimpl = HookimplMarker("datasette") diff --git a/datasette/inspect.py b/datasette/inspect.py index 5e681e03..b126ce5c 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -1,13 +1,13 @@ import hashlib from .utils import ( - detect_spatialite, detect_fts, detect_primary_keys, + detect_spatialite, escape_sqlite, get_all_foreign_keys, - table_columns, sqlite3, + table_columns, ) HASH_BLOCK_SIZE = 1024 * 1024 @@ -95,10 +95,10 @@ def inspect_tables(conn, database_metadata): """) ] - for t in tables.keys(): + for t, table_info in tables.items(): for hidden_table in hidden_tables: if t == hidden_table or t.startswith(hidden_table): - tables[t]["hidden"] = True + table_info["hidden"] = True continue return tables diff --git a/datasette/jump.py b/datasette/jump.py index d138e827..d70d33df 100644 --- a/datasette/jump.py +++ b/datasette/jump.py @@ -21,7 +21,7 @@ class JumpSQL: search_text: str | None = None, display_name: str | None = None, item_type: str = "menu", - ) -> "JumpSQL": + ) -> JumpSQL: if search_text is None: search_text = " ".join( text for text in (label, display_name, description) if text is not None diff --git a/datasette/permissions.py b/datasette/permissions.py index 786dc026..e03b065c 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -1,7 +1,7 @@ +import contextvars from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, NamedTuple -import contextvars # Context variable to track when permission checks should be skipped _skip_permission_checks = contextvars.ContextVar( @@ -72,8 +72,8 @@ class Resource(ABC): ) def __repr__(self) -> str: - return "{}(parent={!r}, child={!r})".format( - self.__class__.__name__, self.parent, self.child + return ( + f"{self.__class__.__name__}(parent={self.parent!r}, child={self.child!r})" ) @property @@ -129,7 +129,6 @@ class Resource(ABC): Must return two columns: parent, child """ - pass class AllowedResource(NamedTuple): diff --git a/datasette/plugins.py b/datasette/plugins.py index ae2cb17d..9cf94079 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,20 +1,14 @@ import importlib +import importlib.metadata as importlib_metadata +import importlib.resources as importlib_resources import os -import pluggy -from pprint import pprint import sys +from pprint import pprint + +import pluggy + from . import hookspecs -if sys.version_info >= (3, 9): - import importlib.resources as importlib_resources -else: - import importlib_resources -if sys.version_info >= (3, 10): - import importlib.metadata as importlib_metadata -else: - import importlib_metadata - - DEFAULT_PLUGINS = ( "datasette.publish.heroku", "datasette.publish.cloudrun", @@ -85,7 +79,7 @@ if DATASETTE_LOAD_PLUGINS is not None: # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) except importlib_metadata.PackageNotFoundError: - sys.stderr.write("Plugin {} could not be found\n".format(package_name)) + sys.stderr.write(f"Plugin {package_name} could not be found\n") # Load default plugins diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 63d22fe8..9ace865b 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -1,15 +1,17 @@ -from datasette import hookimpl -import click import json import os import re from subprocess import CalledProcessError, check_call, check_output +import click + +from datasette import hookimpl + +from ..utils import temporary_docker_directory from .common import ( add_common_publish_arguments_and_options, fail_if_publish_binary_not_installed, ) -from ..utils import temporary_docker_directory @hookimpl @@ -219,7 +221,7 @@ def publish_subcommand(publish): check_call( "gcloud builds submit --tag {}{}".format( - image_id, " --timeout {}".format(timeout) if timeout else "" + image_id, f" --timeout {timeout}" if timeout else "" ), shell=True, ) @@ -231,7 +233,7 @@ def publish_subcommand(publish): ("--min-instances", min_instances), ): if value is not None: - extra_deploy_options.append("{} {}".format(option, value)) + extra_deploy_options.append(f"{option} {value}") check_call( "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format( image_id, @@ -258,24 +260,16 @@ def _ensure_artifact_registry(artifact_project, artifact_region, artifact_reposi ) from exc describe_cmd = ( - "gcloud artifacts repositories describe {repo} --project {project} " - "--location {location} --quiet" - ).format( - repo=artifact_repository, - project=artifact_project, - location=artifact_region, + f"gcloud artifacts repositories describe {artifact_repository} --project {artifact_project} " + f"--location {artifact_region} --quiet" ) try: check_call(describe_cmd, shell=True) return except CalledProcessError: create_cmd = ( - "gcloud artifacts repositories create {repo} --repository-format=docker " - '--location {location} --project {project} --description "Datasette Cloud Run images" --quiet' - ).format( - repo=artifact_repository, - location=artifact_region, - project=artifact_project, + f"gcloud artifacts repositories create {artifact_repository} --repository-format=docker " + f'--location {artifact_region} --project {artifact_project} --description "Datasette Cloud Run images" --quiet' ) try: check_call(create_cmd, shell=True) diff --git a/datasette/publish/common.py b/datasette/publish/common.py index 29665eb3..27dfd4bf 100644 --- a/datasette/publish/common.py +++ b/datasette/publish/common.py @@ -1,9 +1,11 @@ -from ..utils import StaticMount -import click import os import shutil import sys +import click + +from ..utils import StaticMount + def add_common_publish_arguments_and_options(subcommand): for decorator in reversed( @@ -76,9 +78,7 @@ def fail_if_publish_binary_not_installed(binary, publish_target, install_link): """Exit (with error message) if ``binary` isn't installed""" if not shutil.which(binary): click.secho( - "Publishing to {publish_target} requires {binary} to be installed and configured".format( - publish_target=publish_target, binary=binary - ), + f"Publishing to {publish_target} requires {binary} to be installed and configured", bg="red", fg="white", bold=True, diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index f576a346..b0290833 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -1,19 +1,21 @@ -from contextlib import contextmanager -from datasette import hookimpl -import click import json import os import pathlib import shlex import shutil -from subprocess import call, check_output import tempfile +from contextlib import contextmanager +from subprocess import call, check_output + +import click + +from datasette import hookimpl +from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata from .common import ( add_common_publish_arguments_and_options, fail_if_publish_binary_not_installed, ) -from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata @hookimpl @@ -234,7 +236,7 @@ def temporary_heroku_directory( extras.extend(["--static", f"{mount_point}:{mount_point}"]) quoted_files = " ".join( - ["-i {}".format(shlex.quote(file_name)) for file_name in file_names] + [f"-i {shlex.quote(file_name)}" for file_name in file_names] ) procfile_cmd = "web: datasette serve --host 0.0.0.0 {quoted_files} --cors --port $PORT --inspect-file inspect-data.json {extras}".format( quoted_files=quoted_files, extras=" ".join(extras) diff --git a/datasette/renderer.py b/datasette/renderer.py index 7c94f6ee..0e01f52f 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -1,12 +1,13 @@ import json + from datasette.extras import extra_names_from_request from datasette.utils import ( - error_body, - value_as_boolean, - remove_infinites, CustomJSONEncoder, + error_body, path_from_row_pks, + remove_infinites, sqlite3, + value_as_boolean, ) from datasette.utils.asgi import Response diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index f5f977d9..db3c6548 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -1,8 +1,9 @@ from __future__ import annotations -from dataclasses import dataclass import json -from typing import Any, Iterable +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any from .utils import tilde_encode, urlsafe_components @@ -386,7 +387,7 @@ async def count_queries( OR q.sql LIKE :query_search ) """) - params["query_search"] = "%{}%".format(q) + params["query_search"] = f"%{q}%" if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) @@ -462,7 +463,7 @@ async def list_queries( except ValueError: components = [] if database is None and len(components) == 3: - where_clauses.append(""" + where_clauses.append(f""" ( q.database_name > :cursor_database OR ( @@ -476,12 +477,12 @@ async def list_queries( ) ) ) - """.format(sort_key_sql=sort_key_sql)) + """) params["cursor_database"] = components[0] params["cursor_sort_key"] = components[1] params["cursor_name"] = components[2] elif database is not None and len(components) == 2: - where_clauses.append(""" + where_clauses.append(f""" ( {sort_key_sql} > :cursor_sort_key OR ( @@ -489,7 +490,7 @@ async def list_queries( AND q.name > :cursor_name ) ) - """.format(sort_key_sql=sort_key_sql)) + """) params["cursor_sort_key"] = components[0] params["cursor_name"] = components[1] @@ -502,7 +503,7 @@ async def list_queries( OR q.sql LIKE :query_search ) """) - params["query_search"] = "%{}%".format(q) + params["query_search"] = f"%{q}%" if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) diff --git a/datasette/tokens.py b/datasette/tokens.py index 4f905339..79f840d2 100644 --- a/datasette/tokens.py +++ b/datasette/tokens.py @@ -10,7 +10,7 @@ from __future__ import annotations import dataclasses import time -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING import itsdangerous @@ -50,24 +50,24 @@ class TokenRestrictions: database: dict[str, list[str]] = dataclasses.field(default_factory=dict) resource: dict[str, dict[str, list[str]]] = dataclasses.field(default_factory=dict) - def allow_all(self, action: str) -> "TokenRestrictions": + def allow_all(self, action: str) -> TokenRestrictions: """Allow an action across all databases and resources.""" self.all.append(action) return self - def allow_database(self, database: str, action: str) -> "TokenRestrictions": + def allow_database(self, database: str, action: str) -> TokenRestrictions: """Allow an action on a specific database.""" self.database.setdefault(database, []).append(action) return self def allow_resource( self, database: str, resource: str, action: str - ) -> "TokenRestrictions": + ) -> TokenRestrictions: """Allow an action on a specific resource within a database.""" self.resource.setdefault(database, {}).setdefault(resource, []).append(action) return self - def abbreviated(self, datasette: "Datasette") -> Optional[dict]: + def abbreviated(self, datasette: Datasette) -> dict | None: """ Return the abbreviated ``_r`` dictionary shape for this set of restrictions, using action abbreviations registered with ``datasette``. @@ -112,16 +112,16 @@ class TokenHandler: async def create_token( self, - datasette: "Datasette", + datasette: Datasette, actor_id: str, *, - expires_after: Optional[int] = None, - restrictions: Optional[TokenRestrictions] = None, + expires_after: int | None = None, + restrictions: TokenRestrictions | None = None, ) -> str: """Create and return a token string for the given actor.""" raise NotImplementedError - async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: + async def verify_token(self, datasette: Datasette, token: str) -> dict | None: """ Verify a token and return an actor dict. @@ -142,11 +142,11 @@ class SignedTokenHandler(TokenHandler): async def create_token( self, - datasette: "Datasette", + datasette: Datasette, actor_id: str, *, - expires_after: Optional[int] = None, - restrictions: Optional[TokenRestrictions] = None, + expires_after: int | None = None, + restrictions: TokenRestrictions | None = None, ) -> str: if not datasette.setting("allow_signed_tokens"): raise ValueError( @@ -163,7 +163,7 @@ class SignedTokenHandler(TokenHandler): token["_r"] = abbreviated return "dstok_{}".format(datasette.sign(token, namespace="token")) - async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: + async def verify_token(self, datasette: Datasette, token: str) -> dict | None: prefix = "dstok_" if not token.startswith(prefix): @@ -200,9 +200,8 @@ class SignedTokenHandler(TokenHandler): ): duration = max_signed_tokens_ttl - if duration: - if time.time() - created > duration: - raise TokenInvalid("Token has expired") + if duration and time.time() - created > duration: + raise TokenInvalid("Token has expired") actor = {"id": decoded["a"], "token": "dstok"} diff --git a/datasette/tracer.py b/datasette/tracer.py index 28f3cc09..1fbda6f9 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -1,10 +1,11 @@ import asyncio +import json +import time +import traceback from contextlib import contextmanager from contextvars import ContextVar + from markupsafe import escape -import time -import json -import traceback tracers = {} @@ -132,17 +133,17 @@ class AsgiTracer: "num_traces": len(traces), "traces": traces, } - try: - content_type = [ + content_type = next( + ( v.decode("utf8") for k, v in response_headers if k.lower() == b"content-type" - ][0] - except IndexError: - content_type = "" + ), + "", + ) if "text/html" in content_type and b"" in accumulated_body: extra = escape(json.dumps(trace_info, indent=2)) - extra_html = f"
{extra}
".encode("utf8") + extra_html = f"
{extra}
".encode() accumulated_body = accumulated_body.replace(b"", extra_html) elif "json" in content_type and accumulated_body.startswith(b"{"): data = json.loads(accumulated_body.decode("utf8")) diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 16b3d42b..f8da20f3 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -1,6 +1,7 @@ -from .utils import tilde_encode, path_with_format, PrefixedUrlString import urllib +from .utils import PrefixedUrlString, path_with_format, tilde_encode + class Urls: def __init__(self, ds): @@ -8,8 +9,7 @@ class Urls: def path(self, path, format=None): if not isinstance(path, PrefixedUrlString): - if path.startswith("/"): - path = path[1:] + path = path.removeprefix("/") path = self.ds.setting("base_url") + path if format is not None: path = path_with_format(path=path, format=format) @@ -56,6 +56,7 @@ class Urls: return PrefixedUrlString(path) def row_blob(self, database, table, row_path, column): - return self.table(database, table) + "/{}.blob?_blob_column={}".format( - row_path, urllib.parse.quote_plus(column) + return ( + self.table(database, table) + + f"/{row_path}.blob?_blob_column={urllib.parse.quote_plus(column)}" ) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 18d3ba52..eb46d549 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1,29 +1,31 @@ import asyncio +import base64 import binascii -from contextlib import contextmanager -import aiofiles -import click -from collections import OrderedDict, namedtuple, Counter import copy import dataclasses -import base64 import hashlib import inspect import json -import markupsafe -import mergedeep import os import re +import secrets import shlex +import shutil import tempfile -import typing import time import types -import secrets -import shutil -from typing import Iterable, List, Tuple +import typing import urllib +from collections import Counter, OrderedDict, namedtuple +from collections.abc import Iterable +from contextlib import contextmanager + +import aiofiles +import click +import markupsafe +import mergedeep import yaml + from .shutil_backport import copytree from .sqlite import sqlite3, supports_table_xinfo @@ -36,7 +38,7 @@ if typing.TYPE_CHECKING: class PaginatedResources: """Paginated results from allowed_resources query.""" - resources: List["Resource"] + resources: list["Resource"] next: str | None # Keyset token for next page (None if no more results) _datasette: typing.Any = dataclasses.field(default=None, repr=False) _action: str = dataclasses.field(default=None, repr=False) @@ -83,22 +85,132 @@ class PaginatedResources: # From https://www.sqlite.org/lang_keywords.html -reserved_words = set( - ( - "abort action add after all alter analyze and as asc attach autoincrement " - "before begin between by cascade case cast check collate column commit " - "conflict constraint create cross current_date current_time " - "current_timestamp database default deferrable deferred delete desc detach " - "distinct drop each else end escape except exclusive exists explain fail " - "for foreign from full glob group having if ignore immediate in index " - "indexed initially inner insert instead intersect into is isnull join key " - "left like limit match natural no not notnull null of offset on or order " - "outer plan pragma primary query raise recursive references regexp reindex " - "release rename replace restrict right rollback row savepoint select set " - "table temp temporary then to transaction trigger union unique update using " - "vacuum values view virtual when where with without" - ).split() -) +reserved_words = { + "abort", + "action", + "add", + "after", + "all", + "alter", + "analyze", + "and", + "as", + "asc", + "attach", + "autoincrement", + "before", + "begin", + "between", + "by", + "cascade", + "case", + "cast", + "check", + "collate", + "column", + "commit", + "conflict", + "constraint", + "create", + "cross", + "current_date", + "current_time", + "current_timestamp", + "database", + "default", + "deferrable", + "deferred", + "delete", + "desc", + "detach", + "distinct", + "drop", + "each", + "else", + "end", + "escape", + "except", + "exclusive", + "exists", + "explain", + "fail", + "for", + "foreign", + "from", + "full", + "glob", + "group", + "having", + "if", + "ignore", + "immediate", + "in", + "index", + "indexed", + "initially", + "inner", + "insert", + "instead", + "intersect", + "into", + "is", + "isnull", + "join", + "key", + "left", + "like", + "limit", + "match", + "natural", + "no", + "not", + "notnull", + "null", + "of", + "offset", + "on", + "or", + "order", + "outer", + "plan", + "pragma", + "primary", + "query", + "raise", + "recursive", + "references", + "regexp", + "reindex", + "release", + "rename", + "replace", + "restrict", + "right", + "rollback", + "row", + "savepoint", + "select", + "set", + "table", + "temp", + "temporary", + "then", + "to", + "transaction", + "trigger", + "union", + "unique", + "update", + "using", + "vacuum", + "values", + "view", + "virtual", + "when", + "where", + "with", + "without", +} APT_GET_DOCKERFILE_EXTRAS = r""" RUN apt-get update && \ @@ -158,7 +270,7 @@ functions_marked_as_documented = [] def documented(fn=None, *, label=None): def decorate(fn): - fn._datasette_docs_label = label or "internals_utils_{}".format(fn.__name__) + fn._datasette_docs_label = label or f"internals_utils_{fn.__name__}" functions_marked_as_documented.append(fn) return fn @@ -360,7 +472,7 @@ disallawed_sql_res = [ ( re.compile(f"pragma(?!_({'|'.join(allowed_pragmas)}))"), "Statement contained a disallowed PRAGMA. Allowed pragma functions are {}".format( - ", ".join("pragma_{}()".format(pragma) for pragma in allowed_pragmas) + ", ".join(f"pragma_{pragma}()" for pragma in allowed_pragmas) ), ) ] @@ -534,10 +646,7 @@ CMD {cmd}""".format( else "" ), environment_variables="\n".join( - [ - "ENV {} '{}'".format(key, value) - for key, value in environment_variables.items() - ] + [f"ENV {key} '{value}'" for key, value in environment_variables.items()] ), install_from=" ".join(install), files=" ".join(files), @@ -640,7 +749,7 @@ def get_outbound_foreign_keys(conn, table): fks = [] for info in infos: if info is not None: - id, seq, table_name, from_, to_, on_update, on_delete, match = info + id, seq, table_name, from_, to_, _on_update, _on_delete, _match = info fks.append( { "column": from_, @@ -741,7 +850,7 @@ def detect_json1(conn=None): try: conn.execute("SELECT json('{}')") return True - except Exception: + except sqlite3.Error: return False finally: if close_conn: @@ -821,9 +930,7 @@ def is_url(value): if not value.startswith("http://") and not value.startswith("https://"): return False # Any whitespace at all is invalid - if whitespace_re.search(value): - return False - return True + return not whitespace_re.search(value) css_class_re = re.compile(r"^[a-zA-Z]+[_a-zA-Z0-9-]*$") @@ -876,7 +983,9 @@ def module_from_path(path, name): mod.__file__ = path with open(path, "r") as file: code = compile(file.read(), path, "exec", dont_inherit=True) - exec(code, mod.__dict__) + # Executing the file is the whole point - this is how --plugins-dir loads + # plugins and how metadata/config .py files are evaluated + exec(code, mod.__dict__) # noqa: S102 return mod @@ -1033,9 +1142,7 @@ def escape_fts(query): query += '"' bits = _escape_fts_re.split(query) bits = [b for b in bits if b and b != '""'] - return " ".join( - '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits - ) + return " ".join(f'"{bit}"' if not bit.startswith('"') else bit for bit in bits) class MultiParams: @@ -1047,7 +1154,7 @@ class MultiParams: data[key], (list, tuple) ), "dictionary data should be a dictionary of key => [list]" self._data = data - elif isinstance(data, list) or isinstance(data, tuple): + elif isinstance(data, (list, tuple)): new_data = {} for item in data: assert ( @@ -1137,9 +1244,7 @@ def _gather_arguments(fn, kwargs): for parameter in parameters: if parameter not in kwargs: raise TypeError( - "{} requires parameters {}, missing: {}".format( - fn, tuple(parameters), set(parameters) - set(kwargs.keys()) - ) + f"{fn} requires parameters {tuple(parameters)}, missing: {set(parameters) - set(kwargs.keys())}" ) call_with.append(kwargs[parameter]) return call_with @@ -1208,9 +1313,9 @@ def resolve_env_secrets(config, environ): """Create copy that recursively replaces {"$env": "NAME"} with values from environ""" if isinstance(config, dict): if list(config.keys()) == ["$env"]: - return environ.get(list(config.values())[0]) + return environ.get(next(iter(config.values()))) elif list(config.keys()) == ["$file"]: - with open(list(config.values())[0]) as fp: + with open(next(iter(config.values()))) as fp: return fp.read() else: return { @@ -1306,7 +1411,7 @@ _named_param_re = re.compile(r":(\w+)") @documented -def named_parameters(sql: str) -> List[str]: +def named_parameters(sql: str) -> list[str]: """ Given a SQL statement, return a list of named parameters that are used in the statement @@ -1319,7 +1424,7 @@ def named_parameters(sql: str) -> List[str]: return _named_param_re.findall(sql) -async def derive_named_parameters(db: "Database", sql: str) -> List[str]: +async def derive_named_parameters(db: "Database", sql: str) -> list[str]: """ This undocumented but stable method exists for backwards compatibility with plugins that were using it before it switched to named_parameters() @@ -1343,9 +1448,9 @@ def parse_size_limit(value, default, maximum, name="_size"): if size < 0: raise ValueError except ValueError: - raise ValueError("{} must be a positive integer".format(name)) + raise ValueError(f"{name} must be a positive integer") if size > maximum: - raise ValueError("{} must be <= {}".format(name, maximum)) + raise ValueError(f"{name} must be <= {maximum}") return size @@ -1403,7 +1508,7 @@ class TildeEncoder(dict): elif b == _space: res = "+" else: - res = "~{:02X}".format(b) + res = f"~{b:02X}" self[b] = res return res @@ -1498,7 +1603,7 @@ def _combine(base: dict, update: dict) -> dict: return base -def pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> dict: +def pairs_to_nested_config(pairs: list[tuple[str, typing.Any]]) -> dict: """ Parse a list of key-value pairs into a nested dictionary. """ @@ -1513,7 +1618,7 @@ def make_slot_function(name, datasette, request, **kwargs): from datasette.plugins import pm method = getattr(pm.hook, name, None) - assert method is not None, "No hook found for {}".format(name) + assert method is not None, f"No hook found for {name}" async def inner(): html_bits = [] @@ -1537,7 +1642,7 @@ def prune_empty_dicts(d: dict): d.pop(key, None) -def move_plugins_and_allow(source: dict, destination: dict) -> Tuple[dict, dict]: +def move_plugins_and_allow(source: dict, destination: dict) -> tuple[dict, dict]: """ Move 'plugins' and 'allow' keys from source to destination dictionary. Creates hierarchy in destination if needed. After moving, recursively remove any keys diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 610b86f2..812194fd 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -1,28 +1,29 @@ import json -from typing import Optional +import re +from http.cookies import Morsel, SimpleCookie +from mimetypes import guess_type +from pathlib import Path +from urllib.parse import parse_qs, parse_qsl, urlunparse + +import aiofiles +import aiofiles.os + from datasette.utils import MultiParams, calculate_etag, error_body, sha256_file from datasette.utils.multipart import ( - parse_form_data, - MultipartParseError, - FormData, - DEFAULT_MAX_FILE_SIZE, - DEFAULT_MAX_REQUEST_SIZE, - DEFAULT_MAX_FIELDS, - DEFAULT_MAX_FILES, - DEFAULT_MAX_PARTS, DEFAULT_MAX_FIELD_SIZE, + DEFAULT_MAX_FIELDS, + DEFAULT_MAX_FILE_SIZE, + DEFAULT_MAX_FILES, DEFAULT_MAX_MEMORY_FILE_SIZE, DEFAULT_MAX_PART_HEADER_BYTES, DEFAULT_MAX_PART_HEADER_LINES, + DEFAULT_MAX_PARTS, + DEFAULT_MAX_REQUEST_SIZE, DEFAULT_MIN_FREE_DISK_BYTES, + FormData, + MultipartParseError, + parse_form_data, ) -from mimetypes import guess_type -from urllib.parse import parse_qs, urlunparse, parse_qsl -from pathlib import Path -from http.cookies import SimpleCookie, Morsel -import aiofiles -import aiofiles.os -import re # Workaround for adding samesite support to pre 3.8 python Morsel._reserved["samesite"] = "SameSite" @@ -88,7 +89,7 @@ class Request: self.max_post_body_bytes = max_post_body_bytes def __repr__(self): - return ''.format(self.method, self.url) + return f'' @property def method(self): @@ -167,7 +168,7 @@ class Request: if max_bytes is None: max_bytes = self.max_post_body_bytes too_large = PayloadTooLarge( - "Request body exceeded maximum size of {} bytes".format(max_bytes) + f"Request body exceeded maximum size of {max_bytes} bytes" ) if max_bytes: # Reject early if the client declares an oversized body @@ -206,7 +207,7 @@ class Request: max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: Optional[int] = DEFAULT_MAX_PARTS, + max_parts: int | None = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, @@ -529,9 +530,9 @@ class Response: httponly=False, samesite="lax", ): - assert samesite in SAMESITE_VALUES, "samesite should be one of {}".format( - SAMESITE_VALUES - ) + assert ( + samesite in SAMESITE_VALUES + ), f"samesite should be one of {SAMESITE_VALUES}" cookie = SimpleCookie() cookie[key] = value for prop_name, prop_value in ( diff --git a/datasette/utils/baseconv.py b/datasette/utils/baseconv.py index c4b64908..0469d7a8 100644 --- a/datasette/utils/baseconv.py +++ b/datasette/utils/baseconv.py @@ -13,7 +13,7 @@ Originally shared here: https://www.djangosnippets.org/snippets/1431/ """ -class BaseConverter(object): +class BaseConverter: decimal_digits = "0123456789" def __init__(self, digits): diff --git a/datasette/utils/check_callable.py b/datasette/utils/check_callable.py index a0997d20..e21a769b 100644 --- a/datasette/utils/check_callable.py +++ b/datasette/utils/check_callable.py @@ -1,6 +1,6 @@ import inspect import types -from typing import NamedTuple, Any +from typing import Any, NamedTuple class CallableStatus(NamedTuple): @@ -19,7 +19,7 @@ def check_callable(obj: Any) -> CallableStatus: if isinstance(obj, types.FunctionType): return CallableStatus(True, inspect.iscoroutinefunction(obj)) - if hasattr(obj, "__call__"): + if callable(obj): return CallableStatus(True, inspect.iscoroutinefunction(obj.__call__)) - assert False, "obj {} is somehow callable with no __call__ method".format(repr(obj)) + assert False, f"obj {obj!r} is somehow callable with no __call__ method" diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 702b53d8..0ddeb847 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -207,7 +207,8 @@ async def populate_schema_tables(internal_db, db, schema_version): columns = table_column_details(conn, table_name) columns_to_insert.extend( { - **{"database_name": database_name, "table_name": table_name}, + "database_name": database_name, + "table_name": table_name, **column._asdict(), } for column in columns @@ -217,7 +218,8 @@ async def populate_schema_tables(internal_db, db, schema_version): ).fetchall() foreign_keys_to_insert.extend( { - **{"database_name": database_name, "table_name": table_name}, + "database_name": database_name, + "table_name": table_name, **dict(foreign_key), } for foreign_key in foreign_keys @@ -227,7 +229,8 @@ async def populate_schema_tables(internal_db, db, schema_version): ).fetchall() indexes_to_insert.extend( { - **{"database_name": database_name, "table_name": table_name}, + "database_name": database_name, + "table_name": table_name, **dict(index), } for index in indexes @@ -259,7 +262,7 @@ async def populate_schema_tables(internal_db, db, schema_version): "catalog_tables", ): conn.execute( - "DELETE FROM {} WHERE database_name = ?".format(table), + f"DELETE FROM {table} WHERE database_name = ?", [database_name], ) conn.execute( diff --git a/datasette/utils/multipart.py b/datasette/utils/multipart.py index cfa77486..13289b1c 100644 --- a/datasette/utils/multipart.py +++ b/datasette/utils/multipart.py @@ -11,15 +11,10 @@ Supports: import asyncio import shutil import tempfile +from collections.abc import Callable from dataclasses import dataclass, field from typing import ( Any, - Callable, - Dict, - List, - Optional, - Tuple, - Union, ) from urllib.parse import parse_qsl @@ -29,7 +24,7 @@ DEFAULT_MAX_REQUEST_SIZE = 100 * 1024 * 1024 # 100MB DEFAULT_MAX_FIELDS = 1000 DEFAULT_MAX_FILES = 100 # If max_parts is not specified, it defaults to max_fields + max_files -DEFAULT_MAX_PARTS: Optional[int] = None +DEFAULT_MAX_PARTS: int | None = None DEFAULT_MAX_FIELD_SIZE = 100 * 1024 # 100KB DEFAULT_MAX_MEMORY_FILE_SIZE = 1024 * 1024 # 1MB DEFAULT_MAX_PART_HEADER_BYTES = 16 * 1024 # 16KB @@ -40,8 +35,6 @@ DEFAULT_MIN_FREE_DISK_BYTES = 50 * 1024 * 1024 # 50MB class MultipartParseError(Exception): """Raised when multipart parsing fails.""" - pass - @dataclass class UploadedFile: @@ -57,7 +50,7 @@ class UploadedFile: name: str filename: str - content_type: Optional[str] + content_type: str | None size: int _file: tempfile.SpooledTemporaryFile = field(repr=False) @@ -86,7 +79,8 @@ class UploadedFile: def __del__(self): try: self._file.close() - except Exception: + except Exception: # noqa: BLE001, S110 + # __del__ must never raise pass @@ -98,27 +92,27 @@ class FormData: """ def __init__(self): - self._data: List[Tuple[str, Union[str, UploadedFile]]] = [] + self._data: list[tuple[str, str | UploadedFile]] = [] - def append(self, key: str, value: Union[str, UploadedFile]) -> None: + def append(self, key: str, value: str | UploadedFile) -> None: """Add a key-value pair.""" self._data.append((key, value)) - def __getitem__(self, key: str) -> Union[str, UploadedFile]: + def __getitem__(self, key: str) -> str | UploadedFile: """Get the first value for a key.""" for k, v in self._data: if k == key: return v raise KeyError(key) - def get(self, key: str, default: Any = None) -> Optional[Union[str, UploadedFile]]: + def get(self, key: str, default: Any = None) -> str | UploadedFile | None: """Get the first value for a key, or default if not found.""" try: return self[key] except KeyError: return default - def getlist(self, key: str) -> List[Union[str, UploadedFile]]: + def getlist(self, key: str) -> list[str | UploadedFile]: """Get all values for a key.""" return [v for k, v in self._data if k == key] @@ -142,15 +136,15 @@ class FormData: """Return unique keys.""" return list(self) - def items(self) -> List[Tuple[str, Union[str, UploadedFile]]]: + def items(self) -> list[tuple[str, str | UploadedFile]]: """Return all key-value pairs.""" return list(self._data) - def values(self) -> List[Union[str, UploadedFile]]: + def values(self) -> list[str | UploadedFile]: """Return all values.""" return [v for _, v in self._data] - def _uploaded_files(self) -> List[UploadedFile]: + def _uploaded_files(self) -> list[UploadedFile]: """Return UploadedFile instances contained in this form.""" return [v for _, v in self._data if isinstance(v, UploadedFile)] @@ -163,7 +157,7 @@ class FormData: for uploaded in self._uploaded_files(): try: uploaded.close_sync() - except Exception: + except Exception: # noqa: BLE001, S110 # Best-effort cleanup; ignore close errors pass @@ -172,7 +166,7 @@ class FormData: for uploaded in self._uploaded_files(): try: await uploaded.close() - except Exception: + except Exception: # noqa: BLE001, S110 # Best-effort cleanup; ignore close errors pass @@ -189,13 +183,13 @@ class FormData: await self.aclose() -def parse_content_disposition(header: str) -> Dict[str, Optional[str]]: +def parse_content_disposition(header: str) -> dict[str, str | None]: """ Parse Content-Disposition header value. Returns dict with 'name', 'filename' keys (filename may be None). """ - result: Dict[str, Optional[str]] = {"name": None, "filename": None} + result: dict[str, str | None] = {"name": None, "filename": None} # Split on semicolons, handling quoted strings parts = [] @@ -238,7 +232,8 @@ def parse_content_disposition(header: str) -> Dict[str, Optional[str]]: from urllib.parse import unquote result["filename"] = unquote(encoded, encoding="utf-8") - except Exception: + except Exception: # noqa: BLE001, S110 + # Malformed RFC 5987 filename* - fall back to the plain filename pass continue @@ -250,20 +245,19 @@ def parse_content_disposition(header: str) -> Dict[str, Optional[str]]: if key == "name": result["name"] = value - elif key == "filename": - # Only set if filename* hasn't already set it - if result["filename"] is None: - # Strip path components (security) - # Handle both Unix and Windows paths - value = value.replace("\\", "/") - if "/" in value: - value = value.rsplit("/", 1)[-1] - result["filename"] = value + # Only set filename if filename* hasn't already set it + elif key == "filename" and result["filename"] is None: + # Strip path components (security) + # Handle both Unix and Windows paths + value = value.replace("\\", "/") + if "/" in value: + value = value.rsplit("/", 1)[-1] + result["filename"] = value return result -def parse_content_type(header: str) -> Tuple[str, Dict[str, str]]: +def parse_content_type(header: str) -> tuple[str, dict[str, str]]: """ Parse Content-Type header value. @@ -307,7 +301,7 @@ class MultipartParser: max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: Optional[int] = DEFAULT_MAX_PARTS, + max_parts: int | None = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, @@ -348,12 +342,12 @@ class MultipartParser: self._tempdir = tempfile.gettempdir() # Current part state - self.current_headers: Dict[str, str] = {} - self.current_file: Optional[tempfile.SpooledTemporaryFile] = None + self.current_headers: dict[str, str] = {} + self.current_file: tempfile.SpooledTemporaryFile | None = None self.current_body = bytearray() - self.current_name: Optional[str] = None - self.current_filename: Optional[str] = None - self.current_content_type: Optional[str] = None + self.current_name: str | None = None + self.current_filename: str | None = None + self.current_content_type: str | None = None def feed(self, chunk: bytes) -> None: """Feed a chunk of data to the parser.""" @@ -454,7 +448,7 @@ class MultipartParser: # Parse header try: line_str = line.decode("utf-8", errors="replace") - except Exception: + except UnicodeDecodeError: line_str = line.decode("latin-1") if ":" in line_str: @@ -481,7 +475,9 @@ class MultipartParser: if self.file_count > self.max_files: raise MultipartParseError("Too many files") if self.handle_files: - self.current_file = tempfile.SpooledTemporaryFile( + # Outlives this method - it is filled in across parser callbacks + # and then handed to the UploadedFile the caller consumes + self.current_file = tempfile.SpooledTemporaryFile( # noqa: SIM115 max_size=self.max_memory_file_size ) else: @@ -644,7 +640,7 @@ async def parse_form_data( max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: Optional[int] = DEFAULT_MAX_PARTS, + max_parts: int | None = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, diff --git a/datasette/utils/permissions.py b/datasette/utils/permissions.py index fd1e41a1..5a8ee8e2 100644 --- a/datasette/utils/permissions.py +++ b/datasette/utils/permissions.py @@ -2,8 +2,9 @@ from __future__ import annotations import json -from typing import Any, Dict, Iterable, List, Sequence, Tuple import sqlite3 +from collections.abc import Iterable, Sequence +from typing import Any from datasette.permissions import PermissionSQL from datasette.plugins import pm @@ -15,7 +16,7 @@ SKIP_PERMISSION_CHECKS = object() async def gather_permission_sql_from_hooks( *, datasette, actor: dict | None, action: str -) -> List[PermissionSQL] | object: +) -> list[PermissionSQL] | object: """Collect PermissionSQL objects from the permission_resources_sql hook. Ensures that each returned PermissionSQL has a populated ``source``. @@ -34,7 +35,7 @@ async def gather_permission_sql_from_hooks( hookimpls = hook_caller.get_hookimpls() hook_results = list(hook_caller(datasette=datasette, actor=actor, action=action)) - collected: List[PermissionSQL] = [] + collected: list[PermissionSQL] = [] actor_json = json.dumps(actor) if actor is not None else None actor_id = actor.get("id") if isinstance(actor, dict) else None @@ -71,7 +72,7 @@ def _iter_permission_sql_from_result( if isinstance(result, PermissionSQL): return [result] if isinstance(result, (list, tuple)): - collected: List[PermissionSQL] = [] + collected: list[PermissionSQL] = [] for item in result: collected.extend(_iter_permission_sql_from_result(item, action=action)) return collected @@ -90,7 +91,7 @@ def _iter_permission_sql_from_result( def build_rules_union( actor: dict | None, plugins: Sequence[PermissionSQL] -) -> Tuple[str, Dict[str, Any]]: +) -> tuple[str, dict[str, Any]]: """ Compose plugin SQL into a UNION ALL. @@ -102,10 +103,10 @@ def build_rules_union( The system reserves these parameter names: :actor, :actor_id, :action, :filter_parent Plugin parameters should be prefixed with a unique identifier (e.g., source name). """ - parts: List[str] = [] + parts: list[str] = [] actor_json = json.dumps(actor) if actor else None actor_id = actor.get("id") if actor else None - params: Dict[str, Any] = {"actor": actor_json, "actor_id": actor_id} + params: dict[str, Any] = {"actor": actor_json, "actor_id": actor_id} for p in plugins: # No namespacing - just use plugin params as-is @@ -141,10 +142,10 @@ async def resolve_permissions_from_catalog( plugins: Sequence[Any], action: str, candidate_sql: str, - candidate_params: Dict[str, Any] | None = None, + candidate_params: dict[str, Any] | None = None, *, implicit_deny: bool = True, -) -> List[Dict[str, Any]]: +) -> list[dict[str, Any]]: """ Resolve permissions by embedding the provided *candidate_sql* in a CTE. @@ -168,8 +169,8 @@ async def resolve_permissions_from_catalog( - parent, child, allow, reason, source_plugin, depth - resource (rendered "/parent/child" or "/parent" or "/") """ - resolved_plugins: List[PermissionSQL] = [] - restriction_sqls: List[str] = [] + resolved_plugins: list[PermissionSQL] = [] + restriction_sqls: list[str] = [] for plugin in plugins: if callable(plugin) and not isinstance(plugin, PermissionSQL): @@ -398,11 +399,11 @@ async def resolve_permissions_with_candidates( db, actor: dict | None, plugins: Sequence[Any], - candidates: List[Tuple[str, str | None]], + candidates: list[tuple[str, str | None]], action: str, *, implicit_deny: bool = True, -) -> List[Dict[str, Any]]: +) -> list[dict[str, Any]]: """ Resolve permissions without any external candidate table by embedding the candidates as a UNION of parameterized SELECTs in a CTE. @@ -411,8 +412,8 @@ async def resolve_permissions_with_candidates( actor: actor dict (or None), made available as :actor (JSON), :actor_id, and :action """ # Build a small CTE for candidates. - cand_rows_sql: List[str] = [] - cand_params: Dict[str, Any] = {} + cand_rows_sql: list[str] = [] + cand_params: dict[str, Any] = {} for i, (parent, child) in enumerate(candidates): pkey = f"cand_p_{i}" ckey = f"cand_c_{i}" diff --git a/datasette/utils/shutil_backport.py b/datasette/utils/shutil_backport.py index d1fd1bd7..d323f5d6 100644 --- a/datasette/utils/shutil_backport.py +++ b/datasette/utils/shutil_backport.py @@ -6,7 +6,7 @@ https://github.com/python/cpython/blob/v3.8.3/LICENSE """ import os -from shutil import copy, copy2, copystat, Error +from shutil import Error, copy, copy2, copystat def _copytree( diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 1be28982..334545bd 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -413,12 +413,12 @@ def analyze_sql_tables( database=None, table=None, sqlite_schema=sqlite_schema, - target="{} {}".format(arg1, arg2) if arg2 is not None else arg1, + target=f"{arg1} {arg2}" if arg2 is not None else arg1, source=source, ) return sqlite3.SQLITE_OK - action_name = _AUTHORIZER_ACTION_NAMES.get(action, "SQLITE_{}".format(action)) + action_name = _AUTHORIZER_ACTION_NAMES.get(action, f"SQLITE_{action}") record( "unknown", "unknown", @@ -521,9 +521,7 @@ def analyze_sql_tables( and key.target in _SQLITE_INTERNAL_SCHEMA_FUNCTIONS ): return True - if key_is_drop_table_delete(key): - return True - return False + return bool(key_is_drop_table_delete(key)) def table_kind_for(key: OperationKey) -> SQLiteTableType | None: if ( diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index 4743ae4c..d3926f6f 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -100,7 +100,7 @@ def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str] schema_table = _sqlite_schema_table(schema) try: rows = conn.execute( - "select name, sql from {} where type = 'table'".format(schema_table) + f"select name, sql from {schema_table} where type = 'table'" ).fetchall() except sqlite3.DatabaseError: return [] @@ -127,7 +127,7 @@ def _sqlite_table_type_from_schema( schema_table = _sqlite_schema_table(schema) try: row = conn.execute( - "select type, sql from {} where name = ?".format(schema_table), + f"select type, sql from {schema_table} where name = ?", (table,), ).fetchone() except sqlite3.DatabaseError: @@ -155,7 +155,7 @@ def _is_known_shadow_table( schema_table = _sqlite_schema_table(schema) try: rows = conn.execute( - "select name, sql from {} where type = 'table'".format(schema_table) + f"select name, sql from {schema_table} where type = 'table'" ).fetchall() except sqlite3.DatabaseError: return False @@ -174,7 +174,7 @@ def _sqlite_schema_table(schema: str | None) -> str: return "sqlite_master" if schema == "temp": return "sqlite_temp_master" - return "{}.sqlite_master".format(_quote_identifier(schema)) + return f"{_quote_identifier(schema)}.sqlite_master" def _quote_identifier(value: str) -> str: diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index de7e94af..a8be47bf 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -1,6 +1,7 @@ -from asgiref.sync import async_to_sync -from urllib.parse import urlencode import json +from urllib.parse import urlencode + +from asgiref.sync import async_to_sync # These wrapper classes pre-date the introduction of # datasette.client and httpx to Datasette. They could diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index ed7e175f..bac3b39e 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -1,7 +1,7 @@ -from dataclasses import dataclass import dataclasses import types import typing +from dataclasses import dataclass @dataclass(frozen=True) @@ -74,16 +74,14 @@ class Context: extra_class = table_extra_registry.classes_by_name[name] except KeyError: raise KeyError( - "{}.{} is declared with from_extra() but there is no " - "registered extra of that name".format(cls.__name__, name) + f"{cls.__name__}.{name} is declared with from_extra() but there is no " + "registered extra of that name" ) if cls.extras_scope is not None and not extra_class.available_for( cls.extras_scope ): raise ValueError( - "{}.{} is declared with from_extra() but the {} extra is " - "not available for scope {}".format( - cls.__name__, name, name, cls.extras_scope - ) + f"{cls.__name__}.{name} is declared with from_extra() but the {name} extra is " + f"not available for scope {cls.extras_scope}" ) return extra_class.description or "" diff --git a/datasette/views/base.py b/datasette/views/base.py index 66e14a6d..48108cec 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -2,20 +2,20 @@ import csv import hashlib import sys -from datasette.utils.asgi import Request from datasette.utils import ( - add_cors_headers, EscapeHtmlWriter, InvalidSql, LimitedWriter, + add_cors_headers, path_from_row_pks, path_with_format, sqlite3, ) from datasette.utils.asgi import ( AsgiStream, - Response, BadRequest, + Request, + Response, ) @@ -129,12 +129,10 @@ class BaseView: template = environment.select_template(templates) template_context = { **context, - **{ - "select_templates": [ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], - }, + "select_templates": [ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], } headers = {} if self.has_json_alternate: @@ -151,9 +149,7 @@ class BaseView: template_context["alternate_url_json"] = alternate_url_json headers.update( { - "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) + "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' } ) return Response.html( @@ -184,9 +180,7 @@ async def stream_csv(datasette, fetch_data, request, database): stream = request.args.get("_stream") # Do not calculate facets or counts: extra_parameters = [ - "{}=1".format(key) - for key in ("_nofacet", "_nocount") - if not request.args.get(key) + f"{key}=1" for key in ("_nofacet", "_nocount") if not request.args.get(key) ] if extra_parameters: # Replace request object with a new one with modified scope @@ -216,9 +210,6 @@ async def stream_csv(datasette, fetch_data, request, database): except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) - except sqlite3.OperationalError as e: - raise DatasetteError(str(e)) - except DatasetteError: raise @@ -325,8 +316,9 @@ async def stream_csv(datasette, fetch_data, request, database): else: new_row.append(cell) await writer.writerow(new_row) - except Exception as ex: - sys.stderr.write("Caught this error: {}\n".format(ex)) + except Exception as ex: # noqa: BLE001 + # Streaming CSV: report the error into the response body and stop + sys.stderr.write(f"Caught this error: {ex}\n") sys.stderr.flush() await r.write(str(ex)) return diff --git a/datasette/views/database.py b/datasette/views/database.py index 11646f45..f54ffd38 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,49 +1,52 @@ -from dataclasses import asdict, dataclass, field -from urllib.parse import parse_qsl, urlencode import asyncio import hashlib import itertools import json -import markupsafe import os import textwrap +from dataclasses import asdict, dataclass, field +from urllib.parse import parse_qsl, urlencode + +import markupsafe -from datasette.extras import extra_names_from_request, ExtraScope from datasette.database import QueryInterrupted +from datasette.extras import ExtraScope, extra_names_from_request +from datasette.plugins import pm from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, stored_query_to_dict -from datasette.write_sql import QueryWriteRejected from datasette.utils import ( + InvalidSql, add_cors_headers, await_me_maybe, - error_body, call_with_supported_arguments, - named_parameters as derive_named_parameters, + error_body, format_bytes, - make_slot_function, - tilde_decode, - to_css_class, - validate_sql_select, is_url, + make_slot_function, path_with_added_args, path_with_format, path_with_removed_args, sqlite3, + tilde_decode, + to_css_class, truncate_url, - InvalidSql, + validate_sql_select, ) -from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden -from datasette.plugins import pm +from datasette.utils import ( + named_parameters as derive_named_parameters, +) +from datasette.utils.asgi import AsgiFileDownload, Forbidden, NotFound, Response +from datasette.write_sql import QueryWriteRejected +from . import Context from .base import DatasetteError, View, stream_csv from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns +from .table_create_alter import _create_table_ui_context from .table_extras import ( QueryExtraContext, resolve_query_extras, table_extra_registry, ) -from .table_create_alter import _create_table_ui_context -from . import Context @dataclass @@ -100,7 +103,7 @@ class DatabaseView(View): return response if format_ not in ("html", "json"): - raise NotFound("Invalid format: {}".format(format_)) + raise NotFound(f"Invalid format: {format_}") metadata = await datasette.get_database_metadata(database) @@ -164,7 +167,7 @@ class DatabaseView(View): "label": "Create table", "description": "Create a new table in this database.", "attrs": { - "aria-label": "Create table in {}".format(database), + "aria-label": f"Create table in {database}", "data-database-action": "create-table", }, } @@ -271,9 +274,7 @@ class DatabaseView(View): view_name="database", ), headers={ - "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) + "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' }, ) @@ -556,7 +557,7 @@ async def database_download(request, datasette): if datasette.cors: add_cors_headers(headers) if db.hash: - etag = '"{}"'.format(db.hash) + etag = f'"{db.hash}"' headers["Etag"] = etag # Has user seen this already? if_none_match = request.headers.get("if-none-match") @@ -664,8 +665,9 @@ class QueryView(View): ).first() if message_result: message = message_result[0] - except Exception as ex: - message = "Error running on_success_message_sql: {}".format(ex) + except Exception as ex: # noqa: BLE001 + # Stored-query on_success_message_sql is user-authored + message = f"Error running on_success_message_sql: {ex}" message_type = datasette.ERROR if not message: if stored_query.on_success_message: @@ -679,7 +681,8 @@ class QueryView(View): redirect_url = stored_query.on_success_redirect ok = True - except Exception as ex: + except Exception as ex: # noqa: BLE001 + # Stored-query execution is user-authored SQL message = stored_query.on_error_message or str(ex) message_type = datasette.ERROR redirect_url = stored_query.on_error_redirect @@ -813,16 +816,16 @@ class QueryView(View): rows = results.rows except QueryInterrupted as ex: raise DatasetteError( - textwrap.dedent(""" + textwrap.dedent(f"""

SQL query took too long. The time limit is controlled by the sql_time_limit_ms configuration option.

- + - """.format(markupsafe.escape(ex.sql))).strip(), + """).strip(), title="SQL Interrupted", status=400, message_is_html=True, @@ -838,8 +841,6 @@ class QueryView(View): columns = [] except (sqlite3.OperationalError, InvalidSql) as ex: raise DatasetteError(str(ex), title="Invalid SQL", status=400) - except sqlite3.OperationalError as ex: - raise DatasetteError(str(ex)) except DatasetteError: raise @@ -861,7 +862,7 @@ class QueryView(View): return data, None, None return await stream_csv(datasette, fetch_data_for_csv, request, db.name) - elif format_ in datasette.renderers.keys(): + elif format_ in datasette.renderers: if not sql: raise DatasetteError("?sql= is required", status=400) data = {"ok": True, "rows": rows, "columns": columns} @@ -953,9 +954,7 @@ class QueryView(View): } headers.update( { - "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) + "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' } ) metadata = await query_metadata() @@ -1036,9 +1035,7 @@ class QueryView(View): + "?" + urlencode( { - **{ - "sql": sql, - }, + "sql": sql, **named_parameter_values, } ) @@ -1140,7 +1137,7 @@ class QueryView(View): headers=headers, ) else: - assert False, "Invalid format: {}".format(format_) + assert False, f"Invalid format: {format_}" if datasette.cors: add_cors_headers(r.headers) return r @@ -1241,7 +1238,7 @@ async def display_rows(datasette, database, request, rows, columns): '<Binary: {:,} byte{}>'.format( blob_url, ( - ' title="{}"'.format(formatted) + f' title="{formatted}"' if "bytes" not in formatted else "" ), diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index dd35b127..3dfb810c 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -8,8 +8,8 @@ from datasette.utils.asgi import Response from .base import BaseView from .database import display_rows as display_query_rows from .query_helpers import ( - QueryValidationError, SQL_PARAMETER_FORM_PREFIX, + QueryValidationError, _analysis_is_write, _analysis_rows, _analysis_rows_with_permissions, @@ -31,15 +31,7 @@ WRITE_TEMPLATE_LABELS = { "delete": "Delete rows", } WRITE_TEMPLATE_OPERATIONS = tuple(WRITE_TEMPLATE_LABELS) -CREATE_TABLE_TEMPLATE_SQL = "\n".join( - ( - "create table new_table (", - " id integer primary key,", - " name text", - " -- created text default (datetime('now'))", - ")", - ) -) +CREATE_TABLE_TEMPLATE_SQL = "create table new_table (\n id integer primary key,\n name text\n -- created text default (datetime('now'))\n)" def _parameter_names(columns): @@ -49,11 +41,11 @@ def _parameter_names(columns): base = re.sub(r"[^a-z0-9_]+", "_", column.lower()) base = base.strip("_") or "value" if base[0].isdigit(): - base = "p_{}".format(base) + base = f"p_{base}" name = base index = 2 while name in seen: - name = "{}_{}".format(base, index) + name = f"{base}_{index}" index += 1 seen.add(name) names[column] = name @@ -65,7 +57,7 @@ def _quote_identifier(identifier): def _preferred_where_column(table, columns): - lower_table_id = "{}_id".format(table.lower()) + lower_table_id = f"{table.lower()}_id" return ( next((column for column in columns if column.lower() == "id"), None) or next( @@ -90,17 +82,15 @@ def _insert_template_sql(table, columns): auto_pk = _auto_incrementing_primary_key(columns) insert_columns = [column for column in column_names if column != auto_pk] if not insert_columns: - return "insert into {}\ndefault values".format(_quote_identifier(table)) + return f"insert into {_quote_identifier(table)}\ndefault values" names = _parameter_names(insert_columns) return "\n".join( ( - "insert into {} (".format(_quote_identifier(table)), - ",\n".join( - " {}".format(_quote_identifier(column)) for column in insert_columns - ), + f"insert into {_quote_identifier(table)} (", + ",\n".join(f" {_quote_identifier(column)}" for column in insert_columns), ")", "values (", - ",\n".join(" :{}".format(names[column]) for column in insert_columns), + ",\n".join(f" :{names[column]}" for column in insert_columns), ")", ) ) @@ -114,18 +104,14 @@ def _update_template_sql(table, columns): if not set_columns: return "\n".join( ( - "update {}".format(_quote_identifier(table)), - "set {} = :new_{}".format( - _quote_identifier(where_column), names[where_column] - ), - "where {} = :{}".format( - _quote_identifier(where_column), names[where_column] - ), + f"update {_quote_identifier(table)}", + f"set {_quote_identifier(where_column)} = :new_{names[where_column]}", + f"where {_quote_identifier(where_column)} = :{names[where_column]}", ) ) return "\n".join( ( - "update {}".format(_quote_identifier(table)), + f"update {_quote_identifier(table)}", "set " + ",\n".join( "{}{} = :{}".format( @@ -135,9 +121,7 @@ def _update_template_sql(table, columns): ) for index, column in enumerate(set_columns) ), - "where {} = :{}".format( - _quote_identifier(where_column), names[where_column] - ), + f"where {_quote_identifier(where_column)} = :{names[where_column]}", ) ) @@ -148,10 +132,8 @@ def _delete_template_sql(table, columns): where_column = _preferred_where_column(table, column_names) return "\n".join( ( - "delete from {}".format(_quote_identifier(table)), - "where {} = :{}".format( - _quote_identifier(where_column), names[where_column] - ), + f"delete from {_quote_identifier(table)}", + f"where {_quote_identifier(where_column)} = :{names[where_column]}", ) ) diff --git a/datasette/views/index.py b/datasette/views/index.py index 67296cd1..f73ee38a 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -2,11 +2,11 @@ import json from datasette.plugins import pm from datasette.utils import ( + UNSTABLE_API_MESSAGE, + CustomJSONEncoder, add_cors_headers, await_me_maybe, make_slot_function, - CustomJSONEncoder, - UNSTABLE_API_MESSAGE, ) from datasette.utils.asgi import Response from datasette.version import __version__ @@ -46,15 +46,15 @@ class IndexView(BaseView): databases = [] # Iterate over allowed databases instead of all databases - for name in allowed_db_dict.keys(): + for name, allowed_db in allowed_db_dict.items(): db = self.ds.databases[name] - database_private = allowed_db_dict[name].private + database_private = allowed_db.private # Get allowed tables/views for this database allowed_for_db = tables_by_db.get(name, {}) # Get table names from allowed set instead of db.table_names() - table_names = [child_name for child_name in allowed_for_db.keys()] + table_names = [child_name for child_name in allowed_for_db] hidden_table_names = set(await db.hidden_table_names()) @@ -99,7 +99,7 @@ class IndexView(BaseView): # We will be sorting by number of relationships, so populate that field all_foreign_keys = await db.get_all_foreign_keys() for table, foreign_keys in all_foreign_keys.items(): - if table in tables.keys(): + if table in tables: count = len(foreign_keys["incoming"] + foreign_keys["outgoing"]) tables[table]["num_relationships_for_sorting"] = count @@ -121,8 +121,7 @@ class IndexView(BaseView): # Only add views if this is less than TRUNCATE_AT if len(tables_and_views_truncated) < TRUNCATE_AT: num_views_to_add = TRUNCATE_AT - len(tables_and_views_truncated) - for view in views[:num_views_to_add]: - tables_and_views_truncated.append(view) + tables_and_views_truncated.extend(views[:num_views_to_add]) databases.append( { diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 588891d4..725d9cdb 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -5,6 +5,19 @@ from datasette.resources import DatabaseResource from datasette.stored_queries import ( StoredQuery, ) +from datasette.utils import ( + InvalidSql, + escape_sqlite, + parse_size_limit, + path_from_row_pks, + sqlite3, + validate_sql_select, +) +from datasette.utils import ( + named_parameters as derive_named_parameters, +) +from datasette.utils.asgi import Forbidden +from datasette.utils.sql_analysis import Operation, SQLAnalysis from datasette.write_sql import ( IgnoreWriteSqlOperation, QueryWriteRejected, @@ -12,17 +25,6 @@ from datasette.write_sql import ( decision_for_write_sql_operation, operation_is_write, ) -from datasette.utils import ( - parse_size_limit, - named_parameters as derive_named_parameters, - escape_sqlite, - path_from_row_pks, - sqlite3, - validate_sql_select, - InvalidSql, -) -from datasette.utils.asgi import Forbidden -from datasette.utils.sql_analysis import Operation, SQLAnalysis _query_name_re = re.compile(r"^[^/\.\n]+$") @@ -91,7 +93,7 @@ def _as_optional_bool(value, name): return True if lowered in {"0", "false", "f", "no", "off"}: return False - raise QueryValidationError("{} must be 0 or 1".format(name)) + raise QueryValidationError(f"{name} must be 0 or 1") def _query_list_limit(value, default, maximum): @@ -171,7 +173,7 @@ async def _json_or_form_payload(request): try: return json.loads(body or b"{}"), True except json.JSONDecodeError as e: - raise QueryValidationError("Invalid JSON: {}".format(e)) + raise QueryValidationError(f"Invalid JSON: {e}") return await request.post_vars(), False @@ -192,7 +194,7 @@ async def _analyze_user_query(datasette, db, sql, *, actor): try: analysis = await db.analyze_sql(sql, params) except sqlite3.DatabaseError as ex: - raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex + raise QueryValidationError(f"Could not analyze query: {ex}") from ex is_write = _analysis_is_write(analysis) if is_write: @@ -293,8 +295,7 @@ def _coerce_execute_write_payload(data, is_json): for key, value in data.items(): if key in {"sql", "csrftoken", "_json"}: continue - if key.startswith(SQL_PARAMETER_FORM_PREFIX): - key = key[len(SQL_PARAMETER_FORM_PREFIX) :] + key = key.removeprefix(SQL_PARAMETER_FORM_PREFIX) params[key] = value if not isinstance(params, dict): raise QueryValidationError("params must be a dictionary") @@ -314,7 +315,7 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): try: analysis = await db.analyze_sql(sql, params) except sqlite3.DatabaseError as ex: - raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex + raise QueryValidationError(f"Could not analyze query: {ex}") from ex if not _analysis_is_write(analysis): raise QueryValidationError( "Use /-/query for read-only SQL; this endpoint only executes writes" @@ -496,7 +497,7 @@ async def _inserted_row_url(datasette, db, analysis, cursor): ) try: result = await db.execute( - "select {} from {} where rowid = ?".format(select, escape_sqlite(table)), + f"select {select} from {escape_sqlite(table)} where rowid = ?", [lastrowid], ) except sqlite3.DatabaseError: diff --git a/datasette/views/row.py b/datasette/views/row.py index c90a3bbe..b1388299 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -8,34 +8,35 @@ from dataclasses import dataclass, field import markupsafe import sqlite_utils -from datasette.utils.asgi import NotFound, Forbidden, PayloadTooLarge, Response from datasette.database import QueryInterrupted -from datasette.events import UpdateRowEvent, DeleteRowEvent +from datasette.events import DeleteRowEvent, UpdateRowEvent +from datasette.extras import ExtraScope, extra_names_from_request +from datasette.plugins import pm from datasette.resources import TableResource -from .base import BaseView, DatasetteError, stream_csv from datasette.utils import ( + CustomJSONEncoder, + CustomRow, + InvalidSql, + WriteJsonValueError, add_cors_headers, await_me_maybe, call_with_supported_arguments, - CustomJSONEncoder, - CustomRow, decode_write_json_row, - InvalidSql, + escape_sqlite, make_slot_function, path_from_row_pks, path_with_format, path_with_removed_args, - to_css_class, - escape_sqlite, sqlite3, - WriteJsonValueError, + to_css_class, ) -from datasette.plugins import pm -from datasette.extras import extra_names_from_request, ExtraScope +from datasette.utils.asgi import Forbidden, NotFound, PayloadTooLarge, Response + from . import Context, from_extra +from .base import BaseView, DatasetteError, stream_csv from .table import ( - display_columns_and_rows, _table_page_data, + display_columns_and_rows, row_label_from_label_column, ) from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry @@ -187,16 +188,16 @@ class RowView(BaseView): data, extra_template_data, templates = response_or_template_contexts except QueryInterrupted as ex: raise DatasetteError( - textwrap.dedent(""" + textwrap.dedent(f"""

SQL query took too long. The time limit is controlled by the sql_time_limit_ms configuration option.

- + - """.format(markupsafe.escape(ex.sql))).strip(), + """).strip(), title="SQL Interrupted", status=400, message_is_html=True, @@ -207,15 +208,13 @@ class RowView(BaseView): ) except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) - except sqlite3.OperationalError as e: - raise DatasetteError(str(e)) except DatasetteError: raise end = time.perf_counter() data["query_ms"] = (end - start) * 1000 - if format_ in self.ds.renderers.keys(): + if format_ in self.ds.renderers: # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) result = call_with_supported_arguments( @@ -258,7 +257,7 @@ class RowView(BaseView): if status_code is not None: response.status = status_code else: - raise NotFound("Invalid format: {}".format(format_)) + raise NotFound(f"Invalid format: {format_}") ttl = request.args.get("_ttl", None) if ttl is None or not ttl.isdigit(): @@ -373,9 +372,7 @@ class RowView(BaseView): view_name=self.name, ), headers={ - "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) + "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' }, ) @@ -500,7 +497,7 @@ class RowView(BaseView): row_action_label = pk_path if row_label and row_label != pk_path: - row_action_label = "{} {}".format(pk_path, row_label) + row_action_label = f"{pk_path} {row_label}" row_action_permissions = {} if is_table and db.is_mutable: @@ -513,7 +510,7 @@ class RowView(BaseView): row_actions = [] if row_action_permissions.get("update-row"): attrs = { - "aria-label": "Edit row {}".format(row_action_label), + "aria-label": f"Edit row {row_action_label}", "data-row": row_path, "data-row-action": "edit", } @@ -529,7 +526,7 @@ class RowView(BaseView): ) if row_action_permissions.get("delete-row"): attrs = { - "aria-label": "Delete row {}".format(row_action_label), + "aria-label": f"Delete row {row_action_label}", "data-row": row_path, "data-row-action": "delete", } @@ -679,7 +676,7 @@ class RowView(BaseView): key, ",".join(pk_values), ) - foreign_key_tables.append({**fk, **{"count": count, "link": link}}) + foreign_key_tables.append({**fk, "count": count, "link": link}) return foreign_key_tables @@ -705,23 +702,21 @@ async def _row_flash_message(db, action, resolved, row=None): if label: label = _truncated_row_flash_label(label) if label and label != pk_label: - return "{} row {} ({})".format(action, pk_label, label) - return "{} row {}".format(action, pk_label) + return f"{action} row {pk_label} ({label})" + return f"{action} row {pk_label}" async def _resolve_row_and_check_permission(datasette, request, permission): - from datasette.app import DatabaseNotFound, TableNotFound, RowNotFound + from datasette.app import DatabaseNotFound, RowNotFound, TableNotFound try: resolved = await datasette.resolve_row(request) except DatabaseNotFound as e: - return False, Response.error( - ["Database not found: {}".format(e.database_name)], 404 - ) + return False, Response.error([f"Database not found: {e.database_name}"], 404) except TableNotFound as e: - return False, Response.error(["Table not found: {}".format(e.table)], 404) + return False, Response.error([f"Table not found: {e.table}"], 404) except RowNotFound as e: - return False, Response.error(["Record not found: {}".format(e.pk_values)], 404) + return False, Response.error([f"Record not found: {e.pk_values}"], 404) # Ensure user has permission to delete this row if not await datasette.allowed( @@ -753,7 +748,8 @@ class RowDeleteView(BaseView): try: await resolved.db.execute_write_fn(delete_row, request=request) - except Exception as e: + except Exception as e: # noqa: BLE001 + # TODO: narrow to expected write errors so Datasette bugs surface as 500s return Response.error([str(e)], 400) await self.ds.track_event( @@ -793,7 +789,7 @@ class RowUpdateView(BaseView): try: data = await request.json() except json.JSONDecodeError as e: - return Response.error(["Invalid JSON: {}".format(e)]) + return Response.error([f"Invalid JSON: {e}"]) except PayloadTooLarge as e: return Response.error([str(e)], 413) @@ -836,7 +832,8 @@ class RowUpdateView(BaseView): try: await resolved.db.execute_write_fn(update_row, request=request) - except Exception as e: + except Exception as e: # noqa: BLE001 + # TODO: narrow to expected write errors so Datasette bugs surface as 500s return Response.error([str(e)], 400) result = {"ok": True} diff --git a/datasette/views/special.py b/datasette/views/special.py index 28d34208..a77f221f 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -1,23 +1,25 @@ import json import logging +import secrets +import urllib + +from datasette.events import CreateTokenEvent, LoginEvent, LogoutEvent from datasette.jump import JumpSQL, namespace_sql_params from datasette.plugins import pm -from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent from datasette.resources import DatabaseResource, TableResource -from datasette.utils.asgi import Response, Forbidden from datasette.utils import ( UNSTABLE_API_MESSAGE, actor_matches_allow, - parse_size_limit, add_cors_headers, await_me_maybe, error_body, - tilde_encode, + parse_size_limit, tilde_decode, + tilde_encode, ) +from datasette.utils.asgi import Forbidden, Response + from .base import BaseView, View -import secrets -import urllib logger = logging.getLogger(__name__) @@ -179,9 +181,7 @@ class AutocompleteDebugView(BaseView): ) context.update( { - "autocomplete_url": "{}/-/autocomplete".format( - self.ds.urls.table(database_name, table_name) - ), + "autocomplete_url": f"{self.ds.urls.table(database_name, table_name)}/-/autocomplete", "label_column": await db.label_column_for_table(table_name), } ) @@ -420,8 +420,11 @@ class AllowedResourcesView(BaseView): row["reason"] = resource.reasons allowed_rows.append(row) - except Exception: - # If catalog tables don't exist yet, return empty results + except Exception: # noqa: BLE001 + # Returns empty results if the catalog tables don't exist yet, but + # also swallows the AttributeError raised for instance-level actions + # such as view-instance, which have no resource_class. + # TODO: handle that case explicitly and narrow this to sqlite3.Error return ( { "ok": True, @@ -523,7 +526,7 @@ class PermissionRulesView(BaseView): from datasette.utils.actions_sql import build_permission_rules_sql - union_sql, union_params, restriction_sqls = await build_permission_rules_sql( + union_sql, union_params, _restriction_sqls = await build_permission_rules_sql( self.ds, actor, action ) await self.ds.refresh_schemas() @@ -936,7 +939,7 @@ class ApiExplorerView(BaseView): tables.append({"name": table, "links": table_links}) table_links.append( { - "label": "Get rows for {}".format(table), + "label": f"Get rows for {table}", "method": "GET", "path": self.ds.urls.table(name, table, format="json"), } @@ -956,7 +959,7 @@ class ApiExplorerView(BaseView): { "path": self.ds.urls.table(name, table) + "/-/insert", "method": "POST", - "label": "Insert rows into {}".format(table), + "label": f"Insert rows into {table}", "json": { "rows": [ { @@ -970,7 +973,7 @@ class ApiExplorerView(BaseView): { "path": self.ds.urls.table(name, table) + "/-/upsert", "method": "POST", - "label": "Upsert rows into {}".format(table), + "label": f"Upsert rows into {table}", "json": { "rows": [ { @@ -1000,7 +1003,7 @@ class ApiExplorerView(BaseView): table_links.append( { "path": self.ds.urls.table(name, table) + "/-/drop", - "label": "Drop table {}".format(table), + "label": f"Drop table {table}", "json": {"confirm": False}, "method": "POST", } @@ -1017,7 +1020,7 @@ class ApiExplorerView(BaseView): database_links.append( { "path": self.ds.urls.database(name) + "/-/create", - "label": "Create table in {}".format(name), + "label": f"Create table in {name}", "json": { "table": "new_table", "columns": [ diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index d64f37d2..0bbe9f38 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -124,7 +124,7 @@ class QueryListView(BaseView): pairs.append(("_next", page.next)) next_url = self.ds.absolute_url( request, - "{}?{}".format(request.path, urlencode(pairs)), + f"{request.path}?{urlencode(pairs)}", ) current_filters = { @@ -415,7 +415,7 @@ class QueryDefinitionView(BaseView): query_name = tilde_decode(request.url_vars["query"]) query = await self.ds.get_query(db.name, query_name) if query is None: - return Response.error(["Query not found: {}".format(query_name)], 404) + return Response.error([f"Query not found: {query_name}"], 404) if not await self.ds.allowed( action="view-query", resource=QueryResource(db.name, query_name), @@ -439,7 +439,7 @@ class QueryUpdateView(BaseView): query_name = tilde_decode(request.url_vars["query"]) existing = await self.ds.get_query(db.name, query_name) if existing is None: - return Response.error(["Query not found: {}".format(query_name)], 404) + return Response.error([f"Query not found: {query_name}"], 404) if not await self.ds.allowed( action="update-query", resource=QueryResource(db.name, query_name), @@ -532,7 +532,7 @@ class QueryEditView(BaseView): async def get(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error(["Query not found: {}".format(query_name)], 404) + return Response.error([f"Query not found: {query_name}"], 404) await self.ds.ensure_permission( action="update-query", resource=QueryResource(db.name, query_name), @@ -545,7 +545,7 @@ class QueryEditView(BaseView): async def post(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error(["Query not found: {}".format(query_name)], 404) + return Response.error([f"Query not found: {query_name}"], 404) if not await self.ds.allowed( action="update-query", resource=QueryResource(db.name, query_name), @@ -629,7 +629,7 @@ class QueryDeleteView(BaseView): async def get(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error(["Query not found: {}".format(query_name)], 404) + return Response.error([f"Query not found: {query_name}"], 404) await self.ds.ensure_permission( action="delete-query", resource=QueryResource(db.name, query_name), @@ -653,7 +653,7 @@ class QueryDeleteView(BaseView): async def post(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error(["Query not found: {}".format(query_name)], 404) + return Response.error([f"Query not found: {query_name}"], 404) if not await self.ds.allowed( action="delete-query", resource=QueryResource(db.name, query_name), @@ -665,13 +665,13 @@ class QueryDeleteView(BaseView): ["Trusted queries cannot be deleted using the API"], 403 ) - data, is_json = await _json_or_form_payload(request) + _data, is_json = await _json_or_form_payload(request) await self.ds.remove_query(db.name, query_name) if is_json: return Response.json({"ok": True}) self.ds.add_message( request, - "Query “{}” deleted".format(existing.title or query_name), + f"Query “{existing.title or query_name}” deleted", self.ds.INFO, ) return Response.redirect(self.ds.urls.path(self.ds.urls.database(db.name))) diff --git a/datasette/views/table.py b/datasette/views/table.py index f7edd744..3eb80854 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -3,48 +3,51 @@ import itertools import json import urllib import urllib.parse +from dataclasses import dataclass, field import markupsafe +import sqlite_utils +from datasette import tracer from datasette.column_types import SQLiteType -from datasette.extras import extra_names_from_request -from datasette.plugins import pm +from datasette.database import QueryInterrupted from datasette.events import ( AlterTableEvent, DropTableEvent, InsertRowsEvent, UpsertRowsEvent, ) -from datasette.database import QueryInterrupted -from datasette import tracer +from datasette.extras import ExtraScope, extra_names_from_request +from datasette.filters import Filters +from datasette.plugins import pm from datasette.resources import DatabaseResource, TableResource from datasette.utils import ( - add_cors_headers, - await_me_maybe, - call_with_supported_arguments, CustomJSONEncoder, CustomRow, + InvalidSql, + WriteJsonValueError, + add_cors_headers, append_querystring, + await_me_maybe, + call_with_supported_arguments, compound_keys_after_sql, decode_write_json_rows, - format_bytes, - make_slot_function, - tilde_encode, escape_sqlite, filters_should_redirect, + format_bytes, is_url, + make_slot_function, path_from_row_pks, path_with_added_args, path_with_format, path_with_removed_args, path_with_replaced_args, + sqlite3, + tilde_encode, to_css_class, truncate_url, urlsafe_components, value_as_boolean, - InvalidSql, - WriteJsonValueError, - sqlite3, ) from datasette.utils.asgi import ( BadRequest, @@ -54,11 +57,7 @@ from datasette.utils.asgi import ( Request, Response, ) -from datasette.filters import Filters -import sqlite_utils -from dataclasses import dataclass, field -from datasette.extras import ExtraScope from . import Context, from_extra from .base import BaseView, DatasetteError, stream_csv from .database import QueryView @@ -536,7 +535,7 @@ async def _table_insert_ui( columns.append(column_data) data = { - "path": "{}/-/insert".format(datasette.urls.table(database_name, table_name)), + "path": f"{datasette.urls.table(database_name, table_name)}/-/insert", "tableName": table_name, "columns": columns, "bulkColumns": bulk_columns, @@ -544,8 +543,8 @@ async def _table_insert_ui( "maxInsertRows": datasette.setting("max_insert_rows"), } if can_update: - data["upsertPath"] = "{}/-/upsert".format( - datasette.urls.table(database_name, table_name) + data["upsertPath"] = ( + f"{datasette.urls.table(database_name, table_name)}/-/upsert" ) return data @@ -604,7 +603,7 @@ async def _table_alter_ui( columns.append(column_data) data = { - "path": "{}/-/alter".format(datasette.urls.table(database_name, table_name)), + "path": f"{datasette.urls.table(database_name, table_name)}/-/alter", "tableName": table_name, "columns": columns, "primaryKeys": pks, @@ -630,9 +629,7 @@ async def _table_alter_ui( actor=request.actor, ) if can_drop_table: - data["dropPath"] = "{}/-/drop".format( - datasette.urls.table(database_name, table_name) - ) + data["dropPath"] = f"{datasette.urls.table(database_name, table_name)}/-/drop" return data @@ -728,12 +725,10 @@ async def display_columns_and_rows( row_label = row_label_from_label_column(row, label_column) row_action_label = pk_path if row_label and row_label != pk_path: - row_action_label = "{} {}".format(pk_path, row_label) + row_action_label = f"{pk_path} {row_label}" table_path = datasette.urls.table(database_name, table_name) - row_link = '{flat_pks}'.format( - table_path=table_path, - flat_pks=str(markupsafe.escape(pk_path)), - flat_pks_quoted=row_path, + row_link = ( + f'{markupsafe.escape(pk_path)!s}' ) edit_icon = ( '

{}

'.format(error) in response2.text + assert f'

{error}

' in response2.text else: # Check create-token event event = last_event(app_client.ds) @@ -228,7 +231,7 @@ def test_auth_create_token( # And test that token response3 = app_client.get( "/-/actor.json", - headers={"Authorization": "Bearer {}".format("dstok_{}".format(token))}, + headers={"Authorization": "Bearer {}".format(f"dstok_{token}")}, ) assert response3.status == 200 assert response3.json["actor"]["id"] == "test" @@ -241,7 +244,7 @@ async def test_auth_create_token_not_allowed_for_tokens(ds_client): ) response = await ds_client.get( "/-/create-token", - headers={"Authorization": "Bearer dstok_{}".format(ds_tok)}, + headers={"Authorization": f"Bearer dstok_{ds_tok}"}, ) assert response.status_code == 403 @@ -286,12 +289,12 @@ async def test_auth_with_dstok_token(ds_client, scenario, should_work): elif scenario == "invalid_token": token = "invalid" if token: - token = "dstok_{}".format(token) + token = f"dstok_{token}" if scenario == "allow_signed_tokens_off": ds_client.ds._settings["allow_signed_tokens"] = False headers = {} if token: - headers["Authorization"] = "Bearer {}".format(token) + headers["Authorization"] = f"Bearer {token}" response = await ds_client.get("/-/actor.json", headers=headers) try: if should_work: @@ -338,7 +341,7 @@ def test_cli_create_token(app_client, expires): assert details.keys() == expected_keys assert details["a"] == "test" response = app_client.get( - "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} + "/-/actor.json", headers={"Authorization": f"Bearer {token}"} ) if expires is None or expires > 0: expected_actor = { diff --git a/tests/test_base_view.py b/tests/test_base_view.py index c1b0cf20..b46f7ce1 100644 --- a/tests/test_base_view.py +++ b/tests/test_base_view.py @@ -1,8 +1,10 @@ -from datasette.views.base import View +import json + +import pytest + from datasette import Request, Response from datasette.app import Datasette -import json -import pytest +from datasette.views.base import View class GetView(View): diff --git a/tests/test_cli.py b/tests/test_cli.py index cbd8edad..fbd4a8a9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,23 +1,28 @@ -from .fixtures import ( - make_app_client, - TestClient as _TestClient, - EXPECTED_PLUGINS, -) -from datasette.app import SETTINGS -from datasette.plugins import DEFAULT_PLUGINS, pm -from datasette.cli import cli, serve -from datasette.version import __version__ -from datasette.utils import tilde_encode -from datasette.utils.sqlite import sqlite3 -from click.testing import CliRunner import io import json import pathlib -import pytest import sys import textwrap from unittest import mock +import pytest +from click.testing import CliRunner + +from datasette.app import SETTINGS +from datasette.cli import cli, serve +from datasette.plugins import DEFAULT_PLUGINS, pm +from datasette.utils import tilde_encode +from datasette.utils.sqlite import sqlite3 +from datasette.version import __version__ + +from .fixtures import ( + EXPECTED_PLUGINS, + make_app_client, +) +from .fixtures import ( + TestClient as _TestClient, +) + def test_inspect_cli(app_client): runner = CliRunner() @@ -460,7 +465,7 @@ def test_serve_create(tmpdir): @pytest.mark.parametrize("argument", ("-c", "--config")) @pytest.mark.parametrize("format_", ("json", "yaml")) def test_serve_config(tmpdir, argument, format_): - config_path = tmpdir / "datasette.{}".format(format_) + config_path = tmpdir / f"datasette.{format_}" config_path.write_text( ( "settings:\n default_page_size: 5\n" @@ -513,13 +518,13 @@ def test_weird_database_names(tmpdir, filename): result1 = runner.invoke(cli, [db_path, "--get", "/"]) assert result1.exit_code == 0, result1.output filename_no_stem = filename.rsplit(".", 1)[0] - expected_link = '{}'.format( - tilde_encode(filename_no_stem), filename_no_stem + expected_link = ( + f'{filename_no_stem}' ) assert expected_link in result1.output # Now try hitting that database page result2 = runner.invoke( - cli, [db_path, "--get", "/{}".format(tilde_encode(filename_no_stem))] + cli, [db_path, "--get", f"/{tilde_encode(filename_no_stem)}"] ) assert result2.exit_code == 0, result2.output diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index fe9416d6..01b84f59 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -1,8 +1,10 @@ +import json +import textwrap + +from click.testing import CliRunner + from datasette.cli import cli from datasette.plugins import pm -from click.testing import CliRunner -import textwrap -import json def test_serve_with_get(tmp_path_factory): @@ -44,9 +46,9 @@ def test_serve_with_get(tmp_path_factory): # Annoyingly that new test plugin stays resident - we need # to manually unregister it to avoid conflict with other tests - to_unregister = [ + to_unregister = next( p for p in pm.get_plugins() if p.__name__ == "init_for_serve_with_get.py" - ][0] + ) pm.unregister(to_unregister) diff --git a/tests/test_cli_serve_server.py b/tests/test_cli_serve_server.py index 47f23c08..b7604bb8 100644 --- a/tests/test_cli_serve_server.py +++ b/tests/test_cli_serve_server.py @@ -1,6 +1,7 @@ +import socket + import httpx import pytest -import socket @pytest.mark.serial diff --git a/tests/test_column_types.py b/tests/test_column_types.py index cd308ec9..50c6daed 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -1,7 +1,11 @@ import json import logging +import time +import markupsafe +import pytest from bs4 import BeautifulSoup as Soup + from datasette.app import Datasette from datasette.column_types import ( ColumnType, @@ -9,11 +13,7 @@ from datasette.column_types import ( ) from datasette.hookspecs import hookimpl from datasette.plugins import pm -from datasette.utils import error_body, sqlite3 -from datasette.utils import StartupError -import markupsafe -import pytest -import time +from datasette.utils import StartupError, error_body, sqlite3 @pytest.fixture @@ -104,7 +104,7 @@ def write_token(ds, actor_id="root", permissions=None): def _headers(token): return { - "Authorization": "Bearer {}".format(token), + "Authorization": f"Bearer {token}", "Content-Type": "application/json", } diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 636b17eb..00540464 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -1,10 +1,12 @@ import json import pathlib + import pytest from datasette.app import Datasette -from datasette.utils.sqlite import sqlite3 from datasette.utils import StartupError +from datasette.utils.sqlite import sqlite3 + from .fixtures import TestClient as _TestClient PLUGIN = """ diff --git a/tests/test_crossdb.py b/tests/test_crossdb.py index 11e53224..ffd0870c 100644 --- a/tests/test_crossdb.py +++ b/tests/test_crossdb.py @@ -1,7 +1,9 @@ -from datasette.cli import cli -from click.testing import CliRunner -import urllib import sqlite3 +import urllib + +from click.testing import CliRunner + +from datasette.cli import cli def test_crossdb_join(app_client_two_attached_databases_crossdb_enabled): @@ -40,7 +42,7 @@ def test_crossdb_warning_if_too_many_databases(tmp_path_factory): db_dir = tmp_path_factory.mktemp("dbs") dbs = [] for i in range(11): - path = str(db_dir / "db_{}.db".format(i)) + path = str(db_dir / f"db_{i}.db") conn = sqlite3.connect(path) conn.execute("vacuum") conn.close() diff --git a/tests/test_csrf_middleware.py b/tests/test_csrf_middleware.py index 2fcfb216..6c78f69d 100644 --- a/tests/test_csrf_middleware.py +++ b/tests/test_csrf_middleware.py @@ -44,7 +44,7 @@ async def _run_middleware(scope): await mw(scope, None, send) if inner_called: return ("allowed",) - start = [m for m in sent if m["type"] == "http.response.start"][0] + start = next(m for m in sent if m["type"] == "http.response.start") return ("blocked", start["status"]) diff --git a/tests/test_csv.py b/tests/test_csv.py index a2f03776..7758a3c0 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -1,8 +1,10 @@ -from datasette.app import Datasette -from bs4 import BeautifulSoup as Soup -import pytest import urllib.parse +import pytest +from bs4 import BeautifulSoup as Soup + +from datasette.app import Datasette + EXPECTED_TABLE_CSV = """id,content 1,hello 2,world diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 86cdcc6b..32cfc43d 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -1,5 +1,7 @@ import pathlib + import pytest + from .fixtures import make_app_client TEST_TEMPLATE_DIRS = str(pathlib.Path(__file__).parent / "test_templates") diff --git a/tests/test_default_deny.py b/tests/test_default_deny.py index f1e43064..f456a17f 100644 --- a/tests/test_default_deny.py +++ b/tests/test_default_deny.py @@ -1,4 +1,5 @@ import pytest + from datasette.app import Datasette from datasette.resources import DatabaseResource, TableResource diff --git a/tests/test_docs.py b/tests/test_docs.py index 0bcb5e62..16df6a46 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -2,20 +2,22 @@ Tests to ensure certain things are documented. """ -from datasette import app, utils +import re +from pathlib import Path + +import pytest + import datasette.fixtures # noqa: F401 +from datasette import app, utils from datasette.app import Datasette from datasette.filters import Filters -from pathlib import Path -import pytest -import re docs_path = Path(__file__).parent.parent / "docs" label_re = re.compile(r"\.\. _([^\s:]+):") def get_headings(content, underline="-"): - heading_re = re.compile(r"(\w+)(\([^)]*\))?\n\{}+\n".format(underline)) + heading_re = re.compile(rf"(\w+)(\([^)]*\))?\n\{underline}+\n") return {h[0] for h in heading_re.findall(content)} diff --git a/tests/test_docs_plugins.py b/tests/test_docs_plugins.py index 613160ac..4a0014b4 100644 --- a/tests/test_docs_plugins.py +++ b/tests/test_docs_plugins.py @@ -1,10 +1,11 @@ # fmt: off # -- start datasette_with_plugin_fixture -- -from datasette import hookimpl -from datasette.app import Datasette import pytest import pytest_asyncio +from datasette import hookimpl +from datasette.app import Datasette + @pytest_asyncio.fixture async def datasette_with_plugin(): diff --git a/tests/test_error_shape.py b/tests/test_error_shape.py index 768814fd..94c9a7c9 100644 --- a/tests/test_error_shape.py +++ b/tests/test_error_shape.py @@ -17,8 +17,10 @@ present and the legacy "title" key must not be. https://github.com/simonw/datasette/issues - 1.0 API consistency """ -import pytest import time + +import pytest + from datasette.app import Datasette from datasette.utils import sqlite3 @@ -86,7 +88,7 @@ async def test_write_api_validation_error_shape(ds_error_shape): "/data/docs/-/insert", json={"rows": [{"nope": 1}, {"also_nope": 2}]}, headers={ - "Authorization": "Bearer {}".format(token), + "Authorization": f"Bearer {token}", "Content-Type": "application/json", }, ) @@ -410,7 +412,7 @@ async def test_expired_token_returns_401(ds_error_shape): ) ) response = await ds_error_shape.client.get( - "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} + "/-/actor.json", headers={"Authorization": f"Bearer {token}"} ) data = assert_canonical_error(response, 401) assert "expired" in data["error"].lower() @@ -446,7 +448,7 @@ async def test_valid_token_still_authenticates(ds_error_shape): ) ) response = await ds_error_shape.client.get( - "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} + "/-/actor.json", headers={"Authorization": f"Bearer {token}"} ) assert response.status_code == 200 assert response.json()["actor"]["id"] == "root" @@ -477,7 +479,7 @@ async def test_token_when_signed_tokens_disabled_returns_401(tmp_path_factory): ds.sign({"a": "root", "t": int(time.time())}, namespace="token") ) response = await ds.client.get( - "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} + "/-/actor.json", headers={"Authorization": f"Bearer {token}"} ) data = assert_canonical_error(response, 401) assert "not enabled" in data["error"] @@ -642,7 +644,7 @@ async def test_query_list_size_rejects_non_integer(ds_client): @pytest.mark.asyncio @pytest.mark.parametrize("endpoint", ("allowed", "rules")) async def test_debug_endpoints_use_size_and_page_parameters(ds_error_shape, endpoint): - base = "/-/{}.json?action=view-instance".format(endpoint) + base = f"/-/{endpoint}.json?action=view-instance" ok = await ds_error_shape.client.get( base + "&_size=1&_page=1", actor={"id": "root"} ) diff --git a/tests/test_extras.py b/tests/test_extras.py index 73b4965e..4e008926 100644 --- a/tests/test_extras.py +++ b/tests/test_extras.py @@ -1,4 +1,5 @@ import asyncio +from typing import ClassVar import pytest @@ -7,7 +8,7 @@ from datasette.extras import Extra, ExtraRegistry, ExtraScope class SlowValueExtra(Extra): description = "Returns context['value'], optionally slowly" - scopes = {ExtraScope.TABLE} + scopes: ClassVar[set[ExtraScope]] = {ExtraScope.TABLE} async def resolve(self, context): if context["slow"]: @@ -17,7 +18,7 @@ class SlowValueExtra(Extra): class DependentExtra(Extra): description = "Depends on slow_value" - scopes = {ExtraScope.TABLE} + scopes: ClassVar[set[ExtraScope]] = {ExtraScope.TABLE} async def resolve(self, context, slow_value): return slow_value + 1 @@ -25,7 +26,7 @@ class DependentExtra(Extra): class InternalOnlyExtra(Extra): description = "Internal extra for HTML templates only" - scopes = {ExtraScope.TABLE} + scopes: ClassVar[set[ExtraScope]] = {ExtraScope.TABLE} public = False async def resolve(self, context): @@ -52,7 +53,7 @@ def _registered_extra_classes(): @pytest.mark.parametrize("cls", _registered_extra_classes(), ids=lambda cls: cls.key()) def test_registered_extras_have_descriptions(cls): # Every registered extra is part of the documented template/JSON contract - assert cls.description, "{} is missing a description".format(cls.__name__) + assert cls.description, f"{cls.__name__} is missing a description" def test_registry_is_built_once_per_scope(): diff --git a/tests/test_facets.py b/tests/test_facets.py index 8c22ffce..b8eb6e61 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -1,11 +1,14 @@ +import json + +import pytest + from datasette.app import Datasette from datasette.database import Database -from datasette.facets import Facet, ColumnFacet, ArrayFacet, DateFacet -from datasette.utils.asgi import Request +from datasette.facets import ArrayFacet, ColumnFacet, DateFacet, Facet from datasette.utils import detect_json1 +from datasette.utils.asgi import Request + from .fixtures import make_app_client -import json -import pytest @pytest.mark.asyncio @@ -537,7 +540,7 @@ async def test_facet_size(): for j in range(1, 4): await db.execute_write( "insert into neighbourhoods (city, neighbourhood) values (?, ?)", - ["City {}".format(i), "Neighbourhood {}".format(j)], + [f"City {i}", f"Neighbourhood {j}"], ) response = await ds.client.get( "/test_facet_size/neighbourhoods.json?_extra=suggested_facets" diff --git a/tests/test_filters.py b/tests/test_filters.py index eda9e9a1..8d0f3512 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -1,7 +1,8 @@ -from datasette.filters import Filters, through_filters, where_filters, search_filters -from datasette.utils.asgi import Request import pytest +from datasette.filters import Filters, search_filters, through_filters, where_filters +from datasette.utils.asgi import Request + @pytest.mark.parametrize( "args,expected_where,expected_params", diff --git a/tests/test_html.py b/tests/test_html.py index b4c47d80..6a7b4907 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1,16 +1,19 @@ -from bs4 import BeautifulSoup as Soup -from datasette.app import Datasette -from datasette.utils import allowed_pragmas -from .fixtures import make_app_client -from .utils import assert_footer_links, inner_html import copy import hashlib import json import pathlib -import pytest import re import urllib.parse +import pytest +from bs4 import BeautifulSoup as Soup + +from datasette.app import Datasette +from datasette.utils import allowed_pragmas + +from .fixtures import make_app_client +from .utils import assert_footer_links, inner_html + def test_homepage(app_client_two_attached_databases): response = app_client_two_attached_databases.get("/") @@ -142,9 +145,7 @@ def test_static_mounts_hash_cache_control(): ) incorrect_hash = hashlib.sha256(b"incorrect").hexdigest()[:12] - response = client.get( - "/custom-static/test_html.py?_hash={}".format(incorrect_hash) - ) + response = client.get(f"/custom-static/test_html.py?_hash={incorrect_hash}") assert response.status_code == 200 assert "cache-control" not in response.headers @@ -219,11 +220,9 @@ async def test_disallowed_custom_sql_pragma(ds_client): "/fixtures/-/query?sql=SELECT+*+FROM+pragma_not_on_allow_list('idx52')" ) assert response.status_code == 400 - pragmas = ", ".join("pragma_{}()".format(pragma) for pragma in allowed_pragmas) + pragmas = ", ".join(f"pragma_{pragma}()" for pragma in allowed_pragmas) assert ( - "Statement contained a disallowed PRAGMA. Allowed pragma functions are {}".format( - pragmas - ) + f"Statement contained a disallowed PRAGMA. Allowed pragma functions are {pragmas}" in response.text ) @@ -778,8 +777,8 @@ def test_stored_query_show_hide_metadata_option( }, memory=True, ) as client: - expected_show_hide_fragment = '({})'.format( - expected_show_hide_link, expected_show_hide_text + expected_show_hide_fragment = ( + f'({expected_show_hide_text})' ) response = client.get("/_memory/one" + querystring) html = response.text @@ -788,10 +787,7 @@ def test_stored_query_show_hide_metadata_option( )[0] assert show_hide_fragment == expected_show_hide_fragment if expected_hidden: - assert ( - ''.format(expected_hidden) - in html - ) + assert f'' in html else: assert '; rel="alternate"; type="application/json+datasette"'.format( - expected - ) + assert link == f'<{expected}>; rel="alternate"; type="application/json+datasette"' assert ( - ''.format( - expected - ) + f'' in response.text ) @@ -1292,8 +1284,8 @@ async def test_database_color(ds_client): expected_color = ds_client.ds.get_database("fixtures").color # Should be something like #9403e5 expected_fragments = ( - "10px solid #{}".format(expected_color), - "border-color: #{}".format(expected_color), + f"10px solid #{expected_color}", + f"border-color: #{expected_color}", ) assert len(expected_color) == 6 for path in ( diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index e1ab51bb..b4bb964d 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -1,6 +1,7 @@ -import pytest import sqlite3 +import pytest + from datasette.utils import escape_sqlite from datasette.utils.internal_db import INTERNAL_DB_SCHEMA_SQL @@ -137,7 +138,7 @@ async def test_internal_foreign_key_references(ds_client): return { row[1] for row in conn.execute( - "PRAGMA table_info({})".format(escape_sqlite(table_name)) + f"PRAGMA table_info({escape_sqlite(table_name)})" ).fetchall() } @@ -147,7 +148,7 @@ async def test_internal_foreign_key_references(ds_client): for _, name in sorted( (row[5], row[1]) for row in conn.execute( - "PRAGMA table_info({})".format(escape_sqlite(table_name)) + f"PRAGMA table_info({escape_sqlite(table_name)})" ).fetchall() if row[5] ) @@ -159,7 +160,7 @@ async def test_internal_foreign_key_references(ds_client): for table_name in table_names: foreign_key_rows = conn.execute( - "PRAGMA foreign_key_list({})".format(escape_sqlite(table_name)) + f"PRAGMA foreign_key_list({escape_sqlite(table_name)})" ).fetchall() foreign_keys_by_id = {} for foreign_key in foreign_key_rows: @@ -169,25 +170,16 @@ async def test_internal_foreign_key_references(ds_client): foreign_key_rows.sort(key=lambda row: row[1]) other_table = foreign_key_rows[0][2] other_columns = [row[4] for row in foreign_key_rows] - message = 'Column "{}.{}" references other table "{}" which does not exist'.format( - table_name, foreign_key_rows[0][3], other_table - ) + message = f'Column "{table_name}.{foreign_key_rows[0][3]}" references other table "{other_table}" which does not exist' assert other_table in table_names, message + " (bad table)" if all(other_column is None for other_column in other_columns): other_columns = primary_keys_for_table(other_table) - length_message = 'Foreign key from "{}" to "{}" has {} columns but references {} columns'.format( - table_name, - other_table, - len(foreign_key_rows), - len(other_columns), - ) + length_message = f'Foreign key from "{table_name}" to "{other_table}" has {len(foreign_key_rows)} columns but references {len(other_columns)} columns' assert len(other_columns) == len(foreign_key_rows), length_message for foreign_key, other_column in zip(foreign_key_rows, other_columns): column = foreign_key[3] - message = 'Column "{}.{}" references other column "{}.{}" which does not exist'.format( - table_name, column, other_table, other_column - ) + message = f'Column "{table_name}.{column}" references other column "{other_table}.{other_column}" which does not exist' assert other_column in columns_by_table[other_table], ( message + " (bad column)" ) @@ -245,10 +237,10 @@ async def test_stale_catalog_entry_database_fix(tmp_path): @pytest.mark.asyncio async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path): - from datasette.app import Datasette - import sqlite3 + from datasette.app import Datasette + internal_db_path = str(tmp_path / "internal.db") alpha_db_path = str(tmp_path / "alpha.db") bravo_db_path = str(tmp_path / "bravo.db") @@ -293,10 +285,10 @@ async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path @pytest.mark.asyncio async def test_orphan_stale_catalog_child_entries_removed(tmp_path): - from datasette.app import Datasette - import sqlite3 + from datasette.app import Datasette + internal_db_path = str(tmp_path / "internal.db") alpha_db_path = str(tmp_path / "alpha.db") diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index b1a212d9..b1093b1c 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -3,17 +3,23 @@ Tests for the datasette.database.Database class """ import asyncio +import uuid from types import SimpleNamespace -from datasette.app import Datasette -from datasette.database import Database, ExecuteWriteResult, Results, MultipleValues -from datasette.database import DatasetteClosedError -from datasette.database import _deliver_write_result -from datasette.utils.sqlite import sqlite3, supports_returning -from datasette.utils import Column + import pytest import sqlite_utils -import time -import uuid + +from datasette.app import Datasette +from datasette.database import ( + Database, + DatasetteClosedError, + ExecuteWriteResult, + MultipleValues, + Results, + _deliver_write_result, +) +from datasette.utils import Column +from datasette.utils.sqlite import sqlite3, supports_returning requires_sqlite_returning = pytest.mark.skipif( not supports_returning(), reason="SQLite does not support RETURNING" @@ -44,7 +50,7 @@ async def test_results_first(db): @pytest.mark.parametrize("expected", (True, False)) async def test_results_bool(db, expected): where = "" if expected else "where pk = 0" - results = await db.execute("select * from facetable {}".format(where)) + results = await db.execute(f"select * from facetable {where}") assert bool(results) is expected @@ -616,7 +622,7 @@ async def test_execute_write_block_false(db): "update roadside_attractions set name = ? where pk = ?", ["Mystery!", 1], ) - time.sleep(0.1) + await asyncio.sleep(0.1) rows = await db.execute("select name from roadside_attractions where pk = 1") assert "Mystery!" == rows.rows[0][0] @@ -634,7 +640,7 @@ async def test_execute_write_with_returning_block_false(db): ) assert isinstance(task_id, uuid.UUID) - time.sleep(0.1) + await asyncio.sleep(0.1) assert ( await db.execute("select name from write_returning_block_false") ).single_value() == "Cleo" @@ -760,9 +766,10 @@ async def test_execute_write_fn_accepts_any_single_param_name(db, param_name): # Plugins historically relied on the fact that the callback was invoked # positionally, so any parameter name worked. Preserve that contract. scope = {} - exec( - "def write_fn({0}):\n" - " return {0}.execute('select 1 + 1').fetchone()[0]".format(param_name), + # exec() is how we build a function with a parameterized argument name + exec( # noqa: S102 + f"def write_fn({param_name}):\n" + f" return {param_name}.execute('select 1 + 1').fetchone()[0]", scope, ) write_fn = scope["write_fn"] @@ -786,7 +793,9 @@ async def test_execute_write_fn_with_track_event(db): @pytest.mark.asyncio -@pytest.mark.timeout(1) +# func_only so the budget covers the write-thread call under test, not the +# one-off app_client fixture setup this test may be first to trigger +@pytest.mark.timeout(1, func_only=True) async def test_execute_write_fn_connection_exception(tmpdir, app_client): path = str(tmpdir / "immutable.db") conn = sqlite3.connect(path) diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 85598c05..ed2aeaf0 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -9,13 +9,15 @@ import importlib import os import sqlite3 import time + +import pytest +from itsdangerous import BadSignature + from datasette import Context -from datasette.app import Datasette, Database, ResourcesSQL +from datasette.app import Database, Datasette, ResourcesSQL from datasette.database import DatasetteClosedError from datasette.resources import DatabaseResource from datasette.utils import PrefixedUrlString -from itsdangerous import BadSignature -import pytest @pytest.fixture @@ -77,9 +79,7 @@ async def test_static_template_function_hashes_core_asset(tmp_path, monkeypatch) template = ds.get_jinja_environment().from_string("{{ static('demo.js') }}") expected_hash = hashlib.sha256(b"const demo = true;").hexdigest()[:12] - assert await template.render_async() == "/-/static/demo.js?_hash={}".format( - expected_hash - ) + assert await template.render_async() == f"/-/static/demo.js?_hash={expected_hash}" assert isinstance(ds.static("demo.js"), PrefixedUrlString) @@ -101,7 +101,7 @@ def test_static_hash_recalculated_when_cache_headers_disabled(tmp_path, monkeypa asset_path.write_bytes(b"let a = 2;") expected_hash = hashlib.sha256(b"let a = 2;").hexdigest()[:12] - assert ds.static("demo.js") == "/-/static/demo.js?_hash={}".format(expected_hash) + assert ds.static("demo.js") == f"/-/static/demo.js?_hash={expected_hash}" assert ds.static("demo.js") != first_url @@ -114,12 +114,12 @@ def test_static_hashes_mounted_static_file(tmp_path): expected_hash = hashlib.sha256(b"body { color: black; }").hexdigest()[:12] assert ds.static("styles.css", mount="assets") == ( - "/assets/styles.css?_hash={}".format(expected_hash) + f"/assets/styles.css?_hash={expected_hash}" ) ds._settings["base_url"] = "/prefix/" assert ds.static("styles.css", mount="assets") == ( - "/prefix/assets/styles.css?_hash={}".format(expected_hash) + f"/prefix/assets/styles.css?_hash={expected_hash}" ) @@ -144,9 +144,7 @@ def test_static_hashes_plugin_static_file(tmp_path, monkeypatch): expected_hash = hashlib.sha256(b"console.log('plugin');").hexdigest()[:12] assert ds.static("plugin.js", plugin="datasette_cluster_map") == ( - "/-/static-plugins/datasette_cluster_map/plugin.js?_hash={}".format( - expected_hash - ) + f"/-/static-plugins/datasette_cluster_map/plugin.js?_hash={expected_hash}" ) diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py index e9aaaae8..51b38f8d 100644 --- a/tests/test_internals_datasette_client.py +++ b/tests/test_internals_datasette_client.py @@ -1,6 +1,7 @@ import httpx import pytest import pytest_asyncio + from datasette.app import Datasette @@ -238,7 +239,7 @@ async def test_in_client_returns_false_outside_request(datasette): @pytest.mark.asyncio async def test_in_client_returns_true_inside_request(): """Test that datasette.in_client() returns True inside a client request""" - from datasette import hookimpl, Response + from datasette import Response, hookimpl class TestPlugin: __name__ = "test_in_client_plugin" diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py index 6d2dc70a..e982628b 100644 --- a/tests/test_internals_request.py +++ b/tests/test_internals_request.py @@ -1,7 +1,9 @@ -from datasette.utils.asgi import PayloadTooLarge, Request import json + import pytest +from datasette.utils.asgi import PayloadTooLarge, Request + def _post_scope(headers=None): return { diff --git a/tests/test_internals_response.py b/tests/test_internals_response.py index 2366dcde..aa3e1ae2 100644 --- a/tests/test_internals_response.py +++ b/tests/test_internals_response.py @@ -1,7 +1,9 @@ -from datasette.utils.asgi import Response import json + import pytest +from datasette.utils.asgi import Response + def test_response_html(): response = Response.html("Hello from HTML") diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index 24fa745d..50c61995 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -1,6 +1,7 @@ +import pytest + from datasette.app import Datasette from datasette.utils import PrefixedUrlString -import pytest @pytest.fixture(scope="module") diff --git a/tests/test_label_column_for_table.py b/tests/test_label_column_for_table.py index 7667b595..b67b8882 100644 --- a/tests/test_label_column_for_table.py +++ b/tests/test_label_column_for_table.py @@ -1,6 +1,7 @@ import pytest -from datasette.database import Database + from datasette.app import Datasette +from datasette.database import Database @pytest.mark.asyncio diff --git a/tests/test_load_extensions.py b/tests/test_load_extensions.py index cdadb091..61cdb3e0 100644 --- a/tests/test_load_extensions.py +++ b/tests/test_load_extensions.py @@ -1,7 +1,9 @@ -from datasette.app import Datasette -import pytest from pathlib import Path +import pytest + +from datasette.app import Datasette + # not necessarily a full path - the full compiled path looks like "ext.dylib" # or another suffix, but sqlite will, under the hood, decide which file # extension to use based on the operating system (apple=dylib, windows=dll etc) diff --git a/tests/test_messages.py b/tests/test_messages.py index 62d9f647..60eb7938 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -1,6 +1,7 @@ -from .utils import cookie_was_deleted import pytest +from .utils import cookie_was_deleted + @pytest.mark.asyncio @pytest.mark.parametrize( diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 0dc3ecd7..ab38bfb7 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -6,12 +6,12 @@ Uses TDD approach - these tests are written first, then implementation follows. import base64 import json -import pytest from collections import namedtuple +import pytest from multipart_form_data_conformance import get_tests_dir -from datasette.utils.asgi import Request, BadRequest +from datasette.utils.asgi import BadRequest, Request def make_receive(body: bytes): diff --git a/tests/test_package.py b/tests/test_package.py index f05f3ece..43b20589 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -1,9 +1,11 @@ -from click.testing import CliRunner -from datasette import cli -from unittest import mock import os import pathlib +from unittest import mock + import pytest +from click.testing import CliRunner + +from datasette import cli class CaptureDockerfile: diff --git a/tests/test_permission_endpoints.py b/tests/test_permission_endpoints.py index 8726ab62..c54bfbfd 100644 --- a/tests/test_permission_endpoints.py +++ b/tests/test_permission_endpoints.py @@ -6,6 +6,7 @@ Tests for permission endpoints: import pytest import pytest_asyncio + from datasette.app import Datasette @@ -432,8 +433,8 @@ async def test_execute_sql_requires_view_database(): A user who has execute-sql permission but not view-database permission should not be able to execute SQL on that database. """ - from datasette.permissions import PermissionSQL from datasette import hookimpl + from datasette.permissions import PermissionSQL class TestPermissionPlugin: __name__ = "TestPermissionPlugin" diff --git a/tests/test_permissions.py b/tests/test_permissions.py index cd1050d0..73c44682 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1,20 +1,23 @@ import collections +import copy +import json +import re +import time +import urllib +from pprint import pprint + +import pytest +import pytest_asyncio from asgiref.sync import async_to_sync +from bs4 import BeautifulSoup as Soup +from click.testing import CliRunner + from datasette.app import Datasette from datasette.cli import cli from datasette.default_permissions import restrictions_allow_action from datasette.utils import UNSTABLE_API_MESSAGE + from .fixtures import assert_permissions_checked, make_app_client -from click.testing import CliRunner -from bs4 import BeautifulSoup as Soup -import copy -import json -from pprint import pprint -import pytest_asyncio -import pytest -import re -import time -import urllib @pytest.fixture(scope="module") @@ -602,9 +605,7 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta ) assert ( response.status == expected_status - ), "path: {}, permissions: {}, expected_status: {}, status: {}".format( - path, permissions, expected_status, response.status - ) + ), f"path: {path}, permissions: {permissions}, expected_status: {expected_status}, status: {response.status}" finally: cascade_app_client.ds.config = previous_config @@ -2039,7 +2040,7 @@ async def test_databases_json_respects_view_database(tmp_path_factory): paths = [] for name in ("public", "private"): - path = str(db_directory / "{}.db".format(name)) + path = str(db_directory / f"{name}.db") conn = _sqlite3.connect(path) conn.execute("vacuum") conn.close() diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 59b1c0bf..734f0fc2 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,21 +1,3 @@ -from bs4 import BeautifulSoup as Soup -from .fixtures import ( - make_app_client, - TEMP_PLUGIN_SECRET_FILE, - PLUGINS_DIR, - TestClient as _TestClient, -) # noqa -from click.testing import CliRunner -from datasette.app import Datasette -from datasette import cli, hookimpl -from datasette.fixtures import TABLES -from datasette.filters import FilterArguments -from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm -from datasette.permissions import PermissionSQL, Action -from datasette.resources import DatabaseResource -from datasette.utils.sqlite import sqlite3 -from datasette.utils import StartupError, await_me_maybe -from jinja2 import ChoiceLoader, FileSystemLoader import base64 import datetime import importlib @@ -24,9 +6,32 @@ import os import pathlib import re import textwrap -import pytest import urllib +import pytest +from bs4 import BeautifulSoup as Soup +from click.testing import CliRunner +from jinja2 import ChoiceLoader, FileSystemLoader + +from datasette import cli, hookimpl +from datasette.app import Datasette +from datasette.filters import FilterArguments +from datasette.fixtures import TABLES +from datasette.permissions import Action, PermissionSQL +from datasette.plugins import DEFAULT_PLUGINS, get_plugins, pm +from datasette.resources import DatabaseResource +from datasette.utils import StartupError, await_me_maybe +from datasette.utils.sqlite import sqlite3 + +from .fixtures import ( + PLUGINS_DIR, + TEMP_PLUGIN_SECRET_FILE, + make_app_client, +) +from .fixtures import ( + TestClient as _TestClient, +) + at_memory_re = re.compile(r" at 0x\w+") @@ -35,7 +40,7 @@ at_memory_re = re.compile(r" at 0x\w+") ) def test_plugin_hooks_have_tests(plugin_hook): """Every plugin hook should be referenced in this test module""" - tests_in_this_module = [t for t in globals().keys() if t.startswith("test_hook_")] + tests_in_this_module = [t for t in globals() if t.startswith("test_hook_")] ok = False for test in tests_in_this_module: if plugin_hook in test: @@ -125,11 +130,11 @@ async def test_hook_extra_css_urls(ds_client, path, expected_decoded_object): response = await ds_client.get(path) assert response.status_code == 200 links = Soup(response.text, "html.parser").find_all("link") - special_href = [ + special_href = next( link for link in links if link.attrs["href"].endswith("/extra-css-urls-demo.css") - ][0]["href"] + )["href"] # This link has a base64-encoded JSON blob in it encoded = special_href.split("/")[3] actual_decoded_object = json.loads(base64.b64decode(encoded).decode("utf8")) @@ -152,7 +157,7 @@ async def test_hook_extra_js_urls(ds_client): "type": "module", }, ]: - assert any(s == attrs for s in script_attrs), "Expected: {}".format(attrs) + assert any(s == attrs for s in script_attrs), f"Expected: {attrs}" @pytest.mark.asyncio @@ -315,7 +320,8 @@ async def test_plugin_config_env_from_list(ds_client): @pytest.mark.asyncio async def test_plugin_config_file(ds_client): - with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: + # Blocking write is fine here - it is tiny test setup, not request handling + with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: # noqa: ASYNC230 fp.write("FROM_FILE") assert {"foo": "FROM_FILE"} == ds_client.ds.plugin_config("file-plugin") os.remove(TEMP_PLUGIN_SECRET_FILE) @@ -823,7 +829,7 @@ def test_hook_register_routes_with_datasette(configured_path): assert response.status_code == 200 assert configured_path.upper() == response.text # Other one should 404 - other_path = [p for p in ("path1", "path2") if configured_path != p][0] + other_path = next(p for p in ("path1", "path2") if configured_path != p) assert client.get(f"/{other_path}/", follow_redirects=True).status_code == 404 @@ -928,7 +934,7 @@ async def test_plugin_startup_can_add_queries(): await datasette.add_query( "data", "from_startup", - "select {}".format(result.first()[0]), + f"select {result.first()[0]}", source="plugin", ) @@ -1040,7 +1046,7 @@ async def test_hook_handle_exception(ds_client): @pytest.mark.asyncio @pytest.mark.parametrize("param", ("_custom_error", "_custom_error_async")) async def test_hook_handle_exception_custom_response(ds_client, param): - response = await ds_client.get("/trigger-error?{}=1".format(param)) + response = await ds_client.get(f"/trigger-error?{param}=1") assert response.text == param @@ -1374,7 +1380,7 @@ async def test_hook_register_actions_no_duplicates(duplicate): # This should error: with pytest.raises(StartupError) as ex: await ds.invoke_startup() - assert "Duplicate action {}".format(duplicate) in str(ex.value) + assert f"Duplicate action {duplicate}" in str(ex.value) @pytest.mark.asyncio diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 6617bc77..aebcaa33 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -1,10 +1,12 @@ -from click.testing import CliRunner -from datasette import cli -from unittest import mock import json import os -import pytest import textwrap +from unittest import mock + +import pytest +from click.testing import CliRunner + +from datasette import cli @pytest.mark.serial @@ -70,9 +72,7 @@ def test_publish_cloudrun_prompts_for_service( ), mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( - "gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service --max-instances 1".format( - tag - ), + f"gcloud run deploy --allow-unauthenticated --platform=managed --image {tag} input-service --max-instances 1", shell=True, ), ] @@ -107,9 +107,7 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory): ), mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( - "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test --max-instances 1".format( - tag - ), + f"gcloud run deploy --allow-unauthenticated --platform=managed --image {tag} test --max-instances 1", shell=True, ), ] @@ -186,13 +184,13 @@ def test_publish_cloudrun_memory_cpu( tag = f"us-docker.pkg.dev/{mock_output.return_value}/datasette/datasette-test" expected_call = ( "gcloud run deploy --allow-unauthenticated --platform=managed" - " --image {} test".format(tag) + f" --image {tag} test" ) expected_build_call = f"gcloud builds submit --tag {tag}" if memory: - expected_call += " --memory {}".format(memory) + expected_call += f" --memory {memory}" if cpu: - expected_call += " --cpu {}".format(cpu) + expected_call += f" --cpu {cpu}" if timeout: expected_build_call += f" --timeout {timeout}" # max_instances defaults to 1 diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index cab83654..4302ed94 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -1,9 +1,11 @@ -from click.testing import CliRunner -from datasette import cli -from unittest import mock import os import pathlib +from unittest import mock + import pytest +from click.testing import CliRunner + +from datasette import cli @pytest.mark.serial diff --git a/tests/test_pytest_autoclose_plugin.py b/tests/test_pytest_autoclose_plugin.py index 3af1aace..9b17d24b 100644 --- a/tests/test_pytest_autoclose_plugin.py +++ b/tests/test_pytest_autoclose_plugin.py @@ -20,6 +20,7 @@ def _run_pytest(tmp_path: Path) -> subprocess.CompletedProcess: cwd=str(tmp_path), capture_output=True, text=True, + check=False, ) diff --git a/tests/test_queries.py b/tests/test_queries.py index ffa948a9..15b7ad0f 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -15,37 +15,29 @@ from datasette.utils.sqlite import sqlite3, supports_returning requires_sqlite_returning = pytest.mark.skipif( not supports_returning(), reason="SQLite does not support RETURNING" ) -EXPECTED_CREATE_TABLE_TEMPLATE_SQL = "\n".join( - ( - "create table new_table (", - " id integer primary key,", - " name text", - " -- created text default (datetime('now'))", - ")", - ) -) +EXPECTED_CREATE_TABLE_TEMPLATE_SQL = "create table new_table (\n id integer primary key,\n name text\n -- created text default (datetime('now'))\n)" def _template_option_attributes(html, table): - match = re.search(r'
'.format( - i, i - ), + f'', f'', f'', f'', @@ -1667,7 +1668,7 @@ async def test_row_update_sets_message(): assert response.status_code == 200 assert response.json()["rows"][0]["name"] == long_name assert ds.unsign(response.cookies["ds_messages"], "messages") == [ - ["Updated row 1 ({})".format(truncated_name), ds.INFO] + [f"Updated row 1 ({truncated_name})", ds.INFO] ] finally: ds.close() @@ -1680,9 +1681,9 @@ def test_table_data_uses_base_url(app_client_base_url_prefix): import re soup = Soup(response.text, "html.parser") - table_script = [ + table_script = next( s for s in soup.find_all("script") if "_datasetteTableData" in (s.string or "") - ][0] + ) match = re.search( r"window\._datasetteTableData\s*=\s*({.*?});", table_script.string, @@ -1710,16 +1711,17 @@ def test_table_fragment_custom_table_include(): @pytest.mark.asyncio async def test_table_fragment_uses_render_cell_hook(): - from datasette import hookimpl from markupsafe import Markup + from datasette import hookimpl + class TestRenderCellPlugin: __name__ = "TestRenderCellPlugin" @hookimpl def render_cell(self, value, column, table, database): if database == "data" and table == "items" and column == "name": - return Markup("{}".format(value)) + return Markup(f"{value}") return None ds = Datasette(memory=True) @@ -2258,18 +2260,16 @@ def test_allow_facet_off(allow_facet): ) async def test_format_of_binary_links(size, title, length_bytes): ds = Datasette() - db_name = "binary-links-{}".format(size) + db_name = f"binary-links-{size}" db = ds.add_memory_database(db_name) - sql = "select zeroblob({}) as blob".format(size) - await db.execute_write("create table blobs as {}".format(sql)) - response = await ds.client.get("/{}/blobs".format(db_name)) + sql = f"select zeroblob({size}) as blob" + await db.execute_write(f"create table blobs as {sql}") + response = await ds.client.get(f"/{db_name}/blobs") assert response.status_code == 200 - expected = "{}><Binary: {} bytes>".format(title, length_bytes) + expected = f"{title}><Binary: {length_bytes} bytes>" assert expected in response.text # And test with arbitrary SQL query too - sql_response = await ds.client.get( - "{}/-/query".format(db_name), params={"sql": sql} - ) + sql_response = await ds.client.get(f"{db_name}/-/query", params={"sql": sql}) assert sql_response.status_code == 200 assert expected in sql_response.text diff --git a/tests/test_template_context.py b/tests/test_template_context.py index 7923d0e7..691c2d64 100644 --- a/tests/test_template_context.py +++ b/tests/test_template_context.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field import pytest -from datasette.app import Datasette, TEMPLATE_BASE_CONTEXT +from datasette.app import TEMPLATE_BASE_CONTEXT, Datasette from datasette.extras import ExtraScope from datasette.fixtures import write_fixture_database from datasette.template_contexts import PAGES, documented_context_keys @@ -40,17 +40,17 @@ def test_documented_fields(): @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) def test_context_class_fields_all_have_help(klass): for context_field in klass.documented_fields(): - assert context_field.help, "{}.{} is missing documentation".format( - klass.__name__, context_field.name - ) + assert ( + context_field.help + ), f"{klass.__name__}.{context_field.name} is missing documentation" @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__) def test_context_class_has_docstring_and_documented_template(klass): - assert klass.__doc__, "{} is missing a docstring".format(klass.__name__) - assert klass.documented_template, "{} is missing a documented_template".format( - klass.__name__ - ) + assert klass.__doc__, f"{klass.__name__} is missing a docstring" + assert ( + klass.documented_template + ), f"{klass.__name__} is missing a documented_template" def test_from_extra_documentation_comes_from_the_extra_class(): @@ -105,7 +105,7 @@ def isolate_extra_template_vars_plugins(): # for the rest of the process. The contract documents plugin-free # Datasette core, so unregister any non-default plugin that adds # template variables via the extra_template_vars hook - from datasette.plugins import pm, DEFAULT_PLUGINS + from datasette.plugins import DEFAULT_PLUGINS, pm hook_plugins = {impl.plugin for impl in pm.hook.extra_template_vars.get_hookimpls()} removed = [] @@ -182,18 +182,18 @@ async def test_template_context_matches_documented_contract( undocumented = actual - documented no_longer_present = documented - actual assert not undocumented, ( - "Undocumented keys in {} template context: {} - add them to the " - "page's Context class".format(page_name, sorted(undocumented)) + f"Undocumented keys in {page_name} template context: {sorted(undocumented)} - add them to the " + "page's Context class" ) assert not no_longer_present, ( - "Documented keys missing from {} template context: {} - this would " - "break custom templates".format(page_name, sorted(no_longer_present)) + f"Documented keys missing from {page_name} template context: {sorted(no_longer_present)} - this would " + "break custom templates" ) def test_base_context_keys_all_have_docs(): for name, doc in TEMPLATE_BASE_CONTEXT.items(): - assert doc, "Base context key {} is missing docs".format(name) + assert doc, f"Base context key {name} is missing docs" def test_template_context_docs_cover_every_documented_key(): @@ -201,15 +201,14 @@ def test_template_context_docs_cover_every_documented_key(): assert docs_path.exists(), "docs/template_context.rst is missing" docs = docs_path.read_text() for name in TEMPLATE_BASE_CONTEXT: - assert "``{}``".format(name) in docs, name + assert f"``{name}``" in docs, name for page_name, klass in PAGES.items(): title = "{} page".format(klass.__name__.removesuffix("Context")) assert title in docs, title for context_field in klass.documented_fields(): - assert "``{}``".format(context_field.name) in docs, "{} ({} page)".format( - context_field.name, page_name - ) assert ( - "``{}`` - ``{}``".format(context_field.name, context_field.type_name) - in docs - ), "{} type ({} page)".format(context_field.name, page_name) + f"``{context_field.name}``" in docs + ), f"{context_field.name} ({page_name} page)" + assert ( + f"``{context_field.name}`` - ``{context_field.type_name}``" in docs + ), f"{context_field.name} type ({page_name} page)" diff --git a/tests/test_token_handler.py b/tests/test_token_handler.py index f5bbfead..10021ddf 100644 --- a/tests/test_token_handler.py +++ b/tests/test_token_handler.py @@ -2,16 +2,17 @@ Tests for the register_token_handler plugin hook. """ +import pytest + from datasette.app import Datasette from datasette.hookspecs import hookimpl from datasette.plugins import pm from datasette.tokens import ( + SignedTokenHandler, TokenHandler, TokenInvalid, TokenRestrictions, - SignedTokenHandler, ) -import pytest @pytest.fixture diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 9db211d3..21cfa952 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -1,4 +1,5 @@ import pytest + from .fixtures import make_app_client @@ -75,10 +76,9 @@ async def test_trace_child_tasks_resets_contextvar_on_exception(): from datasette import tracer before = tracer.trace_task_id.get() - with pytest.raises(ValueError): - with tracer.trace_child_tasks(): - assert tracer.trace_task_id.get() is not None - raise ValueError("simulated error") + with pytest.raises(ValueError), tracer.trace_child_tasks(): + assert tracer.trace_task_id.get() is not None + raise ValueError("simulated error") # The contextvar must be reset even though the block raised assert tracer.trace_task_id.get() == before diff --git a/tests/test_utils.py b/tests/test_utils.py index a535ca93..1808b3cf 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,8 +2,17 @@ Tests for various datasette helper functions. """ -from datasette.app import Datasette +import hashlib +import json +import os +import pathlib +import tempfile +from unittest.mock import patch + +import pytest + from datasette import utils +from datasette.app import Datasette from datasette.utils.asgi import Request from datasette.utils.sqlite import ( sqlite3, @@ -11,13 +20,6 @@ from datasette.utils.sqlite import ( sqlite_table_type, supports_returning, ) -import hashlib -import json -import os -import pathlib -import pytest -import tempfile -from unittest.mock import patch @pytest.mark.parametrize( @@ -194,7 +196,7 @@ def test_validate_sql_select_good(good_sql): @pytest.mark.parametrize("open_quote,close_quote", [('"', '"'), ("[", "]")]) def test_detect_fts(open_quote, close_quote): - sql = """ + sql = f""" CREATE TABLE "Dumb_Table" ( "TreeID" INTEGER, "qSpecies" TEXT @@ -209,9 +211,9 @@ def test_detect_fts(open_quote, close_quote): "qCaretaker" TEXT ); CREATE VIEW Test_View AS SELECT * FROM Dumb_Table; - CREATE VIRTUAL TABLE {open}Street_Tree_List_fts{close} USING FTS4 ("qAddress", "qCaretaker", "qSpecies", content={open}Street_Tree_List{close}); + CREATE VIRTUAL TABLE {open_quote}Street_Tree_List_fts{close_quote} USING FTS4 ("qAddress", "qCaretaker", "qSpecies", content={open_quote}Street_Tree_List{close_quote}); CREATE VIRTUAL TABLE r USING rtree(a, b, c); - """.format(open=open_quote, close=close_quote) + """ conn = utils.sqlite3.connect(":memory:") conn.executescript(sql) assert None is utils.detect_fts(conn, "Dumb_Table") @@ -262,8 +264,8 @@ def test_escape_sqlite_prevents_injection(): conn.execute("CREATE TABLE users (id INTEGER, password TEXT)") conn.execute("INSERT INTO users VALUES (1, 'super_secret_password')") malicious = "users] UNION SELECT password FROM users--" - conn.execute('CREATE TABLE "{}" (id INTEGER)'.format(malicious)) - sql = "select count(*) from {}".format(utils.escape_sqlite(malicious)) + conn.execute(f'CREATE TABLE "{malicious}" (id INTEGER)') + sql = f"select count(*) from {utils.escape_sqlite(malicious)}" results = conn.execute(sql).fetchall() conn.close() # The injected UNION must not execute - only the empty malicious table @@ -273,16 +275,16 @@ def test_escape_sqlite_prevents_injection(): @pytest.mark.parametrize("table", ("regular", "has'single quote")) def test_detect_fts_different_table_names(table): - sql = """ + sql = f""" CREATE TABLE [{table}] ( "TreeID" INTEGER, "qSpecies" TEXT ); CREATE VIRTUAL TABLE [{table}_fts] USING FTS4 ("qSpecies", content="{table}"); - """.format(table=table) + """ conn = utils.sqlite3.connect(":memory:") conn.executescript(sql) - assert "{table}_fts".format(table=table) == utils.detect_fts(conn, table) + assert f"{table}_fts" == utils.detect_fts(conn, table) conn.close() @@ -690,7 +692,6 @@ def test_resolve_env_secrets(config, expected): [ ({"id": "blah"}, "blah"), ({"id": "blah", "login": "l"}, "l"), - ({"id": "blah", "login": "l"}, "l"), ({"id": "blah", "login": "l", "username": "u"}, "u"), ({"login": "l", "name": "n"}, "n"), ( diff --git a/tests/test_utils_check_callable.py b/tests/test_utils_check_callable.py index 4f72f9ff..857b73cd 100644 --- a/tests/test_utils_check_callable.py +++ b/tests/test_utils_check_callable.py @@ -1,6 +1,7 @@ -from datasette.utils.check_callable import check_callable import pytest +from datasette.utils.check_callable import check_callable + class AsyncClass: async def __call__(self): diff --git a/tests/test_utils_permissions.py b/tests/test_utils_permissions.py index bc3599c2..918dab95 100644 --- a/tests/test_utils_permissions.py +++ b/tests/test_utils_permissions.py @@ -1,14 +1,17 @@ +from collections.abc import Callable + import pytest + from datasette.app import Datasette from datasette.permissions import PermissionSQL from datasette.utils.permissions import resolve_permissions_from_catalog -from typing import Callable, List @pytest.fixture def db(): ds = Datasette() import tempfile + from datasette.database import Database path = tempfile.mktemp(suffix="demo.db") @@ -127,7 +130,7 @@ def plugin_root_deny_for_all() -> Callable[[str], PermissionSQL]: def plugin_conflicting_same_child_rules( user: str, parent: str, child: str -) -> List[Callable[[str], PermissionSQL]]: +) -> list[Callable[[str], PermissionSQL]]: def allow_provider(action: str) -> PermissionSQL: return PermissionSQL( """ @@ -277,9 +280,7 @@ async def test_alice_global_allow_with_specific_denies_catalog(db): # Alice can see everything except accounting/sales and hr/* assert "/accounting/sales" in res_denied(rows) for r in rows: - if r["parent"] == "hr": - assert r["allow"] == 0 - elif r["resource"] == "/accounting/sales": + if r["parent"] == "hr" or r["resource"] == "/accounting/sales": assert r["allow"] == 0 else: assert r["allow"] == 1 diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index 979ff9e1..a6f95e5b 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -1,7 +1,7 @@ import pytest -from datasette.utils.sqlite import sqlite3 from datasette.utils.sql_analysis import analyze_sql_tables +from datasette.utils.sqlite import sqlite3 @pytest.fixture diff --git a/tests/test_write_wrapper.py b/tests/test_write_wrapper.py index 88ce5520..66599c54 100644 --- a/tests/test_write_wrapper.py +++ b/tests/test_write_wrapper.py @@ -3,14 +3,16 @@ Tests for the write_wrapper plugin hook. """ import asyncio +import sqlite3 +import time from dataclasses import dataclass + +import pytest + from datasette.app import Datasette from datasette.events import Event from datasette.hookspecs import hookimpl from datasette.plugins import pm -import pytest -import sqlite3 -import time @dataclass @@ -113,7 +115,8 @@ async def test_write_wrapper_exception_thrown_into_generator(datasette): def wrapper(conn): try: yield - except Exception as e: + except Exception as e: # noqa: BLE001 + # Test helper deliberately captures whatever the wrapped write raised caught["error"] = e return wrapper @@ -232,7 +235,6 @@ async def test_write_wrapper_return_none_skips(datasette): @hookimpl def write_wrapper(datasette, database, request, transaction): log.append("hook-called") - return None pm.register(Plugin(), name="test_skip") try: @@ -339,7 +341,7 @@ async def test_write_wrapper_via_api(tmp_path): "/test/api_test/-/insert", json={"row": {"name": "test"}, "return": True}, headers={ - "Authorization": "Bearer {}".format(token), + "Authorization": f"Bearer {token}", "Content-Type": "application/json", }, ) @@ -466,7 +468,7 @@ async def test_write_wrapper_set_authorizer(datasette, actor, table, should_deny try: request = FakeRequest(actor) if should_deny: - with pytest.raises(Exception): + with pytest.raises(sqlite3.DatabaseError, match="not authorized"): await db.execute_write_fn( lambda conn: conn.execute( f"insert into {table} (value) values ('test')"
{}{i}{i}{i}a{i}