diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 7d8dd37d..b0640ae8 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -57,7 +57,7 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - - name: And the counters writable canned query demo + - name: And the counters writable stored query demo run: | cat > plugins/counters.py < Environment: environment = self._jinja_env if request: @@ -731,6 +751,7 @@ class Datasette: await await_me_maybe(hook) # Ensure internal tables and metadata are populated before startup hooks await self._refresh_schemas() + await self._save_queries_from_config() # Load column_types from config into internal DB await self._apply_column_types_config() for hook in pm.hook.startup(datasette=self): @@ -1007,6 +1028,179 @@ class Datasette: [database_name, resource_name, column_name, key, value], ) + @staticmethod + def _query_row_to_stored_query(row) -> stored_queries.StoredQuery | None: + return stored_queries.query_row_to_stored_query(row) + + @staticmethod + def _query_options_json(options): + return stored_queries.query_options_json(options) + + async def add_query( + self, + database: str, + name: str, + sql: str, + *, + title: str | None = None, + description: str | None = None, + description_html: str | None = None, + hide_sql: bool = False, + fragment: str | None = None, + parameters: Iterable[str] | None = None, + is_write: bool = False, + is_private: bool = False, + is_trusted: bool = False, + source: str = "plugin", + owner_id: str | None = None, + on_success_message: str | None = None, + on_success_message_sql: str | None = None, + on_success_redirect: str | None = None, + on_error_message: str | None = None, + on_error_redirect: str | None = None, + replace: bool = True, + ) -> None: + return await stored_queries.add_query( + self, + database, + name, + sql, + title=title, + description=description, + description_html=description_html, + hide_sql=hide_sql, + fragment=fragment, + parameters=parameters, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + on_success_message=on_success_message, + on_success_message_sql=on_success_message_sql, + on_success_redirect=on_success_redirect, + on_error_message=on_error_message, + on_error_redirect=on_error_redirect, + replace=replace, + ) + + async def update_query( + self, + database: str, + name: str, + *, + sql=stored_queries.UNCHANGED, + title=stored_queries.UNCHANGED, + description=stored_queries.UNCHANGED, + description_html=stored_queries.UNCHANGED, + hide_sql=stored_queries.UNCHANGED, + fragment=stored_queries.UNCHANGED, + parameters=stored_queries.UNCHANGED, + is_write=stored_queries.UNCHANGED, + is_private=stored_queries.UNCHANGED, + is_trusted=stored_queries.UNCHANGED, + source=stored_queries.UNCHANGED, + owner_id=stored_queries.UNCHANGED, + on_success_message=stored_queries.UNCHANGED, + on_success_message_sql=stored_queries.UNCHANGED, + on_success_redirect=stored_queries.UNCHANGED, + on_error_message=stored_queries.UNCHANGED, + on_error_redirect=stored_queries.UNCHANGED, + ) -> None: + return await stored_queries.update_query( + self, + database, + name, + sql=sql, + title=title, + description=description, + description_html=description_html, + hide_sql=hide_sql, + fragment=fragment, + parameters=parameters, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + on_success_message=on_success_message, + on_success_message_sql=on_success_message_sql, + on_success_redirect=on_success_redirect, + on_error_message=on_error_message, + on_error_redirect=on_error_redirect, + ) + + async def remove_query( + self, database: str, name: str, source: str | None = None + ) -> None: + return await stored_queries.remove_query(self, database, name, source=source) + + async def get_query( + self, database: str, name: str + ) -> stored_queries.StoredQuery | None: + return await stored_queries.get_query(self, database, name) + + async def count_queries( + self, + database: str | None = None, + *, + actor: dict[str, Any] | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + ) -> int: + return await stored_queries.count_queries( + self, + database, + actor=actor, + q=q, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + ) + + async def list_queries( + self, + database: str | None = None, + *, + actor: dict[str, Any] | None = None, + limit: int = 50, + cursor: str | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + include_private: bool = False, + ) -> stored_queries.StoredQueryPage: + return await stored_queries.list_queries( + self, + database, + actor=actor, + limit=limit, + cursor=cursor, + q=q, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + include_private=include_private, + ) + + async def ensure_query_write_permissions( + self, database, sql, *, actor=None, params=None, analysis=None + ): + return await stored_queries.ensure_query_write_permissions( + self, database, sql, actor=actor, params=params, analysis=analysis + ) + # Column types API async def _get_resource_column_details(self, database: str, resource: str): @@ -1238,29 +1432,6 @@ class Datasette: def app_css_hash(self): return self.static_hash("app.css") - async def get_canned_queries(self, database_name, actor): - queries = {} - for more_queries in pm.hook.canned_queries( - datasette=self, - database=database_name, - actor=actor, - ): - more_queries = await await_me_maybe(more_queries) - queries.update(more_queries or {}) - # Fix any {"name": "select ..."} queries to be {"name": {"sql": "select ..."}} - for key in queries: - if not isinstance(queries[key], dict): - queries[key] = {"sql": queries[key]} - # Also make sure "name" is available: - queries[key]["name"] = key - return queries - - async def get_canned_query(self, database_name, query_name, actor): - queries = await self.get_canned_queries(database_name, actor) - query = queries.get(query_name) - if query: - return query - def _prepare_connection(self, conn, database): conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") @@ -2236,6 +2407,10 @@ class Datasette: JumpView.as_view(self), r"/-/jump(\.(?Pjson))?$", ) + add_route( + GlobalQueryListView.as_view(self), + r"/-/queries(\.(?Pjson))?$", + ) add_route( InstanceSchemaView.as_view(self), r"/-/schema(\.(?Pjson|md))?$", @@ -2281,14 +2456,50 @@ class Datasette: r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") + add_route( + QueryListView.as_view(self), + r"/(?P[^\/\.]+)/-/queries(\.(?Pjson))?$", + ) + add_route( + QueryCreateAnalyzeView.as_view(self), + r"/(?P[^\/\.]+)/-/queries/analyze$", + ) + add_route( + QueryStoreView.as_view(self), + r"/(?P[^\/\.]+)/-/queries/store$", + ) + add_route( + ExecuteWriteAnalyzeView.as_view(self), + r"/(?P[^\/\.]+)/-/execute-write/analyze$", + ) + add_route( + ExecuteWriteView.as_view(self), + r"/(?P[^\/\.]+)/-/execute-write$", + ) add_route( DatabaseSchemaView.as_view(self), r"/(?P[^\/\.]+)/-/schema(\.(?Pjson|md))?$", ) + add_route( + QueryParametersView.as_view(self), + r"/(?P[^\/\.]+)/-/query/parameters$", + ) add_route( wrap_view(QueryView, self), r"/(?P[^\/\.]+)/-/query(\.(?P\w+))?$", ) + add_route( + QueryDefinitionView.as_view(self), + r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/definition$", + ) + add_route( + QueryUpdateView.as_view(self), + r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/update$", + ) + add_route( + QueryDeleteView.as_view(self), + r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/delete$", + ) add_route( wrap_view(table_view, self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)(\.(?P\w+))?$", diff --git a/datasette/database.py b/datasette/database.py index 66d50ffa..e7e9527e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -25,6 +25,7 @@ from .utils import ( table_columns, table_column_details, ) +from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables from .utils.sqlite import sqlite_version from .inspect import inspect_hash @@ -301,6 +302,13 @@ class Database: # Threaded mode - send to write thread return await self._send_to_write_thread(fn, isolated_connection=True) + async def analyze_sql(self, sql, params=None) -> SQLAnalysis: + self._check_not_closed() + + return await self.execute_isolated_fn( + lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name) + ) + async def execute_write_fn(self, fn, block=True, transaction=True, request=None): self._check_not_closed() pending_events = [] diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 149a4e5f..2f78570b 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -48,12 +48,26 @@ def register_actions(): resource_class=DatabaseResource, also_requires="view-database", ), + Action( + name="execute-write-sql", + abbr="ews", + description="Execute writable SQL queries", + resource_class=DatabaseResource, + also_requires="view-database", + ), Action( name="create-table", abbr="ct", description="Create tables", resource_class=DatabaseResource, ), + Action( + name="store-query", + abbr="sq", + description="Create stored queries", + resource_class=DatabaseResource, + also_requires="execute-sql", + ), # Table-level actions (child-level) Action( name="view-table", @@ -104,4 +118,16 @@ def register_actions(): description="View named query results", resource_class=QueryResource, ), + Action( + name="update-query", + abbr="uq", + description="Update stored queries", + resource_class=QueryResource, + ), + Action( + name="delete-query", + abbr="dq", + description="Delete stored queries", + resource_class=QueryResource, + ), ) diff --git a/datasette/default_database_actions.py b/datasette/default_database_actions.py new file mode 100644 index 00000000..e0cb3cdf --- /dev/null +++ b/datasette/default_database_actions.py @@ -0,0 +1,24 @@ +from datasette import hookimpl +from datasette.resources import DatabaseResource + + +@hookimpl +def database_actions(datasette, actor, database, request): + async def inner(): + if not datasette.get_database(database).is_mutable: + return [] + if not await datasette.allowed( + action="execute-write-sql", + resource=DatabaseResource(database), + actor=actor, + ): + return [] + return [ + { + "href": datasette.urls.database(database) + "/-/execute-write", + "label": "Execute write SQL", + "description": "Run writable SQL with table permission checks.", + } + ] + + return inner diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index 9e3bb648..6cd46f04 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -17,13 +17,6 @@ UNION/INTERSECT operations. The order of evaluation is: from __future__ import annotations -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette import hookimpl - # Re-export all hooks and public utilities from .restrictions import ( actor_restrictions_sql as actor_restrictions_sql, @@ -33,16 +26,9 @@ from .restrictions import ( from .root import root_user_permissions_sql as root_user_permissions_sql from .config import config_permissions_sql as config_permissions_sql from .defaults import ( + # Avoid "datasette.default_permissions" does not explicitly export attribute default_allow_sql_check as default_allow_sql_check, default_action_permissions_sql as default_action_permissions_sql, + default_query_permissions_sql as default_query_permissions_sql, DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, ) - - -@hookimpl -def canned_queries(datasette: "Datasette", database: str, actor) -> dict: - """Return canned queries defined in datasette.yaml configuration.""" - queries = ( - ((datasette.config or {}).get("databases") or {}).get(database) or {} - ).get("queries") or {} - return queries diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 4c74219d..5bc74425 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -67,3 +67,48 @@ async def default_action_permissions_sql( return PermissionSQL.allow(reason=reason) return None + + +@hookimpl(specname="permission_resources_sql") +async def default_query_permissions_sql( + datasette: "Datasette", + actor: Optional[dict], + action: str, +) -> Optional[PermissionSQL]: + actor_id = actor.get("id") if isinstance(actor, dict) else None + + if action not in {"view-query", "update-query", "delete-query"}: + return None + + params = {"query_owner_id": actor_id} + rule_sqls = [] + if actor_id is not None: + if action in {"update-query", "delete-query"}: + # Query owner can update/delete query + rule_sqls.append(""" + SELECT database_name AS parent, name AS child, 1 AS allow, + 'query owner' AS reason + FROM queries + WHERE source = 'user' + AND owner_id = :query_owner_id + """) + else: + # Query owner can view-query + rule_sqls.append(""" + SELECT database_name AS parent, name AS child, 1 AS allow, + 'query owner' AS reason + FROM queries + WHERE owner_id = :query_owner_id + """) + + # restriction_sql enforces private queries ONLY visible/mutable by owner + return PermissionSQL( + sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, + restriction_sql=""" + SELECT database_name AS parent, name AS child + FROM queries + WHERE is_private = 0 + OR owner_id = :query_owner_id + """, + params=params, + ) diff --git a/datasette/facets.py b/datasette/facets.py index bc4b6904..abe0605e 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -83,7 +83,7 @@ class Facet: self.ds = ds self.request = request self.database = database - # For foreign key expansion. Can be None for e.g. canned SQL queries: + # For foreign key expansion. Can be None for e.g. stored SQL queries: self.table = table self.sql = sql or f"select * from [{table}]" self.params = params or [] diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index cf95abcb..dcd502af 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -137,11 +137,6 @@ def permission_resources_sql(datasette, actor, action): """ -@hookspec -def canned_queries(datasette, database, actor): - """Return a dictionary of canned query definitions or an awaitable function that returns them""" - - @hookspec def register_magic_parameters(datasette): """Return a list of (name, function) magic parameter functions""" @@ -179,7 +174,7 @@ def view_actions(datasette, actor, database, view, request): @hookspec def query_actions(datasette, actor, database, query_name, request, sql, params): - """Links for the query and canned query actions menu""" + """Links for the query and stored query actions menu""" @hookspec @@ -233,8 +228,8 @@ def top_query(datasette, request, database, sql): @hookspec -def top_canned_query(datasette, request, database, query_name): - """HTML to include at the top of the canned query page""" +def top_stored_query(datasette, request, database, query_name): + """HTML to include at the top of the stored query page""" @hookspec diff --git a/datasette/plugins.py b/datasette/plugins.py index f532ac60..5a31cdad 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -30,6 +30,7 @@ DEFAULT_PLUGINS = ( "datasette.blob_renderer", "datasette.default_debug_menu", "datasette.default_jump_items", + "datasette.default_database_actions", "datasette.handle_exception", "datasette.forbidden", "datasette.events", diff --git a/datasette/resources.py b/datasette/resources.py index 236b3598..ee2e6d98 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -41,7 +41,7 @@ class TableResource(Resource): class QueryResource(Resource): - """A canned query in a database.""" + """A stored query in a database.""" name = "query" parent_class = DatabaseResource @@ -51,42 +51,8 @@ class QueryResource(Resource): @classmethod async def resources_sql(cls, datasette, actor=None) -> str: - from datasette.plugins import pm - from datasette.utils import await_me_maybe - - # Get all databases from catalog - db = datasette.get_internal_database() - result = await db.execute("SELECT database_name FROM catalog_databases") - databases = [row[0] for row in result.rows] - - # Gather canned queries for this actor from all databases. - # This keeps allowed_resources("view-query", actor=...) consistent with - # actor-specific canned_queries() implementations. - query_pairs = [] - for database_name in databases: - # Call the hook to get queries (including from config via default plugin) - for queries_result in pm.hook.canned_queries( - datasette=datasette, - database=database_name, - actor=actor, - ): - queries = await await_me_maybe(queries_result) - if queries: - for query_name in queries.keys(): - query_pairs.append((database_name, query_name)) - - # Build SQL - if not query_pairs: - return "SELECT NULL AS parent, NULL AS child WHERE 0" - - # Generate UNION ALL query - selects = [] - for db_name, query_name in query_pairs: - # Escape single quotes by doubling them - db_escaped = db_name.replace("'", "''") - query_escaped = query_name.replace("'", "''") - selects.append( - f"SELECT '{db_escaped}' AS parent, '{query_escaped}' AS child" - ) - - return " UNION ALL ".join(selects) + return """ + SELECT q.database_name AS parent, q.name AS child + FROM queries q + JOIN catalog_databases cd ON cd.database_name = q.database_name + """ diff --git a/datasette/static/app.css b/datasette/static/app.css index c21d0dc4..815f6db8 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1409,11 +1409,15 @@ svg.dropdown-menu-icon { border-bottom: 5px solid #666; } -.canned-query-edit-sql { +.stored-query-edit-sql { padding-left: 0.5em; position: relative; top: 1px; } +.save-query { + display: inline-block; + margin-left: 0.45em; +} .blob-download { display: block; diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py new file mode 100644 index 00000000..bcfdfdb4 --- /dev/null +++ b/datasette/stored_queries.py @@ -0,0 +1,623 @@ +from __future__ import annotations + +from dataclasses import dataclass +import json +from typing import Any, Iterable + +from .resources import TableResource +from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components +from .utils.asgi import Forbidden + +UNCHANGED = object() + +QUERY_OPTION_FIELDS = ( + "hide_sql", + "fragment", + "on_success_message", + "on_success_message_sql", + "on_success_redirect", + "on_error_message", + "on_error_redirect", +) + + +@dataclass +class StoredQuery: + database: str + name: str + sql: str + title: str | None + description: str | None + description_html: str | None + hide_sql: bool + fragment: str | None + parameters: list[str] + is_write: bool + is_private: bool + is_trusted: bool + source: str + owner_id: str | None + on_success_message: str | None + on_success_message_sql: str | None + on_success_redirect: str | None + on_error_message: str | None + on_error_redirect: str | None + private: bool | None = None + + +@dataclass +class StoredQueryPage: + queries: list[StoredQuery] + next: str | None + has_more: bool + limit: int + + +def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]: + data = { + "database": query.database, + "name": query.name, + "sql": query.sql, + "title": query.title, + "description": query.description, + "description_html": query.description_html, + "hide_sql": query.hide_sql, + "fragment": query.fragment, + "params": list(query.parameters), + "parameters": list(query.parameters), + "is_write": query.is_write, + "is_private": query.is_private, + "is_trusted": query.is_trusted, + "source": query.source, + "owner_id": query.owner_id, + "on_success_message": query.on_success_message, + "on_success_message_sql": query.on_success_message_sql, + "on_success_redirect": query.on_success_redirect, + "on_error_message": query.on_error_message, + "on_error_redirect": query.on_error_redirect, + } + if query.private is not None: + data["private"] = query.private + return data + + +def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]: + return { + "queries": [stored_query_to_dict(query) for query in page.queries], + "next": page.next, + "has_more": page.has_more, + "limit": page.limit, + } + + +async def save_queries_from_config(datasette: Any) -> None: + # Apply configured query entries from datasette.yaml to the internal table. + await datasette.get_internal_database().execute_write( + "DELETE FROM queries WHERE source = 'config'" + ) + for dbname, db_config in ((datasette.config or {}).get("databases") or {}).items(): + for query_name, query_config in (db_config.get("queries") or {}).items(): + if not isinstance(query_config, dict): + query_config = {"sql": query_config} + await datasette.add_query( + dbname, + query_name, + query_config["sql"], + title=query_config.get("title"), + description=query_config.get("description"), + description_html=query_config.get("description_html"), + hide_sql=bool(query_config.get("hide_sql")), + fragment=query_config.get("fragment"), + parameters=query_config.get("params"), + is_write=bool(query_config.get("write")), + is_private=bool(query_config.get("is_private")), + is_trusted=bool(query_config.get("is_trusted", True)), + source="config", + on_success_message=query_config.get("on_success_message"), + on_success_message_sql=query_config.get("on_success_message_sql"), + on_success_redirect=query_config.get("on_success_redirect"), + on_error_message=query_config.get("on_error_message"), + on_error_redirect=query_config.get("on_error_redirect"), + ) + + +def query_row_to_stored_query( + row: Any, private: bool | None = None +) -> StoredQuery | None: + if row is None: + return None + parameters = json.loads(row["parameters"] or "[]") + options = json.loads(row["options"] or "{}") + return StoredQuery( + database=row["database_name"], + name=row["name"], + sql=row["sql"], + title=row["title"], + description=row["description"], + description_html=row["description_html"], + hide_sql=bool(options.get("hide_sql")), + fragment=options.get("fragment"), + parameters=parameters, + is_write=bool(row["is_write"]), + is_private=bool(row["is_private"]), + is_trusted=bool(row["is_trusted"]), + source=row["source"], + owner_id=row["owner_id"], + on_success_message=options.get("on_success_message"), + on_success_message_sql=options.get("on_success_message_sql"), + on_success_redirect=options.get("on_success_redirect"), + on_error_message=options.get("on_error_message"), + on_error_redirect=options.get("on_error_redirect"), + private=private, + ) + + +def query_options_json(options: dict[str, Any]) -> str: + options_dict = {} + for field in QUERY_OPTION_FIELDS: + value = options.get(field) + if field == "hide_sql": + if value: + options_dict[field] = True + elif value is not None: + options_dict[field] = value + return json.dumps(options_dict, sort_keys=True) + + +async def add_query( + datasette: Any, + database: str, + name: str, + sql: str, + *, + title: str | None = None, + description: str | None = None, + description_html: str | None = None, + hide_sql: bool = False, + fragment: str | None = None, + parameters: Iterable[str] | None = None, + is_write: bool = False, + is_private: bool = False, + is_trusted: bool = False, + source: str = "plugin", + owner_id: str | None = None, + on_success_message: str | None = None, + on_success_message_sql: str | None = None, + on_success_redirect: str | None = None, + on_error_message: str | None = None, + on_error_redirect: str | None = None, + replace: bool = True, +) -> None: + parameters_json = json.dumps(list(parameters or [])) + options_json = query_options_json( + { + "hide_sql": hide_sql, + "fragment": fragment, + "on_success_message": on_success_message, + "on_success_message_sql": on_success_message_sql, + "on_success_redirect": on_success_redirect, + "on_error_message": on_error_message, + "on_error_redirect": on_error_redirect, + } + ) + sql_statement = """ + INSERT INTO queries ( + database_name, name, sql, title, description, description_html, + options, parameters, is_write, is_private, is_trusted, source, owner_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + if replace: + sql_statement += """ + ON CONFLICT(database_name, name) DO UPDATE SET + sql = excluded.sql, + title = excluded.title, + description = excluded.description, + description_html = excluded.description_html, + options = excluded.options, + parameters = excluded.parameters, + is_write = excluded.is_write, + is_private = excluded.is_private, + is_trusted = excluded.is_trusted, + source = excluded.source, + owner_id = excluded.owner_id, + updated_at = CURRENT_TIMESTAMP + """ + await datasette.get_internal_database().execute_write( + sql_statement, + [ + database, + name, + sql, + title, + description, + description_html, + options_json, + parameters_json, + int(bool(is_write)), + int(bool(is_private)), + int(bool(is_trusted)), + source, + owner_id, + ], + ) + + +async def update_query( + datasette: Any, + database: str, + name: str, + *, + sql=UNCHANGED, + title=UNCHANGED, + description=UNCHANGED, + description_html=UNCHANGED, + hide_sql=UNCHANGED, + fragment=UNCHANGED, + parameters=UNCHANGED, + is_write=UNCHANGED, + is_private=UNCHANGED, + is_trusted=UNCHANGED, + source=UNCHANGED, + owner_id=UNCHANGED, + on_success_message=UNCHANGED, + on_success_message_sql=UNCHANGED, + on_success_redirect=UNCHANGED, + on_error_message=UNCHANGED, + on_error_redirect=UNCHANGED, +) -> None: + fields = { + "sql": sql, + "title": title, + "description": description, + "description_html": description_html, + "parameters": parameters, + "is_write": is_write, + "is_private": is_private, + "is_trusted": is_trusted, + "source": source, + "owner_id": owner_id, + } + option_fields = { + "hide_sql": hide_sql, + "fragment": fragment, + "on_success_message": on_success_message, + "on_success_message_sql": on_success_message_sql, + "on_success_redirect": on_success_redirect, + "on_error_message": on_error_message, + "on_error_redirect": on_error_redirect, + } + updates = [] + params = [] + for field, value in fields.items(): + if value is UNCHANGED: + continue + if field in {"is_write", "is_private", "is_trusted"}: + value = int(bool(value)) + elif field == "parameters": + value = json.dumps(list(value or [])) + updates.append(f"{field} = ?") + params.append(value) + changed_options = { + field: value for field, value in option_fields.items() if value is not UNCHANGED + } + if changed_options: + rows = await datasette.get_internal_database().execute( + """ + SELECT options FROM queries + WHERE database_name = ? AND name = ? + """, + [database, name], + ) + row = rows.first() + options = json.loads(row["options"] or "{}") if row is not None else {} + for field, value in changed_options.items(): + if field == "hide_sql": + if value: + options[field] = True + else: + options.pop(field, None) + elif value is None: + options.pop(field, None) + else: + options[field] = value + updates.append("options = ?") + params.append(json.dumps(options, sort_keys=True)) + if not updates: + return + updates.append("updated_at = CURRENT_TIMESTAMP") + params.extend([database, name]) + await datasette.get_internal_database().execute_write( + """ + UPDATE queries + SET {} + WHERE database_name = ? AND name = ? + """.format(", ".join(updates)), + params, + ) + + +async def remove_query( + datasette: Any, database: str, name: str, source: str | None = None +) -> None: + sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" + params = [database, name] + if source is not None: + sql += " AND source = ?" + params.append(source) + await datasette.get_internal_database().execute_write(sql, params) + + +async def get_query(datasette: Any, database: str, name: str) -> StoredQuery | None: + rows = await datasette.get_internal_database().execute( + """ + SELECT * FROM queries + WHERE database_name = ? AND name = ? + """, + [database, name], + ) + return query_row_to_stored_query(rows.first()) + + +async def count_queries( + datasette: Any, + database: str | None = None, + *, + actor: dict[str, Any] | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, +) -> int: + allowed_sql, allowed_params = await datasette.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + ) + params = dict(allowed_params) + where_clauses = [] + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + row = ( + await datasette.get_internal_database().execute( + """ + SELECT count(*) AS count + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + """.format( + allowed_sql=allowed_sql, + where=" AND ".join(where_clauses) or "1 = 1", + ), + params, + ) + ).first() + return row["count"] + + +async def list_queries( + datasette: Any, + database: str | None = None, + *, + actor: dict[str, Any] | None = None, + limit: int = 50, + cursor: str | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + include_private: bool = False, +) -> StoredQueryPage: + limit = min(max(1, int(limit)), 1000) + allowed_sql, allowed_params = await datasette.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + include_is_private=include_private, + ) + params = dict(allowed_params) + params.update({"limit": limit + 1}) + sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" + where_clauses = [] + order_by = "q.database_name, sort_key, q.name" + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + order_by = "sort_key, q.name" + + if cursor: + try: + components = urlsafe_components(cursor) + except ValueError: + components = [] + if database is None and len(components) == 3: + where_clauses.append(""" + ( + q.database_name > :cursor_database + OR ( + q.database_name = :cursor_database + AND ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_database"] = components[0] + params["cursor_sort_key"] = components[1] + params["cursor_name"] = components[2] + elif database is not None and len(components) == 2: + where_clauses.append(""" + ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_sort_key"] = components[0] + params["cursor_name"] = components[1] + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + private_select = ", allowed.is_private AS private" if include_private else "" + rows = list( + ( + await datasette.get_internal_database().execute( + """ + SELECT q.*, {sort_key_sql} AS sort_key{private_select} + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + ORDER BY {order_by} + LIMIT :limit + """.format( + allowed_sql=allowed_sql, + private_select=private_select, + sort_key_sql=sort_key_sql, + where=" AND ".join(where_clauses) or "1 = 1", + order_by=order_by, + ), + params, + ) + ).rows + ) + has_more = len(rows) > limit + if has_more: + rows = rows[:limit] + + queries = [] + for row in rows: + query = query_row_to_stored_query( + row, private=bool(row["private"]) if include_private else None + ) + assert query is not None + queries.append(query) + + next_token = None + if has_more and rows: + last_row = rows[-1] + if database is None: + next_token = "{},{},{}".format( + tilde_encode(last_row["database_name"]), + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + else: + next_token = "{},{}".format( + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + return StoredQueryPage( + queries=queries, + next=next_token, + has_more=has_more, + limit=limit, + ) + + +async def ensure_query_write_permissions( + datasette: Any, + database: str, + sql: str, + *, + actor: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + analysis: Any = None, +) -> Any: + write_actions = { + "insert": "insert-row", + "update": "update-row", + "delete": "delete-row", + } + db = datasette.get_database(database) + if analysis is None: + if params is None: + params = {name: "" for name in named_parameters(sql)} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError as ex: + raise Forbidden(f"Could not analyze query: {ex}") from ex + + for access in analysis.table_accesses: + action = write_actions.get(access.operation) + if action is None: + continue + if access.database != database: + raise Forbidden("Writable queries may not write to attached databases") + if not await datasette.allowed( + action=action, + resource=TableResource(database=access.database, table=access.table), + actor=actor, + ): + raise Forbidden( + f"Permission denied: need {action} on {access.database}/{access.table}" + ) + return analysis diff --git a/datasette/templates/_execute_write_analysis_scripts.html b/datasette/templates/_execute_write_analysis_scripts.html new file mode 100644 index 00000000..a19bae13 --- /dev/null +++ b/datasette/templates/_execute_write_analysis_scripts.html @@ -0,0 +1,111 @@ + diff --git a/datasette/templates/_execute_write_analysis_styles.html b/datasette/templates/_execute_write_analysis_styles.html new file mode 100644 index 00000000..165cfe9f --- /dev/null +++ b/datasette/templates/_execute_write_analysis_styles.html @@ -0,0 +1,41 @@ + diff --git a/datasette/templates/_sql_parameter_scripts.html b/datasette/templates/_sql_parameter_scripts.html new file mode 100644 index 00000000..159a141c --- /dev/null +++ b/datasette/templates/_sql_parameter_scripts.html @@ -0,0 +1,293 @@ + diff --git a/datasette/templates/_sql_parameter_styles.html b/datasette/templates/_sql_parameter_styles.html new file mode 100644 index 00000000..bc6838f5 --- /dev/null +++ b/datasette/templates/_sql_parameter_styles.html @@ -0,0 +1,58 @@ + diff --git a/datasette/templates/_sql_parameters.html b/datasette/templates/_sql_parameters.html new file mode 100644 index 00000000..58801d40 --- /dev/null +++ b/datasette/templates/_sql_parameters.html @@ -0,0 +1,9 @@ +
+ {% if parameter_names %} +

Parameters

+ {% for parameter in parameter_names %} + {% set parameter_id = (sql_parameter_id_prefix|default("qp")) ~ loop.index %} +

{% if sql_parameters_allow_expand|default(false) %} {% endif %}

+ {% endfor %} + {% endif %} +
diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 42b4ca0b..371f6a22 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -5,6 +5,7 @@ {% block extra_head %} {{- super() -}} {% include "_codemirror.html" %} +{% include "_sql_parameter_styles.html" %} {% endblock %} {% block body_class %}db db-{{ database|to_css_class }}{% endblock %} @@ -25,9 +26,13 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if allow_execute_sql %} -
+

Custom SQL query

-

+

+ {% set parameter_names = [] %} + {% set parameter_values = {} %} + {% set sql_parameters_allow_expand = false %} + {% include "_sql_parameters.html" %}

@@ -53,6 +58,9 @@

  • {{ query.title or query.name }}{% if query.private %} 🔒{% endif %}
    • {% endfor %} + {% if queries_more %} +

    View {{ "{:,}".format(queries_count) }} quer{% if queries_count == 1 %}y{% else %}ies{% endif %}

    + {% endif %} {% endif %} {% if tables %} @@ -87,5 +95,11 @@ {% endif %} {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} + {% endblock %} diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html new file mode 100644 index 00000000..6b626f8d --- /dev/null +++ b/datasette/templates/execute_write.html @@ -0,0 +1,314 @@ +{% extends "base.html" %} + +{% block title %}Write to this database{% endblock %} + +{% block extra_head %} +{{- super() -}} +{% include "_codemirror.html" %} + +{% include "_execute_write_analysis_styles.html" %} +{% include "_sql_parameter_styles.html" %} +{% endblock %} + +{% block body_class %}execute-write db-{{ database|to_css_class }}{% endblock %} + +{% block crumbs %} +{{ crumbs.nav(request=request, database=database) }} +{% endblock %} + +{% block content %} + +

    Write to this database

    + +

    Execute SQL to insert, update or delete rows in this database.

    + +{% if execution_message %} +

    {{ execution_message }}{% for link in execution_links %} {{ link.label }}{% endfor %}

    +{% endif %} + + + {% if write_template_tables %} +
    +
    + Start with a template +

    + + + + + +

    +
    +
    + {% endif %} + +

    + + {% set sql_parameters_section_id = "execute-write-parameters-section" %} + {% set sql_parameters_allow_expand = true %} + {% include "_sql_parameters.html" %} + +
    +

    Query operations

    + {% if analysis_error %} +

    {{ analysis_error }}

    + {% elif analysis_rows %} +
    + + + + + + + + + + + {% for row in analysis_rows %} + + + + + + + + {% endfor %} + +
    OperationDatabaseTableRequired permissionAllowed
    {{ row.operation }}{{ row.database }}{{ row.table }}{% if row.required_permission %}{{ row.required_permission }}{% endif %}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}
    + {% else %} +

    Analysis will show each affected table and required permission.

    + {% endif %} + + +

    + + {% if save_query_base_url %}Save this query{% endif %} +

    + + + + +{% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} +{% include "_execute_write_analysis_scripts.html" %} + + + +{% if write_template_tables %} + +{% endif %} + +{% endblock %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 5f85ac6b..168a636b 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -14,9 +14,10 @@ {% endif %} {% include "_codemirror.html" %} +{% include "_sql_parameter_styles.html" %} {% endblock %} -{% block body_class %}query db-{{ database|to_css_class }}{% if canned_query %} query-{{ canned_query|to_css_class }}{% endif %}{% endblock %} +{% block body_class %}query db-{{ database|to_css_class }}{% if stored_query %} query-{{ stored_query|to_css_class }}{% endif %}{% endblock %} {% block crumbs %} {{ crumbs.nav(request=request, database=database) }} @@ -24,19 +25,19 @@ {% block content %} -{% if canned_query_write and db_is_immutable %} +{% if stored_query_write and db_is_immutable %}

    This query cannot be executed because the database is immutable.

    {% endif %} -

    {{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

    +

    {{ metadata.title or database }}{% if stored_query and not metadata.title %}: {{ stored_query }}{% endif %}{% if private %} 🔒{% endif %}

    {% set action_links, action_title = query_actions(), "Query actions" %} {% include "_action_menu.html" %} -{% if canned_query %}{{ top_canned_query() }}{% else %}{{ top_query() }}{% endif %} +{% if stored_query %}{{ top_stored_query() }}{% else %}{{ top_query() }}{% endif %} {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} -
    +

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

    @@ -45,29 +46,28 @@ {% endif %} {% if not hide_sql %} {% if editable and allow_execute_sql %} -

    {% else %} 
    {% if query %}{{ query.sql }}{% endif %}
    {% endif %} {% else %} - {% if not canned_query %} + {% if not stored_query %} {% endif %} {% endif %} - {% if named_parameter_values %} -

    Query parameters

    - {% for name, value in named_parameter_values.items() %} -

    - {% endfor %} - {% endif %} + {% set parameter_names = named_parameter_values.keys()|list %} + {% set parameter_values = named_parameter_values %} + {% set sql_parameters_allow_expand = false %} + {% include "_sql_parameters.html" %}

    {% if not hide_sql %}{% endif %} - + {{ show_hide_hidden }} - {% if canned_query and edit_sql_url %}Edit SQL{% endif %} + {% if save_query_url %}Save this query{% endif %} + {% if stored_query and edit_sql_url %}Edit SQL{% endif %}

    @@ -90,11 +90,17 @@ {% else %} - {% if not canned_query_write and not error %} + {% if not stored_query_write and not error %}

    0 results

    {% endif %} {% endif %} {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} + {% endblock %} diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html new file mode 100644 index 00000000..f5dadbff --- /dev/null +++ b/datasette/templates/query_create.html @@ -0,0 +1,302 @@ +{% extends "base.html" %} + +{% block title %}Create query{% endblock %} + +{% block extra_head %} +{{- super() -}} +{% include "_codemirror.html" %} +{% include "_execute_write_analysis_styles.html" %} + +{% endblock %} + +{% block body_class %}query-create db-{{ database|to_css_class }}{% endblock %} + +{% block crumbs %} +{{ crumbs.nav(request=request, database=database) }} +{% endblock %} + +{% block content %} + +
    + +

    Create query

    + +
    +
    +

    +

    {{ urls.database(database) }}/

    +

    +
    + +

    + +

    + {% if analysis_error %}This query cannot be saved until the SQL is valid.{% elif not has_sql %}Enter SQL to analyze this query.{% elif analysis_is_write %}This query updates data in the database.{% else %}This is a read-only query.{% endif %} + + + Queries marked private can only be seen by you, their creator. +

    + {% if sql and analysis_is_write %} +

    Execute write SQL

    + {% endif %} + +

    + +
    + {% if has_sql %} +

    Query operations

    + {% if analysis_error %} +

    {{ analysis_error }}

    + {% elif analysis_rows %} +
    + + + + + + + + + + + {% for row in analysis_rows %} + + + + + + + + {% endfor %} + +
    OperationDatabaseTableRequired permissionAllowed
    {{ row.operation }}{{ row.database }}{{ row.table }}{% if row.required_permission %}{{ row.required_permission }}{% else %}n/a{% endif %}{% if row.allowed is none %}n/a{% elif row.allowed %}yes{% else %}no{% endif %}
    + {% else %} +

    Analysis will show each affected table and required permission.

    + {% endif %} + {% endif %} +
    +
    + +
    + +{% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} +{% include "_execute_write_analysis_scripts.html" %} + + + + + +{% endblock %} diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html new file mode 100644 index 00000000..fa4859b1 --- /dev/null +++ b/datasette/templates/query_list.html @@ -0,0 +1,281 @@ +{% extends "base.html" %} + +{% block title %}{% if database %}{{ database }}: {% endif %}queries{% endblock %} + +{% block extra_head %} +{{- super() -}} + +{% endblock %} + +{% block body_class %}query-list{% if database %} db-{{ database|to_css_class }}{% endif %}{% endblock %} + +{% block crumbs %} +{{ crumbs.nav(request=request, database=database) }} +{% endblock %} + +{% block content %} + +
    + +

    Queries

    + +
    +

    + + + {% if filters.is_write %}{% endif %} + {% if filters.is_private %}{% endif %} + {% if filters.source %}{% endif %} + {% if filters.owner_id %}{% endif %} + +

    +
    + + + +{% if queries %} +
    + + + {% if show_database %}{% endif %} + + + + + + + {% for query in queries %} + + {% if show_database %} + + {% endif %} + + + + + {% endfor %} + +
    DatabaseQueryOwnerFlags
    {{ query.database }} + {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} + {% if query.description %}

    {{ query.description }}

    {% endif %} +     		{% if query.owner_id is not none %}{{ query.owner_id }}{% else %}-{% endif %} + + {% if query.is_write %}Writable{% else %}Read-only{% endif %} + {% if query.is_private %}Private{% endif %} + {% if query.is_trusted %}Trusted{% endif %} + +
    + {% if show_private_note or show_trusted_note %} +
    + {% if show_private_note %}

    PrivateOnly the owning actor can view this query.

    {% endif %} + {% if show_trusted_note %}

    TrustedExecution skips the usual SQL and write permission checks after view-query allows access.

    {% endif %} +
    + {% endif %} +{% else %} +

    No queries found.

    +{% endif %} + +{% if next_url %} + +{% endif %} + +
    + +{% endblock %} diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index e679ae76..891ee913 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -241,6 +241,14 @@ async def _build_single_action_sql( "),", ] ) + else: + query_parts.extend( + [ + "anon_rules AS (", + " SELECT NULL AS parent, NULL AS child, 0 AS allow, NULL AS reason WHERE 0", + "),", + ] + ) # Continue with the cascading logic query_parts.extend( diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index df149928..bf172667 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -112,6 +112,28 @@ async def initialize_metadata_tables(db): config TEXT, PRIMARY KEY (database_name, resource_name, column_name) ); + + CREATE TABLE IF NOT EXISTS queries ( + database_name TEXT NOT NULL, + name TEXT NOT NULL, + sql TEXT NOT NULL, + title TEXT, + description TEXT, + description_html TEXT, + options TEXT NOT NULL DEFAULT '{}', + parameters TEXT NOT NULL DEFAULT '[]', + is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), + source TEXT NOT NULL DEFAULT 'user', + owner_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (database_name, name) + ); + + CREATE INDEX IF NOT EXISTS queries_owner_idx + ON queries(owner_id); """)) diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py new file mode 100644 index 00000000..b5317b62 --- /dev/null +++ b/datasette/utils/sql_analysis.py @@ -0,0 +1,99 @@ +from dataclasses import dataclass +from typing import Literal + +from datasette.utils.sqlite import sqlite3 + +SQLTableOperation = Literal["read", "insert", "update", "delete"] + + +@dataclass(frozen=True) +class SQLTableAccess: + operation: SQLTableOperation + database: str | None + table: str + sqlite_schema: str | None + columns: tuple[str, ...] = () + source: str | None = None + + +@dataclass(frozen=True) +class SQLAnalysis: + table_accesses: tuple[SQLTableAccess, ...] + + +_ACTION_TO_OPERATION: dict[int, SQLTableOperation] = { + sqlite3.SQLITE_READ: "read", + sqlite3.SQLITE_INSERT: "insert", + sqlite3.SQLITE_UPDATE: "update", + sqlite3.SQLITE_DELETE: "delete", +} + + +def analyze_sql_tables( + conn, + sql: str, + params=None, + *, + database_name: str | None = None, + schema_to_database: dict[str, str] | None = None, +) -> SQLAnalysis: + """ + Return tables accessed by a SQL statement according to SQLite's authorizer. + + This function is synchronous and connection-based. It temporarily installs a + SQLite authorizer, prepares ``EXPLAIN ``, and returns the table access + callbacks observed while SQLite compiles the statement. + """ + accesses: dict[ + tuple[SQLTableOperation, str | None, str, str | None, str | None], set[str] + ] = {} + + def database_for_schema(sqlite_schema): + if schema_to_database and sqlite_schema in schema_to_database: + return schema_to_database[sqlite_schema] + if sqlite_schema == "main" and database_name is not None: + return database_name + return sqlite_schema + + def authorizer(action, arg1, arg2, sqlite_schema, source): + operation = _ACTION_TO_OPERATION.get(action) + if operation is None or arg1 is None: + return sqlite3.SQLITE_OK + + key = ( + operation, + database_for_schema(sqlite_schema), + arg1, + sqlite_schema, + source, + ) + columns = accesses.setdefault(key, set()) + if operation in ("read", "update") and arg2 is not None: + columns.add(arg2) + return sqlite3.SQLITE_OK + + conn.set_authorizer(authorizer) + try: + conn.execute("EXPLAIN " + sql, params if params is not None else {}).fetchall() + finally: + conn.set_authorizer(None) + + return SQLAnalysis( + table_accesses=tuple( + SQLTableAccess( + operation=operation, + database=database, + table=table, + sqlite_schema=sqlite_schema, + columns=tuple(sorted(columns)), + source=source, + ) + for ( + operation, + database, + table, + sqlite_schema, + source, + ), columns in accesses.items() + ) + ) diff --git a/datasette/views/database.py b/datasette/views/database.py index 8e4ea85a..b558b002 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -13,6 +13,7 @@ import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource +from datasette.stored_queries import stored_query_to_dict from datasette.utils import ( add_cors_headers, await_me_maybe, @@ -35,6 +36,7 @@ from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden from datasette.plugins import pm from .base import BaseView, DatasetteError, View, _error, stream_csv +from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns from . import Context @@ -92,24 +94,19 @@ class DatabaseView(View): tables = await get_tables(datasette, request, db, allowed_dict) - # Get allowed queries using the new permission system - allowed_query_page = await datasette.allowed_resources( - "view-query", - request.actor, - parent=database, - include_is_private=True, - limit=1000, + queries_page = await datasette.list_queries( + database, + actor=request.actor, + limit=5, + include_private=True, + ) + stored_queries = queries_page.queries + queries_more = queries_page.has_more + queries_count = ( + await datasette.count_queries(database, actor=request.actor) + if queries_more + else len(stored_queries) ) - - # Build canned_queries list by looking up each allowed query - all_queries = await datasette.get_canned_queries(database, request.actor) - canned_queries = [] - for query_resource in allowed_query_page.resources: - query_name = query_resource.child - if query_name in all_queries: - canned_queries.append( - dict(all_queries[query_name], private=query_resource.private) - ) async def database_actions(): links = [] @@ -140,7 +137,9 @@ class DatabaseView(View): "tables": tables, "hidden_count": len([t for t in tables if t["hidden"]]), "views": sql_views, - "queries": canned_queries, + "queries": [stored_query_to_dict(query) for query in stored_queries], + "queries_more": queries_more, + "queries_count": queries_count, "allow_execute_sql": allow_execute_sql, "table_columns": ( await _table_columns(datasette, database) if allow_execute_sql else {} @@ -173,7 +172,9 @@ class DatabaseView(View): tables=tables, hidden_count=len([t for t in tables if t["hidden"]]), views=sql_views, - queries=canned_queries, + queries=stored_queries, + queries_more=queries_more, + queries_count=queries_count, allow_execute_sql=allow_execute_sql, table_columns=( await _table_columns(datasette, database) @@ -221,7 +222,11 @@ class DatabaseContext(Context): tables: list = field(metadata={"help": "List of table objects in the database"}) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) views: list = field(metadata={"help": "List of view objects in the database"}) - queries: list = field(metadata={"help": "List of canned query objects"}) + queries: list = field(metadata={"help": "List of stored query objects"}) + queries_more: bool = field( + metadata={"help": "Boolean indicating if more stored queries are available"} + ) + queries_count: int = field(metadata={"help": "Count of visible stored queries"}) allow_execute_sql: bool = field( metadata={"help": "Boolean indicating if custom SQL can be executed"} ) @@ -266,8 +271,8 @@ class QueryContext(Context): query: dict = field( metadata={"help": "The SQL query object containing the `sql` string"} ) - canned_query: str = field( - metadata={"help": "The name of the canned query if this is a canned query"} + stored_query: str = field( + metadata={"help": "The name of the stored query if this is a stored query"} ) private: bool = field( metadata={"help": "Boolean indicating if this is a private database"} @@ -275,13 +280,13 @@ class QueryContext(Context): # urls: dict = field( # metadata={"help": "Object containing URL helpers like `database()`"} # ) - canned_query_write: bool = field( + stored_query_write: bool = field( metadata={ - "help": "Boolean indicating if this is a canned query that allows writes" + "help": "Boolean indicating if this is a stored query that allows writes" } ) metadata: dict = field( - metadata={"help": "Metadata about the database or the canned query"} + metadata={"help": "Metadata about the database or the stored query"} ) db_is_immutable: bool = field( metadata={"help": "Boolean indicating if this database is immutable"} @@ -302,12 +307,15 @@ class QueryContext(Context): allow_execute_sql: bool = field( metadata={"help": "Boolean indicating if custom SQL can be executed"} ) + save_query_url: str = field( + metadata={"help": "URL to save the current arbitrary SQL as a query"} + ) tables: list = field(metadata={"help": "List of table objects in the database"}) named_parameter_values: dict = field( metadata={"help": "Dictionary of parameter names/values"} ) edit_sql_url: str = field( - metadata={"help": "URL to edit the SQL for a canned query"} + metadata={"help": "URL to edit the SQL for a stored query"} ) display_rows: list = field(metadata={"help": "List of result rows to display"}) columns: list = field(metadata={"help": "List of column names"}) @@ -331,8 +339,8 @@ class QueryContext(Context): top_query: callable = field( metadata={"help": "Callable to render the top_query slot"} ) - top_canned_query: callable = field( - metadata={"help": "Callable to render the top_canned_query slot"} + top_stored_query: callable = field( + metadata={"help": "Callable to render the top_stored_query slot"} ) query_actions: callable = field( metadata={ @@ -423,21 +431,32 @@ class QueryView(View): db = await datasette.resolve_database(request) - # We must be a canned query + # We must be a stored query table_found = False try: await datasette.resolve_table(request) table_found = True except TableNotFound as table_not_found: - canned_query = await datasette.get_canned_query( - table_not_found.database_name, table_not_found.table, request.actor + stored_query = await datasette.get_query( + table_not_found.database_name, table_not_found.table ) - if canned_query is None: + if stored_query is None: raise if table_found: # That should not have happened raise DatasetteError("Unexpected table found on POST", status=404) + if not await datasette.allowed( + action="view-query", + resource=QueryResource(database=db.name, query=stored_query.name), + actor=request.actor, + ): + raise Forbidden("You do not have permission to view this query") + + await _ensure_stored_query_execution_permissions( + datasette, db, stored_query, request.actor + ) + # If database is immutable, return an error if not db.is_mutable: raise Forbidden("Database is immutable") @@ -462,20 +481,18 @@ class QueryView(View): or request.args.get("_json") or params.get("_json") ) - params_for_query = MagicParameters( - canned_query["sql"], params, request, datasette - ) + params_for_query = MagicParameters(stored_query.sql, params, request, datasette) await params_for_query.execute_params() ok = None redirect_url = None try: cursor = await db.execute_write( - canned_query["sql"], params_for_query, request=request + stored_query.sql, params_for_query, request=request ) # success message can come from on_success_message or on_success_message_sql message = None message_type = datasette.INFO - on_success_message_sql = canned_query.get("on_success_message_sql") + on_success_message_sql = stored_query.on_success_message_sql if on_success_message_sql: try: message_result = ( @@ -487,18 +504,19 @@ class QueryView(View): message = "Error running on_success_message_sql: {}".format(ex) message_type = datasette.ERROR if not message: - message = canned_query.get( - "on_success_message" - ) or "Query executed, {} row{} affected".format( - cursor.rowcount, "" if cursor.rowcount == 1 else "s" + message = ( + stored_query.on_success_message + or "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) ) - redirect_url = canned_query.get("on_success_redirect") + redirect_url = stored_query.on_success_redirect ok = True except Exception as ex: - message = canned_query.get("on_error_message") or str(ex) + message = stored_query.on_error_message or str(ex) message_type = datasette.ERROR - redirect_url = canned_query.get("on_error_redirect") + redirect_url = stored_query.on_error_redirect ok = False if should_return_json: return Response.json( @@ -531,31 +549,35 @@ class QueryView(View): # Create lookup dict for quick access allowed_dict = {r.child: r for r in allowed_tables_page.resources} - # Are we a canned query? - canned_query = None - canned_query_write = False + # Are we a stored query? + stored_query = None + stored_query_write = False if "table" in request.url_vars: try: await datasette.resolve_table(request) except TableNotFound as table_not_found: - # Was this actually a canned query? - canned_query = await datasette.get_canned_query( - table_not_found.database_name, table_not_found.table, request.actor + # Was this actually a stored query? + stored_query = await datasette.get_query( + table_not_found.database_name, table_not_found.table ) - if canned_query is None: + if stored_query is None: raise - canned_query_write = bool(canned_query.get("write")) + stored_query_write = stored_query.is_write private = False - if canned_query: - # Respect canned query permissions + if stored_query: + # Respect stored query permissions visible, private = await datasette.check_visibility( request.actor, action="view-query", - resource=QueryResource(database=database, query=canned_query["name"]), + resource=QueryResource(database=database, query=stored_query.name), ) if not visible: raise Forbidden("You do not have permission to view this query") + if not stored_query_write: + await _ensure_stored_query_execution_permissions( + datasette, db, stored_query, request.actor + ) else: await datasette.ensure_permission( @@ -568,15 +590,15 @@ class QueryView(View): params = {key: request.args.get(key) for key in request.args} sql = None - if canned_query: - sql = canned_query["sql"] + if stored_query: + sql = stored_query.sql elif "sql" in params: sql = params.pop("sql") # Extract any :named parameters named_parameters = [] - if canned_query and canned_query.get("params"): - named_parameters = canned_query["params"] + if stored_query and stored_query.parameters: + named_parameters = stored_query.parameters if not named_parameters and sql: named_parameters = derive_named_parameters(sql) named_parameter_values = { @@ -602,13 +624,13 @@ class QueryView(View): params_for_query = params - if sql and not canned_query_write: + if sql and not stored_query_write: try: - if not canned_query: + if not stored_query: # For regular queries we only allow SELECT, plus other rules validate_sql_select(sql) else: - # Canned queries can run magic parameters + # Stored queries can run magic parameters params_for_query = MagicParameters(sql, params, request, datasette) await params_for_query.execute_params() results = await datasette.execute( @@ -664,7 +686,7 @@ class QueryView(View): columns=columns, rows=rows, sql=sql, - query_name=canned_query["name"] if canned_query else None, + query_name=stored_query.name if stored_query else None, database=database, table=None, request=request, @@ -696,10 +718,10 @@ class QueryView(View): elif format_ == "html": headers = {} templates = [f"query-{to_css_class(database)}.html", "query.html"] - if canned_query: + if stored_query: templates.insert( 0, - f"query-{to_css_class(database)}-{to_css_class(canned_query['name'])}.html", + f"query-{to_css_class(database)}-{to_css_class(stored_query.name)}.html", ) environment = datasette.get_jinja_environment(request) @@ -717,6 +739,9 @@ class QueryView(View): } ) metadata = await datasette.get_database_metadata(database) + if stored_query: + metadata = stored_query_to_dict(stored_query) + metadata.pop("source", None) renderers = {} for key, (_, can_render) in datasette.renderers.items(): @@ -743,9 +768,14 @@ class QueryView(View): resource=DatabaseResource(database=database), actor=request.actor, ) + allow_store_query = await datasette.allowed( + action="store-query", + resource=DatabaseResource(database=database), + actor=request.actor, + ) show_hide_hidden = "" - if canned_query and canned_query.get("hide_sql"): + if stored_query and stored_query.hide_sql: if bool(params.get("_show_sql")): show_hide_link = path_with_removed_args(request, {"_show_sql"}) show_hide_text = "hide" @@ -793,6 +823,19 @@ class QueryView(View): } ) ) + save_query_url = None + if ( + not stored_query + and allow_execute_sql + and allow_store_query + and is_validated_sql + and ":_" not in sql + ): + save_query_url = ( + datasette.urls.database(database) + + "/-/queries/store?" + + urlencode({"sql": sql}) + ) async def query_actions(): query_actions = [] @@ -800,7 +843,7 @@ class QueryView(View): datasette=datasette, actor=request.actor, database=database, - query_name=canned_query["name"] if canned_query else None, + query_name=stored_query.name if stored_query else None, request=request, sql=sql, params=params, @@ -820,16 +863,17 @@ class QueryView(View): "sql": sql, "params": params, }, - canned_query=canned_query["name"] if canned_query else None, + stored_query=stored_query.name if stored_query else None, private=private, - canned_query_write=canned_query_write, + stored_query_write=stored_query_write, db_is_immutable=not db.is_mutable, error=query_error, hide_sql=hide_sql, show_hide_link=datasette.urls.path(show_hide_link), show_hide_text=show_hide_text, - editable=not canned_query, + editable=not stored_query, allow_execute_sql=allow_execute_sql, + save_query_url=save_query_url, tables=await get_tables(datasette, request, db, allowed_dict), named_parameter_values=named_parameter_values, edit_sql_url=edit_sql_url, @@ -849,7 +893,7 @@ class QueryView(View): ) ), show_hide_hidden=markupsafe.Markup(show_hide_hidden), - metadata=canned_query or metadata, + metadata=metadata, alternate_url_json=alternate_url_json, select_templates=[ f"{'*' if template_name == template.name else ''}{template_name}" @@ -858,12 +902,12 @@ class QueryView(View): top_query=make_slot_function( "top_query", datasette, request, database=database, sql=sql ), - top_canned_query=make_slot_function( - "top_canned_query", + top_stored_query=make_slot_function( + "top_stored_query", datasette, request, database=database, - query_name=canned_query["name"] if canned_query else None, + query_name=stored_query.name if stored_query else None, ), query_actions=query_actions, ), @@ -1176,22 +1220,6 @@ class TableCreateView(BaseView): return Response.json(details, status=201) -async def _table_columns(datasette, database_name): - internal_db = datasette.get_internal_database() - result = await internal_db.execute( - "select table_name, name from catalog_columns where database_name = ?", - [database_name], - ) - table_columns = {} - for row in result.rows: - table_columns.setdefault(row["table_name"], []).append(row["name"]) - # Add views - db = datasette.get_database(database_name) - for view_name in await db.view_names(): - table_columns[view_name] = [] - return table_columns - - async def display_rows(datasette, database, request, rows, columns): display_rows = [] truncate_cells = datasette.setting("truncate_cells_html") diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py new file mode 100644 index 00000000..0054300c --- /dev/null +++ b/datasette/views/execute_write.py @@ -0,0 +1,257 @@ +from urllib.parse import urlencode + +from datasette.resources import DatabaseResource +from datasette.utils import sqlite3 +from datasette.utils.asgi import Response + +from .base import BaseView, _error +from .query_helpers import ( + QueryValidationError, + _analysis_is_write, + _analysis_rows, + _analysis_rows_with_permissions, + _block_framing, + _coerce_execute_write_payload, + _derived_query_parameters, + _execute_write_analysis_data, + _inserted_row_url, + _json_or_form_payload, + _prepare_execute_write, + _table_columns, + _wants_json, +) + + +class ExecuteWriteView(BaseView): + name = "execute-write" + has_json_alternate = False + + async def _render_form( + self, + request, + db, + *, + sql="", + parameter_values=None, + analysis=None, + analysis_error=None, + execution_message=None, + execution_links=None, + execution_ok=None, + status=200, + ): + parameter_values = parameter_values or {} + execution_links = execution_links or [] + parameter_names = [] + analysis_rows = [] + table_columns = await _table_columns(self.ds, db.name) + hidden_table_names = set(await db.hidden_table_names()) + write_template_tables = { + table: columns + for table, columns in table_columns.items() + if columns and table not in hidden_table_names + } + if sql and analysis_error is None: + try: + parameter_names = _derived_query_parameters(sql) + if analysis is None: + params = {parameter: "" for parameter in parameter_names} + analysis = await db.analyze_sql(sql, params) + if _analysis_is_write(analysis): + analysis_rows = await _analysis_rows_with_permissions( + self.ds, analysis, request.actor + ) + else: + analysis_error = ( + "Use /-/query for read-only SQL; " + "this endpoint only executes writes" + ) + except (QueryValidationError, sqlite3.DatabaseError) as ex: + analysis_error = getattr(ex, "message", str(ex)) + + allow_save_query = await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ) and await self.ds.allowed( + action="store-query", + resource=DatabaseResource(db.name), + actor=request.actor, + ) + save_query_base_url = None + save_query_url = None + if allow_save_query: + save_query_base_url = self.ds.urls.database(db.name) + "/-/queries/store" + if ( + sql + and analysis_error is None + and not any(row["allowed"] is False for row in analysis_rows) + ): + save_query_url = save_query_base_url + "?" + urlencode({"sql": sql}) + + response = await self.render( + ["execute_write.html"], + request, + { + "database": db.name, + "database_color": db.color, + "sql": sql, + "parameter_names": parameter_names, + "parameter_values": parameter_values, + "analysis_error": analysis_error, + "analysis_rows": [ + row for row in analysis_rows if row["operation"] != "read" + ], + "execution_message": execution_message, + "execution_links": execution_links, + "execution_ok": execution_ok, + "execute_disabled": bool( + (not sql) + or analysis_error + or any(row["allowed"] is False for row in analysis_rows) + ), + "table_columns": table_columns, + "write_template_tables": write_template_tables, + "save_query_url": save_query_url, + "save_query_base_url": save_query_base_url, + }, + ) + response.status = status + return _block_framing(response) + + async def get(self, request): + db = await self.ds.resolve_database(request) + await self.ds.ensure_permission( + action="execute-write-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ) + if not db.is_mutable: + return _block_framing( + _error( + ["Cannot execute write SQL because this database is immutable."], + 403, + ) + ) + return await self._render_form( + request, + db, + sql=request.args.get("sql") or "", + ) + + async def post(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-write-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing( + _error(["Permission denied: need execute-write-sql"], 403) + ) + if not db.is_mutable: + return _block_framing(_error(["Database is immutable"], 403)) + + data = {} + is_json = request.headers.get("content-type", "").startswith("application/json") + sql = "" + provided_params = {} + try: + data, is_json = await _json_or_form_payload(request) + sql, provided_params = _coerce_execute_write_payload(data, is_json) + parameter_names, params, analysis = await _prepare_execute_write( + self.ds, db, sql, provided_params, request.actor + ) + except QueryValidationError as ex: + if _wants_json(request, is_json, data): + return _block_framing(_error([ex.message], ex.status)) + return await self._render_form( + request, + db, + sql=sql or "", + parameter_values=provided_params, + analysis_error=ex.message, + execution_message=ex.message, + execution_ok=False, + status=ex.status, + ) + + try: + cursor = await db.execute_write(sql, params, request=request) + except sqlite3.DatabaseError as ex: + message = str(ex) + if _wants_json(request, is_json, data): + return _block_framing(_error([message], 400)) + return await self._render_form( + request, + db, + sql=sql, + parameter_values=params, + analysis=analysis, + execution_message=message, + execution_ok=False, + status=400, + ) + + message = "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) + if _wants_json(request, is_json, data): + return _block_framing( + Response.json( + { + "ok": True, + "message": message, + "rowcount": cursor.rowcount, + "analysis": _analysis_rows(analysis), + } + ) + ) + + inserted_row_url = await _inserted_row_url(self.ds, db, analysis, cursor) + execution_links = ( + [{"href": inserted_row_url, "label": "View row"}] + if inserted_row_url + else [] + ) + return await self._render_form( + request, + db, + sql=sql, + parameter_values={name: params.get(name, "") for name in parameter_names}, + analysis=analysis, + execution_message=message, + execution_links=execution_links, + execution_ok=True, + ) + + +class ExecuteWriteAnalyzeView(BaseView): + name = "execute-write-analyze" + has_json_alternate = False + + async def get(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-write-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing( + _error(["Permission denied: need execute-write-sql"], 403) + ) + + invalid_keys = set(request.args) - {"sql"} + if invalid_keys: + return _block_framing( + _error( + ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], + 400, + ) + ) + sql = request.args.get("sql") or "" + return _block_framing( + Response.json( + await _execute_write_analysis_data(self.ds, db, sql, request.actor) + ) + ) diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py new file mode 100644 index 00000000..46d71b8e --- /dev/null +++ b/datasette/views/query_helpers.py @@ -0,0 +1,556 @@ +import json +import re + +from datasette.resources import DatabaseResource, TableResource +from datasette.stored_queries import StoredQuery +from datasette.utils import ( + named_parameters as derive_named_parameters, + escape_sqlite, + path_from_row_pks, + sqlite3, + validate_sql_select, + InvalidSql, +) +from datasette.utils.asgi import Forbidden + +_query_name_re = re.compile(r"^[^/\.\n]+$") + +_query_fields = { + "sql", + "title", + "description", + "hide_sql", + "fragment", + "parameters", + "params", + "is_private", + "on_success_message", + "on_success_redirect", + "on_error_message", + "on_error_redirect", +} + +_query_create_fields = _query_fields | {"name", "mode", "csrftoken"} +_query_update_fields = _query_fields +_query_write_fields = { + "on_success_message", + "on_success_redirect", + "on_error_message", + "on_error_redirect", +} + + +class QueryValidationError(Exception): + def __init__(self, message, status=400): + self.message = message + self.status = status + + +def _actor_id(actor): + if isinstance(actor, dict): + return actor.get("id") + return None + + +def _as_bool(value): + if isinstance(value, bool): + return value + if value is None: + return False + if isinstance(value, int): + return bool(value) + if isinstance(value, str): + return value.lower() in {"1", "true", "t", "yes", "on"} + return bool(value) + + +def _as_optional_bool(value, name): + if value is None or value == "": + return None + if isinstance(value, bool): + return value + if isinstance(value, int): + return bool(value) + if isinstance(value, str): + lowered = value.lower() + if lowered in {"1", "true", "t", "yes", "on"}: + return True + if lowered in {"0", "false", "f", "no", "off"}: + return False + raise QueryValidationError("{} must be 0 or 1".format(name)) + + +def _query_list_limit(value, default=50): + if value in (None, ""): + return default + try: + return min(max(1, int(value)), 1000) + except ValueError as ex: + raise QueryValidationError("_size must be an integer") from ex + + +def _derived_query_parameters(sql): + parameters = [] + seen = set() + for parameter in derive_named_parameters(sql): + if parameter.startswith("_"): + raise QueryValidationError("Magic parameters are not allowed") + if parameter not in seen: + parameters.append(parameter) + seen.add(parameter) + return parameters + + +def _coerce_query_parameters(value, derived): + if value is None: + return derived + if isinstance(value, str): + parameters = [ + parameter.strip() + for parameter in re.split(r"[\s,]+", value) + if parameter.strip() + ] + elif isinstance(value, list): + parameters = value + else: + raise QueryValidationError("parameters must be a list of strings") + if not all(isinstance(parameter, str) for parameter in parameters): + raise QueryValidationError("parameters must be a list of strings") + if any(parameter.startswith("_") for parameter in parameters): + raise QueryValidationError("Magic parameters are not allowed") + if set(parameters) != set(derived): + raise QueryValidationError("parameters must match SQL named parameters") + return parameters + + +def _analysis_is_write(analysis): + return any( + access.operation in {"insert", "update", "delete"} + for access in analysis.table_accesses + ) + + +def _block_framing(response): + response.headers["Content-Security-Policy"] = "frame-ancestors 'none'" + response.headers["X-Frame-Options"] = "DENY" + return response + + +def _wants_json(request, is_json, data): + return ( + is_json + or request.headers.get("accept") == "application/json" + or (isinstance(data, dict) and data.get("_json")) + ) + + +def _query_create_form_error_message(message): + return { + "Query name is required": "URL is required", + "Invalid query name": "Invalid URL", + "Query name conflicts with a table or view": ( + "URL conflicts with an existing table or view" + ), + "Query already exists": "A query already exists at that URL", + }.get(message, message) + + +async def _json_or_form_payload(request): + content_type = request.headers.get("content-type", "") + if content_type.startswith("application/json"): + body = await request.post_body() + try: + return json.loads(body or b"{}"), True + except json.JSONDecodeError as e: + raise QueryValidationError("Invalid JSON: {}".format(e)) + return await request.post_vars(), False + + +async def _check_query_name(db, name, *, existing=False): + if not name or not isinstance(name, str): + raise QueryValidationError("Query name is required") + if not _query_name_re.match(name): + raise QueryValidationError("Invalid query name") + if not existing and (await db.table_exists(name) or await db.view_exists(name)): + raise QueryValidationError("Query name conflicts with a table or view") + + +async def _analyze_user_query(datasette, db, sql, *, actor): + if not sql or not isinstance(sql, str): + raise QueryValidationError("SQL is required") + derived = _derived_query_parameters(sql) + params = {parameter: "" for parameter in derived} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError as ex: + raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex + + is_write = _analysis_is_write(analysis) + if is_write: + try: + await datasette.ensure_query_write_permissions( + db.name, sql, actor=actor, analysis=analysis + ) + except Forbidden as ex: + raise QueryValidationError(str(ex), status=403) from ex + else: + try: + validate_sql_select(sql) + except InvalidSql as ex: + raise QueryValidationError(str(ex)) from ex + return is_write, derived, analysis + + +def _analysis_rows(analysis): + write_actions = { + "insert": "insert-row", + "update": "update-row", + "delete": "delete-row", + } + return [ + { + "operation": access.operation, + "database": access.database, + "table": access.table, + "required_permission": write_actions.get(access.operation, ""), + "source": access.source, + } + for access in analysis.table_accesses + ] + + +async def _analysis_rows_with_permissions(datasette, analysis, actor): + rows = _analysis_rows(analysis) + for row in rows: + permission = row["required_permission"] + if permission: + row["allowed"] = await datasette.allowed( + action=permission, + resource=TableResource(row["database"], row["table"]), + actor=actor, + ) + else: + row["allowed"] = None + return rows + + +def _coerce_execute_write_payload(data, is_json): + if not isinstance(data, dict): + raise QueryValidationError("JSON must be a dictionary") + if is_json: + invalid_keys = set(data) - {"sql", "params"} + if invalid_keys: + raise QueryValidationError( + "Invalid keys: {}".format(", ".join(sorted(invalid_keys))) + ) + params = data.get("params") or {} + else: + params = { + key: value + for key, value in data.items() + if key not in {"sql", "csrftoken", "_json"} + } + if not isinstance(params, dict): + raise QueryValidationError("params must be a dictionary") + return data.get("sql"), params + + +async def _prepare_execute_write(datasette, db, sql, params, actor): + if not sql or not isinstance(sql, str): + raise QueryValidationError("SQL is required") + parameter_names = _derived_query_parameters(sql) + extra_params = set(params) - set(parameter_names) + if extra_params: + raise QueryValidationError( + "Unknown parameters: {}".format(", ".join(sorted(extra_params))) + ) + params = {name: params.get(name, "") for name in parameter_names} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError as ex: + raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex + if not _analysis_is_write(analysis): + raise QueryValidationError( + "Use /-/query for read-only SQL; this endpoint only executes writes" + ) + try: + await datasette.ensure_query_write_permissions( + db.name, sql, actor=actor, analysis=analysis + ) + except Forbidden as ex: + raise QueryValidationError(str(ex), status=403) from ex + return parameter_names, params, analysis + + +async def _ensure_stored_query_execution_permissions( + datasette, db, query: StoredQuery, actor +): + if query.is_trusted: + return + if query.is_write: + await datasette.ensure_permission( + action="execute-write-sql", + resource=DatabaseResource(db.name), + actor=actor, + ) + await datasette.ensure_query_write_permissions(db.name, query.sql, actor=actor) + else: + await datasette.ensure_permission( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=actor, + ) + + +async def _execute_write_analysis_data(datasette, db, sql, actor): + parameter_names = [] + analysis_rows = [] + analysis_error = None + if sql: + try: + parameter_names = _derived_query_parameters(sql) + params = {parameter: "" for parameter in parameter_names} + analysis = await db.analyze_sql(sql, params) + if _analysis_is_write(analysis): + analysis_rows = await _analysis_rows_with_permissions( + datasette, analysis, actor + ) + else: + analysis_error = ( + "Use /-/query for read-only SQL; " + "this endpoint only executes writes" + ) + except (QueryValidationError, sqlite3.DatabaseError) as ex: + analysis_error = getattr(ex, "message", str(ex)) + return { + "ok": analysis_error is None, + "parameters": parameter_names, + "analysis_error": analysis_error, + "analysis_rows": [row for row in analysis_rows if row["operation"] != "read"], + "execute_disabled": bool( + (not sql) + or analysis_error + or any(row["allowed"] is False for row in analysis_rows) + ), + } + + +async def _query_create_analysis_data(datasette, db, sql, actor): + has_sql = bool(sql and sql.strip()) + parameter_names = [] + analysis_rows = [] + analysis_error = None + if has_sql: + try: + parameter_names = _derived_query_parameters(sql) + params = {parameter: "" for parameter in parameter_names} + analysis = await db.analyze_sql(sql, params) + analysis_rows = await _analysis_rows_with_permissions( + datasette, analysis, actor + ) + except (QueryValidationError, sqlite3.DatabaseError) as ex: + analysis_error = getattr(ex, "message", str(ex)) + return { + "ok": analysis_error is None, + "parameters": parameter_names, + "analysis_error": analysis_error, + "analysis_rows": analysis_rows, + "has_sql": has_sql, + "analysis_is_write": bool( + analysis_rows and any(row["required_permission"] for row in analysis_rows) + ), + "save_disabled": bool( + (not has_sql) + or analysis_error + or any(row["allowed"] is False for row in analysis_rows) + ), + } + + +async def _query_create_form_context( + datasette, + request, + db, + *, + sql="", + name="", + title="", + description="", + is_private=True, +): + analysis_data = await _query_create_analysis_data(datasette, db, sql, request.actor) + return { + "database": db.name, + "database_color": db.color, + "sql": sql, + "name": name, + "title": title, + "description": description, + "is_private": is_private, + **analysis_data, + } + + +async def _inserted_row_url(datasette, db, analysis, cursor): + if cursor.rowcount != 1: + return None + lastrowid = getattr(cursor, "lastrowid", None) + if lastrowid is None: + return None + direct_inserts = [ + access + for access in analysis.table_accesses + if access.operation == "insert" + and access.source is None + and access.database == db.name + ] + if len(direct_inserts) != 1: + return None + table = direct_inserts[0].table + pks = await db.primary_keys(table) + use_rowid = not pks + select = ( + "rowid" + if use_rowid + else ", ".join(escape_sqlite(primary_key) for primary_key in pks) + ) + try: + result = await db.execute( + "select {} from {} where rowid = ?".format(select, escape_sqlite(table)), + [lastrowid], + ) + except sqlite3.DatabaseError: + return None + row = result.first() + if row is None: + return None + row_path = path_from_row_pks(row, pks, use_rowid) + return datasette.urls.row(db.name, table, row_path) + + +def _apply_query_data_types(data): + typed = dict(data) + for key in ("hide_sql", "is_private"): + if key in typed: + typed[key] = _as_bool(typed[key]) + return typed + + +async def _prepare_query_create(datasette, request, db, data): + invalid_keys = set(data) - _query_create_fields + if invalid_keys: + raise QueryValidationError( + "Invalid keys: {}".format(", ".join(sorted(invalid_keys))) + ) + + data = _apply_query_data_types(data) + name = data.get("name") + await _check_query_name(db, name) + if await datasette.get_query(db.name, name) is not None: + raise QueryValidationError("Query already exists") + + is_write, derived, analysis = await _analyze_user_query( + datasette, + db, + data.get("sql"), + actor=request.actor, + ) + if not is_write and any(data.get(field) for field in _query_write_fields): + raise QueryValidationError("Writable query fields require writable SQL") + + parameters = _coerce_query_parameters( + data.get("parameters", data.get("params")), + derived, + ) + return { + "name": name, + "sql": data["sql"], + "title": data.get("title"), + "description": data.get("description"), + "hide_sql": _as_bool(data.get("hide_sql")), + "fragment": data.get("fragment"), + "parameters": parameters, + "is_write": is_write, + "is_private": _as_bool(data.get("is_private", True)), + "is_trusted": False, + "source": "user", + "owner_id": _actor_id(request.actor), + "on_success_message": data.get("on_success_message"), + "on_success_redirect": data.get("on_success_redirect"), + "on_error_message": data.get("on_error_message"), + "on_error_redirect": data.get("on_error_redirect"), + "analysis": analysis, + } + + +async def _prepare_query_update(datasette, request, db, existing: StoredQuery, update): + invalid_keys = set(update) - _query_update_fields + if invalid_keys: + raise QueryValidationError( + "Invalid keys: {}".format(", ".join(sorted(invalid_keys))) + ) + + update = _apply_query_data_types(update) + sql = update.get("sql", existing.sql) + query_is_write = existing.is_write + derived = _derived_query_parameters(sql) + parameters = None + + if "sql" in update: + query_is_write, derived, _ = await _analyze_user_query( + datasette, + db, + sql, + actor=request.actor, + ) + + if "parameters" in update or "params" in update: + parameters = _coerce_query_parameters( + update.get("parameters", update.get("params")), + derived, + ) + elif "sql" in update: + parameters = derived + + if not query_is_write and any(update.get(field) for field in _query_write_fields): + raise QueryValidationError("Writable query fields require writable SQL") + + field_values = { + "sql": sql, + "title": update.get("title"), + "description": update.get("description"), + "hide_sql": update.get("hide_sql"), + "fragment": update.get("fragment"), + "parameters": parameters, + "is_write": query_is_write, + "is_private": update.get("is_private"), + "on_success_message": update.get("on_success_message"), + "on_success_redirect": update.get("on_success_redirect"), + "on_error_message": update.get("on_error_message"), + "on_error_redirect": update.get("on_error_redirect"), + } + update_kwargs = {} + for field_name, value in field_values.items(): + if field_name in update: + update_kwargs[field_name] = value + if parameters is not None: + update_kwargs["parameters"] = parameters + if "sql" in update: + update_kwargs["is_write"] = query_is_write + return update_kwargs + + +async def _table_columns(datasette, database_name): + internal_db = datasette.get_internal_database() + result = await internal_db.execute( + "select table_name, name from catalog_columns where database_name = ?", + [database_name], + ) + table_columns = {} + for row in result.rows: + table_columns.setdefault(row["table_name"], []).append(row["name"]) + # Add views + db = datasette.get_database(database_name) + for view_name in await db.view_names(): + table_columns[view_name] = [] + return table_columns diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py new file mode 100644 index 00000000..8c4e849e --- /dev/null +++ b/datasette/views/stored_queries.py @@ -0,0 +1,483 @@ +from urllib.parse import parse_qsl, urlencode + +from datasette.resources import DatabaseResource, QueryResource +from datasette.stored_queries import stored_query_to_dict +from datasette.utils import sqlite3, tilde_decode +from datasette.utils.asgi import Response + +from .base import BaseView, _error +from .query_helpers import ( + QueryValidationError, + _as_bool, + _as_optional_bool, + _block_framing, + _derived_query_parameters, + _json_or_form_payload, + _prepare_query_create, + _prepare_query_update, + _query_create_analysis_data, + _query_create_form_context, + _query_create_form_error_message, + _query_list_limit, +) + + +class QueryParametersView(BaseView): + name = "query-parameters" + has_json_alternate = False + + async def get(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing(_error(["Permission denied: need execute-sql"], 403)) + + invalid_keys = set(request.args) - {"sql"} + if invalid_keys: + return _block_framing( + _error( + ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], + 400, + ) + ) + try: + parameters = _derived_query_parameters(request.args.get("sql") or "") + except QueryValidationError as ex: + return _block_framing(_error([ex.message], ex.status)) + return _block_framing(Response.json({"ok": True, "parameters": parameters})) + + +def _query_list_url(path, query_string, *, set_args=None, remove_args=None): + set_args = set_args or {} + remove_args = set(remove_args or ()) + skip = set(set_args) | remove_args | {"_next"} + pairs = [ + (key, value) + for key, value in parse_qsl(query_string, keep_blank_values=True) + if key not in skip + ] + for key, value in set_args.items(): + if value not in (None, ""): + pairs.append((key, value)) + return path + (("?" + urlencode(pairs)) if pairs else "") + + +class QueryListView(BaseView): + name = "query-list" + + async def database_name(self, request): + return (await self.ds.resolve_database(request)).name + + def query_list_path(self, database): + return self.ds.urls.database(database) + "/-/queries" + + async def get(self, request): + database = await self.database_name(request) + format_ = request.url_vars.get("format") or "html" + try: + limit = _query_list_limit( + request.args.get("_size"), + default=20 if format_ == "html" else 50, + ) + is_write = _as_optional_bool(request.args.get("is_write"), "is_write") + is_private = _as_optional_bool(request.args.get("is_private"), "is_private") + except QueryValidationError as ex: + return _error([ex.message], ex.status) + + page = await self.ds.list_queries( + database, + actor=request.actor, + limit=limit, + cursor=request.args.get("_next"), + q=request.args.get("q") or None, + is_write=is_write, + is_private=is_private, + source=request.args.get("source") or None, + owner_id=request.args.get("owner_id") or None, + include_private=True, + ) + query_list_path = self.query_list_path(database) + next_url = None + if page.next: + pairs = [ + (key, value) + for key, value in parse_qsl( + request.query_string, keep_blank_values=True + ) + if key != "_next" + ] + pairs.append(("_next", page.next)) + next_url = "{}?{}".format( + query_list_path, + urlencode(pairs), + ) + + current_filters = { + "actor": request.actor, + "q": request.args.get("q") or None, + "is_write": is_write, + "is_private": is_private, + "source": request.args.get("source") or None, + "owner_id": request.args.get("owner_id") or None, + } + + async def facet_count(field, value): + if current_filters[field] is not None and current_filters[field] != value: + return 0 + filters = dict(current_filters) + filters[field] = value + return await self.ds.count_queries(database, **filters) + + def facet_href(field, value): + if current_filters[field] == value: + return _query_list_url( + query_list_path, + request.query_string, + remove_args=[field], + ) + if current_filters[field] is not None: + return None + return _query_list_url( + query_list_path, + request.query_string, + set_args={field: str(int(value))}, + ) + + async def facet_item(label, field, value): + count = await facet_count(field, value) + active = current_filters[field] == value + if not active and not count: + return None + return { + "label": label, + "count": count, + "href": facet_href(field, value) if active or count else None, + "active": active, + } + + async def facet_items(items): + return [ + item + for item in [ + await facet_item(label, field, value) + for label, field, value in items + ] + if item is not None + ] + + facets = [ + { + "title": "Mode", + "items": await facet_items( + [ + ("Read-only", "is_write", False), + ("Writable", "is_write", True), + ] + ), + }, + { + "title": "Visibility", + "items": await facet_items( + [ + ("Not private", "is_private", False), + ("Private", "is_private", True), + ] + ), + }, + ] + + data = { + "ok": True, + "database": database, + "database_color": ( + self.ds.get_database(database).color if database is not None else None + ), + "queries": page.queries, + "next": page.next, + "next_url": next_url, + "has_more": page.has_more, + "limit": page.limit, + "show_private_note": any(query.is_private for query in page.queries), + "show_trusted_note": any(query.is_trusted for query in page.queries), + "query_list_path": query_list_path, + "show_database": database is None, + "facets": facets, + "filters": { + "q": request.args.get("q") or "", + "is_write": request.args.get("is_write") or "", + "is_private": request.args.get("is_private") or "", + "source": request.args.get("source") or "", + "owner_id": request.args.get("owner_id") or "", + }, + } + if format_ == "json": + return Response.json( + { + **data, + "queries": [stored_query_to_dict(query) for query in page.queries], + } + ) + return await self.render( + ["query_list.html"], + request, + data, + ) + + +class GlobalQueryListView(QueryListView): + name = "global-query-list" + + async def database_name(self, request): + return None + + def query_list_path(self, database): + return self.ds.urls.path("/-/queries") + + +class QueryCreateView(BaseView): + name = "query-create" + has_json_alternate = False + + async def _render_form( + self, + request, + db, + *, + sql="", + name="", + title="", + description="", + is_private=True, + status=200, + ): + response = await self.render( + ["query_create.html"], + request, + await _query_create_form_context( + self.ds, + request, + db, + sql=sql, + name=name, + title=title, + description=description, + is_private=is_private, + ), + ) + response.status = status + return response + + async def get(self, request): + db = await self.ds.resolve_database(request) + await self.ds.ensure_permission( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ) + await self.ds.ensure_permission( + action="store-query", + resource=DatabaseResource(db.name), + actor=request.actor, + ) + + return await self._render_form(request, db, sql=request.args.get("sql") or "") + + +class QueryCreateAnalyzeView(BaseView): + name = "query-create-analyze" + has_json_alternate = False + + async def get(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing(_error(["Permission denied: need execute-sql"], 403)) + if not await self.ds.allowed( + action="store-query", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing(_error(["Permission denied: need store-query"], 403)) + + invalid_keys = set(request.args) - {"sql"} + if invalid_keys: + return _block_framing( + _error( + ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], + 400, + ) + ) + sql = request.args.get("sql") or "" + return _block_framing( + Response.json( + await _query_create_analysis_data(self.ds, db, sql, request.actor) + ) + ) + + +class QueryStoreView(QueryCreateView): + name = "query-store" + + async def _error_response(self, request, db, query_data, message, status): + message = _query_create_form_error_message(message) + self.ds.add_message(request, message, self.ds.ERROR) + return await self._render_form( + request, + db, + sql=query_data.get("sql") or "", + name=query_data.get("name") or "", + title=query_data.get("title") or "", + description=query_data.get("description") or "", + is_private=_as_bool(query_data.get("is_private", True)), + status=status, + ) + + async def post(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _error(["Permission denied: need execute-sql"], 403) + if not await self.ds.allowed( + action="store-query", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _error(["Permission denied: need store-query"], 403) + + is_json = False + query_data = {} + try: + data, is_json = await _json_or_form_payload(request) + if not isinstance(data, dict): + raise QueryValidationError("JSON must be a dictionary") + query_data = data.get("query") if is_json else data + if not isinstance(query_data, dict): + raise QueryValidationError("JSON must contain a query dictionary") + prepared = await _prepare_query_create(self.ds, request, db, query_data) + except QueryValidationError as ex: + if not is_json and isinstance(query_data, dict): + return await self._error_response( + request, db, query_data, ex.message, ex.status + ) + return _error([ex.message], ex.status) + + prepared.pop("analysis") + name = prepared.pop("name") + try: + await self.ds.add_query(db.name, name, replace=False, **prepared) + except sqlite3.IntegrityError as ex: + if not is_json and isinstance(query_data, dict): + return await self._error_response(request, db, query_data, str(ex), 400) + return _error([str(ex)], 400) + + query = await self.ds.get_query(db.name, name) + assert query is not None + if is_json: + return Response.json( + {"ok": True, "query": stored_query_to_dict(query)}, status=201 + ) + self.ds.add_message(request, "Query saved", self.ds.INFO) + return Response.redirect(self.ds.urls.path(self.ds.urls.table(db.name, name))) + + +class QueryDefinitionView(BaseView): + name = "query-definition" + + async def get(self, request): + db = await self.ds.resolve_database(request) + query_name = tilde_decode(request.url_vars["query"]) + query = await self.ds.get_query(db.name, query_name) + if query is None: + return _error(["Query not found: {}".format(query_name)], 404) + if not await self.ds.allowed( + action="view-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ): + return _error(["Permission denied"], 403) + return Response.json({"ok": True, "query": stored_query_to_dict(query)}) + + +class QueryUpdateView(BaseView): + name = "query-update" + + async def post(self, request): + db = await self.ds.resolve_database(request) + query_name = tilde_decode(request.url_vars["query"]) + existing = await self.ds.get_query(db.name, query_name) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + if not await self.ds.allowed( + action="update-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ): + return _error(["Permission denied: need update-query"], 403) + if existing.is_trusted: + return _error(["Trusted queries cannot be updated using the API"], 403) + + try: + data, _ = await _json_or_form_payload(request) + if not isinstance(data, dict): + raise QueryValidationError("JSON must be a dictionary") + invalid_keys = set(data) - {"update", "return"} + if invalid_keys: + raise QueryValidationError( + "Invalid keys: {}".format(", ".join(invalid_keys)) + ) + update = data.get("update") + if not isinstance(update, dict): + raise QueryValidationError("JSON must contain an update dictionary") + if "sql" in update and not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + raise QueryValidationError( + "Permission denied: need execute-sql", status=403 + ) + update_kwargs = await _prepare_query_update( + self.ds, request, db, existing, update + ) + except QueryValidationError as ex: + return _error([ex.message], ex.status) + + await self.ds.update_query(db.name, query_name, **update_kwargs) + if data.get("return"): + query = await self.ds.get_query(db.name, query_name) + assert query is not None + return Response.json( + { + "ok": True, + "query": stored_query_to_dict(query), + } + ) + return Response.json({"ok": True}) + + +class QueryDeleteView(BaseView): + name = "query-delete" + + async def post(self, request): + db = await self.ds.resolve_database(request) + query_name = tilde_decode(request.url_vars["query"]) + existing = await self.ds.get_query(db.name, query_name) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + if not await self.ds.allowed( + action="delete-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ): + return _error(["Permission denied: need delete-query"], 403) + await self.ds.remove_query(db.name, query_name) + return Response.json({"ok": True}) diff --git a/datasette/views/table.py b/datasette/views/table.py index 7027bb10..da69c6b5 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -963,12 +963,12 @@ async def table_view_traced(datasette, request): try: resolved = await datasette.resolve_table(request) except TableNotFound as not_found: - # Was this actually a canned query? - canned_query = await datasette.get_canned_query( - not_found.database_name, not_found.table, request.actor + # Was this actually a stored query? + stored_query = await datasette.get_query( + not_found.database_name, not_found.table ) - # If this is a canned query, not a table, then dispatch to QueryView instead - if canned_query: + # If this is a stored query, not a table, then dispatch to QueryView instead + if stored_query: return await QueryView()(request, datasette) else: raise diff --git a/docs/authentication.rst b/docs/authentication.rst index 7daefab7..f720c12f 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -121,7 +121,7 @@ This configuration will deny access to everyone except the user with ``id`` of ` How permissions are resolved ---------------------------- -Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. +Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified. @@ -468,7 +468,7 @@ You can control the following: * Access to the entire Datasette instance * Access to specific databases * Access to specific tables and views -* Access to specific :ref:`canned_queries` +* Access to specific :ref:`queries ` If a user has permission to view a table they will be able to view that table, independent of if they have permission to view the database or instance that the table exists within. @@ -641,12 +641,12 @@ This works for SQL views as well - you can list their names in the ``"tables"`` .. _authentication_permissions_query: -Access to specific canned queries ---------------------------------- +Access to specific queries +-------------------------- -:ref:`canned_queries` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. +:ref:`Queries ` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. -To limit access to the ``add_name`` canned query in your ``dogs.db`` database to just the :ref:`root user`: +To limit access to the ``add_name`` query in your ``dogs.db`` database to just the :ref:`root user`: .. [[[cog config_example(cog, """ @@ -1020,7 +1020,7 @@ You can also restrict permissions such that they can only be used within specifi The resulting token will only be able to insert rows, and only to tables in the ``mydatabase`` database. -Finally, you can restrict permissions to individual resources - tables, SQL views and :ref:`named queries ` - within a specific database:: +Finally, you can restrict permissions to individual resources - tables, SQL views and :ref:`named queries ` - within a specific database:: datasette create-token root --resource mydatabase mytable insert-row @@ -1285,12 +1285,46 @@ Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.i view-query ---------- -Actor is allowed to view (and execute) a :ref:`canned query ` page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size - this includes executing :ref:`canned_queries_writable`. +Actor is allowed to view a stored query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size. Executing an untrusted stored query also requires ``execute-sql`` or the relevant write permissions; :ref:`trusted stored queries ` can execute with ``view-query`` alone. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) - - ``query`` is the name of the canned query (string) + + ``query`` is the name of the query (string) + +.. _actions_store_query: + +store-query +----------- + +Actor is allowed to create stored queries against a database. + +``resource`` - ``datasette.resources.DatabaseResource(database)`` + ``database`` is the name of the database (string) + +.. _actions_update_query: + +update-query +------------ + +Actor is allowed to update a stored query. + +``resource`` - ``datasette.resources.QueryResource(database, query)`` + ``database`` is the name of the database (string) + + ``query`` is the name of the query (string) + +.. _actions_delete_query: + +delete-query +------------ + +Actor is allowed to delete a stored query. + +``resource`` - ``datasette.resources.QueryResource(database, query)`` + ``database`` is the name of the database (string) + + ``query`` is the name of the query (string) .. _actions_insert_row: @@ -1379,13 +1413,23 @@ Actor is allowed to drop a database table. execute-sql ----------- -Actor is allowed to run arbitrary SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100 +Actor is allowed to run arbitrary read-only SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100 ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) See also :ref:`the default_allow_sql setting `. +.. _actions_execute_write_sql: + +execute-write-sql +----------------- + +Actor is allowed to run arbitrary writable SQL queries against a specific database, subject to table-level write permissions such as ``insert-row``, ``update-row`` and ``delete-row``. + +``resource`` - ``datasette.resources.DatabaseResource(database)`` + ``database`` is the name of the database (string) + .. _actions_permissions_debug: permissions-debug diff --git a/docs/changelog.rst b/docs/changelog.rst index dfb2a736..2ba713ee 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,31 @@ Changelog Unreleased ---------- +Stored queries +~~~~~~~~~~~~~~ + +- The previous "canned queries" feature has been renamed and expanded into :ref:`stored queries `. Queries configured in ``datasette.yaml`` are now loaded into a new ``queries`` table in Datasette's :ref:`internal database `, alongside user-created stored queries. (:issue:`2735`) +- New stored query management APIs: ``datasette.add_query()``, ``datasette.update_query()``, ``datasette.remove_query()``, ``datasette.get_query()``, ``datasette.list_queries()`` and ``datasette.count_queries()``. These replace the removed ``datasette.get_canned_query()`` and ``datasette.get_canned_queries()`` methods. (:issue:`2735`) +- Users with :ref:`store-query ` and :ref:`execute-sql ` permission can create stored queries from the SQL query page or the new ``GET //-/queries/store`` form. (:issue:`2735`) +- The database page now shows a count and preview of stored queries, capped at five, and links to new paginated query browsers at ``/-/queries`` and ``//-/queries``. Those browsers support search. (:issue:`2735`) +- Stored queries created by users default to private and untrusted. Private stored queries can only be viewed, updated or deleted by their owner, even if another actor has broad ``view-query``, ``update-query`` or ``delete-query`` permission. Untrusted stored queries execute using the permissions of the actor running them. See :ref:`stored_queries` and :ref:`trusted_stored_queries` for details. (:issue:`2735`) +- New ``store-query``, ``update-query`` and ``delete-query`` permissions, plus updated semantics for :ref:`view-query `. Trusted stored queries can still execute with ``view-query`` alone; untrusted read queries also require :ref:`execute-sql ` and untrusted writable queries require :ref:`execute-write-sql ` plus the relevant table-level write permissions. (:issue:`2735`) + +Write SQL UI +~~~~~~~~~~~~ + +- New "Write to this database" interface at ``//-/execute-write`` for running arbitrary writable SQL against mutable databases. The form extracts named parameters, analyzes the SQL, shows the table operations that will be attempted and links to a newly inserted row when a single-row insert succeeds. (:issue:`2742`) +- Added the new :ref:`execute-write-sql ` permission for running arbitrary writable SQL. Execution is also gated by table-level permissions such as :ref:`insert-row `, :ref:`update-row ` and :ref:`delete-row `, and writes to attached databases are rejected. (:issue:`2742`) + +Plugin API changes +~~~~~~~~~~~~~~~~~~ + +- The ``top_canned_query()`` plugin hook has been renamed to :ref:`top_stored_query() `. (:issue:`2747`) +- The ``canned_queries()`` plugin hook has been removed. Plugins can use the new :ref:`stored query management methods ` together with :ref:`startup() ` to register queries. (:issue:`2735`) + +Bug fixes +~~~~~~~~~ + - Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) .. _v1_0_a30: @@ -656,7 +681,7 @@ For more information and workarounds, read `the security advisory `` in a ``
    `` element, to help with advanced CSS customization. (:issue:`1446`) - The :ref:`render_cell() ` plugin hook can now return an awaitable function. This means the hook can execute SQL queries. (:issue:`1425`) - :ref:`plugin_register_routes` plugin hook now accepts an optional ``datasette`` argument. (:issue:`1404`) -- New ``hide_sql`` canned query option for defaulting to hiding the SQL query used by a canned query, see :ref:`canned_queries_options`. (:issue:`1422`) +- New ``hide_sql`` canned query option for defaulting to hiding the SQL query used by a canned query, see :ref:`queries_options`. (:issue:`1422`) - New ``--cpu`` option for :ref:`datasette publish cloudrun `. (:issue:`1420`) - If `Rich `__ is installed in the same virtual environment as Datasette, it will be used to provide enhanced display of error tracebacks on the console. (:issue:`1416`) - ``datasette.utils`` :ref:`internals_utils_parse_metadata` function, used by the new `datasette-remote-metadata plugin `__, is now a documented API. (:issue:`1405`) @@ -1424,7 +1449,7 @@ See also `Datasette 0.50: The annotated release notes `__. -- Writable canned queries now expose a JSON API, see :ref:`canned_queries_json_api`. (:issue:`880`) +- Writable canned queries now expose a JSON API, see :ref:`queries_json_api`. (:issue:`880`) - New mechanism for defining page templates with custom path parameters - a template file called ``pages/about/{slug}.html`` will be used to render any requests to ``/about/something``. See :ref:`custom_pages_parameters`. (:issue:`944`) - ``register_output_renderer()`` render functions can now return a ``Response``. (:issue:`953`) - New ``--upgrade`` option for ``datasette install``. (:issue:`945`) @@ -1516,7 +1541,7 @@ Magic parameters for canned queries, a log out feature, improved plugin document Magic parameters for canned queries ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Canned queries now support :ref:`canned_queries_magic_parameters`, which can be used to insert or select automatically generated values. For example:: +Canned queries now support :ref:`queries_magic_parameters`, which can be used to insert or select automatically generated values. For example:: insert into logs (user_id, timestamp) @@ -1547,7 +1572,7 @@ New plugin hooks - :ref:`plugin_hook_register_magic_parameters` can be used to define new types of magic canned query parameters. - :ref:`plugin_hook_startup` can run custom code when Datasette first starts up. `datasette-init `__ is a new plugin that uses this hook to create database tables and views on startup if they have not yet been created. (:issue:`834`) -- :ref:`plugin_hook_canned_queries` lets plugins provide additional canned queries beyond those defined in Datasette's metadata. See `datasette-saved-queries `__ for an example of this hook in action. (:issue:`852`) +- ``canned_queries()`` lets plugins provide additional canned queries beyond those defined in Datasette's metadata. See `datasette-saved-queries `__ for an example of this hook in action. (:issue:`852`) - :ref:`plugin_hook_forbidden` is a hook for customizing how Datasette responds to 403 forbidden errors. (:issue:`812`) Smaller changes @@ -1622,7 +1647,7 @@ A new debug page at ``/-/permissions`` shows recent permission checks, to help a Writable canned queries ~~~~~~~~~~~~~~~~~~~~~~~ -Datasette's :ref:`canned_queries` feature lets you define SQL queries in ``metadata.json`` which can then be executed by users visiting a specific URL. https://latest.datasette.io/fixtures/neighborhood_search for example. +Datasette's :ref:`queries` feature lets you define SQL queries in ``metadata.json`` which can then be executed by users visiting a specific URL. https://latest.datasette.io/fixtures/neighborhood_search for example. Canned queries were previously restricted to ``SELECT``, but Datasette 0.44 introduces the ability for canned queries to execute ``INSERT`` or ``UPDATE`` queries as well, using the new ``"write": true`` property (:issue:`800`): @@ -1641,7 +1666,7 @@ Canned queries were previously restricted to ``SELECT``, but Datasette 0.44 intr } } -See :ref:`canned_queries_writable` for more details. +See :ref:`queries_writable` for more details. Flash messages ~~~~~~~~~~~~~~ @@ -1696,7 +1721,7 @@ Smaller changes - New ``request.cookies`` property. - ``/-/plugins`` endpoint now shows a list of hooks implemented by each plugin, e.g. https://latest.datasette.io/-/plugins?all=1 - ``request.post_vars()`` method no longer discards empty values. -- New "params" canned query key for explicitly setting named parameters, see :ref:`canned_queries_named_parameters`. (:issue:`797`) +- New "params" canned query key for explicitly setting named parameters, see :ref:`queries_named_parameters`. (:issue:`797`) - ``request.args`` is now a :ref:`MultiParams ` object. - Fixed a bug with the ``datasette plugins`` command. (:issue:`802`) - Nicer pattern for using ``make_app_client()`` in tests. (:issue:`395`) @@ -1730,7 +1755,7 @@ The main focus of this release is a major upgrade to the :ref:`plugin_register_o * Visually distinguish float and integer columns - useful for figuring out why order-by-column might be returning unexpected results. (:issue:`729`) * The :ref:`internals_request`, which is passed to several plugin hooks, is now documented. (:issue:`706`) * New ``metadata.json`` option for setting a custom default page size for specific tables and views, see :ref:`table_configuration_size`. (:issue:`751`) -* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega `__, see :ref:`canned_queries_options`. (:issue:`706`) +* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega `__, see :ref:`queries_options`. (:issue:`706`) * Fixed a bug in ``datasette publish`` when running on operating systems where the ``/tmp`` directory lives in a different volume, using a backport of the Python 3.8 ``shutil.copytree()`` function. (:issue:`744`) * Every plugin hook is now covered by the unit tests, and a new unit test checks that each plugin hook has at least one corresponding test. (:issue:`771`, :issue:`773`) @@ -2247,7 +2272,7 @@ A number of small new features: - Documentation for :ref:`datasette publish and datasette package `, closes `#337 `_ - Fixed compatibility with Python 3.7 - ``datasette publish heroku`` now supports app names via the ``-n`` option, which can also be used to overwrite an existing application [Russ Garrett] -- Title and description metadata can now be set for :ref:`canned SQL queries `, closes `#342 `_ +- Title and description metadata can now be set for :ref:`canned SQL queries `, closes `#342 `_ - New ``force_https_on`` config option, fixes ``https://`` API URLs when deploying to Zeit Now - closes `#333 `_ - ``?_json_infinity=1`` query string argument for handling Infinity/-Infinity values in JSON, closes `#332 `_ - URLs displayed in the results of custom SQL queries are now URLified, closes `#298 `_ diff --git a/docs/configuration.rst b/docs/configuration.rst index 8c8c8a67..9213a640 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -87,6 +87,7 @@ This is equivalent to a ``datasette.yaml`` file containing the following: } .. [[[end]]] + .. _configuration_reference: ``datasette.yaml`` reference @@ -433,12 +434,12 @@ Here is a simple example: :ref:`authentication_permissions_config` has the full details. -.. _configuration_reference_canned_queries: +.. _configuration_reference_queries: -Canned queries configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Queries configuration +~~~~~~~~~~~~~~~~~~~~~ -:ref:`Canned queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: +:ref:`Queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: .. [[[cog from metadata_doc import config_example, config_example @@ -483,7 +484,7 @@ Canned queries configuration } .. [[[end]]] -See the :ref:`canned queries documentation ` for more, including how to configure :ref:`writable canned queries `. +See the :ref:`queries documentation ` for more, including how to configure :ref:`writable queries `. .. _configuration_reference_css_js: @@ -1211,4 +1212,3 @@ For column types that accept additional configuration, use an object with ``type } } .. [[[end]]] - diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 8cc40f0f..c324fb79 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -29,7 +29,7 @@ The custom SQL template (``/dbname?sql=...``) gets this: -A canned query template (``/dbname/queryname``) gets this: +A stored query template (``/dbname/queryname``) gets this: .. code-block:: html @@ -193,8 +193,8 @@ The lookup rules Datasette uses are as follows:: query-mydatabase.html query.html - Canned query page (/mydatabase/canned-query): - query-mydatabase-canned-query.html + Stored query page (/mydatabase/query-name): + query-mydatabase-query-name.html query-mydatabase.html query.html @@ -230,7 +230,7 @@ will look something like this:: -This example is from the canned query page for a query called "tz" in the +This example is from the stored query page for a query called "tz" in the database called "mydb". The asterisk shows which template was selected - so in this case, Datasette found a template file called ``query-mydb-tz.html`` and used that - but if that template had not been found, it would have tried for diff --git a/docs/internals.rst b/docs/internals.rst index e0123a7b..4980ee8b 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -725,7 +725,7 @@ The builder methods are: - ``allow_all(action)`` - allow an action across all databases and resources - ``allow_database(database, action)`` - allow an action on a specific database -- ``allow_resource(database, resource, action)`` - allow an action on a specific resource (table, SQL view or :ref:`canned query `) within a database +- ``allow_resource(database, resource, action)`` - allow an action on a specific resource (table, SQL view or :ref:`stored query `) within a database Each method returns the ``TokenRestrictions`` instance so calls can be chained. @@ -837,10 +837,10 @@ await .get_resource_metadata(self, database_name, resource_name) ``database_name`` - string The name of the database to query. ``resource_name`` - string - The name of the resource (table, view, or canned query) inside ``database_name`` to query. + The name of the resource (table, view, or stored query) inside ``database_name`` to query. Returns metadata keys and values for the specified "resource" as a dictionary. -A "resource" in this context can be a table, view, or canned query. +A "resource" in this context can be a table, view, or stored query. Internally queries the ``metadata_resources`` table inside the :ref:`internal database `. .. _datasette_get_column_metadata: @@ -851,7 +851,7 @@ await .get_column_metadata(self, database_name, resource_name, column_name) ``database_name`` - string The name of the database to query. ``resource_name`` - string - The name of the resource (table, view, or canned query) inside ``database_name`` to query. + The name of the resource (table, view, or stored query) inside ``database_name`` to query. ``column_name`` - string The name of the column inside ``resource_name`` to query. @@ -897,7 +897,7 @@ await .set_resource_metadata(self, database_name, resource_name, key, value) ``database_name`` - string The database the metadata entry belongs to. ``resource_name`` - string - The resource (table, view, or canned query) the metadata entry belongs to. + The resource (table, view, or stored query) the metadata entry belongs to. ``key`` - string The metadata entry key to insert (ex ``title``, ``description``, etc.) ``value`` - string @@ -915,7 +915,7 @@ await .set_column_metadata(self, database_name, resource_name, column_name, key, ``database_name`` - string The database the metadata entry belongs to. ``resource_name`` - string - The resource (table, view, or canned query) the metadata entry belongs to. + The resource (table, view, or stored query) the metadata entry belongs to. ``column-name`` - string The column the metadata entry belongs to. ``key`` - string @@ -927,6 +927,200 @@ Adds a new metadata entry for the specified column. Any previous column-level metadata entry with the same ``key`` will be overwritten. Internally upserts the value into the the ``metadata_columns`` table inside the :ref:`internal database `. +.. _datasette_stored_queries: + +Stored queries +-------------- + +:ref:`Stored queries ` are stored in the ``queries`` table in the :ref:`internal database `. Plugins can use the following methods to add, update, list and remove stored queries. + +.. _datasette_add_query: + +await .add_query(database, name, sql, ...) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Adds a stored query. + +.. code-block:: python + + async def add_query( + self, + database, + name, + sql, + *, + title=None, + description=None, + description_html=None, + hide_sql=False, + fragment=None, + parameters=None, + is_write=False, + is_private=False, + is_trusted=False, + source="plugin", + owner_id=None, + on_success_message=None, + on_success_message_sql=None, + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + replace=True, + ): ... + +``database`` - string + The name of the database this query should belong to. +``name`` - string + The name of the stored query, used in the URL for that query. +``sql`` - string + The SQL for the stored query. +``title`` - string, optional + A display title for the query. +``description`` - string, optional + A plain text description. +``description_html`` - string, optional + An HTML description. +``hide_sql`` - boolean, optional + Set to ``True`` to hide the SQL by default on the query page. +``fragment`` - string, optional + A URL fragment to append to query links, for example ``"chart"``. +``parameters`` - list of strings, optional + Explicit parameter names for the query form. If omitted, Datasette derives parameters from the SQL. +``is_write`` - boolean, optional + Set to ``True`` for writable queries. They will the run against the SQLite write connection for the database. +``is_private`` - boolean, optional + Set to ``True`` for private queries. Private queries can only be viewed, updated or deleted by their owner. +``is_trusted`` - boolean, optional + Set to ``True`` for :ref:`trusted stored queries `. +``source`` - string, optional + Identifies where the query came from. Defaults to ``"plugin"``. +``owner_id`` - string, optional + Actor ID of the query owner, used by private query permissions. +``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message``, ``on_error_redirect`` - strings, optional + Options for :ref:`writable queries `. +``replace`` - boolean, optional + Defaults to ``True``, which replaces any existing stored query with the same ``database`` and ``name``. Set this to ``False`` to raise a SQLite integrity error if the query already exists. + +Example: + +.. code-block:: python + + await datasette.add_query( + database="fixtures", + name="recent_rows", + sql="select * from facetable order by created desc limit 10", + title="Recent rows", + source="my-plugin", + ) + +.. _datasette_update_query: + +await .update_query(database, name, ...) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Updates fields for an existing stored query. Only keyword arguments that are provided will be changed. + +The available keyword arguments are the same as those for :ref:`datasette_add_query`, except for ``replace``. Pass ``None`` to clear optional text fields and options such as ``on_success_redirect``. Passing ``hide_sql=False`` removes the ``hide_sql`` option. + +Example: + +.. code-block:: python + + await datasette.update_query( + database="fixtures", + name="recent_rows", + title="Latest rows", + is_private=True, + owner_id="alice", + ) + +.. _datasette_get_query: + +await .get_query(database, name) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Returns a ``StoredQuery`` dataclass instance, or ``None`` if the query does not exist. + +``StoredQuery`` has the following attributes: ``database``, ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write``, ``is_private``, ``is_trusted``, ``source``, ``owner_id``, ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``. + +``parameters`` is a list of explicit parameter names. + +.. _datasette_list_queries: + +await .list_queries(database=None, ...) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Lists stored queries visible to the specified actor. + +.. code-block:: python + + async def list_queries( + self, + database=None, + *, + actor=None, + limit=50, + cursor=None, + q=None, + is_write=None, + is_private=None, + is_trusted=None, + source=None, + owner_id=None, + include_private=False, + ): ... + +``database`` - string, optional + Restrict results to a specific database. Omit this to list queries across all databases. +``actor`` - dictionary, optional + The authenticated actor. Results are filtered using that actor's ``view-query`` permission. +``limit`` - integer, optional + Number of queries to return. Values are clamped to the range 1-1000. +``cursor`` - string, optional + Pagination cursor from the previous page's ``next`` value. +``q`` - string, optional + Search string matched against query name, title, description and SQL. +``is_write``, ``is_private``, ``is_trusted`` - boolean, optional + Filter by those stored query flags. +``source`` - string, optional + Filter by query source. +``owner_id`` - string, optional + Filter by owner actor ID. +``include_private`` - boolean, optional + Set to ``True`` to populate a ``private`` boolean on each returned ``StoredQuery`` indicating if anonymous users would be unable to view that query. + +The return value is a ``StoredQueryPage`` dataclass instance with these attributes: + +``queries`` - list of StoredQuery instances + Stored queries in the same format returned by :ref:`datasette_get_query`. +``next`` - string or None + Pagination cursor for the next page, if one exists. +``has_more`` - boolean + ``True`` if another page of results is available. +``limit`` - integer + The limit used for this page. + +.. _datasette_count_queries: + +await .count_queries(database=None, ...) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Counts stored queries visible to the specified actor. This accepts the same filtering keyword arguments as :ref:`datasette_list_queries`, except for ``limit``, ``cursor`` and ``include_private``. + +.. _datasette_remove_query: + +await .remove_query(database, name, source=None) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Removes a stored query. + +``database`` - string + The database the query belongs to. +``name`` - string + The query name. +``source`` - string, optional + If provided, only a query with this source will be removed. + .. _datasette_column_types: Column types @@ -2148,6 +2342,26 @@ The internal database schema is as follows: config TEXT, PRIMARY KEY (database_name, resource_name, column_name) ); + CREATE TABLE queries ( + database_name TEXT NOT NULL, + name TEXT NOT NULL, + sql TEXT NOT NULL, + title TEXT, + description TEXT, + description_html TEXT, + options TEXT NOT NULL DEFAULT '{}', + parameters TEXT NOT NULL DEFAULT '[]', + is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), + source TEXT NOT NULL DEFAULT 'user', + owner_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (database_name, name) + ); + CREATE INDEX queries_owner_idx + ON queries(owner_id); .. [[[end]]] @@ -2219,8 +2433,8 @@ Note that the space character is a special case: it will be replaced with a ``+` .. _internals_utils_call_with_supported_arguments: -call_with_supported_arguments(fn, **kwargs) -------------------------------------------- +call_with_supported_arguments(fn, \*\*kwargs) +--------------------------------------------- Call ``fn``, passing it only those keyword arguments that match its function signature. This implements a dependency injection pattern - the caller provides all available arguments, and the function receives only the ones it declares as parameters. @@ -2247,8 +2461,8 @@ This is useful in plugins that want to define callback functions that only decla .. _internals_utils_async_call_with_supported_arguments: -await async_call_with_supported_arguments(fn, **kwargs) -------------------------------------------------------- +await async_call_with_supported_arguments(fn, \*\*kwargs) +--------------------------------------------------------- Async version of :ref:`call_with_supported_arguments `. Use this for ``async def`` callback functions. diff --git a/docs/introspection.rst b/docs/introspection.rst index d2eb8efd..7702a4b5 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -149,7 +149,7 @@ Shows currently attached databases. `Databases example ` has its own page, at ``/database-name/query-name``. Viewing this page will execute the query and display the results. +Each configured :ref:`stored query ` has its own page, at ``/database-name/query-name``. Viewing this page will execute the query and display the results. In both cases adding a ``.json`` extension to the URL will return the results as JSON. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 2b8f5eb2..2a0ddc93 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -609,7 +609,7 @@ When a request is received, the ``"render"`` callback function is called with ze The SQL query that was executed. ``query_name`` - string or None - If this was the execution of a :ref:`canned query `, the name of that query. + If this was the execution of a :ref:`stored query `, the name of that query. ``database`` - string The name of the database. @@ -1207,85 +1207,6 @@ Potential use-cases: Examples: `datasette-saved-queries `__, `datasette-init `__ -.. _plugin_hook_canned_queries: - -canned_queries(datasette, database, actor) ------------------------------------------- - -``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. - -``database`` - string - The name of the database. - -``actor`` - dictionary or None - The currently authenticated :ref:`actor `. - -Use this hook to return a dictionary of additional :ref:`canned query ` definitions for the specified database. The return value should be the same shape as the JSON described in the :ref:`canned query ` documentation. - -.. code-block:: python - - from datasette import hookimpl - - - @hookimpl - def canned_queries(datasette, database): - if database == "mydb": - return { - "my_query": { - "sql": "select * from my_table where id > :min_id" - } - } - -The hook can alternatively return an awaitable function that returns a list. Here's an example that returns queries that have been stored in the ``saved_queries`` database table, if one exists: - -.. code-block:: python - - from datasette import hookimpl - - - @hookimpl - def canned_queries(datasette, database): - async def inner(): - db = datasette.get_database(database) - if await db.table_exists("saved_queries"): - results = await db.execute( - "select name, sql from saved_queries" - ) - return { - result["name"]: {"sql": result["sql"]} - for result in results - } - - return inner - -The actor parameter can be used to include the currently authenticated actor in your decision. Here's an example that returns saved queries that were saved by that actor: - -.. code-block:: python - - from datasette import hookimpl - - - @hookimpl - def canned_queries(datasette, database, actor): - async def inner(): - db = datasette.get_database(database) - if actor is not None and await db.table_exists( - "saved_queries" - ): - results = await db.execute( - "select name, sql from saved_queries where actor_id = :id", - {"id": actor["id"]}, - ) - return { - result["name"]: {"sql": result["sql"]} - for result in results - } - - return inner - -Example: `datasette-saved-queries `__ - .. _plugin_hook_actor_from_request: actor_from_request(datasette, request) @@ -1704,7 +1625,7 @@ register_magic_parameters(datasette) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. -:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`canned queries `. This plugin hook allows additional magic parameters to be defined by plugins. +:ref:`queries_magic_parameters` can be used to add automatic parameters to :ref:`configured queries `. This plugin hook allows additional magic parameters to be defined by plugins. Magic parameters all take this format: ``_prefix_rest_of_parameter``. The prefix indicates which magic parameter function should be called - the rest of the parameter is passed as an argument to that function. @@ -1897,7 +1818,7 @@ jump_items_sql(datasette, actor, request) This hook allows plugins to add extra results to Datasette's ``/`` jump menu, which is powered by the ``/-/jump`` JSON endpoint. -Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be searched alongside Datasette's own databases, tables, views and canned query results. The hook can also be an ``async def`` function, or return an awaitable that resolves to one of these values. +Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be searched alongside Datasette's own databases, tables, views and stored query results. The hook can also be an ``async def`` function, or return an awaitable that resolves to one of these values. ``JumpSQL`` queries run against Datasette's internal database by default. To run a query against another database, pass its name as the optional ``database=`` argument. For example, ``JumpSQL(database="content", sql="...")`` runs against the ``content`` database. @@ -2073,7 +1994,7 @@ query_actions(datasette, actor, database, query_name, request, sql, params) The name of the database. ``query_name`` - string or None - The name of the canned query, or ``None`` if this is an arbitrary SQL query. + The name of the stored query, or ``None`` if this is an arbitrary SQL query. ``request`` - :ref:`internals_request` The current HTTP request. @@ -2084,7 +2005,7 @@ query_actions(datasette, actor, database, query_name, request, sql, params) ``params`` - dictionary The parameters passed to the SQL query, if any. -Populates a "Query actions" menu on the canned query and arbitrary SQL query pages. +Populates a "Query actions" menu on the stored query and arbitrary SQL query pages. This example adds a new query action linking to a page for explaining a query: @@ -2348,9 +2269,9 @@ top_query(datasette, request, database, sql) Returns HTML to be displayed at the top of the query results page. -.. _plugin_hook_top_canned_query: +.. _plugin_hook_top_stored_query: -top_canned_query(datasette, request, database, query_name) +top_stored_query(datasette, request, database, query_name) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``datasette`` - :ref:`internals_datasette` @@ -2363,9 +2284,9 @@ top_canned_query(datasette, request, database, query_name) The name of the database. ``query_name`` - string - The name of the canned query. + The name of the stored query. -Returns HTML to be displayed at the top of the canned query page. +Returns HTML to be displayed at the top of the stored query page. .. _plugin_event_tracking: diff --git a/docs/plugins.rst b/docs/plugins.rst index 77958205..d578e9e2 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -216,6 +216,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "register_column_types" ] }, + { + "name": "datasette.default_database_actions", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "database_actions" + ] + }, { "name": "datasette.default_debug_menu", "static": false, @@ -249,7 +258,6 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "templates": false, "version": null, "hooks": [ - "canned_queries", "permission_resources_sql" ] }, diff --git a/docs/spatialite.rst b/docs/spatialite.rst index c93c1e00..bea679aa 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -30,7 +30,7 @@ Warning The following steps are recommended: - Disable arbitrary SQL queries by untrusted users. See :ref:`authentication_permissions_execute_sql` for ways to do this. The easiest is to start Datasette with the ``datasette --setting default_allow_sql off`` option. - - Define :ref:`canned_queries` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. + - Define :ref:`queries ` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. The `Datasette SpatiaLite tutorial `__ includes detailed instructions for running SpatiaLite safely using these techniques diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 7c3cd4ac..f593a534 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -66,12 +66,12 @@ You can also use the `sqlite-utils `__ tool sqlite-utils create-view sf-trees.db demo_view "select qSpecies from Street_Tree_List" -.. _canned_queries: +.. _queries: -Canned queries --------------- +Queries +------- -As an alternative to adding views to your database, you can define canned queries inside your ``datasette.yaml`` file. Here's an example: +As an alternative to adding views to your database, you can define named queries inside your ``datasette.yaml`` file. Here's an example: .. [[[cog from metadata_doc import config_example, config_example @@ -120,24 +120,67 @@ Then run Datasette like this:: datasette sf-trees.db -m metadata.json -Each canned query will be listed on the database index page, and will also get its own URL at:: +Each configured query will be listed on the database index page, and will also get its own URL at:: - /database-name/canned-query-name + /database-name/query-name For the above example, that URL would be:: /sf-trees/just_species -You can optionally include ``"title"`` and ``"description"`` keys to show a title and description on the canned query page. As with regular table metadata you can alternatively specify ``"description_html"`` to have your description rendered as HTML (rather than having HTML special characters escaped). +You can optionally include ``"title"`` and ``"description"`` keys to show a title and description on the query page. As with regular table metadata you can alternatively specify ``"description_html"`` to have your description rendered as HTML (rather than having HTML special characters escaped). -.. _canned_queries_named_parameters: +.. _stored_queries: +.. _saved_queries: -Canned query parameters -~~~~~~~~~~~~~~~~~~~~~~~ +Stored queries +~~~~~~~~~~~~~~ -Canned queries support named parameters, so if you include those in the SQL you will then be able to enter them using the form fields on the canned query page or by adding them to the URL. This means canned queries can be used to create custom JSON APIs based on a carefully designed SQL statement. +Datasette stores both configured queries and user-created queries in the ``queries`` table in the :ref:`internal database `. Configured queries come from the ``queries`` section of ``datasette.yaml``. User-created stored queries can be created from the SQL query page by actors with the :ref:`actions_store_query` and :ref:`actions_execute_sql` permissions. Writable stored queries also require the permissions needed for the writes they perform. -Here's an example of a canned query with a named parameter: +Stored queries created by users default to private. Private stored queries can only be viewed, updated or deleted by the actor that created them. Broad ``view-query``, ``update-query`` or ``delete-query`` permission grants still do not allow other actors to access another actor's private stored queries. + +Stored queries created by users are untrusted. This means they execute using the permissions of the actor who runs them, as if that actor had pasted the SQL into the regular custom SQL interface or write SQL interface. Read-only stored queries require ``execute-sql``. Writable stored queries require ``execute-write-sql`` plus the relevant table-level write permissions. + +.. _trusted_stored_queries: +.. _trusted_saved_queries: + +Trusted stored queries +++++++++++++++++++++++ + +A trusted stored query can execute with ``view-query`` permission alone. It skips the additional ``execute-sql`` and write permission checks that are applied to untrusted stored queries. + +Trusted stored queries should only be used for SQL that has been reviewed by someone trusted to configure the Datasette instance. For that reason, trusted stored queries can only be added using configuration. Users cannot create trusted stored queries through the web interface or the stored query JSON API. + +Queries defined in ``datasette.yaml`` are trusted by default: + +.. code-block:: yaml + + databases: + mydatabase: + queries: + report: + sql: select * from report + +You can opt out of this behavior for a configured query using ``is_trusted: false``: + +.. code-block:: yaml + + databases: + mydatabase: + queries: + report: + sql: select * from report + is_trusted: false + +.. _queries_named_parameters: + +Query parameters +~~~~~~~~~~~~~~~~ + +Configured queries support named parameters, so if you include those in the SQL you will then be able to enter them using the form fields on the query page or by adding them to the URL. This means configured queries can be used to create custom JSON APIs based on a carefully designed SQL statement. + +Here's an example of a configured query with a named parameter: .. code-block:: sql @@ -147,7 +190,7 @@ Here's an example of a canned query with a named parameter: where neighborhood like '%' || :text || '%' order by neighborhood; -In the canned query configuration looks like this: +The query configuration looks like this: .. [[[cog @@ -204,7 +247,7 @@ In the canned query configuration looks like this: Note that we are using SQLite string concatenation here - the ``||`` operator - to add wildcard ``%`` characters to the string provided by the user. -You can try this canned query out here: +You can try this query out here: https://latest.datasette.io/fixtures/neighborhood_search?text=town In this example the ``:text`` named parameter is automatically extracted from the query using a regular expression. @@ -270,17 +313,17 @@ You can alternatively provide an explicit list of named parameters using the ``" } .. [[[end]]] -.. _canned_queries_options: +.. _queries_options: -Additional canned query options -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Additional query options +~~~~~~~~~~~~~~~~~~~~~~~~ -Additional options can be specified for canned queries in the YAML or JSON configuration. +Additional options can be specified for configured queries in the YAML or JSON configuration. hide_sql ++++++++ -Canned queries default to displaying their SQL query at the top of the page. If the query is extremely long you may want to hide it by default, with a "show" link that can be used to make it visible. +Configured queries default to displaying their SQL query at the top of the page. If the query is extremely long you may want to hide it by default, with a "show" link that can be used to make it visible. Add the ``"hide_sql": true`` option to hide the SQL query by default. @@ -289,7 +332,7 @@ fragment Some plugins, such as `datasette-vega `__, can be configured by including additional data in the fragment hash of the URL - the bit that comes after a ``#`` symbol. -You can set a default fragment hash that will be included in the link to the canned query from the database index page using the ``"fragment"`` key. +You can set a default fragment hash that will be included in the link to the query from the database index page using the ``"fragment"`` key. This example demonstrates both ``fragment`` and ``hide_sql``: @@ -346,14 +389,14 @@ This example demonstrates both ``fragment`` and ``hide_sql``: `See here `__ for a demo of this in action. -.. _canned_queries_writable: +.. _queries_writable: -Writable canned queries -~~~~~~~~~~~~~~~~~~~~~~~ +Writable queries +~~~~~~~~~~~~~~~~ -Canned queries by default are read-only. You can use the ``"write": true`` key to indicate that a canned query can write to the database. +Configured queries are read-only by default. You can use the ``"write": true`` key to indicate that a query can write to the database. -See :ref:`authentication_permissions_query` for details on how to add permission checks to canned queries, using the ``"allow"`` key. +See :ref:`authentication_permissions_query` for details on how to add permission checks to queries, using the ``"allow"`` key. .. [[[cog config_example(cog, { @@ -481,14 +524,14 @@ You can pre-populate form fields when the page first loads using a query string, If you specify a query in ``"on_success_message_sql"``, that query will be executed after the main query. The first column of the first row return by that query will be displayed as a success message. Named parameters from the main query will be made available to the success message query as well. -.. _canned_queries_magic_parameters: +.. _queries_magic_parameters: Magic parameters ~~~~~~~~~~~~~~~~ Named parameters that start with an underscore are special: they can be used to automatically add values created by Datasette that are not contained in the incoming form fields or query string. -These magic parameters are only supported for canned queries: to avoid security issues (such as queries that extract the user's private cookies) they are not available to SQL that is executed by the user as a custom SQL query. +These magic parameters are only supported for configured queries: to avoid security issues (such as queries that extract the user's private cookies) they are not available to SQL that is executed by the user as a custom SQL query. Available magic parameters are: @@ -578,14 +621,14 @@ The form presented at ``/mydatabase/add_message`` will have just a field for ``m Additional custom magic parameters can be added by plugins using the :ref:`plugin_hook_register_magic_parameters` hook. -.. _canned_queries_json_api: +.. _queries_json_api: -JSON API for writable canned queries -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +JSON API for writable queries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Writable canned queries can also be accessed using a JSON API. You can POST data to them using JSON, and you can request that their response is returned to you as JSON. +Writable queries can also be accessed using a JSON API. You can POST data to them using JSON, and you can request that their response is returned to you as JSON. -To submit JSON to a writable canned query, encode key/value parameters as a JSON document:: +To submit JSON to a writable query, encode key/value parameters as a JSON document:: POST /mydatabase/add_message diff --git a/docs/upgrade-1.0a20.md b/docs/upgrade-1.0a20.md index fbc3f4a8..b27d8bd5 100644 --- a/docs/upgrade-1.0a20.md +++ b/docs/upgrade-1.0a20.md @@ -244,7 +244,7 @@ except (KeyError, TypeError): New code: ```python try: - query_info = await datasette.get_canned_query(database, query_name, request.actor) + query_info = await datasette.get_query(database, query_name) if query_info and "title" in query_info: title = query_info["title"] except (KeyError, TypeError): @@ -253,7 +253,7 @@ except (KeyError, TypeError): ### Update render functions to async -If your plugin's render function needs to call `datasette.get_canned_query()` or other async Datasette methods, it must be declared as async: +If your plugin's render function needs to call `datasette.get_query()` or other async Datasette methods, it must be declared as async: Old code: ```python @@ -268,7 +268,7 @@ New code: async def render_atom(datasette, request, sql, columns, rows, database, table, query_name, view_name, data): # ... if query_name: - query_info = await datasette.get_canned_query(database, query_name, request.actor) + query_info = await datasette.get_query(database, query_name) if query_info and "title" in query_info: title = query_info["title"] ``` diff --git a/tests/fixtures.py b/tests/fixtures.py index 71884294..8ab3633f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -35,7 +35,6 @@ EXPECTED_PLUGINS = [ "hooks": [ "actor_from_request", "asgi_wrapper", - "canned_queries", "database_actions", "extra_body_script", "extra_css_urls", @@ -68,7 +67,6 @@ EXPECTED_PLUGINS = [ "hooks": [ "actor_from_request", "asgi_wrapper", - "canned_queries", "extra_js_urls", "extra_template_vars", "handle_exception", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index f682e8b9..1d2378b4 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -314,11 +314,6 @@ def startup(datasette): _ = (Response, Forbidden, NotFound, hookimpl, actor_matches_allow) -@hookimpl -def canned_queries(datasette, database, actor): - return {"from_hook": f"select 1, '{actor['id'] if actor else 'null'}' as actor_id"} - - @hookimpl def register_magic_parameters(): from uuid import uuid4 diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index 9e8d9b2b..864637a6 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -143,20 +143,6 @@ def startup(datasette): return inner -@hookimpl -def canned_queries(datasette, database): - async def inner(): - return { - "from_async_hook": "select {}".format( - ( - await datasette.get_database(database).execute("select 1 + 1") - ).first()[0] - ) - } - - return inner - - @hookimpl(trylast=True) def menu_links(datasette, actor): async def inner(): diff --git a/tests/test_docs.py b/tests/test_docs.py index 396ba1a2..9cf39f41 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -66,7 +66,23 @@ def documented_views(): if first_word.endswith("View"): view_labels.add(first_word) # We deliberately don't document these: - view_labels.update(("PatternPortfolioView", "AuthTokenView", "ApiExplorerView")) + view_labels.update( + ( + "PatternPortfolioView", + "AuthTokenView", + "ApiExplorerView", + "ExecuteWriteAnalyzeView", + "ExecuteWriteView", + "GlobalQueryListView", + "QueryCreateAnalyzeView", + "QueryDeleteView", + "QueryDefinitionView", + "QueryListView", + "QueryParametersView", + "QueryStoreView", + "QueryUpdateView", + ) + ) return view_labels diff --git a/tests/test_html.py b/tests/test_html.py index d20796c9..a9de5e79 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -154,12 +154,10 @@ async def test_database_page(ds_client): ("/fixtures/simple_view", "simple_view"), ] == sorted([(a["href"], a.text) for a in views_ul.find_all("a")]) - # And a list of canned queries + # And a list of stored queries queries_ul = soup.find("h2", string="Queries").find_next_sibling("ul") assert queries_ul is not None assert [ - ("/fixtures/from_async_hook", "from_async_hook"), - ("/fixtures/from_hook", "from_hook"), ("/fixtures/magic_parameters", "magic_parameters"), ("/fixtures/neighborhood_search#fragment-goes-here", "Search neighborhoods"), ("/fixtures/pragma_cache_size", "pragma_cache_size"), @@ -344,17 +342,29 @@ async def test_query_parameter_form_fields(ds_client): response = await ds_client.get("/fixtures/-/query?sql=select+:name") assert response.status_code == 200 assert ( - ' ' + ' ' in response.text ) + assert 'data-parameters-url="/fixtures/-/query/parameters"' in response.text + assert 'id="sql-parameters-section"' in response.text + assert "setupSqlParameterRefresh" in response.text response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello") assert response2.status_code == 200 assert ( - ' ' + ' ' in response2.text ) +@pytest.mark.asyncio +async def test_database_page_sql_parameter_refresh_markup(ds_client): + response = await ds_client.get("/fixtures") + assert response.status_code == 200 + assert 'data-parameters-url="/fixtures/-/query/parameters"' in response.text + assert 'id="sql-parameters-section"' in response.text + assert "setupSqlParameterRefresh" in response.text + + @pytest.mark.asyncio async def test_row_html_simple_primary_key(ds_client): response = await ds_client.get("/fixtures/simple_primary_key/1") @@ -623,7 +633,7 @@ async def test_404_content_type(ds_client): @pytest.mark.asyncio -async def test_canned_query_default_title(ds_client): +async def test_stored_query_default_title(ds_client): response = await ds_client.get("/fixtures/magic_parameters") assert response.status_code == 200 soup = Soup(response.content, "html.parser") @@ -631,7 +641,7 @@ async def test_canned_query_default_title(ds_client): @pytest.mark.asyncio -async def test_canned_query_with_custom_metadata(ds_client): +async def test_stored_query_with_custom_metadata(ds_client): response = await ds_client.get("/fixtures/neighborhood_search?text=town") assert response.status_code == 200 soup = Soup(response.content, "html.parser") @@ -690,8 +700,8 @@ async def test_show_hide_sql_query(ds_client): @pytest.mark.asyncio -async def test_canned_query_with_hide_has_no_hidden_sql(ds_client): - # For a canned query the show/hide should NOT have a hidden SQL field +async def test_stored_query_with_hide_has_no_hidden_sql(ds_client): + # For a stored query the show/hide should NOT have a hidden SQL field # https://github.com/simonw/datasette/issues/1411 response = await ds_client.get("/fixtures/pragma_cache_size?_hide_sql=1") soup = Soup(response.content, "html.parser") @@ -710,7 +720,7 @@ async def test_canned_query_with_hide_has_no_hidden_sql(ds_client): (True, "?_show_sql=1", "_show_sql", "/_memory/one", "hide"), ), ) -def test_canned_query_show_hide_metadata_option( +def test_stored_query_show_hide_metadata_option( hide_sql, querystring, expected_hidden, @@ -971,10 +981,10 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix): ("/fixtures/magic_parameters", None), ], ) -async def test_edit_sql_link_on_canned_queries(ds_client, path, expected): +async def test_edit_sql_link_on_stored_queries(ds_client, path, expected): response = await ds_client.get(path) assert response.status_code == 200 - expected_link = f'Edit SQL' + expected_link = f'Edit SQL' if expected: assert expected_link in response.text else: @@ -1096,7 +1106,7 @@ async def test_trace_correctly_escaped(ds_client): "/fixtures/-/query?sql=select+*+from+facetable", "http://localhost/fixtures/-/query.json?sql=select+*+from+facetable", ), - # Canned query page + # Stored query page ( "/fixtures/neighborhood_search?text=town", "http://localhost/fixtures/neighborhood_search.json?text=town", diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 75ae8d39..5481a398 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -688,6 +688,54 @@ async def test_execute_isolated(db, disable_threads): assert not await db.execute_isolated_fn(table_exists_checker("created_by_isolated")) +@pytest.mark.asyncio +async def test_analyze_sql(): + ds = Datasette(memory=True) + db = ds.add_memory_database("test_analyze_sql", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + + analysis = await db.analyze_sql("select name from dogs where id = ?", (1,)) + + assert [ + ( + access.operation, + access.database, + access.sqlite_schema, + access.table, + access.columns, + access.source, + ) + for access in analysis.table_accesses + ] == [ + ("read", "data", "main", "dogs", ("id", "name"), None), + ] + + +@pytest.mark.asyncio +async def test_analyze_sql_insert_select(): + ds = Datasette(memory=True) + db = ds.add_memory_database("test_analyze_sql_insert_select", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await db.execute_write("create table cats (id integer primary key, name text)") + + analysis = await db.analyze_sql("insert into dogs (name) select name from cats") + + assert { + ( + access.operation, + access.database, + access.sqlite_schema, + access.table, + access.columns, + access.source, + ) + for access in analysis.table_accesses + } == { + ("insert", "data", "main", "dogs", (), None), + ("read", "data", "main", "cats", ("name",), None), + } + + @pytest.mark.asyncio async def test_mtime_ns(db): assert isinstance(db.mtime_ns, int) diff --git a/tests/test_jump.py b/tests/test_jump.py index f60af0fd..513a809f 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -76,7 +76,7 @@ async def ds_for_jump(): @pytest.mark.asyncio -async def test_jump_searches_tables_databases_views_and_canned_queries(ds_for_jump): +async def test_jump_searches_tables_databases_views_and_stored_queries(ds_for_jump): response = await ds_for_jump.client.get( "/-/jump.json?q=content", actor={"id": "user"} ) @@ -98,7 +98,7 @@ async def test_jump_searches_tables_databases_views_and_canned_queries(ds_for_ju @pytest.mark.asyncio -async def test_jump_uses_canned_query_names_not_titles(ds_for_jump): +async def test_jump_uses_stored_query_names_not_titles(ds_for_jump): response = await ds_for_jump.client.get( "/-/jump.json?q=datasette", actor={"id": "user"} ) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 8166532f..e5e75432 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1,4 +1,5 @@ import collections +from asgiref.sync import async_to_sync from datasette.app import Datasette from datasette.cli import cli from datasette.default_permissions import restrictions_allow_action @@ -609,6 +610,10 @@ def test_padlocks_on_database_page(cascade_app_client): previous_config = cascade_app_client.ds.config try: cascade_app_client.ds.config = config + async_to_sync(cascade_app_client.ds.invoke_startup)() + async_to_sync(cascade_app_client.ds.add_query)( + "fixtures", "query_two", "select 2", source="config" + ) response = cascade_app_client.get( "/fixtures", cookies={"ds_actor": cascade_app_client.actor_cookie({"id": "test"})}, @@ -617,13 +622,13 @@ def test_padlocks_on_database_page(cascade_app_client): assert ">123_starts_with_digits" in response.text assert ">Table With Space In Name 🔒" in response.text # Queries - assert ">from_async_hook 🔒" in response.text assert ">query_two" in response.text # Views assert ">paginated_view 🔒" in response.text assert ">simple_view" in response.text finally: cascade_app_client.ds.config = previous_config + async_to_sync(cascade_app_client.ds.remove_query)("fixtures", "query_two") @pytest.mark.asyncio @@ -885,7 +890,7 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "t1"), expected_result=True, ), - # view-query on canned query, wrong actor + # view-query on stored query, wrong actor PermConfigTestCase( config={ "databases": { @@ -904,7 +909,7 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "q1"), expected_result=False, ), - # view-query on canned query, right actor + # view-query on stored query, right actor PermConfigTestCase( config={ "databases": { @@ -932,16 +937,24 @@ async def test_permissions_in_config( updated_config = copy.deepcopy(previous_config) updated_config.update(config) perms_ds.config = updated_config + await perms_ds._save_queries_from_config() try: # Convert old-style resource to Resource object - from datasette.resources import DatabaseResource, TableResource + from datasette.resources import DatabaseResource, QueryResource, TableResource resource_obj = None if resource: if isinstance(resource, str): resource_obj = DatabaseResource(database=resource) elif isinstance(resource, tuple) and len(resource) == 2: - resource_obj = TableResource(database=resource[0], table=resource[1]) + if action == "view-query": + resource_obj = QueryResource( + database=resource[0], query=resource[1] + ) + else: + resource_obj = TableResource( + database=resource[0], table=resource[1] + ) result = await perms_ds.allowed( action=action, resource=resource_obj, actor=actor @@ -951,42 +964,24 @@ async def test_permissions_in_config( assert result == expected_result finally: perms_ds.config = previous_config + await perms_ds._save_queries_from_config() @pytest.mark.asyncio -async def test_allowed_resources_view_query_includes_actor_specific_canned_queries(): - """ - Actor-specific canned queries should be listed by allowed_resources("view-query"). - - This test is intentionally explicit about the previous bug: - - the canned query only exists for actor "alice" - - the permission rule only allows actor "alice" to view it - - allowed() succeeds for that specific query resource - - allowed_resources("view-query", actor) must include the same query - - Before the fix, QueryResource.resources_sql() called canned_queries(..., actor=None), - so the query was omitted from resource enumeration and allowed_resources() returned - an empty list even though allowed() returned True. - """ +async def test_allowed_resources_view_query_includes_actor_specific_query_permissions(): from datasette import hookimpl from datasette.permissions import PermissionSQL from datasette.resources import QueryResource - class ActorSpecificQueryPlugin: - __name__ = "ActorSpecificQueryPlugin" - - @hookimpl - def canned_queries(self, datasette, database, actor): - if database == "testdb" and actor and actor.get("id") == "alice": - return {"user_only": {"sql": "select 1 as n"}} - return {} + class ActorSpecificQueryPermissionPlugin: + __name__ = "ActorSpecificQueryPermissionPlugin" @hookimpl def permission_resources_sql(self, datasette, actor, action): if action == "view-query" and actor and actor.get("id") == "alice": return PermissionSQL(sql=""" SELECT 'testdb' AS parent, 'user_only' AS child, 1 AS allow, - 'alice can view her actor-specific canned query' AS reason + 'alice can view this query' AS reason """) return None @@ -994,9 +989,10 @@ async def test_allowed_resources_view_query_includes_actor_specific_canned_queri await ds.invoke_startup() ds.add_memory_database("testdb") await ds._refresh_schemas() + await ds.add_query("testdb", "user_only", "select 1 as n") - plugin = ActorSpecificQueryPlugin() - ds.pm.register(plugin, name="actor_specific_query_plugin") + plugin = ActorSpecificQueryPermissionPlugin() + ds.pm.register(plugin, name="actor_specific_query_permission_plugin") try: actor = {"id": "alice"} @@ -1012,7 +1008,7 @@ async def test_allowed_resources_view_query_includes_actor_specific_canned_queri ("testdb", "user_only") ] finally: - ds.pm.unregister(name="actor_specific_query_plugin") + ds.pm.unregister(name="actor_specific_query_permission_plugin") @pytest.mark.asyncio diff --git a/tests/test_plugins.py b/tests/test_plugins.py index c5b9aef0..32276437 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -885,40 +885,64 @@ async def test_hook_startup_catalog_populated(ds_client): @pytest.mark.asyncio -async def test_hook_canned_queries(ds_client): - queries = (await ds_client.get("/fixtures.json")).json()["queries"] +async def test_plugin_startup_can_add_queries(): + ds = Datasette(memory=True) + ds.add_memory_database("plugin_startup_queries", name="data") + + class AddQueriesPlugin: + __name__ = "AddQueriesPlugin" + + @hookimpl + def startup(self, datasette): + async def inner(): + result = await datasette.get_database("data").execute("select 1 + 1") + await datasette.add_query( + "data", + "from_startup", + "select {}".format(result.first()[0]), + source="plugin", + ) + + return inner + + ds.pm.register(AddQueriesPlugin(), name="add_queries_plugin") + try: + response = await ds.client.get("/data.json") + finally: + ds.pm.unregister(name="add_queries_plugin") + + queries = response.json()["queries"] queries_by_name = {q["name"]: q for q in queries} - assert { - "sql": "select 2", - "name": "from_async_hook", - "private": False, - } == queries_by_name["from_async_hook"] - assert { - "sql": "select 1, 'null' as actor_id", - "name": "from_hook", - "private": False, - } == queries_by_name["from_hook"] + assert queries_by_name["from_startup"]["sql"] == "select 2" + assert queries_by_name["from_startup"]["private"] is False @pytest.mark.asyncio -async def test_hook_canned_queries_non_async(ds_client): - response = await ds_client.get("/fixtures/from_hook.json?_shape=array") - assert [{"1": 1, "actor_id": "null"}] == response.json() +async def test_plugin_startup_query_can_execute(): + ds = Datasette(memory=True) + ds.add_memory_database("plugin_startup_query_execute", name="data") + class AddQueryPlugin: + __name__ = "AddQueryPlugin" + + @hookimpl + def startup(self, datasette): + async def inner(): + await datasette.add_query( + "data", "from_startup", "select 2", source="plugin" + ) + + return inner + + ds.pm.register(AddQueryPlugin(), name="add_query_plugin") + try: + response = await ds.client.get("/data/from_startup.json?_shape=array") + finally: + ds.pm.unregister(name="add_query_plugin") -@pytest.mark.asyncio -async def test_hook_canned_queries_async(ds_client): - response = await ds_client.get("/fixtures/from_async_hook.json?_shape=array") assert [{"2": 2}] == response.json() -@pytest.mark.asyncio -async def test_hook_canned_queries_actor(ds_client): - assert ( - await ds_client.get("/fixtures/from_hook.json?_bot=1&_shape=array") - ).json() == [{"1": 1, "actor_id": "bot"}] - - def test_hook_register_magic_parameters(restore_working_directory): with make_app_client( extra_databases={"data.db": "create table logs (line text)"}, @@ -1462,8 +1486,10 @@ class SlotPlugin: return "Xtop_query:{}:{}:{}".format(database, sql, request.args["z"]) @hookimpl - def top_canned_query(self, request, database, query_name): - return "Xtop_query:{}:{}:{}".format(database, query_name, request.args["z"]) + def top_stored_query(self, request, database, query_name): + return "Xtop_stored_query:{}:{}:{}".format( + database, query_name, request.args["z"] + ) @pytest.mark.asyncio @@ -1524,12 +1550,12 @@ async def test_hook_top_query(ds_client): @pytest.mark.asyncio -async def test_hook_top_canned_query(ds_client): +async def test_hook_top_stored_query(ds_client): try: pm.register(SlotPlugin(), name="SlotPlugin") - response = await ds_client.get("/fixtures/from_hook?z=xyz") + response = await ds_client.get("/fixtures/magic_parameters?z=xyz") assert response.status_code == 200 - assert "Xtop_query:fixtures:from_hook:xyz" in response.text + assert "Xtop_stored_query:fixtures:magic_parameters:xyz" in response.text finally: pm.unregister(name="SlotPlugin") diff --git a/tests/test_queries.py b/tests/test_queries.py new file mode 100644 index 00000000..59fab8c0 --- /dev/null +++ b/tests/test_queries.py @@ -0,0 +1,1874 @@ +import json + +import pytest + +from datasette.app import Datasette +from datasette.resources import DatabaseResource, QueryResource +from datasette.stored_queries import StoredQuery, StoredQueryPage +from datasette.utils.asgi import Forbidden + + +async def add_numbered_queries(ds, database, count): + for i in range(1, count + 1): + await ds.add_query( + database, + "demo_query_{:02d}".format(i), + "select {} as query_number".format(i), + title="Demo query {:02d}".format(i), + description="Seeded demo query number {:02d}".format(i), + source="user", + owner_id="root", + ) + + +@pytest.mark.asyncio +async def test_queries_internal_table_schema(): + ds = Datasette(memory=True) + await ds.invoke_startup() + internal_db = ds.get_internal_database() + + columns = [ + row["name"] + for row in ( + await internal_db.execute("select name from pragma_table_info('queries')") + ) + ] + + assert columns == [ + "database_name", + "name", + "sql", + "title", + "description", + "description_html", + "options", + "parameters", + "is_write", + "is_private", + "is_trusted", + "source", + "owner_id", + "created_at", + "updated_at", + ] + + +@pytest.mark.asyncio +async def test_add_get_and_remove_query(): + ds = Datasette(memory=True) + ds.add_memory_database("query_api", name="data") + await ds.invoke_startup() + + await ds.add_query( + "data", + "top_customers", + "select * from customers where region = :region", + title="Top customers", + description="Customers by region", + hide_sql=True, + fragment="chart", + parameters=["region"], + is_trusted=True, + source="user", + owner_id="alice", + ) + + options_row = ( + await ds.get_internal_database().execute( + """ + SELECT options FROM queries + WHERE database_name = ? AND name = ? + """, + ["data", "top_customers"], + ) + ).first() + assert json.loads(options_row["options"]) == { + "fragment": "chart", + "hide_sql": True, + } + + query = await ds.get_query("data", "top_customers") + assert query == StoredQuery( + database="data", + name="top_customers", + sql="select * from customers where region = :region", + title="Top customers", + description="Customers by region", + description_html=None, + hide_sql=True, + fragment="chart", + parameters=["region"], + is_write=False, + is_private=False, + is_trusted=True, + source="user", + owner_id="alice", + on_success_message=None, + on_success_message_sql=None, + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + ) + + queries_page = await ds.list_queries("data", actor=None) + assert queries_page == StoredQueryPage( + queries=[query], + next=None, + has_more=False, + limit=50, + ) + + await ds.remove_query("data", "top_customers") + assert await ds.get_query("data", "top_customers") is None + queries_page = await ds.list_queries("data", actor=None) + assert queries_page.queries == [] + assert queries_page.next is None + + +@pytest.mark.asyncio +async def test_update_query_only_updates_provided_fields(): + ds = Datasette(memory=True) + ds.add_memory_database("query_api_update", name="data") + await ds.invoke_startup() + + await ds.add_query( + "data", + "redirect", + "select 1", + title="Original", + on_success_redirect="/original", + parameters=["one"], + ) + + options_row = ( + await ds.get_internal_database().execute( + """ + SELECT options FROM queries + WHERE database_name = ? AND name = ? + """, + ["data", "redirect"], + ) + ).first() + assert json.loads(options_row["options"]) == {"on_success_redirect": "/original"} + + await ds.update_query( + "data", + "redirect", + title="Updated", + parameters=[], + on_success_redirect=None, + ) + + query = await ds.get_query("data", "redirect") + assert query.title == "Updated" + assert query.parameters == [] + assert query.on_success_redirect is None + assert query.sql == "select 1" + assert query.is_private is False + assert query.is_trusted is False + options_row = ( + await ds.get_internal_database().execute( + """ + SELECT options FROM queries + WHERE database_name = ? AND name = ? + """, + ["data", "redirect"], + ) + ).first() + assert json.loads(options_row["options"]) == {} + + +@pytest.mark.asyncio +async def test_config_queries_imported_to_internal_table(): + ds = Datasette( + memory=True, + config={ + "databases": { + "data": { + "queries": { + "configured": { + "sql": "select :name as name", + "title": "Configured query", + "description_html": "

    Configured HTML

    ", + "params": ["name"], + "on_success_message_sql": "select 'Hello ' || :name", + } + } + } + } + }, + ) + ds.add_memory_database("query_config", name="data") + await ds.invoke_startup() + + assert await ds.get_query("data", "configured") == StoredQuery( + database="data", + name="configured", + sql="select :name as name", + title="Configured query", + description=None, + description_html="

    Configured HTML

    ", + hide_sql=False, + fragment=None, + parameters=["name"], + is_write=False, + is_private=False, + is_trusted=True, + source="config", + owner_id=None, + on_success_message=None, + on_success_message_sql="select 'Hello ' || :name", + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + ) + + +@pytest.mark.asyncio +async def test_query_resources_come_from_internal_table(): + ds = Datasette(memory=True) + ds.add_memory_database("query_resources", name="data") + await ds.invoke_startup() + await ds.add_query("data", "internal_query", "select 1", source="user") + + page = await ds.allowed_resources("view-query", actor=None) + + assert [(r.parent, r.child) for r in page.resources] == [("data", "internal_query")] + + +@pytest.mark.asyncio +async def test_default_deny_blocks_view_query_even_for_trusted_query(): + ds = Datasette(memory=True, default_deny=True) + ds.add_memory_database("query_permissions", name="data") + await ds.invoke_startup() + await ds.add_query("data", "trusted", "select 1", is_trusted=True) + + assert not await ds.allowed( + action="view-query", + resource=QueryResource("data", "trusted"), + actor=None, + ) + + +@pytest.mark.asyncio +async def test_view_query_default_allow_still_respects_private_restriction(): + ds = Datasette(memory=True) + ds.add_memory_database("default_view_query_permissions", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "private_report", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "shared_report", + "select 2", + is_private=False, + source="user", + owner_id="alice", + ) + + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "shared_report"), + actor=None, + ) + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "bob"}, + ) + + +@pytest.mark.asyncio +async def test_private_query_restriction_blocks_broad_view_query_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-query": {"id": "*"}, + } + } + } + }, + ) + ds.add_memory_database("private_query_permissions", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "private_report", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "shared_report", + "select 2", + is_private=False, + source="user", + owner_id="alice", + ) + + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "bob"}, + ) + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "shared_report"), + actor={"id": "bob"}, + ) + + +@pytest.mark.asyncio +async def test_config_query_restriction_does_not_override_private_internal_query(): + ds = Datasette(memory=True, default_deny=True) + ds.add_memory_database("private_query_with_config_name", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "private_report", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + ds.config = { + "databases": { + "data": { + "permissions": {"view-query": {"id": "*"}}, + "queries": {"private_report": {"sql": "select 2"}}, + } + } + } + + assert not await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "bob"}, + ) + + +@pytest.mark.asyncio +async def test_untrusted_shared_query_execution_requires_execute_sql(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "viewer"}, + "view-query": {"id": "viewer"}, + } + } + } + }, + ) + ds.add_memory_database("untrusted_query_execution", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "shared_report", + "select 1 as one", + is_private=False, + is_trusted=False, + source="user", + owner_id="alice", + ) + + denied_get = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) + denied_post = await ds.client.post( + "/data/shared_report", + actor={"id": "viewer"}, + data={}, + ) + assert denied_get.status_code == 403 + assert denied_post.status_code == 403 + + ds.config["databases"]["data"]["permissions"]["execute-sql"] = {"id": "viewer"} + allowed = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) + assert allowed.status_code == 200 + assert allowed.json()["rows"] == [{"one": 1}] + + +@pytest.mark.asyncio +async def test_config_queries_are_trusted_by_default_but_can_opt_out(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-query": {"id": "viewer"}, + }, + "queries": { + "trusted_report": {"sql": "select 1 as one"}, + "untrusted_report": { + "sql": "select 2 as two", + "is_trusted": False, + }, + }, + } + } + }, + ) + ds.add_memory_database("trusted_query_config", name="data") + await ds.invoke_startup() + + trusted = await ds.client.get("/data/trusted_report.json", actor={"id": "viewer"}) + untrusted = await ds.client.get( + "/data/untrusted_report.json", actor={"id": "viewer"} + ) + + assert trusted.status_code == 200 + assert trusted.json()["rows"] == [{"one": 1}] + assert untrusted.status_code == 403 + + +@pytest.mark.asyncio +async def test_database_page_query_preview_is_limited(): + ds = Datasette(memory=True) + ds.add_memory_database("query_preview", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 25) + + html_response = await ds.client.get("/data") + json_response = await ds.client.get("/data.json") + + assert html_response.status_code == 200 + assert "Demo query 05" in html_response.text + assert "Demo query 06" not in html_response.text + assert 'View 25 queries' in html_response.text + assert len(json_response.json()["queries"]) == 5 + assert json_response.json()["queries_more"] is True + assert json_response.json()["queries_count"] == 25 + + +@pytest.mark.asyncio +async def test_query_actions_are_registered(): + ds = Datasette() + await ds.invoke_startup() + + assert ds.get_action("execute-write-sql").resource_class is DatabaseResource + assert ds.get_action("store-query").resource_class is DatabaseResource + assert ds.get_action("update-query").resource_class is QueryResource + assert ds.get_action("delete-query").resource_class is QueryResource + + +@pytest.mark.asyncio +async def test_analyze_write_query_requires_table_permissions(): + ds = Datasette(memory=True, default_deny=True) + db = ds.add_memory_database("query_write_permissions", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + actor = {"id": "writer"} + await ds.add_query( + "data", + "write_dog", + "insert into dogs (name) values (:name)", + is_write=True, + source="user", + owner_id="writer", + ) + + with pytest.raises(Forbidden): + await ds.ensure_query_write_permissions( + "data", + "insert into dogs (name) values (:name)", + actor=actor, + ) + + ds.config = { + "databases": { + "data": { + "tables": { + "dogs": { + "permissions": { + "insert-row": {"id": "writer"}, + } + } + } + } + } + } + + await ds.ensure_query_write_permissions( + "data", + "insert into dogs (name) values (:name)", + actor=actor, + ) + + +@pytest.mark.asyncio +async def test_analyze_write_query_rejects_writes_to_attached_databases(): + ds = Datasette(memory=True, default_deny=True) + db = ds.add_memory_database("query_attached_writes", name="data") + await db.execute_write("attach database ':memory:' as extra") + await db.execute_write("create table extra.cats (id integer primary key)") + await ds.invoke_startup() + + with pytest.raises(Forbidden): + await ds.ensure_query_write_permissions( + "data", + "insert into extra.cats (id) values (1)", + actor={"id": "writer"}, + ) + + +@pytest.mark.asyncio +async def test_query_store_api_creates_read_only_query(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_store_api", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/queries/store", + actor={"id": "root"}, + json={ + "query": { + "name": "by_name", + "sql": "select * from dogs where name = :name", + "title": "By name", + } + }, + ) + + assert response.status_code == 201 + data = response.json() + assert data["ok"] is True + assert data["query"]["name"] == "by_name" + assert data["query"]["parameters"] == ["name"] + assert data["query"]["is_write"] is False + assert data["query"]["source"] == "user" + assert data["query"]["owner_id"] == "root" + + +@pytest.mark.asyncio +async def test_query_list_and_definition_api(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_api", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 12) + + list_response = await ds.client.get( + "/data/-/queries.json?_size=5", + actor={"id": "root"}, + ) + next_response = await ds.client.get( + "/data/-/queries.json?_size=5&_next={}".format(list_response.json()["next"]), + actor={"id": "root"}, + ) + definition_response = await ds.client.get( + "/data/demo_query_01/-/definition", + actor={"id": "root"}, + ) + + assert list_response.status_code == 200 + assert [query["name"] for query in list_response.json()["queries"]] == [ + "demo_query_01", + "demo_query_02", + "demo_query_03", + "demo_query_04", + "demo_query_05", + ] + assert list_response.json()["next"] + assert [query["name"] for query in next_response.json()["queries"]] == [ + "demo_query_06", + "demo_query_07", + "demo_query_08", + "demo_query_09", + "demo_query_10", + ] + assert definition_response.status_code == 200 + assert definition_response.json()["query"]["title"] == "Demo query 01" + + +@pytest.mark.asyncio +async def test_query_page_does_not_show_internal_source(): + ds = Datasette(memory=True) + ds.add_memory_database("query_page_source", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "stored_report", + "select 1 as one", + title="Stored report", + source="user", + owner_id="root", + ) + + response = await ds.client.get("/data/stored_report", actor={"id": "root"}) + + assert response.status_code == 200 + assert "Stored report" in response.text + assert "Data source:" not in response.text + + +@pytest.mark.asyncio +async def test_query_list_search_filter_and_html(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_html", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 3) + await ds.add_query( + "data", + "private_query", + "select 'private'", + title="Private query", + is_private=True, + source="user", + owner_id="root", + ) + await ds.add_query( + "data", + "trusted_query", + "select 'trusted'", + title="Trusted query", + is_trusted=True, + source="config", + ) + await ds.add_query( + "data", + "writable_query", + "insert into dogs (name) values (:name)", + title="Writable query", + is_write=True, + source="user", + owner_id="root", + ) + + html_response = await ds.client.get( + "/data/-/queries?q=02", + actor={"id": "root"}, + ) + flags_response = await ds.client.get( + "/data/-/queries", + actor={"id": "root"}, + ) + json_response = await ds.client.get( + "/data/-/queries.json?q=02", + actor={"id": "root"}, + ) + filtered_response = await ds.client.get( + "/data/-/queries.json?is_private=1", + actor={"id": "root"}, + ) + filtered_write_response = await ds.client.get( + "/data/-/queries?is_write=1", + actor={"id": "root"}, + ) + filtered_private_response = await ds.client.get( + "/data/-/queries?is_private=1", + actor={"id": "root"}, + ) + + assert html_response.status_code == 200 + assert "Demo query 02" in html_response.text + assert "Demo query 01" not in html_response.text + assert 'class="query-list-results"' in html_response.text + assert 'class="query-list-facets"' in html_response.text + assert 'type="radio"' not in html_response.text + assert "Only the owning actor can view this query." not in html_response.text + assert ( + "Execution skips the usual SQL and write permission checks" + not in html_response.text + ) + assert flags_response.status_code == 200 + assert 'Owner' in flags_response.text + assert 'Flags' in flags_response.text + assert 'Mode' not in flags_response.text + assert 'class="query-list-owner">root' in flags_response.text + assert 'class="query-list-pill">Read-only' in flags_response.text + assert ( + 'class="query-list-pill query-list-pill-write">Writable' + in flags_response.text + ) + assert ( + 'class="query-list-pill query-list-pill-private">Private' + in flags_response.text + ) + assert ( + 'class="query-list-pill query-list-pill-trusted">Trusted' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_write=0">Read-only5' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_write=1">Writable1' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_private=0">Not private5' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_private=1">Private1' + in flags_response.text + ) + assert "Only the owning actor can view this query." in flags_response.text + assert ( + "Execution skips the usual SQL and write permission checks" + in flags_response.text + ) + assert json_response.json()["queries"][0]["name"] == "demo_query_02" + assert [query["name"] for query in filtered_response.json()["queries"]] == [ + "private_query" + ] + assert "Writable query" in filtered_write_response.text + assert "Demo query 01" not in filtered_write_response.text + assert ( + 'query-list-facet-link query-list-facet-link-active" href="/data/-/queries"' + in filtered_write_response.text + ) + assert ( + 'Read-only0' + not in filtered_write_response.text + ) + assert ( + 'href="/data/-/queries?is_write=1&is_private=0">Not private1' + in filtered_write_response.text + ) + assert ( + 'Private0' + not in filtered_write_response.text + ) + assert "Private query" in filtered_private_response.text + assert "Demo query 01" not in filtered_private_response.text + assert ( + 'href="/data/-/queries?is_private=1&is_write=0">Read-only1' + in filtered_private_response.text + ) + assert ( + 'Writable0' + not in filtered_private_response.text + ) + assert ( + 'Not private0' + not in filtered_private_response.text + ) + + +@pytest.mark.asyncio +async def test_query_list_html_defaults_to_twenty_and_shows_pagination(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_html_pagination", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 25) + + response = await ds.client.get("/data/-/queries", actor={"id": "root"}) + json_response = await ds.client.get("/data/-/queries.json", actor={"id": "root"}) + + assert response.status_code == 200 + assert response.text.count('aria-label="Query pagination"') == 1 + assert "Demo query 20" in response.text + assert "Demo query 21" not in response.text + assert 'href="/data/-/queries?_next=' in response.text + assert len(json_response.json()["queries"]) == 25 + + +@pytest.mark.asyncio +async def test_global_query_list_api_and_html(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_global_alpha", name="alpha") + ds.add_memory_database("query_list_global_beta", name="beta") + await ds.invoke_startup() + await ds.add_query( + "alpha", + "alpha_first", + "select 1", + title="Alpha first", + source="user", + owner_id="root", + ) + await ds.add_query( + "alpha", + "alpha_second", + "select 2", + title="Alpha second", + source="user", + owner_id="root", + ) + await ds.add_query( + "beta", + "beta_first", + "select 3", + title="Beta first", + source="user", + owner_id="root", + ) + + list_response = await ds.client.get( + "/-/queries.json?_size=2", + actor={"id": "root"}, + ) + next_response = await ds.client.get( + "/-/queries.json?_size=2&_next={}".format(list_response.json()["next"]), + actor={"id": "root"}, + ) + html_response = await ds.client.get( + "/-/queries?q=Beta", + actor={"id": "root"}, + ) + + assert list_response.status_code == 200 + assert [ + (query["database"], query["name"]) for query in list_response.json()["queries"] + ] == [ + ("alpha", "alpha_first"), + ("alpha", "alpha_second"), + ] + assert list_response.json()["next"] + assert [ + (query["database"], query["name"]) for query in next_response.json()["queries"] + ] == [ + ("beta", "beta_first"), + ] + assert html_response.status_code == 200 + assert 'Database' in html_response.text + assert 'class="query-list-database" href="/beta">beta' in html_response.text + assert "Beta first" in html_response.text + assert "Alpha first" not in html_response.text + + +@pytest.mark.asyncio +async def test_query_store_api_rejects_is_trusted(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-sql": {"id": "writer"}, + "store-query": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("query_trusted_api", name="data") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/queries/store", + actor={"id": "writer"}, + json={"query": {"name": "trusted", "sql": "select 1", "is_trusted": True}}, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == ["Invalid keys: is_trusted"] + + +@pytest.mark.asyncio +async def test_query_store_rejects_config_only_fields(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + ds.add_memory_database("query_config_only_fields_api", name="data") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/queries/store", + actor={"id": "root"}, + json={ + "query": { + "name": "unsafe", + "sql": "select 1", + "description_html": "", + "on_success_message_sql": "select 'secret'", + } + }, + ) + form_response = await ds.client.post( + "/data/-/queries/store", + actor={"id": "root"}, + data={ + "name": "unsafe_form", + "sql": "select 1", + "description_html": "", + }, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == [ + "Invalid keys: description_html, on_success_message_sql" + ] + assert form_response.status_code == 400 + assert "Invalid keys: description_html" in form_response.text + assert await ds.get_query("data", "unsafe") is None + assert await ds.get_query("data", "unsafe_form") is None + + +@pytest.mark.asyncio +async def test_query_store_api_creates_writable_query(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_write_api", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/queries/store", + actor={"id": "root"}, + json={ + "query": { + "name": "insert_dog", + "sql": "insert into dogs (name) values (:name)", + } + }, + ) + + assert response.status_code == 201 + query = response.json()["query"] + assert query["is_write"] is True + assert query["is_private"] is True + assert query["is_trusted"] is False + assert query["parameters"] == ["name"] + + +@pytest.mark.asyncio +async def test_query_update_and_delete_api(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + ds.add_memory_database("query_update_api", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "editable", + "select 1", + title="Original", + source="user", + owner_id="root", + ) + + update_response = await ds.client.post( + "/data/editable/-/update", + actor={"id": "root"}, + json={ + "update": { + "title": "Updated", + "description": "Fresh", + "on_success_redirect": None, + }, + "return": True, + }, + ) + + assert update_response.status_code == 200 + updated = update_response.json()["query"] + assert updated["title"] == "Updated" + assert updated["description"] == "Fresh" + assert updated["on_success_redirect"] is None + + delete_response = await ds.client.post( + "/data/editable/-/delete", + actor={"id": "root"}, + json={}, + ) + + assert delete_response.status_code == 200 + assert delete_response.json() == {"ok": True} + assert await ds.get_query("data", "editable") is None + + +@pytest.mark.asyncio +async def test_query_update_api_rejects_config_only_fields(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_update_config_only_fields", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "editable", + "insert into dogs (name) values (:name)", + is_write=True, + source="user", + owner_id="root", + ) + + response = await ds.client.post( + "/data/editable/-/update", + actor={"id": "root"}, + json={ + "update": { + "description_html": "", + "on_success_message_sql": "select 'secret'", + } + }, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == [ + "Invalid keys: description_html, on_success_message_sql" + ] + query = await ds.get_query("data", "editable") + assert query.description_html is None + assert query.on_success_message_sql is None + + +@pytest.mark.asyncio +async def test_query_update_api_rejects_trusted_queries_but_internal_update_allowed(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "execute-sql": {"id": "editor"}, + "update-query": {"id": "editor"}, + }, + "queries": { + "trusted_report": { + "sql": "select 1 as one", + "title": "Original", + }, + }, + } + } + }, + ) + ds.add_memory_database("query_update_trusted_api", name="data") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/trusted_report/-/update", + actor={"id": "editor"}, + json={"update": {"sql": "select 2 as two", "title": "Edited"}}, + ) + + assert response.status_code == 403 + assert response.json()["errors"] == [ + "Trusted queries cannot be updated using the API" + ] + query = await ds.get_query("data", "trusted_report") + assert query.is_trusted is True + assert query.sql == "select 1 as one" + assert query.title == "Original" + + await ds.update_query( + "data", + "trusted_report", + sql="select 3 as three", + title="Internal", + ) + query = await ds.get_query("data", "trusted_report") + assert query.is_trusted is True + assert query.sql == "select 3 as three" + assert query.title == "Internal" + + +@pytest.mark.asyncio +async def test_query_store_api_rejects_magic_parameters(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + ds.add_memory_database("query_magic_api", name="data") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/queries/store", + actor={"id": "root"}, + json={"query": {"name": "magic", "sql": "select :_actor_id"}}, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == ["Magic parameters are not allowed"] + + +@pytest.mark.asyncio +async def test_create_query_ui_and_arbitrary_sql_save_link(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_create_ui", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + create_response = await ds.client.get( + "/data/-/queries/store?sql=select+*+from+dogs", + actor={"id": "root"}, + ) + write_create_response = await ds.client.get( + "/data/-/queries/store?sql=insert+into+dogs+(name)+values+('Cleo')", + actor={"id": "root"}, + ) + blank_create_response = await ds.client.get( + "/data/-/queries/store", + actor={"id": "root"}, + ) + old_insert_response = await ds.client.get( + "/data/-/queries/insert?sql=select+*+from+dogs", + actor={"id": "root"}, + ) + old_create_response = await ds.client.get( + "/data/-/queries/-/create?sql=select+*+from+dogs", + actor={"id": "root"}, + ) + query_response = await ds.client.get( + "/data/-/query?sql=select+*+from+dogs", + actor={"id": "root"}, + ) + + assert create_response.status_code == 200 + assert "Create query" in create_response.text + assert 'type="radio"' not in create_response.text + assert 'name="parameters"' not in create_response.text + assert 'id="query-parameters"' not in create_response.text + assert 'class="query-create-field"' in create_response.text + assert '' not in create_response.text + assert '' in create_response.text + assert '' in create_response.text + assert '/data/' in create_response.text + assert ( + '' + in create_response.text + ) + assert "function slugify(value)" in create_response.text + assert 'data-analyze-url="/data/-/queries/analyze"' in create_response.text + assert "setupSqlParameterRefresh" in create_response.text + assert "renderParameters: false" in create_response.text + assert "datasetteSqlAnalysis.renderAnalysis" in create_response.text + assert "data-query-create-submit" in create_response.text + assert "data-query-create-writable" not in create_response.text + assert "data-query-create-sql-type" not in create_response.text + assert "data-query-create-analysis-note" in create_response.text + assert "SQL type:" not in create_response.text + assert ( + 'This is a read-only query.' + in create_response.text + ) + assert "disabled> Writable" not in create_response.text + assert ( + "Queries marked private can only be seen by you, their creator." + in create_response.text + ) + assert create_response.text.index( + "This is a read-only query." + ) < create_response.text.index('') + assert "

    Query operations

    " in create_response.text + assert '' in create_response.text + assert '' in create_response.text + assert '' not in create_response.text + assert "" in create_response.text + assert ( + create_response.text.count( + '' + ) + == 2 + ) + assert create_response.text.index( + 'value="Save query"' + ) < create_response.text.index("

    Query operations

    ") + assert blank_create_response.status_code == 200 + assert ( + '
    Required permissionSourcereadn/a
    ' in response.text + assert '' in response.text + assert "" in response.text + assert "" in response.text + assert "" not in response.text + assert 'action="/data/-/execute-write"' in response.text + assert "insert into dogs (name) values ('Cleo')" in response.text + assert (await db.execute("select count(*) from dogs")).first()[0] == 0 + + empty_response = await ds.client.get( + "/data/-/execute-write", + actor={"id": "root"}, + ) + assert '' in empty_response.text + assert 'executeWriteSqlInput.value = "\\n\\n\\n";' in empty_response.text + assert "hidden>Save this query" in empty_response.text + + read_only_response = await ds.client.get( + "/data/-/execute-write?sql=select+*+from+dogs", + actor={"id": "root"}, + ) + assert ( + "Use /-/query for read-only SQL; this endpoint only executes writes" + in read_only_response.text + ) + assert "hidden>Save this query" in read_only_response.text + + +@pytest.mark.asyncio +async def test_execute_write_analyze_endpoint_uses_sql_only(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_analyze", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.get( + "/data/-/execute-write/analyze", + actor={"id": "root"}, + params={"sql": "insert into dogs (name) values (:name)"}, + ) + read_only_response = await ds.client.get( + "/data/-/execute-write/analyze", + actor={"id": "root"}, + params={"sql": "select * from dogs where name = :name"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["ok"] is True + assert data["parameters"] == ["name"] + assert data["analysis_error"] is None + assert data["execute_disabled"] is False + assert data["analysis_rows"] == [ + { + "operation": "insert", + "database": "data", + "table": "dogs", + "required_permission": "insert-row", + "source": None, + "allowed": True, + } + ] + assert "params" not in data + + assert read_only_response.status_code == 200 + read_only_data = read_only_response.json() + assert read_only_data["ok"] is False + assert read_only_data["parameters"] == ["name"] + assert read_only_data["analysis_error"] == ( + "Use /-/query for read-only SQL; this endpoint only executes writes" + ) + assert read_only_data["execute_disabled"] is True + + +@pytest.mark.asyncio +async def test_query_parameters_endpoint_uses_get_sql_only(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_parameters", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.get( + "/data/-/query/parameters", + actor={"id": "root"}, + params={ + "sql": "select * from dogs where name = :name and id = :id", + }, + ) + permission_denied_response = await ds.client.get( + "/data/-/query/parameters", + actor={"id": "not-root"}, + params={"sql": "select * from dogs where name = :name"}, + ) + magic_parameter_response = await ds.client.get( + "/data/-/query/parameters", + actor={"id": "root"}, + params={"sql": "select :_actor_id"}, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": True, "parameters": ["name", "id"]} + assert permission_denied_response.status_code == 403 + assert permission_denied_response.json()["errors"] == [ + "Permission denied: need execute-sql" + ] + assert magic_parameter_response.status_code == 400 + assert magic_parameter_response.json()["errors"] == [ + "Magic parameters are not allowed" + ] + + +@pytest.mark.asyncio +async def test_database_action_menu_links_to_execute_write_for_permitted_actor(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": { + "id": ["writer", "viewer"], + }, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("execute_write_menu", name="data") + await ds.invoke_startup() + + anonymous_response = await ds.client.get("/data") + viewer_response = await ds.client.get("/data", actor={"id": "viewer"}) + writer_response = await ds.client.get("/data", actor={"id": "writer"}) + + assert anonymous_response.status_code == 403 + assert viewer_response.status_code == 200 + assert "Execute write SQL" not in viewer_response.text + assert writer_response.status_code == 200 + assert "Database actions" in writer_response.text + assert 'href="/data/-/execute-write"' in writer_response.text + assert "Execute write SQL" in writer_response.text + + +@pytest.mark.asyncio +async def test_database_action_menu_hides_execute_write_for_immutable_database(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + db = ds.add_memory_database("execute_write_menu_immutable", name="data") + db.is_mutable = False + await ds.invoke_startup() + + response = await ds.client.get("/data", actor={"id": "writer"}) + + assert response.status_code == 200 + assert "Execute write SQL" not in response.text + assert 'href="/data/-/execute-write"' not in response.text + + +@pytest.mark.asyncio +async def test_execute_write_get_rejects_immutable_database(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_get_immutable", name="data") + db.is_mutable = False + await ds.invoke_startup() + + response = await ds.client.get( + "/data/-/execute-write?sql=insert+into+dogs+(name)+values+('Cleo')", + actor={"id": "root"}, + ) + + assert response.status_code == 403 + assert response.json()["errors"] == [ + "Cannot execute write SQL because this database is immutable." + ] + + +@pytest.mark.asyncio +async def test_execute_write_post_requires_database_and_table_permissions(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + db = ds.add_memory_database("execute_write_permissions", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + no_database_permission = await ds.client.post( + "/data/-/execute-write", + actor={"id": "outsider"}, + json={ + "sql": "insert into dogs (name) values (:name)", + "params": {"name": "Cleo"}, + }, + ) + no_table_permission = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={ + "sql": "insert into dogs (name) values (:name)", + "params": {"name": "Cleo"}, + }, + ) + + assert no_database_permission.status_code == 403 + assert no_database_permission.json()["errors"] == [ + "Permission denied: need execute-write-sql" + ] + assert no_table_permission.status_code == 403 + assert no_table_permission.json()["errors"] == [ + "Permission denied: need insert-row on data/dogs" + ] + + ds.config = { + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "dogs": { + "permissions": { + "insert-row": {"id": "writer"}, + } + } + }, + } + } + } + allowed = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={ + "sql": "insert into dogs (name) values (:name)", + "params": {"name": "Cleo"}, + }, + ) + + assert allowed.status_code == 200 + assert allowed.json()["ok"] is True + assert allowed.json()["rowcount"] == 1 + assert allowed.json()["analysis"][0]["operation"] == "insert" + assert (await db.execute("select name from dogs")).first()[0] == "Cleo" + + +@pytest.mark.asyncio +async def test_execute_write_insert_links_to_inserted_row(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_insert_link", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await db.execute_write("create table log (id integer primary key, message text)") + await db.execute_write("insert into log (message) values ('existing')") + await db.execute_write(""" + create trigger dogs_after_insert after insert on dogs begin + insert into log (message) values (new.name); + end + """) + await ds.invoke_startup() + + insert_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "insert into dogs (name) values (:name)", + "name": "Cleo", + }, + ) + update_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "update dogs set name = :name where id = :id", + "name": "Cleo 2", + "id": "1", + }, + ) + + assert insert_response.status_code == 200 + assert "Query executed, 1 row affected" in insert_response.text + assert 'View row' in insert_response.text + assert "/data/log/2" not in insert_response.text + assert update_response.status_code == 200 + assert "Query executed, 1 row affected" in update_response.text + assert "View row" not in update_response.text + + +@pytest.mark.asyncio +async def test_execute_write_post_rejects_read_only_sql(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_read_only", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + json={"sql": "select * from dogs"}, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == [ + "Use /-/query for read-only SQL; this endpoint only executes writes" + ] + + +@pytest.mark.asyncio +async def test_query_owner_gets_update_delete_and_writable_view_defaults(): + ds = Datasette(memory=True, default_deny=True) + ds.add_memory_database("query_owner_defaults", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "insert_dog", + "insert into dogs (name) values (:name)", + is_write=True, + source="user", + owner_id="alice", + ) + + for action in ("view-query", "update-query", "delete-query"): + assert await ds.allowed( + action=action, + resource=QueryResource("data", "insert_dog"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action=action, + resource=QueryResource("data", "insert_dog"), + actor={"id": "bob"}, + ) + + +@pytest.mark.asyncio +async def test_private_query_restricts_broad_update_delete_permissions(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "update-query": {"id": "bob"}, + "delete-query": {"id": "bob"}, + }, + }, + }, + }, + ) + ds.add_memory_database("query_broad_update_delete", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "alice_private", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "alice_public", + "select 2", + is_private=False, + source="user", + owner_id="alice", + ) + + for action in ("update-query", "delete-query"): + assert await ds.allowed( + action=action, + resource=QueryResource("data", "alice_private"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action=action, + resource=QueryResource("data", "alice_private"), + actor={"id": "bob"}, + ) + assert await ds.allowed( + action=action, + resource=QueryResource("data", "alice_public"), + actor={"id": "bob"}, + ) + + private_update_response = await ds.client.post( + "/data/alice_private/-/update", + actor={"id": "bob"}, + json={"update": {"title": "Nope"}}, + ) + private_delete_response = await ds.client.post( + "/data/alice_private/-/delete", + actor={"id": "bob"}, + json={}, + ) + public_update_response = await ds.client.post( + "/data/alice_public/-/update", + actor={"id": "bob"}, + json={"update": {"title": "Bob can edit public queries"}}, + ) + public_delete_response = await ds.client.post( + "/data/alice_public/-/delete", + actor={"id": "bob"}, + json={}, + ) + + assert private_update_response.status_code == 403 + assert private_delete_response.status_code == 403 + assert public_update_response.status_code == 200 + assert public_delete_response.status_code == 200 + assert await ds.get_query("data", "alice_private") is not None + assert await ds.get_query("data", "alice_public") is None + + +@pytest.mark.asyncio +async def test_user_writable_query_execution_rechecks_table_permissions(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": ["alice", "bob"]}, + "execute-write-sql": {"id": ["alice", "bob"]}, + }, + "tables": { + "dogs": { + "permissions": { + "insert-row": {"id": "alice"}, + } + } + }, + } + } + }, + ) + db = ds.add_memory_database("query_write_execution", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "insert_dog", + "insert into dogs (name) values (:name)", + is_write=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "insert_cat", + "insert into dogs (name) values (:name)", + is_write=True, + source="user", + owner_id="bob", + ) + + allowed_response = await ds.client.post( + "/data/insert_dog?_json=1", + actor={"id": "alice"}, + data={"name": "Cleo"}, + ) + denied_response = await ds.client.post( + "/data/insert_cat?_json=1", + actor={"id": "bob"}, + data={"name": "Milo"}, + ) + + assert allowed_response.status_code == 200 + assert allowed_response.json()["ok"] is True + assert denied_response.status_code == 403 + rows = (await db.execute("select name from dogs")).dicts() + assert rows == [{"name": "Cleo"}] diff --git a/tests/test_canned_queries.py b/tests/test_stored_queries.py similarity index 70% rename from tests/test_canned_queries.py rename to tests/test_stored_queries.py index 5e36a87a..2c648d5f 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_stored_queries.py @@ -1,13 +1,19 @@ from bs4 import BeautifulSoup as Soup +from asgiref.sync import async_to_sync import json import pytest import re from .fixtures import make_app_client +def update_query(client, name, **kwargs): + async_to_sync(client.ds.invoke_startup)() + async_to_sync(client.ds.update_query)("data", name, **kwargs) + + @pytest.fixture -def canned_write_client(tmpdir): - template_dir = tmpdir / "canned_write_templates" +def stored_write_client(tmpdir): + template_dir = tmpdir / "stored_write_templates" template_dir.mkdir() (template_dir / "query-data-update_name.html").write_text( """ @@ -23,7 +29,7 @@ def canned_write_client(tmpdir): "databases": { "data": { "queries": { - "canned_read": {"sql": "select * from names"}, + "stored_read": {"sql": "select * from names"}, "add_name": { "sql": "insert into names (name) values (:name)", "write": True, @@ -60,7 +66,7 @@ def canned_write_client(tmpdir): @pytest.fixture -def canned_write_immutable_client(): +def stored_write_immutable_client(): with make_app_client( is_immutable=True, config={ @@ -80,7 +86,7 @@ def canned_write_immutable_client(): @pytest.mark.asyncio -async def test_canned_query_with_named_parameter(ds_client): +async def test_stored_query_with_named_parameter(ds_client): response = await ds_client.get( "/fixtures/neighborhood_search.json?text=town&_shape=arrays" ) @@ -94,14 +100,14 @@ async def test_canned_query_with_named_parameter(ds_client): ] -def test_insert(canned_write_client): - response = canned_write_client.post( +def test_insert(stored_write_client): + response = stored_write_client.post( "/data/add_name", {"name": "Hello"}, csrftoken_from=True, cookies={"foo": "bar"}, ) - messages = canned_write_client.ds.unsign( + messages = stored_write_client.ds.unsign( response.cookies["ds_messages"], "messages" ) assert messages == [["Query executed, 1 row affected", 1]] @@ -109,9 +115,9 @@ def test_insert(canned_write_client): assert response.headers["Location"] == "/data/add_name?success" -def test_insert_blocked_cross_site(canned_write_client): +def test_insert_blocked_cross_site(stored_write_client): # A cross-site POST (browser-originated) must be blocked - response = canned_write_client.post( + response = stored_write_client.post( "/data/add_name", {"name": "Hello"}, headers={"sec-fetch-site": "cross-site"}, @@ -119,74 +125,72 @@ def test_insert_blocked_cross_site(canned_write_client): assert 403 == response.status -def test_insert_no_cookies_no_csrf(canned_write_client): - response = canned_write_client.post("/data/add_name", {"name": "Hello"}) +def test_insert_no_cookies_no_csrf(stored_write_client): + response = stored_write_client.post("/data/add_name", {"name": "Hello"}) assert 302 == response.status assert "/data/add_name?success" == response.headers["Location"] -def test_custom_success_message(canned_write_client): - response = canned_write_client.post( +def test_custom_success_message(stored_write_client): + response = stored_write_client.post( "/data/delete_name", {"rowid": 1}, - cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, + cookies={"ds_actor": stored_write_client.actor_cookie({"id": "root"})}, csrftoken_from=True, ) assert 302 == response.status - messages = canned_write_client.ds.unsign( + messages = stored_write_client.ds.unsign( response.cookies["ds_messages"], "messages" ) assert [["Name deleted", 1]] == messages -def test_insert_error(canned_write_client): - canned_write_client.post("/data/add_name", {"name": "Hello"}, csrftoken_from=True) - response = canned_write_client.post( +def test_insert_error(stored_write_client): + stored_write_client.post("/data/add_name", {"name": "Hello"}, csrftoken_from=True) + response = stored_write_client.post( "/data/add_name_specify_id", {"rowid": 1, "name": "Should fail"}, csrftoken_from=True, ) assert 302 == response.status assert "/data/add_name_specify_id?error" == response.headers["Location"] - messages = canned_write_client.ds.unsign( + messages = stored_write_client.ds.unsign( response.cookies["ds_messages"], "messages" ) assert [["UNIQUE constraint failed: names.rowid", 3]] == messages # How about with a custom error message? - canned_write_client.ds.config["databases"]["data"]["queries"][ - "add_name_specify_id" - ]["on_error_message"] = "ERROR" - response = canned_write_client.post( + update_query(stored_write_client, "add_name_specify_id", on_error_message="ERROR") + response = stored_write_client.post( "/data/add_name_specify_id", {"rowid": 1, "name": "Should fail"}, csrftoken_from=True, ) - assert [["ERROR", 3]] == canned_write_client.ds.unsign( + assert [["ERROR", 3]] == stored_write_client.ds.unsign( response.cookies["ds_messages"], "messages" ) -def test_on_success_message_sql(canned_write_client): - response = canned_write_client.post( +def test_on_success_message_sql(stored_write_client): + response = stored_write_client.post( "/data/add_name_specify_id", {"rowid": 5, "name": "Should be OK"}, csrftoken_from=True, ) assert response.status == 302 assert response.headers["Location"] == "/data/add_name_specify_id" - messages = canned_write_client.ds.unsign( + messages = stored_write_client.ds.unsign( response.cookies["ds_messages"], "messages" ) assert messages == [["Name added: Should be OK with rowid 5", 1]] -def test_error_in_on_success_message_sql(canned_write_client): - response = canned_write_client.post( +def test_error_in_on_success_message_sql(stored_write_client): + response = stored_write_client.post( "/data/add_name_specify_id_with_error_in_on_success_message_sql", {"rowid": 1, "name": "Should fail"}, csrftoken_from=True, ) - messages = canned_write_client.ds.unsign( + messages = stored_write_client.ds.unsign( response.cookies["ds_messages"], "messages" ) assert messages == [ @@ -194,26 +198,29 @@ def test_error_in_on_success_message_sql(canned_write_client): ] -def test_custom_params(canned_write_client): - response = canned_write_client.get("/data/update_name?extra=foo") - assert '' in response.text +def test_custom_params(stored_write_client): + response = stored_write_client.get("/data/update_name?extra=foo") + assert ( + '' + in response.text + ) -def test_canned_query_pages_no_vary_header(canned_write_client): +def test_stored_query_pages_no_vary_header(stored_write_client): # These pages no longer embed per-cookie CSRF tokens, so they must not # set Vary: Cookie - they should be cacheable across users. - assert "vary" not in canned_write_client.get("/data").headers - assert "vary" not in canned_write_client.get("/data/update_name").headers + assert "vary" not in stored_write_client.get("/data").headers + assert "vary" not in stored_write_client.get("/data/update_name").headers -def test_json_post_body(canned_write_client): - response = canned_write_client.post( +def test_json_post_body(stored_write_client): + response = stored_write_client.post( "/data/add_name", body=json.dumps({"name": ["Hello", "there"]}), ) assert 302 == response.status assert "/data/add_name?success" == response.headers["Location"] - rows = canned_write_client.get("/data/names.json?_shape=array").json + rows = stored_write_client.get("/data/names.json?_shape=array").json assert rows == [{"rowid": 1, "name": "['Hello', 'there']"}] @@ -226,8 +233,8 @@ def test_json_post_body(canned_write_client): (None, '{"name": "NameGoesHere", "_json": 1}', None), ), ) -def test_json_response(canned_write_client, headers, body, querystring): - response = canned_write_client.post( +def test_json_response(stored_write_client, headers, body, querystring): + response = stored_write_client.post( "/data/add_name" + (querystring or ""), body=body, headers=headers, @@ -239,29 +246,27 @@ def test_json_response(canned_write_client, headers, body, querystring): "message": "Query executed, 1 row affected", "redirect": "/data/add_name?success", } - rows = canned_write_client.get("/data/names.json?_shape=array").json + rows = stored_write_client.get("/data/names.json?_shape=array").json assert rows == [{"rowid": 1, "name": "NameGoesHere"}] -def test_canned_query_permissions_on_database_page(canned_write_client): - # Without auth only shows three queries - query_names = { - q["name"] for q in canned_write_client.get("/data.json").json["queries"] - } +def test_stored_query_permissions_on_database_page(stored_write_client): + # Without auth shows the five public queries + anon_response = stored_write_client.get("/data.json") + query_names = {q["name"] for q in anon_response.json["queries"]} assert query_names == { "add_name_specify_id_with_error_in_on_success_message_sql", - "from_hook", "update_name", "add_name_specify_id", - "from_async_hook", - "canned_read", + "stored_read", "add_name", } + assert anon_response.json["queries_more"] is False - # With auth shows four - response = canned_write_client.get( + # With auth the database page preview shows the first five queries + response = stored_write_client.get( "/data.json", - cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, + cookies={"ds_actor": stored_write_client.actor_cookie({"id": "root"})}, ) assert response.status == 200 query_names_and_private = sorted( @@ -278,20 +283,43 @@ def test_canned_query_permissions_on_database_page(canned_write_client): "name": "add_name_specify_id_with_error_in_on_success_message_sql", "private": False, }, - {"name": "canned_read", "private": False}, {"name": "delete_name", "private": True}, - {"name": "from_async_hook", "private": False}, - {"name": "from_hook", "private": False}, + {"name": "stored_read", "private": False}, + ] + assert response.json["queries_more"] is True + + # The full query list endpoint includes the remaining query + response = stored_write_client.get( + "/data/-/queries.json?_size=10", + cookies={"ds_actor": stored_write_client.actor_cookie({"id": "root"})}, + ) + assert response.status == 200 + query_names_and_private = sorted( + [ + {"name": q["name"], "private": q["private"]} + for q in response.json["queries"] + ], + key=lambda q: q["name"], + ) + assert query_names_and_private == [ + {"name": "add_name", "private": False}, + {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, + {"name": "delete_name", "private": True}, + {"name": "stored_read", "private": False}, {"name": "update_name", "private": False}, ] -def test_canned_query_permissions(canned_write_client): - assert 403 == canned_write_client.get("/data/delete_name").status - assert 200 == canned_write_client.get("/data/update_name").status - cookies = {"ds_actor": canned_write_client.actor_cookie({"id": "root"})} - assert 200 == canned_write_client.get("/data/delete_name", cookies=cookies).status - assert 200 == canned_write_client.get("/data/update_name", cookies=cookies).status +def test_stored_query_permissions(stored_write_client): + assert 403 == stored_write_client.get("/data/delete_name").status + assert 200 == stored_write_client.get("/data/update_name").status + cookies = {"ds_actor": stored_write_client.actor_cookie({"id": "root"})} + assert 200 == stored_write_client.get("/data/delete_name", cookies=cookies).status + assert 200 == stored_write_client.get("/data/update_name", cookies=cookies).status @pytest.fixture(scope="session") @@ -327,12 +355,16 @@ def magic_parameters_client(): ], ) def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re): - magic_parameters_client.ds.config["databases"]["data"]["queries"]["runme_post"][ - "sql" - ] = f"insert into logs (line) values (:{magic_parameter})" - magic_parameters_client.ds.config["databases"]["data"]["queries"]["runme_get"][ - "sql" - ] = f"select :{magic_parameter} as result" + update_query( + magic_parameters_client, + "runme_post", + sql=f"insert into logs (line) values (:{magic_parameter})", + ) + update_query( + magic_parameters_client, + "runme_get", + sql=f"select :{magic_parameter} as result", + ) cookies = { "ds_actor": magic_parameters_client.actor_cookie({"id": "root"}), "foo": "bar", @@ -366,9 +398,11 @@ def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re) @pytest.mark.parametrize("use_csrf", [True, False]) @pytest.mark.parametrize("return_json", [True, False]) def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_json): - magic_parameters_client.ds.config["databases"]["data"]["queries"]["runme_post"][ - "sql" - ] = "insert into logs (line) values (:_header_host)" + update_query( + magic_parameters_client, + "runme_post", + sql="insert into logs (line) values (:_header_host)", + ) qs = "" if return_json: qs = "?_json=1" @@ -400,8 +434,8 @@ def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_c assert response.json["error"].startswith("You did not supply a value for binding") -def test_canned_write_custom_template(canned_write_client): - response = canned_write_client.get("/data/update_name") +def test_stored_write_custom_template(stored_write_client): + response = stored_write_client.get("/data/update_name") assert response.status == 200 assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text assert ( @@ -419,10 +453,10 @@ def test_canned_write_custom_template(canned_write_client): ) -def test_canned_write_query_disabled_for_immutable_database( - canned_write_immutable_client, +def test_stored_write_query_disabled_for_immutable_database( + stored_write_immutable_client, ): - response = canned_write_immutable_client.get("/fixtures/add") + response = stored_write_immutable_client.get("/fixtures/add") assert response.status == 200 assert ( "This query cannot be executed because the database is immutable." @@ -430,7 +464,7 @@ def test_canned_write_query_disabled_for_immutable_database( ) assert '' in response.text # Submitting form should get a forbidden error - response = canned_write_immutable_client.post( + response = stored_write_immutable_client.post( "/fixtures/add", {"text": "text"}, csrftoken_from=True, diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py new file mode 100644 index 00000000..5730cd0d --- /dev/null +++ b/tests/test_utils_sql_analysis.py @@ -0,0 +1,188 @@ +import pytest + +from datasette.utils.sqlite import sqlite3 +from datasette.utils.sql_analysis import analyze_sql_tables + + +@pytest.fixture +def conn(): + conn = sqlite3.connect(":memory:") + conn.executescript(""" + create table dogs (id integer primary key, name text, age integer); + create table cats (id integer primary key, name text); + create table log (message text); + create view dog_names as select id, name from dogs; + create trigger dogs_after_insert after insert on dogs begin + update cats set name = new.name where id = new.id; + insert into log (message) values (new.name); + end; + create trigger dog_names_instead_of_update instead of update on dog_names begin + update dogs set name = new.name where id = old.id; + end; + """) + try: + yield conn + finally: + conn.close() + + +def as_tuples(analysis): + return [ + ( + access.operation, + access.database, + access.sqlite_schema, + access.table, + access.columns, + access.source, + ) + for access in analysis.table_accesses + ] + + +def test_analyze_select_tables(conn): + analysis = analyze_sql_tables( + conn, + "select dogs.name, cats.name from dogs join cats on dogs.id = cats.id where dogs.age > ?", + (2,), + database_name="data", + ) + + assert set(as_tuples(analysis)) == { + ("read", "data", "main", "cats", ("id", "name"), None), + ("read", "data", "main", "dogs", ("age", "id", "name"), None), + } + + +def test_analyze_uses_sqlite_schema_as_default_database(conn): + analysis = analyze_sql_tables(conn, "select name from dogs") + + assert set(as_tuples(analysis)) == { + ("read", "main", "main", "dogs", ("name",), None), + } + + +def test_analyze_insert_tables(conn): + analysis = analyze_sql_tables( + conn, + "insert into dogs (name, age) values (:name, :age)", + {"name": "Cleo", "age": 4}, + database_name="data", + ) + + assert set(as_tuples(analysis)) == { + ("insert", "data", "main", "dogs", (), None), + ("read", "data", "main", "dogs", ("id", "name"), "dogs_after_insert"), + ("update", "data", "main", "cats", ("name",), "dogs_after_insert"), + ("read", "data", "main", "cats", ("id",), "dogs_after_insert"), + ("insert", "data", "main", "log", (), "dogs_after_insert"), + } + + +def test_analyze_update_tables(conn): + analysis = analyze_sql_tables( + conn, + "update dogs set age = age + 1 where name = ?", + ("Cleo",), + database_name="data", + ) + + assert set(as_tuples(analysis)) == { + ("update", "data", "main", "dogs", ("age",), None), + ("read", "data", "main", "dogs", ("age", "name"), None), + } + + +def test_analyze_delete_tables(conn): + analysis = analyze_sql_tables( + conn, + "delete from dogs where name = ?", + ("Cleo",), + database_name="data", + ) + + assert set(as_tuples(analysis)) == { + ("delete", "data", "main", "dogs", (), None), + ("read", "data", "main", "dogs", ("name",), None), + } + + +def test_analyze_insert_select_with_cte(conn): + analysis = analyze_sql_tables( + conn, + """ + with old_dogs as ( + select name from dogs where age > :age + ) + insert into cats (name) + select name from old_dogs + """, + {"age": 10}, + database_name="data", + ) + + assert set(as_tuples(analysis)) == { + ("insert", "data", "main", "cats", (), None), + ("read", "data", "main", "dogs", ("age", "name"), "old_dogs"), + } + + +def test_analyze_view_with_instead_of_trigger(conn): + analysis = analyze_sql_tables( + conn, + "update dog_names set name = :name where id = :id", + {"name": "Zelda", "id": 1}, + database_name="data", + ) + + assert set(as_tuples(analysis)) == { + ("update", "data", "main", "dog_names", ("name",), None), + ("read", "data", "main", "dogs", ("id", "name"), "dog_names"), + ("read", "data", "main", "dog_names", ("id", "name"), "dog_names"), + ( + "read", + "data", + "main", + "dog_names", + ("id", "name"), + "dog_names_instead_of_update", + ), + ("update", "data", "main", "dogs", ("name",), "dog_names_instead_of_update"), + ("read", "data", "main", "dogs", ("id",), "dog_names_instead_of_update"), + } + + +def test_analyze_attached_database_tables(conn): + conn.execute("attach database ':memory:' as extra") + conn.execute("create table extra.people (id integer primary key, name text)") + + analysis = analyze_sql_tables( + conn, + "insert into extra.people (name) select name from dogs", + database_name="data", + schema_to_database={"extra": "extra_db"}, + ) + + assert set(as_tuples(analysis)) == { + ("insert", "extra_db", "extra", "people", (), None), + ("read", "data", "main", "dogs", ("name",), None), + } + + +def test_analyze_clears_authorizer_on_error(): + class FakeConnection: + def __init__(self): + self.authorizers = [] + + def set_authorizer(self, authorizer): + self.authorizers.append(authorizer) + + def execute(self, sql, params): + raise sqlite3.OperationalError("bad SQL") + + conn = FakeConnection() + + with pytest.raises(sqlite3.OperationalError): + analyze_sql_tables(conn, "bad SQL") + + assert conn.authorizers[-1] is None
    Required permissioninsertupdateread