datasette/datasette/views/execute_write.py

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

483 lines
17 KiB
Python
Raw Normal View History

import re
from urllib.parse import urlencode
from datasette.database import QueryInterrupted
from datasette.resources import DatabaseResource
from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3
from datasette.utils.asgi import Response
from .base import BaseView
from .database import display_rows as display_query_rows
from .query_helpers import (
SQL_PARAMETER_FORM_PREFIX,
QueryValidationError,
_analysis_is_write,
_analysis_rows,
_analysis_rows_with_permissions,
_block_framing,
_coerce_execute_write_payload,
_derived_query_parameters,
_execute_write_analysis_data,
_execute_write_disabled_reason,
_inserted_row_url,
_json_or_form_payload,
_prepare_execute_write,
_table_columns,
_wants_json,
)
WRITE_TEMPLATE_LABELS = {
"insert": "Insert row",
"update": "Update rows",
"delete": "Delete rows",
}
WRITE_TEMPLATE_OPERATIONS = tuple(WRITE_TEMPLATE_LABELS)
CREATE_TABLE_TEMPLATE_SQL = "create table new_table (\n id integer primary key,\n name text\n -- created text default (datetime('now'))\n)"
def _parameter_names(columns):
seen = set()
names = {}
for column in columns:
base = re.sub(r"[^a-z0-9_]+", "_", column.lower())
base = base.strip("_") or "value"
if base[0].isdigit():
base = f"p_{base}"
name = base
index = 2
while name in seen:
name = f"{base}_{index}"
index += 1
seen.add(name)
names[column] = name
return names
def _quote_identifier(identifier):
return '"{}"'.format(identifier.replace('"', '""'))
def _preferred_where_column(table, columns):
lower_table_id = f"{table.lower()}_id"
return (
next((column for column in columns if column.lower() == "id"), None)
or next(
(column for column in columns if column.lower() == lower_table_id), None
)
or columns[0]
)
def _auto_incrementing_primary_key(columns):
primary_keys = [column for column in columns if column.is_pk]
if len(primary_keys) != 1:
return None
primary_key = primary_keys[0]
if primary_key.type and primary_key.type.lower() == "integer":
return primary_key.name
return None
def _insert_template_sql(table, columns):
column_names = [column.name for column in columns]
auto_pk = _auto_incrementing_primary_key(columns)
insert_columns = [column for column in column_names if column != auto_pk]
if not insert_columns:
return f"insert into {_quote_identifier(table)}\ndefault values"
names = _parameter_names(insert_columns)
return "\n".join(
(
f"insert into {_quote_identifier(table)} (",
",\n".join(f" {_quote_identifier(column)}" for column in insert_columns),
")",
"values (",
",\n".join(f" :{names[column]}" for column in insert_columns),
")",
)
)
def _update_template_sql(table, columns):
column_names = [column.name for column in columns]
names = _parameter_names(column_names)
where_column = _preferred_where_column(table, column_names)
set_columns = [column for column in column_names if column != where_column]
if not set_columns:
return "\n".join(
(
f"update {_quote_identifier(table)}",
f"set {_quote_identifier(where_column)} = :new_{names[where_column]}",
f"where {_quote_identifier(where_column)} = :{names[where_column]}",
)
)
return "\n".join(
(
f"update {_quote_identifier(table)}",
"set "
+ ",\n".join(
"{}{} = :{}".format(
" " if index else "",
_quote_identifier(column),
names[column],
)
for index, column in enumerate(set_columns)
),
f"where {_quote_identifier(where_column)} = :{names[where_column]}",
)
)
def _delete_template_sql(table, columns):
column_names = [column.name for column in columns]
names = _parameter_names(column_names)
where_column = _preferred_where_column(table, column_names)
return "\n".join(
(
f"delete from {_quote_identifier(table)}",
f"where {_quote_identifier(where_column)} = :{names[where_column]}",
)
)
def _template_sqls_for_table(table, columns):
return {
"insert": _insert_template_sql(table, columns),
"update": _update_template_sql(table, columns),
"delete": _delete_template_sql(table, columns),
}
async def _template_sql_allowed(datasette, db, sql, actor):
params = {parameter: "" for parameter in _derived_query_parameters(sql)}
try:
analysis = await db.analyze_sql(sql, params)
except sqlite3.DatabaseError:
return False
if not _analysis_is_write(analysis):
return False
analysis_rows = await _analysis_rows_with_permissions(datasette, analysis, actor)
return _execute_write_disabled_reason(sql, None, analysis_rows) is None
async def _write_template_tables(
datasette, db, table_columns, hidden_table_names, actor
):
write_template_tables = {}
for table in table_columns:
if table in hidden_table_names or not table_columns[table]:
continue
column_details = [
column
for column in await db.table_column_details(table)
if not column.hidden
]
if not column_details:
continue
templates = {}
for operation, sql in _template_sqls_for_table(table, column_details).items():
if await _template_sql_allowed(datasette, db, sql, actor):
templates[operation] = sql
if templates:
write_template_tables[table] = {
"templates": templates,
}
return write_template_tables
def _write_template_operations(write_template_tables):
operations = []
for operation in WRITE_TEMPLATE_OPERATIONS:
if any(
operation in table["templates"] for table in write_template_tables.values()
):
operations.append(
{
"name": operation,
"label": WRITE_TEMPLATE_LABELS[operation],
}
)
return operations
async def _create_table_template_sql(datasette, db, actor):
if await datasette.allowed(
action="create-table",
resource=DatabaseResource(db.name),
actor=actor,
):
return CREATE_TABLE_TEMPLATE_SQL
return None
class ExecuteWriteView(BaseView):
name = "execute-write"
has_json_alternate = False
async def _render_form(
self,
request,
db,
*,
sql="",
parameter_values=None,
analysis=None,
analysis_error=None,
execution_message=None,
execution_links=None,
execution_ok=None,
execute_write_returns_rows=False,
execute_write_columns=None,
execute_write_display_rows=None,
execute_write_truncated=False,
status=200,
):
parameter_values = parameter_values or {}
execution_links = execution_links or []
execute_write_columns = execute_write_columns or []
execute_write_display_rows = execute_write_display_rows or []
parameter_names = []
analysis_rows = []
table_columns = await _table_columns(self.ds, db.name)
hidden_table_names = set(await db.hidden_table_names())
write_template_tables = await _write_template_tables(
self.ds, db, table_columns, hidden_table_names, request.actor
)
write_template_operations = _write_template_operations(write_template_tables)
write_create_table_template_sql = await _create_table_template_sql(
self.ds, db, request.actor
)
if sql and analysis_error is None:
try:
parameter_names = _derived_query_parameters(sql)
if analysis is None:
params = {parameter: "" for parameter in parameter_names}
analysis = await db.analyze_sql(sql, params)
if _analysis_is_write(analysis):
analysis_rows = await _analysis_rows_with_permissions(
self.ds, analysis, request.actor
)
else:
analysis_error = (
"Use /-/query for read-only SQL; "
"this endpoint only executes writes"
)
except (QueryValidationError, sqlite3.DatabaseError) as ex:
analysis_error = getattr(ex, "message", str(ex))
allow_save_query = await self.ds.allowed(
action="execute-sql",
resource=DatabaseResource(db.name),
actor=request.actor,
) and await self.ds.allowed(
action="store-query",
resource=DatabaseResource(db.name),
actor=request.actor,
)
save_query_base_url = None
save_query_url = None
execute_disabled_reason = _execute_write_disabled_reason(
sql, analysis_error, analysis_rows
)
if allow_save_query:
save_query_base_url = self.ds.urls.database(db.name) + "/-/queries/store"
if not execute_disabled_reason:
save_query_url = save_query_base_url + "?" + urlencode({"sql": sql})
response = await self.render(
["execute_write.html"],
request,
{
"database": db.name,
"database_color": db.color,
"sql": sql,
"parameter_names": parameter_names,
"parameter_values": parameter_values,
"analysis_error": analysis_error,
"analysis_rows": analysis_rows,
"execution_message": execution_message,
"execution_links": execution_links,
"execution_ok": execution_ok,
"execute_write_returns_rows": execute_write_returns_rows,
"execute_write_columns": execute_write_columns,
"execute_write_display_rows": execute_write_display_rows,
"execute_write_truncated": execute_write_truncated,
"sql_parameter_name_prefix": SQL_PARAMETER_FORM_PREFIX,
"execute_disabled": bool(execute_disabled_reason),
"execute_disabled_reason": execute_disabled_reason,
"table_columns": table_columns,
"write_template_tables": write_template_tables,
"write_template_operations": write_template_operations,
"write_create_table_template_sql": write_create_table_template_sql,
"save_query_url": save_query_url,
"save_query_base_url": save_query_base_url,
},
)
response.status = status
return _block_framing(response)
async def get(self, request):
db = await self.ds.resolve_database(request)
await self.ds.ensure_permission(
action="execute-write-sql",
resource=DatabaseResource(db.name),
actor=request.actor,
)
if not db.is_mutable:
return _block_framing(
Response.error(
["Cannot execute write SQL because this database is immutable."],
403,
)
)
return await self._render_form(
request,
db,
sql=request.args.get("sql") or "",
)
async def post(self, request):
db = await self.ds.resolve_database(request)
if not await self.ds.allowed(
action="execute-write-sql",
resource=DatabaseResource(db.name),
actor=request.actor,
):
return _block_framing(
Response.error(["Permission denied: need execute-write-sql"], 403)
)
if not db.is_mutable:
return _block_framing(Response.error(["Database is immutable"], 403))
data = {}
is_json = request.headers.get("content-type", "").startswith("application/json")
sql = ""
provided_params = {}
try:
data, is_json = await _json_or_form_payload(request)
sql, provided_params = _coerce_execute_write_payload(data, is_json)
parameter_names, params, analysis = await _prepare_execute_write(
self.ds, db, sql, provided_params, request.actor
)
except QueryValidationError as ex:
if _wants_json(request, is_json, data):
return _block_framing(Response.error([ex.message], ex.status))
if ex.flash:
self.ds.add_message(request, ex.message, self.ds.ERROR)
return await self._render_form(
request,
db,
sql=sql or "",
parameter_values=provided_params,
analysis_error=None if ex.flash else ex.message,
execution_message=None if ex.flash else ex.message,
execution_ok=False,
status=ex.status,
)
wants_json = _wants_json(request, is_json, data)
try:
execute_write_kwargs = {"request": request}
cursor = await db.execute_write(sql, params, **execute_write_kwargs)
except (QueryInterrupted, sqlite3.DatabaseError) as ex:
message = str(ex)
if wants_json:
return _block_framing(Response.error([message], 400))
return await self._render_form(
request,
db,
sql=sql,
parameter_values=params,
analysis=analysis,
execution_message=message,
execution_ok=False,
status=400,
)
Connection budget: pages, startup and sweeps only touch the databases they need Adds tests/connection_budget.py, a helper that records which databases a block of code opens, reads, writes or ATTACHes, and tests/test_connection_budget.py which asserts budgets for ~60 endpoints, startup, sweeps and telemetry with 50 attached databases. Fixes for every all-database loop it found: - Index page: tables, views, hidden tables, columns, primary keys, FTS tables and foreign-key relationship counts come from the _internal catalog (a few queries for any number of databases). Row counts are only run when at most 20 databases need counting (COUNT_MAX_DATABASES); above that the page shows table counts plus counts already known. - Derived-table permissions: allowed_resources() no longer queries every database per page; dependency maps are computed by the SchemaWatcher from the rows it scans, cached per database, and read back from the catalog for databases restored from a persisted catalog. - catalog_tables gains a type column (PRAGMA table_list: table, virtual, shadow) so hidden shadow tables can be identified without a connection. - PaginatedResources.all() fetches pages of 1000 after the first. - /-/api and /-/debug/autocomplete use the catalog; /-/schema does one permission query and skips databases with nothing visible. - /-/versions computes SQLite details once per instance, without crossdb ATTACHes; detect_json1() caches its answer. - /-/databases only stats/hashes databases the actor can see. - Immutable table counts are computed when first needed, not at startup. - check_databases() (serve, --get) reuses the catalog scan instead of opening every database a second time, and reports "file is not a database" as a usage error. - SchemaWatcher: failed scans are retried when the file changes, not on every sweep; scan connections skip crossdb ATTACH; catalog_generation counter for caches. - The SQL write UI no longer forces a stat sweep of every database. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EwkLo2rJWidpkDVT3HkvNv
2026-10-04 05:58:31 +00:00
# No refresh_schemas() here: execute_write() already rebuilt this
# database's catalog rows before returning if the SQL changed its
# schema (the SchemaWatcher's post-write check), and a forced
# refresh would stat every attached database on every write
if cursor.rowcount == -1:
message = "Query executed"
else:
message = "Query executed, {} row{} affected".format(
cursor.rowcount, "" if cursor.rowcount == 1 else "s"
)
if wants_json:
data = {
"ok": True,
"message": message,
"rowcount": cursor.rowcount,
"rows": [],
"truncated": False,
"analysis": _analysis_rows(analysis),
}
if cursor.description is not None:
data["rows"] = [dict(row) for row in cursor.fetchall()]
data["truncated"] = cursor.truncated
return _block_framing(Response.json(data))
inserted_row_url = await _inserted_row_url(self.ds, db, analysis, cursor)
execution_links = (
[{"href": inserted_row_url, "label": "View row"}]
if inserted_row_url
else []
)
execute_write_returns_rows = cursor.description is not None
execute_write_columns = []
execute_write_display_rows = []
if execute_write_returns_rows:
execute_write_columns = [
description[0] for description in cursor.description
]
execute_write_display_rows = await display_query_rows(
self.ds,
db.name,
request,
cursor.fetchall(),
execute_write_columns,
)
return await self._render_form(
request,
db,
sql=sql,
parameter_values={name: params.get(name, "") for name in parameter_names},
analysis=analysis,
execution_message=message,
execution_links=execution_links,
execution_ok=True,
execute_write_returns_rows=execute_write_returns_rows,
execute_write_columns=execute_write_columns,
execute_write_display_rows=execute_write_display_rows,
execute_write_truncated=cursor.truncated,
)
class ExecuteWriteAnalyzeView(BaseView):
name = "execute-write-analyze"
has_json_alternate = False
async def get(self, request):
db = await self.ds.resolve_database(request)
if not await self.ds.allowed(
action="execute-write-sql",
resource=DatabaseResource(db.name),
actor=request.actor,
):
return _block_framing(
Response.error(["Permission denied: need execute-write-sql"], 403)
)
invalid_keys = set(request.args) - {"sql"}
if invalid_keys:
return _block_framing(
Response.error(
["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))],
400,
)
)
sql = request.args.get("sql") or ""
analysis = await _execute_write_analysis_data(self.ds, db, sql, request.actor)
analysis["unstable"] = UNSTABLE_API_MESSAGE
return _block_framing(Response.json(analysis))