diff --git a/datasette/app.py b/datasette/app.py index 139e4c34..6ea3d5a4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -84,6 +84,7 @@ from .views.special import ( ) from .views.table import ( TableAutocompleteView, + TableAlterView, TableInsertView, TableUpsertView, TableSetColumnTypeView, @@ -2626,6 +2627,10 @@ class Datasette: TableUpsertView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/upsert$", ) + add_route( + TableAlterView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/alter$", + ) add_route( TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/set-column-type$", diff --git a/datasette/views/table.py b/datasette/views/table.py index c5448c85..11a28323 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,10 +1,12 @@ import asyncio import itertools import json +from typing import Annotated, Any, Literal, Union import urllib import urllib.parse import markupsafe +from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator from datasette.column_types import SQLiteType from datasette.extras import extra_names_from_request @@ -46,6 +48,7 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response from datasette.filters import Filters import sqlite_utils +from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView from .table_extras import ( @@ -649,6 +652,154 @@ async def display_columns_and_rows( return columns, cell_rows +SqliteApiType = Literal["text", "integer", "float", "blob"] +DefaultExpr = Literal["current_timestamp", "current_date", "current_time"] +DEFAULT_EXPR_SQL = { + "current_timestamp": "CURRENT_TIMESTAMP", + "current_date": "CURRENT_DATE", + "current_time": "CURRENT_TIME", +} + + +class _StrictPydanticModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class _DefaultArgsMixin(_StrictPydanticModel): + default: Any | None = None + default_expr: DefaultExpr | None = None + + @model_validator(mode="after") + def validate_default_fields(self): + has_default = "default" in self.model_fields_set + has_default_expr = "default_expr" in self.model_fields_set + if has_default and has_default_expr: + raise ValueError("default and default_expr cannot both be provided") + if has_default_expr and self.default_expr is None: + raise ValueError("default_expr cannot be null") + return self + + +class AddColumnArgs(_DefaultArgsMixin): + name: str + type: SqliteApiType = "text" + not_null: bool = False + + +class RenameColumnArgs(_StrictPydanticModel): + name: str + to: str + + +class AlterColumnArgs(_DefaultArgsMixin): + name: str + type: SqliteApiType | None = None + not_null: bool | None = None + + @model_validator(mode="after") + def require_change(self): + if not ( + {"type", "not_null", "default", "default_expr"} & self.model_fields_set + ): + raise ValueError( + "At least one of type, not_null, default or default_expr must be provided" + ) + return self + + +class DropColumnArgs(_StrictPydanticModel): + name: str + + +class SetPrimaryKeyArgs(_StrictPydanticModel): + columns: list[str] = Field(min_length=1) + + +class ReorderColumnsArgs(_StrictPydanticModel): + columns: list[str] = Field(min_length=1) + + +class AddColumnOperation(_StrictPydanticModel): + op: Literal["add_column"] + args: AddColumnArgs + + +class RenameColumnOperation(_StrictPydanticModel): + op: Literal["rename_column"] + args: RenameColumnArgs + + +class AlterColumnOperation(_StrictPydanticModel): + op: Literal["alter_column"] + args: AlterColumnArgs + + +class DropColumnOperation(_StrictPydanticModel): + op: Literal["drop_column"] + args: DropColumnArgs + + +class SetPrimaryKeyOperation(_StrictPydanticModel): + op: Literal["set_primary_key"] + args: SetPrimaryKeyArgs + + +class ReorderColumnsOperation(_StrictPydanticModel): + op: Literal["reorder_columns"] + args: ReorderColumnsArgs + + +AlterTableOperation = Annotated[ + Union[ + AddColumnOperation, + RenameColumnOperation, + AlterColumnOperation, + DropColumnOperation, + SetPrimaryKeyOperation, + ReorderColumnsOperation, + ], + Field(discriminator="op"), +] + + +class AlterTableRequest(_StrictPydanticModel): + operations: list[AlterTableOperation] = Field(min_length=1) + dry_run: bool = False + + +def _pydantic_errors(validation_error): + errors = [] + for error in validation_error.errors(): + location = ".".join(str(item) for item in error["loc"]) + message = error["msg"] + errors.append("{}: {}".format(location, message) if location else message) + return errors + + +def _table_schema_from_conn(conn, table_name): + row = conn.execute( + "select sql from sqlite_master where type = 'table' and name = ?", + [table_name], + ).fetchone() + return row[0] if row else None + + +def _primary_key_value(columns): + if len(columns) == 1: + return columns[0] + return tuple(columns) + + +def _default_expression_sql(default_expr): + return DEFAULT_EXPR_SQL[default_expr] + + +def _literal_default(db, value): + if isinstance(value, str): + return db.quote(value) + return value + + class TableInsertView(BaseView): name = "table-insert" @@ -946,6 +1097,208 @@ class TableUpsertView(TableInsertView): return await super().post(request, upsert=True) +class TableAlterView(BaseView): + name = "table-alter" + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + try: + resolved = await self.ds.resolve_table(request) + except NotFound as e: + return _error([e.args[0]], 404) + + db = resolved.db + database_name = db.name + table_name = resolved.table + + if not await self.ds.allowed( + action="alter-table", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return _error(["Permission denied: need alter-table"], 403) + + if not db.is_mutable: + return _error(["Database is immutable"], 403) + + content_type = request.headers.get("content-type") or "" + if not content_type.startswith("application/json"): + return _error(["Invalid content-type, must be application/json"], 400) + + try: + data = await request.json() + except json.JSONDecodeError as e: + return _error(["Invalid JSON: {}".format(e)], 400) + + if not isinstance(data, dict): + return _error(["JSON must be a dictionary"], 400) + + try: + alter_request = AlterTableRequest.model_validate(data) + except ValidationError as e: + return _error(_pydantic_errors(e), 400) + + def alter_table(conn): + before_schema = _table_schema_from_conn(conn, table_name) + + def apply_operations(operation_conn): + db_for_write = sqlite_utils.Database(operation_conn) + table = db_for_write[table_name] + + add_columns = [] + types = {} + rename = {} + drop = set() + not_null = {} + defaults = {} + column_order = None + pk = SQLITE_UTILS_DEFAULT + + for operation in alter_request.operations: + args = operation.args + if operation.op == "add_column": + if args.not_null and not ( + ( + "default" in args.model_fields_set + and args.default is not None + ) + or "default_expr" in args.model_fields_set + ): + raise ValueError( + "add_column args.default or args.default_expr is required when not_null is true" + ) + add_columns.append(args) + if "default" in args.model_fields_set and not args.not_null: + defaults[args.name] = _literal_default( + db_for_write, args.default + ) + if "default_expr" in args.model_fields_set and not args.not_null: + defaults[args.name] = _default_expression_sql( + args.default_expr + ) + elif operation.op == "rename_column": + rename[args.name] = args.to + elif operation.op == "alter_column": + if args.type is not None: + types[args.name] = args.type + if args.not_null is not None: + not_null[args.name] = args.not_null + if "default" in args.model_fields_set: + defaults[args.name] = ( + None + if args.default is None + else _literal_default(db_for_write, args.default) + ) + if "default_expr" in args.model_fields_set: + defaults[args.name] = _default_expression_sql( + args.default_expr + ) + elif operation.op == "drop_column": + drop.add(args.name) + elif operation.op == "set_primary_key": + pk = _primary_key_value(args.columns) + elif operation.op == "reorder_columns": + column_order = args.columns + + with operation_conn: + for column in add_columns: + not_null_default = None + if column.not_null: + if "default_expr" in column.model_fields_set: + not_null_default = _default_expression_sql( + column.default_expr + ) + else: + not_null_default = _literal_default( + db_for_write, column.default + ) + table.add_column( + column.name, + column.type, + not_null_default=not_null_default, + ) + + should_transform = any( + ( + types, + rename, + drop, + not_null, + defaults, + column_order is not None, + pk is not SQLITE_UTILS_DEFAULT, + ) + ) + if should_transform: + table.transform( + types=types or None, + rename=rename or None, + drop=drop or None, + pk=pk, + not_null=not_null or None, + defaults=defaults or None, + column_order=column_order, + ) + + return _table_schema_from_conn(operation_conn, table_name) + + if alter_request.dry_run: + memory_conn = sqlite3.connect(":memory:") + try: + conn.backup(memory_conn) + return before_schema, apply_operations(memory_conn) + finally: + memory_conn.close() + + after_schema = apply_operations(conn) + return before_schema, after_schema + + try: + before_schema, after_schema = await db.execute_write_fn( + alter_table, request=request + ) + except Exception as e: + return _error([str(e)], 400) + + altered = before_schema != after_schema + if altered and not alter_request.dry_run: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=before_schema, + after_schema=after_schema, + ) + ) + + table_url = self.ds.absolute_url( + request, self.ds.urls.table(database_name, table_name) + ) + table_api_url = self.ds.absolute_url( + request, self.ds.urls.table(database_name, table_name, format="json") + ) + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "table_url": table_url, + "table_api_url": table_api_url, + "altered": altered, + "schema": after_schema, + "before_schema": before_schema, + "operations_applied": 0 + if alter_request.dry_run + else len(alter_request.operations), + "dry_run": alter_request.dry_run, + }, + status=200, + ) + + class TableSetColumnTypeView(BaseView): name = "table-set-column-type" diff --git a/docs/json_api.rst b/docs/json_api.rst index f7a0caae..4074b479 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -2072,6 +2072,109 @@ To use the ``"replace": true`` option you will also need the :ref:`actions_updat Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`actions_alter_table` permission. +.. _TableAlterView: + +Altering tables +~~~~~~~~~~~~~~~ + +To alter an existing table, make a ``POST`` to ``//
/-/alter``. This requires the :ref:`actions_alter_table` permission. + +:: + + POST //
/-/alter + Content-Type: application/json + Authorization: Bearer dstok_ + +The request body should include an ``operations`` array. Each operation has the same top-level shape: an ``op`` string and an ``args`` object. + +.. code-block:: json + + { + "operations": [ + { + "op": "add_column", + "args": { + "name": "slug", + "type": "text", + "not_null": true, + "default": "" + } + }, + { + "op": "add_column", + "args": { + "name": "created", + "type": "text", + "default_expr": "current_timestamp" + } + }, + { + "op": "rename_column", + "args": { + "name": "title", + "to": "headline" + } + }, + { + "op": "alter_column", + "args": { + "name": "score", + "type": "float" + } + }, + { + "op": "drop_column", + "args": { + "name": "draft_notes" + } + }, + { + "op": "set_primary_key", + "args": { + "columns": ["id"] + } + }, + { + "op": "reorder_columns", + "args": { + "columns": ["id", "headline", "slug", "created", "score"] + } + } + ] + } + +Set ``"dry_run": true`` to validate the operations and return the schema that would be created without modifying the table. + +Supported operations: + +* ``add_column`` adds a new column. ``args`` accepts ``name``, optional ``type`` of ``text``, ``integer``, ``float`` or ``blob``, optional ``not_null``, optional literal ``default`` and optional ``default_expr``. If ``not_null`` is ``true`` either a non-null ``default`` or ``default_expr`` is required. +* ``rename_column`` renames a column. ``args`` accepts ``name`` and ``to``. +* ``alter_column`` changes column properties. ``args`` accepts ``name`` and at least one of ``type``, ``not_null``, literal ``default`` or ``default_expr``. Passing ``"default": null`` removes an existing default. +* ``drop_column`` drops a column. ``args`` accepts ``name``. +* ``set_primary_key`` changes the table primary key. ``args`` accepts ``columns``, a list of one or more column names. +* ``reorder_columns`` reorders columns. ``args`` accepts ``columns``, a list of one or more column names. Columns omitted from this list will appear afterwards in their existing order. + +``default`` is always treated as a literal value. ``default_expr`` accepts one of ``current_timestamp``, ``current_date`` or ``current_time`` and is rendered as the corresponding SQLite default expression. + +A successful response returns the new schema and the previous schema: + +.. code-block:: json + + { + "ok": true, + "database": "data", + "table": "posts", + "table_url": "http://127.0.0.1:8001/data/posts", + "table_api_url": "http://127.0.0.1:8001/data/posts.json", + "altered": true, + "schema": "CREATE TABLE ...", + "before_schema": "CREATE TABLE ...", + "operations_applied": 7, + "dry_run": false + } + +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. + .. _TableSetColumnTypeView: Setting a column type diff --git a/pyproject.toml b/pyproject.toml index a19dc957..38776b2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "asyncinject>=0.7", "setuptools", "pip", + "pydantic>=2", ] [project.urls] diff --git a/tests/test_api_write.py b/tests/test_api_write.py index b7ceb6b2..f117c06e 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -794,6 +794,211 @@ async def test_update_row_alter(ds_write): assert response.json() == {"ok": True} +@pytest.mark.asyncio +async def test_alter_table_operations(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + before_schema = await db.execute_fn( + lambda conn: conn.execute( + "select sql from sqlite_master where type = 'table' and name = 'docs'" + ).fetchone()[0] + ) + + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "operations": [ + { + "op": "add_column", + "args": { + "name": "slug", + "type": "text", + "not_null": True, + "default": "", + }, + }, + { + "op": "add_column", + "args": { + "name": "created", + "type": "text", + "default_expr": "current_timestamp", + }, + }, + { + "op": "add_column", + "args": { + "name": "literal_default", + "type": "text", + "default": "hello)", + }, + }, + {"op": "rename_column", "args": {"name": "title", "to": "headline"}}, + { + "op": "alter_column", + "args": {"name": "age", "type": "text", "default": "0"}, + }, + {"op": "drop_column", "args": {"name": "score"}}, + { + "op": "reorder_columns", + "args": { + "columns": [ + "id", + "headline", + "slug", + "created", + "literal_default", + "age", + ] + }, + }, + {"op": "set_primary_key", "args": {"columns": ["id"]}}, + ] + }, + headers=_headers(token), + ) + + assert response.status_code == 200, response.text + data = response.json() + assert data["ok"] is True + assert data["database"] == "data" + assert data["table"] == "docs" + assert data["altered"] is True + assert data["operations_applied"] == 8 + assert data["before_schema"] == before_schema + assert "headline" in data["schema"] + assert "score" not in data["schema"] + assert "DEFAULT CURRENT_TIMESTAMP" in data["schema"] + assert "DEFAULT 'hello)'" in data["schema"] + + columns = ( + await db.execute("select * from pragma_table_info('docs') order by cid") + ).dicts() + assert [column["name"] for column in columns] == [ + "id", + "headline", + "slug", + "created", + "literal_default", + "age", + ] + assert columns[0]["pk"] == 1 + assert columns[2]["notnull"] == 1 + assert columns[2]["dflt_value"] == "''" + assert columns[3]["dflt_value"] == "CURRENT_TIMESTAMP" + assert columns[4]["dflt_value"] == "'hello)'" + assert columns[5]["type"] == "TEXT" + assert columns[5]["dflt_value"] == "'0'" + + event = last_event(ds_write) + assert event.name == "alter-table" + assert event.database == "data" + assert event.table == "docs" + assert event.before_schema == before_schema + assert event.after_schema == data["schema"] + + +@pytest.mark.asyncio +async def test_alter_table_dry_run(ds_write): + token = write_token(ds_write, permissions=["at"]) + db = ds_write.get_database("data") + response = await ds_write.client.post( + "/data/docs/-/alter", + json={ + "dry_run": True, + "operations": [ + {"op": "add_column", "args": {"name": "slug", "type": "text"}} + ], + }, + headers=_headers(token), + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["ok"] is True + assert data["dry_run"] is True + assert data["altered"] is True + assert data["operations_applied"] == 0 + assert "slug" in data["schema"] + columns = ( + await db.execute("select name from pragma_table_info('docs') order by cid") + ).dicts() + assert [column["name"] for column in columns] == ["id", "title", "score", "age"] + assert last_event(ds_write) is None + + +@pytest.mark.asyncio +async def test_alter_table_permission_denied(ds_write): + token = write_token(ds_write, permissions=["ir"]) + response = await ds_write.client.post( + "/data/docs/-/alter", + json={"operations": [{"op": "add_column", "args": {"name": "slug"}}]}, + headers=_headers(token), + ) + assert response.status_code == 403 + assert response.json() == { + "ok": False, + "errors": ["Permission denied: need alter-table"], + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body,expected_error", + ( + ( + {"operations": [{"op": "add_column", "args": {"type": "text"}}]}, + "operations.0.add_column.args.name: Field required", + ), + ( + { + "operations": [ + {"op": "add_column", "args": {"name": "x", "type": "bad"}} + ] + }, + "operations.0.add_column.args.type: Input should be 'text', 'integer', 'float' or 'blob'", + ), + ( + { + "operations": [ + { + "op": "add_column", + "args": { + "name": "x", + "default_expr": "datetime('now')", + }, + } + ] + }, + "operations.0.add_column.args.default_expr: Input should be 'current_timestamp', 'current_date' or 'current_time'", + ), + ( + { + "operations": [ + { + "op": "add_column", + "args": { + "name": "x", + "default": "x", + "default_expr": "current_timestamp", + }, + } + ] + }, + "operations.0.add_column.args: Value error, default and default_expr cannot both be provided", + ), + ), +) +async def test_alter_table_validation_errors(ds_write, body, expected_error): + response = await ds_write.client.post( + "/data/docs/-/alter", + json=body, + headers=_headers(write_token(ds_write, permissions=["at"])), + ) + assert response.status_code == 400 + assert response.json()["ok"] is False + assert response.json()["errors"] == [expected_error] + + @pytest.mark.asyncio async def test_execute_write_form_parameter_called_sql(): ds = Datasette(memory=True, default_deny=True)