Add alter table JSON API

- Add POST /<database>/<table>/-/alter with Pydantic validation and dry-run support.
- Support add, rename, alter, drop, primary-key and reorder operations, including allow-listed default expressions.
- Document the endpoint and cover schema changes, validation, permissions, events and dry runs.

Refs #2788
This commit is contained in:
Simon Willison 2026-06-17 09:14:19 -07:00
commit b40665dd14
5 changed files with 667 additions and 0 deletions

View file

@ -84,6 +84,7 @@ from .views.special import (
)
from .views.table import (
TableAutocompleteView,
TableAlterView,
TableInsertView,
TableUpsertView,
TableSetColumnTypeView,
@ -2626,6 +2627,10 @@ class Datasette:
TableUpsertView.as_view(self),
r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/upsert$",
)
add_route(
TableAlterView.as_view(self),
r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/alter$",
)
add_route(
TableSetColumnTypeView.as_view(self),
r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/set-column-type$",

View file

@ -1,10 +1,12 @@
import asyncio
import itertools
import json
from typing import Annotated, Any, Literal, Union
import urllib
import urllib.parse
import markupsafe
from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator
from datasette.column_types import SQLiteType
from datasette.extras import extra_names_from_request
@ -46,6 +48,7 @@ from datasette.utils import (
from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response
from datasette.filters import Filters
import sqlite_utils
from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT
from .base import BaseView, DatasetteError, _error, stream_csv
from .database import QueryView
from .table_extras import (
@ -649,6 +652,154 @@ async def display_columns_and_rows(
return columns, cell_rows
SqliteApiType = Literal["text", "integer", "float", "blob"]
DefaultExpr = Literal["current_timestamp", "current_date", "current_time"]
DEFAULT_EXPR_SQL = {
"current_timestamp": "CURRENT_TIMESTAMP",
"current_date": "CURRENT_DATE",
"current_time": "CURRENT_TIME",
}
class _StrictPydanticModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class _DefaultArgsMixin(_StrictPydanticModel):
default: Any | None = None
default_expr: DefaultExpr | None = None
@model_validator(mode="after")
def validate_default_fields(self):
has_default = "default" in self.model_fields_set
has_default_expr = "default_expr" in self.model_fields_set
if has_default and has_default_expr:
raise ValueError("default and default_expr cannot both be provided")
if has_default_expr and self.default_expr is None:
raise ValueError("default_expr cannot be null")
return self
class AddColumnArgs(_DefaultArgsMixin):
name: str
type: SqliteApiType = "text"
not_null: bool = False
class RenameColumnArgs(_StrictPydanticModel):
name: str
to: str
class AlterColumnArgs(_DefaultArgsMixin):
name: str
type: SqliteApiType | None = None
not_null: bool | None = None
@model_validator(mode="after")
def require_change(self):
if not (
{"type", "not_null", "default", "default_expr"} & self.model_fields_set
):
raise ValueError(
"At least one of type, not_null, default or default_expr must be provided"
)
return self
class DropColumnArgs(_StrictPydanticModel):
name: str
class SetPrimaryKeyArgs(_StrictPydanticModel):
columns: list[str] = Field(min_length=1)
class ReorderColumnsArgs(_StrictPydanticModel):
columns: list[str] = Field(min_length=1)
class AddColumnOperation(_StrictPydanticModel):
op: Literal["add_column"]
args: AddColumnArgs
class RenameColumnOperation(_StrictPydanticModel):
op: Literal["rename_column"]
args: RenameColumnArgs
class AlterColumnOperation(_StrictPydanticModel):
op: Literal["alter_column"]
args: AlterColumnArgs
class DropColumnOperation(_StrictPydanticModel):
op: Literal["drop_column"]
args: DropColumnArgs
class SetPrimaryKeyOperation(_StrictPydanticModel):
op: Literal["set_primary_key"]
args: SetPrimaryKeyArgs
class ReorderColumnsOperation(_StrictPydanticModel):
op: Literal["reorder_columns"]
args: ReorderColumnsArgs
AlterTableOperation = Annotated[
Union[
AddColumnOperation,
RenameColumnOperation,
AlterColumnOperation,
DropColumnOperation,
SetPrimaryKeyOperation,
ReorderColumnsOperation,
],
Field(discriminator="op"),
]
class AlterTableRequest(_StrictPydanticModel):
operations: list[AlterTableOperation] = Field(min_length=1)
dry_run: bool = False
def _pydantic_errors(validation_error):
errors = []
for error in validation_error.errors():
location = ".".join(str(item) for item in error["loc"])
message = error["msg"]
errors.append("{}: {}".format(location, message) if location else message)
return errors
def _table_schema_from_conn(conn, table_name):
row = conn.execute(
"select sql from sqlite_master where type = 'table' and name = ?",
[table_name],
).fetchone()
return row[0] if row else None
def _primary_key_value(columns):
if len(columns) == 1:
return columns[0]
return tuple(columns)
def _default_expression_sql(default_expr):
return DEFAULT_EXPR_SQL[default_expr]
def _literal_default(db, value):
if isinstance(value, str):
return db.quote(value)
return value
class TableInsertView(BaseView):
name = "table-insert"
@ -946,6 +1097,208 @@ class TableUpsertView(TableInsertView):
return await super().post(request, upsert=True)
class TableAlterView(BaseView):
name = "table-alter"
def __init__(self, datasette):
self.ds = datasette
async def post(self, request):
try:
resolved = await self.ds.resolve_table(request)
except NotFound as e:
return _error([e.args[0]], 404)
db = resolved.db
database_name = db.name
table_name = resolved.table
if not await self.ds.allowed(
action="alter-table",
resource=TableResource(database=database_name, table=table_name),
actor=request.actor,
):
return _error(["Permission denied: need alter-table"], 403)
if not db.is_mutable:
return _error(["Database is immutable"], 403)
content_type = request.headers.get("content-type") or ""
if not content_type.startswith("application/json"):
return _error(["Invalid content-type, must be application/json"], 400)
try:
data = await request.json()
except json.JSONDecodeError as e:
return _error(["Invalid JSON: {}".format(e)], 400)
if not isinstance(data, dict):
return _error(["JSON must be a dictionary"], 400)
try:
alter_request = AlterTableRequest.model_validate(data)
except ValidationError as e:
return _error(_pydantic_errors(e), 400)
def alter_table(conn):
before_schema = _table_schema_from_conn(conn, table_name)
def apply_operations(operation_conn):
db_for_write = sqlite_utils.Database(operation_conn)
table = db_for_write[table_name]
add_columns = []
types = {}
rename = {}
drop = set()
not_null = {}
defaults = {}
column_order = None
pk = SQLITE_UTILS_DEFAULT
for operation in alter_request.operations:
args = operation.args
if operation.op == "add_column":
if args.not_null and not (
(
"default" in args.model_fields_set
and args.default is not None
)
or "default_expr" in args.model_fields_set
):
raise ValueError(
"add_column args.default or args.default_expr is required when not_null is true"
)
add_columns.append(args)
if "default" in args.model_fields_set and not args.not_null:
defaults[args.name] = _literal_default(
db_for_write, args.default
)
if "default_expr" in args.model_fields_set and not args.not_null:
defaults[args.name] = _default_expression_sql(
args.default_expr
)
elif operation.op == "rename_column":
rename[args.name] = args.to
elif operation.op == "alter_column":
if args.type is not None:
types[args.name] = args.type
if args.not_null is not None:
not_null[args.name] = args.not_null
if "default" in args.model_fields_set:
defaults[args.name] = (
None
if args.default is None
else _literal_default(db_for_write, args.default)
)
if "default_expr" in args.model_fields_set:
defaults[args.name] = _default_expression_sql(
args.default_expr
)
elif operation.op == "drop_column":
drop.add(args.name)
elif operation.op == "set_primary_key":
pk = _primary_key_value(args.columns)
elif operation.op == "reorder_columns":
column_order = args.columns
with operation_conn:
for column in add_columns:
not_null_default = None
if column.not_null:
if "default_expr" in column.model_fields_set:
not_null_default = _default_expression_sql(
column.default_expr
)
else:
not_null_default = _literal_default(
db_for_write, column.default
)
table.add_column(
column.name,
column.type,
not_null_default=not_null_default,
)
should_transform = any(
(
types,
rename,
drop,
not_null,
defaults,
column_order is not None,
pk is not SQLITE_UTILS_DEFAULT,
)
)
if should_transform:
table.transform(
types=types or None,
rename=rename or None,
drop=drop or None,
pk=pk,
not_null=not_null or None,
defaults=defaults or None,
column_order=column_order,
)
return _table_schema_from_conn(operation_conn, table_name)
if alter_request.dry_run:
memory_conn = sqlite3.connect(":memory:")
try:
conn.backup(memory_conn)
return before_schema, apply_operations(memory_conn)
finally:
memory_conn.close()
after_schema = apply_operations(conn)
return before_schema, after_schema
try:
before_schema, after_schema = await db.execute_write_fn(
alter_table, request=request
)
except Exception as e:
return _error([str(e)], 400)
altered = before_schema != after_schema
if altered and not alter_request.dry_run:
await self.ds.track_event(
AlterTableEvent(
request.actor,
database=database_name,
table=table_name,
before_schema=before_schema,
after_schema=after_schema,
)
)
table_url = self.ds.absolute_url(
request, self.ds.urls.table(database_name, table_name)
)
table_api_url = self.ds.absolute_url(
request, self.ds.urls.table(database_name, table_name, format="json")
)
return Response.json(
{
"ok": True,
"database": database_name,
"table": table_name,
"table_url": table_url,
"table_api_url": table_api_url,
"altered": altered,
"schema": after_schema,
"before_schema": before_schema,
"operations_applied": 0
if alter_request.dry_run
else len(alter_request.operations),
"dry_run": alter_request.dry_run,
},
status=200,
)
class TableSetColumnTypeView(BaseView):
name = "table-set-column-type"

View file

@ -2072,6 +2072,109 @@ To use the ``"replace": true`` option you will also need the :ref:`actions_updat
Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`actions_alter_table` permission.
.. _TableAlterView:
Altering tables
~~~~~~~~~~~~~~~
To alter an existing table, make a ``POST`` to ``/<database>/<table>/-/alter``. This requires the :ref:`actions_alter_table` permission.
::
POST /<database>/<table>/-/alter
Content-Type: application/json
Authorization: Bearer dstok_<rest-of-token>
The request body should include an ``operations`` array. Each operation has the same top-level shape: an ``op`` string and an ``args`` object.
.. code-block:: json
{
"operations": [
{
"op": "add_column",
"args": {
"name": "slug",
"type": "text",
"not_null": true,
"default": ""
}
},
{
"op": "add_column",
"args": {
"name": "created",
"type": "text",
"default_expr": "current_timestamp"
}
},
{
"op": "rename_column",
"args": {
"name": "title",
"to": "headline"
}
},
{
"op": "alter_column",
"args": {
"name": "score",
"type": "float"
}
},
{
"op": "drop_column",
"args": {
"name": "draft_notes"
}
},
{
"op": "set_primary_key",
"args": {
"columns": ["id"]
}
},
{
"op": "reorder_columns",
"args": {
"columns": ["id", "headline", "slug", "created", "score"]
}
}
]
}
Set ``"dry_run": true`` to validate the operations and return the schema that would be created without modifying the table.
Supported operations:
* ``add_column`` adds a new column. ``args`` accepts ``name``, optional ``type`` of ``text``, ``integer``, ``float`` or ``blob``, optional ``not_null``, optional literal ``default`` and optional ``default_expr``. If ``not_null`` is ``true`` either a non-null ``default`` or ``default_expr`` is required.
* ``rename_column`` renames a column. ``args`` accepts ``name`` and ``to``.
* ``alter_column`` changes column properties. ``args`` accepts ``name`` and at least one of ``type``, ``not_null``, literal ``default`` or ``default_expr``. Passing ``"default": null`` removes an existing default.
* ``drop_column`` drops a column. ``args`` accepts ``name``.
* ``set_primary_key`` changes the table primary key. ``args`` accepts ``columns``, a list of one or more column names.
* ``reorder_columns`` reorders columns. ``args`` accepts ``columns``, a list of one or more column names. Columns omitted from this list will appear afterwards in their existing order.
``default`` is always treated as a literal value. ``default_expr`` accepts one of ``current_timestamp``, ``current_date`` or ``current_time`` and is rendered as the corresponding SQLite default expression.
A successful response returns the new schema and the previous schema:
.. code-block:: json
{
"ok": true,
"database": "data",
"table": "posts",
"table_url": "http://127.0.0.1:8001/data/posts",
"table_api_url": "http://127.0.0.1:8001/data/posts.json",
"altered": true,
"schema": "CREATE TABLE ...",
"before_schema": "CREATE TABLE ...",
"operations_applied": 7,
"dry_run": false
}
Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error.
.. _TableSetColumnTypeView:
Setting a column type

View file

@ -39,6 +39,7 @@ dependencies = [
"asyncinject>=0.7",
"setuptools",
"pip",
"pydantic>=2",
]
[project.urls]

View file

@ -794,6 +794,211 @@ async def test_update_row_alter(ds_write):
assert response.json() == {"ok": True}
@pytest.mark.asyncio
async def test_alter_table_operations(ds_write):
token = write_token(ds_write, permissions=["at"])
db = ds_write.get_database("data")
before_schema = await db.execute_fn(
lambda conn: conn.execute(
"select sql from sqlite_master where type = 'table' and name = 'docs'"
).fetchone()[0]
)
response = await ds_write.client.post(
"/data/docs/-/alter",
json={
"operations": [
{
"op": "add_column",
"args": {
"name": "slug",
"type": "text",
"not_null": True,
"default": "",
},
},
{
"op": "add_column",
"args": {
"name": "created",
"type": "text",
"default_expr": "current_timestamp",
},
},
{
"op": "add_column",
"args": {
"name": "literal_default",
"type": "text",
"default": "hello)",
},
},
{"op": "rename_column", "args": {"name": "title", "to": "headline"}},
{
"op": "alter_column",
"args": {"name": "age", "type": "text", "default": "0"},
},
{"op": "drop_column", "args": {"name": "score"}},
{
"op": "reorder_columns",
"args": {
"columns": [
"id",
"headline",
"slug",
"created",
"literal_default",
"age",
]
},
},
{"op": "set_primary_key", "args": {"columns": ["id"]}},
]
},
headers=_headers(token),
)
assert response.status_code == 200, response.text
data = response.json()
assert data["ok"] is True
assert data["database"] == "data"
assert data["table"] == "docs"
assert data["altered"] is True
assert data["operations_applied"] == 8
assert data["before_schema"] == before_schema
assert "headline" in data["schema"]
assert "score" not in data["schema"]
assert "DEFAULT CURRENT_TIMESTAMP" in data["schema"]
assert "DEFAULT 'hello)'" in data["schema"]
columns = (
await db.execute("select * from pragma_table_info('docs') order by cid")
).dicts()
assert [column["name"] for column in columns] == [
"id",
"headline",
"slug",
"created",
"literal_default",
"age",
]
assert columns[0]["pk"] == 1
assert columns[2]["notnull"] == 1
assert columns[2]["dflt_value"] == "''"
assert columns[3]["dflt_value"] == "CURRENT_TIMESTAMP"
assert columns[4]["dflt_value"] == "'hello)'"
assert columns[5]["type"] == "TEXT"
assert columns[5]["dflt_value"] == "'0'"
event = last_event(ds_write)
assert event.name == "alter-table"
assert event.database == "data"
assert event.table == "docs"
assert event.before_schema == before_schema
assert event.after_schema == data["schema"]
@pytest.mark.asyncio
async def test_alter_table_dry_run(ds_write):
token = write_token(ds_write, permissions=["at"])
db = ds_write.get_database("data")
response = await ds_write.client.post(
"/data/docs/-/alter",
json={
"dry_run": True,
"operations": [
{"op": "add_column", "args": {"name": "slug", "type": "text"}}
],
},
headers=_headers(token),
)
assert response.status_code == 200, response.text
data = response.json()
assert data["ok"] is True
assert data["dry_run"] is True
assert data["altered"] is True
assert data["operations_applied"] == 0
assert "slug" in data["schema"]
columns = (
await db.execute("select name from pragma_table_info('docs') order by cid")
).dicts()
assert [column["name"] for column in columns] == ["id", "title", "score", "age"]
assert last_event(ds_write) is None
@pytest.mark.asyncio
async def test_alter_table_permission_denied(ds_write):
token = write_token(ds_write, permissions=["ir"])
response = await ds_write.client.post(
"/data/docs/-/alter",
json={"operations": [{"op": "add_column", "args": {"name": "slug"}}]},
headers=_headers(token),
)
assert response.status_code == 403
assert response.json() == {
"ok": False,
"errors": ["Permission denied: need alter-table"],
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"body,expected_error",
(
(
{"operations": [{"op": "add_column", "args": {"type": "text"}}]},
"operations.0.add_column.args.name: Field required",
),
(
{
"operations": [
{"op": "add_column", "args": {"name": "x", "type": "bad"}}
]
},
"operations.0.add_column.args.type: Input should be 'text', 'integer', 'float' or 'blob'",
),
(
{
"operations": [
{
"op": "add_column",
"args": {
"name": "x",
"default_expr": "datetime('now')",
},
}
]
},
"operations.0.add_column.args.default_expr: Input should be 'current_timestamp', 'current_date' or 'current_time'",
),
(
{
"operations": [
{
"op": "add_column",
"args": {
"name": "x",
"default": "x",
"default_expr": "current_timestamp",
},
}
]
},
"operations.0.add_column.args: Value error, default and default_expr cannot both be provided",
),
),
)
async def test_alter_table_validation_errors(ds_write, body, expected_error):
response = await ds_write.client.post(
"/data/docs/-/alter",
json=body,
headers=_headers(write_token(ds_write, permissions=["at"])),
)
assert response.status_code == 400
assert response.json()["ok"] is False
assert response.json()["errors"] == [expected_error]
@pytest.mark.asyncio
async def test_execute_write_form_parameter_called_sql():
ds = Datasette(memory=True, default_deny=True)