Fixed some broken docs/ references

Refs https://github.com/simonw/datasette/pull/2674#issuecomment-4085015792

datasette/app.py

@@ -68,6 +68,7 @@ from .views.special import (
 from .views.table import (
     TableInsertView,
     TableUpsertView,
+    TableSetColumnTypeView,
     TableDropView,
     table_view,
 )
@@ -2240,6 +2241,10 @@ class Datasette:
             TableUpsertView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/upsert$",
         )
+        add_route(
+            TableSetColumnTypeView.as_view(self),
+            r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/set-column-type$",
+        )
         add_route(
             TableDropView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/drop$",

datasette/default_actions.py

@@ -85,6 +85,12 @@ def register_actions():
             description="Alter tables",
             resource_class=TableResource,
         ),
+        Action(
+            name="set-column-type",
+            abbr="sct",
+            description="Set column type",
+            resource_class=TableResource,
+        ),
         Action(
             name="drop-table",
             abbr="dt",

datasette/views/table.py

@@ -666,6 +666,122 @@ class TableUpsertView(TableInsertView):
         return await super().post(request, upsert=True)
 
 
+class TableSetColumnTypeView(BaseView):
+    name = "table-set-column-type"
+
+    def __init__(self, datasette):
+        self.ds = datasette
+
+    async def post(self, request):
+        try:
+            resolved = await self.ds.resolve_table(request)
+        except NotFound as e:
+            return _error([e.args[0]], 404)
+
+        database_name = resolved.db.name
+        table_name = resolved.table
+
+        if not await self.ds.allowed(
+            action="set-column-type",
+            resource=TableResource(database=database_name, table=table_name),
+            actor=request.actor,
+        ):
+            return _error(["Permission denied"], 403)
+
+        content_type = request.headers.get("content-type") or ""
+        if not content_type.startswith("application/json"):
+            return _error(["Invalid content-type, must be application/json"], 400)
+
+        try:
+            data = json.loads(await request.post_body())
+        except json.JSONDecodeError as e:
+            return _error(["Invalid JSON: {}".format(e)], 400)
+
+        if not isinstance(data, dict):
+            return _error(["JSON must be a dictionary"], 400)
+
+        invalid_keys = set(data.keys()) - {"column", "column_type"}
+        if invalid_keys:
+            return _error(
+                ['Invalid parameter: "{}"'.format('", "'.join(sorted(invalid_keys)))],
+                400,
+            )
+
+        if "column" not in data:
+            return _error(['"column" is required'], 400)
+        column = data["column"]
+        if not isinstance(column, str):
+            return _error(['"column" must be a string'], 400)
+
+        if "column_type" not in data:
+            return _error(['"column_type" is required'], 400)
+
+        column_details = await self.ds._get_resource_column_details(
+            database_name, table_name
+        )
+        if column not in column_details:
+            return _error(["Column not found: {}".format(column)], 400)
+
+        column_type_data = data["column_type"]
+        if column_type_data is None:
+            await self.ds.remove_column_type(database_name, table_name, column)
+            return Response.json(
+                {
+                    "ok": True,
+                    "database": database_name,
+                    "table": table_name,
+                    "column": column,
+                    "column_type": None,
+                },
+                status=200,
+            )
+
+        if not isinstance(column_type_data, dict):
+            return _error(['"column_type" must be an object or null'], 400)
+
+        invalid_column_type_keys = set(column_type_data.keys()) - {"type", "config"}
+        if invalid_column_type_keys:
+            return _error(
+                [
+                    'Invalid column_type parameter: "{}"'.format(
+                        '", "'.join(sorted(invalid_column_type_keys))
+                    )
+                ],
+                400,
+            )
+
+        if "type" not in column_type_data:
+            return _error(['"column_type.type" is required'], 400)
+        column_type = column_type_data["type"]
+        if not isinstance(column_type, str):
+            return _error(['"column_type.type" must be a string'], 400)
+
+        config = column_type_data.get("config")
+        if config is not None and not isinstance(config, dict):
+            return _error(['"column_type.config" must be a dictionary'], 400)
+
+        if column_type not in self.ds._column_types:
+            return _error(["Unknown column type: {}".format(column_type)], 400)
+
+        try:
+            await self.ds.set_column_type(
+                database_name, table_name, column, column_type, config
+            )
+        except ValueError as e:
+            return _error([str(e)], 400)
+
+        return Response.json(
+            {
+                "ok": True,
+                "database": database_name,
+                "table": table_name,
+                "column": column,
+                "column_type": {"type": column_type, "config": config},
+            },
+            status=200,
+        )
+
+
 class TableDropView(BaseView):
     name = "table-drop"
 

docs/authentication.rst

@@ -33,7 +33,7 @@ The one exception is the "root" account, which you can sign into while using Dat
 The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including:
 
 * All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.)
-* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``drop-table``)
+* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-type``, ``drop-table``)
 * Debug permissions (``permissions-debug``, ``debug-menu``)
 * Any custom permissions defined by plugins
 
@@ -886,6 +886,8 @@ To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs``
         }
 .. [[[end]]]
 
+Other table-scoped write permissions, including ``set-column-type``, can be configured in the same place.
+
 And for ``insert-row`` against the ``reports`` table in that ``docs`` database:
 
 .. [[[cog
@@ -1343,6 +1345,18 @@ alter-table
 
 Actor is allowed to alter a database table.
 
+``resource`` - ``datasette.resources.TableResource(database, table)``
+    ``database`` is the name of the database (string)
+
+    ``table`` is the name of the table (string)
+
+.. _actions_set_column_type:
+
+set-column-type
+---------------
+
+Actor is allowed to set assigned :ref:`column types <table_configuration_column_types>` for columns in a table.
+
 ``resource`` - ``datasette.resources.TableResource(database, table)``
     ``database`` is the name of the database (string)
 

docs/changelog.rst

@@ -934,7 +934,7 @@ Other small fixes
 0.59 (2021-10-14)
 -----------------
 
-- Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`table_configuration_columns`. (:issue:`942`)
+- Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`metadata_column_descriptions`. (:issue:`942`)
 - New :ref:`register_commands() <plugin_hook_register_commands>` plugin hook allows plugins to register additional Datasette CLI commands, e.g. ``datasette mycommand file.db``. (:issue:`1449`)
 - Adding ``?_facet_size=max`` to a table page now shows the number of unique values in each facet. (:issue:`1423`)
 - Upgraded dependency `httpx 0.20 <https://github.com/encode/httpx/releases/tag/0.20.0>`__ - the undocumented ``allow_redirects=`` parameter to :ref:`internals_datasette_client` is now ``follow_redirects=``, and defaults to ``False`` where it previously defaulted to ``True``. (:issue:`1488`)

docs/internals.rst

@@ -922,7 +922,7 @@ await .get_column_type(database, resource, column)
 ``column`` - string
     The name of the column.
 
-Returns a :ref:`ColumnType <column_types>` subclass instance with ``.config`` populated for the specified column, or ``None`` if no column type is assigned.
+Returns a ``ColumnType`` subclass instance with ``.config`` populated for the specified column, or ``None`` if no column type is assigned.
 
 .. code-block:: python
 
@@ -943,7 +943,7 @@ await .get_column_types(database, resource)
 ``resource`` - string
     The name of the table or view.
 
-Returns a dictionary mapping column names to :ref:`ColumnType <column_types>` subclass instances (with ``.config`` populated) for all columns that have assigned types on the given resource.
+Returns a dictionary mapping column names to ``ColumnType`` subclass instances (with ``.config`` populated) for all columns that have assigned types on the given resource.
 
 .. code-block:: python
 

docs/json_api.rst

@@ -941,6 +941,70 @@ To use the ``"replace": true`` option you will also need the :ref:`actions_updat
 
 Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`actions_alter_table` permission.
 
+.. _TableSetColumnTypeView:
+
+Setting a column type
+~~~~~~~~~~~~~~~~~~~~~
+
+To set a column type for a table column, make a ``POST`` to ``/<database>/<table>/-/set-column-type``. This requires the :ref:`actions_set_column_type` permission.
+
+::
+
+    POST /<database>/<table>/-/set-column-type
+    Content-Type: application/json
+    Authorization: Bearer dstok_<rest-of-token>
+
+.. code-block:: json
+
+    {
+        "column": "title",
+        "column_type": {
+            "type": "email"
+        }
+    }
+
+This will return a ``200`` response like this:
+
+.. code-block:: json
+
+    {
+        "ok": true,
+        "database": "data",
+        "table": "posts",
+        "column": "title",
+        "column_type": {
+            "type": "email",
+            "config": null
+        }
+    }
+
+To provide column type configuration, include a ``config`` object:
+
+.. code-block:: json
+
+    {
+        "column": "title",
+        "column_type": {
+            "type": "url",
+            "config": {
+                "max_length": 200
+            }
+        }
+    }
+
+To clear an existing column type assignment, set ``column_type`` to ``null``:
+
+.. code-block:: json
+
+    {
+        "column": "title",
+        "column_type": null
+    }
+
+This API stores the assignment in Datasette's internal database, so it can be used with immutable databases as well as mutable ones.
+
+Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error.
+
 .. _TableDropView:
 
 Dropping tables

docs/plugin_hooks.rst

@@ -503,10 +503,10 @@ Lets you customize the display of values within table cells in the HTML table vi
 ``request`` - :ref:`internals_request`
     The current request object
 
-``column_type`` - :ref:`ColumnType <column_types>` subclass instance or None
-    The :ref:`ColumnType <column_types>` subclass instance assigned to this column (with ``.config`` populated), or ``None`` if no column type is assigned. You can access ``column_type.name``, ``column_type.config``, etc.
+``column_type`` - :ref:`ColumnType <datasette_column_types>` subclass instance or None
+    The :ref:`ColumnType <datasette_column_types>` subclass instance assigned to this column (with ``.config`` populated), or ``None`` if no column type is assigned. You can access ``column_type.name``, ``column_type.config``, etc.
 
-If a column has a :ref:`column type <column_types>` assigned and that column type's ``render_cell`` method returns a non-``None`` value, it will take priority over this plugin hook.
+If a column has a :ref:`column type <datasette_column_types>` assigned and that column type's ``render_cell`` method returns a non-``None`` value, it will take priority over this plugin hook.
 
 If your hook returns ``None``, it will be ignored. Use this to indicate that your hook is not able to custom render this particular value.
 
@@ -999,7 +999,7 @@ The permission system then uses this query along with rules from plugins to dete
 register_column_types(datasette)
 --------------------------------
 
-Return a list of :ref:`ColumnType <column_types>` **subclasses** (not instances) to register custom column types. Column types define how values in specific columns are rendered, validated, and transformed.
+Return a list of :ref:`ColumnType <datasette_column_types>` **subclasses** (not instances) to register custom column types. Column types define how values in specific columns are rendered, validated, and transformed.
 
 .. code-block:: python
 
@@ -1069,7 +1069,7 @@ And the following methods, all optional:
 
 Per-column configuration is available via ``self.config`` in all methods. When a column type is looked up for a specific column (via :ref:`get_column_type <datasette_get_column_type>` or :ref:`get_column_types <datasette_get_column_types>`), the returned instance has ``config`` set to the parsed JSON config dict for that column assignment, or ``None`` if no config was provided.
 
-Column types are assigned to columns via the ``column_types`` key in :ref:`table configuration <metadata_tables>`:
+Column types are assigned to columns via the :ref:`column_types <table_configuration_column_types>` table configuration option:
 
 .. code-block:: yaml
 

tests/test_auth.py

@@ -191,6 +191,7 @@ def test_auth_create_token(
             "all:view-query",
             "database:fixtures:drop-table",
             "resource:fixtures:foreign_key_references:insert-row",
+            "resource:fixtures:facetable:set-column-type",
         }
     )
     # Now try actually creating one
@@ -427,6 +428,15 @@ async def test_root_with_root_enabled_gets_all_permissions(ds_client):
         is True
     )
 
+    assert (
+        await ds_client.ds.allowed(
+            action="set-column-type",
+            resource=TableResource("fixtures", "facetable"),
+            actor=root_actor,
+        )
+        is True
+    )
+
     assert (
         await ds_client.ds.allowed(
             action="drop-table",
@@ -491,3 +501,12 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client):
         )
         is not True
     ), "Root without root_enabled should not automatically get drop-table"
+
+    assert (
+        await ds_client.ds.allowed(
+            action="set-column-type",
+            resource=TableResource("fixtures", "facetable"),
+            actor=root_actor,
+        )
+        is not True
+    ), "Root without root_enabled should not automatically get set-column-type"

tests/test_column_types.py

@@ -186,6 +186,226 @@ async def test_set_column_type_with_config(ds_ct):
     assert ct.config == {"max_length": 200}
 
 
+@pytest.mark.asyncio
+async def test_set_column_type_api(ds_ct):
+    await ds_ct.invoke_startup()
+    token = write_token(ds_ct, permissions=["sct"])
+    response = await ds_ct.client.post(
+        "/data/posts/-/set-column-type",
+        json={"column": "title", "column_type": {"type": "email"}},
+        headers=_headers(token),
+    )
+    assert response.status_code == 200
+    assert response.json() == {
+        "ok": True,
+        "database": "data",
+        "table": "posts",
+        "column": "title",
+        "column_type": {"type": "email", "config": None},
+    }
+    ct = await ds_ct.get_column_type("data", "posts", "title")
+    assert ct.name == "email"
+    assert ct.config is None
+
+
+@pytest.mark.asyncio
+async def test_set_column_type_api_with_config(ds_ct):
+    await ds_ct.invoke_startup()
+    token = write_token(ds_ct, permissions=["sct"])
+    response = await ds_ct.client.post(
+        "/data/posts/-/set-column-type",
+        json={
+            "column": "title",
+            "column_type": {"type": "url", "config": {"max_length": 200}},
+        },
+        headers=_headers(token),
+    )
+    assert response.status_code == 200
+    assert response.json()["column_type"] == {
+        "type": "url",
+        "config": {"max_length": 200},
+    }
+    ct = await ds_ct.get_column_type("data", "posts", "title")
+    assert ct.name == "url"
+    assert ct.config == {"max_length": 200}
+
+
+@pytest.mark.asyncio
+async def test_clear_column_type_api(ds_ct):
+    await ds_ct.invoke_startup()
+    await ds_ct.set_column_type("data", "posts", "title", "email")
+    token = write_token(ds_ct, permissions=["sct"])
+    response = await ds_ct.client.post(
+        "/data/posts/-/set-column-type",
+        json={"column": "title", "column_type": None},
+        headers=_headers(token),
+    )
+    assert response.status_code == 200
+    assert response.json() == {
+        "ok": True,
+        "database": "data",
+        "table": "posts",
+        "column": "title",
+        "column_type": None,
+    }
+    ct = await ds_ct.get_column_type("data", "posts", "title")
+    assert ct is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "body,special_case,expected_status,expected_errors",
+    (
+        (
+            {"column": "title", "column_type": {"type": "email"}},
+            "no_permission",
+            403,
+            ["Permission denied"],
+        ),
+        (
+            None,
+            "invalid_json",
+            400,
+            [
+                "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)"
+            ],
+        ),
+        (
+            {"column": "title", "column_type": {"type": "email"}},
+            "invalid_content_type",
+            400,
+            ["Invalid content-type, must be application/json"],
+        ),
+        (
+            [],
+            None,
+            400,
+            ["JSON must be a dictionary"],
+        ),
+        (
+            {"column_type": {"type": "email"}},
+            None,
+            400,
+            ['"column" is required'],
+        ),
+        (
+            {"column": 1, "column_type": {"type": "email"}},
+            None,
+            400,
+            ['"column" must be a string'],
+        ),
+        (
+            {"column": "not_a_column", "column_type": {"type": "email"}},
+            None,
+            400,
+            ["Column not found: not_a_column"],
+        ),
+        (
+            {"column": "title", "column_type": "email"},
+            None,
+            400,
+            ['"column_type" must be an object or null'],
+        ),
+        (
+            {"column": "title", "column_type": {}},
+            None,
+            400,
+            ['"column_type.type" is required'],
+        ),
+        (
+            {"column": "title", "column_type": {"type": 1}},
+            None,
+            400,
+            ['"column_type.type" must be a string'],
+        ),
+        (
+            {"column": "title", "column_type": {"type": "url", "config": []}},
+            None,
+            400,
+            ['"column_type.config" must be a dictionary'],
+        ),
+        (
+            {"column": "title", "column_type": {"type": "markdown"}},
+            None,
+            400,
+            ["Unknown column type: markdown"],
+        ),
+        (
+            {"column": "id", "column_type": {"type": "json"}},
+            None,
+            400,
+            [
+                "Column type 'json' is only applicable to SQLite types TEXT but data.posts.id has SQLite type INTEGER"
+            ],
+        ),
+        (
+            {
+                "column": "title",
+                "column_type": {"type": "email"},
+                "extra": True,
+            },
+            None,
+            400,
+            ['Invalid parameter: "extra"'],
+        ),
+    ),
+)
+async def test_set_column_type_api_errors(
+    ds_ct, body, special_case, expected_status, expected_errors
+):
+    await ds_ct.invoke_startup()
+    token = write_token(
+        ds_ct,
+        permissions=(["sct"] if special_case != "no_permission" else ["vi"]),
+    )
+    kwargs = {
+        "headers": {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": (
+                "text/plain"
+                if special_case == "invalid_content_type"
+                else "application/json"
+            ),
+        }
+    }
+    if special_case == "invalid_json":
+        kwargs["content"] = "{bad json"
+    else:
+        kwargs["json"] = body
+    response = await ds_ct.client.post("/data/posts/-/set-column-type", **kwargs)
+    assert response.status_code == expected_status
+    assert response.json() == {"ok": False, "errors": expected_errors}
+
+
+@pytest.mark.asyncio
+async def test_set_column_type_api_works_for_immutable_database(tmp_path_factory):
+    db_directory = tmp_path_factory.mktemp("dbs")
+    db_path = str(db_directory / "immutable.db")
+    db = sqlite3.connect(str(db_path))
+    db.execute("vacuum")
+    db.execute("create table posts (id integer primary key, title text)")
+    db.commit()
+    ds = Datasette([], immutables=[db_path])
+    ds.root_enabled = True
+    try:
+        await ds.invoke_startup()
+        token = write_token(ds, permissions=["sct"])
+        response = await ds.client.post(
+            "/immutable/posts/-/set-column-type",
+            json={"column": "title", "column_type": {"type": "email"}},
+            headers=_headers(token),
+        )
+        assert response.status_code == 200
+        assert response.json()["column_type"] == {"type": "email", "config": None}
+        ct = await ds.get_column_type("immutable", "posts", "title")
+        assert ct.name == "email"
+    finally:
+        db.close()
+        for database in ds.databases.values():
+            if not database.is_memory:
+                database.close()
+
+
 @pytest.mark.asyncio
 async def test_set_column_type_rejects_incompatible_sqlite_type(ds_ct):
     await ds_ct.invoke_startup()

tests/test_internals_datasette.py

@@ -164,7 +164,14 @@ def test_datasette_error_if_string_not_list(tmpdir):
 @pytest.mark.asyncio
 async def test_get_action(ds_client):
     ds = ds_client.ds
-    for name_or_abbr in ("vi", "view-instance", "vt", "view-table"):
+    for name_or_abbr in (
+        "vi",
+        "view-instance",
+        "vt",
+        "view-table",
+        "sct",
+        "set-column-type",
+    ):
         action = ds.get_action(name_or_abbr)
         if "-" in name_or_abbr:
             assert action.name == name_or_abbr

tests/test_permissions.py

@@ -831,6 +831,22 @@ PermConfigTestCase = collections.namedtuple(
             resource=("perms_ds_one", "t1"),
             expected_result=True,
         ),
+        # set-column-type on specific table
+        PermConfigTestCase(
+            config={
+                "databases": {
+                    "perms_ds_one": {
+                        "tables": {
+                            "t1": {"permissions": {"set-column-type": {"id": "user"}}}
+                        }
+                    }
+                }
+            },
+            actor={"id": "user"},
+            action="set-column-type",
+            resource=("perms_ds_one", "t1"),
+            expected_result=True,
+        ),
         # insert-row on database
         PermConfigTestCase(
             config={