diff --git a/datasette/app.py b/datasette/app.py index 6e3e6815..4c98e521 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -68,6 +68,7 @@ from .views.special import ( from .views.table import ( TableInsertView, TableUpsertView, + TableSetColumnTypeView, TableDropView, table_view, ) @@ -2240,6 +2241,10 @@ class Datasette: TableUpsertView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/upsert$", ) + add_route( + TableSetColumnTypeView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/set-column-type$", + ) add_route( TableDropView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/drop$", diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 87d98fac..149a4e5f 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -85,6 +85,12 @@ def register_actions(): description="Alter tables", resource_class=TableResource, ), + Action( + name="set-column-type", + abbr="sct", + description="Set column type", + resource_class=TableResource, + ), Action( name="drop-table", abbr="dt", diff --git a/datasette/views/table.py b/datasette/views/table.py index cf5b5b64..e7a226af 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -666,6 +666,122 @@ class TableUpsertView(TableInsertView): return await super().post(request, upsert=True) +class TableSetColumnTypeView(BaseView): + name = "table-set-column-type" + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + try: + resolved = await self.ds.resolve_table(request) + except NotFound as e: + return _error([e.args[0]], 404) + + database_name = resolved.db.name + table_name = resolved.table + + if not await self.ds.allowed( + action="set-column-type", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return _error(["Permission denied"], 403) + + content_type = request.headers.get("content-type") or "" + if not content_type.startswith("application/json"): + return _error(["Invalid content-type, must be application/json"], 400) + + try: + data = json.loads(await request.post_body()) + except json.JSONDecodeError as e: + return _error(["Invalid JSON: {}".format(e)], 400) + + if not isinstance(data, dict): + return _error(["JSON must be a dictionary"], 400) + + invalid_keys = set(data.keys()) - {"column", "column_type"} + if invalid_keys: + return _error( + ['Invalid parameter: "{}"'.format('", "'.join(sorted(invalid_keys)))], + 400, + ) + + if "column" not in data: + return _error(['"column" is required'], 400) + column = data["column"] + if not isinstance(column, str): + return _error(['"column" must be a string'], 400) + + if "column_type" not in data: + return _error(['"column_type" is required'], 400) + + column_details = await self.ds._get_resource_column_details( + database_name, table_name + ) + if column not in column_details: + return _error(["Column not found: {}".format(column)], 400) + + column_type_data = data["column_type"] + if column_type_data is None: + await self.ds.remove_column_type(database_name, table_name, column) + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "column": column, + "column_type": None, + }, + status=200, + ) + + if not isinstance(column_type_data, dict): + return _error(['"column_type" must be an object or null'], 400) + + invalid_column_type_keys = set(column_type_data.keys()) - {"type", "config"} + if invalid_column_type_keys: + return _error( + [ + 'Invalid column_type parameter: "{}"'.format( + '", "'.join(sorted(invalid_column_type_keys)) + ) + ], + 400, + ) + + if "type" not in column_type_data: + return _error(['"column_type.type" is required'], 400) + column_type = column_type_data["type"] + if not isinstance(column_type, str): + return _error(['"column_type.type" must be a string'], 400) + + config = column_type_data.get("config") + if config is not None and not isinstance(config, dict): + return _error(['"column_type.config" must be a dictionary'], 400) + + if column_type not in self.ds._column_types: + return _error(["Unknown column type: {}".format(column_type)], 400) + + try: + await self.ds.set_column_type( + database_name, table_name, column, column_type, config + ) + except ValueError as e: + return _error([str(e)], 400) + + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "column": column, + "column_type": {"type": column_type, "config": config}, + }, + status=200, + ) + + class TableDropView(BaseView): name = "table-drop" diff --git a/docs/authentication.rst b/docs/authentication.rst index 1b949f9a..7fa3a241 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -33,7 +33,7 @@ The one exception is the "root" account, which you can sign into while using Dat The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including: * All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.) -* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``drop-table``) +* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-type``, ``drop-table``) * Debug permissions (``permissions-debug``, ``debug-menu``) * Any custom permissions defined by plugins @@ -886,6 +886,8 @@ To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` } .. [[[end]]] +Other table-scoped write permissions, including ``set-column-type``, can be configured in the same place. + And for ``insert-row`` against the ``reports`` table in that ``docs`` database: .. [[[cog @@ -1343,6 +1345,18 @@ alter-table Actor is allowed to alter a database table. +``resource`` - ``datasette.resources.TableResource(database, table)`` + ``database`` is the name of the database (string) + + ``table`` is the name of the table (string) + +.. _actions_set_column_type: + +set-column-type +--------------- + +Actor is allowed to set assigned :ref:`column types ` for columns in a table. + ``resource`` - ``datasette.resources.TableResource(database, table)`` ``database`` is the name of the database (string) diff --git a/docs/changelog.rst b/docs/changelog.rst index 71d63e33..3d7d8405 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -934,7 +934,7 @@ Other small fixes 0.59 (2021-10-14) ----------------- -- Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`table_configuration_columns`. (:issue:`942`) +- Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`metadata_column_descriptions`. (:issue:`942`) - New :ref:`register_commands() ` plugin hook allows plugins to register additional Datasette CLI commands, e.g. ``datasette mycommand file.db``. (:issue:`1449`) - Adding ``?_facet_size=max`` to a table page now shows the number of unique values in each facet. (:issue:`1423`) - Upgraded dependency `httpx 0.20 `__ - the undocumented ``allow_redirects=`` parameter to :ref:`internals_datasette_client` is now ``follow_redirects=``, and defaults to ``False`` where it previously defaulted to ``True``. (:issue:`1488`) diff --git a/docs/internals.rst b/docs/internals.rst index 2442e687..704643cc 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -922,7 +922,7 @@ await .get_column_type(database, resource, column) ``column`` - string The name of the column. -Returns a :ref:`ColumnType ` subclass instance with ``.config`` populated for the specified column, or ``None`` if no column type is assigned. +Returns a ``ColumnType`` subclass instance with ``.config`` populated for the specified column, or ``None`` if no column type is assigned. .. code-block:: python @@ -943,7 +943,7 @@ await .get_column_types(database, resource) ``resource`` - string The name of the table or view. -Returns a dictionary mapping column names to :ref:`ColumnType ` subclass instances (with ``.config`` populated) for all columns that have assigned types on the given resource. +Returns a dictionary mapping column names to ``ColumnType`` subclass instances (with ``.config`` populated) for all columns that have assigned types on the given resource. .. code-block:: python diff --git a/docs/json_api.rst b/docs/json_api.rst index 891aa9e0..48c70af6 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -941,6 +941,70 @@ To use the ``"replace": true`` option you will also need the :ref:`actions_updat Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`actions_alter_table` permission. +.. _TableSetColumnTypeView: + +Setting a column type +~~~~~~~~~~~~~~~~~~~~~ + +To set a column type for a table column, make a ``POST`` to ``//
/-/set-column-type``. This requires the :ref:`actions_set_column_type` permission. + +:: + + POST //
/-/set-column-type + Content-Type: application/json + Authorization: Bearer dstok_ + +.. code-block:: json + + { + "column": "title", + "column_type": { + "type": "email" + } + } + +This will return a ``200`` response like this: + +.. code-block:: json + + { + "ok": true, + "database": "data", + "table": "posts", + "column": "title", + "column_type": { + "type": "email", + "config": null + } + } + +To provide column type configuration, include a ``config`` object: + +.. code-block:: json + + { + "column": "title", + "column_type": { + "type": "url", + "config": { + "max_length": 200 + } + } + } + +To clear an existing column type assignment, set ``column_type`` to ``null``: + +.. code-block:: json + + { + "column": "title", + "column_type": null + } + +This API stores the assignment in Datasette's internal database, so it can be used with immutable databases as well as mutable ones. + +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. + .. _TableDropView: Dropping tables diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 53a47334..e375707d 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -503,10 +503,10 @@ Lets you customize the display of values within table cells in the HTML table vi ``request`` - :ref:`internals_request` The current request object -``column_type`` - :ref:`ColumnType ` subclass instance or None - The :ref:`ColumnType ` subclass instance assigned to this column (with ``.config`` populated), or ``None`` if no column type is assigned. You can access ``column_type.name``, ``column_type.config``, etc. +``column_type`` - :ref:`ColumnType ` subclass instance or None + The :ref:`ColumnType ` subclass instance assigned to this column (with ``.config`` populated), or ``None`` if no column type is assigned. You can access ``column_type.name``, ``column_type.config``, etc. -If a column has a :ref:`column type ` assigned and that column type's ``render_cell`` method returns a non-``None`` value, it will take priority over this plugin hook. +If a column has a :ref:`column type ` assigned and that column type's ``render_cell`` method returns a non-``None`` value, it will take priority over this plugin hook. If your hook returns ``None``, it will be ignored. Use this to indicate that your hook is not able to custom render this particular value. @@ -999,7 +999,7 @@ The permission system then uses this query along with rules from plugins to dete register_column_types(datasette) -------------------------------- -Return a list of :ref:`ColumnType ` **subclasses** (not instances) to register custom column types. Column types define how values in specific columns are rendered, validated, and transformed. +Return a list of :ref:`ColumnType ` **subclasses** (not instances) to register custom column types. Column types define how values in specific columns are rendered, validated, and transformed. .. code-block:: python @@ -1069,7 +1069,7 @@ And the following methods, all optional: Per-column configuration is available via ``self.config`` in all methods. When a column type is looked up for a specific column (via :ref:`get_column_type ` or :ref:`get_column_types `), the returned instance has ``config`` set to the parsed JSON config dict for that column assignment, or ``None`` if no config was provided. -Column types are assigned to columns via the ``column_types`` key in :ref:`table configuration `: +Column types are assigned to columns via the :ref:`column_types ` table configuration option: .. code-block:: yaml diff --git a/tests/test_auth.py b/tests/test_auth.py index 1e1cd622..5868a21c 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -191,6 +191,7 @@ def test_auth_create_token( "all:view-query", "database:fixtures:drop-table", "resource:fixtures:foreign_key_references:insert-row", + "resource:fixtures:facetable:set-column-type", } ) # Now try actually creating one @@ -427,6 +428,15 @@ async def test_root_with_root_enabled_gets_all_permissions(ds_client): is True ) + assert ( + await ds_client.ds.allowed( + action="set-column-type", + resource=TableResource("fixtures", "facetable"), + actor=root_actor, + ) + is True + ) + assert ( await ds_client.ds.allowed( action="drop-table", @@ -491,3 +501,12 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client): ) is not True ), "Root without root_enabled should not automatically get drop-table" + + assert ( + await ds_client.ds.allowed( + action="set-column-type", + resource=TableResource("fixtures", "facetable"), + actor=root_actor, + ) + is not True + ), "Root without root_enabled should not automatically get set-column-type" diff --git a/tests/test_column_types.py b/tests/test_column_types.py index 538217b0..4fd30812 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -186,6 +186,226 @@ async def test_set_column_type_with_config(ds_ct): assert ct.config == {"max_length": 200} +@pytest.mark.asyncio +async def test_set_column_type_api(ds_ct): + await ds_ct.invoke_startup() + token = write_token(ds_ct, permissions=["sct"]) + response = await ds_ct.client.post( + "/data/posts/-/set-column-type", + json={"column": "title", "column_type": {"type": "email"}}, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json() == { + "ok": True, + "database": "data", + "table": "posts", + "column": "title", + "column_type": {"type": "email", "config": None}, + } + ct = await ds_ct.get_column_type("data", "posts", "title") + assert ct.name == "email" + assert ct.config is None + + +@pytest.mark.asyncio +async def test_set_column_type_api_with_config(ds_ct): + await ds_ct.invoke_startup() + token = write_token(ds_ct, permissions=["sct"]) + response = await ds_ct.client.post( + "/data/posts/-/set-column-type", + json={ + "column": "title", + "column_type": {"type": "url", "config": {"max_length": 200}}, + }, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json()["column_type"] == { + "type": "url", + "config": {"max_length": 200}, + } + ct = await ds_ct.get_column_type("data", "posts", "title") + assert ct.name == "url" + assert ct.config == {"max_length": 200} + + +@pytest.mark.asyncio +async def test_clear_column_type_api(ds_ct): + await ds_ct.invoke_startup() + await ds_ct.set_column_type("data", "posts", "title", "email") + token = write_token(ds_ct, permissions=["sct"]) + response = await ds_ct.client.post( + "/data/posts/-/set-column-type", + json={"column": "title", "column_type": None}, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json() == { + "ok": True, + "database": "data", + "table": "posts", + "column": "title", + "column_type": None, + } + ct = await ds_ct.get_column_type("data", "posts", "title") + assert ct is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body,special_case,expected_status,expected_errors", + ( + ( + {"column": "title", "column_type": {"type": "email"}}, + "no_permission", + 403, + ["Permission denied"], + ), + ( + None, + "invalid_json", + 400, + [ + "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" + ], + ), + ( + {"column": "title", "column_type": {"type": "email"}}, + "invalid_content_type", + 400, + ["Invalid content-type, must be application/json"], + ), + ( + [], + None, + 400, + ["JSON must be a dictionary"], + ), + ( + {"column_type": {"type": "email"}}, + None, + 400, + ['"column" is required'], + ), + ( + {"column": 1, "column_type": {"type": "email"}}, + None, + 400, + ['"column" must be a string'], + ), + ( + {"column": "not_a_column", "column_type": {"type": "email"}}, + None, + 400, + ["Column not found: not_a_column"], + ), + ( + {"column": "title", "column_type": "email"}, + None, + 400, + ['"column_type" must be an object or null'], + ), + ( + {"column": "title", "column_type": {}}, + None, + 400, + ['"column_type.type" is required'], + ), + ( + {"column": "title", "column_type": {"type": 1}}, + None, + 400, + ['"column_type.type" must be a string'], + ), + ( + {"column": "title", "column_type": {"type": "url", "config": []}}, + None, + 400, + ['"column_type.config" must be a dictionary'], + ), + ( + {"column": "title", "column_type": {"type": "markdown"}}, + None, + 400, + ["Unknown column type: markdown"], + ), + ( + {"column": "id", "column_type": {"type": "json"}}, + None, + 400, + [ + "Column type 'json' is only applicable to SQLite types TEXT but data.posts.id has SQLite type INTEGER" + ], + ), + ( + { + "column": "title", + "column_type": {"type": "email"}, + "extra": True, + }, + None, + 400, + ['Invalid parameter: "extra"'], + ), + ), +) +async def test_set_column_type_api_errors( + ds_ct, body, special_case, expected_status, expected_errors +): + await ds_ct.invoke_startup() + token = write_token( + ds_ct, + permissions=(["sct"] if special_case != "no_permission" else ["vi"]), + ) + kwargs = { + "headers": { + "Authorization": f"Bearer {token}", + "Content-Type": ( + "text/plain" + if special_case == "invalid_content_type" + else "application/json" + ), + } + } + if special_case == "invalid_json": + kwargs["content"] = "{bad json" + else: + kwargs["json"] = body + response = await ds_ct.client.post("/data/posts/-/set-column-type", **kwargs) + assert response.status_code == expected_status + assert response.json() == {"ok": False, "errors": expected_errors} + + +@pytest.mark.asyncio +async def test_set_column_type_api_works_for_immutable_database(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "immutable.db") + db = sqlite3.connect(str(db_path)) + db.execute("vacuum") + db.execute("create table posts (id integer primary key, title text)") + db.commit() + ds = Datasette([], immutables=[db_path]) + ds.root_enabled = True + try: + await ds.invoke_startup() + token = write_token(ds, permissions=["sct"]) + response = await ds.client.post( + "/immutable/posts/-/set-column-type", + json={"column": "title", "column_type": {"type": "email"}}, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json()["column_type"] == {"type": "email", "config": None} + ct = await ds.get_column_type("immutable", "posts", "title") + assert ct.name == "email" + finally: + db.close() + for database in ds.databases.values(): + if not database.is_memory: + database.close() + + @pytest.mark.asyncio async def test_set_column_type_rejects_incompatible_sqlite_type(ds_ct): await ds_ct.invoke_startup() diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index b378a158..ec0180a7 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -164,7 +164,14 @@ def test_datasette_error_if_string_not_list(tmpdir): @pytest.mark.asyncio async def test_get_action(ds_client): ds = ds_client.ds - for name_or_abbr in ("vi", "view-instance", "vt", "view-table"): + for name_or_abbr in ( + "vi", + "view-instance", + "vt", + "view-table", + "sct", + "set-column-type", + ): action = ds.get_action(name_or_abbr) if "-" in name_or_abbr: assert action.name == name_or_abbr diff --git a/tests/test_permissions.py b/tests/test_permissions.py index be8969d7..f9303759 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -831,6 +831,22 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "t1"), expected_result=True, ), + # set-column-type on specific table + PermConfigTestCase( + config={ + "databases": { + "perms_ds_one": { + "tables": { + "t1": {"permissions": {"set-column-type": {"id": "user"}}} + } + } + } + }, + actor={"id": "user"}, + action="set-column-type", + resource=("perms_ds_one", "t1"), + expected_result=True, + ), # insert-row on database PermConfigTestCase( config={