"$env": "X" mechanism now works with nested lists, closes #837

datasette/app.py

@@ -45,6 +45,7 @@ from .utils import (
     format_bytes,
     module_from_path,
     parse_metadata,
+    resolve_env_secrets,
     sqlite3,
     to_css_class,
 )
@@ -367,18 +368,7 @@ class Datasette:
             return None
         plugin_config = plugins.get(plugin_name)
         # Resolve any $file and $env keys
-        if isinstance(plugin_config, dict):
-            # Create a copy so we don't mutate the version visible at /-/metadata.json
-            plugin_config_copy = dict(plugin_config)
-            for key, value in plugin_config_copy.items():
-                if isinstance(value, dict):
-                    if list(value.keys()) == ["$env"]:
-                        plugin_config_copy[key] = os.environ.get(
-                            list(value.values())[0]
-                        )
-                    elif list(value.keys()) == ["$file"]:
-                        plugin_config_copy[key] = open(list(value.values())[0]).read()
-            return plugin_config_copy
+        plugin_config = resolve_env_secrets(plugin_config, os.environ)
         return plugin_config
 
     def app_css_hash(self):

datasette/utils/__init__.py

@@ -904,3 +904,19 @@ async def check_visibility(datasette, actor, action, resource, default=True):
         None, action, resource=resource, default=default,
     )
     return visible, private
+
+
+def resolve_env_secrets(config, environ):
+    'Create copy that recursively replaces {"$env": "NAME"} with values from environ'
+    if isinstance(config, dict):
+        if list(config.keys()) == ["$env"]:
+            return environ.get(list(config.values())[0])
+        else:
+            return {
+                key: resolve_env_secrets(value, environ)
+                for key, value in config.items()
+            }
+    elif isinstance(config, list):
+        return [resolve_env_secrets(value, environ) for value in config]
+    else:
+        return config

docs/changelog.rst

@@ -94,6 +94,8 @@ Both flash messages and user authentication needed a way to sign values and set
 
 Datasette will generate a secret automatically when it starts up, but to avoid resetting the secret (and hence invalidating any cookies) every time the server restarts you should set your own secret. You can pass a secret to Datasette using the new ``--secret`` option or with a ``DATASETTE_SECRET`` environment variable. See :ref:`config_secret` for more details.
 
+You can also set a secret when you deploy Datasette using ``datasette publish`` or ``datasette package`` - see :ref:`config_publish_secrets`.
+
 Plugins can now sign value and verify their signatures using the :ref:`datasette.sign() <datasette_sign>` and :ref:`datasette.unsign() <datasette_unsign>` methods.
 
 CSRF protection

tests/fixtures.py

@@ -408,6 +408,7 @@ METADATA = {
     "plugins": {
         "name-of-plugin": {"depth": "root"},
         "env-plugin": {"foo": {"$env": "FOO_ENV"}},
+        "env-plugin-list": [{"in_a_list": {"$env": "FOO_ENV"}}],
         "file-plugin": {"foo": {"$file": TEMP_PLUGIN_SECRET_FILE}},
     },
     "databases": {

tests/test_plugins.py

@@ -173,6 +173,19 @@ def test_plugin_config_env(app_client):
     del os.environ["FOO_ENV"]
 
 
+def test_plugin_config_env_from_list(app_client):
+    os.environ["FOO_ENV"] = "FROM_ENVIRONMENT"
+    assert [{"in_a_list": "FROM_ENVIRONMENT"}] == app_client.ds.plugin_config(
+        "env-plugin-list"
+    )
+    # Ensure secrets aren't visible in /-/metadata.json
+    metadata = app_client.get("/-/metadata.json")
+    assert [{"in_a_list": {"$env": "FOO_ENV"}}] == metadata.json["plugins"][
+        "env-plugin-list"
+    ]
+    del os.environ["FOO_ENV"]
+
+
 def test_plugin_config_file(app_client):
     open(TEMP_PLUGIN_SECRET_FILE, "w").write("FROM_FILE")
     assert {"foo": "FROM_FILE"} == app_client.ds.plugin_config("file-plugin")

tests/test_utils.py

@@ -503,3 +503,17 @@ def test_multi_params(data, should_raise):
 )
 def test_actor_matches_allow(actor, allow, expected):
     assert expected == utils.actor_matches_allow(actor, allow)
+
+
+@pytest.mark.parametrize(
+    "config,expected",
+    [
+        ({"foo": "bar"}, {"foo": "bar"}),
+        ({"$env": "FOO"}, "x"),
+        ({"k": {"$env": "FOO"}}, {"k": "x"}),
+        ([{"k": {"$env": "FOO"}}, {"z": {"$env": "FOO"}}], [{"k": "x"}, {"z": "x"}]),
+        ({"k": [{"in_a_list": {"$env": "FOO"}}]}, {"k": [{"in_a_list": "x"}]}),
+    ],
+)
+def test_resolve_env_secrets(config, expected):
+    assert expected == utils.resolve_env_secrets(config, {"FOO": "x"})