New get_metadata() plugin hook for dynamic metadata

The following hook is added:

    get_metadata(
      datasette=self, key=key, database=database, table=table,
      fallback=fallback
    )

This gets called when we're building our metdata for the rest
of the system to use. We merge whatever the plugins return
with any local metadata (from metadata.yml/yaml/json) allowing
for a live-editable dynamic Datasette.

As a security precation, local meta is *not* overwritable by
plugin hooks. The workflow for transitioning to live-meta would
be to load the plugin with the full metadata.yaml and save.
Then remove the parts of the metadata that you want to be able
to change from the file.

* Avoid race condition: don't mutate databases list

This avoids the nasty "RuntimeError: OrderedDict mutated during
iteration" error that randomly happens when a plugin adds a
new database to Datasette, using `add_database`. This change
makes the add and remove database functions more expensive, but
it prevents the random explosion race conditions that make for
confusing user experience when importing live databases.

Thanks, @brandonrobertz

tests/test_permissions.py

@@ -440,7 +440,7 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta
     """Test that e.g. having view-table but NOT view-database lets you view table page, etc"""
     allow = {"id": "*"}
     deny = {}
-    previous_metadata = cascade_app_client.ds._metadata
+    previous_metadata = cascade_app_client.ds.metadata()
     updated_metadata = copy.deepcopy(previous_metadata)
     actor = {"id": "test"}
     if "download" in permissions:
@@ -457,11 +457,11 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta
         updated_metadata["databases"]["fixtures"]["queries"]["magic_parameters"][
             "allow"
         ] = (allow if "query" in permissions else deny)
-        cascade_app_client.ds._metadata = updated_metadata
+        cascade_app_client.ds._metadata_local = updated_metadata
         response = cascade_app_client.get(
             path,
             cookies={"ds_actor": cascade_app_client.actor_cookie(actor)},
         )
         assert expected_status == response.status
     finally:
-        cascade_app_client.ds._metadata = previous_metadata
+        cascade_app_client.ds._metadata_local = previous_metadata

tests/test_plugins.py

@@ -850,3 +850,32 @@ def test_hook_skip_csrf(app_client):
         "/skip-csrf-2", post_data={"this is": "post data"}, cookies={"ds_actor": cookie}
     )
     assert second_missing_csrf_response.status == 403
+
+
+def test_hook_get_metadata(app_client):
+    app_client.ds._metadata_local = {
+        "title": "Testing get_metadata hook!",
+        "databases": {
+            "from-local": {
+                "title": "Hello from local metadata"
+            }
+        }
+    }
+    og_pm_hook_get_metadata = pm.hook.get_metadata
+    def get_metadata_mock(*args, **kwargs):
+        return [{
+            "databases": {
+                "from-hook": {
+                    "title": "Hello from the plugin hook"
+                },
+                "from-local": {
+                    "title": "This will be overwritten!"
+                }
+            }
+        }]
+    pm.hook.get_metadata = get_metadata_mock
+    meta = app_client.ds.metadata()
+    assert "Testing get_metadata hook!" == meta["title"]
+    assert "Hello from local metadata" == meta["databases"]["from-local"]["title"]
+    assert "Hello from the plugin hook" == meta["databases"]["from-hook"]["title"]
+    pm.hook.get_metadata = og_pm_hook_get_metadata