Stringify plugin definitions so they can be pickled during caching (#2835)

RELEASE.md

@@ -0,0 +1,3 @@
+Release type: patch
+
+Replace plugin definitions in settings with string representations after registering, so they can be cached correctly (#2828).

pelican/__init__.py

@@ -20,7 +20,7 @@ from pelican.generators import (ArticlesGenerator,  # noqa: I100
                                 PagesGenerator, SourceFileGenerator,
                                 StaticGenerator, TemplatePagesGenerator)
 from pelican.plugins import signals
-from pelican.plugins._utils import load_plugins
+from pelican.plugins._utils import get_plugin_name, load_plugins
 from pelican.readers import Readers
 from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer
 from pelican.settings import coerce_overrides, read_settings
@@ -65,14 +65,18 @@ class Pelican:
             sys.path.insert(0, '')
 
     def init_plugins(self):
-        self.plugins = load_plugins(self.settings)
+        self.plugins = []
-        for plugin in self.plugins:
+        for plugin in load_plugins(self.settings):
-            logger.debug('Registering plugin `%s`', plugin.__name__)
+            name = get_plugin_name(plugin)
+            logger.debug('Registering plugin `%s`', name)
             try:
                 plugin.register()
+                self.plugins.append(plugin)
             except Exception as e:
                 logger.error('Cannot register plugin `%s`\n%s',
-                             plugin.__name__, e)
+                             name, e)
+
+        self.settings['PLUGINS'] = [get_plugin_name(p) for p in self.plugins]
 
     def run(self):
         """Run the generators and return"""

pelican/plugins/_utils.py

@@ -1,6 +1,7 @@
 import importlib
 import importlib.machinery
 import importlib.util
+import inspect
 import logging
 import pkgutil
 import sys
@@ -107,3 +108,18 @@ def load_plugins(settings):
         plugins = list(namespace_plugins.values())
 
     return plugins
+
+
+def get_plugin_name(plugin):
+    """
+    Plugins can be passed as module objects, however this breaks caching as
+    module objects cannot be pickled. To work around this, all plugins are
+    stringified post-initialization.
+    """
+    if inspect.isclass(plugin):
+        return plugin.__qualname__
+
+    if inspect.ismodule(plugin):
+        return plugin.__name__
+
+    return type(plugin).__qualname__

pelican/tests/dummy_plugins/normal_plugin/normal_plugin/__init__.py

@@ -1,7 +1,5 @@
 from .submodule import noop  # noqa: F401
 
-NAME = 'normal plugin'
-
 
 def register():
     pass

pelican/tests/dummy_plugins/normal_plugin/normal_submodule_plugin/subpackage/subpackage.py

@@ -1,5 +1,2 @@
-NAME = 'normal subpackage plugin'
-
-
 def register():
     pass

pelican/tests/dummy_plugins/normal_plugin/normal_submodule_plugin/subplugin.py

@@ -1,5 +1,2 @@
-NAME = 'normal submodule plugin'
-
-
 def register():
     pass

pelican/tests/test_plugins.py

@@ -1,7 +1,9 @@
 import os
 from contextlib import contextmanager
 
-from pelican.plugins._utils import get_namespace_plugins, load_plugins
+import pelican.tests.dummy_plugins.normal_plugin.normal_plugin as normal_plugin
+from pelican.plugins._utils import (get_namespace_plugins, get_plugin_name,
+                                    load_plugins)
 from pelican.tests.support import unittest
 
 
@@ -81,9 +83,7 @@ class PluginTest(unittest.TestCase):
 
     def test_load_plugins(self):
         def get_plugin_names(plugins):
-            return {
+            return {get_plugin_name(p) for p in plugins}
-                plugin.NAME if hasattr(plugin, 'NAME') else plugin.__name__
-                for plugin in plugins}
 
         # existing namespace plugins
         existing_ns_plugins = load_plugins({})
@@ -93,7 +93,7 @@ class PluginTest(unittest.TestCase):
             plugins = load_plugins({})
             self.assertEqual(len(plugins), len(existing_ns_plugins)+1, plugins)
             self.assertEqual(
-                {'namespace plugin'} | get_plugin_names(existing_ns_plugins),
+                {'pelican.plugins.ns_plugin'} | get_plugin_names(existing_ns_plugins),
                 get_plugin_names(plugins))
 
             # disable namespace plugins with `PLUGINS = []`
@@ -113,7 +113,7 @@ class PluginTest(unittest.TestCase):
             plugins = load_plugins(SETTINGS)
             self.assertEqual(len(plugins), 1, plugins)
             self.assertEqual(
-                {'normal plugin'},
+                {'normal_plugin'},
                 get_plugin_names(plugins))
 
             # normal submodule/subpackage plugins
@@ -127,8 +127,8 @@ class PluginTest(unittest.TestCase):
             plugins = load_plugins(SETTINGS)
             self.assertEqual(len(plugins), 2, plugins)
             self.assertEqual(
-                {'normal submodule plugin',
+                {'normal_submodule_plugin.subplugin',
-                 'normal subpackage plugin'},
+                 'normal_submodule_plugin.subpackage.subpackage'},
                 get_plugin_names(plugins))
 
             # ensure normal plugins are loaded only once
@@ -149,7 +149,7 @@ class PluginTest(unittest.TestCase):
             plugins = load_plugins(SETTINGS)
             self.assertEqual(len(plugins), 1, plugins)
             self.assertEqual(
-                {'namespace plugin'},
+                {'pelican.plugins.ns_plugin'},
                 get_plugin_names(plugins))
 
             # namespace plugin long
@@ -159,7 +159,7 @@ class PluginTest(unittest.TestCase):
             plugins = load_plugins(SETTINGS)
             self.assertEqual(len(plugins), 1, plugins)
             self.assertEqual(
-                {'namespace plugin'},
+                {'pelican.plugins.ns_plugin'},
                 get_plugin_names(plugins))
 
             # normal and namespace plugin
@@ -170,5 +170,22 @@ class PluginTest(unittest.TestCase):
             plugins = load_plugins(SETTINGS)
             self.assertEqual(len(plugins), 2, plugins)
             self.assertEqual(
-                {'normal plugin', 'namespace plugin'},
+                {'normal_plugin', 'pelican.plugins.ns_plugin'},
                 get_plugin_names(plugins))
+
+    def test_get_plugin_name(self):
+        self.assertEqual(
+            get_plugin_name(normal_plugin),
+            'pelican.tests.dummy_plugins.normal_plugin.normal_plugin',
+        )
+
+        class NoopPlugin:
+            def register(self):
+                pass
+
+        self.assertEqual(
+            get_plugin_name(NoopPlugin),
+            'PluginTest.test_get_plugin_name.<locals>.NoopPlugin')
+        self.assertEqual(
+            get_plugin_name(NoopPlugin()),
+            'PluginTest.test_get_plugin_name.<locals>.NoopPlugin')