Namespace plugin implementation

* Creates pelican.plugins
* Moves plugin related code under pelican.plugins
  * pelican.plugins.signals is now the location for signals, pelican.signals is kept
    for backwards compatibility
  * pelican.plugins._utils contains necessary bits for plugin discovery and loading.
    Logic from Pelican class is moved here. Pelican class now just asks for plugins
    and registers them
* Contains tests for old and new plugin loading

pelican/__init__.py

@@ -9,6 +9,11 @@ import sys
 import time
 import traceback
 from collections.abc import Iterable
+# Combines all paths to `pelican` package accessible from `sys.path`
+# Makes it possible to install `pelican` and namespace plugins into different
+# locations in the file system (e.g. pip with `-e` or `--user`)
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
 
 # pelican.log has to be the first pelican module to be loaded
 # because logging.setLoggerClass has to be called before logging.getLogger
@@ -17,6 +22,7 @@ from pelican import signals  # noqa
 from pelican.generators import (ArticlesGenerator, PagesGenerator,
                                 SourceFileGenerator, StaticGenerator,
                                 TemplatePagesGenerator)
+from pelican.plugins._utils import load_plugins
 from pelican.readers import Readers
 from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer
 from pelican.settings import read_settings
@@ -62,27 +68,14 @@ class Pelican(object):
             sys.path.insert(0, '')
 
     def init_plugins(self):
-        self.plugins = []
+        self.plugins = load_plugins(self.settings)
-        logger.debug('Temporarily adding PLUGIN_PATHS to system path')
+        for plugin in self.plugins:
-        _sys_path = sys.path[:]
+            logger.debug('Registering plugin `%s`', plugin.__name__)
-        for pluginpath in self.settings['PLUGIN_PATHS']:
+            try:
-            sys.path.insert(0, pluginpath)
+                plugin.register()
-        for plugin in self.settings['PLUGINS']:
+            except Exception as e:
-            # if it's a string, then import it
+                logger.error('Cannot register plugin `%s`\n%s',
-            if isinstance(plugin, str):
+                             plugin.__name__, e)
-                logger.debug("Loading plugin `%s`", plugin)
-                try:
-                    plugin = __import__(plugin, globals(), locals(), 'module')
-                except ImportError as e:
-                    logger.error(
-                        "Cannot load plugin `%s`\n%s", plugin, e)
-                    continue
-
-            logger.debug("Registering plugin `%s`", plugin.__name__)
-            plugin.register()
-            self.plugins.append(plugin)
-        logger.debug('Restoring system path')
-        sys.path = _sys_path
 
     def run(self):
         """Run the generators and return"""

pelican/plugins/_utils.py

@@ -0,0 +1,77 @@
+import importlib
+import logging
+import pkgutil
+import sys
+
+import six
+
+
+logger = logging.getLogger(__name__)
+
+
+def iter_namespace(ns_pkg):
+    # Specifying the second argument (prefix) to iter_modules makes the
+    # returned name an absolute name instead of a relative one. This allows
+    # import_module to work without having to do additional modification to
+    # the name.
+    return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".")
+
+
+def get_namespace_plugins(ns_pkg=None):
+    if ns_pkg is None:
+        import pelican.plugins as ns_pkg
+
+    return {
+        name: importlib.import_module(name)
+        for finder, name, ispkg
+        in iter_namespace(ns_pkg)
+        if ispkg
+    }
+
+
+def list_plugins(ns_pkg=None):
+    from pelican.log import init as init_logging
+    init_logging(logging.INFO)
+    ns_plugins = get_namespace_plugins(ns_pkg)
+    if ns_plugins:
+        logger.info('Plugins found:\n' + '\n'.join(ns_plugins))
+    else:
+        logger.info('No plugins are installed')
+
+
+def load_plugins(settings):
+    logger.debug('Finding namespace plugins')
+    namespace_plugins = get_namespace_plugins()
+    if namespace_plugins:
+        logger.debug('Namespace plugins found:\n' +
+                     '\n'.join(namespace_plugins))
+    plugins = []
+    if settings.get('PLUGINS') is not None:
+        _sys_path = sys.path[:]
+
+        for path in settings.get('PLUGIN_PATHS', []):
+            sys.path.insert(0, path)
+
+        for plugin in settings['PLUGINS']:
+            if isinstance(plugin, six.string_types):
+                logger.debug('Loading plugin `%s`', plugin)
+                # try to find in namespace plugins
+                if plugin in namespace_plugins:
+                    plugin = namespace_plugins[plugin]
+                elif 'pelican.plugins.{}'.format(plugin) in namespace_plugins:
+                    plugin = namespace_plugins['pelican.plugins.{}'.format(
+                        plugin)]
+                # try to import it
+                else:
+                    try:
+                        plugin = __import__(plugin, globals(), locals(),
+                                            str('module'))
+                    except ImportError as e:
+                        logger.error('Cannot load plugin `%s`\n%s', plugin, e)
+                        continue
+            plugins.append(plugin)
+        sys.path = _sys_path
+    else:
+        plugins = list(namespace_plugins.values())
+
+    return plugins

pelican/plugins/signals.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function, unicode_literals
+
+from blinker import signal
+
+# Run-level signals:
+
+initialized = signal('pelican_initialized')
+get_generators = signal('get_generators')
+all_generators_finalized = signal('all_generators_finalized')
+get_writer = signal('get_writer')
+finalized = signal('pelican_finalized')
+
+# Reader-level signals
+
+readers_init = signal('readers_init')
+
+# Generator-level signals
+
+generator_init = signal('generator_init')
+
+article_generator_init = signal('article_generator_init')
+article_generator_pretaxonomy = signal('article_generator_pretaxonomy')
+article_generator_finalized = signal('article_generator_finalized')
+article_generator_write_article = signal('article_generator_write_article')
+article_writer_finalized = signal('article_writer_finalized')
+
+page_generator_init = signal('page_generator_init')
+page_generator_finalized = signal('page_generator_finalized')
+page_generator_write_page = signal('page_generator_write_page')
+page_writer_finalized = signal('page_writer_finalized')
+
+static_generator_init = signal('static_generator_init')
+static_generator_finalized = signal('static_generator_finalized')
+
+# Page-level signals
+
+article_generator_preread = signal('article_generator_preread')
+article_generator_context = signal('article_generator_context')
+
+page_generator_preread = signal('page_generator_preread')
+page_generator_context = signal('page_generator_context')
+
+static_generator_preread = signal('static_generator_preread')
+static_generator_context = signal('static_generator_context')
+
+content_object_init = signal('content_object_init')
+
+# Writers signals
+content_written = signal('content_written')
+feed_generated = signal('feed_generated')
+feed_written = signal('feed_written')

pelican/settings.py

@@ -138,7 +138,7 @@ DEFAULT_CONFIG = {
     'TYPOGRIFY_IGNORE_TAGS': [],
     'SUMMARY_MAX_LENGTH': 50,
     'PLUGIN_PATHS': [],
-    'PLUGINS': [],
+    'PLUGINS': None,
     'PYGMENTS_RST_OPTIONS': {},
     'TEMPLATE_PAGES': {},
     'TEMPLATE_EXTENSIONS': ['.html'],

pelican/signals.py

@@ -1,51 +1,4 @@
 # -*- coding: utf-8 -*-
+# This file is for backwards compatibility
 
-from blinker import signal
+from pelican.plugins.signals import * # noqa
-
-# Run-level signals:
-
-initialized = signal('pelican_initialized')
-get_generators = signal('get_generators')
-all_generators_finalized = signal('all_generators_finalized')
-get_writer = signal('get_writer')
-finalized = signal('pelican_finalized')
-
-# Reader-level signals
-
-readers_init = signal('readers_init')
-
-# Generator-level signals
-
-generator_init = signal('generator_init')
-
-article_generator_init = signal('article_generator_init')
-article_generator_pretaxonomy = signal('article_generator_pretaxonomy')
-article_generator_finalized = signal('article_generator_finalized')
-article_generator_write_article = signal('article_generator_write_article')
-article_writer_finalized = signal('article_writer_finalized')
-
-page_generator_init = signal('page_generator_init')
-page_generator_finalized = signal('page_generator_finalized')
-page_generator_write_page = signal('page_generator_write_page')
-page_writer_finalized = signal('page_writer_finalized')
-
-static_generator_init = signal('static_generator_init')
-static_generator_finalized = signal('static_generator_finalized')
-
-# Page-level signals
-
-article_generator_preread = signal('article_generator_preread')
-article_generator_context = signal('article_generator_context')
-
-page_generator_preread = signal('page_generator_preread')
-page_generator_context = signal('page_generator_context')
-
-static_generator_preread = signal('static_generator_preread')
-static_generator_context = signal('static_generator_context')
-
-content_object_init = signal('content_object_init')
-
-# Writers signals
-content_written = signal('content_written')
-feed_generated = signal('feed_generated')
-feed_written = signal('feed_written')

pelican/tests/dummy_plugins/namespace_plugin/pelican/plugins/ns_plugin/__init__.py

@@ -0,0 +1,5 @@
+NAME = 'namespace plugin'
+
+
+def register():
+    pass

pelican/tests/dummy_plugins/normal_plugin/normal_plugin/__init__.py

@@ -0,0 +1,5 @@
+NAME = 'normal plugin'
+
+
+def register():
+    pass

pelican/tests/test_plugins.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function, unicode_literals
+
+import os
+from contextlib import contextmanager
+
+from pelican.plugins._utils import get_namespace_plugins, load_plugins
+from pelican.tests.support import unittest
+
+
+@contextmanager
+def tmp_namespace_path(path):
+    '''Context manager for temporarily appending namespace plugin packages
+
+    path: path containing the `pelican` folder
+
+    This modifies the `pelican.__path__` and lets the `pelican.plugins`
+    namespace package resolve it from that.
+    '''
+    # This avoids calls to internal `pelican.plugins.__path__._recalculate()`
+    # as it should not be necessary
+    import pelican
+
+    old_path = pelican.__path__[:]
+    try:
+        pelican.__path__.append(os.path.join(path, 'pelican'))
+        yield
+    finally:
+        pelican.__path__ = old_path
+
+
+class PluginTest(unittest.TestCase):
+    _PLUGIN_FOLDER = os.path.join(
+        os.path.abspath(os.path.dirname(__file__)),
+        'dummy_plugins')
+    _NS_PLUGIN_FOLDER = os.path.join(_PLUGIN_FOLDER, 'namespace_plugin')
+    _NORMAL_PLUGIN_FOLDER = os.path.join(_PLUGIN_FOLDER, 'normal_plugin')
+
+    def test_namespace_path_modification(self):
+        import pelican
+        import pelican.plugins
+        old_path = pelican.__path__[:]
+
+        # not existing path
+        path = os.path.join(self._PLUGIN_FOLDER, 'foo')
+        with tmp_namespace_path(path):
+            self.assertIn(
+                os.path.join(path, 'pelican'),
+                pelican.__path__)
+            # foo/pelican does not exist, so it won't propagate
+            self.assertNotIn(
+                os.path.join(path, 'pelican', 'plugins'),
+                pelican.plugins.__path__)
+        # verify that we restored path back
+        self.assertEqual(pelican.__path__, old_path)
+
+        # existing path
+        with tmp_namespace_path(self._NS_PLUGIN_FOLDER):
+            self.assertIn(
+                os.path.join(self._NS_PLUGIN_FOLDER, 'pelican'),
+                pelican.__path__)
+            # /namespace_plugin/pelican exists, so it should be in
+            self.assertIn(
+                os.path.join(self._NS_PLUGIN_FOLDER, 'pelican', 'plugins'),
+                pelican.plugins.__path__)
+        self.assertEqual(pelican.__path__, old_path)
+
+    def test_get_namespace_plugins(self):
+        # without plugins
+        ns_plugins = get_namespace_plugins()
+        self.assertEqual(len(ns_plugins), 0)
+
+        # with plugin
+        with tmp_namespace_path(self._NS_PLUGIN_FOLDER):
+            ns_plugins = get_namespace_plugins()
+            self.assertEqual(len(ns_plugins), 1)
+            self.assertIn('pelican.plugins.ns_plugin', ns_plugins)
+            self.assertEqual(
+                ns_plugins['pelican.plugins.ns_plugin'].NAME,
+                'namespace plugin')
+
+        # should be back to 0 outside `with`
+        ns_plugins = get_namespace_plugins()
+        self.assertEqual(len(ns_plugins), 0)
+
+    def test_load_plugins(self):
+        # no plugins
+        plugins = load_plugins({})
+        self.assertEqual(len(plugins), 0)
+
+        with tmp_namespace_path(self._NS_PLUGIN_FOLDER):
+            # with no `PLUGINS` setting, load namespace plugins
+            plugins = load_plugins({})
+            self.assertEqual(len(plugins), 1, plugins)
+            self.assertEqual(
+                {'namespace plugin'},
+                set(plugin.NAME for plugin in plugins))
+
+            # disable namespace plugins with `PLUGINS = []`
+            SETTINGS = {
+                'PLUGINS': []
+            }
+            plugins = load_plugins(SETTINGS)
+            self.assertEqual(len(plugins), 0, plugins)
+
+            # using `PLUGINS`
+
+            # normal plugin
+            SETTINGS = {
+                'PLUGINS': ['normal_plugin'],
+                'PLUGIN_PATHS': [self._NORMAL_PLUGIN_FOLDER]
+            }
+            plugins = load_plugins(SETTINGS)
+            self.assertEqual(len(plugins), 1, plugins)
+            self.assertEqual(
+                {'normal plugin'},
+                set(plugin.NAME for plugin in plugins))
+
+            # namespace plugin short
+            SETTINGS = {
+                'PLUGINS': ['ns_plugin']
+            }
+            plugins = load_plugins(SETTINGS)
+            self.assertEqual(len(plugins), 1, plugins)
+            self.assertEqual(
+                {'namespace plugin'},
+                set(plugin.NAME for plugin in plugins))
+
+            # namespace plugin long
+            SETTINGS = {
+                'PLUGINS': ['pelican.plugins.ns_plugin']
+            }
+            plugins = load_plugins(SETTINGS)
+            self.assertEqual(len(plugins), 1, plugins)
+            self.assertEqual(
+                {'namespace plugin'},
+                set(plugin.NAME for plugin in plugins))
+
+            # normal and namespace plugin
+            SETTINGS = {
+                'PLUGINS': ['normal_plugin', 'ns_plugin'],
+                'PLUGIN_PATHS': [self._NORMAL_PLUGIN_FOLDER]
+            }
+            plugins = load_plugins(SETTINGS)
+            self.assertEqual(len(plugins), 2, plugins)
+            self.assertEqual(
+                {'normal plugin', 'namespace plugin'},
+                set(plugin.NAME for plugin in plugins))

setup.py

@@ -18,7 +18,8 @@ entry_points = {
         'pelican = pelican.__main__:main',
         'pelican-import = pelican.tools.pelican_import:main',
         'pelican-quickstart = pelican.tools.pelican_quickstart:main',
-        'pelican-themes = pelican.tools.pelican_themes:main'
+        'pelican-themes = pelican.tools.pelican_themes:main',
+        'pelican-plugins = pelican.plugins._utils:list_plugins'
     ]
 }
 
@@ -44,7 +45,7 @@ setup(
     keywords='static web site generator SSG reStructuredText Markdown',
     license='AGPLv3',
     long_description=description,
-    packages=['pelican', 'pelican.tools'],
+    packages=['pelican', 'pelican.tools', 'pelican.plugins'],
     package_data={
         # we manually collect the package data, as opposed to using,
         # include_package_data=True because we don't want the tests to be