Merge pull request #2644 from avaris/namespace

Namespace plugin implementation

docs/plugins.rst

@@ -9,16 +9,31 @@ features to Pelican without having to directly modify the Pelican core.
 How to use plugins
 ==================
 
-To load plugins, you have to specify them in your settings file. There are two
+Starting with version 5.0, Pelican moved to a new plugin structure utilizing
-ways to do so. The first method is to specify strings with the path to the
+namespace packages. Plugins supporting this structure will install under the
-callables::
+namespace package ``pelican.plugins`` and can be automatically discovered
+by Pelican.
 
-    PLUGINS = ['package.myplugin',]
+If you leave the ``PLUGINS`` setting as default (``None``), Pelican will then
+collect the namespace plugins and register them. If on the other hand you
+specify a ``PLUGINS`` settings as a list of plugins, this autodiscovery will
+be disabled and only listed plugins will be registered and you will have to
+explicitly list the namespace plugins as well.
 
-Alternatively, another method is to import them and add them to the list::
+If you are using ``PLUGINS`` setting, you can specify plugins in two ways.
+The first method specifies plugins as a list of strings. Namespace plugins can
+be specified either by their full names (``pelican.plugins.myplugin``) or by
+their short names (``myplugin``)::
+
+    PLUGINS = ['package.myplugin',
+               'namespace_plugin1',
+               'pelican.plugins.namespace_plugin2']
+
+Alternatively, you can import them in your settings file and pass the modules::
 
     from package import myplugin
-    PLUGINS = [myplugin,]
+    from pelican.plugins import namespace_plugin1, namespace_plugin2
+    PLUGINS = [myplugin, namespace_plugin1, namespace_plugin2]
 
 .. note::
 
@@ -36,11 +51,13 @@ the ``PLUGIN_PATHS`` list can be absolute or relative to the settings file::
 
 Where to find plugins
 =====================
+Namespace plugins can be found in the `pelican-plugins organization`_ as
+individual repositories. Legacy plugins are collected in the `pelican-plugins
+repository`_ and they will be slowly phased out in favor of the namespace
+versions.
 
-We maintain a separate repository of plugins for people to share and use.
+.. _pelican-plugins organization: https://github.com/pelican-plugins
-Please visit the `pelican-plugins`_ repository for a list of available plugins.
+.. _pelican-plugins repository: https://github.com/getpelican/pelican-plugins
-
-.. _pelican-plugins: https://github.com/getpelican/pelican-plugins
 
 Please note that while we do our best to review and maintain these plugins,
 they are submitted by the Pelican community and thus may have varying levels of
@@ -70,6 +87,33 @@ which you map the signals to your plugin logic. Let's take a simple example::
     your ``register`` callable or they will be garbage-collected before the
     signal is emitted.
 
+Namespace plugin structure
+--------------------------
+
+Namespace plugins must adhere to a certain structure in order to function
+properly. They need to be installable (i.e. contain ``setup.py`` or equivalent)
+and have a folder structure as follows::
+
+    myplugin
+    ├── pelican
+    │   └── plugins
+    │       └── myplugin
+    │           ├── __init__.py
+    │           └── ...
+    ├── ...
+    └── setup.py
+
+It is crucial that ``pelican`` or ``pelican/plugins`` folder **not**
+contain an ``__init__.py`` file. In fact, it is best to have those folders
+empty besides the listed folders in the above structure and keep your
+plugin related files contained solely in the ``pelican/plugins/myplugin``
+folder to avoid any issues.
+
+For easily setting up the proper structure, a `cookiecutter template for
+plugins`_ is provided. Refer to the README in the link for how to use it.
+
+.. _cookiecutter template for plugins: https://github.com/getpelican/cookiecutter-pelican-plugin
+
 List of signals
 ===============
 

docs/settings.rst

@@ -194,7 +194,7 @@ Basic settings
    Controls the extension that will be used by the SourcesGenerator.  Defaults
    to ``.text``. If not a valid string the default value will be used.
 
-.. data:: PLUGINS = []
+.. data:: PLUGINS = None
 
    The list of plugins to load. See :ref:`plugins`.
 

pelican/__init__.py

@@ -9,14 +9,20 @@ import sys
 import time
 import traceback
 from collections.abc import Iterable
+# Combines all paths to `pelican` package accessible from `sys.path`
+# Makes it possible to install `pelican` and namespace plugins into different
+# locations in the file system (e.g. pip with `-e` or `--user`)
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
 
 # pelican.log has to be the first pelican module to be loaded
 # because logging.setLoggerClass has to be called before logging.getLogger
 from pelican.log import init as init_logging
-from pelican import signals  # noqa
+from pelican.generators import (ArticlesGenerator,  # noqa: I100
-from pelican.generators import (ArticlesGenerator, PagesGenerator,
+                                PagesGenerator, SourceFileGenerator,
-                                SourceFileGenerator, StaticGenerator,
+                                StaticGenerator, TemplatePagesGenerator)
-                                TemplatePagesGenerator)
+from pelican.plugins import signals
+from pelican.plugins._utils import load_plugins
 from pelican.readers import Readers
 from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer
 from pelican.settings import read_settings
@@ -62,27 +68,14 @@ class Pelican(object):
             sys.path.insert(0, '')
 
     def init_plugins(self):
-        self.plugins = []
+        self.plugins = load_plugins(self.settings)
-        logger.debug('Temporarily adding PLUGIN_PATHS to system path')
+        for plugin in self.plugins:
-        _sys_path = sys.path[:]
+            logger.debug('Registering plugin `%s`', plugin.__name__)
-        for pluginpath in self.settings['PLUGIN_PATHS']:
+            try:
-            sys.path.insert(0, pluginpath)
+                plugin.register()
-        for plugin in self.settings['PLUGINS']:
+            except Exception as e:
-            # if it's a string, then import it
+                logger.error('Cannot register plugin `%s`\n%s',
-            if isinstance(plugin, str):
+                             plugin.__name__, e)
-                logger.debug("Loading plugin `%s`", plugin)
-                try:
-                    plugin = __import__(plugin, globals(), locals(), 'module')
-                except ImportError as e:
-                    logger.error(
-                        "Cannot load plugin `%s`\n%s", plugin, e)
-                    continue
-
-            logger.debug("Registering plugin `%s`", plugin.__name__)
-            plugin.register()
-            self.plugins.append(plugin)
-        logger.debug('Restoring system path')
-        sys.path = _sys_path
 
     def run(self):
         """Run the generators and return"""

pelican/contents.py

@@ -10,7 +10,7 @@ from urllib.parse import urljoin, urlparse, urlunparse
 
 import pytz
 
-from pelican import signals
+from pelican.plugins import signals
 from pelican.settings import DEFAULT_CONFIG
 from pelican.utils import (deprecated_attribute, memoized, path_to_url,
                            posixize_path, sanitised_join, set_date_tzinfo,

pelican/generators.py

@@ -13,9 +13,9 @@ from operator import attrgetter
 from jinja2 import (BaseLoader, ChoiceLoader, Environment, FileSystemLoader,
                     PrefixLoader, TemplateNotFound)
 
-from pelican import signals
 from pelican.cache import FileStampDataCacher
 from pelican.contents import Article, Page, Static
+from pelican.plugins import signals
 from pelican.readers import Readers
 from pelican.utils import (DateFormatter, copy, mkdir_p, order_content,
                            posixize_path, process_translations)

pelican/plugins/_utils.py

@@ -0,0 +1,85 @@
+import importlib
+import importlib.machinery
+import importlib.util
+import logging
+import pkgutil
+
+
+logger = logging.getLogger(__name__)
+
+
+def iter_namespace(ns_pkg):
+    # Specifying the second argument (prefix) to iter_modules makes the
+    # returned name an absolute name instead of a relative one. This allows
+    # import_module to work without having to do additional modification to
+    # the name.
+    return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".")
+
+
+def get_namespace_plugins(ns_pkg=None):
+    if ns_pkg is None:
+        import pelican.plugins as ns_pkg
+
+    return {
+        name: importlib.import_module(name)
+        for finder, name, ispkg
+        in iter_namespace(ns_pkg)
+        if ispkg
+    }
+
+
+def list_plugins(ns_pkg=None):
+    from pelican.log import init as init_logging
+    init_logging(logging.INFO)
+    ns_plugins = get_namespace_plugins(ns_pkg)
+    if ns_plugins:
+        logger.info('Plugins found:\n' + '\n'.join(ns_plugins))
+    else:
+        logger.info('No plugins are installed')
+
+
+def load_legacy_plugin(plugin, plugin_paths):
+    # Try to find plugin in PLUGIN_PATHS
+    spec = importlib.machinery.PathFinder.find_spec(plugin, plugin_paths)
+    if spec is None:
+        # If failed, try to find it in normal importable locations
+        spec = importlib.util.find_spec(plugin)
+    if spec is None:
+        raise ImportError('Cannot import plugin `{}`'.format(plugin))
+    else:
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+        return mod
+
+
+def load_plugins(settings):
+    logger.debug('Finding namespace plugins')
+    namespace_plugins = get_namespace_plugins()
+    if namespace_plugins:
+        logger.debug('Namespace plugins found:\n' +
+                     '\n'.join(namespace_plugins))
+    plugins = []
+    if settings.get('PLUGINS') is not None:
+        for plugin in settings['PLUGINS']:
+            if isinstance(plugin, str):
+                logger.debug('Loading plugin `%s`', plugin)
+                # try to find in namespace plugins
+                if plugin in namespace_plugins:
+                    plugin = namespace_plugins[plugin]
+                elif 'pelican.plugins.{}'.format(plugin) in namespace_plugins:
+                    plugin = namespace_plugins['pelican.plugins.{}'.format(
+                        plugin)]
+                # try to import it
+                else:
+                    try:
+                        plugin = load_legacy_plugin(
+                            plugin,
+                            settings.get('PLUGIN_PATHS', []))
+                    except ImportError as e:
+                        logger.error('Cannot load plugin `%s`\n%s', plugin, e)
+                        continue
+            plugins.append(plugin)
+    else:
+        plugins = list(namespace_plugins.values())
+
+    return plugins

pelican/plugins/signals.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from __future__ import print_function, unicode_literals
 
 from blinker import signal
 

pelican/readers.py

@@ -16,9 +16,9 @@ from docutils.parsers.rst.languages import get_language as get_docutils_lang
 from docutils.writers.html4css1 import HTMLTranslator, Writer
 
 from pelican import rstdirectives  # NOQA
-from pelican import signals
 from pelican.cache import FileStampDataCacher
 from pelican.contents import Author, Category, Page, Tag
+from pelican.plugins import signals
 from pelican.utils import get_date, pelican_open, posixize_path
 
 try:

pelican/settings.py

@@ -138,7 +138,7 @@ DEFAULT_CONFIG = {
     'TYPOGRIFY_IGNORE_TAGS': [],
     'SUMMARY_MAX_LENGTH': 50,
     'PLUGIN_PATHS': [],
-    'PLUGINS': [],
+    'PLUGINS': None,
     'PYGMENTS_RST_OPTIONS': {},
     'TEMPLATE_PAGES': {},
     'TEMPLATE_EXTENSIONS': ['.html'],

pelican/tests/dummy_plugins/namespace_plugin/pelican/plugins/ns_plugin/__init__.py

@@ -0,0 +1,5 @@
+NAME = 'namespace plugin'
+
+
+def register():
+    pass

pelican/tests/dummy_plugins/normal_plugin/normal_plugin/__init__.py

@@ -0,0 +1,5 @@
+NAME = 'normal plugin'
+
+
+def register():
+    pass

pelican/tests/test_contents.py

@@ -10,8 +10,8 @@ from sys import platform
 from jinja2.utils import generate_lorem_ipsum
 
 from pelican.contents import Article, Author, Category, Page, Static
+from pelican.plugins.signals import content_object_init
 from pelican.settings import DEFAULT_CONFIG
-from pelican.signals import content_object_init
 from pelican.tests.support import (LoggedTestCase, get_context, get_settings,
                                    unittest)
 from pelican.utils import (path_to_url, posixize_path, truncate_html_words)

pelican/tests/test_plugins.py

@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function, unicode_literals
+
+import os
+from contextlib import contextmanager
+
+from pelican.plugins._utils import get_namespace_plugins, load_plugins
+from pelican.tests.support import unittest
+
+
+@contextmanager
+def tmp_namespace_path(path):
+    '''Context manager for temporarily appending namespace plugin packages
+
+    path: path containing the `pelican` folder
+
+    This modifies the `pelican.__path__` and lets the `pelican.plugins`
+    namespace package resolve it from that.
+    '''
+    # This avoids calls to internal `pelican.plugins.__path__._recalculate()`
+    # as it should not be necessary
+    import pelican
+
+    old_path = pelican.__path__[:]
+    try:
+        pelican.__path__.append(os.path.join(path, 'pelican'))
+        yield
+    finally:
+        pelican.__path__ = old_path
+
+
+class PluginTest(unittest.TestCase):
+    _PLUGIN_FOLDER = os.path.join(
+        os.path.abspath(os.path.dirname(__file__)),
+        'dummy_plugins')
+    _NS_PLUGIN_FOLDER = os.path.join(_PLUGIN_FOLDER, 'namespace_plugin')
+    _NORMAL_PLUGIN_FOLDER = os.path.join(_PLUGIN_FOLDER, 'normal_plugin')
+
+    def test_namespace_path_modification(self):
+        import pelican
+        import pelican.plugins
+        old_path = pelican.__path__[:]
+
+        # not existing path
+        path = os.path.join(self._PLUGIN_FOLDER, 'foo')
+        with tmp_namespace_path(path):
+            self.assertIn(
+                os.path.join(path, 'pelican'),
+                pelican.__path__)
+            # foo/pelican does not exist, so it won't propagate
+            self.assertNotIn(
+                os.path.join(path, 'pelican', 'plugins'),
+                pelican.plugins.__path__)
+        # verify that we restored path back
+        self.assertEqual(pelican.__path__, old_path)
+
+        # existing path
+        with tmp_namespace_path(self._NS_PLUGIN_FOLDER):
+            self.assertIn(
+                os.path.join(self._NS_PLUGIN_FOLDER, 'pelican'),
+                pelican.__path__)
+            # /namespace_plugin/pelican exists, so it should be in
+            self.assertIn(
+                os.path.join(self._NS_PLUGIN_FOLDER, 'pelican', 'plugins'),
+                pelican.plugins.__path__)
+        self.assertEqual(pelican.__path__, old_path)
+
+    def test_get_namespace_plugins(self):
+        # without plugins
+        ns_plugins = get_namespace_plugins()
+        self.assertEqual(len(ns_plugins), 0)
+
+        # with plugin
+        with tmp_namespace_path(self._NS_PLUGIN_FOLDER):
+            ns_plugins = get_namespace_plugins()
+            self.assertEqual(len(ns_plugins), 1)
+            self.assertIn('pelican.plugins.ns_plugin', ns_plugins)
+            self.assertEqual(
+                ns_plugins['pelican.plugins.ns_plugin'].NAME,
+                'namespace plugin')
+
+        # should be back to 0 outside `with`
+        ns_plugins = get_namespace_plugins()
+        self.assertEqual(len(ns_plugins), 0)
+
+    def test_load_plugins(self):
+        # no plugins
+        plugins = load_plugins({})
+        self.assertEqual(len(plugins), 0)
+
+        with tmp_namespace_path(self._NS_PLUGIN_FOLDER):
+            # with no `PLUGINS` setting, load namespace plugins
+            plugins = load_plugins({})
+            self.assertEqual(len(plugins), 1, plugins)
+            self.assertEqual(
+                {'namespace plugin'},
+                set(plugin.NAME for plugin in plugins))
+
+            # disable namespace plugins with `PLUGINS = []`
+            SETTINGS = {
+                'PLUGINS': []
+            }
+            plugins = load_plugins(SETTINGS)
+            self.assertEqual(len(plugins), 0, plugins)
+
+            # using `PLUGINS`
+
+            # normal plugin
+            SETTINGS = {
+                'PLUGINS': ['normal_plugin'],
+                'PLUGIN_PATHS': [self._NORMAL_PLUGIN_FOLDER]
+            }
+            plugins = load_plugins(SETTINGS)
+            self.assertEqual(len(plugins), 1, plugins)
+            self.assertEqual(
+                {'normal plugin'},
+                set(plugin.NAME for plugin in plugins))
+
+            # namespace plugin short
+            SETTINGS = {
+                'PLUGINS': ['ns_plugin']
+            }
+            plugins = load_plugins(SETTINGS)
+            self.assertEqual(len(plugins), 1, plugins)
+            self.assertEqual(
+                {'namespace plugin'},
+                set(plugin.NAME for plugin in plugins))
+
+            # namespace plugin long
+            SETTINGS = {
+                'PLUGINS': ['pelican.plugins.ns_plugin']
+            }
+            plugins = load_plugins(SETTINGS)
+            self.assertEqual(len(plugins), 1, plugins)
+            self.assertEqual(
+                {'namespace plugin'},
+                set(plugin.NAME for plugin in plugins))
+
+            # normal and namespace plugin
+            SETTINGS = {
+                'PLUGINS': ['normal_plugin', 'ns_plugin'],
+                'PLUGIN_PATHS': [self._NORMAL_PLUGIN_FOLDER]
+            }
+            plugins = load_plugins(SETTINGS)
+            self.assertEqual(len(plugins), 2, plugins)
+            self.assertEqual(
+                {'normal plugin', 'namespace plugin'},
+                set(plugin.NAME for plugin in plugins))

pelican/writers.py

@@ -8,8 +8,8 @@ from feedgenerator import Atom1Feed, Rss201rev2Feed, get_tag_uri
 
 from jinja2 import Markup
 
-from pelican import signals
 from pelican.paginator import Paginator
+from pelican.plugins import signals
 from pelican.utils import (get_relative_path, is_selected_for_writing,
                            path_to_url, sanitised_join, set_date_tzinfo)
 

setup.py

@@ -18,7 +18,8 @@ entry_points = {
         'pelican = pelican.__main__:main',
         'pelican-import = pelican.tools.pelican_import:main',
         'pelican-quickstart = pelican.tools.pelican_quickstart:main',
-        'pelican-themes = pelican.tools.pelican_themes:main'
+        'pelican-themes = pelican.tools.pelican_themes:main',
+        'pelican-plugins = pelican.plugins._utils:list_plugins'
     ]
 }
 
@@ -44,7 +45,7 @@ setup(
     keywords='static web site generator SSG reStructuredText Markdown',
     license='AGPLv3',
     long_description=description,
-    packages=['pelican', 'pelican.tools'],
+    packages=['pelican', 'pelican.tools', 'pelican.plugins'],
     package_data={
         # we manually collect the package data, as opposed to using,
         # include_package_data=True because we don't want the tests to be