diff --git a/docs/plugins.rst b/docs/plugins.rst index 5a6f9e55..4d44eea7 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -9,16 +9,31 @@ features to Pelican without having to directly modify the Pelican core. How to use plugins ================== -To load plugins, you have to specify them in your settings file. There are two -ways to do so. The first method is to specify strings with the path to the -callables:: +Starting with version 5.0, Pelican moved to a new plugin structure utilizing +namespace packages. Plugins supporting this structure will install under the +namespace package ``pelican.plugins`` and can be automatically discovered +by Pelican. - PLUGINS = ['package.myplugin',] +If you leave the ``PLUGINS`` setting as default (``None``), Pelican will then +collect the namespace plugins and register them. If on the other hand you +specify a ``PLUGINS`` settings as a list of plugins, this autodiscovery will +be disabled and only listed plugins will be registered and you will have to +explicitly list the namespace plugins as well. -Alternatively, another method is to import them and add them to the list:: +If you are using ``PLUGINS`` setting, you can specify plugins in two ways. +The first method specifies plugins as a list of strings. Namespace plugins can +be specified either by their full names (``pelican.plugins.myplugin``) or by +their short names (``myplugin``):: + + PLUGINS = ['package.myplugin', + 'namespace_plugin1', + 'pelican.plugins.namespace_plugin2'] + +Alternatively, you can import them in your settings file and pass the modules:: from package import myplugin - PLUGINS = [myplugin,] + from pelican.plugins import namespace_plugin1, namespace_plugin2 + PLUGINS = [myplugin, namespace_plugin1, namespace_plugin2] .. note:: @@ -36,11 +51,13 @@ the ``PLUGIN_PATHS`` list can be absolute or relative to the settings file:: Where to find plugins ===================== +Namespace plugins can be found in the `pelican-plugins organization`_ as +individual repositories. Legacy plugins are collected in the `pelican-plugins +repository`_ and they will be slowly phased out in favor of the namespace +versions. -We maintain a separate repository of plugins for people to share and use. -Please visit the `pelican-plugins`_ repository for a list of available plugins. - -.. _pelican-plugins: https://github.com/getpelican/pelican-plugins +.. _pelican-plugins organization: https://github.com/pelican-plugins +.. _pelican-plugins repository: https://github.com/getpelican/pelican-plugins Please note that while we do our best to review and maintain these plugins, they are submitted by the Pelican community and thus may have varying levels of @@ -70,6 +87,33 @@ which you map the signals to your plugin logic. Let's take a simple example:: your ``register`` callable or they will be garbage-collected before the signal is emitted. +Namespace plugin structure +-------------------------- + +Namespace plugins must adhere to a certain structure in order to function +properly. They need to be installable (i.e. contain ``setup.py`` or equivalent) +and have a folder structure as follows:: + + myplugin + ├── pelican + │   └── plugins + │   └── myplugin + │   ├── __init__.py + │   └── ... + ├── ... + └── setup.py + +It is crucial that ``pelican`` or ``pelican/plugins`` folder **not** +contain an ``__init__.py`` file. In fact, it is best to have those folders +empty besides the listed folders in the above structure and keep your +plugin related files contained solely in the ``pelican/plugins/myplugin`` +folder to avoid any issues. + +For easily setting up the proper structure, a `cookiecutter template for +plugins`_ is provided. Refer to the README in the link for how to use it. + +.. _cookiecutter template for plugins: https://github.com/getpelican/cookiecutter-pelican-plugin + List of signals =============== diff --git a/docs/settings.rst b/docs/settings.rst index 5d090a62..39982315 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -194,7 +194,7 @@ Basic settings Controls the extension that will be used by the SourcesGenerator. Defaults to ``.text``. If not a valid string the default value will be used. -.. data:: PLUGINS = [] +.. data:: PLUGINS = None The list of plugins to load. See :ref:`plugins`. diff --git a/pelican/__init__.py b/pelican/__init__.py index 83160684..17f4f922 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -9,14 +9,20 @@ import sys import time import traceback from collections.abc import Iterable +# Combines all paths to `pelican` package accessible from `sys.path` +# Makes it possible to install `pelican` and namespace plugins into different +# locations in the file system (e.g. pip with `-e` or `--user`) +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) # pelican.log has to be the first pelican module to be loaded # because logging.setLoggerClass has to be called before logging.getLogger from pelican.log import init as init_logging -from pelican import signals # noqa -from pelican.generators import (ArticlesGenerator, PagesGenerator, - SourceFileGenerator, StaticGenerator, - TemplatePagesGenerator) +from pelican.generators import (ArticlesGenerator, # noqa: I100 + PagesGenerator, SourceFileGenerator, + StaticGenerator, TemplatePagesGenerator) +from pelican.plugins import signals +from pelican.plugins._utils import load_plugins from pelican.readers import Readers from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer from pelican.settings import read_settings @@ -62,27 +68,14 @@ class Pelican(object): sys.path.insert(0, '') def init_plugins(self): - self.plugins = [] - logger.debug('Temporarily adding PLUGIN_PATHS to system path') - _sys_path = sys.path[:] - for pluginpath in self.settings['PLUGIN_PATHS']: - sys.path.insert(0, pluginpath) - for plugin in self.settings['PLUGINS']: - # if it's a string, then import it - if isinstance(plugin, str): - logger.debug("Loading plugin `%s`", plugin) - try: - plugin = __import__(plugin, globals(), locals(), 'module') - except ImportError as e: - logger.error( - "Cannot load plugin `%s`\n%s", plugin, e) - continue - - logger.debug("Registering plugin `%s`", plugin.__name__) - plugin.register() - self.plugins.append(plugin) - logger.debug('Restoring system path') - sys.path = _sys_path + self.plugins = load_plugins(self.settings) + for plugin in self.plugins: + logger.debug('Registering plugin `%s`', plugin.__name__) + try: + plugin.register() + except Exception as e: + logger.error('Cannot register plugin `%s`\n%s', + plugin.__name__, e) def run(self): """Run the generators and return""" diff --git a/pelican/contents.py b/pelican/contents.py index 6edf5152..594cd3b5 100644 --- a/pelican/contents.py +++ b/pelican/contents.py @@ -10,7 +10,7 @@ from urllib.parse import urljoin, urlparse, urlunparse import pytz -from pelican import signals +from pelican.plugins import signals from pelican.settings import DEFAULT_CONFIG from pelican.utils import (deprecated_attribute, memoized, path_to_url, posixize_path, sanitised_join, set_date_tzinfo, diff --git a/pelican/generators.py b/pelican/generators.py index 8bd2656f..02667cd7 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -13,9 +13,9 @@ from operator import attrgetter from jinja2 import (BaseLoader, ChoiceLoader, Environment, FileSystemLoader, PrefixLoader, TemplateNotFound) -from pelican import signals from pelican.cache import FileStampDataCacher from pelican.contents import Article, Page, Static +from pelican.plugins import signals from pelican.readers import Readers from pelican.utils import (DateFormatter, copy, mkdir_p, order_content, posixize_path, process_translations) diff --git a/pelican/plugins/_utils.py b/pelican/plugins/_utils.py new file mode 100644 index 00000000..a3c9ee77 --- /dev/null +++ b/pelican/plugins/_utils.py @@ -0,0 +1,85 @@ +import importlib +import importlib.machinery +import importlib.util +import logging +import pkgutil + + +logger = logging.getLogger(__name__) + + +def iter_namespace(ns_pkg): + # Specifying the second argument (prefix) to iter_modules makes the + # returned name an absolute name instead of a relative one. This allows + # import_module to work without having to do additional modification to + # the name. + return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".") + + +def get_namespace_plugins(ns_pkg=None): + if ns_pkg is None: + import pelican.plugins as ns_pkg + + return { + name: importlib.import_module(name) + for finder, name, ispkg + in iter_namespace(ns_pkg) + if ispkg + } + + +def list_plugins(ns_pkg=None): + from pelican.log import init as init_logging + init_logging(logging.INFO) + ns_plugins = get_namespace_plugins(ns_pkg) + if ns_plugins: + logger.info('Plugins found:\n' + '\n'.join(ns_plugins)) + else: + logger.info('No plugins are installed') + + +def load_legacy_plugin(plugin, plugin_paths): + # Try to find plugin in PLUGIN_PATHS + spec = importlib.machinery.PathFinder.find_spec(plugin, plugin_paths) + if spec is None: + # If failed, try to find it in normal importable locations + spec = importlib.util.find_spec(plugin) + if spec is None: + raise ImportError('Cannot import plugin `{}`'.format(plugin)) + else: + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def load_plugins(settings): + logger.debug('Finding namespace plugins') + namespace_plugins = get_namespace_plugins() + if namespace_plugins: + logger.debug('Namespace plugins found:\n' + + '\n'.join(namespace_plugins)) + plugins = [] + if settings.get('PLUGINS') is not None: + for plugin in settings['PLUGINS']: + if isinstance(plugin, str): + logger.debug('Loading plugin `%s`', plugin) + # try to find in namespace plugins + if plugin in namespace_plugins: + plugin = namespace_plugins[plugin] + elif 'pelican.plugins.{}'.format(plugin) in namespace_plugins: + plugin = namespace_plugins['pelican.plugins.{}'.format( + plugin)] + # try to import it + else: + try: + plugin = load_legacy_plugin( + plugin, + settings.get('PLUGIN_PATHS', [])) + except ImportError as e: + logger.error('Cannot load plugin `%s`\n%s', plugin, e) + continue + plugins.append(plugin) + else: + plugins = list(namespace_plugins.values()) + + return plugins diff --git a/pelican/signals.py b/pelican/plugins/signals.py similarity index 96% rename from pelican/signals.py rename to pelican/plugins/signals.py index 253b5fc3..18a745b4 100644 --- a/pelican/signals.py +++ b/pelican/plugins/signals.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import print_function, unicode_literals from blinker import signal diff --git a/pelican/readers.py b/pelican/readers.py index b26bd381..673b637e 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -16,9 +16,9 @@ from docutils.parsers.rst.languages import get_language as get_docutils_lang from docutils.writers.html4css1 import HTMLTranslator, Writer from pelican import rstdirectives # NOQA -from pelican import signals from pelican.cache import FileStampDataCacher from pelican.contents import Author, Category, Page, Tag +from pelican.plugins import signals from pelican.utils import get_date, pelican_open, posixize_path try: diff --git a/pelican/settings.py b/pelican/settings.py index a4033001..0cdcefc7 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -138,7 +138,7 @@ DEFAULT_CONFIG = { 'TYPOGRIFY_IGNORE_TAGS': [], 'SUMMARY_MAX_LENGTH': 50, 'PLUGIN_PATHS': [], - 'PLUGINS': [], + 'PLUGINS': None, 'PYGMENTS_RST_OPTIONS': {}, 'TEMPLATE_PAGES': {}, 'TEMPLATE_EXTENSIONS': ['.html'], diff --git a/pelican/tests/dummy_plugins/namespace_plugin/pelican/plugins/ns_plugin/__init__.py b/pelican/tests/dummy_plugins/namespace_plugin/pelican/plugins/ns_plugin/__init__.py new file mode 100644 index 00000000..c514861d --- /dev/null +++ b/pelican/tests/dummy_plugins/namespace_plugin/pelican/plugins/ns_plugin/__init__.py @@ -0,0 +1,5 @@ +NAME = 'namespace plugin' + + +def register(): + pass diff --git a/pelican/tests/dummy_plugins/normal_plugin/normal_plugin/__init__.py b/pelican/tests/dummy_plugins/normal_plugin/normal_plugin/__init__.py new file mode 100644 index 00000000..15896087 --- /dev/null +++ b/pelican/tests/dummy_plugins/normal_plugin/normal_plugin/__init__.py @@ -0,0 +1,5 @@ +NAME = 'normal plugin' + + +def register(): + pass diff --git a/pelican/tests/test_contents.py b/pelican/tests/test_contents.py index 256f08bb..62608b7b 100644 --- a/pelican/tests/test_contents.py +++ b/pelican/tests/test_contents.py @@ -10,8 +10,8 @@ from sys import platform from jinja2.utils import generate_lorem_ipsum from pelican.contents import Article, Author, Category, Page, Static +from pelican.plugins.signals import content_object_init from pelican.settings import DEFAULT_CONFIG -from pelican.signals import content_object_init from pelican.tests.support import (LoggedTestCase, get_context, get_settings, unittest) from pelican.utils import (path_to_url, posixize_path, truncate_html_words) diff --git a/pelican/tests/test_plugins.py b/pelican/tests/test_plugins.py new file mode 100644 index 00000000..1adbbb5c --- /dev/null +++ b/pelican/tests/test_plugins.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function, unicode_literals + +import os +from contextlib import contextmanager + +from pelican.plugins._utils import get_namespace_plugins, load_plugins +from pelican.tests.support import unittest + + +@contextmanager +def tmp_namespace_path(path): + '''Context manager for temporarily appending namespace plugin packages + + path: path containing the `pelican` folder + + This modifies the `pelican.__path__` and lets the `pelican.plugins` + namespace package resolve it from that. + ''' + # This avoids calls to internal `pelican.plugins.__path__._recalculate()` + # as it should not be necessary + import pelican + + old_path = pelican.__path__[:] + try: + pelican.__path__.append(os.path.join(path, 'pelican')) + yield + finally: + pelican.__path__ = old_path + + +class PluginTest(unittest.TestCase): + _PLUGIN_FOLDER = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + 'dummy_plugins') + _NS_PLUGIN_FOLDER = os.path.join(_PLUGIN_FOLDER, 'namespace_plugin') + _NORMAL_PLUGIN_FOLDER = os.path.join(_PLUGIN_FOLDER, 'normal_plugin') + + def test_namespace_path_modification(self): + import pelican + import pelican.plugins + old_path = pelican.__path__[:] + + # not existing path + path = os.path.join(self._PLUGIN_FOLDER, 'foo') + with tmp_namespace_path(path): + self.assertIn( + os.path.join(path, 'pelican'), + pelican.__path__) + # foo/pelican does not exist, so it won't propagate + self.assertNotIn( + os.path.join(path, 'pelican', 'plugins'), + pelican.plugins.__path__) + # verify that we restored path back + self.assertEqual(pelican.__path__, old_path) + + # existing path + with tmp_namespace_path(self._NS_PLUGIN_FOLDER): + self.assertIn( + os.path.join(self._NS_PLUGIN_FOLDER, 'pelican'), + pelican.__path__) + # /namespace_plugin/pelican exists, so it should be in + self.assertIn( + os.path.join(self._NS_PLUGIN_FOLDER, 'pelican', 'plugins'), + pelican.plugins.__path__) + self.assertEqual(pelican.__path__, old_path) + + def test_get_namespace_plugins(self): + # without plugins + ns_plugins = get_namespace_plugins() + self.assertEqual(len(ns_plugins), 0) + + # with plugin + with tmp_namespace_path(self._NS_PLUGIN_FOLDER): + ns_plugins = get_namespace_plugins() + self.assertEqual(len(ns_plugins), 1) + self.assertIn('pelican.plugins.ns_plugin', ns_plugins) + self.assertEqual( + ns_plugins['pelican.plugins.ns_plugin'].NAME, + 'namespace plugin') + + # should be back to 0 outside `with` + ns_plugins = get_namespace_plugins() + self.assertEqual(len(ns_plugins), 0) + + def test_load_plugins(self): + # no plugins + plugins = load_plugins({}) + self.assertEqual(len(plugins), 0) + + with tmp_namespace_path(self._NS_PLUGIN_FOLDER): + # with no `PLUGINS` setting, load namespace plugins + plugins = load_plugins({}) + self.assertEqual(len(plugins), 1, plugins) + self.assertEqual( + {'namespace plugin'}, + set(plugin.NAME for plugin in plugins)) + + # disable namespace plugins with `PLUGINS = []` + SETTINGS = { + 'PLUGINS': [] + } + plugins = load_plugins(SETTINGS) + self.assertEqual(len(plugins), 0, plugins) + + # using `PLUGINS` + + # normal plugin + SETTINGS = { + 'PLUGINS': ['normal_plugin'], + 'PLUGIN_PATHS': [self._NORMAL_PLUGIN_FOLDER] + } + plugins = load_plugins(SETTINGS) + self.assertEqual(len(plugins), 1, plugins) + self.assertEqual( + {'normal plugin'}, + set(plugin.NAME for plugin in plugins)) + + # namespace plugin short + SETTINGS = { + 'PLUGINS': ['ns_plugin'] + } + plugins = load_plugins(SETTINGS) + self.assertEqual(len(plugins), 1, plugins) + self.assertEqual( + {'namespace plugin'}, + set(plugin.NAME for plugin in plugins)) + + # namespace plugin long + SETTINGS = { + 'PLUGINS': ['pelican.plugins.ns_plugin'] + } + plugins = load_plugins(SETTINGS) + self.assertEqual(len(plugins), 1, plugins) + self.assertEqual( + {'namespace plugin'}, + set(plugin.NAME for plugin in plugins)) + + # normal and namespace plugin + SETTINGS = { + 'PLUGINS': ['normal_plugin', 'ns_plugin'], + 'PLUGIN_PATHS': [self._NORMAL_PLUGIN_FOLDER] + } + plugins = load_plugins(SETTINGS) + self.assertEqual(len(plugins), 2, plugins) + self.assertEqual( + {'normal plugin', 'namespace plugin'}, + set(plugin.NAME for plugin in plugins)) diff --git a/pelican/writers.py b/pelican/writers.py index daeb9dec..7bbd216e 100644 --- a/pelican/writers.py +++ b/pelican/writers.py @@ -8,8 +8,8 @@ from feedgenerator import Atom1Feed, Rss201rev2Feed, get_tag_uri from jinja2 import Markup -from pelican import signals from pelican.paginator import Paginator +from pelican.plugins import signals from pelican.utils import (get_relative_path, is_selected_for_writing, path_to_url, sanitised_join, set_date_tzinfo) diff --git a/setup.py b/setup.py index 7c583da2..dc1cc527 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,8 @@ entry_points = { 'pelican = pelican.__main__:main', 'pelican-import = pelican.tools.pelican_import:main', 'pelican-quickstart = pelican.tools.pelican_quickstart:main', - 'pelican-themes = pelican.tools.pelican_themes:main' + 'pelican-themes = pelican.tools.pelican_themes:main', + 'pelican-plugins = pelican.plugins._utils:list_plugins' ] } @@ -44,7 +45,7 @@ setup( keywords='static web site generator SSG reStructuredText Markdown', license='AGPLv3', long_description=description, - packages=['pelican', 'pelican.tools'], + packages=['pelican', 'pelican.tools', 'pelican.plugins'], package_data={ # we manually collect the package data, as opposed to using, # include_package_data=True because we don't want the tests to be