Merge pull request #795 from wking/read-file

Generate context objects in read_file()

docs/getting_started.rst

@@ -213,6 +213,8 @@ The idea behind "pages" is that they are usually not temporal in nature and are
 used for content that does not change very often (e.g., "About" or "Contact"
 pages).
 
+.. _internal_metadata:
+
 File metadata
 -------------
 

docs/plugins.rst

@@ -79,9 +79,9 @@ article_generator_finalized     article_generator               invoked at the e
 get_generators                  generators                      invoked in Pelican.get_generator_classes,
                                                                 can return a Generator, or several
                                                                 generator in a tuple or in a list.
-pages_generate_context          pages_generator, metadata
+page_generate_context           page_generator, metadata
-pages_generator_init            pages_generator                 invoked in the PagesGenerator.__init__
+page_generator_init             page_generator                  invoked in the PagesGenerator.__init__
-pages_generator_finalized       pages_generator                 invoked at the end of PagesGenerator.generate_context
+page_generator_finalized        page_generator                  invoked at the end of PagesGenerator.generate_context
 content_object_init             content_object                  invoked at the end of Content.__init__ (see note below)
 =============================   ============================   ===========================================================================
 
@@ -104,3 +104,22 @@ request if you need them!
 
        def register():
                signals.content_object_init.connect(test, sender=contents.Article)
+
+.. note::
+
+   After Pelican 3.2, signal names were standardized.  Older plugins
+   may need to be updated to use the new names:
+
+   ==========================  ===========================
+   Old name                    New name
+   ==========================  ===========================
+   article_generate_context    article_generator_context
+   article_generate_finalized  article_generator_finalized
+   article_generate_preread    article_generator_preread
+   pages_generate_context      page_generator_context
+   pages_generate_preread      page_generator_preread
+   pages_generator_finalized   page_generator_finalized
+   pages_generator_init        page_generator_init
+   static_generate_context     static_generator_context
+   static_generate_preread     static_generator_preread
+   ==========================  ===========================

docs/settings.rst

@@ -62,15 +62,16 @@ Setting name (default value)                                            What doe
                                                                         For example, if you would like to extract both the
                                                                         date and the slug, you could set something like:
                                                                         ``'(?P<date>\d{4}-\d{2}-\d{2})_(?P<slug>.*)'``.
+                                                                        See :ref:`path_metadata`.
 `PATH_METADATA` (``''``)                                                Like ``FILENAME_METADATA``, but parsed from a page's
                                                                         full path relative to the content source directory.
+                                                                        See :ref:`path_metadata`.
+`EXTRA_PATH_METADATA` (``{}``)                                          Extra metadata dictionaries keyed by relative path.
+                                                                        See :ref:`path_metadata`.
 `DELETE_OUTPUT_DIRECTORY` (``False``)                                   Delete the output directory, and **all** of its contents, before
                                                                         generating new files. This can be useful in preventing older,
                                                                         unnecessary files from persisting in your output. However, **this is
                                                                         a destructive setting and should be handled with extreme care.**
-`FILES_TO_COPY` (``()``)                                                A list of files (or directories) to copy from the source (inside the
-                                                                        content directory) to the destination (inside the output directory).
-                                                                        For example: ``(('extra/robots.txt', 'robots.txt'),)``.
 `JINJA_EXTENSIONS` (``[]``)                                             A list of any Jinja2 extensions you want to use.
 `JINJA_FILTERS` (``{}``)                                                A list of custom Jinja2 filters you want to use.
                                                                         The dictionary should map the filtername to the filter function.
@@ -337,6 +338,52 @@ your resume, and a contact page — you could have::
                       'src/resume.html': 'dest/resume.html',
                       'src/contact.html': 'dest/contact.html'}
 
+
+.. _path_metadata:
+
+Path metadata
+=============
+
+Not all metadata needs to be `embedded in source file itself`__.  For
+example, blog posts are often named following a ``YYYY-MM-DD-SLUG.rst``
+pattern, or nested into ``YYYY/MM/DD-SLUG`` directories.  To extract
+metadata from the filename or path, set ``FILENAME_METADATA`` or
+``PATH_METADATA`` to regular expressions that use Python's `group name
+notation`_ ``(?P<name>…)``.  If you want to attach additional metadata
+but don't want to encode it in the path, you can set
+``EXTRA_PATH_METADATA``:
+
+.. parsed-literal::
+
+    EXTRA_PATH_METADATA = {
+        'relative/path/to/file-1': {
+            'key-1a': 'value-1a',
+            'key-1b': 'value-1b',
+            },
+        'relative/path/to/file-2': {
+            'key-2': 'value-2',
+            },
+        }
+
+This can be a convenient way to shift the installed location of a
+particular file:
+
+.. parsed-literal::
+
+    # Take advantage of the following defaults
+    # STATIC_SAVE_AS = '{path}'
+    # STATIC_URL = '{path}'
+    STATIC_PATHS = [
+        'extra/robots.txt',
+        ]
+    EXTRA_PATH_METADATA = {
+        'extra/robots.txt': {'path': 'robots.txt'},
+        }
+
+__ internal_metadata__
+.. _group name notation:
+   http://docs.python.org/3/library/re.html#regular-expression-syntax
+
 Feed settings
 =============
 

pelican/contents.py

@@ -48,6 +48,8 @@ class Content(object):
 
         self.settings = settings
         self._content = content
+        if context is None:
+            context = {}
         self._context = context
         self.translations = []
 
@@ -169,6 +171,9 @@ class Content(object):
         :param siteurl: siteurl which is locally generated by the writer in
                         case of RELATIVE_URLS.
         """
+        if not content:
+            return content
+
         hrefs = re.compile(r"""
             (?P<markup><\s*[^\>]*  # match tag with src and href attr
                 (?:href|src)\s*=)
@@ -220,7 +225,7 @@ class Content(object):
 
     @property
     def content(self):
-        return self.get_content(self._context['localsiteurl'])
+        return self.get_content(self._context.get('localsiteurl', ''))
 
     def _get_summary(self):
         """Returns the summary of an article.

pelican/generators.py

@@ -5,7 +5,6 @@ import os
 import math
 import random
 import logging
-import datetime
 import shutil
 
 from codecs import open
@@ -19,9 +18,7 @@ from jinja2 import (
         TemplateNotFound
 )
 
-from pelican.contents import (
+from pelican.contents import Article, Page, Static, is_valid_content
-        Article, Page, Category, Static, is_valid_content
-)
 from pelican.readers import read_file
 from pelican.utils import copy, process_translations, mkdir_p, DateFormatter
 from pelican import signals
@@ -105,23 +102,25 @@ class Generator(object):
     def get_files(self, path, exclude=[], extensions=None):
         """Return a list of files to use, based on rules
 
-        :param path: the path to search the file on
+        :param path: the path to search (relative to self.path)
         :param exclude: the list of path to exclude
         :param extensions: the list of allowed extensions (if False, all
             extensions are allowed)
         """
         files = []
+        root = os.path.join(self.path, path)
 
-        if os.path.isdir(path):
+        if os.path.isdir(root):
-            for root, dirs, temp_files in os.walk(path, followlinks=True):
+            for dirpath, dirs, temp_files in os.walk(root, followlinks=True):
                 for e in exclude:
                     if e in dirs:
                         dirs.remove(e)
+                reldir = os.path.relpath(dirpath, self.path)
                 for f in temp_files:
-                    fp = os.path.join(root, f)
+                    fp = os.path.join(reldir, f)
                     if self._include_path(fp, extensions):
                         files.append(fp)
-        elif os.path.exists(path) and self._include_path(path, extensions):
+        elif os.path.exists(root) and self._include_path(path, extensions):
             files.append(path)  # can't walk non-directories
         return files
 
@@ -375,45 +374,22 @@ class ArticlesGenerator(Generator):
     def generate_context(self):
         """Add the articles into the shared context"""
 
-        article_path = os.path.normpath(  # we have to remove trailing slashes
-            os.path.join(self.path, self.settings['ARTICLE_DIR'])
-        )
         all_articles = []
         for f in self.get_files(
-                article_path,
+                self.settings['ARTICLE_DIR'],
                 exclude=self.settings['ARTICLE_EXCLUDES']):
             try:
-                signals.article_generate_preread.send(self)
+                article = read_file(
-                content, metadata = read_file(f, settings=self.settings)
+                    base_path=self.path, path=f, content_class=Article,
+                    settings=self.settings, context=self.context,
+                    preread_signal=signals.article_generator_preread,
+                    preread_sender=self,
+                    context_signal=signals.article_generator_context,
+                    context_sender=self)
             except Exception as e:
-                logger.warning('Could not process %s\n%s' % (f, str(e)))
+                logger.warning('Could not process {}\n{}'.format(f, e))
                 continue
 
-            # if no category is set, use the name of the path as a category
-            if 'category' not in metadata:
-
-                if (self.settings['USE_FOLDER_AS_CATEGORY']
-                    and os.path.dirname(f) != article_path):
-                    # if the article is in a subdirectory
-                    category = os.path.basename(os.path.dirname(f))
-                else:
-                    # if the article is not in a subdirectory
-                    category = self.settings['DEFAULT_CATEGORY']
-
-                if category != '':
-                    metadata['category'] = Category(category, self.settings)
-
-            if 'date' not in metadata and self.settings.get('DEFAULT_DATE'):
-                if self.settings['DEFAULT_DATE'] == 'fs':
-                    metadata['date'] = datetime.datetime.fromtimestamp(
-                            os.stat(f).st_ctime)
-                else:
-                    metadata['date'] = datetime.datetime(
-                            *self.settings['DEFAULT_DATE'])
-
-            signals.article_generate_context.send(self, metadata=metadata)
-            article = Article(content, metadata, settings=self.settings,
-                              source_path=f, context=self.context)
             if not is_valid_content(article, f):
                 continue
 
@@ -502,22 +478,26 @@ class PagesGenerator(Generator):
         self.hidden_pages = []
         self.hidden_translations = []
         super(PagesGenerator, self).__init__(*args, **kwargs)
-        signals.pages_generator_init.send(self)
+        signals.page_generator_init.send(self)
 
     def generate_context(self):
         all_pages = []
         hidden_pages = []
         for f in self.get_files(
-                os.path.join(self.path, self.settings['PAGE_DIR']),
+                self.settings['PAGE_DIR'],
                 exclude=self.settings['PAGE_EXCLUDES']):
             try:
-                content, metadata = read_file(f, settings=self.settings)
+                page = read_file(
+                    base_path=self.path, path=f, content_class=Page,
+                    settings=self.settings, context=self.context,
+                    preread_signal=signals.page_generator_preread,
+                    preread_sender=self,
+                    context_signal=signals.page_generator_context,
+                    context_sender=self)
             except Exception as e:
-                logger.warning('Could not process %s\n%s' % (f, str(e)))
+                logger.warning('Could not process {}\n{}'.format(f, e))
                 continue
-            signals.pages_generate_context.send(self, metadata=metadata)
+
-            page = Page(content, metadata, settings=self.settings,
-                        source_path=f, context=self.context)
             if not is_valid_content(page, f):
                 continue
 
@@ -539,7 +519,7 @@ class PagesGenerator(Generator):
         self._update_context(('pages', ))
         self.context['PAGES'] = self.pages
 
-        signals.pages_generator_finalized.send(self)
+        signals.page_generator_finalized.send(self)
 
     def generate_output(self, writer):
         for page in chain(self.translations, self.pages,
@@ -566,33 +546,17 @@ class StaticGenerator(Generator):
         # walk static paths
         for static_path in self.settings['STATIC_PATHS']:
             for f in self.get_files(
-                    os.path.join(self.path, static_path), extensions=False):
+                    static_path, extensions=False):
-                f_rel = os.path.relpath(f, self.path)
+                static = read_file(
-                content, metadata = read_file(
+                    base_path=self.path, path=f, content_class=Static,
-                    f, fmt='static', settings=self.settings)
+                    fmt='static',
-                # TODO remove this hardcoded 'static' subdirectory
+                    settings=self.settings, context=self.context,
-                metadata['save_as'] = os.path.join('static', f_rel)
+                    preread_signal=signals.static_generator_preread,
-                metadata['url'] = pelican.utils.path_to_url(metadata['save_as'])
+                    preread_sender=self,
-                sc = Static(
+                    context_signal=signals.static_generator_context,
-                    content=None,
+                    context_sender=self)
-                    metadata=metadata,
+                self.staticfiles.append(static)
-                    settings=self.settings,
+                self.add_source_path(static)
-                    source_path=f_rel)
-                self.staticfiles.append(sc)
-                self.add_source_path(sc)
-        # same thing for FILES_TO_COPY
-        for src, dest in self.settings['FILES_TO_COPY']:
-            content, metadata = read_file(
-                src, fmt='static', settings=self.settings)
-            metadata['save_as'] = dest
-            metadata['url'] = pelican.utils.path_to_url(metadata['save_as'])
-            sc = Static(
-                content=None,
-                metadata={'save_as': dest},
-                settings=self.settings,
-                source_path=src)
-            self.staticfiles.append(sc)
-            self.add_source_path(sc)
 
     def generate_output(self, writer):
         self._copy_paths(self.settings['THEME_STATIC_PATHS'], self.theme,

pelican/readers.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, print_function
 
+import datetime
+import logging
 import os
 import re
 try:
@@ -31,10 +33,12 @@ try:
 except ImportError:
     from HTMLParser import HTMLParser
 
-from pelican.contents import Category, Tag, Author
+from pelican.contents import Page, Category, Tag, Author
 from pelican.utils import get_date, pelican_open
 
 
+logger = logging.getLogger(__name__)
+
 METADATA_PROCESSORS = {
     'tags': lambda x, y: [Tag(tag, y) for tag in x.split(',')],
     'date': lambda x, y: get_date(x),
@@ -333,29 +337,48 @@ for cls in [Reader] + Reader.__subclasses__():
         EXTENSIONS[ext] = cls
 
 
-def read_file(path, fmt=None, settings=None):
+def read_file(base_path, path, content_class=Page, fmt=None,
-    """Return a reader object using the given format."""
+              settings=None, context=None,
+              preread_signal=None, preread_sender=None,
+              context_signal=None, context_sender=None):
+    """Return a content object parsed with the given format."""
+    path = os.path.abspath(os.path.join(base_path, path))
+    source_path = os.path.relpath(path, base_path)
     base, ext = os.path.splitext(os.path.basename(path))
+    logger.debug('read file {} -> {}'.format(
+            source_path, content_class.__name__))
     if not fmt:
         fmt = ext[1:]
 
     if fmt not in EXTENSIONS:
         raise TypeError('Pelican does not know how to parse {}'.format(path))
 
+    if preread_signal:
+        logger.debug('signal {}.send({})'.format(
+                preread_signal, preread_sender))
+        preread_signal.send(preread_sender)
+
     if settings is None:
         settings = {}
 
-    reader = EXTENSIONS[fmt](settings)
+    reader_class = EXTENSIONS[fmt]
+    if not reader_class.enabled:
+        raise ValueError('Missing dependencies for {}'.format(fmt))
+
+    reader = reader_class(settings)
+
     settings_key = '%s_EXTENSIONS' % fmt.upper()
 
     if settings and settings_key in settings:
         reader.extensions = settings[settings_key]
 
-    if not reader.enabled:
+    metadata = default_metadata(
-        raise ValueError("Missing dependencies for %s" % fmt)
+        settings=settings, process=reader.process_metadata)
-
+    metadata.update(path_metadata(
-    metadata = parse_path_metadata(
+            full_path=path, source_path=source_path, settings=settings))
-        path=path, settings=settings, process=reader.process_metadata)
+    metadata.update(parse_path_metadata(
+            source_path=source_path, settings=settings,
+            process=reader.process_metadata))
     content, reader_metadata = reader.read(path)
     metadata.update(reader_metadata)
 
@@ -365,9 +388,43 @@ def read_file(path, fmt=None, settings=None):
         content = typogrify(content)
         metadata['title'] = typogrify(metadata['title'])
 
-    return content, metadata
+    if context_signal:
+        logger.debug('signal {}.send({}, <metadata>)'.format(
+                context_signal, context_sender))
+        context_signal.send(context_sender, metadata=metadata)
+    return content_class(
+        content=content,
+        metadata=metadata,
+        settings=settings,
+        source_path=path,
+        context=context)
 
-def parse_path_metadata(path, settings=None, process=None):
+
+def default_metadata(settings=None, process=None):
+    metadata = {}
+    if settings:
+        if 'DEFAULT_CATEGORY' in settings:
+            value = settings['DEFAULT_CATEGORY']
+            if process:
+                value = process('category', value)
+            metadata['category'] = value
+        if 'DEFAULT_DATE' in settings and settings['DEFAULT_DATE'] != 'fs':
+            metadata['date'] = datetime.datetime(*settings['DEFAULT_DATE'])
+    return metadata
+
+
+def path_metadata(full_path, source_path, settings=None):
+    metadata = {}
+    if settings:
+        if settings.get('DEFAULT_DATE', None) == 'fs':
+            metadata['date'] = datetime.datetime.fromtimestamp(
+                os.stat(full_path).st_ctime)
+        metadata.update(settings.get('EXTRA_PATH_METADATA', {}).get(
+                source_path, {}))
+    return metadata
+
+
+def parse_path_metadata(source_path, settings=None, process=None):
     """Extract a metadata dictionary from a file's path
 
     >>> import pprint
@@ -378,7 +435,7 @@ def parse_path_metadata(path, settings=None, process=None):
     ...     }
     >>> reader = Reader(settings=settings)
     >>> metadata = parse_path_metadata(
-    ...     path='my-cat/2013-01-01/my-slug.html',
+    ...     source_path='my-cat/2013-01-01/my-slug.html',
     ...     settings=settings,
     ...     process=reader.process_metadata)
     >>> pprint.pprint(metadata)  # doctest: +ELLIPSIS
@@ -387,13 +444,19 @@ def parse_path_metadata(path, settings=None, process=None):
      'slug': 'my-slug'}
     """
     metadata = {}
-    base, ext = os.path.splitext(os.path.basename(path))
+    dirname, basename = os.path.split(source_path)
+    base, ext = os.path.splitext(basename)
+    subdir = os.path.basename(dirname)
     if settings:
+        checks = []
         for key,data in [('FILENAME_METADATA', base),
-                         ('PATH_METADATA', path),
+                         ('PATH_METADATA', source_path),
                          ]:
-            regexp = settings.get(key)
+            checks.append((settings.get(key, None), data))
-            if regexp:
+        if settings.get('USE_FOLDER_AS_CATEGORY', None):
+            checks.insert(0, ('(?P<category>.*)', subdir))
+        for regexp,data in checks:
+            if regexp and data:
                 match = re.match(regexp, data)
                 if match:
                     # .items() for py3k compat.

pelican/settings.py

@@ -95,7 +95,7 @@ DEFAULT_CONFIG = {
     'DEFAULT_METADATA': (),
     'FILENAME_METADATA': '(?P<date>\d{4}-\d{2}-\d{2}).*',
     'PATH_METADATA': '',
-    'FILES_TO_COPY': (),
+    'EXTRA_PATH_METADATA': {},
     'DEFAULT_STATUS': 'published',
     'ARTICLE_PERMALINK_STRUCTURE': '',
     'TYPOGRIFY': False,
@@ -257,11 +257,14 @@ def configure_settings(settings):
 
     for old,new,doc in [
             ('LESS_GENERATOR', 'the Webassets plugin', None),
+            ('FILES_TO_COPY', 'STATIC_PATHS and EXTRA_PATH_METADATA',
+             'https://github.com/getpelican/pelican/blob/master/docs/settings.rst#path-metadata'),
             ]:
         if old in settings:
-            message = 'The {} setting has been removed in favor of {}'
+            message = 'The {} setting has been removed in favor of {}'.format(
+                old, new)
             if doc:
-                message += ', see {} for details'
+                message += ', see {} for details'.format(doc)
             logger.warning(message)
 
     return settings

pelican/signals.py

@@ -2,15 +2,34 @@
 from __future__ import unicode_literals, print_function
 from blinker import signal
 
+# Run-level signals:
+
 initialized = signal('pelican_initialized')
-finalized = signal('pelican_finalized')
-article_generate_preread = signal('article_generate_preread')
-generator_init = signal('generator_init')
-article_generate_context = signal('article_generate_context')
-article_generator_init = signal('article_generator_init')
-article_generator_finalized = signal('article_generate_finalized')
 get_generators = signal('get_generators')
-pages_generate_context = signal('pages_generate_context')
+finalized = signal('pelican_finalized')
-pages_generator_init = signal('pages_generator_init')
+
-pages_generator_finalized = signal('pages_generator_finalized')
+# Generator-level signals
+
+generator_init = signal('generator_init')
+
+article_generator_init = signal('article_generator_init')
+article_generator_finalized = signal('article_generator_finalized')
+
+page_generator_init = signal('page_generator_init')
+page_generator_finalized = signal('page_generator_finalized')
+
+static_generator_init = signal('static_generator_init')
+static_generator_finalized = signal('static_generator_finalized')
+
+# Page-level signals
+
+article_generator_preread = signal('article_generator_preread')
+article_generator_context = signal('article_generator_context')
+
+page_generator_preread = signal('page_generator_preread')
+page_generator_context = signal('page_generator_context')
+
+static_generator_preread = signal('static_generator_preread')
+static_generator_context = signal('static_generator_context')
+
 content_object_init = signal('content_object_init')

pelican/tests/default_conf.py

@@ -29,11 +29,16 @@ SOCIAL = (('twitter', 'http://twitter.com/ametaireau'),
 # global metadata to all the contents
 DEFAULT_METADATA = (('yeah', 'it is'),)
 
-# static paths will be copied under the same name
+# path-specific metadata
-STATIC_PATHS = ["pictures", ]
+EXTRA_PATH_METADATA = {
+    'extra/robots.txt': {'path': 'robots.txt'},
+    }
 
-# A list of files to copy from the source to the destination
+# static paths will be copied without parsing their contents
-FILES_TO_COPY = (('extra/robots.txt', 'robots.txt'),)
+STATIC_PATHS = [
+    'pictures',
+    'extra/robots.txt',
+    ]
 
 # foobar will not be used, because it's not in caps. All configuration keys
 # have to be in caps

pelican/tests/test_generators.py

@@ -15,6 +15,7 @@ from pelican.settings import DEFAULT_CONFIG
 from pelican.tests.support import unittest, get_settings
 
 CUR_DIR = os.path.dirname(__file__)
+CONTENT_DIR = os.path.join(CUR_DIR, 'content')
 
 
 class TestArticlesGenerator(unittest.TestCase):
@@ -30,12 +31,10 @@ class TestArticlesGenerator(unittest.TestCase):
         """
         if self.generator is None:
             settings = get_settings(filenames={})
-            settings['ARTICLE_DIR'] = 'content'
             settings['DEFAULT_CATEGORY'] = 'Default'
             settings['DEFAULT_DATE'] = (1970, 1, 1)
             self.generator = ArticlesGenerator(settings.copy(), settings,
-                                CUR_DIR, settings['THEME'], None,
+                CONTENT_DIR, settings['THEME'], None,  settings['MARKUP'])
-                                settings['MARKUP'])
             self.generator.generate_context()
         return self.generator
 
@@ -118,14 +117,13 @@ class TestArticlesGenerator(unittest.TestCase):
     def test_do_not_use_folder_as_category(self):
 
         settings = DEFAULT_CONFIG.copy()
-        settings['ARTICLE_DIR'] = 'content'
         settings['DEFAULT_CATEGORY'] = 'Default'
         settings['DEFAULT_DATE'] = (1970, 1, 1)
         settings['USE_FOLDER_AS_CATEGORY'] = False
         settings['filenames'] = {}
         generator = ArticlesGenerator(
-            settings.copy(), settings, CUR_DIR, DEFAULT_CONFIG['THEME'], None,
+            settings.copy(), settings, CONTENT_DIR, DEFAULT_CONFIG['THEME'],
-            DEFAULT_CONFIG['MARKUP'])
+            None, DEFAULT_CONFIG['MARKUP'])
         generator.generate_context()
         # test for name
         # categories are grouped by slug; if two categories have the same slug
@@ -213,12 +211,12 @@ class TestPageGenerator(unittest.TestCase):
 
     def test_generate_context(self):
         settings = get_settings(filenames={})
-        settings['PAGE_DIR'] = 'TestPages'
+        settings['PAGE_DIR'] = 'TestPages'  # relative to CUR_DIR
         settings['DEFAULT_DATE'] = (1970, 1, 1)
 
-        generator = PagesGenerator(settings.copy(), settings, CUR_DIR,
+        generator = PagesGenerator(
-                                      settings['THEME'], None,
+            settings.copy(), settings, CUR_DIR, settings['THEME'], None,
-                                      settings['MARKUP'])
+            settings['MARKUP'])
         generator.generate_context()
         pages = self.distill_pages(generator.pages)
         hidden_pages = self.distill_pages(generator.hidden_pages)

pelican/tests/test_readers.py

@@ -20,13 +20,13 @@ class ReaderTest(unittest.TestCase):
     def read_file(self, path, **kwargs):
         # Isolate from future API changes to readers.read_file
         return readers.read_file(
-            _path(path), settings=get_settings(**kwargs))
+            base_path=CONTENT_PATH, path=path, settings=get_settings(**kwargs))
 
 
 class RstReaderTest(ReaderTest):
 
     def test_article_with_metadata(self):
-        content, metadata = self.read_file(path='article_with_metadata.rst')
+        page = self.read_file(path='article_with_metadata.rst')
         expected = {
             'category': 'yeah',
             'author': 'Alexis Métaireau',
@@ -40,10 +40,10 @@ class RstReaderTest(ReaderTest):
         }
 
         for key, value in expected.items():
-            self.assertEqual(value, metadata[key], key)
+            self.assertEqual(value, page.metadata[key], key)
 
     def test_article_with_filename_metadata(self):
-        content, metadata = self.read_file(
+        page = self.read_file(
             path='2012-11-29_rst_w_filename_meta#foo-bar.rst',
             FILENAME_METADATA=None)
         expected = {
@@ -51,10 +51,10 @@ class RstReaderTest(ReaderTest):
             'author': 'Alexis Métaireau',
             'title': 'Rst with filename metadata',
         }
-        for key, value in metadata.items():
+        for key, value in page.metadata.items():
             self.assertEqual(value, expected[key], key)
 
-        content, metadata = self.read_file(
+        page = self.read_file(
             path='2012-11-29_rst_w_filename_meta#foo-bar.rst',
             FILENAME_METADATA='(?P<date>\d{4}-\d{2}-\d{2}).*')
         expected = {
@@ -63,10 +63,10 @@ class RstReaderTest(ReaderTest):
             'title': 'Rst with filename metadata',
             'date': datetime.datetime(2012, 11, 29),
         }
-        for key, value in metadata.items():
+        for key, value in page.metadata.items():
             self.assertEqual(value, expected[key], key)
 
-        content, metadata = self.read_file(
+        page = self.read_file(
             path='2012-11-29_rst_w_filename_meta#foo-bar.rst',
             FILENAME_METADATA=(
                 '(?P<date>\d{4}-\d{2}-\d{2})_'
@@ -80,7 +80,7 @@ class RstReaderTest(ReaderTest):
             'slug': 'article_with_filename_metadata',
             'mymeta': 'foo',
         }
-        for key, value in metadata.items():
+        for key, value in page.metadata.items():
             self.assertEqual(value, expected[key], key)
 
     def test_article_metadata_key_lowercase(self):
@@ -96,23 +96,23 @@ class RstReaderTest(ReaderTest):
     def test_typogrify(self):
         # if nothing is specified in the settings, the content should be
         # unmodified
-        content, _ = self.read_file(path='article.rst')
+        page = self.read_file(path='article.rst')
         expected = ('<p>This is some content. With some stuff to '
                     '&quot;typogrify&quot;.</p>\n<p>Now with added '
                     'support for <abbr title="three letter acronym">'
                     'TLA</abbr>.</p>\n')
 
-        self.assertEqual(content, expected)
+        self.assertEqual(page.content, expected)
 
         try:
             # otherwise, typogrify should be applied
-            content, _ = self.read_file(path='article.rst', TYPOGRIFY=True)
+            page = self.read_file(path='article.rst', TYPOGRIFY=True)
             expected = ('<p>This is some content. With some stuff to&nbsp;'
                         '&#8220;typogrify&#8221;.</p>\n<p>Now with added '
                         'support for <abbr title="three letter acronym">'
                         '<span class="caps">TLA</span></abbr>.</p>\n')
 
-            self.assertEqual(content, expected)
+            self.assertEqual(page.content, expected)
         except ImportError:
             return unittest.skip('need the typogrify distribution')
 
@@ -225,7 +225,7 @@ class MdReaderTest(ReaderTest):
     def test_article_with_markdown_markup_extension(self):
         # test to ensure the markdown markup extension is being processed as
         # expected
-        content, metadata = self.read_file(
+        page = self.read_file(
             path='article_with_markdown_markup_extensions.md',
             MD_EXTENSIONS=['toc', 'codehilite', 'extra'])
         expected = ('<div class="toc">\n'
@@ -239,11 +239,11 @@ class MdReaderTest(ReaderTest):
                     '<h2 id="level1">Level1</h2>\n'
                     '<h3 id="level2">Level2</h3>')
 
-        self.assertEqual(content, expected)
+        self.assertEqual(page.content, expected)
 
     @unittest.skipUnless(readers.Markdown, "markdown isn't installed")
     def test_article_with_filename_metadata(self):
-        content, metadata = self.read_file(
+        page = self.read_file(
             path='2012-11-30_md_w_filename_meta#foo-bar.md',
             FILENAME_METADATA=None)
         expected = {
@@ -251,9 +251,9 @@ class MdReaderTest(ReaderTest):
             'author': 'Alexis Métaireau',
         }
         for key, value in expected.items():
-            self.assertEqual(value, metadata[key], key)
+            self.assertEqual(value, page.metadata[key], key)
 
-        content, metadata = self.read_file(
+        page = self.read_file(
             path='2012-11-30_md_w_filename_meta#foo-bar.md',
             FILENAME_METADATA='(?P<date>\d{4}-\d{2}-\d{2}).*')
         expected = {
@@ -262,9 +262,9 @@ class MdReaderTest(ReaderTest):
             'date': datetime.datetime(2012, 11, 30),
         }
         for key, value in expected.items():
-            self.assertEqual(value, metadata[key], key)
+            self.assertEqual(value, page.metadata[key], key)
 
-        content, metadata = self.read_file(
+        page = self.read_file(
             path='2012-11-30_md_w_filename_meta#foo-bar.md',
             FILENAME_METADATA=(
                 '(?P<date>\d{4}-\d{2}-\d{2})'
@@ -278,7 +278,7 @@ class MdReaderTest(ReaderTest):
             'mymeta': 'foo',
         }
         for key, value in expected.items():
-            self.assertEqual(value, metadata[key], key)
+            self.assertEqual(value, page.metadata[key], key)
 
 
 class AdReaderTest(ReaderTest):
@@ -286,13 +286,13 @@ class AdReaderTest(ReaderTest):
     @unittest.skipUnless(readers.asciidoc, "asciidoc isn't installed")
     def test_article_with_asc_extension(self):
         # Ensure the asc extension is being processed by the correct reader
-        content, metadata = self.read_file(
+        page = self.read_file(
             path='article_with_asc_extension.asc')
         expected = ('<hr>\n<h2><a name="_used_for_pelican_test">'
                     '</a>Used for pelican test</h2>\n'
                     '<p>The quick brown fox jumped over'
                     ' the lazy dog&#8217;s back.</p>\n')
-        self.assertEqual(content, expected)
+        self.assertEqual(page.content, expected)
         expected = {
             'category': 'Blog',
             'author': 'Author O. Article',
@@ -302,7 +302,7 @@ class AdReaderTest(ReaderTest):
         }
 
         for key, value in expected.items():
-            self.assertEqual(value, metadata[key], key)
+            self.assertEqual(value, page.metadata[key], key)
 
     @unittest.skipUnless(readers.asciidoc, "asciidoc isn't installed")
     def test_article_with_asc_options(self):
@@ -319,24 +319,24 @@ class AdReaderTest(ReaderTest):
 
 class HTMLReaderTest(ReaderTest):
     def test_article_with_comments(self):
-        content, metadata = self.read_file(path='article_with_comments.html')
+        page = self.read_file(path='article_with_comments.html')
 
         self.assertEqual('''
         Body content
         <!--  This comment is included (including extra whitespace)   -->
-    ''', content)
+    ''', page.content)
 
     def test_article_with_keywords(self):
-        content, metadata = self.read_file(path='article_with_keywords.html')
+        page = self.read_file(path='article_with_keywords.html')
         expected = {
             'tags': ['foo', 'bar', 'foobar'],
         }
 
         for key, value in expected.items():
-            self.assertEqual(value, metadata[key], key)
+            self.assertEqual(value, page.metadata[key], key)
 
     def test_article_with_metadata(self):
-        content, metadata = self.read_file(path='article_with_metadata.html')
+        page = self.read_file(path='article_with_metadata.html')
         expected = {
             'category': 'yeah',
             'author': 'Alexis Métaireau',
@@ -348,21 +348,19 @@ class HTMLReaderTest(ReaderTest):
         }
 
         for key, value in expected.items():
-            self.assertEqual(value, metadata[key], key)
+            self.assertEqual(value, page.metadata[key], key)
 
     def test_article_with_null_attributes(self):
-        content, metadata = self.read_file(
+        page = self.read_file(path='article_with_null_attributes.html')
-            path='article_with_null_attributes.html')
 
         self.assertEqual('''
         Ensure that empty attributes are copied properly.
         <input name="test" disabled style="" />
-    ''', content)
+    ''', page.content)
 
     def test_article_metadata_key_lowercase(self):
         # Keys of metadata should be lowercase.
-        content, metadata = self.read_file(
+        page = self.read_file(path='article_with_uppercase_metadata.html')
-            path='article_with_uppercase_metadata.html')
+        self.assertIn('category', page.metadata, 'Key should be lowercase.')
-        self.assertIn('category', metadata, 'Key should be lowercase.')
+        self.assertEqual('Yeah', page.metadata.get('category'),
-        self.assertEqual('Yeah', metadata.get('category'),
                           'Value keeps cases.')

samples/pelican.conf.py

@@ -34,11 +34,19 @@ SOCIAL = (('twitter', 'http://twitter.com/ametaireau'),
 # global metadata to all the contents
 DEFAULT_METADATA = (('yeah', 'it is'),)
 
-# static paths will be copied under the same name
+# path-specific metadata
-STATIC_PATHS = ["pictures", ]
+EXTRA_PATH_METADATA = {
+    'extra/robots.txt': {'path': 'robots.txt'},
+    'pictures/Fat_Cat.jpg': {'path': 'static/pictures/Fat_Cat.jpg'},
+    'pictures/Sushi.jpg': {'path': 'static/pictures/Sushi.jpg'},
+    'pictures/Sushi_Macro.jpg': {'path': 'static/pictures/Sushi_Macro.jpg'},
+    }
 
-# A list of files to copy from the source to the destination
+# static paths will be copied without parsing their contents
-FILES_TO_COPY = (('extra/robots.txt', 'robots.txt'),)
+STATIC_PATHS = [
+    'pictures',
+    'extra/robots.txt',
+    ]
 
 # custom page generated with a jinja2 template
 TEMPLATE_PAGES = {'pages/jinja2_template.html': 'jinja2_template.html'}