From ecf56829300b731711b46177cee86345d504096f Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 4 Jan 2013 14:21:38 -0500 Subject: [PATCH 01/22] readers: Instrument read_file to return Content objects The assorted generators all use read_file() to read in the file contents and metadata. Previously, they sometimes parse additional metadata, fire off signals, and initialize a pelican.contents.Content subclass on their own. We can reduce duplicated code and increase consistency by shifting all that stuff into read_file() itself, and this commit is a step in that direction. --- pelican/readers.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/pelican/readers.py b/pelican/readers.py index 816464ef..beaa8814 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -31,7 +31,7 @@ try: except ImportError: from HTMLParser import HTMLParser -from pelican.contents import Category, Tag, Author +from pelican.contents import Page, Category, Tag, Author from pelican.utils import get_date, pelican_open @@ -333,8 +333,15 @@ for cls in [Reader] + Reader.__subclasses__(): EXTENSIONS[ext] = cls -def read_file(path, fmt=None, settings=None): - """Return a reader object using the given format.""" +def read_file(base_path, path, content_class=Page, fmt=None, + settings=None, context=None, + preread_signal=None, preread_sender=None, + context_signal=None, context_sender=None): + """Return a content object parsed with the given format.""" + if preread_signal: + preread_signal.send(preread_sender) + path = os.path.abspath(os.path.join(base_path, path)) + source_path = os.path.relpath(path, base_path) base, ext = os.path.splitext(os.path.basename(path)) if not fmt: fmt = ext[1:] @@ -355,7 +362,7 @@ def read_file(path, fmt=None, settings=None): raise ValueError("Missing dependencies for %s" % fmt) metadata = parse_path_metadata( - path=path, settings=settings, process=reader.process_metadata) + path=source_path, settings=settings, process=reader.process_metadata) content, reader_metadata = reader.read(path) metadata.update(reader_metadata) @@ -365,7 +372,14 @@ def read_file(path, fmt=None, settings=None): content = typogrify(content) metadata['title'] = typogrify(metadata['title']) - return content, metadata + if context_signal: + context_signal.send(context_sender, metadata=metadata) + return content_class( + content=content, + metadata=metadata, + settings=settings, + source_path=path, + context=context) def parse_path_metadata(path, settings=None, process=None): """Extract a metadata dictionary from a file's path From 4e118eff010c4048889063ecdad804f0435ad00b Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 5 Jan 2013 21:27:30 -0500 Subject: [PATCH 02/22] readers: Add debugging logging to read_file --- pelican/readers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pelican/readers.py b/pelican/readers.py index beaa8814..974290e6 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals, print_function +import logging import os import re try: @@ -35,6 +36,8 @@ from pelican.contents import Page, Category, Tag, Author from pelican.utils import get_date, pelican_open +logger = logging.getLogger(__name__) + METADATA_PROCESSORS = { 'tags': lambda x, y: [Tag(tag, y) for tag in x.split(',')], 'date': lambda x, y: get_date(x), @@ -343,6 +346,8 @@ def read_file(base_path, path, content_class=Page, fmt=None, path = os.path.abspath(os.path.join(base_path, path)) source_path = os.path.relpath(path, base_path) base, ext = os.path.splitext(os.path.basename(path)) + logger.debug('read file {} -> {}'.format( + source_path, content_class.__name__)) if not fmt: fmt = ext[1:] From e38e170656f174528601601c5315484525b6abd9 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Mon, 3 Jun 2013 15:09:18 -0400 Subject: [PATCH 03/22] readers: Log signal sending for read_file() --- pelican/readers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pelican/readers.py b/pelican/readers.py index 974290e6..2de00b51 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -341,8 +341,6 @@ def read_file(base_path, path, content_class=Page, fmt=None, preread_signal=None, preread_sender=None, context_signal=None, context_sender=None): """Return a content object parsed with the given format.""" - if preread_signal: - preread_signal.send(preread_sender) path = os.path.abspath(os.path.join(base_path, path)) source_path = os.path.relpath(path, base_path) base, ext = os.path.splitext(os.path.basename(path)) @@ -354,6 +352,11 @@ def read_file(base_path, path, content_class=Page, fmt=None, if fmt not in EXTENSIONS: raise TypeError('Pelican does not know how to parse {}'.format(path)) + if preread_signal: + logger.debug('signal {}.send({})'.format( + preread_signal, preread_sender)) + preread_signal.send(preread_sender) + if settings is None: settings = {} @@ -378,6 +381,8 @@ def read_file(base_path, path, content_class=Page, fmt=None, metadata['title'] = typogrify(metadata['title']) if context_signal: + logger.debug('signal {}.send({}, )'.format( + context_signal, context_sender)) context_signal.send(context_sender, metadata=metadata) return content_class( content=content, From effe7e5e126343e9b787d0434e79599c499b8443 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 4 Jan 2013 18:53:12 -0500 Subject: [PATCH 04/22] signals: Sort signals into categories --- pelican/signals.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pelican/signals.py b/pelican/signals.py index 92bc6249..ad2985b5 100644 --- a/pelican/signals.py +++ b/pelican/signals.py @@ -2,15 +2,27 @@ from __future__ import unicode_literals, print_function from blinker import signal +# Run-level signals: + initialized = signal('pelican_initialized') +get_generators = signal('get_generators') finalized = signal('pelican_finalized') -article_generate_preread = signal('article_generate_preread') + +# Generator-level signals + generator_init = signal('generator_init') -article_generate_context = signal('article_generate_context') + article_generator_init = signal('article_generator_init') article_generator_finalized = signal('article_generate_finalized') -get_generators = signal('get_generators') -pages_generate_context = signal('pages_generate_context') + pages_generator_init = signal('pages_generator_init') pages_generator_finalized = signal('pages_generator_finalized') + +# Page-level signals + +article_generate_preread = signal('article_generate_preread') +article_generate_context = signal('article_generate_context') + +pages_generate_context = signal('pages_generate_context') + content_object_init = signal('content_object_init') From 386cd1f3f0e6c573cc8965ace2e9bf492452cf70 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 4 Jan 2013 19:01:41 -0500 Subject: [PATCH 05/22] signals: Add missing signals Note that the `pages_*` names are plural , while the `article_*` names are singular. I'll fix this once I update the PagesGenerator. --- pelican/signals.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pelican/signals.py b/pelican/signals.py index ad2985b5..34bde68a 100644 --- a/pelican/signals.py +++ b/pelican/signals.py @@ -18,11 +18,18 @@ article_generator_finalized = signal('article_generate_finalized') pages_generator_init = signal('pages_generator_init') pages_generator_finalized = signal('pages_generator_finalized') +static_generator_init = signal('static_generator_init') +static_generator_finalized = signal('static_generator_finalized') + # Page-level signals article_generate_preread = signal('article_generate_preread') article_generate_context = signal('article_generate_context') +pages_generate_preread = signal('pages_generate_preread') pages_generate_context = signal('pages_generate_context') +static_generate_preread = signal('static_generate_preread') +static_generate_context = signal('static_generate_context') + content_object_init = signal('content_object_init') From f2d6f77462976f5ea291481cba40e041d42d9e8c Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 5 Jan 2013 21:33:28 -0500 Subject: [PATCH 06/22] signals: Fix *_generate_* signals -> *_generator_* For example, article_generate_preread is now article_generator_preread for consistency with the other preread and context signals. --- pelican/signals.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pelican/signals.py b/pelican/signals.py index 34bde68a..877b0a6b 100644 --- a/pelican/signals.py +++ b/pelican/signals.py @@ -13,7 +13,7 @@ finalized = signal('pelican_finalized') generator_init = signal('generator_init') article_generator_init = signal('article_generator_init') -article_generator_finalized = signal('article_generate_finalized') +article_generator_finalized = signal('article_generator_finalized') pages_generator_init = signal('pages_generator_init') pages_generator_finalized = signal('pages_generator_finalized') @@ -23,13 +23,13 @@ static_generator_finalized = signal('static_generator_finalized') # Page-level signals -article_generate_preread = signal('article_generate_preread') -article_generate_context = signal('article_generate_context') +article_generator_preread = signal('article_generator_preread') +article_generator_context = signal('article_generator_context') -pages_generate_preread = signal('pages_generate_preread') -pages_generate_context = signal('pages_generate_context') +pages_generator_preread = signal('pages_generator_preread') +pages_generator_context = signal('pages_generator_context') -static_generate_preread = signal('static_generate_preread') -static_generate_context = signal('static_generate_context') +static_generator_preread = signal('static_generator_preread') +static_generator_context = signal('static_generator_context') content_object_init = signal('content_object_init') From a9c530281e1928e9f9f3c2cb78312db784505ca1 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 4 Jan 2013 18:14:28 -0500 Subject: [PATCH 07/22] Move Article metadata extraction from generators to readers There's no reason why this information should be Article-specific. This commit breaks the other generators for the moment. I'll fix them shortly. --- pelican/generators.py | 41 ++++++++--------------------------- pelican/readers.py | 50 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 40 deletions(-) diff --git a/pelican/generators.py b/pelican/generators.py index 75b61df2..7daa55d2 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -5,7 +5,6 @@ import os import math import random import logging -import datetime import shutil from codecs import open @@ -19,9 +18,7 @@ from jinja2 import ( TemplateNotFound ) -from pelican.contents import ( - Article, Page, Category, Static, is_valid_content -) +from pelican.contents import Article, Page, Static, is_valid_content from pelican.readers import read_file from pelican.utils import copy, process_translations, mkdir_p, DateFormatter from pelican import signals @@ -383,37 +380,17 @@ class ArticlesGenerator(Generator): article_path, exclude=self.settings['ARTICLE_EXCLUDES']): try: - signals.article_generate_preread.send(self) - content, metadata = read_file(f, settings=self.settings) + article = read_file( + base_path=self.path, path=f, content_class=Article, + settings=self.settings, context=self.context, + preread_signal=signals.article_generator_preread, + preread_sender=self, + context_signal=signals.article_generator_context, + context_sender=self) except Exception as e: - logger.warning('Could not process %s\n%s' % (f, str(e))) + logger.warning('Could not process {}\n{}'.format(f, e)) continue - # if no category is set, use the name of the path as a category - if 'category' not in metadata: - - if (self.settings['USE_FOLDER_AS_CATEGORY'] - and os.path.dirname(f) != article_path): - # if the article is in a subdirectory - category = os.path.basename(os.path.dirname(f)) - else: - # if the article is not in a subdirectory - category = self.settings['DEFAULT_CATEGORY'] - - if category != '': - metadata['category'] = Category(category, self.settings) - - if 'date' not in metadata and self.settings.get('DEFAULT_DATE'): - if self.settings['DEFAULT_DATE'] == 'fs': - metadata['date'] = datetime.datetime.fromtimestamp( - os.stat(f).st_ctime) - else: - metadata['date'] = datetime.datetime( - *self.settings['DEFAULT_DATE']) - - signals.article_generate_context.send(self, metadata=metadata) - article = Article(content, metadata, settings=self.settings, - source_path=f, context=self.context) if not is_valid_content(article, f): continue diff --git a/pelican/readers.py b/pelican/readers.py index 2de00b51..3cf69dcf 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals, print_function +import datetime import logging import os import re @@ -369,8 +370,13 @@ def read_file(base_path, path, content_class=Page, fmt=None, if not reader.enabled: raise ValueError("Missing dependencies for %s" % fmt) - metadata = parse_path_metadata( - path=source_path, settings=settings, process=reader.process_metadata) + metadata = default_metadata( + settings=settings, process=reader.process_metadata) + metadata.update(path_metadata( + full_path=path, source_path=source_path, settings=settings)) + metadata.update(parse_path_metadata( + source_path=source_path, settings=settings, + process=reader.process_metadata)) content, reader_metadata = reader.read(path) metadata.update(reader_metadata) @@ -391,7 +397,29 @@ def read_file(base_path, path, content_class=Page, fmt=None, source_path=path, context=context) -def parse_path_metadata(path, settings=None, process=None): + +def default_metadata(settings=None, process=None): + metadata = {} + if settings: + if 'DEFAULT_CATEGORY' in settings: + value = settings['DEFAULT_CATEGORY'] + if process: + value = process('category', value) + metadata['category'] = value + if 'DEFAULT_DATE' in settings and settings['DEFAULT_DATE'] != 'fs': + metadata['date'] = datetime.datetime(*settings['DEFAULT_DATE']) + return metadata + + +def path_metadata(full_path, source_path, settings=None): + metadata = {} + if settings and settings.get('DEFAULT_DATE', None) == 'fs': + metadata['date'] = datetime.datetime.fromtimestamp( + os.stat(path).st_ctime) + return metadata + + +def parse_path_metadata(source_path, settings=None, process=None): """Extract a metadata dictionary from a file's path >>> import pprint @@ -402,7 +430,7 @@ def parse_path_metadata(path, settings=None, process=None): ... } >>> reader = Reader(settings=settings) >>> metadata = parse_path_metadata( - ... path='my-cat/2013-01-01/my-slug.html', + ... source_path='my-cat/2013-01-01/my-slug.html', ... settings=settings, ... process=reader.process_metadata) >>> pprint.pprint(metadata) # doctest: +ELLIPSIS @@ -411,13 +439,19 @@ def parse_path_metadata(path, settings=None, process=None): 'slug': 'my-slug'} """ metadata = {} - base, ext = os.path.splitext(os.path.basename(path)) + dirname, basename = os.path.split(source_path) + base, ext = os.path.splitext(basename) + subdir = os.path.basename(dirname) if settings: + checks = [] for key,data in [('FILENAME_METADATA', base), - ('PATH_METADATA', path), + ('PATH_METADATA', source_path), ]: - regexp = settings.get(key) - if regexp: + checks.append((settings.get(key, None), data)) + if settings.get('USE_FOLDER_AS_CATEGORY', None): + checks.insert(0, ('(?P.*)', subdir)) + for regexp,data in checks: + if regexp and data: match = re.match(regexp, data) if match: # .items() for py3k compat. From 7be16dd5247e07952c009cb439734722acf019ac Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 4 Jan 2013 18:19:26 -0500 Subject: [PATCH 08/22] generators: Update PagesGenerator to use new read_file Also standardize signal names. If `article_generator_*` is singular, `page_generator_*` should be as well. Fix it from the older `pages_generator_*`. --- docs/plugins.rst | 6 +++--- pelican/generators.py | 18 +++++++++++------- pelican/signals.py | 8 ++++---- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 064ba73d..9e262962 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -79,9 +79,9 @@ article_generator_finalized article_generator invoked at the e get_generators generators invoked in Pelican.get_generator_classes, can return a Generator, or several generator in a tuple or in a list. -pages_generate_context pages_generator, metadata -pages_generator_init pages_generator invoked in the PagesGenerator.__init__ -pages_generator_finalized pages_generator invoked at the end of PagesGenerator.generate_context +page_generate_context page_generator, metadata +page_generator_init page_generator invoked in the PagesGenerator.__init__ +page_generator_finalized page_generator invoked at the end of PagesGenerator.generate_context content_object_init content_object invoked at the end of Content.__init__ (see note below) ============================= ============================ =========================================================================== diff --git a/pelican/generators.py b/pelican/generators.py index 7daa55d2..a4cd273e 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -479,7 +479,7 @@ class PagesGenerator(Generator): self.hidden_pages = [] self.hidden_translations = [] super(PagesGenerator, self).__init__(*args, **kwargs) - signals.pages_generator_init.send(self) + signals.page_generator_init.send(self) def generate_context(self): all_pages = [] @@ -488,13 +488,17 @@ class PagesGenerator(Generator): os.path.join(self.path, self.settings['PAGE_DIR']), exclude=self.settings['PAGE_EXCLUDES']): try: - content, metadata = read_file(f, settings=self.settings) + page = read_file( + base_path=self.path, path=f, content_class=Page, + settings=self.settings, context=self.context, + preread_signal=signals.page_generator_preread, + preread_sender=self, + context_signal=signals.page_generator_context, + context_sender=self) except Exception as e: - logger.warning('Could not process %s\n%s' % (f, str(e))) + logger.warning('Could not process {}\n{}'.format(f, e)) continue - signals.pages_generate_context.send(self, metadata=metadata) - page = Page(content, metadata, settings=self.settings, - source_path=f, context=self.context) + if not is_valid_content(page, f): continue @@ -516,7 +520,7 @@ class PagesGenerator(Generator): self._update_context(('pages', )) self.context['PAGES'] = self.pages - signals.pages_generator_finalized.send(self) + signals.page_generator_finalized.send(self) def generate_output(self, writer): for page in chain(self.translations, self.pages, diff --git a/pelican/signals.py b/pelican/signals.py index 877b0a6b..cb010d37 100644 --- a/pelican/signals.py +++ b/pelican/signals.py @@ -15,8 +15,8 @@ generator_init = signal('generator_init') article_generator_init = signal('article_generator_init') article_generator_finalized = signal('article_generator_finalized') -pages_generator_init = signal('pages_generator_init') -pages_generator_finalized = signal('pages_generator_finalized') +page_generator_init = signal('page_generator_init') +page_generator_finalized = signal('page_generator_finalized') static_generator_init = signal('static_generator_init') static_generator_finalized = signal('static_generator_finalized') @@ -26,8 +26,8 @@ static_generator_finalized = signal('static_generator_finalized') article_generator_preread = signal('article_generator_preread') article_generator_context = signal('article_generator_context') -pages_generator_preread = signal('pages_generator_preread') -pages_generator_context = signal('pages_generator_context') +page_generator_preread = signal('page_generator_preread') +page_generator_context = signal('page_generator_context') static_generator_preread = signal('static_generator_preread') static_generator_context = signal('static_generator_context') From 1bc5b100eceab6b0ad90ab7ebecd896dbbc2bc93 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 5 Jan 2013 07:21:47 -0500 Subject: [PATCH 09/22] generators: Convert StaticGenerator to use the new read_file --- pelican/generators.py | 45 ++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/pelican/generators.py b/pelican/generators.py index a4cd273e..b777efec 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -548,32 +548,29 @@ class StaticGenerator(Generator): for static_path in self.settings['STATIC_PATHS']: for f in self.get_files( os.path.join(self.path, static_path), extensions=False): - f_rel = os.path.relpath(f, self.path) - content, metadata = read_file( - f, fmt='static', settings=self.settings) - # TODO remove this hardcoded 'static' subdirectory - metadata['save_as'] = os.path.join('static', f_rel) - metadata['url'] = pelican.utils.path_to_url(metadata['save_as']) - sc = Static( - content=None, - metadata=metadata, - settings=self.settings, - source_path=f_rel) - self.staticfiles.append(sc) - self.add_source_path(sc) + static = read_file( + base_path=self.path, path=f, content_class=Static, + fmt='static', + settings=self.settings, context=self.context, + preread_signal=signals.static_generator_preread, + preread_sender=self, + context_signal=signals.static_generator_context, + context_sender=self) + self.staticfiles.append(static) + self.add_source_path(static) + # same thing for FILES_TO_COPY for src, dest in self.settings['FILES_TO_COPY']: - content, metadata = read_file( - src, fmt='static', settings=self.settings) - metadata['save_as'] = dest - metadata['url'] = pelican.utils.path_to_url(metadata['save_as']) - sc = Static( - content=None, - metadata={'save_as': dest}, - settings=self.settings, - source_path=src) - self.staticfiles.append(sc) - self.add_source_path(sc) + static = read_file( + base_path=self.path, path=f, content_class=Static, + fmt='static', + settings=self.settings, context=self.context, + preread_signal=signals.static_generator_preread, + preread_sender=self, + context_signal=signals.static_generator_context, + context_sender=self) + self.staticfiles.append(static) + self.add_source_path(static) def generate_output(self, writer): self._copy_paths(self.settings['THEME_STATIC_PATHS'], self.theme, From 1ca0e06a273c45a52b4aab019ae95e580c6c58da Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Mon, 3 Jun 2013 15:14:11 -0400 Subject: [PATCH 10/22] Remove the FILES_TO_COPY setting We no longer instantiate the Static object in the StaticGenerator, so we can't set the save_as argument anymore. If you want to adjust the output path, use the upcoming EXTRA_PATH_METADATA setting. --- docs/settings.rst | 3 --- pelican/generators.py | 13 ------------- pelican/settings.py | 1 - 3 files changed, 17 deletions(-) diff --git a/docs/settings.rst b/docs/settings.rst index 0a04f804..a75badfc 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -68,9 +68,6 @@ Setting name (default value) What doe generating new files. This can be useful in preventing older, unnecessary files from persisting in your output. However, **this is a destructive setting and should be handled with extreme care.** -`FILES_TO_COPY` (``()``) A list of files (or directories) to copy from the source (inside the - content directory) to the destination (inside the output directory). - For example: ``(('extra/robots.txt', 'robots.txt'),)``. `JINJA_EXTENSIONS` (``[]``) A list of any Jinja2 extensions you want to use. `JINJA_FILTERS` (``{}``) A list of custom Jinja2 filters you want to use. The dictionary should map the filtername to the filter function. diff --git a/pelican/generators.py b/pelican/generators.py index b777efec..c48f8067 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -559,19 +559,6 @@ class StaticGenerator(Generator): self.staticfiles.append(static) self.add_source_path(static) - # same thing for FILES_TO_COPY - for src, dest in self.settings['FILES_TO_COPY']: - static = read_file( - base_path=self.path, path=f, content_class=Static, - fmt='static', - settings=self.settings, context=self.context, - preread_signal=signals.static_generator_preread, - preread_sender=self, - context_signal=signals.static_generator_context, - context_sender=self) - self.staticfiles.append(static) - self.add_source_path(static) - def generate_output(self, writer): self._copy_paths(self.settings['THEME_STATIC_PATHS'], self.theme, 'theme', self.output_path, os.curdir) diff --git a/pelican/settings.py b/pelican/settings.py index 34a2b42a..5386d172 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -95,7 +95,6 @@ DEFAULT_CONFIG = { 'DEFAULT_METADATA': (), 'FILENAME_METADATA': '(?P\d{4}-\d{2}-\d{2}).*', 'PATH_METADATA': '', - 'FILES_TO_COPY': (), 'DEFAULT_STATUS': 'published', 'ARTICLE_PERMALINK_STRUCTURE': '', 'TYPOGRIFY': False, From d43dc1b9d6ba5f0f99c6de44db214f296558c701 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Mon, 3 Jun 2013 15:15:53 -0400 Subject: [PATCH 11/22] Add the EXTRA_PATH_METADATA setting Useful for altering static output paths when you don't want to encode extra metadata in the static file's source path. --- docs/settings.rst | 1 + pelican/readers.py | 9 ++++++--- pelican/settings.py | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/settings.rst b/docs/settings.rst index a75badfc..c7d7f199 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -64,6 +64,7 @@ Setting name (default value) What doe ``'(?P\d{4}-\d{2}-\d{2})_(?P.*)'``. `PATH_METADATA` (``''``) Like ``FILENAME_METADATA``, but parsed from a page's full path relative to the content source directory. +`EXTRA_PATH_METADATA` (``{}``) Extra metadata dictionaries keyed by relative path. `DELETE_OUTPUT_DIRECTORY` (``False``) Delete the output directory, and **all** of its contents, before generating new files. This can be useful in preventing older, unnecessary files from persisting in your output. However, **this is diff --git a/pelican/readers.py b/pelican/readers.py index 3cf69dcf..3e00b430 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -413,9 +413,12 @@ def default_metadata(settings=None, process=None): def path_metadata(full_path, source_path, settings=None): metadata = {} - if settings and settings.get('DEFAULT_DATE', None) == 'fs': - metadata['date'] = datetime.datetime.fromtimestamp( - os.stat(path).st_ctime) + if settings: + if settings.get('DEFAULT_DATE', None) == 'fs': + metadata['date'] = datetime.datetime.fromtimestamp( + os.stat(full_path).st_ctime) + metadata.update(settings.get('EXTRA_PATH_METADATA', {}).get( + source_path, {})) return metadata diff --git a/pelican/settings.py b/pelican/settings.py index 5386d172..13896032 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -95,6 +95,7 @@ DEFAULT_CONFIG = { 'DEFAULT_METADATA': (), 'FILENAME_METADATA': '(?P\d{4}-\d{2}-\d{2}).*', 'PATH_METADATA': '', + 'EXTRA_PATH_METADATA': {}, 'DEFAULT_STATUS': 'published', 'ARTICLE_PERMALINK_STRUCTURE': '', 'TYPOGRIFY': False, From 7de7bd0e3715f2227ce84fcf198dcc98cc3a0582 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 5 Jan 2013 07:43:11 -0500 Subject: [PATCH 12/22] Update sample configurations from FILES_TO_COPY to EXTRA_PATH_METADATA --- pelican/tests/default_conf.py | 13 +++++++++---- samples/pelican.conf.py | 16 ++++++++++++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/pelican/tests/default_conf.py b/pelican/tests/default_conf.py index bc3a7dff..80a990b5 100644 --- a/pelican/tests/default_conf.py +++ b/pelican/tests/default_conf.py @@ -29,11 +29,16 @@ SOCIAL = (('twitter', 'http://twitter.com/ametaireau'), # global metadata to all the contents DEFAULT_METADATA = (('yeah', 'it is'),) -# static paths will be copied under the same name -STATIC_PATHS = ["pictures", ] +# path-specific metadata +EXTRA_PATH_METADATA = { + 'extra/robots.txt': {'path': 'robots.txt'}, + } -# A list of files to copy from the source to the destination -FILES_TO_COPY = (('extra/robots.txt', 'robots.txt'),) +# static paths will be copied without parsing their contents +STATIC_PATHS = [ + 'pictures', + 'extra/robots.txt', + ] # foobar will not be used, because it's not in caps. All configuration keys # have to be in caps diff --git a/samples/pelican.conf.py b/samples/pelican.conf.py index 70edd5f8..ad2042fd 100755 --- a/samples/pelican.conf.py +++ b/samples/pelican.conf.py @@ -34,11 +34,19 @@ SOCIAL = (('twitter', 'http://twitter.com/ametaireau'), # global metadata to all the contents DEFAULT_METADATA = (('yeah', 'it is'),) -# static paths will be copied under the same name -STATIC_PATHS = ["pictures", ] +# path-specific metadata +EXTRA_PATH_METADATA = { + 'extra/robots.txt': {'path': 'robots.txt'}, + 'pictures/Fat_Cat.jpg': {'path': 'static/pictures/Fat_Cat.jpg'}, + 'pictures/Sushi.jpg': {'path': 'static/pictures/Sushi.jpg'}, + 'pictures/Sushi_Macro.jpg': {'path': 'static/pictures/Sushi_Macro.jpg'}, + } -# A list of files to copy from the source to the destination -FILES_TO_COPY = (('extra/robots.txt', 'robots.txt'),) +# static paths will be copied without parsing their contents +STATIC_PATHS = [ + 'pictures', + 'extra/robots.txt', + ] # custom page generated with a jinja2 template TEMPLATE_PAGES = {'pages/jinja2_template.html': 'jinja2_template.html'} From 06121eda76f22e03d43ce40c5c69d45d97447253 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Mon, 3 Jun 2013 15:21:17 -0400 Subject: [PATCH 13/22] test_readers: Update to new readers.read_file --- pelican/tests/test_readers.py | 72 +++++++++++++++++------------------ 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/pelican/tests/test_readers.py b/pelican/tests/test_readers.py index a48301f9..14d42325 100644 --- a/pelican/tests/test_readers.py +++ b/pelican/tests/test_readers.py @@ -20,13 +20,13 @@ class ReaderTest(unittest.TestCase): def read_file(self, path, **kwargs): # Isolate from future API changes to readers.read_file return readers.read_file( - _path(path), settings=get_settings(**kwargs)) + base_path=CONTENT_PATH, path=path, settings=get_settings(**kwargs)) class RstReaderTest(ReaderTest): def test_article_with_metadata(self): - content, metadata = self.read_file(path='article_with_metadata.rst') + page = self.read_file(path='article_with_metadata.rst') expected = { 'category': 'yeah', 'author': 'Alexis Métaireau', @@ -40,10 +40,10 @@ class RstReaderTest(ReaderTest): } for key, value in expected.items(): - self.assertEqual(value, metadata[key], key) + self.assertEqual(value, page.metadata[key], key) def test_article_with_filename_metadata(self): - content, metadata = self.read_file( + page = self.read_file( path='2012-11-29_rst_w_filename_meta#foo-bar.rst', FILENAME_METADATA=None) expected = { @@ -51,10 +51,10 @@ class RstReaderTest(ReaderTest): 'author': 'Alexis Métaireau', 'title': 'Rst with filename metadata', } - for key, value in metadata.items(): + for key, value in page.metadata.items(): self.assertEqual(value, expected[key], key) - content, metadata = self.read_file( + page = self.read_file( path='2012-11-29_rst_w_filename_meta#foo-bar.rst', FILENAME_METADATA='(?P\d{4}-\d{2}-\d{2}).*') expected = { @@ -63,10 +63,10 @@ class RstReaderTest(ReaderTest): 'title': 'Rst with filename metadata', 'date': datetime.datetime(2012, 11, 29), } - for key, value in metadata.items(): + for key, value in page.metadata.items(): self.assertEqual(value, expected[key], key) - content, metadata = self.read_file( + page = self.read_file( path='2012-11-29_rst_w_filename_meta#foo-bar.rst', FILENAME_METADATA=( '(?P\d{4}-\d{2}-\d{2})_' @@ -80,7 +80,7 @@ class RstReaderTest(ReaderTest): 'slug': 'article_with_filename_metadata', 'mymeta': 'foo', } - for key, value in metadata.items(): + for key, value in page.metadata.items(): self.assertEqual(value, expected[key], key) def test_article_metadata_key_lowercase(self): @@ -96,23 +96,23 @@ class RstReaderTest(ReaderTest): def test_typogrify(self): # if nothing is specified in the settings, the content should be # unmodified - content, _ = self.read_file(path='article.rst') + page = self.read_file(path='article.rst') expected = ('

This is some content. With some stuff to ' '"typogrify".

\n

Now with added ' 'support for ' 'TLA.

\n') - self.assertEqual(content, expected) + self.assertEqual(page.content, expected) try: # otherwise, typogrify should be applied - content, _ = self.read_file(path='article.rst', TYPOGRIFY=True) + page = self.read_file(path='article.rst', TYPOGRIFY=True) expected = ('

This is some content. With some stuff to ' '“typogrify”.

\n

Now with added ' 'support for ' 'TLA.

\n') - self.assertEqual(content, expected) + self.assertEqual(page.content, expected) except ImportError: return unittest.skip('need the typogrify distribution') @@ -225,7 +225,7 @@ class MdReaderTest(ReaderTest): def test_article_with_markdown_markup_extension(self): # test to ensure the markdown markup extension is being processed as # expected - content, metadata = self.read_file( + page = self.read_file( path='article_with_markdown_markup_extensions.md', MD_EXTENSIONS=['toc', 'codehilite', 'extra']) expected = ('
\n' @@ -239,11 +239,11 @@ class MdReaderTest(ReaderTest): '

Level1

\n' '

Level2

') - self.assertEqual(content, expected) + self.assertEqual(page.content, expected) @unittest.skipUnless(readers.Markdown, "markdown isn't installed") def test_article_with_filename_metadata(self): - content, metadata = self.read_file( + page = self.read_file( path='2012-11-30_md_w_filename_meta#foo-bar.md', FILENAME_METADATA=None) expected = { @@ -251,9 +251,9 @@ class MdReaderTest(ReaderTest): 'author': 'Alexis Métaireau', } for key, value in expected.items(): - self.assertEqual(value, metadata[key], key) + self.assertEqual(value, page.metadata[key], key) - content, metadata = self.read_file( + page = self.read_file( path='2012-11-30_md_w_filename_meta#foo-bar.md', FILENAME_METADATA='(?P\d{4}-\d{2}-\d{2}).*') expected = { @@ -262,9 +262,9 @@ class MdReaderTest(ReaderTest): 'date': datetime.datetime(2012, 11, 30), } for key, value in expected.items(): - self.assertEqual(value, metadata[key], key) + self.assertEqual(value, page.metadata[key], key) - content, metadata = self.read_file( + page = self.read_file( path='2012-11-30_md_w_filename_meta#foo-bar.md', FILENAME_METADATA=( '(?P\d{4}-\d{2}-\d{2})' @@ -278,7 +278,7 @@ class MdReaderTest(ReaderTest): 'mymeta': 'foo', } for key, value in expected.items(): - self.assertEqual(value, metadata[key], key) + self.assertEqual(value, page.metadata[key], key) class AdReaderTest(ReaderTest): @@ -286,13 +286,13 @@ class AdReaderTest(ReaderTest): @unittest.skipUnless(readers.asciidoc, "asciidoc isn't installed") def test_article_with_asc_extension(self): # Ensure the asc extension is being processed by the correct reader - content, metadata = self.read_file( + page = self.read_file( path='article_with_asc_extension.asc') expected = ('
\n

' 'Used for pelican test

\n' '

The quick brown fox jumped over' ' the lazy dog’s back.

\n') - self.assertEqual(content, expected) + self.assertEqual(page.content, expected) expected = { 'category': 'Blog', 'author': 'Author O. Article', @@ -302,7 +302,7 @@ class AdReaderTest(ReaderTest): } for key, value in expected.items(): - self.assertEqual(value, metadata[key], key) + self.assertEqual(value, page.metadata[key], key) @unittest.skipUnless(readers.asciidoc, "asciidoc isn't installed") def test_article_with_asc_options(self): @@ -319,24 +319,24 @@ class AdReaderTest(ReaderTest): class HTMLReaderTest(ReaderTest): def test_article_with_comments(self): - content, metadata = self.read_file(path='article_with_comments.html') + page = self.read_file(path='article_with_comments.html') self.assertEqual(''' Body content - ''', content) + ''', page.content) def test_article_with_keywords(self): - content, metadata = self.read_file(path='article_with_keywords.html') + page = self.read_file(path='article_with_keywords.html') expected = { 'tags': ['foo', 'bar', 'foobar'], } for key, value in expected.items(): - self.assertEqual(value, metadata[key], key) + self.assertEqual(value, page.metadata[key], key) def test_article_with_metadata(self): - content, metadata = self.read_file(path='article_with_metadata.html') + page = self.read_file(path='article_with_metadata.html') expected = { 'category': 'yeah', 'author': 'Alexis Métaireau', @@ -348,21 +348,19 @@ class HTMLReaderTest(ReaderTest): } for key, value in expected.items(): - self.assertEqual(value, metadata[key], key) + self.assertEqual(value, page.metadata[key], key) def test_article_with_null_attributes(self): - content, metadata = self.read_file( - path='article_with_null_attributes.html') + page = self.read_file(path='article_with_null_attributes.html') self.assertEqual(''' Ensure that empty attributes are copied properly. - ''', content) + ''', page.content) def test_article_metadata_key_lowercase(self): # Keys of metadata should be lowercase. - content, metadata = self.read_file( - path='article_with_uppercase_metadata.html') - self.assertIn('category', metadata, 'Key should be lowercase.') - self.assertEqual('Yeah', metadata.get('category'), + page = self.read_file(path='article_with_uppercase_metadata.html') + self.assertIn('category', page.metadata, 'Key should be lowercase.') + self.assertEqual('Yeah', page.metadata.get('category'), 'Value keeps cases.') From f63325aa9a459b2c510bd16297a381544a1571ea Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Mon, 3 Jun 2013 15:26:23 -0400 Subject: [PATCH 14/22] test_generators: Replace CUR_DIR with CONTENT_DIR for subdir cat. detection In situations where I've cleared ARTICLE_DIR, I've done so to ensure that there are no directories that will override the DEFAULT_CATEGORY due to USE_FOLDER_AS_CATEGORY. --- pelican/tests/test_generators.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pelican/tests/test_generators.py b/pelican/tests/test_generators.py index 48aff498..f5ed6b85 100644 --- a/pelican/tests/test_generators.py +++ b/pelican/tests/test_generators.py @@ -15,6 +15,7 @@ from pelican.settings import DEFAULT_CONFIG from pelican.tests.support import unittest, get_settings CUR_DIR = os.path.dirname(__file__) +CONTENT_DIR = os.path.join(CUR_DIR, 'content') class TestArticlesGenerator(unittest.TestCase): @@ -30,12 +31,10 @@ class TestArticlesGenerator(unittest.TestCase): """ if self.generator is None: settings = get_settings(filenames={}) - settings['ARTICLE_DIR'] = 'content' settings['DEFAULT_CATEGORY'] = 'Default' settings['DEFAULT_DATE'] = (1970, 1, 1) self.generator = ArticlesGenerator(settings.copy(), settings, - CUR_DIR, settings['THEME'], None, - settings['MARKUP']) + CONTENT_DIR, settings['THEME'], None, settings['MARKUP']) self.generator.generate_context() return self.generator @@ -118,14 +117,13 @@ class TestArticlesGenerator(unittest.TestCase): def test_do_not_use_folder_as_category(self): settings = DEFAULT_CONFIG.copy() - settings['ARTICLE_DIR'] = 'content' settings['DEFAULT_CATEGORY'] = 'Default' settings['DEFAULT_DATE'] = (1970, 1, 1) settings['USE_FOLDER_AS_CATEGORY'] = False settings['filenames'] = {} generator = ArticlesGenerator( - settings.copy(), settings, CUR_DIR, DEFAULT_CONFIG['THEME'], None, - DEFAULT_CONFIG['MARKUP']) + settings.copy(), settings, CONTENT_DIR, DEFAULT_CONFIG['THEME'], + None, DEFAULT_CONFIG['MARKUP']) generator.generate_context() # test for name # categories are grouped by slug; if two categories have the same slug @@ -213,12 +211,12 @@ class TestPageGenerator(unittest.TestCase): def test_generate_context(self): settings = get_settings(filenames={}) - settings['PAGE_DIR'] = 'TestPages' + settings['PAGE_DIR'] = 'TestPages' # relative to CUR_DIR settings['DEFAULT_DATE'] = (1970, 1, 1) - generator = PagesGenerator(settings.copy(), settings, CUR_DIR, - settings['THEME'], None, - settings['MARKUP']) + generator = PagesGenerator( + settings.copy(), settings, CUR_DIR, settings['THEME'], None, + settings['MARKUP']) generator.generate_context() pages = self.distill_pages(generator.pages) hidden_pages = self.distill_pages(generator.hidden_pages) From fdde17281d36f28cf7084a21daafa77eea9fe3cb Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 4 Jan 2013 19:47:55 -0500 Subject: [PATCH 15/22] contents: Page fallbacks for context and localsiteurl --- pelican/contents.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pelican/contents.py b/pelican/contents.py index 5f2e66b0..fd33bc4c 100644 --- a/pelican/contents.py +++ b/pelican/contents.py @@ -48,6 +48,8 @@ class Content(object): self.settings = settings self._content = content + if context is None: + context = {} self._context = context self.translations = [] @@ -220,7 +222,7 @@ class Content(object): @property def content(self): - return self.get_content(self._context['localsiteurl']) + return self.get_content(self._context.get('localsiteurl', '')) def _get_summary(self): """Returns the summary of an article. From 29f0aa39d2239122782fa5c1a547827cf7acedb0 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 22 Mar 2013 00:13:37 -0400 Subject: [PATCH 16/22] content: Don't update static content --- pelican/contents.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pelican/contents.py b/pelican/contents.py index fd33bc4c..1b604f19 100644 --- a/pelican/contents.py +++ b/pelican/contents.py @@ -171,6 +171,9 @@ class Content(object): :param siteurl: siteurl which is locally generated by the writer in case of RELATIVE_URLS. """ + if not content: + return content + hrefs = re.compile(r""" (?P<\s*[^\>]* # match tag with src and href attr (?:href|src)\s*=) From 4058cfdea44725c520dc574c033476892c235d58 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sun, 24 Mar 2013 10:19:01 -0400 Subject: [PATCH 17/22] settings: Add a warning for folks using FILES_TO_COPY. --- pelican/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pelican/settings.py b/pelican/settings.py index 13896032..35e76346 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -257,6 +257,8 @@ def configure_settings(settings): for old,new,doc in [ ('LESS_GENERATOR', 'the Webassets plugin', None), + ('FILES_TO_COPY', 'STATIC_PATHS and EXTRA_PATH_METADATA', + 'https://github.com/getpelican/pelican/pull/795'), ]: if old in settings: message = 'The {} setting has been removed in favor of {}' From 38c22e83b6bfe7fdcd4f8987063cd240d5655539 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Mon, 3 Jun 2013 15:29:54 -0400 Subject: [PATCH 18/22] readers: Ensure the reader class is enabled before instantiating Otherwise the MarkdownReader fails with: 'bool' object is not callable if Markdown is not installed. Reported-by: Deniz Turgut --- pelican/readers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pelican/readers.py b/pelican/readers.py index 3e00b430..bd9f5914 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -361,15 +361,17 @@ def read_file(base_path, path, content_class=Page, fmt=None, if settings is None: settings = {} - reader = EXTENSIONS[fmt](settings) + reader_class = EXTENSIONS[fmt] + if not reader_class.enabled: + raise ValueError('Missing dependencies for {}'.format(fmt)) + + reader = reader_class(settings) + settings_key = '%s_EXTENSIONS' % fmt.upper() if settings and settings_key in settings: reader.extensions = settings[settings_key] - if not reader.enabled: - raise ValueError("Missing dependencies for %s" % fmt) - metadata = default_metadata( settings=settings, process=reader.process_metadata) metadata.update(path_metadata( From 8797f0ebefee5df4edef6f9a8a98685e1cc1ee92 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 7 Jun 2013 14:56:21 -0400 Subject: [PATCH 19/22] docs/plugins.rst: Document signal name changes --- docs/plugins.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/plugins.rst b/docs/plugins.rst index 9e262962..9bf08ff3 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -104,3 +104,22 @@ request if you need them! def register(): signals.content_object_init.connect(test, sender=contents.Article) + +.. note:: + + After Pelican 3.2, signal names were standardized. Older plugins + may need to be updated to use the new names: + + ========================== =========================== + Old name New name + ========================== =========================== + article_generate_context article_generator_context + article_generate_finalized article_generator_finalized + article_generate_preread article_generator_preread + pages_generate_context page_generator_context + pages_generate_preread page_generator_preread + pages_generator_finalized page_generator_finalized + pages_generator_init page_generator_init + static_generate_context static_generator_context + static_generate_preread static_generator_preread + ========================== =========================== From ce0a9b697e08fee069cda6c12a8b9921c183830c Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 18 Jan 2013 19:00:45 -0500 Subject: [PATCH 20/22] Document path metadata extraction --- docs/getting_started.rst | 2 ++ docs/settings.rst | 49 ++++++++++++++++++++++++++++++++++++++++ pelican/settings.py | 2 +- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index b41f8c18..b2e0aeda 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -213,6 +213,8 @@ The idea behind "pages" is that they are usually not temporal in nature and are used for content that does not change very often (e.g., "About" or "Contact" pages). +.. _internal_metadata: + File metadata ------------- diff --git a/docs/settings.rst b/docs/settings.rst index c7d7f199..ffcddc7a 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -62,9 +62,12 @@ Setting name (default value) What doe For example, if you would like to extract both the date and the slug, you could set something like: ``'(?P\d{4}-\d{2}-\d{2})_(?P.*)'``. + See :ref:`path_metadata`. `PATH_METADATA` (``''``) Like ``FILENAME_METADATA``, but parsed from a page's full path relative to the content source directory. + See :ref:`path_metadata`. `EXTRA_PATH_METADATA` (``{}``) Extra metadata dictionaries keyed by relative path. + See :ref:`path_metadata`. `DELETE_OUTPUT_DIRECTORY` (``False``) Delete the output directory, and **all** of its contents, before generating new files. This can be useful in preventing older, unnecessary files from persisting in your output. However, **this is @@ -335,6 +338,52 @@ your resume, and a contact page — you could have:: 'src/resume.html': 'dest/resume.html', 'src/contact.html': 'dest/contact.html'} + +.. _path_metadata: + +Path metadata +============= + +Not all metadata needs to be `embedded in source file itself`__. For +example, blog posts are often named following a ``YYYY-MM-DD-SLUG.rst`` +pattern, or nested into ``YYYY/MM/DD-SLUG`` directories. To extract +metadata from the filename or path, set ``FILENAME_METADATA`` or +``PATH_METADATA`` to regular expressions that use Python's `group name +notation`_ ``(?P…)``. If you want to attach additional metadata +but don't want to encode it in the path, you can set +``EXTRA_PATH_METADATA``: + +.. parsed-literal:: + + EXTRA_PATH_METADATA = { + 'relative/path/to/file-1': { + 'key-1a': 'value-1a', + 'key-1b': 'value-1b', + }, + 'relative/path/to/file-2': { + 'key-2': 'value-2', + }, + } + +This can be a convenient way to shift the installed location of a +particular file: + +.. parsed-literal:: + + # Take advantage of the following defaults + # STATIC_SAVE_AS = '{path}' + # STATIC_URL = '{path}' + STATIC_PATHS = [ + 'extra/robots.txt', + ] + EXTRA_PATH_METADATA = { + 'extra/robots.txt': {'path': 'robots.txt'}, + } + +__ internal_metadata__ +.. _group name notation: + http://docs.python.org/3/library/re.html#regular-expression-syntax + Feed settings ============= diff --git a/pelican/settings.py b/pelican/settings.py index 35e76346..df3ad6b6 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -258,7 +258,7 @@ def configure_settings(settings): for old,new,doc in [ ('LESS_GENERATOR', 'the Webassets plugin', None), ('FILES_TO_COPY', 'STATIC_PATHS and EXTRA_PATH_METADATA', - 'https://github.com/getpelican/pelican/pull/795'), + 'https://github.com/getpelican/pelican/blob/master/docs/settings.rst#path-metadata'), ]: if old in settings: message = 'The {} setting has been removed in favor of {}' From 9b42b2a13082e325fabf3f8e4b3f86b3b775bfe7 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 12 Jun 2013 16:07:24 -0400 Subject: [PATCH 21/22] generators: get_files() should use paths relative to Generator.path All paths should be relative to Generator.path unless we're actively accessing the filesystem. This makes the argument less ambiguous, so we have less likelyhood of joining paths multiple times. --- pelican/generators.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/pelican/generators.py b/pelican/generators.py index c48f8067..a01281dc 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -102,23 +102,25 @@ class Generator(object): def get_files(self, path, exclude=[], extensions=None): """Return a list of files to use, based on rules - :param path: the path to search the file on + :param path: the path to search (relative to self.path) :param exclude: the list of path to exclude :param extensions: the list of allowed extensions (if False, all extensions are allowed) """ files = [] + root = os.path.join(self.path, path) - if os.path.isdir(path): - for root, dirs, temp_files in os.walk(path, followlinks=True): + if os.path.isdir(root): + for dirpath, dirs, temp_files in os.walk(root, followlinks=True): for e in exclude: if e in dirs: dirs.remove(e) + reldir = os.path.relpath(dirpath, self.path) for f in temp_files: - fp = os.path.join(root, f) + fp = os.path.join(reldir, f) if self._include_path(fp, extensions): files.append(fp) - elif os.path.exists(path) and self._include_path(path, extensions): + elif os.path.exists(root) and self._include_path(path, extensions): files.append(path) # can't walk non-directories return files @@ -372,12 +374,9 @@ class ArticlesGenerator(Generator): def generate_context(self): """Add the articles into the shared context""" - article_path = os.path.normpath( # we have to remove trailing slashes - os.path.join(self.path, self.settings['ARTICLE_DIR']) - ) all_articles = [] for f in self.get_files( - article_path, + self.settings['ARTICLE_DIR'], exclude=self.settings['ARTICLE_EXCLUDES']): try: article = read_file( @@ -485,7 +484,7 @@ class PagesGenerator(Generator): all_pages = [] hidden_pages = [] for f in self.get_files( - os.path.join(self.path, self.settings['PAGE_DIR']), + self.settings['PAGE_DIR'], exclude=self.settings['PAGE_EXCLUDES']): try: page = read_file( @@ -547,7 +546,7 @@ class StaticGenerator(Generator): # walk static paths for static_path in self.settings['STATIC_PATHS']: for f in self.get_files( - os.path.join(self.path, static_path), extensions=False): + static_path, extensions=False): static = read_file( base_path=self.path, path=f, content_class=Static, fmt='static', From 180cf9165fd0f35d57c51b4cc4111f1a5a92858c Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 13 Jun 2013 21:12:43 -0400 Subject: [PATCH 22/22] settings: Fix deprecation warning in configure_settings() The broken code came from my 1d4d86c (settings: Rework the LESS_GENERATOR removal warning for easy extension, 2013-03-24), where I put formatting placeholders ({}) into the warning message, but forgot to fill them in :/. --- pelican/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pelican/settings.py b/pelican/settings.py index df3ad6b6..c6cc6c3c 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -261,9 +261,10 @@ def configure_settings(settings): 'https://github.com/getpelican/pelican/blob/master/docs/settings.rst#path-metadata'), ]: if old in settings: - message = 'The {} setting has been removed in favor of {}' + message = 'The {} setting has been removed in favor of {}'.format( + old, new) if doc: - message += ', see {} for details' + message += ', see {} for details'.format(doc) logger.warning(message) return settings