diff --git a/pelican/__init__.py b/pelican/__init__.py index c0f33687..52d371ec 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -134,6 +134,7 @@ class Pelican(object): """Run the generators and return""" context = self.settings.copy() + filenames = {} # share the dict between all the generators generators = [ cls( context, @@ -142,7 +143,8 @@ class Pelican(object): self.theme, self.output_path, self.markup, - self.delete_outputdir + self.delete_outputdir, + filenames=filenames ) for cls in self.get_generator_classes() ] diff --git a/pelican/contents.py b/pelican/contents.py index bb2b5a6e..0d599771 100644 --- a/pelican/contents.py +++ b/pelican/contents.py @@ -3,13 +3,16 @@ import copy import locale import logging import functools +import os +import re +import urlparse from datetime import datetime from sys import platform, stdin from pelican.settings import _DEFAULT_CONFIG -from pelican.utils import slugify, truncate_html_words +from pelican.utils import slugify, truncate_html_words, memoized from pelican import signals logger = logging.getLogger(__name__) @@ -25,7 +28,7 @@ class Page(object): default_template = 'page' def __init__(self, content, metadata=None, settings=None, - filename=None): + filename=None, context=None): # init parameters if not metadata: metadata = {} @@ -34,6 +37,7 @@ class Page(object): self.settings = settings self._content = content + self._context = context self.translations = [] local_metadata = dict(settings.get('DEFAULT_METADATA', ())) @@ -128,12 +132,56 @@ class Page(object): key = key if self.in_default_lang else 'lang_%s' % key return self._expand_settings(key) + def _update_content(self, content): + """Change all the relative paths of the content to relative paths + suitable for the ouput content. + + :param content: content resource that will be passed to the templates. + """ + hrefs = re.compile(r""" + (?P<\s*[^\>]* # match tag with src and href attr + (?:href|src)\s*=) + + (?P["\']) # require value to be quoted + (?P\|(?P.*?)\|(?P.*?)) # the url value + \2""", re.X) + + def replacer(m): + what = m.group('what') + value = m.group('value') + origin = m.group('path') + # we support only filename for now. the plan is to support + # categories, tags, etc. in the future, but let's keep things + # simple for now. + if what == 'filename': + if value.startswith('/'): + value = value[1:] + else: + # relative to the filename of this content + value = self.get_relative_filename( + os.path.join(self.relative_dir, value) + ) + + if value in self._context['filenames']: + origin = urlparse.urljoin(self._context['SITEURL'], + self._context['filenames'][value].url) + else: + logger.warning(u"Unable to find {fn}, skipping url" + " replacement".format(fn=value)) + + return m.group('markup') + m.group('quote') + origin \ + + m.group('quote') + + return hrefs.sub(replacer, content) + @property + @memoized def content(self): if hasattr(self, "_get_content"): content = self._get_content() else: content = self._content + content = self._update_content(content) return content def _get_summary(self): @@ -143,7 +191,8 @@ class Page(object): return self._summary else: if self.settings['SUMMARY_MAX_LENGTH']: - return truncate_html_words(self.content, self.settings['SUMMARY_MAX_LENGTH']) + return truncate_html_words(self.content, + self.settings['SUMMARY_MAX_LENGTH']) return self.content def _set_summary(self, summary): @@ -162,6 +211,27 @@ class Page(object): else: return self.default_template + def get_relative_filename(self, filename=None): + """Return the relative path (from the content path) to the given + filename. + + If no filename is specified, use the filename of this content object. + """ + if not filename: + filename = self.filename + + return os.path.relpath( + os.path.abspath(os.path.join(self.settings['PATH'], filename)), + os.path.abspath(self.settings['PATH']) + ) + + @property + def relative_dir(self): + return os.path.dirname(os.path.relpath( + os.path.abspath(self.filename), + os.path.abspath(self.settings['PATH'])) + ) + class Article(Page): mandatory_properties = ('title', 'date', 'category') @@ -227,11 +297,27 @@ class Author(URLWrapper): pass +class StaticContent(object): + def __init__(self, src, dst=None, settings=None): + if not settings: + settings = copy.deepcopy(_DEFAULT_CONFIG) + self.src = src + self.url = dst or src + self.filepath = os.path.join(settings['PATH'], src) + self.save_as = os.path.join(settings['OUTPUT_PATH'], self.url) + + def __str__(self): + return str(self.filepath.encode('utf-8', 'replace')) + + def __unicode__(self): + return self.filepath + + def is_valid_content(content, f): try: content.check_properties() return True except NameError, e: - logger.error(u"Skipping %s: impossible to find informations about '%s'"\ - % (f, e)) + logger.error(u"Skipping %s: impossible to find informations about" + "'%s'" % (f, e)) return False diff --git a/pelican/generators.py b/pelican/generators.py index 020a3711..2da7d8bd 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -5,6 +5,7 @@ import random import logging import datetime import subprocess +import shutil from codecs import open from collections import defaultdict @@ -15,9 +16,10 @@ from operator import attrgetter, itemgetter from jinja2 import (Environment, FileSystemLoader, PrefixLoader, ChoiceLoader, BaseLoader, TemplateNotFound) -from pelican.contents import Article, Page, Category, is_valid_content +from pelican.contents import Article, Page, Category, StaticContent, \ + is_valid_content from pelican.readers import read_file -from pelican.utils import copy, process_translations +from pelican.utils import copy, process_translations, mkdir_p from pelican import signals @@ -61,6 +63,7 @@ class Generator(object): # get custom Jinja filters from user settings custom_filters = self.settings.get('JINJA_FILTERS', {}) self.env.filters.update(custom_filters) + self.context['filenames'] = kwargs.get('filenames', {}) signals.generator_init.send(self) @@ -82,8 +85,10 @@ class Generator(object): :param path: the path to search the file on :param exclude: the list of path to exclude + :param extensions: the list of allowed extensions (if False, all + extensions are allowed) """ - if not extensions: + if extensions is None: extensions = self.markup files = [] @@ -97,10 +102,17 @@ class Generator(object): for e in exclude: if e in dirs: dirs.remove(e) - files.extend([os.sep.join((root, f)) for f in temp_files - if True in [f.endswith(ext) for ext in extensions]]) + for f in temp_files: + if extensions is False or \ + (True in [f.endswith(ext) for ext in extensions]): + files.append(os.sep.join((root, f))) return files + def add_filename(self, content): + location = os.path.relpath(os.path.abspath(content.filename), + os.path.abspath(self.path)) + self.context['filenames'][location] = content + def _update_context(self, items): """Update the context with the given items from the currrent processor. @@ -300,7 +312,7 @@ class ArticlesGenerator(Generator): self.generate_drafts(write) def generate_context(self): - """change the context""" + """Add the articles into the shared context""" article_path = os.path.normpath( # we have to remove trailing slashes os.path.join(self.path, self.settings['ARTICLE_DIR']) @@ -341,10 +353,12 @@ class ArticlesGenerator(Generator): signals.article_generate_context.send(self, metadata=metadata) article = Article(content, metadata, settings=self.settings, - filename=f) + filename=f, context=self.context) if not is_valid_content(article, f): continue + self.add_filename(article) + if article.status == "published": if hasattr(article, 'tags'): for tag in article.tags: @@ -440,11 +454,14 @@ class PagesGenerator(Generator): except Exception, e: logger.warning(u'Could not process %s\n%s' % (f, str(e))) continue - signals.pages_generate_context.send(self, metadata=metadata ) + signals.pages_generate_context.send(self, metadata=metadata) page = Page(content, metadata, settings=self.settings, - filename=f) + filename=f, context=self.context) if not is_valid_content(page, f): continue + + self.add_filename(page) + if page.status == "published": all_pages.append(page) elif page.status == "hidden": @@ -479,17 +496,33 @@ class StaticGenerator(Generator): copy(path, source, os.path.join(output_path, destination), final_path, overwrite=True) - def generate_output(self, writer): + def generate_context(self): + self.staticfiles = [] - self._copy_paths(self.settings['STATIC_PATHS'], self.path, - 'static', self.output_path) + # walk static paths + for static_path in self.settings['STATIC_PATHS']: + for f in self.get_files( + os.path.join(self.path, static_path), extensions=False): + f_rel = os.path.relpath(f, self.path) + # TODO remove this hardcoded 'static' subdirectory + sc = StaticContent(f_rel, os.path.join('static', f_rel), + settings=self.settings) + self.staticfiles.append(sc) + self.context['filenames'][f_rel] = sc + # same thing for FILES_TO_COPY + for src, dest in self.settings['FILES_TO_COPY']: + sc = StaticContent(src, dest, settings=self.settings) + self.staticfiles.append(sc) + self.context['filenames'][src] = sc + + def generate_output(self, writer): self._copy_paths(self.settings['THEME_STATIC_PATHS'], self.theme, 'theme', self.output_path, '.') - - # copy all the files needed - for source, destination in self.settings['FILES_TO_COPY']: - copy(source, self.path, self.output_path, destination, - overwrite=True) + # copy all StaticContent files + for sc in self.staticfiles: + mkdir_p(os.path.dirname(sc.save_as)) + shutil.copy(sc.filepath, sc.save_as) + logger.info('copying %s to %s' % (sc.filepath, sc.save_as)) class PdfGenerator(Generator): @@ -532,8 +565,8 @@ class PdfGenerator(Generator): try: os.mkdir(pdf_path) except OSError: - logger.error("Couldn't create the pdf output folder in " + pdf_path) - pass + logger.error("Couldn't create the pdf output folder in " + + pdf_path) for article in self.context['articles']: self._create_pdf(article, pdf_path) diff --git a/pelican/utils.py b/pelican/utils.py index 79387357..6ca797c4 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -4,7 +4,9 @@ import re import pytz import shutil import logging -from collections import defaultdict +import errno +from collections import defaultdict, Hashable +from functools import partial from codecs import open from datetime import datetime @@ -19,6 +21,32 @@ class NoFilesError(Exception): pass +class memoized(object): + '''Decorator. Caches a function's return value each time it is called. + If called later with the same arguments, the cached value is returned + (not reevaluated). + ''' + def __init__(self, func): + self.func = func + self.cache = {} + def __call__(self, *args): + if not isinstance(args, Hashable): + # uncacheable. a list, for instance. + # better to not cache than blow up. + return self.func(*args) + if args in self.cache: + return self.cache[args] + else: + value = self.func(*args) + self.cache[args] = value + return value + def __repr__(self): + '''Return the function's docstring.''' + return self.func.__doc__ + def __get__(self, obj, objtype): + '''Support instance methods.''' + return partial(self.__call__, obj) + def get_date(string): """Return a datetime object from a string. @@ -300,3 +328,11 @@ def set_date_tzinfo(d, tz_name=None): return tz.localize(d) else: return d + + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError, e: + if e.errno != errno.EEXIST: + raise diff --git a/pelican/writers.py b/pelican/writers.py index b932a805..b24a90dd 100644 --- a/pelican/writers.py +++ b/pelican/writers.py @@ -2,12 +2,10 @@ from __future__ import with_statement import os -import re import locale import logging from codecs import open -from functools import partial from feedgenerator import Atom1Feed, Rss201rev2Feed from jinja2 import Markup from pelican.paginator import Paginator @@ -129,8 +127,6 @@ class Writer(object): localcontext['SITEURL'] = get_relative_path(name) localcontext.update(kwargs) - if relative_urls: - self.update_context_contents(name, localcontext) # check paginated paginated = paginated or {} @@ -168,66 +164,3 @@ class Writer(object): else: # no pagination _write_file(template, localcontext, self.output_path, name) - - def update_context_contents(self, name, context): - """Recursively run the context to find elements (articles, pages, etc) - whose content getter needs to be modified in order to deal with - relative paths. - - :param name: name of the file to output. - :param context: dict that will be passed to the templates, which need - to be updated. - """ - def _update_content(name, input): - """Change all the relatives paths of the input content to relatives - paths suitable fot the ouput content - - :param name: path of the output. - :param input: input resource that will be passed to the templates. - """ - content = input._content - - hrefs = re.compile(r""" - (?P<\s*[^\>]* # match tag with src and href attr - (?:href|src)\s*=\s* - ) - (?P["\']) # require value to be quoted - (?![#?]) # don't match fragment or query URLs - (?![a-z]+:) # don't match protocol URLS - (?P.*?) # the url value - \2""", re.X) - - def replacer(m): - relative_path = m.group('path') - dest_path = os.path.normpath( - os.sep.join((get_relative_path(name), "static", - relative_path))) - - # On Windows, make sure we end up with Unix-like paths. - if os.name == 'nt': - dest_path = dest_path.replace('\\', '/') - - return m.group('markup') + m.group('quote') + dest_path \ - + m.group('quote') - - return hrefs.sub(replacer, content) - - if context is None: - return - if hasattr(context, 'values'): - context = context.values() - - for item in context: - # run recursively on iterables - if hasattr(item, '__iter__'): - self.update_context_contents(name, item) - - # if it is a content, patch it - elif hasattr(item, '_content'): - relative_path = get_relative_path(name) - - paths = self.reminder.setdefault(item, []) - if relative_path not in paths: - paths.append(relative_path) - setattr(item, "_get_content", - partial(_update_content, name, item)) diff --git a/samples/content/another_super_article.rst b/samples/content/another_super_article.rst index 5ec1e2b8..e6e0a92c 100644 --- a/samples/content/another_super_article.rst +++ b/samples/content/another_super_article.rst @@ -14,7 +14,7 @@ Why not ? After all, why not ? It's pretty simple to do it, and it will allow me to write my blogposts in rst ! YEAH ! -.. image:: pictures/Sushi.jpg +.. image:: |filename|/pictures/Sushi.jpg :height: 450 px :width: 600 px :alt: alternate text diff --git a/samples/content/cat1/markdown-article.md b/samples/content/cat1/markdown-article.md index 3bf56dc0..5307b47a 100644 --- a/samples/content/cat1/markdown-article.md +++ b/samples/content/cat1/markdown-article.md @@ -2,3 +2,6 @@ Title: A markdown powered article Date: 2011-04-20 You're mutually oblivious. + +[a root-relative link to unbelievable](|filename|/unbelievable.rst) +[a file-relative link to unbelievable](|filename|../unbelievable.rst) diff --git a/samples/content/pages/test_page.rst b/samples/content/pages/test_page.rst index 06f91c10..2285f17b 100644 --- a/samples/content/pages/test_page.rst +++ b/samples/content/pages/test_page.rst @@ -5,7 +5,7 @@ This is a test page Just an image. -.. image:: pictures/Fat_Cat.jpg +.. image:: |filename|/pictures/Fat_Cat.jpg :height: 450 px :width: 600 px :alt: alternate text diff --git a/samples/content/super_article.rst b/samples/content/super_article.rst index 1dfd8e34..76e57683 100644 --- a/samples/content/super_article.rst +++ b/samples/content/super_article.rst @@ -16,12 +16,12 @@ This is a simple title And here comes the cool stuff_. -.. image:: pictures/Sushi.jpg +.. image:: |filename|/pictures/Sushi.jpg :height: 450 px :width: 600 px :alt: alternate text -.. image:: pictures/Sushi_Macro.jpg +.. image:: |filename|/pictures/Sushi_Macro.jpg :height: 450 px :width: 600 px :alt: alternate text diff --git a/samples/content/unbelievable.rst b/samples/content/unbelievable.rst index 11443e9a..20cb9dc7 100644 --- a/samples/content/unbelievable.rst +++ b/samples/content/unbelievable.rst @@ -4,3 +4,6 @@ Unbelievable ! :date: 2010-10-15 20:30 Or completely awesome. Depends the needs. + +`a root-relative link to markdown-article <|filename|/cat1/markdown-article.md>`_ +`a file-relative link to markdown-article <|filename|cat1/markdown-article.md>`_ diff --git a/tests/support.py b/tests/support.py index 5b63eeba..b6db5195 100644 --- a/tests/support.py +++ b/tests/support.py @@ -15,6 +15,7 @@ from tempfile import mkdtemp from shutil import rmtree from pelican.contents import Article +from pelican.settings import _DEFAULT_CONFIG try: import unittest2 as unittest @@ -149,3 +150,10 @@ def module_exists(module_name): return False else: return True + + +def get_settings(): + settings = _DEFAULT_CONFIG.copy() + settings['DIRECT_TEMPLATES'] = ['archives'] + settings['filenames'] = {} + return settings diff --git a/tests/test_generators.py b/tests/test_generators.py index f8f6a3f4..d2ad6f01 100644 --- a/tests/test_generators.py +++ b/tests/test_generators.py @@ -11,7 +11,7 @@ from pelican.generators import ArticlesGenerator, PagesGenerator, \ TemplatePagesGenerator from pelican.writers import Writer from pelican.settings import _DEFAULT_CONFIG -from .support import unittest +from .support import unittest, get_settings CUR_DIR = os.path.dirname(__file__) @@ -28,20 +28,20 @@ class TestArticlesGenerator(unittest.TestCase): for each test. """ if self.generator is None: - settings = _DEFAULT_CONFIG.copy() + settings = get_settings() settings['ARTICLE_DIR'] = 'content' settings['DEFAULT_CATEGORY'] = 'Default' settings['DEFAULT_DATE'] = (1970, 01, 01) self.generator = ArticlesGenerator(settings.copy(), settings, - CUR_DIR, _DEFAULT_CONFIG['THEME'], None, - _DEFAULT_CONFIG['MARKUP']) + CUR_DIR, settings['THEME'], None, + settings['MARKUP']) self.generator.generate_context() return self.generator def distill_articles(self, articles): distilled = [] for page in articles: - distilled.append([ + distilled.append([ page.title, page.status, page.category.name, @@ -51,16 +51,16 @@ class TestArticlesGenerator(unittest.TestCase): return distilled def test_generate_feeds(self): - - generator = ArticlesGenerator(None, {'FEED_ALL_ATOM': _DEFAULT_CONFIG['FEED_ALL_ATOM']}, - None, _DEFAULT_CONFIG['THEME'], None, - _DEFAULT_CONFIG['MARKUP']) + settings = get_settings() + generator = ArticlesGenerator(settings, + {'FEED_ALL_ATOM': settings['FEED_ALL_ATOM']}, None, + settings['THEME'], None, settings['MARKUP']) writer = MagicMock() generator.generate_feeds(writer) - writer.write_feed.assert_called_with([], None, 'feeds/all.atom.xml') + writer.write_feed.assert_called_with([], settings, 'feeds/all.atom.xml') - generator = ArticlesGenerator(None, {'FEED_ALL_ATOM': None}, None, - _DEFAULT_CONFIG['THEME'], None, None) + generator = ArticlesGenerator(settings, {'FEED_ALL_ATOM': None}, None, + settings['THEME'], None, None) writer = MagicMock() generator.generate_feeds(writer) self.assertFalse(writer.write_feed.called) @@ -106,11 +106,10 @@ class TestArticlesGenerator(unittest.TestCase): def test_direct_templates_save_as_default(self): - settings = _DEFAULT_CONFIG.copy() - settings['DIRECT_TEMPLATES'] = ['archives'] - generator = ArticlesGenerator(settings.copy(), settings, None, - _DEFAULT_CONFIG['THEME'], None, - _DEFAULT_CONFIG['MARKUP']) + settings = get_settings() + generator = ArticlesGenerator(settings, settings, None, + settings['THEME'], None, + settings['MARKUP']) write = MagicMock() generator.generate_direct_templates(write) write.assert_called_with("archives.html", @@ -119,12 +118,12 @@ class TestArticlesGenerator(unittest.TestCase): def test_direct_templates_save_as_modified(self): - settings = _DEFAULT_CONFIG.copy() + settings = get_settings() settings['DIRECT_TEMPLATES'] = ['archives'] settings['ARCHIVES_SAVE_AS'] = 'archives/index.html' generator = ArticlesGenerator(settings, settings, None, - _DEFAULT_CONFIG['THEME'], None, - _DEFAULT_CONFIG['MARKUP']) + settings['THEME'], None, + settings['MARKUP']) write = MagicMock() generator.generate_direct_templates(write) write.assert_called_with("archives/index.html", @@ -133,12 +132,12 @@ class TestArticlesGenerator(unittest.TestCase): def test_direct_templates_save_as_false(self): - settings = _DEFAULT_CONFIG.copy() + settings = get_settings() settings['DIRECT_TEMPLATES'] = ['archives'] settings['ARCHIVES_SAVE_AS'] = 'archives/index.html' generator = ArticlesGenerator(settings, settings, None, - _DEFAULT_CONFIG['THEME'], None, - _DEFAULT_CONFIG['MARKUP']) + settings['THEME'], None, + settings['MARKUP']) write = MagicMock() generator.generate_direct_templates(write) write.assert_called_count == 0 @@ -174,13 +173,13 @@ class TestPageGenerator(unittest.TestCase): return distilled def test_generate_context(self): - settings = _DEFAULT_CONFIG.copy() - + settings = get_settings() settings['PAGE_DIR'] = 'TestPages' settings['DEFAULT_DATE'] = (1970, 01, 01) + generator = PagesGenerator(settings.copy(), settings, CUR_DIR, - _DEFAULT_CONFIG['THEME'], None, - _DEFAULT_CONFIG['MARKUP']) + settings['THEME'], None, + settings['MARKUP']) generator.generate_context() pages = self.distill_pages(generator.pages) hidden_pages = self.distill_pages(generator.hidden_pages) @@ -214,7 +213,7 @@ class TestTemplatePagesGenerator(unittest.TestCase): def test_generate_output(self): - settings = _DEFAULT_CONFIG.copy() + settings = get_settings() settings['STATIC_PATHS'] = ['static'] settings['TEMPLATE_PAGES'] = { 'template/source.html': 'generated/file.html'