Don't rewrite URLs

Remove the code that was appending ../static in front of some URLs, and add a
way to do cross-content linking.

pelican/__init__.py

@@ -134,6 +134,7 @@ class Pelican(object):
         """Run the generators and return"""
 
         context = self.settings.copy()
+        filenames = {}  # share the dict between all the generators
         generators = [
             cls(
                 context,
@@ -142,7 +143,8 @@ class Pelican(object):
                 self.theme,
                 self.output_path,
                 self.markup,
-                self.delete_outputdir
+                self.delete_outputdir,
+                filenames=filenames
             ) for cls in self.get_generator_classes()
         ]
 

pelican/contents.py

@@ -3,13 +3,16 @@ import copy
 import locale
 import logging
 import functools
+import os
+import re
+import urlparse
 
 from datetime import datetime
 from sys import platform, stdin
 
 
 from pelican.settings import _DEFAULT_CONFIG
-from pelican.utils import slugify, truncate_html_words
+from pelican.utils import slugify, truncate_html_words, memoized
 from pelican import signals
 
 logger = logging.getLogger(__name__)
@@ -25,7 +28,7 @@ class Page(object):
     default_template = 'page'
 
     def __init__(self, content, metadata=None, settings=None,
-                 filename=None):
+                 filename=None, context=None):
         # init parameters
         if not metadata:
             metadata = {}
@@ -34,6 +37,7 @@ class Page(object):
 
         self.settings = settings
         self._content = content
+        self._context = context
         self.translations = []
 
         local_metadata = dict(settings.get('DEFAULT_METADATA', ()))
@@ -128,12 +132,56 @@ class Page(object):
         key = key if self.in_default_lang else 'lang_%s' % key
         return self._expand_settings(key)
 
+    def _update_content(self, content):
+        """Change all the relative paths of the content to relative paths
+        suitable for the ouput content.
+
+        :param content: content resource that will be passed to the templates.
+        """
+        hrefs = re.compile(r"""
+            (?P<markup><\s*[^\>]*  # match tag with src and href attr
+                (?:href|src)\s*=)
+
+            (?P<quote>["\'])      # require value to be quoted
+            (?P<path>\|(?P<what>.*?)\|(?P<value>.*?))  # the url value
+            \2""", re.X)
+
+        def replacer(m):
+            what = m.group('what')
+            value = m.group('value')
+            origin = m.group('path')
+            # we support only filename for now. the plan is to support
+            # categories, tags, etc. in the future, but let's keep things
+            # simple for now.
+            if what == 'filename':
+                if value.startswith('/'):
+                    value = value[1:]
+                else:
+                    # relative to the filename of this content
+                    value = self.get_relative_filename(
+                        os.path.join(self.relative_dir, value)
+                    )
+
+                if value in self._context['filenames']:
+                    origin = urlparse.urljoin(self._context['SITEURL'],
+                             self._context['filenames'][value].url)
+                else:
+                    logger.warning(u"Unable to find {fn}, skipping url"
+                                    " replacement".format(fn=value))
+
+            return m.group('markup') + m.group('quote') + origin \
+                    + m.group('quote')
+
+        return hrefs.sub(replacer, content)
+
     @property
+    @memoized
     def content(self):
         if hasattr(self, "_get_content"):
             content = self._get_content()
         else:
             content = self._content
+        content = self._update_content(content)
         return content
 
     def _get_summary(self):
@@ -143,7 +191,8 @@ class Page(object):
             return self._summary
         else:
             if self.settings['SUMMARY_MAX_LENGTH']:
-                return truncate_html_words(self.content, self.settings['SUMMARY_MAX_LENGTH'])
+                return truncate_html_words(self.content,
+                        self.settings['SUMMARY_MAX_LENGTH'])
             return self.content
 
     def _set_summary(self, summary):
@@ -162,6 +211,27 @@ class Page(object):
         else:
             return self.default_template
 
+    def get_relative_filename(self, filename=None):
+        """Return the relative path (from the content path) to the given
+        filename.
+
+        If no filename is specified, use the filename of this content object.
+        """
+        if not filename:
+            filename = self.filename
+
+        return os.path.relpath(
+            os.path.abspath(os.path.join(self.settings['PATH'], filename)),
+            os.path.abspath(self.settings['PATH'])
+        )
+
+    @property
+    def relative_dir(self):
+        return os.path.dirname(os.path.relpath(
+            os.path.abspath(self.filename),
+            os.path.abspath(self.settings['PATH']))
+        )
+
 
 class Article(Page):
     mandatory_properties = ('title', 'date', 'category')
@@ -227,11 +297,27 @@ class Author(URLWrapper):
     pass
 
 
+class StaticContent(object):
+    def __init__(self, src, dst=None, settings=None):
+        if not settings:
+            settings = copy.deepcopy(_DEFAULT_CONFIG)
+        self.src = src
+        self.url = dst or src
+        self.filepath = os.path.join(settings['PATH'], src)
+        self.save_as = os.path.join(settings['OUTPUT_PATH'], self.url)
+
+    def __str__(self):
+        return str(self.filepath.encode('utf-8', 'replace'))
+
+    def __unicode__(self):
+        return self.filepath
+
+
 def is_valid_content(content, f):
     try:
         content.check_properties()
         return True
     except NameError, e:
-        logger.error(u"Skipping %s: impossible to find informations about '%s'"\
+        logger.error(u"Skipping %s: impossible to find informations about"
-                % (f, e))
+                      "'%s'" % (f, e))
         return False

pelican/generators.py

@@ -5,6 +5,7 @@ import random
 import logging
 import datetime
 import subprocess
+import shutil
 
 from codecs import open
 from collections import defaultdict
@@ -15,9 +16,10 @@ from operator import attrgetter, itemgetter
 from jinja2 import (Environment, FileSystemLoader, PrefixLoader, ChoiceLoader,
                     BaseLoader, TemplateNotFound)
 
-from pelican.contents import Article, Page, Category, is_valid_content
+from pelican.contents import Article, Page, Category, StaticContent, \
+        is_valid_content
 from pelican.readers import read_file
-from pelican.utils import copy, process_translations
+from pelican.utils import copy, process_translations, mkdir_p
 from pelican import signals
 
 
@@ -61,6 +63,7 @@ class Generator(object):
         # get custom Jinja filters from user settings
         custom_filters = self.settings.get('JINJA_FILTERS', {})
         self.env.filters.update(custom_filters)
+        self.context['filenames'] = kwargs.get('filenames', {})
 
         signals.generator_init.send(self)
 
@@ -82,8 +85,10 @@ class Generator(object):
 
         :param path: the path to search the file on
         :param exclude: the list of path to exclude
+        :param extensions: the list of allowed extensions (if False, all
+            extensions are allowed)
         """
-        if not extensions:
+        if extensions is None:
             extensions = self.markup
 
         files = []
@@ -97,10 +102,17 @@ class Generator(object):
             for e in exclude:
                 if e in dirs:
                     dirs.remove(e)
-            files.extend([os.sep.join((root, f)) for f in temp_files
+            for f in temp_files:
-                if True in [f.endswith(ext) for ext in extensions]])
+                if extensions is False or \
+                        (True in [f.endswith(ext) for ext in extensions]):
+                    files.append(os.sep.join((root, f)))
         return files
 
+    def add_filename(self, content):
+        location = os.path.relpath(os.path.abspath(content.filename),
+                                   os.path.abspath(self.path))
+        self.context['filenames'][location] = content
+
     def _update_context(self, items):
         """Update the context with the given items from the currrent
         processor.
@@ -300,7 +312,7 @@ class ArticlesGenerator(Generator):
         self.generate_drafts(write)
 
     def generate_context(self):
-        """change the context"""
+        """Add the articles into the shared context"""
 
         article_path = os.path.normpath(  # we have to remove trailing slashes
             os.path.join(self.path, self.settings['ARTICLE_DIR'])
@@ -341,10 +353,12 @@ class ArticlesGenerator(Generator):
 
             signals.article_generate_context.send(self, metadata=metadata)
             article = Article(content, metadata, settings=self.settings,
-                              filename=f)
+                              filename=f, context=self.context)
             if not is_valid_content(article, f):
                 continue
 
+            self.add_filename(article)
+
             if article.status == "published":
                 if hasattr(article, 'tags'):
                     for tag in article.tags:
@@ -440,11 +454,14 @@ class PagesGenerator(Generator):
             except Exception, e:
                 logger.warning(u'Could not process %s\n%s' % (f, str(e)))
                 continue
-            signals.pages_generate_context.send(self, metadata=metadata )
+            signals.pages_generate_context.send(self, metadata=metadata)
             page = Page(content, metadata, settings=self.settings,
-                        filename=f)
+                        filename=f, context=self.context)
             if not is_valid_content(page, f):
                 continue
+
+            self.add_filename(page)
+
             if page.status == "published":
                 all_pages.append(page)
             elif page.status == "hidden":
@@ -479,17 +496,33 @@ class StaticGenerator(Generator):
             copy(path, source, os.path.join(output_path, destination),
                  final_path, overwrite=True)
 
-    def generate_output(self, writer):
+    def generate_context(self):
+        self.staticfiles = []
 
-        self._copy_paths(self.settings['STATIC_PATHS'], self.path,
+        # walk static paths
-                         'static', self.output_path)
+        for static_path in self.settings['STATIC_PATHS']:
+            for f in self.get_files(
+                    os.path.join(self.path, static_path), extensions=False):
+                f_rel = os.path.relpath(f, self.path)
+                # TODO remove this hardcoded 'static' subdirectory
+                sc = StaticContent(f_rel, os.path.join('static', f_rel),
+                        settings=self.settings)
+                self.staticfiles.append(sc)
+                self.context['filenames'][f_rel] = sc
+        # same thing for FILES_TO_COPY
+        for src, dest in self.settings['FILES_TO_COPY']:
+            sc = StaticContent(src, dest, settings=self.settings)
+            self.staticfiles.append(sc)
+            self.context['filenames'][src] = sc
+
+    def generate_output(self, writer):
         self._copy_paths(self.settings['THEME_STATIC_PATHS'], self.theme,
                          'theme', self.output_path, '.')
-
+        # copy all StaticContent files
-        # copy all the files needed
+        for sc in self.staticfiles:
-        for source, destination in self.settings['FILES_TO_COPY']:
+            mkdir_p(os.path.dirname(sc.save_as))
-            copy(source, self.path, self.output_path, destination,
+            shutil.copy(sc.filepath, sc.save_as)
-                 overwrite=True)
+            logger.info('copying %s to %s' % (sc.filepath, sc.save_as))
 
 
 class PdfGenerator(Generator):
@@ -532,8 +565,8 @@ class PdfGenerator(Generator):
             try:
                 os.mkdir(pdf_path)
             except OSError:
-                logger.error("Couldn't create the pdf output folder in " + pdf_path)
+                logger.error("Couldn't create the pdf output folder in " +
-                pass
+                             pdf_path)
 
         for article in self.context['articles']:
             self._create_pdf(article, pdf_path)

pelican/utils.py

@@ -4,7 +4,9 @@ import re
 import pytz
 import shutil
 import logging
-from collections import defaultdict
+import errno
+from collections import defaultdict, Hashable
+from functools import partial
 
 from codecs import open
 from datetime import datetime
@@ -19,6 +21,32 @@ class NoFilesError(Exception):
     pass
 
 
+class memoized(object):
+   '''Decorator. Caches a function's return value each time it is called.
+   If called later with the same arguments, the cached value is returned
+   (not reevaluated).
+   '''
+   def __init__(self, func):
+      self.func = func
+      self.cache = {}
+   def __call__(self, *args):
+      if not isinstance(args, Hashable):
+         # uncacheable. a list, for instance.
+         # better to not cache than blow up.
+         return self.func(*args)
+      if args in self.cache:
+         return self.cache[args]
+      else:
+         value = self.func(*args)
+         self.cache[args] = value
+         return value
+   def __repr__(self):
+      '''Return the function's docstring.'''
+      return self.func.__doc__
+   def __get__(self, obj, objtype):
+      '''Support instance methods.'''
+      return partial(self.__call__, obj)
+
 def get_date(string):
     """Return a datetime object from a string.
 
@@ -300,3 +328,11 @@ def set_date_tzinfo(d, tz_name=None):
         return tz.localize(d)
     else:
         return d
+
+
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+    except OSError, e:
+        if e.errno != errno.EEXIST:
+            raise

pelican/writers.py

@@ -2,12 +2,10 @@
 from __future__ import with_statement
 
 import os
-import re
 import locale
 import logging
 
 from codecs import open
-from functools import partial
 from feedgenerator import Atom1Feed, Rss201rev2Feed
 from jinja2 import Markup
 from pelican.paginator import Paginator
@@ -129,8 +127,6 @@ class Writer(object):
             localcontext['SITEURL'] = get_relative_path(name)
 
         localcontext.update(kwargs)
-        if relative_urls:
-            self.update_context_contents(name, localcontext)
 
         # check paginated
         paginated = paginated or {}
@@ -168,66 +164,3 @@ class Writer(object):
         else:
             # no pagination
             _write_file(template, localcontext, self.output_path, name)
-
-    def update_context_contents(self, name, context):
-        """Recursively run the context to find elements (articles, pages, etc)
-        whose content getter needs to be modified in order to deal with
-        relative paths.
-
-        :param name: name of the file to output.
-        :param context: dict that will be passed to the templates, which need
-                        to be updated.
-        """
-        def _update_content(name, input):
-            """Change all the relatives paths of the input content to relatives
-            paths suitable fot the ouput content
-
-            :param name: path of the output.
-            :param input: input resource that will be passed to the templates.
-            """
-            content = input._content
-
-            hrefs = re.compile(r"""
-                (?P<markup><\s*[^\>]*  # match tag with src and href attr
-                    (?:href|src)\s*=\s*
-                )
-                (?P<quote>["\'])       # require value to be quoted
-                (?![#?])               # don't match fragment or query URLs
-                (?![a-z]+:)            # don't match protocol URLS
-                (?P<path>.*?)          # the url value
-                \2""", re.X)
-
-            def replacer(m):
-                relative_path = m.group('path')
-                dest_path = os.path.normpath(
-                                os.sep.join((get_relative_path(name), "static",
-                                relative_path)))
-
-                # On Windows, make sure we end up with Unix-like paths.
-                if os.name == 'nt':
-                    dest_path = dest_path.replace('\\', '/')
-
-                return m.group('markup') + m.group('quote') + dest_path \
-                        + m.group('quote')
-
-            return hrefs.sub(replacer, content)
-
-        if context is None:
-            return
-        if hasattr(context, 'values'):
-            context = context.values()
-
-        for item in context:
-            # run recursively on iterables
-            if hasattr(item, '__iter__'):
-                self.update_context_contents(name, item)
-
-            # if it is a content, patch it
-            elif hasattr(item, '_content'):
-                relative_path = get_relative_path(name)
-
-                paths = self.reminder.setdefault(item, [])
-                if relative_path not in paths:
-                    paths.append(relative_path)
-                    setattr(item, "_get_content",
-                        partial(_update_content, name, item))

samples/content/another_super_article.rst

@@ -14,7 +14,7 @@ Why not ?
 After all, why not ? It's pretty simple to do it, and it will allow me to write my blogposts in rst !
 YEAH !
 
-.. image:: pictures/Sushi.jpg
+.. image:: |filename|/pictures/Sushi.jpg
    :height: 450 px
    :width: 600 px
    :alt: alternate text

samples/content/cat1/markdown-article.md

@@ -2,3 +2,6 @@ Title: A markdown powered article
 Date: 2011-04-20
 
 You're mutually oblivious.
+
+[a root-relative link to unbelievable](|filename|/unbelievable.rst)
+[a file-relative link to unbelievable](|filename|../unbelievable.rst)

samples/content/pages/test_page.rst

@@ -5,7 +5,7 @@ This is a test page
 
 Just an image.
 
-.. image:: pictures/Fat_Cat.jpg
+.. image:: |filename|/pictures/Fat_Cat.jpg
    :height: 450 px
    :width: 600 px
    :alt: alternate text

samples/content/super_article.rst

@@ -16,12 +16,12 @@ This is a simple title
 
 And here comes the cool stuff_.
 
-.. image:: pictures/Sushi.jpg
+.. image:: |filename|/pictures/Sushi.jpg
    :height: 450 px
    :width: 600 px
    :alt: alternate text
 
-.. image:: pictures/Sushi_Macro.jpg
+.. image:: |filename|/pictures/Sushi_Macro.jpg
    :height: 450 px
    :width: 600 px
    :alt: alternate text

samples/content/unbelievable.rst

@@ -4,3 +4,6 @@ Unbelievable !
 :date: 2010-10-15 20:30
 
 Or completely awesome. Depends the needs.
+
+`a root-relative link to markdown-article <|filename|/cat1/markdown-article.md>`_
+`a file-relative link to markdown-article <|filename|cat1/markdown-article.md>`_

tests/support.py

@@ -15,6 +15,7 @@ from tempfile import mkdtemp
 from shutil import rmtree
 
 from pelican.contents import Article
+from pelican.settings import _DEFAULT_CONFIG
 
 try:
     import unittest2 as unittest
@@ -149,3 +150,10 @@ def module_exists(module_name):
         return False
     else:
         return True
+
+
+def get_settings():
+    settings = _DEFAULT_CONFIG.copy()
+    settings['DIRECT_TEMPLATES'] = ['archives']
+    settings['filenames'] = {}
+    return settings

tests/test_generators.py

@@ -11,7 +11,7 @@ from pelican.generators import ArticlesGenerator, PagesGenerator, \
     TemplatePagesGenerator
 from pelican.writers import Writer
 from pelican.settings import _DEFAULT_CONFIG
-from .support import unittest
+from .support import unittest, get_settings
 
 CUR_DIR = os.path.dirname(__file__)
 
@@ -28,20 +28,20 @@ class TestArticlesGenerator(unittest.TestCase):
          for each test.
         """
         if self.generator is None:
-            settings = _DEFAULT_CONFIG.copy()
+            settings = get_settings()
             settings['ARTICLE_DIR'] = 'content'
             settings['DEFAULT_CATEGORY'] = 'Default'
             settings['DEFAULT_DATE'] = (1970, 01, 01)
             self.generator = ArticlesGenerator(settings.copy(), settings,
-                                CUR_DIR, _DEFAULT_CONFIG['THEME'], None,
+                                CUR_DIR, settings['THEME'], None,
-                                _DEFAULT_CONFIG['MARKUP'])
+                                settings['MARKUP'])
             self.generator.generate_context()
         return self.generator
 
     def distill_articles(self, articles):
         distilled = []
         for page in articles:
-           distilled.append([
+            distilled.append([
                     page.title,
                     page.status,
                     page.category.name,
@@ -51,16 +51,16 @@ class TestArticlesGenerator(unittest.TestCase):
         return distilled
 
     def test_generate_feeds(self):
-
+        settings = get_settings()
-        generator = ArticlesGenerator(None, {'FEED_ALL_ATOM': _DEFAULT_CONFIG['FEED_ALL_ATOM']},
+        generator = ArticlesGenerator(settings,
-                                      None, _DEFAULT_CONFIG['THEME'], None,
+                {'FEED_ALL_ATOM': settings['FEED_ALL_ATOM']}, None,
-                                      _DEFAULT_CONFIG['MARKUP'])
+                settings['THEME'], None, settings['MARKUP'])
         writer = MagicMock()
         generator.generate_feeds(writer)
-        writer.write_feed.assert_called_with([], None, 'feeds/all.atom.xml')
+        writer.write_feed.assert_called_with([], settings, 'feeds/all.atom.xml')
 
-        generator = ArticlesGenerator(None, {'FEED_ALL_ATOM': None}, None,
+        generator = ArticlesGenerator(settings, {'FEED_ALL_ATOM': None}, None,
-                                      _DEFAULT_CONFIG['THEME'], None, None)
+                                      settings['THEME'], None, None)
         writer = MagicMock()
         generator.generate_feeds(writer)
         self.assertFalse(writer.write_feed.called)
@@ -106,11 +106,10 @@ class TestArticlesGenerator(unittest.TestCase):
 
     def test_direct_templates_save_as_default(self):
 
-        settings = _DEFAULT_CONFIG.copy()
+        settings = get_settings()
-        settings['DIRECT_TEMPLATES'] = ['archives']
+        generator = ArticlesGenerator(settings, settings, None,
-        generator = ArticlesGenerator(settings.copy(), settings, None,
+                                      settings['THEME'], None,
-                                      _DEFAULT_CONFIG['THEME'], None,
+                                      settings['MARKUP'])
-                                      _DEFAULT_CONFIG['MARKUP'])
         write = MagicMock()
         generator.generate_direct_templates(write)
         write.assert_called_with("archives.html",
@@ -119,12 +118,12 @@ class TestArticlesGenerator(unittest.TestCase):
 
     def test_direct_templates_save_as_modified(self):
 
-        settings = _DEFAULT_CONFIG.copy()
+        settings = get_settings()
         settings['DIRECT_TEMPLATES'] = ['archives']
         settings['ARCHIVES_SAVE_AS'] = 'archives/index.html'
         generator = ArticlesGenerator(settings, settings, None,
-                                      _DEFAULT_CONFIG['THEME'], None,
+                                      settings['THEME'], None,
-                                      _DEFAULT_CONFIG['MARKUP'])
+                                      settings['MARKUP'])
         write = MagicMock()
         generator.generate_direct_templates(write)
         write.assert_called_with("archives/index.html",
@@ -133,12 +132,12 @@ class TestArticlesGenerator(unittest.TestCase):
 
     def test_direct_templates_save_as_false(self):
 
-        settings = _DEFAULT_CONFIG.copy()
+        settings = get_settings()
         settings['DIRECT_TEMPLATES'] = ['archives']
         settings['ARCHIVES_SAVE_AS'] = 'archives/index.html'
         generator = ArticlesGenerator(settings, settings, None,
-                                      _DEFAULT_CONFIG['THEME'], None,
+                                      settings['THEME'], None,
-                                      _DEFAULT_CONFIG['MARKUP'])
+                                      settings['MARKUP'])
         write = MagicMock()
         generator.generate_direct_templates(write)
         write.assert_called_count == 0
@@ -174,13 +173,13 @@ class TestPageGenerator(unittest.TestCase):
         return distilled
 
     def test_generate_context(self):
-        settings = _DEFAULT_CONFIG.copy()
+        settings = get_settings()
-
         settings['PAGE_DIR'] = 'TestPages'
         settings['DEFAULT_DATE'] = (1970, 01, 01)
+
         generator = PagesGenerator(settings.copy(), settings, CUR_DIR,
-                                      _DEFAULT_CONFIG['THEME'], None,
+                                      settings['THEME'], None,
-                                      _DEFAULT_CONFIG['MARKUP'])
+                                      settings['MARKUP'])
         generator.generate_context()
         pages = self.distill_pages(generator.pages)
         hidden_pages = self.distill_pages(generator.hidden_pages)
@@ -214,7 +213,7 @@ class TestTemplatePagesGenerator(unittest.TestCase):
 
     def test_generate_output(self):
 
-        settings = _DEFAULT_CONFIG.copy()
+        settings = get_settings()
         settings['STATIC_PATHS'] = ['static']
         settings['TEMPLATE_PAGES'] = {
                 'template/source.html': 'generated/file.html'