From 24a1254f034a238f481f8b889a961dbde99c551d Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Fri, 30 Sep 2016 15:29:14 +0200 Subject: [PATCH 01/93] Explicitly disallow duplications of URL and save_as. --- pelican/readers.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pelican/readers.py b/pelican/readers.py index 415e7558..56e54355 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -31,6 +31,20 @@ except ImportError: # This means that _filter_discardable_metadata() must be called on processed # metadata dicts before use, to remove the items with the special value. _DISCARD = object() + +DUPLICATES_DEFINITIONS_ALLOWED = { + 'tags': False, + 'date': False, + 'modified': False, + 'status': False, + 'category': False, + 'author': False, + 'save_as': False, + 'URL': False, + 'authors': False, + 'slug': False +} + METADATA_PROCESSORS = { 'tags': lambda x, y: ([ Tag(tag, y) @@ -264,7 +278,7 @@ class MarkdownReader(BaseReader): self._md.reset() formatted = self._md.convert(formatted_values) output[name] = self.process_metadata(name, formatted) - elif name in METADATA_PROCESSORS: + elif not DUPLICATES_DEFINITIONS_ALLOWED.get(name, True): if len(value) > 1: logger.warning( 'Duplicate definition of `%s` ' From 9e574e9d8c6e4d6377a6fcefe1579bb998c7223a Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Fri, 30 Sep 2016 15:33:05 +0200 Subject: [PATCH 02/93] Just in case someone forgot the DUPLICATES_DEFINITIONS_ALLOWED but add in METADATA_PROCESSORS. --- pelican/readers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pelican/readers.py b/pelican/readers.py index 56e54355..5220f1a0 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -278,7 +278,8 @@ class MarkdownReader(BaseReader): self._md.reset() formatted = self._md.convert(formatted_values) output[name] = self.process_metadata(name, formatted) - elif not DUPLICATES_DEFINITIONS_ALLOWED.get(name, True): + elif (not DUPLICATES_DEFINITIONS_ALLOWED.get(name, True) or + name in METADATA_PROCESSORS): if len(value) > 1: logger.warning( 'Duplicate definition of `%s` ' From e8a87e5d3cb82081127a8a07da574059668688de Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Mon, 10 Oct 2016 12:23:26 +0200 Subject: [PATCH 03/93] As not allowing duplicates in processed items is counter intuitive, let's allow it. Also it may be allowed in the future (to process multiple values). Also @avaris think it's bad to test something twice (see https://github.com/getpelican/pelican/pull/2017), but for me confusion lies in the "Why is list processing forbidden?", so, in a way, our ideas converges in "let's not disallow processed items to be lists". This reverts commit 9e574e9d8c6e4d6377a6fcefe1579bb998c7223a. --- pelican/readers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pelican/readers.py b/pelican/readers.py index 503db679..a7712f0b 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -278,8 +278,7 @@ class MarkdownReader(BaseReader): self._md.reset() formatted = self._md.convert(formatted_values) output[name] = self.process_metadata(name, formatted) - elif (not DUPLICATES_DEFINITIONS_ALLOWED.get(name, True) or - name in METADATA_PROCESSORS): + elif not DUPLICATES_DEFINITIONS_ALLOWED.get(name, True): if len(value) > 1: logger.warning( 'Duplicate definition of `%s` ' From e07c53a09dab16ec33e44a84a1c5db5e58deb61c Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Wed, 26 Oct 2016 08:34:52 +0200 Subject: [PATCH 04/93] FIX: Those keys are looked up lowercased. --- pelican/readers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pelican/readers.py b/pelican/readers.py index a7712f0b..74b617e6 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -40,7 +40,7 @@ DUPLICATES_DEFINITIONS_ALLOWED = { 'category': False, 'author': False, 'save_as': False, - 'URL': False, + 'url': False, 'authors': False, 'slug': False } From 32c154be95cd71ea96284b1dbee1d3fa04a9adb1 Mon Sep 17 00:00:00 2001 From: Brandon B Date: Fri, 28 Apr 2017 19:18:44 -0700 Subject: [PATCH 05/93] Update tips.rst I found that one of the easiest ways to publish to GitHub User Pages is to make Pelican a subdirectory within the ``.github.io`` project, then generate the output files in the root level of ``.github.io`` and push the entire project to GitHub. --- docs/tips.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/tips.rst b/docs/tips.rst index 50160380..bf0b3599 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -98,6 +98,18 @@ by the ``ghp-import`` command) to the ``elemoine.github.io`` repository's To publish your Pelican site as User Pages, feel free to adjust the ``github`` target of the Makefile. + +Another option for publishing to User Pages is to generate the output files in the root directory of the project. + +For example, your main project folder is ``.github.io`` and you can create the Pelican project in a subdirectory called ``Pelican``. Then from inside the ``Pelican`` folder you can run:: + + $ pelican content -o .. -s pelicanconf.py + +Now you can push the whole project ``.github.io`` to the master branch of your GitHub repository:: + + $ git push origin master + +(assuming origin is set to your remote repository). Custom 404 Pages ---------------- From b1bc9a6f5fa24dd7c9350a9805123f1386301559 Mon Sep 17 00:00:00 2001 From: Jorge Maldonado Ventura Date: Tue, 8 Aug 2017 01:16:13 +0200 Subject: [PATCH 06/93] Typofix: - paragraph missing stop - Remove unnecessary colon --- docs/content.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content.rst b/docs/content.rst index 507593bf..ae2cd6f2 100644 --- a/docs/content.rst +++ b/docs/content.rst @@ -11,7 +11,7 @@ The idea behind "pages" is that they are usually not temporal in nature and are used for content that does not change very often (e.g., "About" or "Contact" pages). -You can find sample content in the repository at: ``pelican/samples/content/`` +You can find sample content in the repository at ``pelican/samples/content/``. .. _internal_metadata: From d6dca3403d1a0cb45bf7f352080b6c233ac69825 Mon Sep 17 00:00:00 2001 From: Jorge Maldonado Ventura Date: Tue, 8 Aug 2017 23:45:00 +0200 Subject: [PATCH 07/93] pelican/ is not necessary (documentation content.rst) --- docs/content.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content.rst b/docs/content.rst index ae2cd6f2..69212540 100644 --- a/docs/content.rst +++ b/docs/content.rst @@ -11,7 +11,7 @@ The idea behind "pages" is that they are usually not temporal in nature and are used for content that does not change very often (e.g., "About" or "Contact" pages). -You can find sample content in the repository at ``pelican/samples/content/``. +You can find sample content in the repository at ``samples/content/``. .. _internal_metadata: From 50af2ed45d3db08e4ac85cdcdeae5e30a7b50a24 Mon Sep 17 00:00:00 2001 From: Pedro H Date: Sun, 16 Oct 2016 15:47:22 +0800 Subject: [PATCH 08/93] Add THEME_TEMPLATE_OVERRIDES. Refs 2021 Allow for overriding individual templates from the theme by configuring the Jinja2 `Environment` loader to search for templates in the `THEME_TEMPLATES_OVERRIDES` path before the theme's `templates/` directory. --- docs/settings.rst | 24 ++++++-- pelican/generators.py | 21 ++++--- pelican/settings.py | 18 +++++- pelican/tests/test_generators.py | 59 ++++++++++++++++++- pelican/tests/test_settings.py | 17 ++++++ .../tests/theme_overrides/level1/article.html | 4 ++ .../tests/theme_overrides/level2/article.html | 4 ++ .../tests/theme_overrides/level2/authors.html | 4 ++ 8 files changed, 134 insertions(+), 17 deletions(-) create mode 100644 pelican/tests/theme_overrides/level1/article.html create mode 100644 pelican/tests/theme_overrides/level2/article.html create mode 100644 pelican/tests/theme_overrides/level2/authors.html diff --git a/docs/settings.rst b/docs/settings.rst index e5ce19f5..e59e42d4 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -706,16 +706,13 @@ Template pages tags and category index pages). If the tag and category collections are not needed, set ``DIRECT_TEMPLATES = ['index', 'archives']`` + ``DIRECT_TEMPLATES`` are searched for over paths maintained in + ``THEME_TEMPLATES_OVERRIDES``. + .. data:: PAGINATED_DIRECT_TEMPLATES = ['index'] Provides the direct templates that should be paginated. -.. data:: EXTRA_TEMPLATES_PATHS = [] - - A list of paths you want Jinja2 to search for templates. Can be used to - separate templates from the theme. Example: projects, resume, profile ... - These templates need to use ``DIRECT_TEMPLATES`` setting. - Metadata ======== @@ -1011,6 +1008,21 @@ However, here are the settings that are related to themes. with the same names are included in the paths defined in this settings, they will be progressively overwritten. +.. data:: THEME_TEMPLATES_OVERRIDES = [] + + A list of paths you want Jinja2 to search for templates before searching the + theme's ``templates/`` directory. Allows for overriding individual theme + template files without having to fork an existing theme. Jinja2 searches in + the following order: files in ``THEME_TEMPLATES_OVERRIDES`` first, then the + theme's ``templates/``. + + You can also extend templates from the theme using the ``{% extends %}`` + directive utilizing the ``!theme`` prefix as shown in the following example: + + .. parsed-literal:: + + {% extends '!theme/article.html' %} + .. data:: CSS_FILE = 'main.css' Specify the CSS file you want to load. diff --git a/pelican/generators.py b/pelican/generators.py index eb97c115..1dbca007 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -51,20 +51,25 @@ class Generator(object): # templates cache self._templates = {} - self._templates_path = [] - self._templates_path.append(os.path.expanduser( - os.path.join(self.theme, 'templates'))) - self._templates_path += self.settings['EXTRA_TEMPLATES_PATHS'] + self._templates_path = list(self.settings['THEME_TEMPLATES_OVERRIDES']) - theme_path = os.path.dirname(os.path.abspath(__file__)) + theme_templates_path = os.path.expanduser( + os.path.join(self.theme, 'templates')) + self._templates_path.append(theme_templates_path) + theme_loader = FileSystemLoader(theme_templates_path) + + simple_theme_path = os.path.dirname(os.path.abspath(__file__)) + simple_loader = FileSystemLoader( + os.path.join(simple_theme_path, "themes", "simple", "templates")) - simple_loader = FileSystemLoader(os.path.join(theme_path, - "themes", "simple", "templates")) self.env = Environment( loader=ChoiceLoader([ FileSystemLoader(self._templates_path), simple_loader, # implicit inheritance - PrefixLoader({'!simple': simple_loader}) # explicit one + PrefixLoader({ + '!simple': simple_loader, + '!theme': theme_loader + }) # explicit ones ]), **self.settings['JINJA_ENVIRONMENT'] ) diff --git a/pelican/settings.py b/pelican/settings.py index 417de79a..d025cf7a 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -99,7 +99,7 @@ DEFAULT_CONFIG = { 'RELATIVE_URLS': False, 'DEFAULT_LANG': 'en', 'DIRECT_TEMPLATES': ['index', 'tags', 'categories', 'authors', 'archives'], - 'EXTRA_TEMPLATES_PATHS': [], + 'THEME_TEMPLATES_OVERRIDES': [], 'PAGINATED_DIRECT_TEMPLATES': ['index'], 'PELICAN_CLASS': 'pelican.Pelican', 'DEFAULT_DATE_FORMAT': '%a %d %B %Y', @@ -376,12 +376,26 @@ def configure_settings(settings): settings[new_key] = [settings[old_key]] # also make a list del settings[old_key] + # Deprecated warning of EXTRA_TEMPLATES_PATHS + if 'EXTRA_TEMPLATES_PATHS' in settings: + logger.warning('EXTRA_TEMPLATES_PATHS is deprecated use ' + 'THEME_TEMPLATES_OVERRIDES instead.') + if ('THEME_TEMPLATES_OVERRIDES' in settings and + settings['THEME_TEMPLATES_OVERRIDES']): + raise Exception( + 'Setting both EXTRA_TEMPLATES_PATHS and ' + 'THEME_TEMPLATES_OVERRIDES is not permitted. Please move to ' + 'only setting THEME_TEMPLATES_OVERRIDES.') + settings['THEME_TEMPLATES_OVERRIDES'] = \ + settings['EXTRA_TEMPLATES_PATHS'] + del settings['EXTRA_TEMPLATES_PATHS'] + # Save people from accidentally setting a string rather than a list path_keys = ( 'ARTICLE_EXCLUDES', 'DEFAULT_METADATA', 'DIRECT_TEMPLATES', - 'EXTRA_TEMPLATES_PATHS', + 'THEME_TEMPLATES_OVERRIDES', 'FILES_TO_COPY', 'IGNORE_FILES', 'PAGINATED_DIRECT_TEMPLATES', diff --git a/pelican/tests/test_generators.py b/pelican/tests/test_generators.py index 5f2151c3..100b1c5a 100644 --- a/pelican/tests/test_generators.py +++ b/pelican/tests/test_generators.py @@ -9,7 +9,8 @@ from shutil import copy, rmtree from tempfile import mkdtemp from pelican.generators import (ArticlesGenerator, Generator, PagesGenerator, - StaticGenerator, TemplatePagesGenerator) + PelicanTemplateNotFound, StaticGenerator, + TemplatePagesGenerator) from pelican.tests.support import get_settings, unittest from pelican.writers import Writer @@ -117,6 +118,62 @@ class TestGenerator(unittest.TestCase): self.assertEqual(comment_end_string, generator.env.comment_end_string) + def test_theme_overrides(self): + """ + Test that the THEME_TEMPLATES_OVERRIDES configuration setting is + utilized correctly in the Generator. + """ + override_dirs = (os.path.join(CUR_DIR, 'theme_overrides', 'level1'), + os.path.join(CUR_DIR, 'theme_overrides', 'level2')) + self.settings['THEME_TEMPLATES_OVERRIDES'] = override_dirs + generator = Generator( + context=self.settings.copy(), + settings=self.settings, + path=CUR_DIR, + theme=self.settings['THEME'], + output_path=None) + + filename = generator.get_template('article').filename + self.assertEqual(override_dirs[0], os.path.dirname(filename)) + self.assertEqual('article.html', os.path.basename(filename)) + + filename = generator.get_template('authors').filename + self.assertEqual(override_dirs[1], os.path.dirname(filename)) + self.assertEqual('authors.html', os.path.basename(filename)) + + filename = generator.get_template('taglist').filename + self.assertEqual(os.path.join(self.settings['THEME'], 'templates'), + os.path.dirname(filename)) + self.assertNotIn(os.path.dirname(filename), override_dirs) + self.assertEqual('taglist.html', os.path.basename(filename)) + + def test_simple_prefix(self): + """ + Test `!simple` theme prefix. + """ + filename = self.generator.get_template('!simple/authors').filename + expected_path = os.path.join( + os.path.dirname(CUR_DIR), 'themes', 'simple', 'templates') + self.assertEqual(expected_path, os.path.dirname(filename)) + self.assertEqual('authors.html', os.path.basename(filename)) + + def test_theme_prefix(self): + """ + Test `!theme` theme prefix. + """ + filename = self.generator.get_template('!theme/authors').filename + expected_path = os.path.join( + os.path.dirname(CUR_DIR), 'themes', 'notmyidea', 'templates') + self.assertEqual(expected_path, os.path.dirname(filename)) + self.assertEqual('authors.html', os.path.basename(filename)) + + def test_bad_prefix(self): + """ + Test unknown/bad theme prefix throws exception. + """ + self.assertRaises(PelicanTemplateNotFound, self.generator.get_template, + '!UNKNOWN/authors') + class TestArticlesGenerator(unittest.TestCase): diff --git a/pelican/tests/test_settings.py b/pelican/tests/test_settings.py index 7b1e36df..4ec16c0f 100644 --- a/pelican/tests/test_settings.py +++ b/pelican/tests/test_settings.py @@ -166,3 +166,20 @@ class TestSettingsConfiguration(unittest.TestCase): settings['THEME'] = 'foo' self.assertRaises(Exception, configure_settings, settings) + + def test_deprecated_extra_templates_paths(self): + settings = self.settings + settings['EXTRA_TEMPLATES_PATHS'] = ['/foo/bar', '/ha'] + + configure_settings(settings) + + self.assertEqual(settings['THEME_TEMPLATES_OVERRIDES'], + ['/foo/bar', '/ha']) + self.assertNotIn('EXTRA_TEMPLATES_PATHS', settings) + + def test_theme_and_extra_templates_exception(self): + settings = self.settings + settings['EXTRA_TEMPLATES_PATHS'] = ['/ha'] + settings['THEME_TEMPLATES_OVERRIDES'] = ['/foo/bar'] + + self.assertRaises(Exception, configure_settings, settings) diff --git a/pelican/tests/theme_overrides/level1/article.html b/pelican/tests/theme_overrides/level1/article.html new file mode 100644 index 00000000..12f6b7bf --- /dev/null +++ b/pelican/tests/theme_overrides/level1/article.html @@ -0,0 +1,4 @@ + diff --git a/pelican/tests/theme_overrides/level2/article.html b/pelican/tests/theme_overrides/level2/article.html new file mode 100644 index 00000000..12f6b7bf --- /dev/null +++ b/pelican/tests/theme_overrides/level2/article.html @@ -0,0 +1,4 @@ + diff --git a/pelican/tests/theme_overrides/level2/authors.html b/pelican/tests/theme_overrides/level2/authors.html new file mode 100644 index 00000000..12f6b7bf --- /dev/null +++ b/pelican/tests/theme_overrides/level2/authors.html @@ -0,0 +1,4 @@ + From 17b37358e9046ad840b37ff9fa20ed1a673ca727 Mon Sep 17 00:00:00 2001 From: Sergei K Date: Tue, 17 Oct 2017 07:38:18 +0500 Subject: [PATCH 09/93] Add a new signal: page_generator_write_page --- docs/plugins.rst | 1 + pelican/generators.py | 1 + pelican/signals.py | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/plugins.rst b/docs/plugins.rst index 008e7551..10780b66 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -108,6 +108,7 @@ page_generator_preread page_generator invoked befor use if code needs to do something before every page is parsed. page_generator_init page_generator invoked in the PagesGenerator.__init__ page_generator_finalized page_generator invoked at the end of PagesGenerator.generate_context +page_generator_write_page page_generator, content invoked before writing each page, the page is passed as content page_writer_finalized page_generator, writer invoked after all pages have been written, but before the page generator is closed. static_generator_context static_generator, metadata diff --git a/pelican/generators.py b/pelican/generators.py index eb97c115..19c1d5a8 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -642,6 +642,7 @@ class PagesGenerator(CachingGenerator): def generate_output(self, writer): for page in chain(self.translations, self.pages, self.hidden_translations, self.hidden_pages): + signals.page_generator_write_page.send(self, content=page) writer.write_file( page.save_as, self.get_template(page.template), self.context, page=page, diff --git a/pelican/signals.py b/pelican/signals.py index 0b10fdfa..18a745b4 100644 --- a/pelican/signals.py +++ b/pelican/signals.py @@ -27,6 +27,7 @@ article_writer_finalized = signal('article_writer_finalized') page_generator_init = signal('page_generator_init') page_generator_finalized = signal('page_generator_finalized') +page_generator_write_page = signal('page_generator_write_page') page_writer_finalized = signal('page_writer_finalized') static_generator_init = signal('static_generator_init') From 88da1b89cbebe66278aca7933f2c5165511b8207 Mon Sep 17 00:00:00 2001 From: Dan Bate Date: Sun, 15 Jan 2017 22:57:01 +0000 Subject: [PATCH 10/93] made template extensions configurable --- docs/settings.rst | 4 ++++ pelican/generators.py | 17 ++++++++++++----- pelican/settings.py | 1 + 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/settings.rst b/docs/settings.rst index e5ce19f5..d32a9869 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -699,6 +699,10 @@ Template pages 'src/resume.html': 'dest/resume.html', 'src/contact.html': 'dest/contact.html'} +.. data:: TEMPLATE_EXTENSION = ['.html'] + + The extensions to use when looking up template files from template names. + .. data:: DIRECT_TEMPLATES = ['index', 'categories', 'authors', 'archives'] List of templates that are used directly to render content. Typically direct diff --git a/pelican/generators.py b/pelican/generators.py index eb97c115..b9e40243 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -86,12 +86,19 @@ class Generator(object): templates ready to use with Jinja2. """ if name not in self._templates: - try: - self._templates[name] = self.env.get_template(name + '.html') - except TemplateNotFound: + for ext in self.settings['TEMPLATE_EXTENSIONS']: + try: + self._templates[name] = self.env.get_template(name + ext) + break + except TemplateNotFound: + continue + + if name not in self._templates: raise PelicanTemplateNotFound( - '[templates] unable to load {}.html from {}'.format( - name, self._templates_path)) + '[templates] unable to load {}[{}] from {}'.format( + name, ', '.join(self.settings['TEMPLATE_EXTENSIONS']), + self._templates_path)) + return self._templates[name] def _include_path(self, path, extensions=None): diff --git a/pelican/settings.py b/pelican/settings.py index 417de79a..0d09340b 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -134,6 +134,7 @@ DEFAULT_CONFIG = { 'PLUGINS': [], 'PYGMENTS_RST_OPTIONS': {}, 'TEMPLATE_PAGES': {}, + 'TEMPLATE_EXTENSIONS': ['.html'], 'IGNORE_FILES': ['.#*'], 'SLUG_SUBSTITUTIONS': (), 'INTRASITE_LINK_REGEX': '[{|](?P.*?)[|}]', From 1f30306e2329e5a1f0c5dd39844d9bb0a0c04573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Tue, 19 Sep 2017 18:22:56 +0200 Subject: [PATCH 11/93] Make the internal link replacer function public. So it can be used from outside. --- pelican/contents.py | 126 ++++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/pelican/contents.py b/pelican/contents.py index 15770fc8..a534dbaa 100644 --- a/pelican/contents.py +++ b/pelican/contents.py @@ -228,6 +228,68 @@ class Content(object): key = key if self.in_default_lang else 'lang_%s' % key return self._expand_settings(key) + def _link_replacer(self, siteurl, m): + what = m.group('what') + value = urlparse(m.group('value')) + path = value.path + origin = m.group('path') + + # XXX Put this in a different location. + if what in {'filename', 'attach'}: + if path.startswith('/'): + path = path[1:] + else: + # relative to the source path of this content + path = self.get_relative_source_path( + os.path.join(self.relative_dir, path) + ) + + if path not in self._context['filenames']: + unquoted_path = path.replace('%20', ' ') + + if unquoted_path in self._context['filenames']: + path = unquoted_path + + linked_content = self._context['filenames'].get(path) + if linked_content: + if what == 'attach': + if isinstance(linked_content, Static): + linked_content.attach_to(self) + else: + logger.warning( + "%s used {attach} link syntax on a " + "non-static file. Use {filename} instead.", + self.get_relative_source_path()) + origin = '/'.join((siteurl, linked_content.url)) + origin = origin.replace('\\', '/') # for Windows paths. + else: + logger.warning( + "Unable to find '%s', skipping url replacement.", + value.geturl(), extra={ + 'limit_msg': ("Other resources were not found " + "and their urls not replaced")}) + elif what == 'category': + origin = '/'.join((siteurl, Category(path, self.settings).url)) + elif what == 'tag': + origin = '/'.join((siteurl, Tag(path, self.settings).url)) + elif what == 'index': + origin = '/'.join((siteurl, self.settings['INDEX_SAVE_AS'])) + elif what == 'author': + origin = '/'.join((siteurl, Author(path, self.settings).url)) + else: + logger.warning( + "Replacement Indicator '%s' not recognized, " + "skipping replacement", + what) + + # keep all other parts, such as query, fragment, etc. + parts = list(value) + parts[2] = origin + origin = urlunparse(parts) + + return ''.join((m.group('markup'), m.group('quote'), origin, + m.group('quote'))) + def _update_content(self, content, siteurl): """Update the content attribute. @@ -251,69 +313,7 @@ class Content(object): \2""".format(instrasite_link_regex) hrefs = re.compile(regex, re.X) - def replacer(m): - what = m.group('what') - value = urlparse(m.group('value')) - path = value.path - origin = m.group('path') - - # XXX Put this in a different location. - if what in {'filename', 'attach'}: - if path.startswith('/'): - path = path[1:] - else: - # relative to the source path of this content - path = self.get_relative_source_path( - os.path.join(self.relative_dir, path) - ) - - if path not in self._context['filenames']: - unquoted_path = path.replace('%20', ' ') - - if unquoted_path in self._context['filenames']: - path = unquoted_path - - linked_content = self._context['filenames'].get(path) - if linked_content: - if what == 'attach': - if isinstance(linked_content, Static): - linked_content.attach_to(self) - else: - logger.warning( - "%s used {attach} link syntax on a " - "non-static file. Use {filename} instead.", - self.get_relative_source_path()) - origin = '/'.join((siteurl, linked_content.url)) - origin = origin.replace('\\', '/') # for Windows paths. - else: - logger.warning( - "Unable to find '%s', skipping url replacement.", - value.geturl(), extra={ - 'limit_msg': ("Other resources were not found " - "and their urls not replaced")}) - elif what == 'category': - origin = '/'.join((siteurl, Category(path, self.settings).url)) - elif what == 'tag': - origin = '/'.join((siteurl, Tag(path, self.settings).url)) - elif what == 'index': - origin = '/'.join((siteurl, self.settings['INDEX_SAVE_AS'])) - elif what == 'author': - origin = '/'.join((siteurl, Author(path, self.settings).url)) - else: - logger.warning( - "Replacement Indicator '%s' not recognized, " - "skipping replacement", - what) - - # keep all other parts, such as query, fragment, etc. - parts = list(value) - parts[2] = origin - origin = urlunparse(parts) - - return ''.join((m.group('markup'), m.group('quote'), origin, - m.group('quote'))) - - return hrefs.sub(replacer, content) + return hrefs.sub(lambda m: self._link_replacer(siteurl, m), content) def get_siteurl(self): return self._context.get('localsiteurl', '') From 0b13aa9b4610a8d26ece75cfcebebaa5fb3fa3de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Thu, 20 Jul 2017 23:21:17 +0200 Subject: [PATCH 12/93] Make URL part joining aware of absolute URLs. Previously, with RELATIVE_URLS disabled, when both SITEURL and STATIC_URL were absolute, the final generate data URLs looked wrong like this (two absolute URLs joined by `/`): http://your.site/http://static.your.site/image.png With this patch, the data URLs are correctly: http://static.your.site/image.png This also applies to all *_URL configuration options (for example, ability to have pages and articles on different domains) and behaves like one expects even with URLs starting with just `//`, thanks to making use of urllib.parse.urljoin(). However, when RELATIVE_URLS are enabled, urllib.parse.urljoin() doesn't handle the relative base correctly. In that case, simple os.path.join() is used. That, however, breaks the above case, but as RELATIVE_URLS are meant for local development (thus no data scattered across multiple domains), I don't see any problem. Just to clarify, this is a fully backwards-compatible change, it only enables new use cases that were impossible before. --- pelican/contents.py | 31 +++++++++++++++++----- pelican/tests/test_contents.py | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/pelican/contents.py b/pelican/contents.py index a534dbaa..e84c8296 100644 --- a/pelican/contents.py +++ b/pelican/contents.py @@ -11,7 +11,7 @@ import sys import pytz import six -from six.moves.urllib.parse import urlparse, urlunparse +from six.moves.urllib.parse import urljoin, urlparse, urlunparse from pelican import signals from pelican.settings import DEFAULT_CONFIG @@ -234,6 +234,25 @@ class Content(object): path = value.path origin = m.group('path') + # urllib.parse.urljoin() produces `a.html` for urljoin("..", "a.html") + # so if RELATIVE_URLS are enabled, we fall back to os.path.join() to + # properly get `../a.html`. However, os.path.join() produces + # `baz/http://foo/bar.html` for join("baz", "http://foo/bar.html") + # instead of correct "http://foo/bar.html", so one has to pick a side + # as there is no silver bullet. + if self.settings['RELATIVE_URLS']: + joiner = os.path.join + else: + joiner = urljoin + + # However, it's not *that* simple: urljoin("blog", "index.html") + # produces just `index.html` instead of `blog/index.html` (unlike + # os.path.join()), so in order to get a correct answer one needs to + # append a trailing slash to siteurl in that case. This also makes + # the new behavior fully compatible with Pelican 3.7.1. + if not siteurl.endswith('/'): + siteurl += '/' + # XXX Put this in a different location. if what in {'filename', 'attach'}: if path.startswith('/'): @@ -260,7 +279,7 @@ class Content(object): "%s used {attach} link syntax on a " "non-static file. Use {filename} instead.", self.get_relative_source_path()) - origin = '/'.join((siteurl, linked_content.url)) + origin = joiner(siteurl, linked_content.url) origin = origin.replace('\\', '/') # for Windows paths. else: logger.warning( @@ -269,13 +288,13 @@ class Content(object): 'limit_msg': ("Other resources were not found " "and their urls not replaced")}) elif what == 'category': - origin = '/'.join((siteurl, Category(path, self.settings).url)) + origin = joiner(siteurl, Category(path, self.settings).url) elif what == 'tag': - origin = '/'.join((siteurl, Tag(path, self.settings).url)) + origin = joiner(siteurl, Tag(path, self.settings).url) elif what == 'index': - origin = '/'.join((siteurl, self.settings['INDEX_SAVE_AS'])) + origin = joiner(siteurl, self.settings['INDEX_SAVE_AS']) elif what == 'author': - origin = '/'.join((siteurl, Author(path, self.settings).url)) + origin = joiner(siteurl, Author(path, self.settings).url) else: logger.warning( "Replacement Indicator '%s' not recognized, " diff --git a/pelican/tests/test_contents.py b/pelican/tests/test_contents.py index d028c7a1..04c82f61 100644 --- a/pelican/tests/test_contents.py +++ b/pelican/tests/test_contents.py @@ -397,6 +397,54 @@ class TestPage(LoggedTestCase): '' ) + def test_intrasite_link_absolute(self): + """Test that absolute URLs are merged properly.""" + + args = self.page_kwargs.copy() + args['settings'] = get_settings( + STATIC_URL='http://static.cool.site/{path}', + ARTICLE_URL='http://blog.cool.site/{slug}.html') + args['source_path'] = 'content' + args['context']['filenames'] = { + 'images/poster.jpg': Static('', + settings=args['settings'], + source_path='images/poster.jpg'), + 'article.rst': Article('', + settings=args['settings'], + metadata={'slug': 'article', + 'title': 'Article'}) + } + + # Article link will go to blog + args['content'] = ( + 'Article' + ) + content = Page(**args).get_content('http://cool.site') + self.assertEqual( + content, + 'Article' + ) + + # Page link will go to the main site + args['content'] = ( + 'Index' + ) + content = Page(**args).get_content('http://cool.site') + self.assertEqual( + content, + 'Index' + ) + + # Image link will go to static + args['content'] = ( + '' + ) + content = Page(**args).get_content('http://cool.site') + self.assertEqual( + content, + '' + ) + def test_intrasite_link_markdown_spaces(self): # Markdown introduces %20 instead of spaces, this tests that # we support markdown doing this. From fc4b3e44d8a92635cabcff57bdf078cb2385ad80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sun, 29 Oct 2017 20:31:09 +0100 Subject: [PATCH 13/93] Make use of SITESUBTITLE setting also in Atom feeds. For some reason, feedgenerator.py uses the `description` argument only for RSS and the `subtitle` argument only for Atom. So setting both to the same value. In order to avoid unnecessary changes, if SITESUBTITLE is not present, the subtitle is set to None instead of '', so the generated Atom feed doesn't contain an empty tag (in contrast, the RSS feed contains an empty tag now, though). --- pelican/writers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pelican/writers.py b/pelican/writers.py index fef0d5ca..ff3207fa 100644 --- a/pelican/writers.py +++ b/pelican/writers.py @@ -40,7 +40,8 @@ class Writer(object): title=Markup(feed_title).striptags(), link=(self.site_url + '/'), feed_url=self.feed_url, - description=context.get('SITESUBTITLE', '')) + description=context.get('SITESUBTITLE', ''), + subtitle=context.get('SITESUBTITLE', None)) return feed def _add_item_to_the_feed(self, feed, item): From d04b777159a7a28238a5bb989bbae1bce1a9c6b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Sun, 29 Oct 2017 21:28:11 +0100 Subject: [PATCH 14/93] Test for SITESUBTITLE in feeds. It's now present in both RSS and Atom feeds. --- pelican/tests/output/custom/a-markdown-powered-article.html | 2 +- pelican/tests/output/custom/archives.html | 2 +- pelican/tests/output/custom/article-1.html | 2 +- pelican/tests/output/custom/article-2.html | 2 +- pelican/tests/output/custom/article-3.html | 2 +- pelican/tests/output/custom/author/alexis-metaireau.html | 2 +- pelican/tests/output/custom/author/alexis-metaireau2.html | 2 +- pelican/tests/output/custom/author/alexis-metaireau3.html | 2 +- pelican/tests/output/custom/authors.html | 2 +- pelican/tests/output/custom/categories.html | 2 +- pelican/tests/output/custom/category/bar.html | 2 +- pelican/tests/output/custom/category/cat1.html | 2 +- pelican/tests/output/custom/category/misc.html | 2 +- pelican/tests/output/custom/category/yeah.html | 2 +- pelican/tests/output/custom/drafts/a-draft-article.html | 2 +- pelican/tests/output/custom/feeds/alexis-metaireau.atom.xml | 2 +- pelican/tests/output/custom/feeds/alexis-metaireau.rss.xml | 2 +- pelican/tests/output/custom/feeds/all-en.atom.xml | 2 +- pelican/tests/output/custom/feeds/all-fr.atom.xml | 2 +- pelican/tests/output/custom/feeds/all.atom.xml | 2 +- pelican/tests/output/custom/feeds/all.rss.xml | 2 +- pelican/tests/output/custom/feeds/bar.atom.xml | 2 +- pelican/tests/output/custom/feeds/bar.rss.xml | 2 +- pelican/tests/output/custom/feeds/cat1.atom.xml | 2 +- pelican/tests/output/custom/feeds/cat1.rss.xml | 2 +- pelican/tests/output/custom/feeds/misc.atom.xml | 2 +- pelican/tests/output/custom/feeds/misc.rss.xml | 2 +- pelican/tests/output/custom/feeds/yeah.atom.xml | 2 +- pelican/tests/output/custom/feeds/yeah.rss.xml | 2 +- pelican/tests/output/custom/filename_metadata-example.html | 2 +- pelican/tests/output/custom/index.html | 2 +- pelican/tests/output/custom/index2.html | 2 +- pelican/tests/output/custom/index3.html | 2 +- pelican/tests/output/custom/jinja2_template.html | 2 +- pelican/tests/output/custom/oh-yeah-fr.html | 2 +- pelican/tests/output/custom/oh-yeah.html | 2 +- pelican/tests/output/custom/override/index.html | 2 +- .../tests/output/custom/pages/this-is-a-test-hidden-page.html | 2 +- pelican/tests/output/custom/pages/this-is-a-test-page.html | 2 +- pelican/tests/output/custom/second-article-fr.html | 2 +- pelican/tests/output/custom/second-article.html | 2 +- pelican/tests/output/custom/tag/bar.html | 2 +- pelican/tests/output/custom/tag/baz.html | 2 +- pelican/tests/output/custom/tag/foo.html | 2 +- pelican/tests/output/custom/tag/foobar.html | 2 +- pelican/tests/output/custom/tag/oh.html | 2 +- pelican/tests/output/custom/tag/yeah.html | 2 +- pelican/tests/output/custom/tags.html | 2 +- pelican/tests/output/custom/this-is-a-super-article.html | 2 +- pelican/tests/output/custom/unbelievable.html | 2 +- samples/pelican.conf.py | 1 + 51 files changed, 51 insertions(+), 50 deletions(-) diff --git a/pelican/tests/output/custom/a-markdown-powered-article.html b/pelican/tests/output/custom/a-markdown-powered-article.html index 86c6a98d..5c416f2f 100644 --- a/pelican/tests/output/custom/a-markdown-powered-article.html +++ b/pelican/tests/output/custom/a-markdown-powered-article.html @@ -13,7 +13,7 @@ Fork me on GitHub