diff --git a/docs/fr/configuration.rst b/docs/fr/configuration.rst index 151eff3a..abfc7ef5 100644 --- a/docs/fr/configuration.rst +++ b/docs/fr/configuration.rst @@ -155,7 +155,5 @@ SITEURL : STATIC_PATHS : Les chemins statiques que vous voulez avoir accès sur le chemin de sortie "statique" ; - - - - +MARKDOWN_EXTENSIONS : + Liste des extentions Markdown que vous souhaitez utiliser ; diff --git a/docs/settings.rst b/docs/settings.rst index 03f1e4fa..ff59e0d3 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -69,6 +69,12 @@ Setting name (default value) What doe `PDF_GENERATOR` (``False``) Set to True if you want to have PDF versions of your documents. You will need to install `rst2pdf`. +`OUTPUT_SOURCES` (``False``) Set to True if you want to copy the articles and pages in their + original format (e.g. Markdown or ReStructeredText) to the + specified OUTPUT_PATH. +`OUTPUT_SOURCES_EXTENSION` (``.text``) Controls the extension that will be used by the SourcesGenerator. + Defaults to ``.text``. If not a valid string the default value + will be used. `RELATIVE_URLS` (``True``) Defines whether Pelican should use document-relative URLs or not. If set to ``False``, Pelican will use the SITEURL setting to construct absolute URLs. @@ -108,6 +114,7 @@ Setting name (default value) What doe Example: projects, resume, profile ... This templates need to use ``DIRECT_TEMPLATES`` setting +`MARKDOWN_EXTENSIONS` (``['toc',]``) A list of any Markdown extensions you want to use. ===================================================================== ===================================================================== .. [#] Default is the system locale. diff --git a/pelican/__init__.py b/pelican/__init__.py index 6281675b..9bcac779 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -8,11 +8,13 @@ import argparse from pelican import signals -from pelican.generators import (Generator, ArticlesGenerator, PagesGenerator, - StaticGenerator, PdfGenerator, LessCSSGenerator) +from pelican.generators import (ArticlesGenerator, PagesGenerator, + StaticGenerator, PdfGenerator, + LessCSSGenerator, SourceFileGenerator) from pelican.log import init from pelican.settings import read_settings, _DEFAULT_CONFIG -from pelican.utils import clean_output_dir, files_changed, file_changed, NoFilesError +from pelican.utils import (clean_output_dir, files_changed, file_changed, + NoFilesError) from pelican.writers import Writer __major__ = 3 @@ -78,7 +80,7 @@ class Pelican(object): logger.debug("Loading plugin `{0}' ...".format(plugin)) plugin = __import__(plugin, globals(), locals(), 'module') - logger.debug("Registering plugin `{0}' ...".format(plugin.__name__)) + logger.debug("Registering plugin `{0}'".format(plugin.__name__)) plugin.register() def _handle_deprecation(self): @@ -139,8 +141,8 @@ class Pelican(object): 'Modify CATEGORY_FEED to CATEGORY_FEED_ATOM in your settings and ' 'theme for the same behavior. Temporarily setting ' 'CATEGORY_FEED_ATOM for backwards compatibility.') - self.settings['CATEGORY_FEED_ATOM'] = self.settings['CATEGORY_FEED'] - + self.settings['CATEGORY_FEED_ATOM'] =\ + self.settings['CATEGORY_FEED'] def run(self): """Run the generators and return""" @@ -187,6 +189,8 @@ class Pelican(object): generators.append(PdfGenerator) if self.settings['LESS_GENERATOR']: # can be True or PATH to lessc generators.append(LessCSSGenerator) + if self.settings['OUTPUT_SOURCES']: + generators.append(SourceFileGenerator) for pair in signals.get_generators.send(self): (funct, value) = pair @@ -290,7 +294,7 @@ def main(): # have. if files_changed(pelican.path, pelican.markup) or \ files_changed(pelican.theme, ['']): - if files_found_error == False: + if not files_found_error: files_found_error = True pelican.run() @@ -306,8 +310,9 @@ def main(): logger.warning("Keyboard interrupt, quitting.") break except NoFilesError: - if files_found_error == True: - logger.warning("No valid files found in content. Nothing to generate.") + if files_found_error: + logger.warning("No valid files found in content. " + "Nothing to generate.") files_found_error = False time.sleep(1) # sleep to avoid cpu load except Exception, e: diff --git a/pelican/generators.py b/pelican/generators.py index f4366f53..461c8b46 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -518,6 +518,19 @@ class PdfGenerator(Generator): for page in self.context['pages']: self._create_pdf(page, pdf_path) +class SourceFileGenerator(Generator): + def generate_context(self): + self.output_extension = self.settings['OUTPUT_SOURCES_EXTENSION'] + + def _create_source(self, obj, output_path): + filename = os.path.splitext(obj.save_as)[0] + dest = os.path.join(output_path, filename + self.output_extension) + copy('', obj.filename, dest) + + def generate_output(self, writer=None): + logger.info(u' Generating source files...') + for object in chain(self.context['articles'], self.context['pages']): + self._create_source(object, self.output_path) class LessCSSGenerator(Generator): """Compile less css files.""" diff --git a/pelican/plugins/sitemap.py b/pelican/plugins/sitemap.py index 6402ba9c..ebce1f04 100644 --- a/pelican/plugins/sitemap.py +++ b/pelican/plugins/sitemap.py @@ -1,7 +1,8 @@ +import collections import os.path from datetime import datetime -from logging import debug, warning, error, info +from logging import warning, info from codecs import open from pelican import signals, contents @@ -14,45 +15,17 @@ TXT_HEADER = u"""{0}/index.html XML_HEADER = u""" - - - {0}/index.html - {1} - {2} - {3} - - - - {0}/archives.html - {1} - {2} - {3} - - - - {0}/tags.html - {1} - {2} - {3} - - - - {0}/categories.html - {1} - {2} - {3} - +xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd" +xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> """ XML_URL = u""" - - {0}/{1} - {2} - {3} - {4} - + +{0}/{1} +{2} +{3} +{4} + """ XML_FOOTER = u""" @@ -69,7 +42,6 @@ def format_date(date): return date.strftime("%Y-%m-%dT%H:%M:%S") + tz - class SitemapGenerator(object): def __init__(self, context, settings, path, theme, output_path, *null): @@ -146,6 +118,10 @@ class SitemapGenerator(object): if getattr(page, 'status', 'published') != 'published': return + page_path = os.path.join(self.output_path, page.url) + if not os.path.exists(page_path): + return + lastmod = format_date(getattr(page, 'date', self.now)) if isinstance(page, contents.Article): @@ -176,22 +152,29 @@ class SitemapGenerator(object): for article in self.context['articles']: pages += article.translations - info('writing {0}'.format(path)) with open(path, 'w', encoding='utf-8') as fd: if self.format == 'xml': - fd.write(XML_HEADER.format( - self.siteurl, - format_date(self.now), - self.changefreqs['indexes'], - self.priorities['indexes'] - ) - ) + fd.write(XML_HEADER) else: fd.write(TXT_HEADER.format(self.siteurl)) + FakePage = collections.namedtuple('FakePage', + ['status', + 'date', + 'url']) + + for standard_page_url in ['index.html', + 'archives.html', + 'tags.html', + 'categories.html']: + fake = FakePage(status='published', + date=self.now, + url=standard_page_url) + self.write_url(fake, fd) + for page in pages: self.write_url(page, fd) @@ -199,7 +182,6 @@ class SitemapGenerator(object): fd.write(XML_FOOTER) - def get_generators(generators): return SitemapGenerator diff --git a/pelican/readers.py b/pelican/readers.py index 30038f7a..dab829b9 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -102,7 +102,7 @@ class RstReader(Reader): def _get_publisher(self, filename): extra_params = {'initial_header_level': '2'} pub = docutils.core.Publisher( - destination_class=docutils.io.StringOutput) + destination_class=docutils.io.StringOutput) pub.set_components('standalone', 'restructuredtext', 'html') pub.writer.translator_class = PelicanHTMLTranslator pub.process_programmatic_settings(None, extra_params, None) @@ -129,8 +129,13 @@ class MarkdownReader(Reader): def read(self, filename): """Parse content and metadata of markdown files""" + markdown_extensions = self.settings.get('MARKDOWN_EXTENSIONS', []) + if isinstance(markdown_extensions, (str, unicode)): + markdown_extensions = [m.strip() for m in + markdown_extensions.split(',')] text = pelican_open(filename) - md = Markdown(extensions=set(self.extensions + ['meta'])) + md = Markdown(extensions=set( + self.extensions + markdown_extensions + ['meta'])) content = md.convert(text) metadata = {} diff --git a/pelican/settings.py b/pelican/settings.py index c1ebe9b6..7d0f6cdb 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -32,6 +32,8 @@ _DEFAULT_CONFIG = {'PATH': '.', 'SITENAME': 'A Pelican Blog', 'DISPLAY_PAGES_ON_MENU': True, 'PDF_GENERATOR': False, + 'OUTPUT_SOURCES': False, + 'OUTPUT_SOURCES_EXTENSION': '.text', 'DEFAULT_CATEGORY': 'misc', 'DEFAULT_DATE': 'fs', 'WITH_FUTURE_DATES': True, @@ -76,6 +78,7 @@ _DEFAULT_CONFIG = {'PATH': '.', 'SUMMARY_MAX_LENGTH': 50, 'WEBASSETS': False, 'PLUGINS': [], + 'MARKDOWN_EXTENSIONS': ['toc', ], } @@ -175,4 +178,11 @@ def configure_settings(settings, default_settings=None, filename=None): logger.warn("You must install the webassets module to use WEBASSETS.") settings['WEBASSETS'] = False + if 'OUTPUT_SOURCES_EXTENSION' in settings: + if not isinstance(settings['OUTPUT_SOURCES_EXTENSION'], str): + settings['OUTPUT_SOURCES_EXTENSION'] = _DEFAULT_CONFIG['OUTPUT_SOURCES_EXTENSION'] + logger.warn("Detected misconfiguration with OUTPUT_SOURCES_EXTENSION." + " falling back to the default extension " + + _DEFAULT_CONFIG['OUTPUT_SOURCES_EXTENSION']) + return settings diff --git a/pelican/tools/pelican_import.py b/pelican/tools/pelican_import.py index c2d259f2..fc28c6a4 100755 --- a/pelican/tools/pelican_import.py +++ b/pelican/tools/pelican_import.py @@ -184,6 +184,8 @@ def build_header(title, date, author, categories, tags): header = '%s\n%s\n' % (title, '#' * len(title)) if date: header += ':date: %s\n' % date + if author: + header += ':author: %s\n' % author if categories: header += ':category: %s\n' % ', '.join(categories) if tags: @@ -196,6 +198,8 @@ def build_markdown_header(title, date, author, categories, tags): header = 'Title: %s\n' % title if date: header += 'Date: %s\n' % date + if author: + header += 'Author: %s\n' % author if categories: header += 'Category: %s\n' % ', '.join(categories) if tags: diff --git a/pelican/utils.py b/pelican/utils.py index 60ecee34..7e9ab4cb 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -97,6 +97,17 @@ def copy(path, source, destination, destination_path=None, overwrite=False): def clean_output_dir(path): """Remove all the files from the output directory""" + if not os.path.exists(path): + logger.debug("Directory already removed: %s" % path) + return + + if not os.path.isdir(path): + try: + os.remove(path) + except Exception, e: + logger.error("Unable to delete file %s; %e" % path, e) + return + # remove all the existing content from the output folder for filename in os.listdir(path): file = os.path.join(path, filename) diff --git a/pelican/writers.py b/pelican/writers.py index 75971ee9..65f0b338 100644 --- a/pelican/writers.py +++ b/pelican/writers.py @@ -148,9 +148,9 @@ class Writer(object): paginators[key] = Paginator(object_list, len(object_list)) # generated pages, and write + name_root, ext = os.path.splitext(name) for page_num in range(paginators.values()[0].num_pages): paginated_localcontext = localcontext.copy() - paginated_name = name for key in paginators.iterkeys(): paginator = paginators[key] page = paginator.page(page_num + 1) @@ -158,9 +158,10 @@ class Writer(object): {'%s_paginator' % key: paginator, '%s_page' % key: page}) if page_num > 0: - ext = '.' + paginated_name.rsplit('.')[-1] - paginated_name = paginated_name.replace(ext, - '%s%s' % (page_num + 1, ext)) + paginated_name = '%s%s%s' % ( + name_root, page_num + 1, ext) + else: + paginated_name = name _write_file(template, paginated_localcontext, self.output_path, paginated_name) diff --git a/tests/content/article_with_markdown_markup_extensions.md b/tests/content/article_with_markdown_markup_extensions.md new file mode 100644 index 00000000..6cf56403 --- /dev/null +++ b/tests/content/article_with_markdown_markup_extensions.md @@ -0,0 +1,8 @@ +Title: Test Markdown extensions + +[TOC] + +## Level1 + +### Level2 + diff --git a/tests/test_generators.py b/tests/test_generators.py index 3a4ea1e3..374172d8 100644 --- a/tests/test_generators.py +++ b/tests/test_generators.py @@ -69,6 +69,7 @@ class TestArticlesGenerator(unittest.TestCase): [u'Article title', 'published', 'Default', 'article'], [u'Article with template', 'published', 'Default', 'custom'], [u'Test md File', 'published', 'test', 'article'], + [u'Test Markdown extensions', 'published', u'Default', 'article'], [u'This is a super article !', 'published', 'Yeah', 'article'], [u'This is an article with category !', 'published', 'yeah', 'article'], [u'This is an article without category !', 'published', 'Default', 'article'], diff --git a/tests/test_readers.py b/tests/test_readers.py index 299aa378..406027c1 100644 --- a/tests/test_readers.py +++ b/tests/test_readers.py @@ -70,7 +70,7 @@ class RstReaderTest(unittest.TestCase): class MdReaderTest(unittest.TestCase): @unittest.skipUnless(readers.Markdown, "markdown isn't installed") - def test_article_with_md_extention(self): + def test_article_with_md_extension(self): # test to ensure the md extension is being processed by the correct reader reader = readers.MarkdownReader({}) content, metadata = reader.read(_filename('article_with_md_extension.md')) @@ -90,3 +90,22 @@ class MdReaderTest(unittest.TestCase): "

This is another markdown test file. Uses the mkd extension.

" self.assertEqual(content, expected) + + @unittest.skipUnless(readers.Markdown, "markdown isn't installed") + def test_article_with_markdown_markup_extension(self): + # test to ensure the markdown markup extension is being processed as expected + reader = readers.MarkdownReader({}) + reader.settings.update(dict(MARKDOWN_EXTENSIONS=['toc', ])) + content, metadata = reader.read(_filename('article_with_markdown_markup_extensions.md')) + expected = '
\n'\ + '\n'\ + '
\n'\ + '

Level1

\n'\ + '

Level2

' + + self.assertEqual(content, expected) diff --git a/tests/test_utils.py b/tests/test_utils.py index 148e322a..58118d4f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -112,3 +112,16 @@ class TestUtils(unittest.TestCase): self.assertTrue(os.path.isdir(test_directory)) self.assertListEqual([], os.listdir(test_directory)) shutil.rmtree(test_directory) + + def test_clean_output_dir_not_there(self): + test_directory = os.path.join(os.path.dirname(__file__), 'does_not_exist') + utils.clean_output_dir(test_directory) + self.assertTrue(not os.path.exists(test_directory)) + + def test_clean_output_dir_is_file(self): + test_directory = os.path.join(os.path.dirname(__file__), 'this_is_a_file') + f = open(test_directory, 'w') + f.write('') + f.close() + utils.clean_output_dir(test_directory) + self.assertTrue(not os.path.exists(test_directory))