diff --git a/docs/internals.rst b/docs/internals.rst index 12127646..17541c0c 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -49,14 +49,14 @@ Take a look to the Markdown reader:: md = Markdown(extensions = ['meta', 'codehilite']) content = md.convert(text) - metadatas = {} + metadata = {} for name, value in md.Meta.items(): - if name in _METADATAS_FIELDS: - meta = _METADATAS_FIELDS[name](value[0]) + if name in _METADATA_FIELDS: + meta = _METADATA_FIELDS[name](value[0]) else: meta = value[0] - metadatas[name.lower()] = meta - return content, metadatas + metadata[name.lower()] = meta + return content, metadata Simple isn't it ? diff --git a/docs/settings.rst b/docs/settings.rst index 363ffc87..971c55dc 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -21,7 +21,7 @@ this file will be passed to the templates as well. ================================================ ===================================================== -Setting name (default value) what does it do? +Setting name (default value) what does it do? ================================================ ===================================================== `AUTHOR` Default author (put your name) `CATEGORY_FEED` ('feeds/%s.atom.xml'[1]_) Where to put the atom categories feeds. @@ -32,6 +32,8 @@ Setting name (default value) what does it do? `DEFAULT_CATEGORY` (``'misc'``) The default category to fallback on. `DEFAULT_DATE_FORMAT` (``'%a %d %B %Y'``) The default date format you want to use. `DEFAULT_LANG` (``'en'``) The default language to use. +`DEFAULT_METADATA` (``()``) A list containing the default metadata for + each content (articles, pages, etc.) `DEFAULT_ORPHANS` (0) The minimum number of articles allowed on the last page. Use this when you don't want to have a last page with very few articles. @@ -45,8 +47,11 @@ Setting name (default value) what does it do? informations from the metadata `FEED` (``'feeds/all.atom.xml'``) relative url to output the atom feed. `FEED_RSS` (``None``, i.e. no RSS) relative url to output the rss feed. +`FILES_TO_COPY` (``()``, no files) A list of tuples (source, destination) of files + to copy from the source directory to the + output path `JINJA_EXTENSIONS` (``[]``) A list of any Jinja2 extensions you want to use. -`KEEP_OUTPUT_DIRECTORY` (``False``) Keep the output directory and just update all +`DELETE_OUTPUT_DIRECTORY` (``False``) Delete the output directory instead of just updating all the generated files. `LOCALE` (''[2]_) Change the locale. `MARKUP` (``('rst', 'md')``) A list of available markup languages you want diff --git a/pelican/__init__.py b/pelican/__init__.py index 12d12210..74bb1058 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -1,5 +1,6 @@ import argparse import os +import time from pelican.generators import (ArticlesGenerator, PagesGenerator, StaticGenerator, PdfGenerator) @@ -13,7 +14,7 @@ VERSION = "2.6.0" class Pelican(object): def __init__(self, settings=None, path=None, theme=None, output_path=None, - markup=None, keep=False): + markup=None, delete_outputdir=False): """Read the settings, and performs some checks on the environment before doing anything else. """ @@ -31,7 +32,7 @@ class Pelican(object): output_path = output_path or settings['OUTPUT_PATH'] self.output_path = os.path.realpath(output_path) self.markup = markup or settings['MARKUP'] - self.keep = keep or settings['KEEP_OUTPUT_DIRECTORY'] + self.delete_outputdir = delete_outputdir or settings['DELETE_OUTPUT_DIRECTORY'] # find the theme in pelican.theme if the given one does not exists if not os.path.exists(self.theme): @@ -54,7 +55,7 @@ class Pelican(object): self.theme, self.output_path, self.markup, - self.keep + self.delete_outputdir ) for cls in self.get_generator_classes() ] @@ -62,8 +63,10 @@ class Pelican(object): if hasattr(p, 'generate_context'): p.generate_context() - # erase the directory if it is not the source - if os.path.realpath(self.path).startswith(self.output_path) and not self.keep: + # erase the directory if it is not the source and if that's + # explicitely asked + if (self.delete_outputdir and + os.path.realpath(self.path).startswith(self.output_path)): clean_output_dir(self.output_path) writer = self.get_writer() @@ -100,11 +103,9 @@ def main(): help='the list of markup language to use (rst or md). Please indicate ' 'them separated by commas') parser.add_argument('-s', '--settings', dest='settings', - help='the settings of the application. Default to None.') - parser.add_argument('-k', '--keep-output-directory', dest='keep', - action='store_true', - help='Keep the output directory and just update all the generated files.' - 'Default is to delete the output directory.') + help='the settings of the application. Default to False.') + parser.add_argument('-d', '--delete-output-directory', dest='delete_outputdir', + action='store_true', help='Delete the output directory.') parser.add_argument('-v', '--verbose', action='store_const', const=log.INFO, dest='verbosity', help='Show all messages') parser.add_argument('-q', '--quiet', action='store_const', const=log.CRITICAL, dest='verbosity', @@ -134,12 +135,14 @@ def main(): cls = getattr(module, cls_name) try: - pelican = cls(settings, args.path, args.theme, args.output, markup, args.keep) + pelican = cls(settings, args.path, args.theme, args.output, markup, + args.delete_outputdir) if args.autoreload: while True: try: if files_changed(pelican.path, pelican.markup): pelican.run() + time.sleep(.5) # sleep to avoid cpu load except KeyboardInterrupt: break else: diff --git a/pelican/contents.py b/pelican/contents.py index dc6bfceb..a3c6670b 100644 --- a/pelican/contents.py +++ b/pelican/contents.py @@ -4,19 +4,21 @@ from pelican.log import * class Page(object): """Represents a page - Given a content, and metadatas, create an adequate object. + Given a content, and metadata, create an adequate object. - :param string: the string to parse, containing the original content. - :param markup: the markup language to use while parsing. + :param content: the string to parse, containing the original content. """ mandatory_properties = ('title',) - def __init__(self, content, metadatas={}, settings={}, filename=None): + def __init__(self, content, metadata={}, settings={}, filename=None): self._content = content self.translations = [] self.status = "published" # default value - for key, value in metadatas.items(): + + local_metadata = dict(settings['DEFAULT_METADATA']) + local_metadata.update(metadata) + for key, value in local_metadata.items(): setattr(self, key.lower(), value) if not hasattr(self, 'author'): @@ -90,6 +92,6 @@ def is_valid_content(content, f): try: content.check_properties() return True - except NameError as e: + except NameError, e: error(u"Skipping %s: impossible to find informations about '%s'" % (f, e)) return False diff --git a/pelican/generators.py b/pelican/generators.py index 569d5f50..7657261b 100755 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -11,7 +11,7 @@ import random from jinja2 import Environment, FileSystemLoader from jinja2.exceptions import TemplateNotFound -from pelican.utils import copytree, get_relative_path, process_translations, open +from pelican.utils import copy, get_relative_path, process_translations, open from pelican.contents import Article, Page, is_valid_content from pelican.readers import read_file from pelican.log import * @@ -35,7 +35,7 @@ class Generator(object): loader=FileSystemLoader(self._templates_path), extensions=self.settings.get('JINJA_EXTENSIONS', []), ) - + # get custom Jinja filters from user settings custom_filters = self.settings.get('JINJA_FILTERS', {}) self._env.filters.update(custom_filters) @@ -63,7 +63,13 @@ class Generator(object): extensions = self.markup files = [] - for root, dirs, temp_files in os.walk(path, followlinks=True): + + try: + iter = os.walk(path, followlinks=True) + except TypeError: # python 2.5 does not support followlinks + iter = os.walk(path) + + for root, dirs, temp_files in iter: for e in exclude: if e in dirs: dirs.remove(e) @@ -116,11 +122,11 @@ class ArticlesGenerator(Generator): if 'TAG_FEED' in self.settings: for tag, arts in self.tags.items(): arts.sort(key=attrgetter('date'), reverse=True) - writer.write_feed(arts, self.context, + writer.write_feed(arts, self.context, self.settings['TAG_FEED'] % tag) if 'TAG_FEED_RSS' in self.settings: - writer.write_feed(arts, self.context, + writer.write_feed(arts, self.context, self.settings['TAG_FEED_RSS'] % tag, feed_type='rss') translations_feeds = defaultdict(list) @@ -142,7 +148,7 @@ class ArticlesGenerator(Generator): relative_urls = self.settings.get('RELATIVE_URLS') ) - # to minimize the number of relative path stuff modification + # to minimize the number of relative path stuff modification # in writer, articles pass first article_template = self.get_template('article') for article in chain(self.translations, self.articles): @@ -183,10 +189,10 @@ class ArticlesGenerator(Generator): files = self.get_files(self.path, exclude=['pages',]) all_articles = [] for f in files: - content, metadatas = read_file(f) + content, metadata = read_file(f) # if no category is set, use the name of the path as a category - if 'category' not in metadatas.keys(): + if 'category' not in metadata.keys(): if os.path.dirname(f) == self.path: category = self.settings['DEFAULT_CATEGORY'] @@ -194,13 +200,13 @@ class ArticlesGenerator(Generator): category = os.path.basename(os.path.dirname(f)) if category != '': - metadatas['category'] = unicode(category) + metadata['category'] = unicode(category) - if 'date' not in metadatas.keys()\ + if 'date' not in metadata.keys()\ and self.settings['FALLBACK_ON_FS_DATE']: - metadatas['date'] = datetime.fromtimestamp(os.stat(f).st_ctime) + metadata['date'] = datetime.fromtimestamp(os.stat(f).st_ctime) - article = Article(content, metadatas, settings=self.settings, + article = Article(content, metadata, settings=self.settings, filename=f) if not is_valid_content(article, f): continue @@ -220,7 +226,7 @@ class ArticlesGenerator(Generator): # sort the articles by date self.articles.sort(key=attrgetter('date'), reverse=True) self.dates = list(self.articles) - self.dates.sort(key=attrgetter('date'), + self.dates.sort(key=attrgetter('date'), reverse=self.context['REVERSE_ARCHIVE_ORDER']) # create tag cloud @@ -236,7 +242,7 @@ class ArticlesGenerator(Generator): if tags: max_count = max(tags) steps = self.settings.get('TAG_CLOUD_STEPS') - + # calculate word sizes self.tag_cloud = [ ( @@ -273,8 +279,8 @@ class PagesGenerator(Generator): def generate_context(self): all_pages = [] for f in self.get_files(os.sep.join((self.path, 'pages'))): - content, metadatas = read_file(f) - page = Page(content, metadatas, settings=self.settings, + content, metadata = read_file(f) + page = Page(content, metadata, settings=self.settings, filename=f) if not is_valid_content(page, f): continue @@ -298,9 +304,10 @@ class StaticGenerator(Generator): def _copy_paths(self, paths, source, destination, output_path, final_path=None): + """Copy all the paths from source to destination""" for path in paths: - copytree(path, source, os.path.join(output_path, destination), - final_path) + copy(path, source, os.path.join(output_path, destination), final_path, + overwrite=True) def generate_output(self, writer): self._copy_paths(self.settings['STATIC_PATHS'], self.path, @@ -308,6 +315,10 @@ class StaticGenerator(Generator): self._copy_paths(self.settings['THEME_STATIC_PATHS'], self.theme, 'theme', self.output_path, '.') + # copy all the files needed + for source, destination in self.settings['FILES_TO_COPY']: + copy(source, self.path, self.output_path, destination, overwrite=True) + class PdfGenerator(Generator): """Generate PDFs on the output dir, for all articles and pages coming from @@ -327,7 +338,7 @@ class PdfGenerator(Generator): # print "Generating pdf for", obj.filename, " in ", output_pdf self.pdfcreator.createPdf(text=open(obj.filename), output=output_pdf) info(u' [ok] writing %s' % output_pdf) - + def generate_context(self): pass diff --git a/pelican/log.py b/pelican/log.py index fce2b681..ac4420ba 100644 --- a/pelican/log.py +++ b/pelican/log.py @@ -1,4 +1,6 @@ -from logging import * +from logging import CRITICAL, ERROR, WARN, INFO, DEBUG +from logging import critical, error, info, warning, warn, debug +from logging import Formatter, getLogger, StreamHandler import sys import os diff --git a/pelican/readers.py b/pelican/readers.py index 4e1d7b2e..7d799f88 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -3,7 +3,7 @@ try: from docutils import core # import the directives to have pygments support - import rstdirectives + from pelican import rstdirectives except ImportError: core = False try: @@ -11,15 +11,14 @@ try: except ImportError: Markdown = False import re -import string from pelican.utils import get_date, open -_METADATAS_PROCESSORS = { - 'tags': lambda x: map(string.strip, x.split(',')), +_METADATA_PROCESSORS = { + 'tags': lambda x: map(unicode.strip, x.split(',')), 'date': lambda x: get_date(x), - 'status': string.strip, + 'status': unicode.strip, } @@ -31,11 +30,11 @@ class RstReader(Reader): extension = "rst" def _parse_metadata(self, content): - """Return the dict containing metadatas""" + """Return the dict containing metadata""" output = {} for m in re.compile('^:([a-z]+): (.*)\s', re.M).finditer(content): name, value = m.group(1).lower(), m.group(2) - output[name] = _METADATAS_PROCESSORS.get( + output[name] = _METADATA_PROCESSORS.get( name, lambda x:x )(value) return output @@ -43,16 +42,18 @@ class RstReader(Reader): def read(self, filename): """Parse restructured text""" text = open(filename) - metadatas = self._parse_metadata(text) + metadata = self._parse_metadata(text) extra_params = {'input_encoding': 'unicode', 'initial_header_level': '2'} - rendered_content = core.publish_parts(text, writer_name='html', + rendered_content = core.publish_parts(text, + source_path=filename, + writer_name='html', settings_overrides=extra_params) title = rendered_content.get('title') content = rendered_content.get('body') - if not metadatas.has_key('title'): - metadatas['title'] = title - return content, metadatas + if not metadata.has_key('title'): + metadata['title'] = title + return content, metadata class MarkdownReader(Reader): enabled = bool(Markdown) @@ -64,13 +65,13 @@ class MarkdownReader(Reader): md = Markdown(extensions = ['meta', 'codehilite']) content = md.convert(text) - metadatas = {} + metadata = {} for name, value in md.Meta.items(): name = name.lower() - metadatas[name] = _METADATAS_PROCESSORS.get( + metadata[name] = _METADATA_PROCESSORS.get( name, lambda x:x )(value[0]) - return content, metadatas + return content, metadata class HtmlReader(Reader): @@ -80,13 +81,13 @@ class HtmlReader(Reader): def read(self, filename): """Parse content and metadata of (x)HTML files""" content = open(filename) - metadatas = {'title':'unnamed'} + metadata = {'title':'unnamed'} for i in self._re.findall(content): key = i.split(':')[0][5:].strip() value = i.split(':')[-1][:-3].strip() - metadatas[key.lower()] = value + metadata[key.lower()] = value - return content, metadatas + return content, metadata diff --git a/pelican/settings.py b/pelican/settings.py index 6c0918a0..a744f465 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -21,7 +21,7 @@ _DEFAULT_CONFIG = {'PATH': None, 'CSS_FILE': 'main.css', 'REVERSE_ARCHIVE_ORDER': False, 'REVERSE_CATEGORY_ORDER': False, - 'KEEP_OUTPUT_DIRECTORY': False, + 'DELETE_OUTPUT_DIRECTORY': False, 'CLEAN_URLS': False, # use /blah/ instead /blah.html in urls 'RELATIVE_URLS': True, 'DEFAULT_LANG': 'en', @@ -37,6 +37,8 @@ _DEFAULT_CONFIG = {'PATH': None, 'WITH_PAGINATION': False, 'DEFAULT_PAGINATION': 5, 'DEFAULT_ORPHANS': 0, + 'DEFAULT_METADATA': (), + 'FILES_TO_COPY': (), } def read_settings(filename): diff --git a/pelican/utils.py b/pelican/utils.py index 732b1830..41733149 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -6,7 +6,7 @@ from datetime import datetime from codecs import open as _open from itertools import groupby from operator import attrgetter -from pelican.log import * +from pelican.log import warning, info def get_date(string): @@ -42,20 +42,38 @@ def slugify(value): value = unicode(re.sub('[^\w\s-]', '', value).strip().lower()) return re.sub('[-\s]+', '-', value) -def copytree(path, origin, destination, topath=None): - """Copy path from origin to destination, silent any errors""" - - if not topath: - topath = path - try: - fromp = os.path.expanduser(os.path.join(origin, path)) - to = os.path.expanduser(os.path.join(destination, topath)) - shutil.copytree(fromp, to) - info('copying %s to %s' % (fromp, to)) +def copy(path, source, destination, destination_path=None, overwrite=False): + """Copy path from origin to destination. - except OSError: - pass + The function is able to copy either files or directories. + :param path: the path to be copied from the source to the destination + :param source: the source dir + :param destination: the destination dir + :param destination_path: the destination path (optional) + :param overwrite: wether to overwrite the destination if already exists or not + + """ + if not destination_path: + destination_path = path + + source_ = os.path.abspath(os.path.expanduser(os.path.join(source, path))) + destination_ = os.path.abspath( + os.path.expanduser(os.path.join(destination, destination_path))) + + if os.path.isdir(source_): + try: + shutil.copytree(source_, destination_) + info('copying %s to %s' % (source_, destination_)) + except OSError: + if overwrite: + shutil.rmtree(destination_) + shutil.copytree(source_, destination_) + info('replacement of %s with %s' % (source_, destination_)) + + elif os.path.isfile(source_): + shutil.copy(source_, destination_) + info('copying %s to %s' % (source_, destination_)) def clean_output_dir(path): """Remove all the files from the output directory""" @@ -164,9 +182,13 @@ def process_translations(content_list): len_ = len(default_lang_items) if len_ > 1: warning(u'there are %s variants of "%s"' % (len_, slug)) + for x in default_lang_items: + warning(' %s' % x.filename) elif len_ == 0: default_lang_items = items[:1] + if not slug: + warning('empty slug for %r' %( default_lang_items[0].filename,)) index.extend(default_lang_items) translations.extend(filter( lambda x: x not in default_lang_items, @@ -188,9 +210,10 @@ def files_changed(path, extensions): def file_times(path): """Return the last time files have been modified""" - for top_level in os.listdir(path): - for root, dirs, files in os.walk(top_level): - for file in filter(with_extension, files): + for root, dirs, files in os.walk(path): + dirs[:] = [x for x in dirs if x[0] != '.'] + for file in files: + if any(file.endswith(ext) for ext in extensions): yield os.stat(os.path.join(root, file)).st_mtime global LAST_MTIME diff --git a/pelican/writers.py b/pelican/writers.py index 3679e249..65cfb202 100644 --- a/pelican/writers.py +++ b/pelican/writers.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import with_statement import os import re from codecs import open @@ -44,9 +45,8 @@ class Writer(object): Return the feed. If no output_path or filename is specified, just return the feed object. - :param articles: the articles to put on the feed. + :param elements: the articles to put on the feed. :param context: the context to get the feed metadata. - :param output_path: where to output the file. :param filename: the filename to output. :param feed_type: the feed type to use (atom or rss) """ @@ -139,7 +139,7 @@ class Writer(object): '%s_page' % key: page}) if page_num > 0: ext = '.' + paginated_name.rsplit('.')[-1] - paginated_name = paginated_name.replace(ext, + paginated_name = paginated_name.replace(ext, '%s%s' % (page_num + 1, ext)) _write_file(template, paginated_localcontext, self.output_path, @@ -149,7 +149,7 @@ class Writer(object): _write_file(template, localcontext, self.output_path, name) def update_context_contents(self, name, context): - """Recursively run the context to find elements (articles, pages, etc) + """Recursively run the context to find elements (articles, pages, etc) whose content getter needs to be modified in order to deal with relative paths. @@ -188,12 +188,12 @@ class Writer(object): return context def inject_update_method(self, name, item): - """Replace the content attribute getter of an element by a function + """Replace the content attribute getter of an element by a function that will deals with its relatives paths. """ def _update_object_content(name, input): - """Change all the relatives paths of the input content to relatives + """Change all the relatives paths of the input content to relatives paths suitable fot the ouput content :param name: path of the output. diff --git a/samples/content/cat1/article1.rst b/samples/content/cat1/article1.rst index 4789543b..1148a8f9 100644 --- a/samples/content/cat1/article1.rst +++ b/samples/content/cat1/article1.rst @@ -2,5 +2,6 @@ Article 1 ######### :date: 2011-02-17 +:yeah: oh yeah ! Article 1 diff --git a/samples/content/extra/robots.txt b/samples/content/extra/robots.txt new file mode 100644 index 00000000..ae5b0d05 --- /dev/null +++ b/samples/content/extra/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: /static/pictures diff --git a/samples/pelican.conf.py b/samples/pelican.conf.py index 07c49d01..2cb9df27 100755 --- a/samples/pelican.conf.py +++ b/samples/pelican.conf.py @@ -24,4 +24,11 @@ SOCIAL = (('twitter', 'http://twitter.com/ametaireau'), ('lastfm', 'http://lastfm.com/user/akounet'), ('github', 'http://github.com/ametaireau'),) +# global metadata to all the contents +DEFAULT_METADATA = (('yeah', 'it is'),) + +# static paths will be copied under the same name STATIC_PATHS = ["pictures",] + +# A list of files to copy from the source to the destination +FILES_TO_COPY = (('extra/robots.txt', 'robots.txt'),) diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 936a3171..b90802d8 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python from setuptools import setup import sys @@ -17,7 +18,7 @@ setup( long_description=open('README.rst').read(), packages = ['pelican'], include_package_data = True, - install_requires = requires, + install_requires = requires, scripts = ['bin/pelican'], classifiers = ['Development Status :: 5 - Production/Stable', 'Environment :: Console',