diff --git a/.hgignore b/.hgignore deleted file mode 100644 index a0f6b7c5..00000000 --- a/.hgignore +++ /dev/null @@ -1,9 +0,0 @@ -syntax: glob -output/* -*.pyc -MANIFEST -build -dist -docs/_build -Paste-* -*.egg-info diff --git a/.hgtags b/.hgtags deleted file mode 100644 index 6d4c8d99..00000000 --- a/.hgtags +++ /dev/null @@ -1,29 +0,0 @@ -7acafbf7e47b1287525026ad8b4f1efe443d5403 1.2 -7acafbf7e47b1287525026ad8b4f1efe443d5403 1.2 -ae850ab0fd62a98a98da7ce74ac794319c6a5066 1.2 -54a0309f79d6c5b54d8e1e3b5e3f744856b68a73 1.1 -8f5e0eb037768351eb08840e588a4364266a69b3 1.1.1 -bb986ed591734ca469f726753cbc48ebbfce0dcc 1.2.1 -8a3dad99cbfa6bb5d0ef073213d0d86e0b4c5dba 1.2.2 -4a20105a242ab154f6202aa6651979bfbb4cf95e 1.2.3 -803aa0976cca3dd737777c640722988b1f3769fe 1.2.4 -703c4511105fd9c8b85afda951a294c194e7cf3e 1.2.5 -6e46a40aaa850a979f5d09dd95d02791ec7ab0ef 2.0 -bf14d1a5c1fae9475447698f0f9b8d35c551f732 2.1 -da86343ebd543e5865050e47ecb0937755528d13 2.1.1 -760187f048bb23979402f950ecb5d3c5493995b1 2.2 -20aa16fe4daa3b70f6c063f170edc916b49837ed 2.3 -f9c1d94081504f21f5b2ba147a38099e45db1769 2.4 -e65199a0b2706d2fb48f7a3c015e869716e0bec1 2.4.1 -89dbd7b6f114508eae62fc821326f4797dfc8b23 2.4.2 -979b4473af56a191a278c83058bc9c8fa1fde30e 2.4.3 -26a444fbb78becae358afa0a5b47587db8739b21 2.4.4 -3542b65fd1963ae7065b6a3bc912fbb6c150e98c 2.4.5 -87745dfdd51b96bf18eaaf6c402effa902c1b856 2.5.0 -294a2830a393d5a97671dc211dbdb5254a15e604 2.5.1 -294a2830a393d5a97671dc211dbdb5254a15e604 2.5.1 -92b31e41134cb2c1a156ce623338cf634d2ebc3e 2.5.1 -7d728f8e771cbbc802ce81e424e08a8eecbd48dc 2.5.2 -7d728f8e771cbbc802ce81e424e08a8eecbd48dc 2.5.2 -6d368a1739a4ce48d2d04b00db04fa538e2bf90a 2.5.2 -1f9dd44b546425216b1fa35fd88d3d532da8916b 2.5.3 diff --git a/.travis.yml b/.travis.yml index 1ff512b6..1df32baa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,15 @@ before_install: - sudo locale-gen fr_FR.UTF-8 tr_TR.UTF-8 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then ln -s /usr/share/asciidoc/asciidocapi.py ~/virtualenv/python2.7/lib/python2.7/site-packages/; fi - - pip install mock --use-mirrors - - pip install . --use-mirrors - - pip install --use-mirrors Markdown -script: python -m unittest discover + - pip install mock nose nose-cov Markdown + - pip install . +script: nosetests -sv --with-coverage --cover-package=pelican pelican +after_success: + # Report coverage results to coveralls.io + - pip install coveralls + - coveralls +notifications: + irc: + channels: + - "irc.freenode.org#pelican" + on_success: change diff --git a/MANIFEST.in b/MANIFEST.in index 136243c0..dcf9ea45 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,3 @@ include *.rst -global-include *.py -recursive-include pelican *.html *.css *png *.in *.rst *.md *.mkd *.xml +recursive-include pelican *.html *.css *png *.in *.rst *.md *.mkd *.xml *.py include LICENSE THANKS docs/changelog.rst diff --git a/bumpr.rc b/bumpr.rc new file mode 100644 index 00000000..cfc90fd7 --- /dev/null +++ b/bumpr.rc @@ -0,0 +1,30 @@ +[bumpr] +file = pelican/__init__.py +vcs = git +clean = + python setup.py clean + rm -rf *egg-info build dist +tests = tox +publish = python setup.py sdist register upload +files = README.rst + +[bump] +unsuffix = true +message = Bump version {version} + +[prepare] +part = patch +suffix = dev +message = Prepare version {version} for next development cycle + +[changelog] +file = docs/changelog.rst +separator = = +bump = {version} ({date:%Y-%m-%d}) +prepare = Next release + +[readthedoc] +url = http://docs.getpelican.com/{tag} + +[commands] +bump = sed -i "s/last_stable\s*=.*/last_stable = '{version}'/" docs/conf.py diff --git a/dev_requirements.txt b/dev_requirements.txt index fa2634a0..c90ac630 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -6,3 +6,6 @@ Markdown BeautifulSoup4 lxml typogrify + +# To perform release +bumpr==0.2.0 diff --git a/docs/changelog.rst b/docs/changelog.rst index fcec0b53..064c8fd2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,8 +1,21 @@ Release history ############### -3.3 (XXXX-XX-XX) -================ +Next release +============ + +* Rename signals for better consistency (some plugins may need to be updated) +* Move metadata extraction from generators to readers; metadata extraction no + longer article-specific +* Deprecate ``FILES_TO_COPY`` in favor of ``STATIC_PATHS`` and + ``EXTRA_PATH_METADATA`` +* Add support for ``{}`` in relative links syntax, besides ``||`` +* Add support for ``{tag}`` and ``{category}`` relative links + +3.2.1 and 3.2.2 +=============== + +* Facilitate inclusion in FreeBSD Ports Collection 3.2 (2013-04-24) ================ diff --git a/docs/conf.py b/docs/conf.py index 40de84c7..5ac81b9e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,18 +4,26 @@ import sys, os sys.path.append(os.path.abspath(os.pardir)) -from pelican import __version__, __major__ +from pelican import __version__ # -- General configuration ----------------------------------------------------- templates_path = ['_templates'] -extensions = ['sphinx.ext.autodoc',] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.ifconfig', 'sphinx.ext.extlinks'] source_suffix = '.rst' master_doc = 'index' project = 'Pelican' copyright = '2010, Alexis Metaireau and contributors' exclude_patterns = ['_build'] -version = __version__ -release = __major__ +release = __version__ +version = '.'.join(release.split('.')[:1]) +last_stable = '3.2.2' +rst_prolog = ''' +.. |last_stable| replace:: :pelican-doc:`{0}` +'''.format(last_stable) + +extlinks = { + 'pelican-doc': ('http://docs.getpelican.com/%s/', '') +} # -- Options for HTML output --------------------------------------------------- diff --git a/docs/contribute.rst b/docs/contribute.rst index 80d07644..304d1de8 100644 --- a/docs/contribute.rst +++ b/docs/contribute.rst @@ -88,9 +88,10 @@ Pelican, and the changes to that output are expected and deemed correct given the nature of your changes, then you should update the output used by the functional tests. To do so, you can use the following two commands:: - $ pelican -o pelican/tests/output/custom/ -s samples/pelican.conf.py \ + $ LC_ALL=en_US.utf8 pelican -o pelican/tests/output/custom/ \ + -s samples/pelican.conf.py samples/content/ + $ LC_ALL=en_US.utf8 pelican -o pelican/tests/output/basic/ \ samples/content/ - $ pelican -o pelican/tests/output/basic/ samples/content/ Testing on Python 2 and 3 ------------------------- diff --git a/docs/faq.rst b/docs/faq.rst index a8043e07..c1d4f5a0 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -90,12 +90,16 @@ For reStructuredText, this metadata should of course be prefixed with a colon:: :Modified: 2012-08-08 -This metadata can then be accessed in the template:: +This metadata can then be accessed in templates such as ``article.html`` via:: {% if article.modified %} Last modified: {{ article.modified }} {% endif %} +If you want to include metadata in templates outside the article context (e.g., ``base.html``), the ``if`` statement should instead be:: + + {% if article and article.modified %} + How do I assign custom templates on a per-page basis? ===================================================== @@ -187,3 +191,15 @@ Older themes that referenced the old setting names may not link properly. In order to rectify this, please update your theme for compatibility by changing the relevant values in your template files. For an example of complete feed headers and usage please check out the ``simple`` theme. + +Is Pelican only suitable for blogs? +=================================== + +No. Pelican can be easily configured to create and maintain any type of static site. +This may require little customization of your theme and Pelican configuration. +For example, if you are building a launch site for your product and do not need +tags on your site. You can hide tags by removing relevant html code from your theme. +You can also disable generation of tags pages:: + + TAGS_SAVE_AS = '' + TAG_SAVE_AS = '' diff --git a/docs/getting_started.rst b/docs/getting_started.rst index ddffb5ff..2605c3a1 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -41,14 +41,14 @@ method:: If you have Git installed and prefer to install the latest bleeding-edge version of Pelican rather than a stable release, use the following command:: - $ pip install -e git://github.com/getpelican/pelican#egg=pelican + $ pip install -e git+https://github.com/getpelican/pelican.git#egg=pelican If you plan on using Markdown as a markup format, you'll need to install the Markdown library as well:: $ pip install Markdown -If you want to use AsciiDoc you need to install it from `source +If you want to use AsciiDoc_ you need to install it from `source `_ or use your operating system's package manager. @@ -121,6 +121,10 @@ automatically installed without any action on your part: broadcast signaling system * `unidecode `_, for ASCII transliterations of Unicode text +* `six `_, for Python 2 and 3 compatibility + utilities +* `MarkupSafe `_, for a markup safe + string implementation If you want the following optional packages, you will need to install them manually via ``pip``: @@ -148,21 +152,87 @@ if you plan to create non-chronological content):: │   └── (pages) ├── output ├── develop_server.sh + ├── fabfile.py ├── Makefile ├── pelicanconf.py # Main settings file └── publishconf.py # Settings to use when ready to publish The next step is to begin to adding content to the *content* folder that has -been created for you. (See *Writing articles using Pelican* section below for -more information about how to format your content.) +been created for you. (See the **Writing content using Pelican** section below +for more information about how to format your content.) Once you have written some content to generate, you can use the ``pelican`` command to generate your site, which will be placed in the output folder. -Alternatively, you can use automation tools that "wrap" the ``pelican`` command -to simplify the process of generating, previewing, and uploading your site. One -such tool is the ``Makefile`` that's automatically created for you when you use -``pelican-quickstart`` to create a skeleton project. To use ``make`` to -generate your site, run:: + +Automation tools +================ + +While the ``pelican`` command is the canonical way to generate your site, +automation tools can be used to streamline the generation and publication +flow. One of the questions asked during the ``pelican-quickstart`` process +described above pertains to whether you want to automate site generation and +publication. If you answered "yes" to that question, a ``fabfile.py`` and +``Makefile`` will be generated in the root of your project. These files, +pre-populated with certain information gleaned from other answers provided +during the ``pelican-quickstart`` process, are meant as a starting point and +should be customized to fit your particular needs and usage patterns. If you +find one or both of these automation tools to be of limited utility, these +files can deleted at any time and will not affect usage of the canonical +``pelican`` command. + +Following are automation tools that "wrap" the ``pelican`` command and can +simplify the process of generating, previewing, and uploading your site. + +Fabric +------ + +The advantage of Fabric_ is that it is written in Python and thus can be used +in a wide range of environments. The downside is that it must be installed +separately. Use the following command to install Fabric, prefixing with +``sudo`` if your environment requires it:: + + $ pip install Fabric + +Take a moment to open the ``fabfile.py`` file that was generated in your +project root. You will see a number of commands, any one of which can be +renamed, removed, and/or customized to your liking. Using the out-of-the-box +configuration, you can generate your site via:: + + $ fab build + +If you'd prefer to have Pelican automatically regenerate your site every time a +change is detected (which is handy when testing locally), use the following +command instead:: + + $ fab regenerate + +To serve the generated site so it can be previewed in your browser at +http://localhost:8000/:: + + $ fab serve + +If during the ``pelican-quickstart`` process you answered "yes" when asked +whether you want to upload your site via SSH, you can use the following command +to publish your site via rsync over SSH:: + + $ fab publish + +These are just a few of the commands available by default, so feel free to +explore ``fabfile.py`` and see what other commands are available. More +importantly, don't hesitate to customize ``fabfile.py`` to suit your specific +needs and preferences. + +Make +---- + +A ``Makefile`` is also automatically created for you when you say "yes" to +the relevant question during the ``pelican-quickstart`` process. The advantage +of this method is that the ``make`` command is built into most POSIX systems +and thus doesn't require installing anything else in order to use it. The +downside is that non-POSIX systems (e.g., Windows) do not include ``make``, +and installing it on those systems can be a non-trivial task. + +If you want to use ``make`` to generate your site, run:: $ make html @@ -173,7 +243,7 @@ command instead:: $ make regenerate To serve the generated site so it can be previewed in your browser at -http://localhost:8000:: +http://localhost:8000/:: $ make serve @@ -209,6 +279,8 @@ The idea behind "pages" is that they are usually not temporal in nature and are used for content that does not change very often (e.g., "About" or "Contact" pages). +.. _internal_metadata: + File metadata ------------- @@ -251,6 +323,9 @@ pattern:: This is the content of my super blog post. +Conventions for AsciiDoc_ posts, which should have an ``.asc`` extension, can +be found on the AsciiDoc_ site. + Pelican can also process HTML files ending in ``.html`` and ``.htm``. Pelican interprets the HTML in a very straightforward manner, reading metadata from ``meta`` tags, the title from the ``title`` tag, and the body out from the @@ -259,11 +334,11 @@ interprets the HTML in a very straightforward manner, reading metadata from My super title - - - - - + + + + + This is the content of my super blog post. @@ -282,7 +357,10 @@ by the directory in which the file resides. For example, a file located at ``python/foobar/myfoobar.rst`` will have a category of ``foobar``. If you would like to organize your files in other ways where the name of the subfolder would not be a good category name, you can set the setting ``USE_FOLDER_AS_CATEGORY`` -to ``False``. +to ``False``. When parsing dates given in the page metadata, Pelican supports +the W3C's `suggested subset ISO 8601`__. + +__ `W3C ISO 8601`_ If you do not explicitly specify summary metadata for a given post, the ``SUMMARY_MAX_LENGTH`` setting can be used to specify how many words from the @@ -312,6 +390,8 @@ If you want to exclude any pages from being linked to or listed in the menu then add a ``status: hidden`` attribute to its metadata. This is useful for things like making error pages that fit the generated theme of your site. +.. _ref-linking-to-internal-content: + Linking to internal content --------------------------- @@ -322,7 +402,7 @@ and images that may be sitting alongside the current post (instead of having to determine where those resources will be placed after site generation). To link to internal content (files in the ``content`` directory), use the -following syntax: ``|filename|path/to/file``:: +following syntax: ``{filename}path/to/file``:: website/ @@ -343,8 +423,8 @@ In this example, ``article1.rst`` could look like:: See below intra-site link examples in reStructuredText format. - `a link relative to content root <|filename|/cat/article2.md>`_ - `a link relative to current file <|filename|cat/article2.md>`_ + `a link relative to content root <{filename}/cat/article2.rst>`_ + `a link relative to current file <{filename}cat/article2.rst>`_ and ``article2.md``:: @@ -353,8 +433,8 @@ and ``article2.md``:: See below intra-site link examples in Markdown format. - [a link relative to content root](|filename|/article1.rst) - [a link relative to current file](|filename|../article1.rst) + [a link relative to content root]({filename}/article1.md) + [a link relative to current file]({filename}../article1.md) Embedding non-article or non-page content is slightly different in that the directories need to be specified in ``pelicanconf.py`` file. The ``images`` @@ -369,7 +449,7 @@ manually:: And ``image-test.md`` would include:: - ![Alt Text](|filename|/images/han.jpg) + ![Alt Text]({filename}/images/han.jpg) Any content can be linked in this way. What happens is that the ``images`` directory gets copied to ``output/static/`` upon publishing. This is @@ -381,6 +461,17 @@ following to ``pelicanconf.py``:: And then the ``pdfs`` directory would also be copied to ``output/static/``. +You can also link to categories or tags, using the ``{tag}tagname`` and +``{category}foobar`` syntax. + +For backward compatibility, Pelican also supports bars ``||``, besides ``{}``, +i.e. the ``filename``, ``tag`` and ``category`` identifiers can be enclosed +in bars ``|`` instead of braces ``{}``, for example, ``|filename|an_article.rst``, +``|tag|tagname``, ``|category|foobar``. + +Using ``{}`` ensures that the syntax will not collide with markdown extensions or +reST directives. + Importing an existing blog -------------------------- @@ -442,6 +533,9 @@ which posts are translations:: That's true, foobar is still alive! + +.. _internal_pygments_options: + Syntax highlighting ------------------- @@ -465,6 +559,65 @@ indenting both the identifier and code:: The specified identifier (e.g. ``python``, ``ruby``) should be one that appears on the `list of available lexers `_. +When using reStructuredText the following options are available in the +code-block directive: + +============= ============ ========================================= +Option Valid values Description +============= ============ ========================================= +anchorlinenos N/A If present wrap line numbers in tags. +classprefix string String to prepend to token class names +hl_lines numbers List of lines to be highlighted. +lineanchors string Wrap each line in an anchor using this + string and -linenumber. +linenos string If present or set to "table" output line + numbers in a table, if set to + "inline" output them inline. "none" means + do not output the line numbers for this + table. +linenospecial number If set every nth line will be given the + 'special' css class. +linenostart number Line number for the first line. +linenostep number Print every nth line number. +lineseparator string String to print between lines of code, + '\n' by default. +linespans string Wrap each line in a span using this and + -linenumber. +nobackground N/A If set do not output background color for + the wrapping element +nowrap N/A If set do not wrap the tokens at all. +tagsfile string ctags file to use for name definitions. +tagurlformat string format for the ctag links. +============= ============ ========================================= + +Note that, depending on its version, your pygments module might not have +all of these available. See the `Pygments documentation +`_ for the HTML formatter for more +details on each of the options. + +for example the below code block enables line numbers, starting at 153, +and prefixes the Pygments CSS classes with *pgcss* to make the names +more unique and avoid possible CSS conflicts:: + + .. code-block:: identifier + :classprefix: pgcss + :linenos: table + :linenostart: 153 + + + +It is also possible to specify the ``PYGMENTS_RST_OPTIONS`` variable +in your Pelican configuration file for settings that will be +automatically applied to every code block. + +For example, if you wanted to have line numbers on for every code block +and a CSS prefix you would set this variable to:: + + PYGMENTS_RST_OPTIONS = { 'classprefix': 'pgcss', 'linenos': 'table'} + +If specified, settings for individual code blocks will override the +defaults in the configuration file. + Publishing drafts ----------------- @@ -474,3 +627,6 @@ metadata. That article will then be output to the ``drafts`` folder and not listed on the index page nor on any category page. .. _virtualenv: http://www.virtualenv.org/ +.. _W3C ISO 8601: http://www.w3.org/TR/NOTE-datetime +.. _Fabric: http://fabfile.org/ +.. _AsciiDoc: http://www.methods.co.nz/asciidoc/ diff --git a/docs/importer.rst b/docs/importer.rst index 9a0c513e..b1d1b926 100644 --- a/docs/importer.rst +++ b/docs/importer.rst @@ -14,6 +14,7 @@ software to reStructuredText or Markdown. The supported import formats are: - WordPress XML export - Dotclear export - Posterous API +- Tumblr API - RSS/Atom feed The conversion from HTML to reStructuredText or Markdown relies on `Pandoc`_. @@ -41,16 +42,17 @@ Usage :: - pelican-import [-h] [--wpfile] [--dotclear] [--posterous] [--feed] [-o OUTPUT] + pelican-import [-h] [--wpfile] [--dotclear] [--posterous] [--tumblr] [--feed] [-o OUTPUT] [-m MARKUP] [--dir-cat] [--dir-page] [--strip-raw] [--disable-slugs] - [-e EMAIL] [-p PASSWORD] - input|api_token + [-e EMAIL] [-p PASSWORD] [-b BLOGNAME] + input|api_token|api_key Positional arguments -------------------- input The input file to read api_token [Posterous only] api_token can be obtained from http://posterous.com/api/ + api_key [Tumblr only] api_key can be obtained from http://www.tumblr.com/oauth/apps Optional arguments ------------------ @@ -59,6 +61,7 @@ Optional arguments --wpfile WordPress XML export (default: False) --dotclear Dotclear export (default: False) --posterous Posterous API (default: False) + --tumblr Tumblr API (default: False) --feed Feed to parse (default: False) -o OUTPUT, --output OUTPUT Output path (default: output) @@ -69,6 +72,7 @@ Optional arguments (default: False) --dir-page Put files recognised as pages in "pages/" sub- directory (wordpress import only) (default: False) + --filter-author Import only post from the specified author. --strip-raw Strip raw HTML code that can't be converted to markup such as flash embeds or iframes (wordpress import only) (default: False) @@ -80,6 +84,8 @@ Optional arguments Email used to authenticate Posterous API -p PASSWORD, --password=PASSWORD Password used to authenticate Posterous API + -b BLOGNAME, --blogname=BLOGNAME + Blog name used in Tumblr API Examples @@ -97,6 +103,9 @@ for Posterous:: $ pelican-import --posterous -o ~/output --email= --password= +For Tumblr:: + + $ pelican-import --tumblr -o ~/output --blogname= Tests ===== diff --git a/docs/index.rst b/docs/index.rst index eceb407f..eb8883ce 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,5 +1,14 @@ -Pelican -======= +Pelican |release| +================= + + +.. ifconfig:: release.endswith('.dev') + + .. warning:: + + This documentation is for the version of Pelican currently under development. + Were you looking for version |last_stable| documentation? + Pelican is a static site generator, written in Python_. @@ -12,7 +21,7 @@ Pelican is a static site generator, written in Python_. Features -------- -Pelican currently supports: +Pelican |version| currently supports: * Articles (e.g., blog posts) and pages (e.g., "About", "Projects", "Contact") * Comments, via an external service (Disqus). (Please note that while @@ -22,7 +31,6 @@ Pelican currently supports: * Publication of articles in multiple languages * Atom/RSS feeds * Code syntax highlighting -* PDF generation of the articles/pages (optional) * Import from WordPress, Dotclear, or RSS feeds * Integration with external tools: Twitter, Google Analytics, etc. (optional) diff --git a/docs/internals.rst b/docs/internals.rst index 704122ba..f69a9bb8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -24,7 +24,7 @@ The logic is separated into different classes and concepts: then passed to the generators. * **Readers** are used to read from various formats (AsciiDoc, HTML, Markdown and - reStructuredText for now, but the system is extensible). Given a file, they + reStructuredText for now, but the system is extensible). Given a file, they return metadata (author, tags, category, etc.) and content (HTML-formatted). * **Generators** generate the different outputs. For instance, Pelican comes with @@ -44,7 +44,7 @@ method that returns HTML content and some metadata. Take a look at the Markdown reader:: - class MarkdownReader(Reader): + class MarkdownReader(BaseReader): enabled = bool(Markdown) def read(self, source_path): diff --git a/docs/plugins.rst b/docs/plugins.rst index 064ba73d..29d67e24 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -71,18 +71,20 @@ finalized pelican object invoked after al - minifying js/css assets. - notify/ping search engines with an updated sitemap. generator_init generator invoked in the Generator.__init__ -article_generate_context article_generator, metadata -article_generate_preread article_generator invoked before a article is read in ArticlesGenerator.generate_context; +readers_init readers invoked in the Readers.__init__ +article_generator_context article_generator, metadata +article_generator_preread article_generator invoked before a article is read in ArticlesGenerator.generate_context; use if code needs to do something before every article is parsed article_generator_init article_generator invoked in the ArticlesGenerator.__init__ article_generator_finalized article_generator invoked at the end of ArticlesGenerator.generate_context get_generators generators invoked in Pelican.get_generator_classes, can return a Generator, or several generator in a tuple or in a list. -pages_generate_context pages_generator, metadata -pages_generator_init pages_generator invoked in the PagesGenerator.__init__ -pages_generator_finalized pages_generator invoked at the end of PagesGenerator.generate_context +page_generate_context page_generator, metadata +page_generator_init page_generator invoked in the PagesGenerator.__init__ +page_generator_finalized page_generator invoked at the end of PagesGenerator.generate_context content_object_init content_object invoked at the end of Content.__init__ (see note below) +content_written path, context invoked each time a content file is written. ============================= ============================ =========================================================================== The list is currently small, so don't hesitate to add signals and make a pull @@ -104,3 +106,88 @@ request if you need them! def register(): signals.content_object_init.connect(test, sender=contents.Article) + +.. note:: + + After Pelican 3.2, signal names were standardized. Older plugins + may need to be updated to use the new names: + + ========================== =========================== + Old name New name + ========================== =========================== + article_generate_context article_generator_context + article_generate_finalized article_generator_finalized + article_generate_preread article_generator_preread + pages_generate_context page_generator_context + pages_generate_preread page_generator_preread + pages_generator_finalized page_generator_finalized + pages_generator_init page_generator_init + static_generate_context static_generator_context + static_generate_preread static_generator_preread + ========================== =========================== + +Recipes +======= + +We eventually realised some of the recipes to create plugins would be best +shared in the documentation somewhere, so here they are! + +How to create a new reader +-------------------------- + +One thing you might want is to add the support for your very own input +format. While it might make sense to add this feature in pelican core, we +wisely chose to avoid this situation, and have the different readers defined in +plugins. + +The rationale behind this choice is mainly that plugins are really easy to +write and don't slow down pelican itself when they're not active. + +No more talking, here is the example:: + + from pelican import signals + from pelican.readers import BaseReader + + # Create a new reader class, inheriting from the pelican.reader.BaseReader + class NewReader(BaseReader): + enabled = True # Yeah, you probably want that :-) + + # The list of file extensions you want this reader to match with. + # In the case multiple readers use the same extensions, the latest will + # win (so the one you're defining here, most probably). + file_extensions = ['yeah'] + + # You need to have a read method, which takes a filename and returns + # some content and the associated metadata. + def read(self, filename): + metadata = {'title': 'Oh yeah', + 'category': 'Foo', + 'date': '2012-12-01'} + + parsed = {} + for key, value in metadata.items(): + parsed[key] = self.process_metadata(key, value) + + return "Some content", parsed + + def add_reader(readers): + readers.reader_classes['yeah'] = NewReader + + # this is how pelican works. + def register(): + signals.readers_init.connect(add_reader) + + +Adding a new generator +---------------------- + +Adding a new generator is also really easy. You might want to have a look at +:doc:`internals` for more information on how to create your own generator. + +:: + + def get_generators(generators): + # define a new generator here if you need to + return generators + + signals.get_generators.connect(get_generators) diff --git a/docs/settings.rst b/docs/settings.rst index 3a32ad22..e8965731 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -62,15 +62,19 @@ Setting name (default value) What doe For example, if you would like to extract both the date and the slug, you could set something like: ``'(?P\d{4}-\d{2}-\d{2})_(?P.*)'``. + See :ref:`path_metadata`. `PATH_METADATA` (``''``) Like ``FILENAME_METADATA``, but parsed from a page's full path relative to the content source directory. + See :ref:`path_metadata`. +`EXTRA_PATH_METADATA` (``{}``) Extra metadata dictionaries keyed by relative path. + See :ref:`path_metadata`. `DELETE_OUTPUT_DIRECTORY` (``False``) Delete the output directory, and **all** of its contents, before generating new files. This can be useful in preventing older, unnecessary files from persisting in your output. However, **this is a destructive setting and should be handled with extreme care.** -`FILES_TO_COPY` (``()``) A list of files (or directories) to copy from the source (inside the - content directory) to the destination (inside the output directory). - For example: ``(('extra/robots.txt', 'robots.txt'),)``. +`OUTPUT_RETENTION` (``()``) A tuple of filenames that should be retained and not deleted from the + output directory. One use case would be the preservation of version + control data. For example: ``(".hg", ".git", ".bzr")`` `JINJA_EXTENSIONS` (``[]``) A list of any Jinja2 extensions you want to use. `JINJA_FILTERS` (``{}``) A list of custom Jinja2 filters you want to use. The dictionary should map the filtername to the filter function. @@ -80,9 +84,10 @@ Setting name (default value) What doe here or a single string representing one locale. When providing a list, all the locales will be tried until one works. -`MARKUP` (``('rst', 'md')``) A list of available markup languages you want - to use. For the moment, the only available values - are `rst`, `md`, `markdown`, `mkd`, `mdown`, `html`, and `htm`. +`READERS` (``{}``) A dict of file extensions / Reader classes to overwrite or + add file readers. for instance, to avoid processing .html files: + ``READERS = {'html': None}``. Or to add a custom reader for the + `foo` extension: ``READERS = {'foo': FooReader}`` `IGNORE_FILES` (``['.#*']``) A list of file globbing patterns to match against the source files to be ignored by the processor. For example, the default ``['.#*']`` will ignore emacs lock files. @@ -134,8 +139,9 @@ Setting name (default value) What doe library, which can be installed via: ``pip install typogrify`` `DIRECT_TEMPLATES` (``('index', 'tags', 'categories', 'archives')``) List of templates that are used directly to render content. Typically direct templates are used to generate - index pages for collections of content (e.g. tags and - category index pages). + index pages for collections of content (e.g., tags and + category index pages). If the tag and category collections + are not needed, set ``DIRECT_TEMPLATES = ('index', 'archives')`` `PAGINATED_DIRECT_TEMPLATES` (``('index',)``) Provides the direct templates that should be paginated. `SUMMARY_MAX_LENGTH` (``50``) When creating a short summary of an article, this will be the default length in words of the text created. @@ -148,6 +154,16 @@ Setting name (default value) What doe These templates need to use ``DIRECT_TEMPLATES`` setting. `ASCIIDOC_OPTIONS` (``[]``) A list of options to pass to AsciiDoc. See the `manpage `_ +`WITH_FUTURE_DATES` (``True``) If disabled, content with dates in the future will get a + default status of draft. +`INTRASITE_LINK_REGEX` (``'[{|](?P.*?)[|}]'``) Regular expression that is used to parse internal links. + Default syntax of links to internal files, tags, etc. is + to enclose the identifier, say ``filename``, in ``{}`` or ``||``. + Identifier between ``{`` and ``}`` goes into the ``what`` capturing group. + For details see :ref:`ref-linking-to-internal-content`. +`PYGMENTS_RST_OPTIONS` (``[]``) A list of default Pygments settings for your reStructuredText + code blocks. See :ref:`internal_pygments_options` for a list of + supported options. ===================================================================== ===================================================================== .. [#] Default is the system locale. @@ -233,26 +249,38 @@ Setting name (default value) What does it do? use the default language. `PAGE_LANG_SAVE_AS` (``'pages/{slug}-{lang}.html'``) The location we will save the page which doesn't use the default language. -`AUTHOR_URL` (``'author/{slug}.html'``) The URL to use for an author. -`AUTHOR_SAVE_AS` (``'author/{slug}.html'``) The location to save an author. `CATEGORY_URL` (``'category/{slug}.html'``) The URL to use for a category. `CATEGORY_SAVE_AS` (``'category/{slug}.html'``) The location to save a category. `TAG_URL` (``'tag/{slug}.html'``) The URL to use for a tag. `TAG_SAVE_AS` (``'tag/{slug}.html'``) The location to save the tag page. +`TAGS_URL` (``'tags.html'``) The URL to use for the tag list. +`TAGS_SAVE_AS` (``'tags.html'``) The location to save the tag list. +`AUTHOR_URL` (``'author/{slug}.html'``) The URL to use for an author. +`AUTHOR_SAVE_AS` (``'author/{slug}.html'``) The location to save an author. +`AUTHORS_URL` (``'authors.html'``) The URL to use for the author list. +`AUTHORS_SAVE_AS` (``'authors.html'``) The location to save the author list. `_SAVE_AS` The location to save content generated from direct templates. Where is the upper case template name. +`ARCHIVES_SAVE_AS` (``'archives.html'``) The location to save the article archives page. `YEAR_ARCHIVE_SAVE_AS` (False) The location to save per-year archives of your posts. `MONTH_ARCHIVE_SAVE_AS` (False) The location to save per-month archives of your posts. `DAY_ARCHIVE_SAVE_AS` (False) The location to save per-day archives of your posts. +`SLUG_SUBSTITUTIONS` (``()``) Substitutions to make prior to stripping out + non-alphanumerics when generating slugs. Specified + as a list of 2-tuples of ``(from, to)`` which are + applied in order. ==================================================== ===================================================== .. note:: - When any of the `*_SAVE_AS` settings is set to False, files will not be created. + If you do not want one or more of the default pages to be created (e.g., + you are the only author on your site and thus do not need an Authors page), + set the corresponding ``*_SAVE_AS`` setting to ``False`` to prevent the + relevant page from being generated. Timezone -------- @@ -336,6 +364,52 @@ your resume, and a contact page — you could have:: 'src/resume.html': 'dest/resume.html', 'src/contact.html': 'dest/contact.html'} + +.. _path_metadata: + +Path metadata +============= + +Not all metadata needs to be `embedded in source file itself`__. For +example, blog posts are often named following a ``YYYY-MM-DD-SLUG.rst`` +pattern, or nested into ``YYYY/MM/DD-SLUG`` directories. To extract +metadata from the filename or path, set ``FILENAME_METADATA`` or +``PATH_METADATA`` to regular expressions that use Python's `group name +notation`_ ``(?P…)``. If you want to attach additional metadata +but don't want to encode it in the path, you can set +``EXTRA_PATH_METADATA``: + +.. parsed-literal:: + + EXTRA_PATH_METADATA = { + 'relative/path/to/file-1': { + 'key-1a': 'value-1a', + 'key-1b': 'value-1b', + }, + 'relative/path/to/file-2': { + 'key-2': 'value-2', + }, + } + +This can be a convenient way to shift the installed location of a +particular file: + +.. parsed-literal:: + + # Take advantage of the following defaults + # STATIC_SAVE_AS = '{path}' + # STATIC_URL = '{path}' + STATIC_PATHS = [ + 'extra/robots.txt', + ] + EXTRA_PATH_METADATA = { + 'extra/robots.txt': {'path': 'robots.txt'}, + } + +__ internal_metadata__ +.. _group name notation: + http://docs.python.org/3/library/re.html#regular-expression-syntax + Feed settings ============= @@ -414,8 +488,32 @@ Setting name (default value) What does it do? `DEFAULT_PAGINATION` (``False``) The maximum number of articles to include on a page, not including orphans. False to disable pagination. +`PAGINATION_PATTERNS` A set of patterns that are used to determine advanced + pagination output. ================================================ ===================================================== +Using Pagination Patterns +------------------------- + +The ``PAGINATION_PATTERNS`` setting can be used to configure where +subsequent pages are created. The setting is a sequence of three +element tuples, where each tuple consists of:: + + (minimum page, URL setting, SAVE_AS setting,) + +For example, if you wanted the first page to just be ``/``, and the +second (and subsequent) pages to be ``/page/2/``, you would set +``PAGINATION_PATTERNS`` as follows:: + + PAGINATION_PATTERNS = ( + (1, '{base_name}/', '{base_name}/index.html'), + (2, '{base_name}/page/{number}/', '{base_name}/page/{number}/index.html'), + ) + +This would cause the first page to be written to +``{base_name}/index.html``, and subsequent ones would be written into +``page/{number}`` directories. + Tag cloud ========= @@ -430,16 +528,36 @@ Setting name (default value) What does it do? `TAG_CLOUD_MAX_ITEMS` (``100``) Maximum number of tags in the cloud. ================================================ ===================================================== -The default theme does not support tag clouds, but it is pretty easy to add:: +The default theme does not include a tag cloud, but it is pretty easy to add:: -
    + -You should then also define a CSS style with the appropriate classes (tag-0 to tag-N, where -N matches `TAG_CLOUD_STEPS` -1). +You should then also define CSS styles with appropriate classes (tag-0 to tag-N, where +N matches `TAG_CLOUD_STEPS` -1), tag-0 being the most frequent, and define a ul.tagcloud +class with appropriate list-style to create the cloud, for example:: + + ul.tagcloud { + list-style: none; + padding: 0; + } + + ul.tagcloud li { + display: inline-block; + } + + li.tag-0 { + font-size: 150%; + } + + li.tag-1 { + font-size: 120%; + } + + ... Translations ============ @@ -482,9 +600,15 @@ Setting name (default value) What does it do? or absolute path to a theme folder, or the name of a default theme or a theme installed via ``pelican-themes`` (see below). +`THEME_STATIC_DIR` (``'theme'``) Destination directory in the output path where + Pelican will place the files collected from + `THEME_STATIC_PATHS`. Default is `theme`. `THEME_STATIC_PATHS` (``['static']``) Static theme paths you want to copy. Default value is `static`, but if your theme has - other static paths, you can put them here. + other static paths, you can put them here. If files + or directories with the same names are included in + the paths defined in this settings, they will be + progressively overwritten. `CSS_FILE` (``'main.css'``) Specify the CSS file you want to load. ================================================ ===================================================== diff --git a/docs/tips.rst b/docs/tips.rst index 64695db0..1864f0dd 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -26,7 +26,7 @@ For example, if the sources of your Pelican site are contained in a GitHub repository, and if you want to publish your Pelican site as Project Pages of this repository, you can then use the following:: - $ pelican content -o output pelicanconf.py + $ pelican content -o output -s pelicanconf.py $ ghp-import output $ git push origin gh-pages @@ -49,7 +49,7 @@ To publish a Pelican site as User Pages you need to *push* the content of the Again, you can take advantage of ``ghp-import``:: - $ pelican content -o output pelicanconf.py + $ pelican content -o output -s pelicanconf.py $ ghp-import output $ git push git@github.com:elemoine/elemoine.github.com.git gh-pages:master @@ -71,7 +71,7 @@ To automatically update your Pelican site on each commit you can create a post-commit hook. For example, you can add the following to ``.git/hooks/post-commit``:: - pelican pelican content -o output pelicanconf.py && ghp-import output && git push origin gh-pages + pelican pelican content -o output -s pelicanconf.py && ghp-import output && git push origin gh-pages Tip #2: @@ -83,3 +83,12 @@ that you will add ``CNAME`` file to your ``content``, dir and use the to the ``output`` dir. For example:: FILES_TO_COPY = (('extra/CNAME', 'CNAME'),) + +How to add Youtube or Vimeo Videos +================================== + +The easiest way is to paste embed code of the video from these sites in your +markup file. + +Alternatively, you can also use Pelican plugins like ``liquid_tags`` or ``pelican_youtube`` +or ``pelican_vimeo`` to embed videos in your blog. diff --git a/pelican/__init__.py b/pelican/__init__.py index 7f406c4f..cecab54b 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -9,21 +9,20 @@ import time import logging import argparse import locale +import collections from pelican import signals from pelican.generators import (ArticlesGenerator, PagesGenerator, - StaticGenerator, PdfGenerator, - SourceFileGenerator, TemplatePagesGenerator) + StaticGenerator, SourceFileGenerator, + TemplatePagesGenerator) from pelican.log import init +from pelican.readers import Readers from pelican.settings import read_settings from pelican.utils import clean_output_dir, folder_watcher, file_watcher from pelican.writers import Writer -__major__ = 3 -__minor__ = 2 -__micro__ = 0 -__version__ = "{0}.{1}.{2}".format(__major__, __minor__, __micro__) +__version__ = "3.2.3.dev" DEFAULT_CONFIG_NAME = 'pelicanconf.py' @@ -45,9 +44,9 @@ class Pelican(object): self.path = settings['PATH'] self.theme = settings['THEME'] self.output_path = settings['OUTPUT_PATH'] - self.markup = settings['MARKUP'] self.ignore_files = settings['IGNORE_FILES'] self.delete_outputdir = settings['DELETE_OUTPUT_DIRECTORY'] + self.output_retention = settings['OUTPUT_RETENTION'] self.init_path() self.init_plugins() @@ -157,24 +156,23 @@ class Pelican(object): context['localsiteurl'] = self.settings['SITEURL'] # share generators = [ cls( - context, - self.settings, - self.path, - self.theme, - self.output_path, - self.markup, + context=context, + settings=self.settings, + path=self.path, + theme=self.theme, + output_path=self.output_path, ) for cls in self.get_generator_classes() ] - for p in generators: - if hasattr(p, 'generate_context'): - p.generate_context() - # erase the directory if it is not the source and if that's # explicitely asked if (self.delete_outputdir and not os.path.realpath(self.path).startswith(self.output_path)): - clean_output_dir(self.output_path) + clean_output_dir(self.output_path, self.output_retention) + + for p in generators: + if hasattr(p, 'generate_context'): + p.generate_context() writer = self.get_writer() @@ -197,15 +195,13 @@ class Pelican(object): if self.settings['TEMPLATE_PAGES']: generators.append(TemplatePagesGenerator) - if self.settings['PDF_GENERATOR']: - generators.append(PdfGenerator) if self.settings['OUTPUT_SOURCES']: generators.append(SourceFileGenerator) for pair in signals.get_generators.send(self): (funct, value) = pair - if not isinstance(value, (tuple, list)): + if not isinstance(value, collections.Iterable): value = (value, ) for v in value: @@ -236,10 +232,6 @@ def parse_arguments(): help='Where to output the generated files. If not specified, a ' 'directory will be created, named "output" in the current path.') - parser.add_argument('-m', '--markup', dest='markup', - help='The list of markup language to use (rst or md). Please indicate ' - 'them separated by commas.') - parser.add_argument('-s', '--settings', dest='settings', help='The settings of the application, this is automatically set to ' '{0} if a file exists with this name.'.format(DEFAULT_CONFIG_NAME)) @@ -279,8 +271,6 @@ def get_config(args): if args.output: config['OUTPUT_PATH'] = \ os.path.abspath(os.path.expanduser(args.output)) - if args.markup: - config['MARKUP'] = [a.strip().lower() for a in args.markup.split(',')] if args.theme: abstheme = os.path.abspath(os.path.expanduser(args.theme)) config['THEME'] = abstheme if os.path.exists(abstheme) else args.theme @@ -296,8 +286,6 @@ def get_config(args): for key in config: if key in ('PATH', 'OUTPUT_PATH', 'THEME'): config[key] = config[key].decode(enc) - if key == "MARKUP": - config[key] = [a.decode(enc) for a in config[key]] return config @@ -315,16 +303,17 @@ def get_instance(args): module = __import__(module) cls = getattr(module, cls_name) - return cls(settings) + return cls(settings), settings def main(): args = parse_arguments() init(args.verbosity) - pelican = get_instance(args) + pelican, settings = get_instance(args) + readers = Readers(settings) watchers = {'content': folder_watcher(pelican.path, - pelican.markup, + readers.extensions, pelican.ignore_files), 'theme': folder_watcher(pelican.theme, [''], @@ -333,8 +322,8 @@ def main(): try: if args.autoreload: - print(' --- AutoReload Mode: Monitoring `content`, `theme` and `settings`' - ' for changes. ---') + print(' --- AutoReload Mode: Monitoring `content`, `theme` and' + ' `settings` for changes. ---') while True: try: @@ -346,7 +335,7 @@ def main(): modified = {k: next(v) for k, v in watchers.items()} if modified['settings']: - pelican = get_instance(args) + pelican, settings = get_instance(args) if any(modified.values()): print('\n-> Modified: {}. re-generating...'.format( @@ -388,7 +377,7 @@ def main(): # so convert the message to unicode with the correct encoding msg = str(e) if not six.PY3: - msg = msg.decode(locale.getpreferredencoding(False)) + msg = msg.decode(locale.getpreferredencoding()) logger.critical(msg) diff --git a/pelican/contents.py b/pelican/contents.py index 5f2e66b0..b453f61b 100644 --- a/pelican/contents.py +++ b/pelican/contents.py @@ -48,6 +48,8 @@ class Content(object): self.settings = settings self._content = content + if context is None: + context = {} self._context = context self.translations = [] @@ -84,7 +86,8 @@ class Content(object): # create the slug if not existing, from the title if not hasattr(self, 'slug') and hasattr(self, 'title'): - self.slug = slugify(self.title) + self.slug = slugify(self.title, + settings.get('SLUG_SUBSTITUTIONS', ())) self.source_path = source_path @@ -138,14 +141,21 @@ class Content(object): """Returns the URL, formatted with the proper values""" metadata = copy.copy(self.metadata) path = self.metadata.get('path', self.get_relative_source_path()) + default_category = self.settings['DEFAULT_CATEGORY'] + slug_substitutions = self.settings.get('SLUG_SUBSTITUTIONS', ()) metadata.update({ 'path': path_to_url(path), 'slug': getattr(self, 'slug', ''), 'lang': getattr(self, 'lang', 'en'), 'date': getattr(self, 'date', datetime.now()), - 'author': getattr(self, 'author', ''), - 'category': getattr(self, 'category', - self.settings['DEFAULT_CATEGORY']), + 'author': slugify( + getattr(self, 'author', ''), + slug_substitutions + ), + 'category': slugify( + getattr(self, 'category', default_category), + slug_substitutions + ) }) return metadata @@ -169,13 +179,18 @@ class Content(object): :param siteurl: siteurl which is locally generated by the writer in case of RELATIVE_URLS. """ - hrefs = re.compile(r""" + if not content: + return content + + instrasite_link_regex = self.settings['INTRASITE_LINK_REGEX'] + regex = r""" (?P<\s*[^\>]* # match tag with src and href attr (?:href|src)\s*=) (?P["\']) # require value to be quoted - (?P\|(?P.*?)\|(?P.*?)) # the url value - \2""", re.X) + (?P{0}(?P.*?)) # the url value + \2""".format(instrasite_link_regex) + hrefs = re.compile(regex, re.X) def replacer(m): what = m.group('what') @@ -203,6 +218,10 @@ class Content(object): else: logger.warning("Unable to find {fn}, skipping url" " replacement".format(fn=value)) + elif what == 'category': + origin = Category(value, self.settings).url + elif what == 'tag': + origin = Tag(value, self.settings).url return ''.join((m.group('markup'), m.group('quote'), origin, m.group('quote'))) @@ -220,7 +239,7 @@ class Content(object): @property def content(self): - return self.get_content(self._context['localsiteurl']) + return self.get_content(self._context.get('localsiteurl', '')) def _get_summary(self): """Returns the summary of an article. diff --git a/pelican/generators.py b/pelican/generators.py index 75b61df2..d695c7c8 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -5,7 +5,6 @@ import os import math import random import logging -import datetime import shutil from codecs import open @@ -14,18 +13,13 @@ from functools import partial from itertools import chain, groupby from operator import attrgetter, itemgetter -from jinja2 import ( - Environment, FileSystemLoader, PrefixLoader, ChoiceLoader, BaseLoader, - TemplateNotFound -) +from jinja2 import (Environment, FileSystemLoader, PrefixLoader, ChoiceLoader, + BaseLoader, TemplateNotFound) -from pelican.contents import ( - Article, Page, Category, Static, is_valid_content -) -from pelican.readers import read_file +from pelican.contents import Article, Page, Static, is_valid_content +from pelican.readers import Readers from pelican.utils import copy, process_translations, mkdir_p, DateFormatter from pelican import signals -import pelican.utils logger = logging.getLogger(__name__) @@ -34,19 +28,23 @@ logger = logging.getLogger(__name__) class Generator(object): """Baseclass generator""" - def __init__(self, *args, **kwargs): - for idx, item in enumerate(('context', 'settings', 'path', 'theme', - 'output_path', 'markup')): - setattr(self, item, args[idx]) + def __init__(self, context, settings, path, theme, output_path, **kwargs): + self.context = context + self.settings = settings + self.path = path + self.theme = theme + self.output_path = output_path for arg, value in kwargs.items(): setattr(self, arg, value) + self.readers = Readers(self.settings) + # templates cache self._templates = {} self._templates_path = [] self._templates_path.append(os.path.expanduser( - os.path.join(self.theme, 'templates'))) + os.path.join(self.theme, 'templates'))) self._templates_path += self.settings['EXTRA_TEMPLATES_PATHS'] theme_path = os.path.dirname(os.path.abspath(__file__)) @@ -55,6 +53,7 @@ class Generator(object): "themes", "simple", "templates")) self.env = Environment( trim_blocks=True, + lstrip_blocks=True, loader=ChoiceLoader([ FileSystemLoader(self._templates_path), simple_loader, # implicit inheritance @@ -83,9 +82,8 @@ class Generator(object): try: self._templates[name] = self.env.get_template(name + '.html') except TemplateNotFound: - raise Exception( - ('[templates] unable to load %s.html from %s' - % (name, self._templates_path))) + raise Exception('[templates] unable to load %s.html from %s' + % (name, self._templates_path)) return self._templates[name] def _include_path(self, path, extensions=None): @@ -96,7 +94,7 @@ class Generator(object): extensions are allowed) """ if extensions is None: - extensions = self.markup + extensions = tuple(self.readers.extensions) basename = os.path.basename(path) if extensions is False or basename.endswith(extensions): return True @@ -105,23 +103,25 @@ class Generator(object): def get_files(self, path, exclude=[], extensions=None): """Return a list of files to use, based on rules - :param path: the path to search the file on + :param path: the path to search (relative to self.path) :param exclude: the list of path to exclude :param extensions: the list of allowed extensions (if False, all extensions are allowed) """ files = [] + root = os.path.join(self.path, path) - if os.path.isdir(path): - for root, dirs, temp_files in os.walk(path, followlinks=True): + if os.path.isdir(root): + for dirpath, dirs, temp_files in os.walk(root, followlinks=True): for e in exclude: if e in dirs: dirs.remove(e) + reldir = os.path.relpath(dirpath, self.path) for f in temp_files: - fp = os.path.join(root, f) + fp = os.path.join(reldir, f) if self._include_path(fp, extensions): files.append(fp) - elif os.path.exists(path) and self._include_path(path, extensions): + elif os.path.exists(root) and self._include_path(path, extensions): files.append(path) # can't walk non-directories return files @@ -164,7 +164,8 @@ class TemplatePagesGenerator(Generator): try: template = self.env.get_template(source) rurls = self.settings['RELATIVE_URLS'] - writer.write_file(dest, template, self.context, rurls) + writer.write_file(dest, template, self.context, rurls, + override_output=True) finally: del self.env.loader.loaders[0] @@ -258,7 +259,8 @@ class ArticlesGenerator(Generator): """Generate the articles.""" for article in chain(self.translations, self.articles): write(article.save_as, self.get_template(article.template), - self.context, article=article, category=article.category) + self.context, article=article, category=article.category, + override_output=hasattr(article, 'override_save_as')) def generate_period_archives(self, write): """Generate per-year, per-month, and per-day archives.""" @@ -331,6 +333,7 @@ class ArticlesGenerator(Generator): """Generate category pages.""" category_template = self.get_template('category') for cat, articles in self.categories: + articles.sort(key=attrgetter('date'), reverse=True) dates = [article for article in self.dates if article in articles] write(cat.save_as, category_template, self.context, category=cat, articles=articles, dates=dates, @@ -341,6 +344,7 @@ class ArticlesGenerator(Generator): """Generate Author pages.""" author_template = self.get_template('author') for aut, articles in self.authors: + articles.sort(key=attrgetter('date'), reverse=True) dates = [article for article in self.dates if article in articles] write(aut.save_as, author_template, self.context, author=aut, articles=articles, dates=dates, @@ -375,45 +379,22 @@ class ArticlesGenerator(Generator): def generate_context(self): """Add the articles into the shared context""" - article_path = os.path.normpath( # we have to remove trailing slashes - os.path.join(self.path, self.settings['ARTICLE_DIR']) - ) all_articles = [] for f in self.get_files( - article_path, + self.settings['ARTICLE_DIR'], exclude=self.settings['ARTICLE_EXCLUDES']): try: - signals.article_generate_preread.send(self) - content, metadata = read_file(f, settings=self.settings) + article = self.readers.read_file( + base_path=self.path, path=f, content_class=Article, + context=self.context, + preread_signal=signals.article_generator_preread, + preread_sender=self, + context_signal=signals.article_generator_context, + context_sender=self) except Exception as e: - logger.warning('Could not process %s\n%s' % (f, str(e))) + logger.warning('Could not process {}\n{}'.format(f, e)) continue - # if no category is set, use the name of the path as a category - if 'category' not in metadata: - - if (self.settings['USE_FOLDER_AS_CATEGORY'] - and os.path.dirname(f) != article_path): - # if the article is in a subdirectory - category = os.path.basename(os.path.dirname(f)) - else: - # if the article is not in a subdirectory - category = self.settings['DEFAULT_CATEGORY'] - - if category != '': - metadata['category'] = Category(category, self.settings) - - if 'date' not in metadata and self.settings.get('DEFAULT_DATE'): - if self.settings['DEFAULT_DATE'] == 'fs': - metadata['date'] = datetime.datetime.fromtimestamp( - os.stat(f).st_ctime) - else: - metadata['date'] = datetime.datetime( - *self.settings['DEFAULT_DATE']) - - signals.article_generate_context.send(self, metadata=metadata) - article = Article(content, metadata, settings=self.settings, - source_path=f, context=self.context) if not is_valid_content(article, f): continue @@ -502,22 +483,26 @@ class PagesGenerator(Generator): self.hidden_pages = [] self.hidden_translations = [] super(PagesGenerator, self).__init__(*args, **kwargs) - signals.pages_generator_init.send(self) + signals.page_generator_init.send(self) def generate_context(self): all_pages = [] hidden_pages = [] for f in self.get_files( - os.path.join(self.path, self.settings['PAGE_DIR']), + self.settings['PAGE_DIR'], exclude=self.settings['PAGE_EXCLUDES']): try: - content, metadata = read_file(f, settings=self.settings) + page = self.readers.read_file( + base_path=self.path, path=f, content_class=Page, + context=self.context, + preread_signal=signals.page_generator_preread, + preread_sender=self, + context_signal=signals.page_generator_context, + context_sender=self) except Exception as e: - logger.warning('Could not process %s\n%s' % (f, str(e))) + logger.warning('Could not process {}\n{}'.format(f, e)) continue - signals.pages_generate_context.send(self, metadata=metadata) - page = Page(content, metadata, settings=self.settings, - source_path=f, context=self.context) + if not is_valid_content(page, f): continue @@ -539,14 +524,15 @@ class PagesGenerator(Generator): self._update_context(('pages', )) self.context['PAGES'] = self.pages - signals.pages_generator_finalized.send(self) + signals.page_generator_finalized.send(self) def generate_output(self, writer): for page in chain(self.translations, self.pages, self.hidden_translations, self.hidden_pages): writer.write_file(page.save_as, self.get_template(page.template), self.context, page=page, - relative_urls=self.settings['RELATIVE_URLS']) + relative_urls=self.settings['RELATIVE_URLS'], + override_output=hasattr(page, 'override_save_as')) class StaticGenerator(Generator): @@ -558,7 +544,7 @@ class StaticGenerator(Generator): """Copy all the paths from source to destination""" for path in paths: copy(path, source, os.path.join(output_path, destination), - final_path, overwrite=True) + final_path) def generate_context(self): self.staticfiles = [] @@ -566,39 +552,24 @@ class StaticGenerator(Generator): # walk static paths for static_path in self.settings['STATIC_PATHS']: for f in self.get_files( - os.path.join(self.path, static_path), extensions=False): - f_rel = os.path.relpath(f, self.path) - content, metadata = read_file( - f, fmt='static', settings=self.settings) - # TODO remove this hardcoded 'static' subdirectory - metadata['save_as'] = os.path.join('static', f_rel) - metadata['url'] = pelican.utils.path_to_url(metadata['save_as']) - sc = Static( - content=None, - metadata=metadata, - settings=self.settings, - source_path=f_rel) - self.staticfiles.append(sc) - self.add_source_path(sc) - # same thing for FILES_TO_COPY - for src, dest in self.settings['FILES_TO_COPY']: - content, metadata = read_file( - src, fmt='static', settings=self.settings) - metadata['save_as'] = dest - metadata['url'] = pelican.utils.path_to_url(metadata['save_as']) - sc = Static( - content=None, - metadata={'save_as': dest}, - settings=self.settings, - source_path=src) - self.staticfiles.append(sc) - self.add_source_path(sc) + static_path, extensions=False): + static = self.readers.read_file( + base_path=self.path, path=f, content_class=Static, + fmt='static', context=self.context, + preread_signal=signals.static_generator_preread, + preread_sender=self, + context_signal=signals.static_generator_context, + context_sender=self) + self.staticfiles.append(static) + self.add_source_path(static) + self._update_context(('staticfiles',)) def generate_output(self, writer): self._copy_paths(self.settings['THEME_STATIC_PATHS'], self.theme, - 'theme', self.output_path, os.curdir) + self.settings['THEME_STATIC_DIR'], self.output_path, + os.curdir) # copy all Static files - for sc in self.staticfiles: + for sc in self.context['staticfiles']: source_path = os.path.join(self.path, sc.source_path) save_as = os.path.join(self.output_path, sc.save_as) mkdir_p(os.path.dirname(save_as)) @@ -606,52 +577,6 @@ class StaticGenerator(Generator): logger.info('copying {} to {}'.format(sc.source_path, sc.save_as)) -class PdfGenerator(Generator): - """Generate PDFs on the output dir, for all articles and pages coming from - rst""" - def __init__(self, *args, **kwargs): - super(PdfGenerator, self).__init__(*args, **kwargs) - try: - from rst2pdf.createpdf import RstToPdf - pdf_style_path = os.path.join(self.settings['PDF_STYLE_PATH']) - pdf_style = self.settings['PDF_STYLE'] - self.pdfcreator = RstToPdf(breakside=0, - stylesheets=[pdf_style], - style_path=[pdf_style_path]) - except ImportError: - raise Exception("unable to find rst2pdf") - - def _create_pdf(self, obj, output_path): - if obj.source_path.endswith('.rst'): - filename = obj.slug + ".pdf" - output_pdf = os.path.join(output_path, filename) - # print('Generating pdf for', obj.source_path, 'in', output_pdf) - with open(obj.source_path) as f: - self.pdfcreator.createPdf(text=f.read(), output=output_pdf) - logger.info(' [ok] writing %s' % output_pdf) - - def generate_context(self): - pass - - def generate_output(self, writer=None): - # we don't use the writer passed as argument here - # since we write our own files - logger.info(' Generating PDF files...') - pdf_path = os.path.join(self.output_path, 'pdf') - if not os.path.exists(pdf_path): - try: - os.mkdir(pdf_path) - except OSError: - logger.error("Couldn't create the pdf output folder in " + - pdf_path) - - for article in self.context['articles']: - self._create_pdf(article, pdf_path) - - for page in self.context['pages']: - self._create_pdf(page, pdf_path) - - class SourceFileGenerator(Generator): def generate_context(self): self.output_extension = self.settings['OUTPUT_SOURCES_EXTENSION'] diff --git a/pelican/paginator.py b/pelican/paginator.py index 067215c2..df8606ec 100644 --- a/pelican/paginator.py +++ b/pelican/paginator.py @@ -1,15 +1,37 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals, print_function +import six # From django.core.paginator +from collections import namedtuple +import functools +import logging +import os + from math import ceil +logger = logging.getLogger(__name__) + + +PaginationRule = namedtuple( + 'PaginationRule', + 'min_page URL SAVE_AS', +) + class Paginator(object): - def __init__(self, object_list, per_page, orphans=0): + def __init__(self, name, object_list, settings): + self.name = name self.object_list = object_list - self.per_page = per_page - self.orphans = orphans + self.settings = settings + + if settings.get('DEFAULT_PAGINATION'): + self.per_page = settings.get('DEFAULT_PAGINATION') + self.orphans = settings.get('DEFAULT_ORPHANS') + else: + self.per_page = len(object_list) + self.orphans = 0 + self._num_pages = self._count = None def page(self, number): @@ -18,7 +40,8 @@ class Paginator(object): top = bottom + self.per_page if top + self.orphans >= self.count: top = self.count - return Page(self.object_list[bottom:top], number, self) + return Page(self.name, self.object_list[bottom:top], number, self, + self.settings) def _get_count(self): "Returns the total number of objects, across all pages." @@ -45,10 +68,12 @@ class Paginator(object): class Page(object): - def __init__(self, object_list, number, paginator): + def __init__(self, name, object_list, number, paginator, settings): + self.name = name self.object_list = object_list self.number = number self.paginator = paginator + self.settings = settings def __repr__(self): return '' % (self.number, self.paginator.num_pages) @@ -87,3 +112,48 @@ class Page(object): if self.number == self.paginator.num_pages: return self.paginator.count return self.number * self.paginator.per_page + + def _from_settings(self, key): + """Returns URL information as defined in settings. Similar to + URLWrapper._from_settings, but specialized to deal with pagination + logic.""" + + rule = None + + # find the last matching pagination rule + for p in self.settings['PAGINATION_PATTERNS']: + if p.min_page <= self.number: + rule = p + + if not rule: + return '' + + prop_value = getattr(rule, key) + + if not isinstance(prop_value, six.string_types): + logger.warning('%s is set to %s' % (key, prop_value)) + return prop_value + + # URL or SAVE_AS is a string, format it with a controlled context + context = { + 'name': self.name, + 'object_list': self.object_list, + 'number': self.number, + 'paginator': self.paginator, + 'settings': self.settings, + 'base_name': os.path.dirname(self.name), + 'number_sep': '/', + } + + if self.number == 1: + # no page numbers on the first page + context['number'] = '' + context['number_sep'] = '' + + ret = prop_value.format(**context) + if ret[0] == '/': + ret = ret[1:] + return ret + + url = property(functools.partial(_from_settings, key='URL')) + save_as = property(functools.partial(_from_settings, key='SAVE_AS')) diff --git a/pelican/readers.py b/pelican/readers.py index 816464ef..067bbb85 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals, print_function +import datetime +import logging import os import re + try: import docutils import docutils.core @@ -12,7 +15,7 @@ try: # import the directives to have pygments support from pelican import rstdirectives # NOQA except ImportError: - core = False + docutils = False try: from markdown import Markdown except ImportError: @@ -31,7 +34,8 @@ try: except ImportError: from HTMLParser import HTMLParser -from pelican.contents import Category, Tag, Author +from pelican import signals +from pelican.contents import Page, Category, Tag, Author from pelican.utils import get_date, pelican_open @@ -43,8 +47,22 @@ METADATA_PROCESSORS = { 'author': Author, } +logger = logging.getLogger(__name__) -class Reader(object): + +class BaseReader(object): + """Base class to read files. + + This class is used to process static files, and it can be inherited for + other types of file. A Reader class must have the following attributes: + + - enabled: (boolean) tell if the Reader class is enabled. It + generally depends on the import of some dependency. + - file_extensions: a list of file extensions that the Reader will process. + - extensions: a list of extensions to use in the reader (typical use is + Markdown). + + """ enabled = True file_extensions = ['static'] extensions = None @@ -97,8 +115,16 @@ class PelicanHTMLTranslator(HTMLTranslator): def depart_abbreviation(self, node): self.body.append('') + def visit_image(self, node): + # set an empty alt if alt is not specified + # avoids that alt is taken from src + node['alt'] = node.get('alt', '') + return HTMLTranslator.visit_image(self, node) + + +class RstReader(BaseReader): + """Reader for reStructuredText files""" -class RstReader(Reader): enabled = bool(docutils) file_extensions = ['rst'] @@ -154,15 +180,17 @@ class RstReader(Reader): return content, metadata -class MarkdownReader(Reader): +class MarkdownReader(BaseReader): + """Reader for Markdown files""" + enabled = bool(Markdown) file_extensions = ['md', 'markdown', 'mkd', 'mdown'] def __init__(self, *args, **kwargs): super(MarkdownReader, self).__init__(*args, **kwargs) - self.extensions = self.settings['MD_EXTENSIONS'] - self.extensions.append('meta') - self._md = Markdown(extensions=self.extensions) + self.extensions = list(self.settings['MD_EXTENSIONS']) + if 'meta' not in self.extensions: + self.extensions.append('meta') def _parse_metadata(self, meta): """Return the dict containing document metadata""" @@ -182,6 +210,7 @@ class MarkdownReader(Reader): def read(self, source_path): """Parse content and metadata of markdown files""" + self._md = Markdown(extensions=self.extensions) with pelican_open(source_path) as text: content = self._md.convert(text) @@ -189,13 +218,14 @@ class MarkdownReader(Reader): return content, metadata -class HTMLReader(Reader): +class HTMLReader(BaseReader): """Parses HTML files as input, looking for meta, title, and body tags""" + file_extensions = ['htm', 'html'] enabled = True class _HTMLParser(HTMLParser): - def __init__(self, settings): + def __init__(self, settings, filename): HTMLParser.__init__(self) self.body = '' self.metadata = {} @@ -203,6 +233,8 @@ class HTMLReader(Reader): self._data_buffer = '' + self._filename = filename + self._in_top_level = True self._in_head = False self._in_title = False @@ -271,7 +303,11 @@ class HTMLReader(Reader): def _handle_meta_tag(self, attrs): name = self._attr_value(attrs, 'name').lower() - contents = self._attr_value(attrs, 'contents', '') + contents = self._attr_value(attrs, 'content', '') + if not contents: + contents = self._attr_value(attrs, 'contents', '') + if contents: + logger.warning("Meta tag attribute 'contents' used in file %s, should be changed to 'content'", self._filename) if name == 'keywords': name = 'tags' @@ -284,7 +320,7 @@ class HTMLReader(Reader): def read(self, filename): """Parse content and metadata of HTML files""" with pelican_open(filename) as content: - parser = self._HTMLParser(self.settings) + parser = self._HTMLParser(self.settings, filename) parser.feed(content) parser.close() @@ -294,7 +330,9 @@ class HTMLReader(Reader): return parser.body, metadata -class AsciiDocReader(Reader): +class AsciiDocReader(BaseReader): + """Reader for AsciiDoc files""" + enabled = bool(asciidoc) file_extensions = ['asc'] default_options = ["--no-header-footer", "-a newline=\\n"] @@ -326,48 +364,173 @@ class AsciiDocReader(Reader): return content, metadata -EXTENSIONS = {} +class Readers(object): + """Interface for all readers. -for cls in [Reader] + Reader.__subclasses__(): - for ext in cls.file_extensions: - EXTENSIONS[ext] = cls + This class contains a mapping of file extensions / Reader classes, to know + which Reader class must be used to read a file (based on its extension). + This is customizable both with the 'READERS' setting, and with the + 'readers_init' signall for plugins. + + """ + + # used to warn about missing dependencies only once, at the first + # instanciation of a Readers object. + warn_missing_deps = True + + def __init__(self, settings=None): + self.settings = settings or {} + self.readers = {} + self.reader_classes = {} + + for cls in [BaseReader] + BaseReader.__subclasses__(): + if not cls.enabled: + if self.__class__.warn_missing_deps: + logger.debug('Missing dependencies for {}' + .format(', '.join(cls.file_extensions))) + continue + + for ext in cls.file_extensions: + self.reader_classes[ext] = cls + + self.__class__.warn_missing_deps = False + + if self.settings['READERS']: + self.reader_classes.update(self.settings['READERS']) + + signals.readers_init.send(self) + + for fmt, reader_class in self.reader_classes.items(): + if not reader_class: + continue + + self.readers[fmt] = reader_class(self.settings) + + @property + def extensions(self): + return self.readers.keys() + + def read_file(self, base_path, path, content_class=Page, fmt=None, + context=None, preread_signal=None, preread_sender=None, + context_signal=None, context_sender=None): + """Return a content object parsed with the given format.""" + + path = os.path.abspath(os.path.join(base_path, path)) + source_path = os.path.relpath(path, base_path) + logger.debug('read file {} -> {}'.format( + source_path, content_class.__name__)) + + if not fmt: + _, ext = os.path.splitext(os.path.basename(path)) + fmt = ext[1:] + + if fmt not in self.readers: + raise TypeError( + 'Pelican does not know how to parse {}'.format(path)) + + if preread_signal: + logger.debug('signal {}.send({})'.format( + preread_signal, preread_sender)) + preread_signal.send(preread_sender) + + reader = self.readers[fmt] + + metadata = default_metadata( + settings=self.settings, process=reader.process_metadata) + metadata.update(path_metadata( + full_path=path, source_path=source_path, + settings=self.settings)) + metadata.update(parse_path_metadata( + source_path=source_path, settings=self.settings, + process=reader.process_metadata)) + + content, reader_metadata = reader.read(path) + metadata.update(reader_metadata) + + if content: + # find images with empty alt + find_empty_alt(content, path) + + # eventually filter the content with typogrify if asked so + if content and self.settings['TYPOGRIFY']: + from typogrify.filters import typogrify + content = typogrify(content) + metadata['title'] = typogrify(metadata['title']) + + if context_signal: + logger.debug('signal {}.send({}, )'.format( + context_signal, context_sender)) + context_signal.send(context_sender, metadata=metadata) + + return content_class(content=content, metadata=metadata, + settings=self.settings, source_path=path, + context=context) -def read_file(path, fmt=None, settings=None): - """Return a reader object using the given format.""" - base, ext = os.path.splitext(os.path.basename(path)) - if not fmt: - fmt = ext[1:] +def find_empty_alt(content, path): + """Find images with empty alt - if fmt not in EXTENSIONS: - raise TypeError('Pelican does not know how to parse {}'.format(path)) + Create warnings for all images with empty alt (up to a certain number), + as they are really likely to be accessibility flaws. - if settings is None: - settings = {} + """ + imgs = re.compile(r""" + (?: + # src before alt + ]* + src=(['"])(.*)\1 + [^\>]* + alt=(['"])\3 + )|(?: + # alt before src + ]* + alt=(['"])\4 + [^\>]* + src=(['"])(.*)\5 + ) + """, re.X) + matches = re.findall(imgs, content) + # find a correct threshold + nb_warnings = 10 + if len(matches) == nb_warnings + 1: + nb_warnings += 1 # avoid bad looking case + # print one warning per image with empty alt until threshold + for match in matches[:nb_warnings]: + logger.warning('Empty alt attribute for image {} in {}'.format( + os.path.basename(match[1] + match[5]), path)) + # print one warning for the other images with empty alt + if len(matches) > nb_warnings: + logger.warning('{} other images with empty alt attributes' + .format(len(matches) - nb_warnings)) - reader = EXTENSIONS[fmt](settings) - settings_key = '%s_EXTENSIONS' % fmt.upper() - if settings and settings_key in settings: - reader.extensions = settings[settings_key] +def default_metadata(settings=None, process=None): + metadata = {} + if settings: + if 'DEFAULT_CATEGORY' in settings: + value = settings['DEFAULT_CATEGORY'] + if process: + value = process('category', value) + metadata['category'] = value + if 'DEFAULT_DATE' in settings and settings['DEFAULT_DATE'] != 'fs': + metadata['date'] = datetime.datetime(*settings['DEFAULT_DATE']) + return metadata - if not reader.enabled: - raise ValueError("Missing dependencies for %s" % fmt) - metadata = parse_path_metadata( - path=path, settings=settings, process=reader.process_metadata) - content, reader_metadata = reader.read(path) - metadata.update(reader_metadata) +def path_metadata(full_path, source_path, settings=None): + metadata = {} + if settings: + if settings.get('DEFAULT_DATE', None) == 'fs': + metadata['date'] = datetime.datetime.fromtimestamp( + os.stat(full_path).st_ctime) + metadata.update(settings.get('EXTRA_PATH_METADATA', {}).get( + source_path, {})) + return metadata - # eventually filter the content with typogrify if asked so - if content and settings and settings['TYPOGRIFY']: - from typogrify.filters import typogrify - content = typogrify(content) - metadata['title'] = typogrify(metadata['title']) - return content, metadata - -def parse_path_metadata(path, settings=None, process=None): +def parse_path_metadata(source_path, settings=None, process=None): """Extract a metadata dictionary from a file's path >>> import pprint @@ -376,9 +539,9 @@ def parse_path_metadata(path, settings=None, process=None): ... 'PATH_METADATA': ... '(?P[^/]*)/(?P\d{4}-\d{2}-\d{2})/.*', ... } - >>> reader = Reader(settings=settings) + >>> reader = BaseReader(settings=settings) >>> metadata = parse_path_metadata( - ... path='my-cat/2013-01-01/my-slug.html', + ... source_path='my-cat/2013-01-01/my-slug.html', ... settings=settings, ... process=reader.process_metadata) >>> pprint.pprint(metadata) # doctest: +ELLIPSIS @@ -387,13 +550,18 @@ def parse_path_metadata(path, settings=None, process=None): 'slug': 'my-slug'} """ metadata = {} - base, ext = os.path.splitext(os.path.basename(path)) + dirname, basename = os.path.split(source_path) + base, ext = os.path.splitext(basename) + subdir = os.path.basename(dirname) if settings: - for key,data in [('FILENAME_METADATA', base), - ('PATH_METADATA', path), - ]: - regexp = settings.get(key) - if regexp: + checks = [] + for key, data in [('FILENAME_METADATA', base), + ('PATH_METADATA', source_path)]: + checks.append((settings.get(key, None), data)) + if settings.get('USE_FOLDER_AS_CATEGORY', None): + checks.insert(0, ('(?P.*)', subdir)) + for regexp, data in checks: + if regexp and data: match = re.match(regexp, data) if match: # .items() for py3k compat. diff --git a/pelican/rstdirectives.py b/pelican/rstdirectives.py index fb4a6c93..1bf6971c 100644 --- a/pelican/rstdirectives.py +++ b/pelican/rstdirectives.py @@ -7,21 +7,32 @@ from pygments.formatters import HtmlFormatter from pygments import highlight from pygments.lexers import get_lexer_by_name, TextLexer import re - -INLINESTYLES = False -DEFAULT = HtmlFormatter(noclasses=INLINESTYLES) -VARIANTS = { - 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True), -} +import six +import pelican.settings as pys class Pygments(Directive): - """ Source code syntax hightlighting. + """ Source code syntax highlighting. """ required_arguments = 1 optional_arguments = 0 final_argument_whitespace = True - option_spec = dict([(key, directives.flag) for key in VARIANTS]) + option_spec = { + 'anchorlinenos': directives.flag, + 'classprefix': directives.unchanged, + 'hl_lines': directives.unchanged, + 'lineanchors': directives.unchanged, + 'linenos': directives.unchanged, + 'linenospecial': directives.nonnegative_int, + 'linenostart': directives.nonnegative_int, + 'linenostep': directives.nonnegative_int, + 'lineseparator': directives.unchanged, + 'linespans': directives.unchanged, + 'nobackground': directives.flag, + 'nowrap': directives.flag, + 'tagsfile': directives.unchanged, + 'tagurlformat': directives.unchanged, + } has_content = True def run(self): @@ -31,9 +42,27 @@ class Pygments(Directive): except ValueError: # no lexer found - use the text one instead of an exception lexer = TextLexer() - # take an arbitrary option if more than one is given - formatter = self.options and VARIANTS[self.options.keys()[0]] \ - or DEFAULT + + # Fetch the defaults + if pys.PYGMENTS_RST_OPTIONS is not None: + for k, v in six.iteritems(pys.PYGMENTS_RST_OPTIONS): + # Locally set options overrides the defaults + if k not in self.options: + self.options[k] = v + + if ('linenos' in self.options and + self.options['linenos'] not in ('table', 'inline')): + if self.options['linenos'] == 'none': + self.options.pop('linenos') + else: + self.options['linenos'] = 'table' + + for flag in ('nowrap', 'nobackground', 'anchorlinenos'): + if flag in self.options: + self.options[flag] = True + + # noclasses should already default to False, but just in case... + formatter = HtmlFormatter(noclasses=False, **self.options) parsed = highlight('\n'.join(self.content), lexer, formatter) return [nodes.raw('', parsed, format='html')] @@ -41,63 +70,6 @@ directives.register_directive('code-block', Pygments) directives.register_directive('sourcecode', Pygments) -class YouTube(Directive): - """ Embed YouTube video in posts. - - Courtesy of Brian Hsu: https://gist.github.com/1422773 - - VIDEO_ID is required, with / height are optional integer, - and align could be left / center / right. - - Usage: - .. youtube:: VIDEO_ID - :width: 640 - :height: 480 - :align: center - """ - - def align(argument): - """Conversion function for the "align" option.""" - return directives.choice(argument, ('left', 'center', 'right')) - - required_arguments = 1 - optional_arguments = 2 - option_spec = { - 'width': directives.positive_int, - 'height': directives.positive_int, - 'align': align - } - - final_argument_whitespace = False - has_content = False - - def run(self): - videoID = self.arguments[0].strip() - width = 420 - height = 315 - align = 'left' - - if 'width' in self.options: - width = self.options['width'] - - if 'height' in self.options: - height = self.options['height'] - - if 'align' in self.options: - align = self.options['align'] - - url = 'http://www.youtube.com/embed/%s' % videoID - div_block = '
    ' % align - embed_block = '' % (width, height, url) - - return [ - nodes.raw('', div_block, format='html'), - nodes.raw('', embed_block, format='html'), - nodes.raw('', '
    ', format='html')] - -directives.register_directive('youtube', YouTube) - _abbr_re = re.compile('\((.*)\)$') diff --git a/pelican/server.py b/pelican/server.py index fd99b209..0f7472c5 100644 --- a/pelican/server.py +++ b/pelican/server.py @@ -1,5 +1,7 @@ from __future__ import print_function +import os import sys +import logging try: import SimpleHTTPServer as srvmod except ImportError: @@ -10,20 +12,37 @@ try: except ImportError: import socketserver # NOQA -PORT = 8000 +PORT = len(sys.argv) == 2 and int(sys.argv[1]) or 8000 +SUFFIXES = ['','.html','/index.html'] -Handler = srvmod.SimpleHTTPRequestHandler +class ComplexHTTPRequestHandler(srvmod.SimpleHTTPRequestHandler): + def do_GET(self): + # we are trying to detect the file by having a fallback mechanism + r = None + for suffix in SUFFIXES: + if not hasattr(self,'original_path'): + self.original_path = self.path + self.path = self.original_path + suffix + path = self.translate_path(self.path) + if os.path.exists(path): + r = srvmod.SimpleHTTPRequestHandler.do_GET(self) + if r is not None: + break + logging.warning("Unable to find %s file." % self.path) + return r + +Handler = ComplexHTTPRequestHandler try: httpd = socketserver.TCPServer(("", PORT), Handler) except OSError as e: - print("Could not listen on port", PORT) + logging.error("Could not listen on port %s" % PORT) sys.exit(getattr(e, 'exitcode', 1)) -print("serving at port", PORT) +logging.info("serving at port %s" % PORT) try: httpd.serve_forever() except KeyboardInterrupt as e: - print("shutting down server") + logging.info("shutting down server") httpd.socket.close() \ No newline at end of file diff --git a/pelican/settings.py b/pelican/settings.py index 34a2b42a..99828935 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -33,8 +33,9 @@ DEFAULT_CONFIG = { 'PAGE_EXCLUDES': (), 'THEME': DEFAULT_THEME, 'OUTPUT_PATH': 'output', - 'MARKUP': ('rst', 'md'), + 'READERS': {}, 'STATIC_PATHS': ['images', ], + 'THEME_STATIC_DIR': 'theme', 'THEME_STATIC_PATHS': ['static', ], 'FEED_ALL_ATOM': os.path.join('feeds', 'all.atom.xml'), 'CATEGORY_FEED_ATOM': os.path.join('feeds', '%s.atom.xml'), @@ -44,7 +45,6 @@ DEFAULT_CONFIG = { 'SITENAME': 'A Pelican Blog', 'DISPLAY_PAGES_ON_MENU': True, 'DISPLAY_CATEGORIES_ON_MENU': True, - 'PDF_GENERATOR': False, 'OUTPUT_SOURCES': False, 'OUTPUT_SOURCES_EXTENSION': '.text', 'USE_FOLDER_AS_CATEGORY': True, @@ -54,6 +54,7 @@ DEFAULT_CONFIG = { 'NEWEST_FIRST_ARCHIVES': True, 'REVERSE_CATEGORY_ORDER': False, 'DELETE_OUTPUT_DIRECTORY': False, + 'OUTPUT_RETENTION': (), 'ARTICLE_URL': '{slug}.html', 'ARTICLE_SAVE_AS': '{slug}.html', 'ARTICLE_LANG_URL': '{slug}-{lang}.html', @@ -64,6 +65,7 @@ DEFAULT_CONFIG = { 'PAGE_LANG_SAVE_AS': os.path.join('pages', '{slug}-{lang}.html'), 'STATIC_URL': '{path}', 'STATIC_SAVE_AS': '{path}', + 'PDF_GENERATOR': False, 'PDF_STYLE_PATH': '', 'PDF_STYLE': 'twelvepoint', 'CATEGORY_URL': 'category/{slug}.html', @@ -72,6 +74,9 @@ DEFAULT_CONFIG = { 'TAG_SAVE_AS': os.path.join('tag', '{slug}.html'), 'AUTHOR_URL': 'author/{slug}.html', 'AUTHOR_SAVE_AS': os.path.join('author', '{slug}.html'), + 'PAGINATION_PATTERNS': [ + (0, '{name}{number}.html', '{name}{number}.html'), + ], 'YEAR_ARCHIVE_SAVE_AS': False, 'MONTH_ARCHIVE_SAVE_AS': False, 'DAY_ARCHIVE_SAVE_AS': False, @@ -79,7 +84,7 @@ DEFAULT_CONFIG = { 'DEFAULT_LANG': 'en', 'TAG_CLOUD_STEPS': 4, 'TAG_CLOUD_MAX_ITEMS': 100, - 'DIRECT_TEMPLATES': ('index', 'tags', 'categories', 'archives'), + 'DIRECT_TEMPLATES': ('index', 'tags', 'categories', 'authors', 'archives'), 'EXTRA_TEMPLATES_PATHS': [], 'PAGINATED_DIRECT_TEMPLATES': ('index', ), 'PELICAN_CLASS': 'pelican.Pelican', @@ -89,23 +94,29 @@ DEFAULT_CONFIG = { 'MD_EXTENSIONS': ['codehilite(css_class=highlight)', 'extra'], 'JINJA_EXTENSIONS': [], 'JINJA_FILTERS': {}, - 'LOCALE': [], # defaults to user locale + 'LOCALE': [''], # defaults to user locale 'DEFAULT_PAGINATION': False, 'DEFAULT_ORPHANS': 0, 'DEFAULT_METADATA': (), 'FILENAME_METADATA': '(?P\d{4}-\d{2}-\d{2}).*', 'PATH_METADATA': '', - 'FILES_TO_COPY': (), + 'EXTRA_PATH_METADATA': {}, 'DEFAULT_STATUS': 'published', 'ARTICLE_PERMALINK_STRUCTURE': '', 'TYPOGRIFY': False, 'SUMMARY_MAX_LENGTH': 50, 'PLUGIN_PATH': '', 'PLUGINS': [], + 'PYGMENTS_RST_OPTIONS': {}, 'TEMPLATE_PAGES': {}, 'IGNORE_FILES': ['.#*'], + 'SLUG_SUBSTITUTIONS': (), + 'INTRASITE_LINK_REGEX': '[{|](?P.*?)[|}]', } +PYGMENTS_RST_OPTIONS = None + + def read_settings(path=None, override=None): if path: local_settings = get_settings_from_file(path) @@ -114,7 +125,7 @@ def read_settings(path=None, override=None): if p in local_settings and local_settings[p] is not None \ and not isabs(local_settings[p]): absp = os.path.abspath(os.path.normpath(os.path.join( - os.path.dirname(path), local_settings[p]))) + os.path.dirname(path), local_settings[p]))) if p not in ('THEME', 'PLUGIN_PATH') or os.path.exists(absp): local_settings[p] = absp else: @@ -123,7 +134,14 @@ def read_settings(path=None, override=None): if override: local_settings.update(override) - return configure_settings(local_settings) + parsed_settings = configure_settings(local_settings) + # This is because there doesn't seem to be a way to pass extra + # parameters to docutils directive handlers, so we have to have a + # variable here that we'll import from within Pygments.run (see + # rstdirectives.py) to see what the user defaults were. + global PYGMENTS_RST_OPTIONS + PYGMENTS_RST_OPTIONS = parsed_settings.get('PYGMENTS_RST_OPTIONS', None) + return parsed_settings def get_settings_from_module(module=None, default_settings=DEFAULT_CONFIG): @@ -132,7 +150,7 @@ def get_settings_from_module(module=None, default_settings=DEFAULT_CONFIG): context = copy.deepcopy(default_settings) if module is not None: context.update( - (k, v) for k, v in inspect.getmembers(module) if k.isupper()) + (k, v) for k, v in inspect.getmembers(module) if k.isupper()) return context @@ -215,17 +233,18 @@ def configure_settings(settings): settings['FEED_DOMAIN'] = settings['SITEURL'] # Warn if feeds are generated with both SITEURL & FEED_DOMAIN undefined - feed_keys = ['FEED_ATOM', 'FEED_RSS', - 'FEED_ALL_ATOM', 'FEED_ALL_RSS', - 'CATEGORY_FEED_ATOM', 'CATEGORY_FEED_RSS', - 'TAG_FEED_ATOM', 'TAG_FEED_RSS', - 'TRANSLATION_FEED_ATOM', 'TRANSLATION_FEED_RSS', - ] + feed_keys = [ + 'FEED_ATOM', 'FEED_RSS', + 'FEED_ALL_ATOM', 'FEED_ALL_RSS', + 'CATEGORY_FEED_ATOM', 'CATEGORY_FEED_RSS', + 'TAG_FEED_ATOM', 'TAG_FEED_RSS', + 'TRANSLATION_FEED_ATOM', 'TRANSLATION_FEED_RSS', + ] if any(settings.get(k) for k in feed_keys): if not settings.get('SITEURL'): - logger.warning('Feeds generated without SITEURL set properly may not' - ' be valid') + logger.warning('Feeds generated without SITEURL set properly may' + ' not be valid') if not 'TIMEZONE' in settings: logger.warning( @@ -234,34 +253,50 @@ def configure_settings(settings): 'http://docs.getpelican.com/en/latest/settings.html#timezone ' 'for more information') + # fix up pagination rules + from pelican.paginator import PaginationRule + pagination_rules = [ + PaginationRule(*r) for r in settings.get( + 'PAGINATION_PATTERNS', + DEFAULT_CONFIG['PAGINATION_PATTERNS'], + ) + ] + settings['PAGINATION_PATTERNS'] = sorted( + pagination_rules, + key=lambda r: r[0], + ) + # Save people from accidentally setting a string rather than a list path_keys = ( - 'ARTICLE_EXCLUDES', - 'DEFAULT_METADATA', - 'DIRECT_TEMPLATES', - 'EXTRA_TEMPLATES_PATHS', - 'FILES_TO_COPY', - 'IGNORE_FILES', - 'JINJA_EXTENSIONS', - 'MARKUP', - 'PAGINATED_DIRECT_TEMPLATES', - 'PLUGINS', - 'STATIC_PATHS', - 'THEME_STATIC_PATHS',) + 'ARTICLE_EXCLUDES', + 'DEFAULT_METADATA', + 'DIRECT_TEMPLATES', + 'EXTRA_TEMPLATES_PATHS', + 'FILES_TO_COPY', + 'IGNORE_FILES', + 'JINJA_EXTENSIONS', + 'PAGINATED_DIRECT_TEMPLATES', + 'PLUGINS', + 'STATIC_PATHS', + 'THEME_STATIC_PATHS', + ) for PATH_KEY in filter(lambda k: k in settings, path_keys): if isinstance(settings[PATH_KEY], six.string_types): - logger.warning("Detected misconfiguration with %s setting (must " - "be a list), falling back to the default" - % PATH_KEY) + logger.warning("Detected misconfiguration with %s setting " + "(must be a list), falling back to the default" + % PATH_KEY) settings[PATH_KEY] = DEFAULT_CONFIG[PATH_KEY] - for old,new,doc in [ + for old, new, doc in [ ('LESS_GENERATOR', 'the Webassets plugin', None), + ('FILES_TO_COPY', 'STATIC_PATHS and EXTRA_PATH_METADATA', + 'https://github.com/getpelican/pelican/blob/master/docs/settings.rst#path-metadata'), ]: if old in settings: - message = 'The {} setting has been removed in favor of {}' + message = 'The {} setting has been removed in favor of {}'.format( + old, new) if doc: - message += ', see {} for details' + message += ', see {} for details'.format(doc) logger.warning(message) return settings diff --git a/pelican/signals.py b/pelican/signals.py index 92bc6249..e92272c9 100644 --- a/pelican/signals.py +++ b/pelican/signals.py @@ -2,15 +2,41 @@ from __future__ import unicode_literals, print_function from blinker import signal +# Run-level signals: + initialized = signal('pelican_initialized') -finalized = signal('pelican_finalized') -article_generate_preread = signal('article_generate_preread') -generator_init = signal('generator_init') -article_generate_context = signal('article_generate_context') -article_generator_init = signal('article_generator_init') -article_generator_finalized = signal('article_generate_finalized') get_generators = signal('get_generators') -pages_generate_context = signal('pages_generate_context') -pages_generator_init = signal('pages_generator_init') -pages_generator_finalized = signal('pages_generator_finalized') +finalized = signal('pelican_finalized') + +# Reader-level signals + +readers_init = signal('readers_init') + +# Generator-level signals + +generator_init = signal('generator_init') + +article_generator_init = signal('article_generator_init') +article_generator_finalized = signal('article_generator_finalized') + +page_generator_init = signal('page_generator_init') +page_generator_finalized = signal('page_generator_finalized') + +static_generator_init = signal('static_generator_init') +static_generator_finalized = signal('static_generator_finalized') + +# Page-level signals + +article_generator_preread = signal('article_generator_preread') +article_generator_context = signal('article_generator_context') + +page_generator_preread = signal('page_generator_preread') +page_generator_context = signal('page_generator_context') + +static_generator_preread = signal('static_generator_preread') +static_generator_context = signal('static_generator_context') + content_object_init = signal('content_object_init') + +# Writers signals +content_written = signal('content_written') diff --git a/pelican/tests/content/article_with_keywords.html b/pelican/tests/content/article_with_keywords.html index c869f514..0744c754 100644 --- a/pelican/tests/content/article_with_keywords.html +++ b/pelican/tests/content/article_with_keywords.html @@ -1,6 +1,6 @@ This is a super article ! - + diff --git a/pelican/tests/content/article_with_metadata.html b/pelican/tests/content/article_with_metadata.html index b108ac8a..b501ea29 100644 --- a/pelican/tests/content/article_with_metadata.html +++ b/pelican/tests/content/article_with_metadata.html @@ -1,12 +1,12 @@ This is a super article ! - - - - - - + + + + + + Multi-line metadata should be supported diff --git a/pelican/tests/content/article_with_metadata.unknownextension b/pelican/tests/content/article_with_metadata.unknownextension new file mode 100644 index 00000000..d4bac1c0 --- /dev/null +++ b/pelican/tests/content/article_with_metadata.unknownextension @@ -0,0 +1,12 @@ + +This is a super article ! +######################### + +:tags: foo, bar, foobar +:date: 2010-12-02 10:14 +:category: yeah +:author: Alexis Métaireau +:summary: + Multi-line metadata should be supported + as well as **inline markup**. +:custom_field: http://notmyidea.org diff --git a/pelican/tests/content/article_with_metadata_and_contents.html b/pelican/tests/content/article_with_metadata_and_contents.html new file mode 100644 index 00000000..b108ac8a --- /dev/null +++ b/pelican/tests/content/article_with_metadata_and_contents.html @@ -0,0 +1,15 @@ + + + This is a super article ! + + + + + + + + + Multi-line metadata should be supported + as well as inline markup. + + diff --git a/pelican/tests/content/article_with_uppercase_metadata.html b/pelican/tests/content/article_with_uppercase_metadata.html index 4fe5a9ee..b4cedf39 100644 --- a/pelican/tests/content/article_with_uppercase_metadata.html +++ b/pelican/tests/content/article_with_uppercase_metadata.html @@ -1,6 +1,6 @@ This is a super article ! - + diff --git a/pelican/tests/default_conf.py b/pelican/tests/default_conf.py index bc3a7dff..62594894 100644 --- a/pelican/tests/default_conf.py +++ b/pelican/tests/default_conf.py @@ -9,7 +9,6 @@ GITHUB_URL = 'http://github.com/ametaireau/' DISQUS_SITENAME = "blog-notmyidea" PDF_GENERATOR = False REVERSE_CATEGORY_ORDER = True -LOCALE = "" DEFAULT_PAGINATION = 2 FEED_RSS = 'feeds/all.rss.xml' @@ -29,11 +28,16 @@ SOCIAL = (('twitter', 'http://twitter.com/ametaireau'), # global metadata to all the contents DEFAULT_METADATA = (('yeah', 'it is'),) -# static paths will be copied under the same name -STATIC_PATHS = ["pictures", ] +# path-specific metadata +EXTRA_PATH_METADATA = { + 'extra/robots.txt': {'path': 'robots.txt'}, + } -# A list of files to copy from the source to the destination -FILES_TO_COPY = (('extra/robots.txt', 'robots.txt'),) +# static paths will be copied without parsing their contents +STATIC_PATHS = [ + 'pictures', + 'extra/robots.txt', + ] # foobar will not be used, because it's not in caps. All configuration keys # have to be in caps diff --git a/pelican/tests/output/basic/a-markdown-powered-article.html b/pelican/tests/output/basic/a-markdown-powered-article.html index 94b6e4ca..6bc29eba 100644 --- a/pelican/tests/output/basic/a-markdown-powered-article.html +++ b/pelican/tests/output/basic/a-markdown-powered-article.html @@ -1,11 +1,11 @@ - + A markdown powered article - - - + + + @@ -15,46 +15,47 @@
-
+
-
+