Make StaticGenerator skip content sources. Refs #1019.

This change partially addresses issue #1019, by teaching Pelican to distinguish
between static files and content source files. A user can now safely add the
same directory to both STATIC_PATHS and PAGE_PATHS (or ARTICLE_PATHS). Pelican
will then process the content source files in that directory normally, and
treat the remaining files as static, without copying the raw content source
files to the output directory. (The OUTPUT_SOURCES setting still works.)

In other words, images and markdown/reST files can now safely live together.

To keep those files together in the generated site, STATIC_SAVE_AS and
PAGE_SAVE_AS (or ARTICLE_SAVE_AS) should point to the same output directory.

There are two new configuration settings:

STATIC_EXCLUDES=[]  # This works just like PAGE_EXCLUDES and ARTICLE_EXCLUDES.
STATIC_EXCLUDE_SOURCES=True  # Set this to False to get the old behavior.

Two small but noteworthy internal changes:

StaticGenerator now runs after all the other generators. This allows it to see
which files are meant to be processed by other generators, and avoid them.

Generators now include files that they fail to process (e.g. those with missing
mandatory metadata) along with all the other paths in context['filenames'].
This allows such files to be excluded from StaticGenerator's file list, so they
won't end up accidentally published. Since these files have no Content object,
their value in context['filenames'] is None. The code that uses that dict has
been updated accordingly.

pelican/tests/mixed_content/short_page.md

@@ -0,0 +1,3 @@
+Title: Short Page
+
+This is a page with little text.

pelican/tests/test_generators.py

@@ -12,7 +12,7 @@ from shutil import rmtree
 from tempfile import mkdtemp
 
 from pelican.generators import (Generator, ArticlesGenerator, PagesGenerator,
-                                TemplatePagesGenerator)
+                                StaticGenerator, TemplatePagesGenerator)
 from pelican.writers import Writer
 from pelican.tests.support import unittest, get_settings
 import locale
@@ -558,3 +558,69 @@ class TestTemplatePagesGenerator(unittest.TestCase):
         # output content is correct
         with open(output_path, 'r') as output_file:
             self.assertEqual(output_file.read(), 'foo: bar')
+
+
+class TestStaticGenerator(unittest.TestCase):
+
+    def setUp(self):
+        self.content_path = os.path.join(CUR_DIR, 'mixed_content')
+
+    def test_static_excludes(self):
+        """Test that StaticGenerator respects STATIC_EXCLUDES.
+        """
+        settings = get_settings(STATIC_EXCLUDES=['subdir'],
+            PATH=self.content_path, STATIC_PATHS=[''])
+        context = settings.copy()
+        context['filenames'] = {}
+
+        StaticGenerator(context=context, settings=settings,
+            path=settings['PATH'], output_path=None,
+            theme=settings['THEME']).generate_context()
+
+        staticnames = [os.path.basename(c.source_path)
+            for c in context['staticfiles']]
+
+        self.assertNotIn('subdir_fake_image.jpg', staticnames,
+            "StaticGenerator processed a file in a STATIC_EXCLUDES directory")
+        self.assertIn('fake_image.jpg', staticnames,
+            "StaticGenerator skipped a file that it should have included")
+
+    def test_static_exclude_sources(self):
+        """Test that StaticGenerator respects STATIC_EXCLUDE_SOURCES.
+        """
+        # Test STATIC_EXCLUDE_SOURCES=True
+
+        settings = get_settings(STATIC_EXCLUDE_SOURCES=True,
+            PATH=self.content_path, PAGE_PATHS=[''], STATIC_PATHS=[''],
+            CACHE_CONTENT=False)
+        context = settings.copy()
+        context['filenames'] = {}
+
+        for generator_class in (PagesGenerator, StaticGenerator):
+            generator_class(context=context, settings=settings,
+                path=settings['PATH'], output_path=None,
+                theme=settings['THEME']).generate_context()
+
+        staticnames = [os.path.basename(c.source_path)
+            for c in context['staticfiles']]
+
+        self.assertFalse(any(name.endswith(".md") for name in staticnames),
+            "STATIC_EXCLUDE_SOURCES=True failed to exclude a markdown file")
+
+        # Test STATIC_EXCLUDE_SOURCES=False
+
+        settings.update(STATIC_EXCLUDE_SOURCES=False)
+        context = settings.copy()
+        context['filenames'] = {}
+
+        for generator_class in (PagesGenerator, StaticGenerator):
+            generator_class(context=context, settings=settings,
+                path=settings['PATH'], output_path=None,
+                theme=settings['THEME']).generate_context()
+
+        staticnames = [os.path.basename(c.source_path)
+            for c in context['staticfiles']]
+
+        self.assertTrue(any(name.endswith(".md") for name in staticnames),
+            "STATIC_EXCLUDE_SOURCES=False failed to include a markdown file")
+

pelican/tests/test_pelican.py

@@ -10,6 +10,7 @@ import logging
 import subprocess
 
 from pelican import Pelican
+from pelican.generators import StaticGenerator
 from pelican.settings import read_settings
 from pelican.tests.support import LoggedTestCase, mute, locale_available, unittest
 
@@ -75,6 +76,16 @@ class TestPelican(LoggedTestCase):
         assert not out, out
         assert not err, err
 
+    def test_order_of_generators(self):
+        # StaticGenerator must run last, so it can find files that were
+        # skipped by the other generators.
+
+        pelican = Pelican(settings=read_settings(path=None))
+        generator_classes = pelican.get_generator_classes()
+
+        self.assertTrue(generator_classes[-1] is StaticGenerator,
+            "StaticGenerator must be the last generator, but it isn't!")
+
     def test_basic_generation_works(self):
         # when running pelican without settings, it should pick up the default
         # ones and generate correct output without raising any exception