Merge pull request #1982 from adeverteuil/feature_check_static_modified

Add static file options: hard/symlink & only-when-modified

docs/settings.rst

@@ -230,6 +230,20 @@ Basic settings
    ``PAGE_PATHS``. If you are trying to publish your site's source files,
    consider using the ``OUTPUT_SOURCES`` setting instead.
 
+.. data:: STATIC_CREATE_LINKS = False
+
+   Create links instead of copying files. If the content and output
+   directories are on the same device, then create hard links.  Falls
+   back to symbolic links if the output directory is on a different
+   filesystem. If symlinks are created, don't forget to add the ``-L``
+   or ``--copy-links`` option to rsync when uploading your site.
+
+.. data:: STATIC_CHECK_IF_MODIFIED = False
+
+   If set to ``True``, and ``STATIC_CREATE_LINKS`` is ``False``, compare
+   mtimes of content and output files, and only copy content files that
+   are newer than existing output files.
+
 .. data:: TYPOGRIFY = False
 
    If set to True, several typographical improvements will be incorporated into

pelican/generators.py

@@ -2,6 +2,7 @@
 from __future__ import print_function, unicode_literals
 
 import calendar
+import errno
 import fnmatch
 import logging
 import os
@@ -20,9 +21,8 @@ from pelican import signals
 from pelican.cache import FileStampDataCacher
 from pelican.contents import Article, Draft, Page, Static, is_valid_content
 from pelican.readers import Readers
-from pelican.utils import (DateFormatter, copy, copy_file_metadata, mkdir_p,
+from pelican.utils import (DateFormatter, copy, mkdir_p, posixize_path,
-                           posixize_path, process_translations,
+                           process_translations, python_2_unicode_compatible)
-                           python_2_unicode_compatible)
 
 
 logger = logging.getLogger(__name__)
@@ -682,21 +682,9 @@ class StaticGenerator(Generator):
 
     def __init__(self, *args, **kwargs):
         super(StaticGenerator, self).__init__(*args, **kwargs)
+        self.fallback_to_symlinks = False
         signals.static_generator_init.send(self)
 
-    def _copy_paths(self, paths, source, destination, output_path,
-                    final_path=None):
-        """Copy all the paths from source to destination"""
-        for path in paths:
-            if final_path:
-                copy(os.path.join(source, path),
-                     os.path.join(output_path, destination, final_path),
-                     self.settings['IGNORE_FILES'])
-            else:
-                copy(os.path.join(source, path),
-                     os.path.join(output_path, destination, path),
-                     self.settings['IGNORE_FILES'])
-
     def generate_context(self):
         self.staticfiles = []
         for f in self.get_files(self.settings['STATIC_PATHS'],
@@ -724,13 +712,88 @@ class StaticGenerator(Generator):
         self._copy_paths(self.settings['THEME_STATIC_PATHS'], self.theme,
                          self.settings['THEME_STATIC_DIR'], self.output_path,
                          os.curdir)
-        # copy all Static files
         for sc in self.context['staticfiles']:
+            if self._file_update_required(sc):
+                self._link_or_copy_staticfile(sc)
+            else:
+                logger.debug('%s is up to date, not copying', sc.source_path)
+
+    def _copy_paths(self, paths, source, destination, output_path,
+                    final_path=None):
+        """Copy all the paths from source to destination"""
+        for path in paths:
+            if final_path:
+                copy(os.path.join(source, path),
+                     os.path.join(output_path, destination, final_path),
+                     self.settings['IGNORE_FILES'])
+            else:
+                copy(os.path.join(source, path),
+                     os.path.join(output_path, destination, path),
+                     self.settings['IGNORE_FILES'])
+
+    def _file_update_required(self, staticfile):
+        source_path = os.path.join(self.path, staticfile.source_path)
+        save_as = os.path.join(self.output_path, staticfile.save_as)
+        if not os.path.exists(save_as):
+            return True
+        elif (self.settings['STATIC_CREATE_LINKS'] and
+              os.path.samefile(source_path, save_as)):
+            return False
+        elif (self.settings['STATIC_CREATE_LINKS'] and
+              os.path.realpath(save_as) == source_path):
+            return False
+        elif not self.settings['STATIC_CHECK_IF_MODIFIED']:
+            return True
+        else:
+            return self._source_is_newer(staticfile)
+
+    def _source_is_newer(self, staticfile):
+        source_path = os.path.join(self.path, staticfile.source_path)
+        save_as = os.path.join(self.output_path, staticfile.save_as)
+        s_mtime = os.path.getmtime(source_path)
+        d_mtime = os.path.getmtime(save_as)
+        return s_mtime > d_mtime
+
+    def _link_or_copy_staticfile(self, sc):
+        if self.settings['STATIC_CREATE_LINKS']:
+            self._link_staticfile(sc)
+        else:
+            self._copy_staticfile(sc)
+
+    def _copy_staticfile(self, sc):
         source_path = os.path.join(self.path, sc.source_path)
         save_as = os.path.join(self.output_path, sc.save_as)
-            mkdir_p(os.path.dirname(save_as))
+        self._mkdir(os.path.dirname(save_as))
+        copy(source_path, save_as)
         logger.info('Copying %s to %s', sc.source_path, sc.save_as)
-            copy_file_metadata(source_path, save_as)
+
+    def _link_staticfile(self, sc):
+        source_path = os.path.join(self.path, sc.source_path)
+        save_as = os.path.join(self.output_path, sc.save_as)
+        self._mkdir(os.path.dirname(save_as))
+        try:
+            if os.path.lexists(save_as):
+                os.unlink(save_as)
+            logger.info('Linking %s and %s', sc.source_path, sc.save_as)
+            if self.fallback_to_symlinks:
+                os.symlink(source_path, save_as)
+            else:
+                os.link(source_path, save_as)
+        except OSError as err:
+            if err.errno == errno.EXDEV:  # 18: Invalid cross-device link
+                logger.debug(
+                    "Cross-device links not valid. "
+                    "Creating symbolic links instead."
+                    )
+                self.fallback_to_symlinks = True
+                self._link_staticfile(sc)
+            else:
+                raise err
+
+    def _mkdir(self, path):
+        if os.path.lexists(path) and not os.path.isdir(path):
+            os.unlink(path)
+        mkdir_p(path)
 
 
 class SourceFileGenerator(Generator):

pelican/settings.py

@@ -82,6 +82,8 @@ DEFAULT_CONFIG = {
     'PAGE_LANG_SAVE_AS': posix_join('pages', '{slug}-{lang}.html'),
     'STATIC_URL': '{path}',
     'STATIC_SAVE_AS': '{path}',
+    'STATIC_CREATE_LINKS': False,
+    'STATIC_CHECK_IF_MODIFIED': False,
     'CATEGORY_URL': 'category/{slug}.html',
     'CATEGORY_SAVE_AS': posix_join('category', '{slug}.html'),
     'TAG_URL': 'tag/{slug}.html',

pelican/tests/test_generators.py

@@ -5,7 +5,7 @@ import locale
 import os
 
 from codecs import open
-from shutil import rmtree
+from shutil import copy, rmtree
 from tempfile import mkdtemp
 
 from pelican.generators import (ArticlesGenerator, Generator, PagesGenerator,
@@ -674,6 +674,30 @@ class TestStaticGenerator(unittest.TestCase):
 
     def setUp(self):
         self.content_path = os.path.join(CUR_DIR, 'mixed_content')
+        self.temp_content = mkdtemp(prefix='testcontent.')
+        self.temp_output = mkdtemp(prefix='testoutput.')
+        self.settings = get_settings()
+        self.settings['PATH'] = self.temp_content
+        self.settings['STATIC_PATHS'] = ["static"]
+        self.settings['OUTPUT_PATH'] = self.temp_output
+        os.mkdir(os.path.join(self.temp_content, "static"))
+        self.startfile = os.path.join(self.temp_content,
+                                      "static", "staticfile")
+        self.endfile = os.path.join(self.temp_output, "static", "staticfile")
+        self.generator = StaticGenerator(
+            context={'filenames': {}},
+            settings=self.settings,
+            path=self.temp_content,
+            theme="",
+            output_path=self.temp_output,
+            )
+
+    def tearDown(self):
+        rmtree(self.temp_content)
+        rmtree(self.temp_output)
+
+    def set_ancient_mtime(self, path, timestamp=1):
+        os.utime(path, (timestamp, timestamp))
 
     def test_static_excludes(self):
         """Test that StaticGenerator respects STATIC_EXCLUDES.
@@ -687,7 +711,7 @@ class TestStaticGenerator(unittest.TestCase):
 
         StaticGenerator(
             context=context, settings=settings,
-            path=settings['PATH'], output_path=None,
+            path=settings['PATH'], output_path=self.temp_output,
             theme=settings['THEME']).generate_context()
 
         staticnames = [os.path.basename(c.source_path)
@@ -716,7 +740,7 @@ class TestStaticGenerator(unittest.TestCase):
         for generator_class in (PagesGenerator, StaticGenerator):
             generator_class(
                 context=context, settings=settings,
-                path=settings['PATH'], output_path=None,
+                path=settings['PATH'], output_path=self.temp_output,
                 theme=settings['THEME']).generate_context()
 
         staticnames = [os.path.basename(c.source_path)
@@ -733,7 +757,7 @@ class TestStaticGenerator(unittest.TestCase):
         for generator_class in (PagesGenerator, StaticGenerator):
             generator_class(
                 context=context, settings=settings,
-                path=settings['PATH'], output_path=None,
+                path=settings['PATH'], output_path=self.temp_output,
                 theme=settings['THEME']).generate_context()
 
         staticnames = [os.path.basename(c.source_path)
@@ -742,3 +766,135 @@ class TestStaticGenerator(unittest.TestCase):
         self.assertTrue(
             any(name.endswith(".md") for name in staticnames),
             "STATIC_EXCLUDE_SOURCES=False failed to include a markdown file")
+
+    def test_copy_one_file(self):
+        with open(self.startfile, "w") as f:
+            f.write("staticcontent")
+        self.generator.generate_context()
+        self.generator.generate_output(None)
+        with open(self.endfile, "r") as f:
+            self.assertEqual(f.read(), "staticcontent")
+
+    @unittest.skipUnless(MagicMock, 'Needs Mock module')
+    def test_file_update_required_when_dest_does_not_exist(self):
+        staticfile = MagicMock()
+        staticfile.source_path = self.startfile
+        staticfile.save_as = self.endfile
+        with open(staticfile.source_path, "w") as f:
+            f.write("a")
+        update_required = self.generator._file_update_required(staticfile)
+        self.assertTrue(update_required)
+
+    @unittest.skipUnless(MagicMock, 'Needs Mock module')
+    def test_dest_and_source_mtimes_are_equal(self):
+        staticfile = MagicMock()
+        staticfile.source_path = self.startfile
+        staticfile.save_as = self.endfile
+        self.settings['STATIC_CHECK_IF_MODIFIED'] = True
+        with open(staticfile.source_path, "w") as f:
+            f.write("a")
+        os.mkdir(os.path.join(self.temp_output, "static"))
+        copy(staticfile.source_path, staticfile.save_as)
+        isnewer = self.generator._source_is_newer(staticfile)
+        self.assertFalse(isnewer)
+
+    @unittest.skipUnless(MagicMock, 'Needs Mock module')
+    def test_source_is_newer(self):
+        staticfile = MagicMock()
+        staticfile.source_path = self.startfile
+        staticfile.save_as = self.endfile
+        with open(staticfile.source_path, "w") as f:
+            f.write("a")
+        os.mkdir(os.path.join(self.temp_output, "static"))
+        copy(staticfile.source_path, staticfile.save_as)
+        self.set_ancient_mtime(staticfile.save_as)
+        isnewer = self.generator._source_is_newer(staticfile)
+        self.assertTrue(isnewer)
+
+    def test_skip_file_when_source_is_not_newer(self):
+        self.settings['STATIC_CHECK_IF_MODIFIED'] = True
+        with open(self.startfile, "w") as f:
+            f.write("staticcontent")
+        os.mkdir(os.path.join(self.temp_output, "static"))
+        with open(self.endfile, "w") as f:
+            f.write("staticcontent")
+        expected = os.path.getmtime(self.endfile)
+        self.set_ancient_mtime(self.startfile)
+        self.generator.generate_context()
+        self.generator.generate_output(None)
+        self.assertEqual(os.path.getmtime(self.endfile), expected)
+
+    def test_dont_link_by_default(self):
+        with open(self.startfile, "w") as f:
+            f.write("staticcontent")
+        self.generator.generate_context()
+        self.generator.generate_output(None)
+        self.assertFalse(os.path.samefile(self.startfile, self.endfile))
+
+    def test_output_file_is_linked_to_source(self):
+        self.settings['STATIC_CREATE_LINKS'] = True
+        with open(self.startfile, "w") as f:
+            f.write("staticcontent")
+        self.generator.generate_context()
+        self.generator.generate_output(None)
+        self.assertTrue(os.path.samefile(self.startfile, self.endfile))
+
+    def test_output_file_exists_and_is_newer(self):
+        self.settings['STATIC_CREATE_LINKS'] = True
+        with open(self.startfile, "w") as f:
+            f.write("staticcontent")
+        os.mkdir(os.path.join(self.temp_output, "static"))
+        with open(self.endfile, "w") as f:
+            f.write("othercontent")
+        self.generator.generate_context()
+        self.generator.generate_output(None)
+        self.assertTrue(os.path.samefile(self.startfile, self.endfile))
+
+    def test_can_symlink_when_hardlink_not_possible(self):
+        self.settings['STATIC_CREATE_LINKS'] = True
+        with open(self.startfile, "w") as f:
+            f.write("staticcontent")
+        os.mkdir(os.path.join(self.temp_output, "static"))
+        self.generator.fallback_to_symlinks = True
+        self.generator.generate_context()
+        self.generator.generate_output(None)
+        self.assertTrue(os.path.islink(self.endfile))
+
+    def test_existing_symlink_is_considered_up_to_date(self):
+        self.settings['STATIC_CREATE_LINKS'] = True
+        with open(self.startfile, "w") as f:
+            f.write("staticcontent")
+        os.mkdir(os.path.join(self.temp_output, "static"))
+        os.symlink(self.startfile, self.endfile)
+        staticfile = MagicMock()
+        staticfile.source_path = self.startfile
+        staticfile.save_as = self.endfile
+        requires_update = self.generator._file_update_required(staticfile)
+        self.assertFalse(requires_update)
+
+    def test_invalid_symlink_is_overwritten(self):
+        self.settings['STATIC_CREATE_LINKS'] = True
+        with open(self.startfile, "w") as f:
+            f.write("staticcontent")
+        os.mkdir(os.path.join(self.temp_output, "static"))
+        os.symlink("invalid", self.endfile)
+        staticfile = MagicMock()
+        staticfile.source_path = self.startfile
+        staticfile.save_as = self.endfile
+        requires_update = self.generator._file_update_required(staticfile)
+        self.assertTrue(requires_update)
+        self.generator.fallback_to_symlinks = True
+        self.generator.generate_context()
+        self.generator.generate_output(None)
+        self.assertEqual(os.path.realpath(self.endfile), self.startfile)
+
+    def test_delete_existing_file_before_mkdir(self):
+        with open(self.startfile, "w") as f:
+            f.write("staticcontent")
+        with open(os.path.join(self.temp_output, "static"), "w") as f:
+            f.write("This file should be a directory")
+        self.generator.generate_context()
+        self.generator.generate_output(None)
+        self.assertTrue(
+            os.path.isdir(os.path.join(self.temp_output, "static")))
+        self.assertTrue(os.path.isfile(self.endfile))