Fix #1042 enable (sym)linking of static content and sources

This can greatly speed up generation for people with lots of static
files and/or sources output.

docs/settings.rst

@@ -119,6 +119,7 @@ Setting name (followed by default value, if any)
 ``OUTPUT_SOURCES_EXTENSION = '.text'``                                           Controls the extension that will be used by the SourcesGenerator.
                                                                                  Defaults to ``.text``. If not a valid string the default value
                                                                                  will be used.
+``OUTPUT_SOURCES_JUST_LINK = ''``                                                Works like ``STATIC_JUST_LINK``.
 ``RELATIVE_URLS = False``                                                        Defines whether Pelican should use document-relative URLs or
                                                                                  not. Only set this to ``True`` when developing/testing and only
                                                                                  if you fully understand the effect it can have on links/feeds.
@@ -135,6 +136,10 @@ Setting name (followed by default value, if any)
                                                                                  on the output path "static". By default,
                                                                                  Pelican will copy the "images" folder to the
                                                                                  output folder.
+``STATIC_JUST_LINK = ''``                                                        Instead of copying the static files to the output directory, they can
+                                                                                 be linked as symbolic links if set to ``symbolic`` (``rsync`` based uploads may require the ``--copy-links`` option) or as hard links if
+                                                                                 set to ``hard``. Note that this functionality may be available only on
+                                                                                 some operating systems and Python distributions.
 ``TIMEZONE``                                                                     The timezone used in the date information, to
                                                                                  generate Atom and RSS feeds. See the *Timezone*
                                                                                  section below for more info.

pelican/generators.py

@@ -650,13 +650,16 @@ class StaticGenerator(Generator):
     def _copy_paths(self, paths, source, destination, output_path,
                     final_path=None):
         """Copy all the paths from source to destination"""
+        just_link = self.settings['STATIC_JUST_LINK']
         for path in paths:
             if final_path:
                 copy(os.path.join(source, path),
-                     os.path.join(output_path, destination, final_path))
+                     os.path.join(output_path, destination, final_path),
+                    just_link)
             else:
                 copy(os.path.join(source, path),
-                     os.path.join(output_path, destination, path))
+                     os.path.join(output_path, destination, path),
+                    just_link)
 
     def generate_context(self):
         self.staticfiles = []
@@ -680,14 +683,14 @@ class StaticGenerator(Generator):
     def generate_output(self, writer):
         self._copy_paths(self.settings['THEME_STATIC_PATHS'], self.theme,
                          self.settings['THEME_STATIC_DIR'], self.output_path,
-                         os.curdir)
+            os.curdir)
         # copy all Static files
+        just_link = self.settings['STATIC_JUST_LINK']
         for sc in self.context['staticfiles']:
             source_path = os.path.join(self.path, sc.source_path)
             save_as = os.path.join(self.output_path, sc.save_as)
             mkdir_p(os.path.dirname(save_as))
-            shutil.copy2(source_path, save_as)
-            logger.info('copying {} to {}'.format(sc.source_path, sc.save_as))
+            copy(source_path, save_as, just_link)
 
 
 class SourceFileGenerator(Generator):
@@ -699,7 +702,8 @@ class SourceFileGenerator(Generator):
         output_path, _ = os.path.splitext(obj.save_as)
         dest = os.path.join(self.output_path,
                             output_path + self.output_extension)
-        copy(obj.source_path, dest)
+        just_link = self.settings['SOURCES_JUST_LINK']
+        copy(obj.source_path, dest, just_link)
 
     def generate_output(self, writer=None):
         logger.info(' Generating source files...')

pelican/settings.py

@@ -39,6 +39,7 @@ DEFAULT_CONFIG = {
     'STATIC_PATHS': ['images', ],
     'THEME_STATIC_DIR': 'theme',
     'THEME_STATIC_PATHS': ['static', ],
+    'STATIC_JUST_LINK': '',
     'FEED_ALL_ATOM': os.path.join('feeds', 'all.atom.xml'),
     'CATEGORY_FEED_ATOM': os.path.join('feeds', '%s.atom.xml'),
     'AUTHOR_FEED_ATOM': os.path.join('feeds', '%s.atom.xml'),
@@ -51,6 +52,7 @@ DEFAULT_CONFIG = {
     'DISPLAY_CATEGORIES_ON_MENU': True,
     'OUTPUT_SOURCES': False,
     'OUTPUT_SOURCES_EXTENSION': '.text',
+    'SOURCES_JUST_LINK': '',
     'USE_FOLDER_AS_CATEGORY': True,
     'DEFAULT_CATEGORY': 'misc',
     'WITH_FUTURE_DATES': True,

pelican/tests/test_pelican.py

@@ -102,6 +102,39 @@ class TestPelican(LoggedTestCase):
         mute(True)(pelican.run)()
         self.assertDirsEqual(self.temp_path, os.path.join(OUTPUT_PATH, 'custom'))
 
+    def test_static_symlinking(self):
+        '''Test that symbolic linking of static files works'''
+        settings = read_settings(path=SAMPLE_CONFIG, override={
+            'PATH': INPUT_PATH,
+            'OUTPUT_PATH': self.temp_path,
+            'CACHE_PATH': self.temp_cache,
+            'LOCALE': locale.normalize('en_US'),
+            'STATIC_JUST_LINK': 'symbolic',
+            })
+        pelican = Pelican(settings=settings)
+        mute(True)(pelican.run)()
+
+        for fname in ['pictures/Fat_Cat.jpg', 'pictures/Sushi_Macro.jpg', 'robots.txt']:
+            dest = os.path.join(self.temp_path, fname)
+            self.assertTrue(os.path.exists(dest) and os.path.islink(dest))
+
+    def test_static_hardlinking(self):
+        '''Test that hard linking of static files works'''
+        settings = read_settings(path=SAMPLE_CONFIG, override={
+            'PATH': INPUT_PATH,
+            'OUTPUT_PATH': self.temp_path,
+            'CACHE_PATH': self.temp_cache,
+            'LOCALE': locale.normalize('en_US'),
+            'STATIC_JUST_LINK': 'hard',
+            })
+        pelican = Pelican(settings=settings)
+        mute(True)(pelican.run)()
+
+        for fname in ['pictures/Fat_Cat.jpg', 'pictures/Sushi_Macro.jpg']:
+            src = os.path.join(INPUT_PATH, fname)
+            dest = os.path.join(self.temp_path, fname)
+            self.assertTrue(os.path.exists(dest) and os.path.samefile(src, dest))
+
     def test_theme_static_paths_copy(self):
         # the same thing with a specified set of settings should work
         settings = read_settings(path=SAMPLE_CONFIG, override={

pelican/utils.py

@@ -229,7 +229,12 @@ def slugify(value, substitutions=()):
     return value.decode('ascii')
 
 
-def copy(source, destination):
+_LINK_FUNCS = {             # map: link type -> func name in os module
+    'hard': 'link',
+    'symbolic': 'symlink',
+    }
+
+def copy(source, destination, just_link=''):
     """Recursively copy source into destination.
 
     If source is a file, destination has to be a file as well.
@@ -238,11 +243,40 @@ def copy(source, destination):
 
     :param source: the source file or directory
     :param destination: the destination file or directory
+    :param just_link: type of link to use instead of copying,
+                      'hard' or 'symbolic'
     """
 
     source_ = os.path.abspath(os.path.expanduser(source))
     destination_ = os.path.abspath(os.path.expanduser(destination))
 
+    if just_link:
+        try:
+            dest = destination_
+            link_func = getattr(os, _LINK_FUNCS[just_link])
+            if just_link == 'symbolic' and six.PY3:
+                link_func = partial(link_func,
+                                    target_is_directory=os.path.isdir(source_))
+            if os.path.exists(dest) and os.path.isdir(dest):
+                dest = os.path.join(dest, os.path.basename(source_))
+            else:
+                dest_dir = os.path.dirname(dest)
+                if not os.path.exists(dest_dir):
+                    os.makedirs(dest_dir)
+            link_func(source_, dest)
+            logger.info('linking ({}) {} -> {}'.format(
+                        just_link, source_, dest))
+            return
+        except KeyError:
+            logger.error('Unknown link type: {}'.format(just_link))
+        except AttributeError as err:
+            logger.error(('{} linking not supported by platform, '
+                         'falling back to copying\n{}').format(just_link, err))
+        except (OSError, IOError) as err:
+            logger.error(('Cannot make {} link {} -> {}, '
+                         'falling back to copying\n{}').format(
+                             just_link, source_, dest ,err))
+
     if not os.path.exists(destination_) and not os.path.isfile(source_):
         os.makedirs(destination_)
 
@@ -684,4 +718,3 @@ def is_selected_for_writing(settings, path):
         return path in settings['WRITE_SELECTED']
     else:
         return True
-