weeheavy/pelican-theme
1
0
Fork 0
forked from github/pelican
Code Pull requests Activity

Fix #1042 enable (sym)linking of static content and sources

Browse source 
This can greatly speed up generation for people with lots of static
files and/or sources output.
This commit is contained in:
Ondrej Grover 2014-05-09 11:42:12 +02:00
commit 25a8ab1a5f
5 changed files with 85 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

5
docs/settings.rst
View file

@ -119,6 +119,7 @@ Setting name (followed by default value, if any)
``OUTPUT_SOURCES_EXTENSION = '.text'`` Controls the extension that will be used by the SourcesGenerator.
Defaults to ``.text``. If not a valid string the default value
will be used.
``OUTPUT_SOURCES_JUST_LINK = ''`` Works like ``STATIC_JUST_LINK``.
``RELATIVE_URLS = False`` Defines whether Pelican should use document-relative URLs or
not. Only set this to ``True`` when developing/testing and only
if you fully understand the effect it can have on links/feeds.
@ -135,6 +136,10 @@ Setting name (followed by default value, if any)
on the output path "static". By default,
Pelican will copy the "images" folder to the
output folder.
``STATIC_JUST_LINK = ''`` Instead of copying the static files to the output directory, they can
be linked as symbolic links if set to ``symbolic`` (``rsync`` based uploads may require the ``--copy-links`` option) or as hard links if
set to ``hard``. Note that this functionality may be available only on
some operating systems and Python distributions.
``TIMEZONE`` The timezone used in the date information, to
generate Atom and RSS feeds. See the *Timezone*
section below for more info.

14
pelican/generators.py
View file

@ -650,13 +650,16 @@ class StaticGenerator(Generator):
def _copy_paths(self, paths, source, destination, output_path,
final_path=None):
"""Copy all the paths from source to destination"""
just_link = self.settings['STATIC_JUST_LINK']
for path in paths:
if final_path:
copy(os.path.join(source, path),
os.path.join(output_path, destination, final_path))
os.path.join(output_path, destination, final_path),
just_link)
else:
copy(os.path.join(source, path),
os.path.join(output_path, destination, path))
os.path.join(output_path, destination, path),
just_link)
def generate_context(self):
self.staticfiles = []
@ -682,12 +685,12 @@ class StaticGenerator(Generator):
self.settings['THEME_STATIC_DIR'], self.output_path,
os.curdir)
# copy all Static files
just_link = self.settings['STATIC_JUST_LINK']
for sc in self.context['staticfiles']:
source_path = os.path.join(self.path, sc.source_path)
save_as = os.path.join(self.output_path, sc.save_as)
mkdir_p(os.path.dirname(save_as))
shutil.copy2(source_path, save_as)
logger.info('copying {} to {}'.format(sc.source_path, sc.save_as))
copy(source_path, save_as, just_link)
class SourceFileGenerator(Generator):
@ -699,7 +702,8 @@ class SourceFileGenerator(Generator):
output_path, _ = os.path.splitext(obj.save_as)
dest = os.path.join(self.output_path,
output_path + self.output_extension)
copy(obj.source_path, dest)
just_link = self.settings['SOURCES_JUST_LINK']
copy(obj.source_path, dest, just_link)
def generate_output(self, writer=None):
logger.info(' Generating source files...')

2
pelican/settings.py
View file

@ -39,6 +39,7 @@ DEFAULT_CONFIG = {
'STATIC_PATHS': ['images', ],
'THEME_STATIC_DIR': 'theme',
'THEME_STATIC_PATHS': ['static', ],
'STATIC_JUST_LINK': '',
'FEED_ALL_ATOM': os.path.join('feeds', 'all.atom.xml'),
'CATEGORY_FEED_ATOM': os.path.join('feeds', '%s.atom.xml'),
'AUTHOR_FEED_ATOM': os.path.join('feeds', '%s.atom.xml'),
@ -51,6 +52,7 @@ DEFAULT_CONFIG = {
'DISPLAY_CATEGORIES_ON_MENU': True,
'OUTPUT_SOURCES': False,
'OUTPUT_SOURCES_EXTENSION': '.text',
'SOURCES_JUST_LINK': '',
'USE_FOLDER_AS_CATEGORY': True,
'DEFAULT_CATEGORY': 'misc',
'WITH_FUTURE_DATES': True,

33
pelican/tests/test_pelican.py
View file

@ -102,6 +102,39 @@ class TestPelican(LoggedTestCase):
mute(True)(pelican.run)()
self.assertDirsEqual(self.temp_path, os.path.join(OUTPUT_PATH, 'custom'))
def test_static_symlinking(self):
'''Test that symbolic linking of static files works'''
settings = read_settings(path=SAMPLE_CONFIG, override={
'PATH': INPUT_PATH,
'OUTPUT_PATH': self.temp_path,
'CACHE_PATH': self.temp_cache,
'LOCALE': locale.normalize('en_US'),
'STATIC_JUST_LINK': 'symbolic',
})
pelican = Pelican(settings=settings)
mute(True)(pelican.run)()
for fname in ['pictures/Fat_Cat.jpg', 'pictures/Sushi_Macro.jpg', 'robots.txt']:
dest = os.path.join(self.temp_path, fname)
self.assertTrue(os.path.exists(dest) and os.path.islink(dest))
def test_static_hardlinking(self):
'''Test that hard linking of static files works'''
settings = read_settings(path=SAMPLE_CONFIG, override={
'PATH': INPUT_PATH,
'OUTPUT_PATH': self.temp_path,
'CACHE_PATH': self.temp_cache,
'LOCALE': locale.normalize('en_US'),
'STATIC_JUST_LINK': 'hard',
})
pelican = Pelican(settings=settings)
mute(True)(pelican.run)()
for fname in ['pictures/Fat_Cat.jpg', 'pictures/Sushi_Macro.jpg']:
src = os.path.join(INPUT_PATH, fname)
dest = os.path.join(self.temp_path, fname)
self.assertTrue(os.path.exists(dest) and os.path.samefile(src, dest))
def test_theme_static_paths_copy(self):
# the same thing with a specified set of settings should work
settings = read_settings(path=SAMPLE_CONFIG, override={

37
pelican/utils.py
View file

@ -229,7 +229,12 @@ def slugify(value, substitutions=()):
return value.decode('ascii')
def copy(source, destination):
_LINK_FUNCS = { # map: link type -> func name in os module
'hard': 'link',
'symbolic': 'symlink',
}
def copy(source, destination, just_link=''):
"""Recursively copy source into destination.
If source is a file, destination has to be a file as well.
@ -238,11 +243,40 @@ def copy(source, destination):
:param source: the source file or directory
:param destination: the destination file or directory
:param just_link: type of link to use instead of copying,
'hard' or 'symbolic'
"""
source_ = os.path.abspath(os.path.expanduser(source))
destination_ = os.path.abspath(os.path.expanduser(destination))
if just_link:
try:
dest = destination_
link_func = getattr(os, _LINK_FUNCS[just_link])
if just_link == 'symbolic' and six.PY3:
link_func = partial(link_func,
target_is_directory=os.path.isdir(source_))
if os.path.exists(dest) and os.path.isdir(dest):
dest = os.path.join(dest, os.path.basename(source_))
else:
dest_dir = os.path.dirname(dest)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
link_func(source_, dest)
logger.info('linking ({}) {} -> {}'.format(
just_link, source_, dest))
return
except KeyError:
logger.error('Unknown link type: {}'.format(just_link))
except AttributeError as err:
logger.error(('{} linking not supported by platform, '
'falling back to copying\n{}').format(just_link, err))
except (OSError, IOError) as err:
logger.error(('Cannot make {} link {} -> {}, '
'falling back to copying\n{}').format(
just_link, source_, dest ,err))
if not os.path.exists(destination_) and not os.path.isfile(source_):
os.makedirs(destination_)
@ -684,4 +718,3 @@ def is_selected_for_writing(settings, path):
return path in settings['WRITE_SELECTED']
else:
return True