github/pelican
1
0
Fork 1
mirror of https://github.com/getpelican/pelican.git synced 2025-10-15 20:28:56 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

rename CACHE_DIR -> CACHE_PATH to unify with rest of Pelican

Browse source 
CACHE_PATH can now be relative to settings file like OUTPUT_PATH.
Also add --cache-path commandline option.
Change cache loading warning to a less scary and more helpful message.
This commit is contained in:
Ondrej Grover 2014-04-27 08:53:56 +02:00
commit 5bad061c19
6 changed files with 41 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

4
docs/settings.rst
View file

@ -178,7 +178,7 @@ Setting name (followed by default value, if any)
``CONTENT_CACHING_LAYER = 'reader'`` If set to ``'reader'``, save only the raw content and metadata ``CONTENT_CACHING_LAYER = 'reader'`` If set to ``'reader'``, save only the raw content and metadata
returned by readers. If set to ``'generator'``, save processed returned by readers. If set to ``'generator'``, save processed
content objects. content objects.
``CACHE_DIRECTORY = 'cache'`` Directory in which to store cache files. ``CACHE_PATH = 'cache'`` Directory in which to store cache files.
``GZIP_CACHE = True`` If ``True``, use gzip to (de)compress the cache files. ``GZIP_CACHE = True`` If ``True``, use gzip to (de)compress the cache files.
``CHECK_MODIFIED_METHOD = 'mtime'`` Controls how files are checked for modifications. ``CHECK_MODIFIED_METHOD = 'mtime'`` Controls how files are checked for modifications.
``LOAD_CONTENT_CACHE = True`` If ``True``, load unmodified content from cache. ``LOAD_CONTENT_CACHE = True`` If ``True``, load unmodified content from cache.
@ -739,7 +739,7 @@ When Pelican is about to read some content source file:
1. The hash or modification time information for the file from a 1. The hash or modification time information for the file from a
previous build are loaded from a cache file if ``LOAD_CONTENT_CACHE`` previous build are loaded from a cache file if ``LOAD_CONTENT_CACHE``
is ``True``. These files are stored in the ``CACHE_DIRECTORY`` is ``True``. These files are stored in the ``CACHE_PATH``
directory. If the file has no record in the cache file, it is read directory. If the file has no record in the cache file, it is read
as usual. as usual.
2. The file is checked according to ``CHECK_MODIFIED_METHOD``: 2. The file is checked according to ``CHECK_MODIFIED_METHOD``:

8
pelican/__init__.py
View file

@ -275,7 +275,11 @@ def parse_arguments():
help='Relaunch pelican each time a modification occurs' help='Relaunch pelican each time a modification occurs'
' on the content files.') ' on the content files.')
parser.add_argument('-c', '--ignore-cache', action='store_true', parser.add_argument('--cache-path', dest='cache_path',
help=('Directory in which to store cache files. '
'If not specified, defaults to "cache".'))
parser.add_argument('--ignore-cache', action='store_true',
dest='ignore_cache', help='Ignore content cache ' dest='ignore_cache', help='Ignore content cache '
'from previous runs by not loading cache files.') 'from previous runs by not loading cache files.')
@ -300,6 +304,8 @@ def get_config(args):
config['DELETE_OUTPUT_DIRECTORY'] = args.delete_outputdir config['DELETE_OUTPUT_DIRECTORY'] = args.delete_outputdir
if args.ignore_cache: if args.ignore_cache:
config['LOAD_CONTENT_CACHE'] = False config['LOAD_CONTENT_CACHE'] = False
if args.cache_path:
config['CACHE_PATH'] = args.cache_path
if args.selected_paths: if args.selected_paths:
config['WRITE_SELECTED'] = args.selected_paths.split(',') config['WRITE_SELECTED'] = args.selected_paths.split(',')

4
pelican/settings.py
View file

@ -124,7 +124,7 @@ DEFAULT_CONFIG = {
'SLUGIFY_SOURCE': 'title', 'SLUGIFY_SOURCE': 'title',
'CACHE_CONTENT': True, 'CACHE_CONTENT': True,
'CONTENT_CACHING_LAYER': 'reader', 'CONTENT_CACHING_LAYER': 'reader',
'CACHE_DIRECTORY': 'cache', 'CACHE_PATH': 'cache',
'GZIP_CACHE': True, 'GZIP_CACHE': True,
'CHECK_MODIFIED_METHOD': 'mtime', 'CHECK_MODIFIED_METHOD': 'mtime',
'LOAD_CONTENT_CACHE': True, 'LOAD_CONTENT_CACHE': True,
@ -139,7 +139,7 @@ def read_settings(path=None, override=None):
if path: if path:
local_settings = get_settings_from_file(path) local_settings = get_settings_from_file(path)
# Make the paths relative to the settings file # Make the paths relative to the settings file
for p in ['PATH', 'OUTPUT_PATH', 'THEME']: for p in ['PATH', 'OUTPUT_PATH', 'THEME', 'CACHE_PATH']:
if p in local_settings and local_settings[p] is not None \ if p in local_settings and local_settings[p] is not None \
and not isabs(local_settings[p]): and not isabs(local_settings[p]):
absp = os.path.abspath(os.path.normpath(os.path.join( absp = os.path.abspath(os.path.normpath(os.path.join(

26
pelican/tests/test_generators.py
View file

@ -66,7 +66,7 @@ class TestArticlesGenerator(unittest.TestCase):
def test_generate_feeds(self): def test_generate_feeds(self):
settings = get_settings() settings = get_settings()
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
generator = ArticlesGenerator( generator = ArticlesGenerator(
context=settings, settings=settings, context=settings, settings=settings,
path=None, theme=settings['THEME'], output_path=None) path=None, theme=settings['THEME'], output_path=None)
@ -142,7 +142,7 @@ class TestArticlesGenerator(unittest.TestCase):
settings['DEFAULT_CATEGORY'] = 'Default' settings['DEFAULT_CATEGORY'] = 'Default'
settings['DEFAULT_DATE'] = (1970, 1, 1) settings['DEFAULT_DATE'] = (1970, 1, 1)
settings['USE_FOLDER_AS_CATEGORY'] = False settings['USE_FOLDER_AS_CATEGORY'] = False
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
settings['READERS'] = {'asc': None} settings['READERS'] = {'asc': None}
settings['filenames'] = {} settings['filenames'] = {}
generator = ArticlesGenerator( generator = ArticlesGenerator(
@ -167,7 +167,7 @@ class TestArticlesGenerator(unittest.TestCase):
def test_direct_templates_save_as_default(self): def test_direct_templates_save_as_default(self):
settings = get_settings(filenames={}) settings = get_settings(filenames={})
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
generator = ArticlesGenerator( generator = ArticlesGenerator(
context=settings, settings=settings, context=settings, settings=settings,
path=None, theme=settings['THEME'], output_path=None) path=None, theme=settings['THEME'], output_path=None)
@ -182,7 +182,7 @@ class TestArticlesGenerator(unittest.TestCase):
settings = get_settings() settings = get_settings()
settings['DIRECT_TEMPLATES'] = ['archives'] settings['DIRECT_TEMPLATES'] = ['archives']
settings['ARCHIVES_SAVE_AS'] = 'archives/index.html' settings['ARCHIVES_SAVE_AS'] = 'archives/index.html'
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
generator = ArticlesGenerator( generator = ArticlesGenerator(
context=settings, settings=settings, context=settings, settings=settings,
path=None, theme=settings['THEME'], output_path=None) path=None, theme=settings['THEME'], output_path=None)
@ -198,7 +198,7 @@ class TestArticlesGenerator(unittest.TestCase):
settings = get_settings() settings = get_settings()
settings['DIRECT_TEMPLATES'] = ['archives'] settings['DIRECT_TEMPLATES'] = ['archives']
settings['ARCHIVES_SAVE_AS'] = 'archives/index.html' settings['ARCHIVES_SAVE_AS'] = 'archives/index.html'
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
generator = ArticlesGenerator( generator = ArticlesGenerator(
context=settings, settings=settings, context=settings, settings=settings,
path=None, theme=settings['THEME'], output_path=None) path=None, theme=settings['THEME'], output_path=None)
@ -225,7 +225,7 @@ class TestArticlesGenerator(unittest.TestCase):
settings = get_settings(filenames={}) settings = get_settings(filenames={})
settings['YEAR_ARCHIVE_SAVE_AS'] = 'posts/{date:%Y}/index.html' settings['YEAR_ARCHIVE_SAVE_AS'] = 'posts/{date:%Y}/index.html'
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
generator = ArticlesGenerator( generator = ArticlesGenerator(
context=settings, settings=settings, context=settings, settings=settings,
path=CONTENT_DIR, theme=settings['THEME'], output_path=None) path=CONTENT_DIR, theme=settings['THEME'], output_path=None)
@ -291,7 +291,7 @@ class TestArticlesGenerator(unittest.TestCase):
def test_article_object_caching(self): def test_article_object_caching(self):
"""Test Article objects caching at the generator level""" """Test Article objects caching at the generator level"""
settings = get_settings(filenames={}) settings = get_settings(filenames={})
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
settings['CONTENT_CACHING_LAYER'] = 'generator' settings['CONTENT_CACHING_LAYER'] = 'generator'
settings['READERS'] = {'asc': None} settings['READERS'] = {'asc': None}
@ -311,7 +311,7 @@ class TestArticlesGenerator(unittest.TestCase):
def test_reader_content_caching(self): def test_reader_content_caching(self):
"""Test raw content caching at the reader level""" """Test raw content caching at the reader level"""
settings = get_settings(filenames={}) settings = get_settings(filenames={})
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
settings['READERS'] = {'asc': None} settings['READERS'] = {'asc': None}
generator = ArticlesGenerator( generator = ArticlesGenerator(
@ -335,7 +335,7 @@ class TestArticlesGenerator(unittest.TestCase):
used in --ignore-cache or autoreload mode""" used in --ignore-cache or autoreload mode"""
settings = get_settings(filenames={}) settings = get_settings(filenames={})
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
settings['READERS'] = {'asc': None} settings['READERS'] = {'asc': None}
generator = ArticlesGenerator( generator = ArticlesGenerator(
@ -373,7 +373,7 @@ class TestPageGenerator(unittest.TestCase):
def test_generate_context(self): def test_generate_context(self):
settings = get_settings(filenames={}) settings = get_settings(filenames={})
settings['PAGE_DIR'] = 'TestPages' # relative to CUR_DIR settings['PAGE_DIR'] = 'TestPages' # relative to CUR_DIR
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
settings['DEFAULT_DATE'] = (1970, 1, 1) settings['DEFAULT_DATE'] = (1970, 1, 1)
generator = PagesGenerator( generator = PagesGenerator(
@ -402,7 +402,7 @@ class TestPageGenerator(unittest.TestCase):
def test_page_object_caching(self): def test_page_object_caching(self):
"""Test Page objects caching at the generator level""" """Test Page objects caching at the generator level"""
settings = get_settings(filenames={}) settings = get_settings(filenames={})
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
settings['CONTENT_CACHING_LAYER'] = 'generator' settings['CONTENT_CACHING_LAYER'] = 'generator'
settings['READERS'] = {'asc': None} settings['READERS'] = {'asc': None}
@ -422,7 +422,7 @@ class TestPageGenerator(unittest.TestCase):
def test_reader_content_caching(self): def test_reader_content_caching(self):
"""Test raw content caching at the reader level""" """Test raw content caching at the reader level"""
settings = get_settings(filenames={}) settings = get_settings(filenames={})
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
settings['READERS'] = {'asc': None} settings['READERS'] = {'asc': None}
generator = PagesGenerator( generator = PagesGenerator(
@ -446,7 +446,7 @@ class TestPageGenerator(unittest.TestCase):
used in --ignore_cache or autoreload mode""" used in --ignore_cache or autoreload mode"""
settings = get_settings(filenames={}) settings = get_settings(filenames={})
settings['CACHE_DIRECTORY'] = self.temp_cache settings['CACHE_PATH'] = self.temp_cache
settings['READERS'] = {'asc': None} settings['READERS'] = {'asc': None}
generator = PagesGenerator( generator = PagesGenerator(

10
pelican/tests/test_pelican.py
View file

@ -79,7 +79,7 @@ class TestPelican(LoggedTestCase):
settings = read_settings(path=None, override={ settings = read_settings(path=None, override={
'PATH': INPUT_PATH, 'PATH': INPUT_PATH,
'OUTPUT_PATH': self.temp_path, 'OUTPUT_PATH': self.temp_path,
'CACHE_DIRECTORY': self.temp_cache, 'CACHE_PATH': self.temp_cache,
'LOCALE': locale.normalize('en_US'), 'LOCALE': locale.normalize('en_US'),
}) })
pelican = Pelican(settings=settings) pelican = Pelican(settings=settings)
@ -95,7 +95,7 @@ class TestPelican(LoggedTestCase):
settings = read_settings(path=SAMPLE_CONFIG, override={ settings = read_settings(path=SAMPLE_CONFIG, override={
'PATH': INPUT_PATH, 'PATH': INPUT_PATH,
'OUTPUT_PATH': self.temp_path, 'OUTPUT_PATH': self.temp_path,
'CACHE_DIRECTORY': self.temp_cache, 'CACHE_PATH': self.temp_cache,
'LOCALE': locale.normalize('en_US'), 'LOCALE': locale.normalize('en_US'),
}) })
pelican = Pelican(settings=settings) pelican = Pelican(settings=settings)
@ -107,7 +107,7 @@ class TestPelican(LoggedTestCase):
settings = read_settings(path=SAMPLE_CONFIG, override={ settings = read_settings(path=SAMPLE_CONFIG, override={
'PATH': INPUT_PATH, 'PATH': INPUT_PATH,
'OUTPUT_PATH': self.temp_path, 'OUTPUT_PATH': self.temp_path,
'CACHE_DIRECTORY': self.temp_cache, 'CACHE_PATH': self.temp_cache,
'THEME_STATIC_PATHS': [os.path.join(SAMPLES_PATH, 'very'), 'THEME_STATIC_PATHS': [os.path.join(SAMPLES_PATH, 'very'),
os.path.join(SAMPLES_PATH, 'kinda'), os.path.join(SAMPLES_PATH, 'kinda'),
os.path.join(SAMPLES_PATH, 'theme_standard')] os.path.join(SAMPLES_PATH, 'theme_standard')]
@ -128,7 +128,7 @@ class TestPelican(LoggedTestCase):
settings = read_settings(path=SAMPLE_CONFIG, override={ settings = read_settings(path=SAMPLE_CONFIG, override={
'PATH': INPUT_PATH, 'PATH': INPUT_PATH,
'OUTPUT_PATH': self.temp_path, 'OUTPUT_PATH': self.temp_path,
'CACHE_DIRECTORY': self.temp_cache, 'CACHE_PATH': self.temp_cache,
'THEME_STATIC_PATHS': [os.path.join(SAMPLES_PATH, 'theme_standard')] 'THEME_STATIC_PATHS': [os.path.join(SAMPLES_PATH, 'theme_standard')]
}) })
@ -144,7 +144,7 @@ class TestPelican(LoggedTestCase):
settings = read_settings(path=None, override={ settings = read_settings(path=None, override={
'PATH': INPUT_PATH, 'PATH': INPUT_PATH,
'OUTPUT_PATH': self.temp_path, 'OUTPUT_PATH': self.temp_path,
'CACHE_DIRECTORY': self.temp_cache, 'CACHE_PATH': self.temp_cache,
'WRITE_SELECTED': [ 'WRITE_SELECTED': [
os.path.join(self.temp_path, 'oh-yeah.html'), os.path.join(self.temp_path, 'oh-yeah.html'),
os.path.join(self.temp_path, 'categories.html'), os.path.join(self.temp_path, 'categories.html'),

18
pelican/utils.py
View file

@ -553,14 +553,14 @@ class FileDataCacher(object):
'''Class that can cache data contained in files''' '''Class that can cache data contained in files'''
def __init__(self, settings, cache_name, caching_policy, load_policy): def __init__(self, settings, cache_name, caching_policy, load_policy):
'''Load the specified cache within CACHE_DIRECTORY in settings '''Load the specified cache within CACHE_PATH in settings
only if *load_policy* is True, only if *load_policy* is True,
May use gzip if GZIP_CACHE ins settings is True. May use gzip if GZIP_CACHE ins settings is True.
Sets caching policy according to *caching_policy*. Sets caching policy according to *caching_policy*.
''' '''
self.settings = settings self.settings = settings
self._cache_path = os.path.join(self.settings['CACHE_DIRECTORY'], self._cache_path = os.path.join(self.settings['CACHE_PATH'],
cache_name) cache_name)
self._cache_data_policy = caching_policy self._cache_data_policy = caching_policy
if self.settings['GZIP_CACHE']: if self.settings['GZIP_CACHE']:
@ -572,9 +572,15 @@ class FileDataCacher(object):
try: try:
with self._cache_open(self._cache_path, 'rb') as fhandle: with self._cache_open(self._cache_path, 'rb') as fhandle:
self._cache = pickle.load(fhandle) self._cache = pickle.load(fhandle)
except (IOError, OSError, pickle.UnpicklingError) as err: except (IOError, OSError) as err:
logger.warning(('Cannot load cache {}, ' logger.debug(('Cannot load cache {} (this is normal on first '
'proceeding with empty cache.\n{}').format( 'run). Proceeding with empty cache.\n{}').format(
self._cache_path, err))
self._cache = {}
except pickle.UnpicklingError as err:
logger.warning(('Cannot unpickle cache {}, cache may be using '
'an incompatible protocol (see pelican caching docs). '
'Proceeding with empty cache.\n{}').format(
self._cache_path, err)) self._cache_path, err))
self._cache = {} self._cache = {}
else: else:
@ -596,7 +602,7 @@ class FileDataCacher(object):
'''Save the updated cache''' '''Save the updated cache'''
if self._cache_data_policy: if self._cache_data_policy:
try: try:
mkdir_p(self.settings['CACHE_DIRECTORY']) mkdir_p(self.settings['CACHE_PATH'])
with self._cache_open(self._cache_path, 'wb') as fhandle: with self._cache_open(self._cache_path, 'wb') as fhandle:
pickle.dump(self._cache, fhandle) pickle.dump(self._cache, fhandle)
except (IOError, OSError, pickle.PicklingError) as err: except (IOError, OSError, pickle.PicklingError) as err: