From 5bad061c19947fde41a98bccb56a9a00ef46a40a Mon Sep 17 00:00:00 2001 From: Ondrej Grover Date: Sun, 27 Apr 2014 08:53:56 +0200 Subject: [PATCH] rename CACHE_DIR -> CACHE_PATH to unify with rest of Pelican CACHE_PATH can now be relative to settings file like OUTPUT_PATH. Also add --cache-path commandline option. Change cache loading warning to a less scary and more helpful message. --- docs/settings.rst | 4 ++-- pelican/__init__.py | 8 +++++++- pelican/settings.py | 4 ++-- pelican/tests/test_generators.py | 26 +++++++++++++------------- pelican/tests/test_pelican.py | 10 +++++----- pelican/utils.py | 18 ++++++++++++------ 6 files changed, 41 insertions(+), 29 deletions(-) diff --git a/docs/settings.rst b/docs/settings.rst index d5802d1d..2782977c 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -178,7 +178,7 @@ Setting name (followed by default value, if any) ``CONTENT_CACHING_LAYER = 'reader'`` If set to ``'reader'``, save only the raw content and metadata returned by readers. If set to ``'generator'``, save processed content objects. -``CACHE_DIRECTORY = 'cache'`` Directory in which to store cache files. +``CACHE_PATH = 'cache'`` Directory in which to store cache files. ``GZIP_CACHE = True`` If ``True``, use gzip to (de)compress the cache files. ``CHECK_MODIFIED_METHOD = 'mtime'`` Controls how files are checked for modifications. ``LOAD_CONTENT_CACHE = True`` If ``True``, load unmodified content from cache. @@ -739,7 +739,7 @@ When Pelican is about to read some content source file: 1. The hash or modification time information for the file from a previous build are loaded from a cache file if ``LOAD_CONTENT_CACHE`` - is ``True``. These files are stored in the ``CACHE_DIRECTORY`` + is ``True``. These files are stored in the ``CACHE_PATH`` directory. If the file has no record in the cache file, it is read as usual. 2. The file is checked according to ``CHECK_MODIFIED_METHOD``: diff --git a/pelican/__init__.py b/pelican/__init__.py index 74f55c46..d6417391 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -275,7 +275,11 @@ def parse_arguments(): help='Relaunch pelican each time a modification occurs' ' on the content files.') - parser.add_argument('-c', '--ignore-cache', action='store_true', + parser.add_argument('--cache-path', dest='cache_path', + help=('Directory in which to store cache files. ' + 'If not specified, defaults to "cache".')) + + parser.add_argument('--ignore-cache', action='store_true', dest='ignore_cache', help='Ignore content cache ' 'from previous runs by not loading cache files.') @@ -300,6 +304,8 @@ def get_config(args): config['DELETE_OUTPUT_DIRECTORY'] = args.delete_outputdir if args.ignore_cache: config['LOAD_CONTENT_CACHE'] = False + if args.cache_path: + config['CACHE_PATH'] = args.cache_path if args.selected_paths: config['WRITE_SELECTED'] = args.selected_paths.split(',') diff --git a/pelican/settings.py b/pelican/settings.py index abf16b32..f759ff9e 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -124,7 +124,7 @@ DEFAULT_CONFIG = { 'SLUGIFY_SOURCE': 'title', 'CACHE_CONTENT': True, 'CONTENT_CACHING_LAYER': 'reader', - 'CACHE_DIRECTORY': 'cache', + 'CACHE_PATH': 'cache', 'GZIP_CACHE': True, 'CHECK_MODIFIED_METHOD': 'mtime', 'LOAD_CONTENT_CACHE': True, @@ -139,7 +139,7 @@ def read_settings(path=None, override=None): if path: local_settings = get_settings_from_file(path) # Make the paths relative to the settings file - for p in ['PATH', 'OUTPUT_PATH', 'THEME']: + for p in ['PATH', 'OUTPUT_PATH', 'THEME', 'CACHE_PATH']: if p in local_settings and local_settings[p] is not None \ and not isabs(local_settings[p]): absp = os.path.abspath(os.path.normpath(os.path.join( diff --git a/pelican/tests/test_generators.py b/pelican/tests/test_generators.py index 9463047e..7b79e8f3 100644 --- a/pelican/tests/test_generators.py +++ b/pelican/tests/test_generators.py @@ -66,7 +66,7 @@ class TestArticlesGenerator(unittest.TestCase): def test_generate_feeds(self): settings = get_settings() - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache generator = ArticlesGenerator( context=settings, settings=settings, path=None, theme=settings['THEME'], output_path=None) @@ -142,7 +142,7 @@ class TestArticlesGenerator(unittest.TestCase): settings['DEFAULT_CATEGORY'] = 'Default' settings['DEFAULT_DATE'] = (1970, 1, 1) settings['USE_FOLDER_AS_CATEGORY'] = False - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache settings['READERS'] = {'asc': None} settings['filenames'] = {} generator = ArticlesGenerator( @@ -167,7 +167,7 @@ class TestArticlesGenerator(unittest.TestCase): def test_direct_templates_save_as_default(self): settings = get_settings(filenames={}) - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache generator = ArticlesGenerator( context=settings, settings=settings, path=None, theme=settings['THEME'], output_path=None) @@ -182,7 +182,7 @@ class TestArticlesGenerator(unittest.TestCase): settings = get_settings() settings['DIRECT_TEMPLATES'] = ['archives'] settings['ARCHIVES_SAVE_AS'] = 'archives/index.html' - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache generator = ArticlesGenerator( context=settings, settings=settings, path=None, theme=settings['THEME'], output_path=None) @@ -198,7 +198,7 @@ class TestArticlesGenerator(unittest.TestCase): settings = get_settings() settings['DIRECT_TEMPLATES'] = ['archives'] settings['ARCHIVES_SAVE_AS'] = 'archives/index.html' - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache generator = ArticlesGenerator( context=settings, settings=settings, path=None, theme=settings['THEME'], output_path=None) @@ -225,7 +225,7 @@ class TestArticlesGenerator(unittest.TestCase): settings = get_settings(filenames={}) settings['YEAR_ARCHIVE_SAVE_AS'] = 'posts/{date:%Y}/index.html' - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache generator = ArticlesGenerator( context=settings, settings=settings, path=CONTENT_DIR, theme=settings['THEME'], output_path=None) @@ -291,7 +291,7 @@ class TestArticlesGenerator(unittest.TestCase): def test_article_object_caching(self): """Test Article objects caching at the generator level""" settings = get_settings(filenames={}) - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache settings['CONTENT_CACHING_LAYER'] = 'generator' settings['READERS'] = {'asc': None} @@ -311,7 +311,7 @@ class TestArticlesGenerator(unittest.TestCase): def test_reader_content_caching(self): """Test raw content caching at the reader level""" settings = get_settings(filenames={}) - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache settings['READERS'] = {'asc': None} generator = ArticlesGenerator( @@ -335,7 +335,7 @@ class TestArticlesGenerator(unittest.TestCase): used in --ignore-cache or autoreload mode""" settings = get_settings(filenames={}) - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache settings['READERS'] = {'asc': None} generator = ArticlesGenerator( @@ -373,7 +373,7 @@ class TestPageGenerator(unittest.TestCase): def test_generate_context(self): settings = get_settings(filenames={}) settings['PAGE_DIR'] = 'TestPages' # relative to CUR_DIR - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache settings['DEFAULT_DATE'] = (1970, 1, 1) generator = PagesGenerator( @@ -402,7 +402,7 @@ class TestPageGenerator(unittest.TestCase): def test_page_object_caching(self): """Test Page objects caching at the generator level""" settings = get_settings(filenames={}) - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache settings['CONTENT_CACHING_LAYER'] = 'generator' settings['READERS'] = {'asc': None} @@ -422,7 +422,7 @@ class TestPageGenerator(unittest.TestCase): def test_reader_content_caching(self): """Test raw content caching at the reader level""" settings = get_settings(filenames={}) - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache settings['READERS'] = {'asc': None} generator = PagesGenerator( @@ -446,7 +446,7 @@ class TestPageGenerator(unittest.TestCase): used in --ignore_cache or autoreload mode""" settings = get_settings(filenames={}) - settings['CACHE_DIRECTORY'] = self.temp_cache + settings['CACHE_PATH'] = self.temp_cache settings['READERS'] = {'asc': None} generator = PagesGenerator( diff --git a/pelican/tests/test_pelican.py b/pelican/tests/test_pelican.py index 294cf399..411fb7da 100644 --- a/pelican/tests/test_pelican.py +++ b/pelican/tests/test_pelican.py @@ -79,7 +79,7 @@ class TestPelican(LoggedTestCase): settings = read_settings(path=None, override={ 'PATH': INPUT_PATH, 'OUTPUT_PATH': self.temp_path, - 'CACHE_DIRECTORY': self.temp_cache, + 'CACHE_PATH': self.temp_cache, 'LOCALE': locale.normalize('en_US'), }) pelican = Pelican(settings=settings) @@ -95,7 +95,7 @@ class TestPelican(LoggedTestCase): settings = read_settings(path=SAMPLE_CONFIG, override={ 'PATH': INPUT_PATH, 'OUTPUT_PATH': self.temp_path, - 'CACHE_DIRECTORY': self.temp_cache, + 'CACHE_PATH': self.temp_cache, 'LOCALE': locale.normalize('en_US'), }) pelican = Pelican(settings=settings) @@ -107,7 +107,7 @@ class TestPelican(LoggedTestCase): settings = read_settings(path=SAMPLE_CONFIG, override={ 'PATH': INPUT_PATH, 'OUTPUT_PATH': self.temp_path, - 'CACHE_DIRECTORY': self.temp_cache, + 'CACHE_PATH': self.temp_cache, 'THEME_STATIC_PATHS': [os.path.join(SAMPLES_PATH, 'very'), os.path.join(SAMPLES_PATH, 'kinda'), os.path.join(SAMPLES_PATH, 'theme_standard')] @@ -128,7 +128,7 @@ class TestPelican(LoggedTestCase): settings = read_settings(path=SAMPLE_CONFIG, override={ 'PATH': INPUT_PATH, 'OUTPUT_PATH': self.temp_path, - 'CACHE_DIRECTORY': self.temp_cache, + 'CACHE_PATH': self.temp_cache, 'THEME_STATIC_PATHS': [os.path.join(SAMPLES_PATH, 'theme_standard')] }) @@ -144,7 +144,7 @@ class TestPelican(LoggedTestCase): settings = read_settings(path=None, override={ 'PATH': INPUT_PATH, 'OUTPUT_PATH': self.temp_path, - 'CACHE_DIRECTORY': self.temp_cache, + 'CACHE_PATH': self.temp_cache, 'WRITE_SELECTED': [ os.path.join(self.temp_path, 'oh-yeah.html'), os.path.join(self.temp_path, 'categories.html'), diff --git a/pelican/utils.py b/pelican/utils.py index 7b58a231..2af34ecf 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -553,14 +553,14 @@ class FileDataCacher(object): '''Class that can cache data contained in files''' def __init__(self, settings, cache_name, caching_policy, load_policy): - '''Load the specified cache within CACHE_DIRECTORY in settings + '''Load the specified cache within CACHE_PATH in settings only if *load_policy* is True, May use gzip if GZIP_CACHE ins settings is True. Sets caching policy according to *caching_policy*. ''' self.settings = settings - self._cache_path = os.path.join(self.settings['CACHE_DIRECTORY'], + self._cache_path = os.path.join(self.settings['CACHE_PATH'], cache_name) self._cache_data_policy = caching_policy if self.settings['GZIP_CACHE']: @@ -572,9 +572,15 @@ class FileDataCacher(object): try: with self._cache_open(self._cache_path, 'rb') as fhandle: self._cache = pickle.load(fhandle) - except (IOError, OSError, pickle.UnpicklingError) as err: - logger.warning(('Cannot load cache {}, ' - 'proceeding with empty cache.\n{}').format( + except (IOError, OSError) as err: + logger.debug(('Cannot load cache {} (this is normal on first ' + 'run). Proceeding with empty cache.\n{}').format( + self._cache_path, err)) + self._cache = {} + except pickle.UnpicklingError as err: + logger.warning(('Cannot unpickle cache {}, cache may be using ' + 'an incompatible protocol (see pelican caching docs). ' + 'Proceeding with empty cache.\n{}').format( self._cache_path, err)) self._cache = {} else: @@ -596,7 +602,7 @@ class FileDataCacher(object): '''Save the updated cache''' if self._cache_data_policy: try: - mkdir_p(self.settings['CACHE_DIRECTORY']) + mkdir_p(self.settings['CACHE_PATH']) with self._cache_open(self._cache_path, 'wb') as fhandle: pickle.dump(self._cache, fhandle) except (IOError, OSError, pickle.PicklingError) as err: