Add a caching mechnism

pelican/__init__.py

@@ -13,13 +13,16 @@ import collections
 
 from pelican import signals
 
-from pelican.generators import (ArticlesGenerator, PagesGenerator,
+from pelican.generators import (
-                                StaticGenerator, SourceFileGenerator,
+    ArticlesGenerator, PagesGenerator, StaticGenerator, SourceFileGenerator,
-                                TemplatePagesGenerator)
+    TemplatePagesGenerator
+)
 from pelican.log import init
 from pelican.readers import Readers
 from pelican.settings import read_settings
-from pelican.utils import clean_output_dir, folder_watcher, file_watcher
+from pelican.utils import (
+    clean_output_dir, folder_watcher, file_watcher, mkdir_p
+)
 from pelican.writers import Writer
 
 __version__ = "3.3.1.dev"
@@ -48,10 +51,17 @@ class Pelican(object):
         self.delete_outputdir = settings['DELETE_OUTPUT_DIRECTORY']
         self.output_retention = settings['OUTPUT_RETENTION']
 
+        self.init_filesystem()
         self.init_path()
         self.init_plugins()
         signals.initialized.send(self)
 
+    def init_filesystem(self):
+        cache_dir = self.settings['CACHE_PATH']
+        if self.settings['USE_CACHE'] and not os.path.exists(cache_dir):
+            logger.debug('Creating directory {0}'.format(cache_dir))
+            mkdir_p(cache_dir)
+
     def init_path(self):
         if not any(p in sys.path for p in ['', os.curdir]):
             logger.debug("Adding current directory to system path")
@@ -261,6 +271,9 @@ def parse_arguments():
         action='store_true',
         help="Relaunch pelican each time a modification occurs"
                              " on the content files.")
+
+    parser.add_argument('--cache', dest='use_cache', action='store_true',
+        help='Cache the file rendering between runs')
     return parser.parse_args()
 
 
@@ -276,6 +289,8 @@ def get_config(args):
         config['THEME'] = abstheme if os.path.exists(abstheme) else args.theme
     if args.delete_outputdir is not None:
         config['DELETE_OUTPUT_DIRECTORY'] = args.delete_outputdir
+    if args.use_cache:
+        config['USE_CACHE'] = True
 
     # argparse returns bytes in Py2. There is no definite answer as to which
     # encoding argparse (or sys.argv) uses.

pelican/cache.py

@@ -0,0 +1,37 @@
+import os.path
+import hashlib
+import pickle
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class CachedReader(object):
+
+    def __init__(self, reader, cache_path):
+        self._reader = reader
+        self._cache_path = cache_path
+
+    def process_metadata(self, *args, **kwargs):
+        return self._reader.process_metadata(*args, **kwargs)
+
+    def read(self, path):
+        mtime = os.stat(path).st_mtime
+
+        m = hashlib.md5()
+        # We want to hash path + mtime
+        m.update(path)
+        m.update(str(mtime))
+        hash_ = m.hexdigest()
+
+        cache_file = os.path.join(self._cache_path, hash_)
+        if os.path.exists(cache_file):
+            logger.debug('reading {0} from cache'.format(path))
+            with open(cache_file) as f:
+                content, metadata = pickle.load(f)
+        else:
+            content, metadata = self._reader.read(path)
+            with open(cache_file, 'w+') as f:
+                pickle.dump((content, metadata), f)
+            logger.debug('stored {0} in the cache'.format(path))
+        return content, metadata

pelican/readers.py

@@ -35,6 +35,7 @@ except ImportError:
     from HTMLParser import HTMLParser
 
 from pelican import signals
+from pelican.cache import CachedReader
 from pelican.contents import Page, Category, Tag, Author
 from pelican.utils import get_date, pelican_open
 
@@ -435,6 +436,12 @@ class Readers(object):
 
         reader = self.readers[fmt]
 
+        if self.settings['USE_CACHE']:
+            # If we are using a cache, then the reader class should be a cached
+            # one.
+            reader = CachedReader(reader=reader,
+                                  cache_path=self.settings['CACHE_PATH'])
+
         metadata = default_metadata(
             settings=self.settings, process=reader.process_metadata)
         metadata.update(path_metadata(

pelican/settings.py

@@ -112,6 +112,8 @@ DEFAULT_CONFIG = {
     'IGNORE_FILES': ['.#*'],
     'SLUG_SUBSTITUTIONS': (),
     'INTRASITE_LINK_REGEX': '[{|](?P<what>.*?)[|}]',
+    'CACHE_PATH': '_pelican_cache',
+    'USE_CACHE': False,
     }
 
 PYGMENTS_RST_OPTIONS = None
@@ -121,7 +123,7 @@ def read_settings(path=None, override=None):
     if path:
         local_settings = get_settings_from_file(path)
         # Make the paths relative to the settings file
-        for p in ['PATH', 'OUTPUT_PATH', 'THEME', 'PLUGIN_PATH']:
+        for p in ['PATH', 'OUTPUT_PATH', 'THEME', 'PLUGIN_PATH', 'CACHE_PATH']:
             if p in local_settings and local_settings[p] is not None \
                     and not isabs(local_settings[p]):
                 absp = os.path.abspath(os.path.normpath(os.path.join(