diff --git a/pelican/__init__.py b/pelican/__init__.py index 70013804..097cc200 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -141,7 +141,7 @@ class Pelican(object): ) self.settings[old] = self.settings[new] - def run(self): + def run(self, modified=None): """Run the generators and return""" start_time = time.time() @@ -164,11 +164,13 @@ class Pelican(object): # explicitly asked if (self.delete_outputdir and not os.path.realpath(self.path).startswith(self.output_path)): - clean_output_dir(self.output_path, self.output_retention) + clean_output_dir(self.output_path, + self.output_retention, + files_to_clean=modified) for p in generators: if hasattr(p, 'generate_context'): - p.generate_context() + p.generate_context(modified=modified) signals.all_generators_finalized.send(generators) @@ -462,7 +464,9 @@ def main(): logger.warning('Empty theme folder. Using `basic` ' 'theme.') - pelican.run() + modified_files = [v for vals in list(modified.values()) + for v in vals] + pelican.run(modified=modified_files) except KeyboardInterrupt: logger.warning("Keyboard interrupt, quitting.") @@ -484,6 +488,8 @@ def main(): if next(watchers['theme']) is None: logger.warning('Empty theme folder. Using `basic` theme.') + modified_files = [val for vals in list(modified.values()) + for val in vals] pelican.run() except Exception as e: diff --git a/pelican/generators.py b/pelican/generators.py index 88752392..9043594c 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -94,6 +94,19 @@ class Generator(object): name, self._templates_path)) return self._templates[name] + def _filter_child_paths(self, ancestor_paths, child_paths): + """Return set of elements in child_paths which are children of at least + one of ancestor_paths. + """ + paths = set() + for ancestor_path in ancestor_paths: + ancestor_path = os.path.join(self.path, ancestor_path) + paths.update( + filter(lambda p: os.path.realpath(p).startswith(ancestor_path), + child_paths) + ) + return paths + def _include_path(self, path, extensions=None): """Inclusion logic for .get_files(), returns True/False @@ -501,14 +514,17 @@ class ArticlesGenerator(CachingGenerator): self.generate_authors(write) self.generate_drafts(write) - def generate_context(self): + def generate_context(self, modified=None): """Add the articles into the shared context""" all_articles = [] all_drafts = [] - for f in self.get_files( - self.settings['ARTICLE_PATHS'], - exclude=self.settings['ARTICLE_EXCLUDES']): + files = self.get_files( + (self.settings['ARTICLE_PATHS'] if modified is None + else self._filter_child_paths(self.settings['ARTICLE_PATHS'], + modified)), + exclude=self.settings['ARTICLE_EXCLUDES']) + for f in files: article_or_draft = self.get_cached_data(f, None) if article_or_draft is None: # TODO needs overhaul, maybe nomad for read_file @@ -611,12 +627,15 @@ class PagesGenerator(CachingGenerator): super(PagesGenerator, self).__init__(*args, **kwargs) signals.page_generator_init.send(self) - def generate_context(self): + def generate_context(self, modified=None): all_pages = [] hidden_pages = [] - for f in self.get_files( - self.settings['PAGE_PATHS'], - exclude=self.settings['PAGE_EXCLUDES']): + files = self.get_files( + (self.settings['PAGE_PATHS'] if modified is None + else self._filter_child_paths(self.settings['PAGE_PATHS'], + modified)), + exclude=self.settings['PAGE_EXCLUDES']) + for f in files: page = self.get_cached_data(f, None) if page is None: try: @@ -697,11 +716,15 @@ class StaticGenerator(Generator): os.path.join(output_path, destination, path), self.settings['IGNORE_FILES']) - def generate_context(self): + def generate_context(self, modified=None): self.staticfiles = [] - for f in self.get_files(self.settings['STATIC_PATHS'], - exclude=self.settings['STATIC_EXCLUDES'], - extensions=False): + files = self.get_files( + (self.settings['STATIC_PATHS'] if modified is None + else self._filter_child_paths(self.settings['STATIC_PATHS'], + modified)), + exclude=self.settings['STATIC_EXCLUDES'], + extensions=False) + for f in files: # skip content source files unless the user explicitly wants them if self.settings['STATIC_EXCLUDE_SOURCES']: @@ -735,7 +758,8 @@ class StaticGenerator(Generator): class SourceFileGenerator(Generator): - def generate_context(self): + def generate_context(self, modified=None): + """`modified` is unused; only here for consistency.""" self.output_extension = self.settings['OUTPUT_SOURCES_EXTENSION'] def _create_source(self, obj): diff --git a/pelican/tests/test_utils.py b/pelican/tests/test_utils.py index 9a7109d6..bbbd65dd 100644 --- a/pelican/tests/test_utils.py +++ b/pelican/tests/test_utils.py @@ -286,22 +286,22 @@ class TestUtils(LoggedTestCase): file_watcher = utils.file_watcher(path) # first check returns True - self.assertEqual(next(folder_watcher), True) - self.assertEqual(next(file_watcher), True) + self.assertTrue(next(folder_watcher)) + self.assertTrue(next(file_watcher)) # next check without modification returns False - self.assertEqual(next(folder_watcher), False) - self.assertEqual(next(file_watcher), False) + self.assertFalse(next(folder_watcher)) + self.assertFalse(next(file_watcher)) # after modification, returns True t = time.time() os.utime(path, (t, t)) - self.assertEqual(next(folder_watcher), True) - self.assertEqual(next(file_watcher), True) + self.assertTrue(next(folder_watcher)) + self.assertTrue(next(file_watcher)) # file watcher with None or empty path should return None - self.assertEqual(next(utils.file_watcher('')), None) - self.assertEqual(next(utils.file_watcher(None)), None) + self.assertIsNone(next(utils.file_watcher(''))) + self.assertIsNone(next(utils.file_watcher(None))) empty_path = os.path.join(os.path.dirname(__file__), 'empty') try: @@ -309,9 +309,9 @@ class TestUtils(LoggedTestCase): os.mkdir(os.path.join(empty_path, "empty_folder")) shutil.copy(__file__, empty_path) - # if no files of interest, returns None + # if no files of interest, returns empty list watcher = utils.folder_watcher(empty_path, ['rst']) - self.assertEqual(next(watcher), None) + self.assertEqual(next(watcher), []) except OSError: self.fail("OSError Exception in test_files_changed test") finally: diff --git a/pelican/utils.py b/pelican/utils.py index ef9da23b..ee3c6f6a 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -398,8 +398,11 @@ def copy_file_metadata(source, destination): source, destination, e) -def clean_output_dir(path, retention): - """Remove all files from output directory except those in retention list""" +def clean_output_dir(path, retention, files_to_clean=None): + """Remove all files from output directory except those in retention list. + If files_to_clean is provided, only clean these files (but still skip over + the ones in retention list). + """ if not os.path.exists(path): logger.debug("Directory already removed: %s", path) @@ -414,6 +417,8 @@ def clean_output_dir(path, retention): # remove existing content from output folder unless in retention list for filename in os.listdir(path): + if files_to_clean is not None and filename not in files_to_clean: + continue file = os.path.join(path, filename) if any(filename == retain for retain in retention): logger.debug("Skipping deletion; %s is on retention list: %s", @@ -727,11 +732,13 @@ def process_translations(content_list, order_by=None): def folder_watcher(path, extensions, ignores=[]): '''Generator for monitoring a folder for modifications. - Returns a boolean indicating if files are changed since last check. + Returns a list indicating the files that were changed since last check. Returns None if there are no matching files in the folder''' - def file_times(path): - '''Return `mtime` for each file in path''' + def file_times(path, after=0): + '''Return a (`mtime`, `file_path`) tuple for each file in path. + If `after` kwarg is provided, only return files with `mtime` > `after`. + ''' for root, dirs, files in os.walk(path, followlinks=True): dirs[:] = [x for x in dirs if not x.startswith(os.curdir)] @@ -740,21 +747,24 @@ def folder_watcher(path, extensions, ignores=[]): if f.endswith(tuple(extensions)) and \ not any(fnmatch.fnmatch(f, ignore) for ignore in ignores): try: - yield os.stat(os.path.join(root, f)).st_mtime + mtime = os.stat(os.path.join(root, f)).st_mtime + if mtime > after: + yield (mtime, os.path.join(root, f)) except OSError as e: logger.warning('Caught Exception: %s', e) LAST_MTIME = 0 while True: try: - mtime = max(file_times(path)) - if mtime > LAST_MTIME: - LAST_MTIME = mtime - yield True + modified_files = sorted(file_times(path, after=LAST_MTIME), + reverse=True) + if modified_files: + LAST_MTIME = modified_files[0][0] + yield [mf[1] for mf in modified_files] except ValueError: yield None else: - yield False + yield [] def file_watcher(path): @@ -770,9 +780,9 @@ def file_watcher(path): if mtime > LAST_MTIME: LAST_MTIME = mtime - yield True + yield path else: - yield False + yield '' else: yield None