diff --git a/pelican/__init__.py b/pelican/__init__.py index 09ecf26a..6469c607 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -24,8 +24,7 @@ from pelican.plugins._utils import load_plugins from pelican.readers import Readers from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer from pelican.settings import read_settings -from pelican.utils import (clean_output_dir, file_watcher, - folder_watcher, maybe_pluralize) +from pelican.utils import (FileSystemWatcher, clean_output_dir, maybe_pluralize) from pelican.writers import Writer try: @@ -381,65 +380,36 @@ def get_instance(args): return cls(settings), settings -def autoreload(watchers, args, old_static, reader_descs, excqueue=None): +def autoreload(args, excqueue=None): + print(' --- AutoReload Mode: Monitoring `content`, `theme` and' + ' `settings` for changes. ---') + pelican, settings = get_instance(args) + watcher = FileSystemWatcher(args.settings, Readers, settings) + sleep = False while True: try: - # Check source dir for changed files ending with the given - # extension in the settings. In the theme dir is no such - # restriction; all files are recursively checked if they - # have changed, no matter what extension the filenames - # have. - modified = {k: next(v) for k, v in watchers.items()} + # Don't sleep first time, but sleep afterwards to reduce cpu load + if sleep: + time.sleep(0.5) + else: + sleep = True + + modified = watcher.check() if modified['settings']: pelican, settings = get_instance(args) - - # Adjust static watchers if there are any changes - new_static = settings.get("STATIC_PATHS", []) - - # Added static paths - # Add new watchers and set them as modified - new_watchers = set(new_static).difference(old_static) - for static_path in new_watchers: - static_key = '[static]%s' % static_path - watchers[static_key] = folder_watcher( - os.path.join(pelican.path, static_path), - [''], - pelican.ignore_files) - modified[static_key] = next(watchers[static_key]) - - # Removed static paths - # Remove watchers and modified values - old_watchers = set(old_static).difference(new_static) - for static_path in old_watchers: - static_key = '[static]%s' % static_path - watchers.pop(static_key) - modified.pop(static_key) - - # Replace old_static with the new one - old_static = new_static + watcher.update_watchers(settings) if any(modified.values()): print('\n-> Modified: {}. re-generating...'.format( ', '.join(k for k, v in modified.items() if v))) - - if modified['content'] is None: - logger.warning( - 'No valid files found in content for ' - + 'the active readers:\n' - + '\n'.join(reader_descs)) - - if modified['theme'] is None: - logger.warning('Empty theme folder. Using `basic` ' - 'theme.') - pelican.run() - except KeyboardInterrupt as e: - logger.warning("Keyboard interrupt, quitting.") + except KeyboardInterrupt: if excqueue is not None: - excqueue.put(traceback.format_exception_only(type(e), e)[-1]) - return + excqueue.put(None) + return + raise except Exception as e: if (args.verbosity == logging.DEBUG): @@ -449,10 +419,8 @@ def autoreload(watchers, args, old_static, reader_descs, excqueue=None): else: raise logger.warning( - 'Caught exception "%s". Reloading.', e) - - finally: - time.sleep(.5) # sleep to avoid cpu load + 'Caught exception:\n"%s".', e, + exc_info=settings.get('DEBUG', False)) def listen(server, port, output, excqueue=None): @@ -476,8 +444,10 @@ def listen(server, port, output, excqueue=None): return except KeyboardInterrupt: - print("\nKeyboard interrupt received. Shutting down server.") httpd.socket.close() + if excqueue is not None: + return + raise def main(argv=None): @@ -492,37 +462,11 @@ def main(argv=None): try: pelican, settings = get_instance(args) - readers = Readers(settings) - reader_descs = sorted( - { - '%s (%s)' % (type(r).__name__, ', '.join(r.file_extensions)) - for r in readers.readers.values() - if r.enabled - } - ) - - watchers = {'content': folder_watcher(pelican.path, - readers.extensions, - pelican.ignore_files), - 'theme': folder_watcher(pelican.theme, - [''], - pelican.ignore_files), - 'settings': file_watcher(args.settings)} - - old_static = settings.get("STATIC_PATHS", []) - for static_path in old_static: - # use a prefix to avoid possible overriding of standard watchers - # above - watchers['[static]%s' % static_path] = folder_watcher( - os.path.join(pelican.path, static_path), - [''], - pelican.ignore_files) - if args.autoreload and args.listen: excqueue = multiprocessing.Queue() p1 = multiprocessing.Process( target=autoreload, - args=(watchers, args, old_static, reader_descs, excqueue)) + args=(args, excqueue)) p2 = multiprocessing.Process( target=listen, args=(settings.get('BIND'), settings.get('PORT'), @@ -532,26 +476,19 @@ def main(argv=None): exc = excqueue.get() p1.terminate() p2.terminate() - logger.critical(exc) + if exc is not None: + logger.critical(exc) elif args.autoreload: - print(' --- AutoReload Mode: Monitoring `content`, `theme` and' - ' `settings` for changes. ---') - autoreload(watchers, args, old_static, reader_descs) + autoreload(args) elif args.listen: listen(settings.get('BIND'), settings.get('PORT'), settings.get("OUTPUT_PATH")) else: - if next(watchers['content']) is None: - logger.warning( - 'No valid files found in content for ' - + 'the active readers:\n' - + '\n'.join(reader_descs)) - - if next(watchers['theme']) is None: - logger.warning('Empty theme folder. Using `basic` theme.') - + watcher = FileSystemWatcher(args.settings, Readers, settings) + watcher.check() pelican.run() - + except KeyboardInterrupt: + logger.warning('Keyboard interrupt received. Exiting.') except Exception as e: logger.critical('%s', e) diff --git a/pelican/tests/test_utils.py b/pelican/tests/test_utils.py index f0bc290d..ed205e6c 100644 --- a/pelican/tests/test_utils.py +++ b/pelican/tests/test_utils.py @@ -10,6 +10,7 @@ import pytz from pelican import utils from pelican.generators import TemplatePagesGenerator +from pelican.readers import Readers from pelican.settings import read_settings from pelican.tests.support import (LoggedTestCase, get_article, locale_available, unittest) @@ -361,47 +362,91 @@ class TestUtils(LoggedTestCase): self.assertNotIn(a_arts[4], b_arts[5].translations) self.assertNotIn(a_arts[5], b_arts[4].translations) - def test_watchers(self): - # Test if file changes are correctly detected - # Make sure to handle not getting any files correctly. + def test_filesystemwatcher(self): + def create_file(name, content): + with open(name, 'w') as f: + f.write(content) - dirname = os.path.join(os.path.dirname(__file__), 'content') - folder_watcher = utils.folder_watcher(dirname, ['rst']) + # disable logger filter + from pelican.utils import logger + logger.disable_filter() - path = os.path.join(dirname, 'article_with_metadata.rst') - file_watcher = utils.file_watcher(path) + # create a temp "project" dir + root = mkdtemp() + content_path = os.path.join(root, 'content') + static_path = os.path.join(root, 'content', 'static') + config_file = os.path.join(root, 'config.py') + theme_path = os.path.join(root, 'mytheme') - # first check returns True - self.assertEqual(next(folder_watcher), True) - self.assertEqual(next(file_watcher), True) + # populate + os.mkdir(content_path) + os.mkdir(theme_path) + create_file(config_file, + 'PATH = "content"\n' + 'THEME = "mytheme"\n' + 'STATIC_PATHS = ["static"]') - # next check without modification returns False - self.assertEqual(next(folder_watcher), False) - self.assertEqual(next(file_watcher), False) + t = time.time() - 1000 # make sure it's in the "past" + os.utime(config_file, (t, t)) + settings = read_settings(config_file) - # after modification, returns True - t = time.time() - os.utime(path, (t, t)) - self.assertEqual(next(folder_watcher), True) - self.assertEqual(next(file_watcher), True) + watcher = utils.FileSystemWatcher(config_file, Readers, settings) + # should get a warning for static not not existing + self.assertLogCountEqual(1, 'Watched path does not exist: .*static') - # file watcher with None or empty path should return None - self.assertEqual(next(utils.file_watcher('')), None) - self.assertEqual(next(utils.file_watcher(None)), None) + # create it and update config + os.mkdir(static_path) + watcher.update_watchers(settings) + # no new warning + self.assertLogCountEqual(1, 'Watched path does not exist: .*static') - empty_path = os.path.join(os.path.dirname(__file__), 'empty') - try: - os.mkdir(empty_path) - os.mkdir(os.path.join(empty_path, "empty_folder")) - shutil.copy(__file__, empty_path) + # get modified values + modified = watcher.check() + # empty theme and content should raise warnings + self.assertLogCountEqual(1, 'No valid files found in content') + self.assertLogCountEqual(1, 'Empty theme folder. Using `basic` theme') - # if no files of interest, returns None - watcher = utils.folder_watcher(empty_path, ['rst']) - self.assertEqual(next(watcher), None) - except OSError: - self.fail("OSError Exception in test_files_changed test") - finally: - shutil.rmtree(empty_path, True) + self.assertIsNone(modified['content']) # empty + self.assertIsNone(modified['theme']) # empty + self.assertIsNone(modified['[static]static']) # empty + self.assertTrue(modified['settings']) # modified, first time + + # add a content, add file to theme and check again + create_file(os.path.join(content_path, 'article.md'), + 'Title: test\n' + 'Date: 01-01-2020') + + create_file(os.path.join(theme_path, 'dummy'), + 'test') + + modified = watcher.check() + # no new warning + self.assertLogCountEqual(1, 'No valid files found in content') + self.assertLogCountEqual(1, 'Empty theme folder. Using `basic` theme') + + self.assertIsNone(modified['[static]static']) # empty + self.assertFalse(modified['settings']) # not modified + self.assertTrue(modified['theme']) # modified + self.assertTrue(modified['content']) # modified + + # change config, remove static path + create_file(config_file, + 'PATH = "content"\n' + 'THEME = "mytheme"\n' + 'STATIC_PATHS = []') + + settings = read_settings(config_file) + watcher.update_watchers(settings) + + modified = watcher.check() + self.assertNotIn('[static]static', modified) # should be gone + self.assertTrue(modified['settings']) # modified + self.assertFalse(modified['content']) # not modified + self.assertFalse(modified['theme']) # not modified + + # cleanup + logger.enable_filter() + shutil.rmtree(root) def test_clean_output_dir(self): retention = () diff --git a/pelican/utils.py b/pelican/utils.py index 1f3d502c..9b414863 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -734,60 +734,167 @@ def order_content(content_list, order_by='slug'): return content_list -def folder_watcher(path, extensions, ignores=[]): - '''Generator for monitoring a folder for modifications. +class FileSystemWatcher: + def __init__(self, settings_file, reader_class, settings=None): + self.watchers = { + 'settings': FileSystemWatcher.file_watcher(settings_file) + } - Returns a boolean indicating if files are changed since last check. - Returns None if there are no matching files in the folder''' + self.settings = None + self.reader_class = reader_class + self._extensions = None + self._content_path = None + self._theme_path = None + self._ignore_files = None - def file_times(path): - '''Return `mtime` for each file in path''' + if settings is not None: + self.update_watchers(settings) - for root, dirs, files in os.walk(path, followlinks=True): - dirs[:] = [x for x in dirs if not x.startswith(os.curdir)] + def update_watchers(self, settings): + new_extensions = set(self.reader_class(settings).extensions) + new_content_path = settings.get('PATH', '') + new_theme_path = settings.get('THEME', '') + new_ignore_files = set(settings.get('IGNORE_FILES', [])) - for f in files: - valid_extension = f.endswith(tuple(extensions)) - file_ignored = any( - fnmatch.fnmatch(f, ignore) for ignore in ignores - ) - if valid_extension and not file_ignored: - try: - yield os.stat(os.path.join(root, f)).st_mtime - except OSError as e: - logger.warning('Caught Exception: %s', e) + extensions_changed = new_extensions != self._extensions + content_changed = new_content_path != self._content_path + theme_changed = new_theme_path != self._theme_path + ignore_changed = new_ignore_files != self._ignore_files - LAST_MTIME = 0 - while True: - try: - mtime = max(file_times(path)) - if mtime > LAST_MTIME: - LAST_MTIME = mtime - yield True - except ValueError: - yield None + # Refresh content watcher if related settings changed + if extensions_changed or content_changed or ignore_changed: + self.add_watcher('content', + new_content_path, + new_extensions, + new_ignore_files) + + # Refresh theme watcher if related settings changed + if theme_changed or ignore_changed: + self.add_watcher('theme', + new_theme_path, + [''], + new_ignore_files) + + # Watch STATIC_PATHS + old_static_watchers = set(key + for key in self.watchers + if key.startswith('[static]')) + + for path in settings.get('STATIC_PATHS', []): + key = '[static]{}'.format(path) + if ignore_changed or (key not in self.watchers): + self.add_watcher( + key, + os.path.join(new_content_path, path), + [''], + new_ignore_files) + if key in old_static_watchers: + old_static_watchers.remove(key) + + # cleanup removed static watchers + for key in old_static_watchers: + del self.watchers[key] + + # update values + self.settings = settings + self._extensions = new_extensions + self._content_path = new_content_path + self._theme_path = new_theme_path + self._ignore_files = new_ignore_files + + def check(self): + '''return a key:watcher_status dict for all watchers''' + result = {key: next(watcher) for key, watcher in self.watchers.items()} + + # Various warnings + if result.get('content') is None: + reader_descs = sorted( + { + '%s (%s)' % (type(r).__name__, ', '.join(r.file_extensions)) + for r in self.reader_class(self.settings).readers.values() + if r.enabled + } + ) + logger.warning( + 'No valid files found in content for the active readers:\n' + + '\n'.join(reader_descs)) + + if result.get('theme') is None: + logger.warning('Empty theme folder. Using `basic` theme.') + + return result + + def add_watcher(self, key, path, extensions=[''], ignores=[]): + watcher = self.get_watcher(path, extensions, ignores) + if watcher is not None: + self.watchers[key] = watcher + + def get_watcher(self, path, extensions=[''], ignores=[]): + '''return a watcher depending on path type (file or folder)''' + if not os.path.exists(path): + logger.warning("Watched path does not exist: %s", path) + return None + + if os.path.isdir(path): + return self.folder_watcher(path, extensions, ignores) else: - yield False + return self.file_watcher(path) + @staticmethod + def folder_watcher(path, extensions, ignores=[]): + '''Generator for monitoring a folder for modifications. -def file_watcher(path): - '''Generator for monitoring a file for modifications''' - LAST_MTIME = 0 - while True: - if path: + Returns a boolean indicating if files are changed since last check. + Returns None if there are no matching files in the folder''' + + def file_times(path): + '''Return `mtime` for each file in path''' + + for root, dirs, files in os.walk(path, followlinks=True): + dirs[:] = [x for x in dirs if not x.startswith(os.curdir)] + + for f in files: + valid_extension = f.endswith(tuple(extensions)) + file_ignored = any( + fnmatch.fnmatch(f, ignore) for ignore in ignores + ) + if valid_extension and not file_ignored: + try: + yield os.stat(os.path.join(root, f)).st_mtime + except OSError as e: + logger.warning('Caught Exception: %s', e) + + LAST_MTIME = 0 + while True: try: - mtime = os.stat(path).st_mtime - except OSError as e: - logger.warning('Caught Exception: %s', e) - continue - - if mtime > LAST_MTIME: - LAST_MTIME = mtime - yield True + mtime = max(file_times(path)) + if mtime > LAST_MTIME: + LAST_MTIME = mtime + yield True + except ValueError: + yield None else: yield False - else: - yield None + + @staticmethod + def file_watcher(path): + '''Generator for monitoring a file for modifications''' + LAST_MTIME = 0 + while True: + if path: + try: + mtime = os.stat(path).st_mtime + except OSError as e: + logger.warning('Caught Exception: %s', e) + continue + + if mtime > LAST_MTIME: + LAST_MTIME = mtime + yield True + else: + yield False + else: + yield None def set_date_tzinfo(d, tz_name=None):