Merge pull request #2750 from avaris/autoreload

Refactor file/folder watchers and autoreload

pelican/__init__.py

@@ -24,8 +24,7 @@ from pelican.plugins._utils import load_plugins
 from pelican.readers import Readers
 from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer
 from pelican.settings import read_settings
-from pelican.utils import (clean_output_dir, file_watcher,
+from pelican.utils import (FileSystemWatcher, clean_output_dir, maybe_pluralize)
-                           folder_watcher, maybe_pluralize)
 from pelican.writers import Writer
 
 try:
@@ -381,65 +380,36 @@ def get_instance(args):
     return cls(settings), settings
 
 
-def autoreload(watchers, args, old_static, reader_descs, excqueue=None):
+def autoreload(args, excqueue=None):
+    print('  --- AutoReload Mode: Monitoring `content`, `theme` and'
+          ' `settings` for changes. ---')
+    pelican, settings = get_instance(args)
+    watcher = FileSystemWatcher(args.settings, Readers, settings)
+    sleep = False
     while True:
         try:
-            # Check source dir for changed files ending with the given
+            # Don't sleep first time, but sleep afterwards to reduce cpu load
-            # extension in the settings. In the theme dir is no such
+            if sleep:
-            # restriction; all files are recursively checked if they
+                time.sleep(0.5)
-            # have changed, no matter what extension the filenames
+            else:
-            # have.
+                sleep = True
-            modified = {k: next(v) for k, v in watchers.items()}
+
+            modified = watcher.check()
 
             if modified['settings']:
                 pelican, settings = get_instance(args)
-
+                watcher.update_watchers(settings)
-                # Adjust static watchers if there are any changes
-                new_static = settings.get("STATIC_PATHS", [])
-
-                # Added static paths
-                # Add new watchers and set them as modified
-                new_watchers = set(new_static).difference(old_static)
-                for static_path in new_watchers:
-                    static_key = '[static]%s' % static_path
-                    watchers[static_key] = folder_watcher(
-                        os.path.join(pelican.path, static_path),
-                        [''],
-                        pelican.ignore_files)
-                    modified[static_key] = next(watchers[static_key])
-
-                # Removed static paths
-                # Remove watchers and modified values
-                old_watchers = set(old_static).difference(new_static)
-                for static_path in old_watchers:
-                    static_key = '[static]%s' % static_path
-                    watchers.pop(static_key)
-                    modified.pop(static_key)
-
-                # Replace old_static with the new one
-                old_static = new_static
 
             if any(modified.values()):
                 print('\n-> Modified: {}. re-generating...'.format(
                     ', '.join(k for k, v in modified.items() if v)))
-
-                if modified['content'] is None:
-                    logger.warning(
-                        'No valid files found in content for '
-                        + 'the active readers:\n'
-                        + '\n'.join(reader_descs))
-
-                if modified['theme'] is None:
-                    logger.warning('Empty theme folder. Using `basic` '
-                                   'theme.')
-
                 pelican.run()
 
-        except KeyboardInterrupt as e:
+        except KeyboardInterrupt:
-            logger.warning("Keyboard interrupt, quitting.")
             if excqueue is not None:
-                excqueue.put(traceback.format_exception_only(type(e), e)[-1])
+                excqueue.put(None)
-            return
+                return
+            raise
 
         except Exception as e:
             if (args.verbosity == logging.DEBUG):
@@ -449,10 +419,8 @@ def autoreload(watchers, args, old_static, reader_descs, excqueue=None):
                 else:
                     raise
             logger.warning(
-                'Caught exception "%s". Reloading.', e)
+                'Caught exception:\n"%s".', e,
-
+                exc_info=settings.get('DEBUG', False))
-        finally:
-            time.sleep(.5)  # sleep to avoid cpu load
 
 
 def listen(server, port, output, excqueue=None):
@@ -476,8 +444,10 @@ def listen(server, port, output, excqueue=None):
         return
 
     except KeyboardInterrupt:
-        print("\nKeyboard interrupt received. Shutting down server.")
         httpd.socket.close()
+        if excqueue is not None:
+            return
+        raise
 
 
 def main(argv=None):
@@ -492,37 +462,11 @@ def main(argv=None):
     try:
         pelican, settings = get_instance(args)
 
-        readers = Readers(settings)
-        reader_descs = sorted(
-            {
-                '%s (%s)' % (type(r).__name__, ', '.join(r.file_extensions))
-                for r in readers.readers.values()
-                if r.enabled
-            }
-        )
-
-        watchers = {'content': folder_watcher(pelican.path,
-                                              readers.extensions,
-                                              pelican.ignore_files),
-                    'theme': folder_watcher(pelican.theme,
-                                            [''],
-                                            pelican.ignore_files),
-                    'settings': file_watcher(args.settings)}
-
-        old_static = settings.get("STATIC_PATHS", [])
-        for static_path in old_static:
-            # use a prefix to avoid possible overriding of standard watchers
-            # above
-            watchers['[static]%s' % static_path] = folder_watcher(
-                os.path.join(pelican.path, static_path),
-                [''],
-                pelican.ignore_files)
-
         if args.autoreload and args.listen:
             excqueue = multiprocessing.Queue()
             p1 = multiprocessing.Process(
                 target=autoreload,
-                args=(watchers, args, old_static, reader_descs, excqueue))
+                args=(args, excqueue))
             p2 = multiprocessing.Process(
                 target=listen,
                 args=(settings.get('BIND'), settings.get('PORT'),
@@ -532,26 +476,19 @@ def main(argv=None):
             exc = excqueue.get()
             p1.terminate()
             p2.terminate()
-            logger.critical(exc)
+            if exc is not None:
+                logger.critical(exc)
         elif args.autoreload:
-            print('  --- AutoReload Mode: Monitoring `content`, `theme` and'
+            autoreload(args)
-                  ' `settings` for changes. ---')
-            autoreload(watchers, args, old_static, reader_descs)
         elif args.listen:
             listen(settings.get('BIND'), settings.get('PORT'),
                    settings.get("OUTPUT_PATH"))
         else:
-            if next(watchers['content']) is None:
+            watcher = FileSystemWatcher(args.settings, Readers, settings)
-                logger.warning(
+            watcher.check()
-                    'No valid files found in content for '
-                    + 'the active readers:\n'
-                    + '\n'.join(reader_descs))
-
-            if next(watchers['theme']) is None:
-                logger.warning('Empty theme folder. Using `basic` theme.')
-
             pelican.run()
-
+    except KeyboardInterrupt:
+        logger.warning('Keyboard interrupt received. Exiting.')
     except Exception as e:
         logger.critical('%s', e)
 

pelican/tests/test_utils.py

@@ -10,6 +10,7 @@ import pytz
 
 from pelican import utils
 from pelican.generators import TemplatePagesGenerator
+from pelican.readers import Readers
 from pelican.settings import read_settings
 from pelican.tests.support import (LoggedTestCase, get_article,
                                    locale_available, unittest)
@@ -361,47 +362,91 @@ class TestUtils(LoggedTestCase):
                             self.assertNotIn(a_arts[4], b_arts[5].translations)
                             self.assertNotIn(a_arts[5], b_arts[4].translations)
 
-    def test_watchers(self):
+    def test_filesystemwatcher(self):
-        # Test if file changes are correctly detected
+        def create_file(name, content):
-        # Make sure to handle not getting any files correctly.
+            with open(name, 'w') as f:
+                f.write(content)
 
-        dirname = os.path.join(os.path.dirname(__file__), 'content')
+        # disable logger filter
-        folder_watcher = utils.folder_watcher(dirname, ['rst'])
+        from pelican.utils import logger
+        logger.disable_filter()
 
-        path = os.path.join(dirname, 'article_with_metadata.rst')
+        # create a temp "project" dir
-        file_watcher = utils.file_watcher(path)
+        root = mkdtemp()
+        content_path = os.path.join(root, 'content')
+        static_path = os.path.join(root, 'content', 'static')
+        config_file = os.path.join(root, 'config.py')
+        theme_path = os.path.join(root, 'mytheme')
 
-        # first check returns True
+        # populate
-        self.assertEqual(next(folder_watcher), True)
+        os.mkdir(content_path)
-        self.assertEqual(next(file_watcher), True)
+        os.mkdir(theme_path)
+        create_file(config_file,
+                    'PATH = "content"\n'
+                    'THEME = "mytheme"\n'
+                    'STATIC_PATHS = ["static"]')
 
-        # next check without modification returns False
+        t = time.time() - 1000  # make sure it's in the "past"
-        self.assertEqual(next(folder_watcher), False)
+        os.utime(config_file, (t, t))
-        self.assertEqual(next(file_watcher), False)
+        settings = read_settings(config_file)
 
-        # after modification, returns True
+        watcher = utils.FileSystemWatcher(config_file, Readers, settings)
-        t = time.time()
+        # should get a warning for static not not existing
-        os.utime(path, (t, t))
+        self.assertLogCountEqual(1, 'Watched path does not exist: .*static')
-        self.assertEqual(next(folder_watcher), True)
-        self.assertEqual(next(file_watcher), True)
 
-        # file watcher with None or empty path should return None
+        # create it and update config
-        self.assertEqual(next(utils.file_watcher('')), None)
+        os.mkdir(static_path)
-        self.assertEqual(next(utils.file_watcher(None)), None)
+        watcher.update_watchers(settings)
+        # no new warning
+        self.assertLogCountEqual(1, 'Watched path does not exist: .*static')
 
-        empty_path = os.path.join(os.path.dirname(__file__), 'empty')
+        # get modified values
-        try:
+        modified = watcher.check()
-            os.mkdir(empty_path)
+        # empty theme and content should raise warnings
-            os.mkdir(os.path.join(empty_path, "empty_folder"))
+        self.assertLogCountEqual(1, 'No valid files found in content')
-            shutil.copy(__file__, empty_path)
+        self.assertLogCountEqual(1, 'Empty theme folder. Using `basic` theme')
 
-            # if no files of interest, returns None
+        self.assertIsNone(modified['content'])  # empty
-            watcher = utils.folder_watcher(empty_path, ['rst'])
+        self.assertIsNone(modified['theme'])  # empty
-            self.assertEqual(next(watcher), None)
+        self.assertIsNone(modified['[static]static'])  # empty
-        except OSError:
+        self.assertTrue(modified['settings'])  # modified, first time
-            self.fail("OSError Exception in test_files_changed test")
+
-        finally:
+        # add a content, add file to theme and check again
-            shutil.rmtree(empty_path, True)
+        create_file(os.path.join(content_path, 'article.md'),
+                    'Title: test\n'
+                    'Date: 01-01-2020')
+
+        create_file(os.path.join(theme_path, 'dummy'),
+                    'test')
+
+        modified = watcher.check()
+        # no new warning
+        self.assertLogCountEqual(1, 'No valid files found in content')
+        self.assertLogCountEqual(1, 'Empty theme folder. Using `basic` theme')
+
+        self.assertIsNone(modified['[static]static'])  # empty
+        self.assertFalse(modified['settings'])  # not modified
+        self.assertTrue(modified['theme'])  # modified
+        self.assertTrue(modified['content'])  # modified
+
+        # change config, remove static path
+        create_file(config_file,
+                    'PATH = "content"\n'
+                    'THEME = "mytheme"\n'
+                    'STATIC_PATHS = []')
+
+        settings = read_settings(config_file)
+        watcher.update_watchers(settings)
+
+        modified = watcher.check()
+        self.assertNotIn('[static]static', modified)  # should be gone
+        self.assertTrue(modified['settings'])  # modified
+        self.assertFalse(modified['content'])  # not modified
+        self.assertFalse(modified['theme'])  # not modified
+
+        # cleanup
+        logger.enable_filter()
+        shutil.rmtree(root)
 
     def test_clean_output_dir(self):
         retention = ()

pelican/utils.py

@@ -735,60 +735,167 @@ def order_content(content_list, order_by='slug'):
     return content_list
 
 
-def folder_watcher(path, extensions, ignores=[]):
+class FileSystemWatcher:
-    '''Generator for monitoring a folder for modifications.
+    def __init__(self, settings_file, reader_class, settings=None):
+        self.watchers = {
+            'settings': FileSystemWatcher.file_watcher(settings_file)
+        }
 
-    Returns a boolean indicating if files are changed since last check.
+        self.settings = None
-    Returns None if there are no matching files in the folder'''
+        self.reader_class = reader_class
+        self._extensions = None
+        self._content_path = None
+        self._theme_path = None
+        self._ignore_files = None
 
-    def file_times(path):
+        if settings is not None:
-        '''Return `mtime` for each file in path'''
+            self.update_watchers(settings)
 
-        for root, dirs, files in os.walk(path, followlinks=True):
+    def update_watchers(self, settings):
-            dirs[:] = [x for x in dirs if not x.startswith(os.curdir)]
+        new_extensions = set(self.reader_class(settings).extensions)
+        new_content_path = settings.get('PATH', '')
+        new_theme_path = settings.get('THEME', '')
+        new_ignore_files = set(settings.get('IGNORE_FILES', []))
 
-            for f in files:
+        extensions_changed = new_extensions != self._extensions
-                valid_extension = f.endswith(tuple(extensions))
+        content_changed = new_content_path != self._content_path
-                file_ignored = any(
+        theme_changed = new_theme_path != self._theme_path
-                    fnmatch.fnmatch(f, ignore) for ignore in ignores
+        ignore_changed = new_ignore_files != self._ignore_files
-                )
-                if valid_extension and not file_ignored:
-                    try:
-                        yield os.stat(os.path.join(root, f)).st_mtime
-                    except OSError as e:
-                        logger.warning('Caught Exception: %s', e)
 
-    LAST_MTIME = 0
+        # Refresh content watcher if related settings changed
-    while True:
+        if extensions_changed or content_changed or ignore_changed:
-        try:
+            self.add_watcher('content',
-            mtime = max(file_times(path))
+                             new_content_path,
-            if mtime > LAST_MTIME:
+                             new_extensions,
-                LAST_MTIME = mtime
+                             new_ignore_files)
-                yield True
+
-        except ValueError:
+        # Refresh theme watcher if related settings changed
-            yield None
+        if theme_changed or ignore_changed:
+            self.add_watcher('theme',
+                             new_theme_path,
+                             [''],
+                             new_ignore_files)
+
+        # Watch STATIC_PATHS
+        old_static_watchers = set(key
+                                  for key in self.watchers
+                                  if key.startswith('[static]'))
+
+        for path in settings.get('STATIC_PATHS', []):
+            key = '[static]{}'.format(path)
+            if ignore_changed or (key not in self.watchers):
+                self.add_watcher(
+                    key,
+                    os.path.join(new_content_path, path),
+                    [''],
+                    new_ignore_files)
+            if key in old_static_watchers:
+                old_static_watchers.remove(key)
+
+        # cleanup removed static watchers
+        for key in old_static_watchers:
+            del self.watchers[key]
+
+        # update values
+        self.settings = settings
+        self._extensions = new_extensions
+        self._content_path = new_content_path
+        self._theme_path = new_theme_path
+        self._ignore_files = new_ignore_files
+
+    def check(self):
+        '''return a key:watcher_status dict for all watchers'''
+        result = {key: next(watcher) for key, watcher in self.watchers.items()}
+
+        # Various warnings
+        if result.get('content') is None:
+            reader_descs = sorted(
+                {
+                    '%s (%s)' % (type(r).__name__, ', '.join(r.file_extensions))
+                    for r in self.reader_class(self.settings).readers.values()
+                    if r.enabled
+                }
+            )
+            logger.warning(
+                    'No valid files found in content for the active readers:\n'
+                    + '\n'.join(reader_descs))
+
+        if result.get('theme') is None:
+            logger.warning('Empty theme folder. Using `basic` theme.')
+
+        return result
+
+    def add_watcher(self, key, path, extensions=[''], ignores=[]):
+        watcher = self.get_watcher(path, extensions, ignores)
+        if watcher is not None:
+            self.watchers[key] = watcher
+
+    def get_watcher(self, path, extensions=[''], ignores=[]):
+        '''return a watcher depending on path type (file or folder)'''
+        if not os.path.exists(path):
+            logger.warning("Watched path does not exist: %s", path)
+            return None
+
+        if os.path.isdir(path):
+            return self.folder_watcher(path, extensions, ignores)
         else:
-            yield False
+            return self.file_watcher(path)
 
+    @staticmethod
+    def folder_watcher(path, extensions, ignores=[]):
+        '''Generator for monitoring a folder for modifications.
 
-def file_watcher(path):
+        Returns a boolean indicating if files are changed since last check.
-    '''Generator for monitoring a file for modifications'''
+        Returns None if there are no matching files in the folder'''
-    LAST_MTIME = 0
+
-    while True:
+        def file_times(path):
-        if path:
+            '''Return `mtime` for each file in path'''
+
+            for root, dirs, files in os.walk(path, followlinks=True):
+                dirs[:] = [x for x in dirs if not x.startswith(os.curdir)]
+
+                for f in files:
+                    valid_extension = f.endswith(tuple(extensions))
+                    file_ignored = any(
+                        fnmatch.fnmatch(f, ignore) for ignore in ignores
+                    )
+                    if valid_extension and not file_ignored:
+                        try:
+                            yield os.stat(os.path.join(root, f)).st_mtime
+                        except OSError as e:
+                            logger.warning('Caught Exception: %s', e)
+
+        LAST_MTIME = 0
+        while True:
             try:
-                mtime = os.stat(path).st_mtime
+                mtime = max(file_times(path))
-            except OSError as e:
+                if mtime > LAST_MTIME:
-                logger.warning('Caught Exception: %s', e)
+                    LAST_MTIME = mtime
-                continue
+                    yield True
-
+            except ValueError:
-            if mtime > LAST_MTIME:
+                yield None
-                LAST_MTIME = mtime
-                yield True
             else:
                 yield False
-        else:
+
-            yield None
+    @staticmethod
+    def file_watcher(path):
+        '''Generator for monitoring a file for modifications'''
+        LAST_MTIME = 0
+        while True:
+            if path:
+                try:
+                    mtime = os.stat(path).st_mtime
+                except OSError as e:
+                    logger.warning('Caught Exception: %s', e)
+                    continue
+
+                if mtime > LAST_MTIME:
+                    LAST_MTIME = mtime
+                    yield True
+                else:
+                    yield False
+            else:
+                yield None
 
 
 def set_date_tzinfo(d, tz_name=None):