diff --git a/pelican/__init__.py b/pelican/__init__.py index f0af3429..a4f5b38e 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -27,7 +27,7 @@ from pelican.plugins._utils import get_plugin_name, load_plugins from pelican.readers import Readers from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer from pelican.settings import read_settings -from pelican.utils import (FileSystemWatcher, clean_output_dir, maybe_pluralize) +from pelican.utils import (wait_for_changes, clean_output_dir, maybe_pluralize) from pelican.writers import Writer try: @@ -452,26 +452,19 @@ def autoreload(args, excqueue=None): console.print(' --- AutoReload Mode: Monitoring `content`, `theme` and' ' `settings` for changes. ---') pelican, settings = get_instance(args) - watcher = FileSystemWatcher(args.settings, Readers, settings) - sleep = False + settings_file = os.path.abspath(args.settings) while True: try: - # Don't sleep first time, but sleep afterwards to reduce cpu load - if sleep: - time.sleep(0.5) - else: - sleep = True + changed_files = wait_for_changes(args.settings, Readers, settings) - modified = watcher.check() + changed_files = {c[1] for c in changed_files} - if modified['settings']: + if settings_file in changed_files: pelican, settings = get_instance(args) - watcher.update_watchers(settings) - if any(modified.values()): - console.print('\n-> Modified: {}. re-generating...'.format( - ', '.join(k for k, v in modified.items() if v))) - pelican.run() + console.print('\n-> Modified: {}. re-generating...'.format( + ', '.join(changed_files))) + pelican.run() except KeyboardInterrupt: if excqueue is not None: @@ -558,8 +551,6 @@ def main(argv=None): listen(settings.get('BIND'), settings.get('PORT'), settings.get("OUTPUT_PATH")) else: - watcher = FileSystemWatcher(args.settings, Readers, settings) - watcher.check() with console.status("Generating..."): pelican.run() except KeyboardInterrupt: diff --git a/pelican/tests/test_utils.py b/pelican/tests/test_utils.py index e1758726..7ff1af7a 100644 --- a/pelican/tests/test_utils.py +++ b/pelican/tests/test_utils.py @@ -412,92 +412,6 @@ class TestUtils(LoggedTestCase): self.assertNotIn(a_arts[4], b_arts[5].translations) self.assertNotIn(a_arts[5], b_arts[4].translations) - def test_filesystemwatcher(self): - def create_file(name, content): - with open(name, 'w') as f: - f.write(content) - - # disable logger filter - from pelican.utils import logger - logger.disable_filter() - - # create a temp "project" dir - root = mkdtemp() - content_path = os.path.join(root, 'content') - static_path = os.path.join(root, 'content', 'static') - config_file = os.path.join(root, 'config.py') - theme_path = os.path.join(root, 'mytheme') - - # populate - os.mkdir(content_path) - os.mkdir(theme_path) - create_file(config_file, - 'PATH = "content"\n' - 'THEME = "mytheme"\n' - 'STATIC_PATHS = ["static"]') - - t = time.time() - 1000 # make sure it's in the "past" - os.utime(config_file, (t, t)) - settings = read_settings(config_file) - - watcher = utils.FileSystemWatcher(config_file, Readers, settings) - # should get a warning for static not not existing - self.assertLogCountEqual(1, 'Watched path does not exist: .*static') - - # create it and update config - os.mkdir(static_path) - watcher.update_watchers(settings) - # no new warning - self.assertLogCountEqual(1, 'Watched path does not exist: .*static') - - # get modified values - modified = watcher.check() - # empty theme and content should raise warnings - self.assertLogCountEqual(1, 'No valid files found in content') - self.assertLogCountEqual(1, 'Empty theme folder. Using `basic` theme') - - self.assertIsNone(modified['content']) # empty - self.assertIsNone(modified['theme']) # empty - self.assertIsNone(modified['[static]static']) # empty - self.assertTrue(modified['settings']) # modified, first time - - # add a content, add file to theme and check again - create_file(os.path.join(content_path, 'article.md'), - 'Title: test\n' - 'Date: 01-01-2020') - - create_file(os.path.join(theme_path, 'dummy'), - 'test') - - modified = watcher.check() - # no new warning - self.assertLogCountEqual(1, 'No valid files found in content') - self.assertLogCountEqual(1, 'Empty theme folder. Using `basic` theme') - - self.assertIsNone(modified['[static]static']) # empty - self.assertFalse(modified['settings']) # not modified - self.assertTrue(modified['theme']) # modified - self.assertTrue(modified['content']) # modified - - # change config, remove static path - create_file(config_file, - 'PATH = "content"\n' - 'THEME = "mytheme"\n' - 'STATIC_PATHS = []') - - settings = read_settings(config_file) - watcher.update_watchers(settings) - - modified = watcher.check() - self.assertNotIn('[static]static', modified) # should be gone - self.assertTrue(modified['settings']) # modified - self.assertFalse(modified['content']) # not modified - self.assertFalse(modified['theme']) # not modified - - # cleanup - logger.enable_filter() - shutil.rmtree(root) - def test_clean_output_dir(self): retention = () test_directory = os.path.join(self.temp_output, diff --git a/pelican/utils.py b/pelican/utils.py index d8cf15b4..4832e0c1 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -24,6 +24,8 @@ except ModuleNotFoundError: from backports.zoneinfo import ZoneInfo from markupsafe import Markup +import watchfiles + logger = logging.getLogger(__name__) @@ -755,167 +757,31 @@ def order_content(content_list, order_by='slug'): return content_list -class FileSystemWatcher: - def __init__(self, settings_file, reader_class, settings=None): - self.watchers = { - 'settings': FileSystemWatcher.file_watcher(settings_file) - } +def wait_for_changes(settings_file, reader_class, settings): + new_extensions = set(reader_class(settings).extensions) + content_path = settings.get('PATH', '') + theme_path = settings.get('THEME', '') + ignore_files = set(settings.get('IGNORE_FILES', [])) - self.settings = None - self.reader_class = reader_class - self._extensions = None - self._content_path = None - self._theme_path = None - self._ignore_files = None + watching_paths = [ + settings_file, + theme_path, + content_path, + ] - if settings is not None: - self.update_watchers(settings) + watching_paths.extend( + os.path.join(content_path, path) for path in settings.get('STATIC_PATHS', []) + ) - def update_watchers(self, settings): - new_extensions = set(self.reader_class(settings).extensions) - new_content_path = settings.get('PATH', '') - new_theme_path = settings.get('THEME', '') - new_ignore_files = set(settings.get('IGNORE_FILES', [])) + watching_paths = [os.path.abspath(p) for p in watching_paths if p and os.path.exists(p)] - extensions_changed = new_extensions != self._extensions - content_changed = new_content_path != self._content_path - theme_changed = new_theme_path != self._theme_path - ignore_changed = new_ignore_files != self._ignore_files - - # Refresh content watcher if related settings changed - if extensions_changed or content_changed or ignore_changed: - self.add_watcher('content', - new_content_path, - new_extensions, - new_ignore_files) - - # Refresh theme watcher if related settings changed - if theme_changed or ignore_changed: - self.add_watcher('theme', - new_theme_path, - [''], - new_ignore_files) - - # Watch STATIC_PATHS - old_static_watchers = set(key - for key in self.watchers - if key.startswith('[static]')) - - for path in settings.get('STATIC_PATHS', []): - key = '[static]{}'.format(path) - if ignore_changed or (key not in self.watchers): - self.add_watcher( - key, - os.path.join(new_content_path, path), - [''], - new_ignore_files) - if key in old_static_watchers: - old_static_watchers.remove(key) - - # cleanup removed static watchers - for key in old_static_watchers: - del self.watchers[key] - - # update values - self.settings = settings - self._extensions = new_extensions - self._content_path = new_content_path - self._theme_path = new_theme_path - self._ignore_files = new_ignore_files - - def check(self): - '''return a key:watcher_status dict for all watchers''' - result = {key: next(watcher) for key, watcher in self.watchers.items()} - - # Various warnings - if result.get('content') is None: - reader_descs = sorted( - { - ' | %s (%s)' % (type(r).__name__, ', '.join(r.file_extensions)) - for r in self.reader_class(self.settings).readers.values() - if r.enabled - } - ) - logger.warning( - 'No valid files found in content for the active readers:\n' - + '\n'.join(reader_descs)) - - if result.get('theme') is None: - logger.warning('Empty theme folder. Using `basic` theme.') - - return result - - def add_watcher(self, key, path, extensions=[''], ignores=[]): - watcher = self.get_watcher(path, extensions, ignores) - if watcher is not None: - self.watchers[key] = watcher - - def get_watcher(self, path, extensions=[''], ignores=[]): - '''return a watcher depending on path type (file or folder)''' - if not os.path.exists(path): - logger.warning("Watched path does not exist: %s", path) - return None - - if os.path.isdir(path): - return self.folder_watcher(path, extensions, ignores) - else: - return self.file_watcher(path) - - @staticmethod - def folder_watcher(path, extensions, ignores=[]): - '''Generator for monitoring a folder for modifications. - - Returns a boolean indicating if files are changed since last check. - Returns None if there are no matching files in the folder''' - - def file_times(path): - '''Return `mtime` for each file in path''' - - for root, dirs, files in os.walk(path, followlinks=True): - dirs[:] = [x for x in dirs if not x.startswith(os.curdir)] - - for f in files: - valid_extension = f.endswith(tuple(extensions)) - file_ignored = any( - fnmatch.fnmatch(f, ignore) for ignore in ignores - ) - if valid_extension and not file_ignored: - try: - yield os.stat(os.path.join(root, f)).st_mtime - except OSError as e: - logger.warning('Caught Exception: %s', e) - - LAST_MTIME = 0 - while True: - try: - mtime = max(file_times(path)) - if mtime > LAST_MTIME: - LAST_MTIME = mtime - yield True - except ValueError: - yield None - else: - yield False - - @staticmethod - def file_watcher(path): - '''Generator for monitoring a file for modifications''' - LAST_MTIME = 0 - while True: - if path: - try: - mtime = os.stat(path).st_mtime - except OSError as e: - logger.warning('Caught Exception: %s', e) - continue - - if mtime > LAST_MTIME: - LAST_MTIME = mtime - yield True - else: - yield False - else: - yield None + return next(watchfiles.watch( + *watching_paths, + watch_filter=watchfiles.DefaultFilter( + ignore_entity_patterns=[fnmatch.translate(pattern) for pattern in ignore_files] + ), + rust_timeout=0 + )) def set_date_tzinfo(d, tz_name=None): diff --git a/pyproject.toml b/pyproject.toml index 826c1179..788b8dcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ rich = ">=10.1" unidecode = ">=1.1" markdown = {version = ">=3.1", optional = true} backports-zoneinfo = {version = "^0.2.1", python = "<3.9"} +watchfiles = "^0.19.0" [tool.poetry.dev-dependencies] BeautifulSoup4 = "^4.9"