mirror of
https://github.com/getpelican/pelican.git
synced 2025-10-15 20:28:56 +02:00
Merge pull request #2750 from avaris/autoreload
Refactor file/folder watchers and autoreload
This commit is contained in:
Justin Mayer
GitHub
commit
2eb9c26cdb
3 changed files with 273 additions and 184 deletions
|
|
@ -24,8 +24,7 @@ from pelican.plugins._utils import load_plugins
|
|||
from pelican.readers import Readers
|
||||
from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer
|
||||
from pelican.settings import read_settings
|
||||
from pelican.utils import (clean_output_dir, file_watcher,
|
||||
folder_watcher, maybe_pluralize)
|
||||
from pelican.utils import (FileSystemWatcher, clean_output_dir, maybe_pluralize)
|
||||
from pelican.writers import Writer
|
||||
|
||||
try:
|
||||
|
|
@ -381,65 +380,36 @@ def get_instance(args):
|
|||
return cls(settings), settings
|
||||
|
||||
|
||||
def autoreload(watchers, args, old_static, reader_descs, excqueue=None):
|
||||
def autoreload(args, excqueue=None):
|
||||
print(' --- AutoReload Mode: Monitoring `content`, `theme` and'
|
||||
' `settings` for changes. ---')
|
||||
pelican, settings = get_instance(args)
|
||||
watcher = FileSystemWatcher(args.settings, Readers, settings)
|
||||
sleep = False
|
||||
while True:
|
||||
try:
|
||||
# Check source dir for changed files ending with the given
|
||||
# extension in the settings. In the theme dir is no such
|
||||
# restriction; all files are recursively checked if they
|
||||
# have changed, no matter what extension the filenames
|
||||
# have.
|
||||
modified = {k: next(v) for k, v in watchers.items()}
|
||||
# Don't sleep first time, but sleep afterwards to reduce cpu load
|
||||
if sleep:
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
sleep = True
|
||||
|
||||
modified = watcher.check()
|
||||
|
||||
if modified['settings']:
|
||||
pelican, settings = get_instance(args)
|
||||
|
||||
# Adjust static watchers if there are any changes
|
||||
new_static = settings.get("STATIC_PATHS", [])
|
||||
|
||||
# Added static paths
|
||||
# Add new watchers and set them as modified
|
||||
new_watchers = set(new_static).difference(old_static)
|
||||
for static_path in new_watchers:
|
||||
static_key = '[static]%s' % static_path
|
||||
watchers[static_key] = folder_watcher(
|
||||
os.path.join(pelican.path, static_path),
|
||||
[''],
|
||||
pelican.ignore_files)
|
||||
modified[static_key] = next(watchers[static_key])
|
||||
|
||||
# Removed static paths
|
||||
# Remove watchers and modified values
|
||||
old_watchers = set(old_static).difference(new_static)
|
||||
for static_path in old_watchers:
|
||||
static_key = '[static]%s' % static_path
|
||||
watchers.pop(static_key)
|
||||
modified.pop(static_key)
|
||||
|
||||
# Replace old_static with the new one
|
||||
old_static = new_static
|
||||
watcher.update_watchers(settings)
|
||||
|
||||
if any(modified.values()):
|
||||
print('\n-> Modified: {}. re-generating...'.format(
|
||||
', '.join(k for k, v in modified.items() if v)))
|
||||
|
||||
if modified['content'] is None:
|
||||
logger.warning(
|
||||
'No valid files found in content for '
|
||||
+ 'the active readers:\n'
|
||||
+ '\n'.join(reader_descs))
|
||||
|
||||
if modified['theme'] is None:
|
||||
logger.warning('Empty theme folder. Using `basic` '
|
||||
'theme.')
|
||||
|
||||
pelican.run()
|
||||
|
||||
except KeyboardInterrupt as e:
|
||||
logger.warning("Keyboard interrupt, quitting.")
|
||||
except KeyboardInterrupt:
|
||||
if excqueue is not None:
|
||||
excqueue.put(traceback.format_exception_only(type(e), e)[-1])
|
||||
return
|
||||
excqueue.put(None)
|
||||
return
|
||||
raise
|
||||
|
||||
except Exception as e:
|
||||
if (args.verbosity == logging.DEBUG):
|
||||
|
|
@ -449,10 +419,8 @@ def autoreload(watchers, args, old_static, reader_descs, excqueue=None):
|
|||
else:
|
||||
raise
|
||||
logger.warning(
|
||||
'Caught exception "%s". Reloading.', e)
|
||||
|
||||
finally:
|
||||
time.sleep(.5) # sleep to avoid cpu load
|
||||
'Caught exception:\n"%s".', e,
|
||||
exc_info=settings.get('DEBUG', False))
|
||||
|
||||
|
||||
def listen(server, port, output, excqueue=None):
|
||||
|
|
@ -476,8 +444,10 @@ def listen(server, port, output, excqueue=None):
|
|||
return
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nKeyboard interrupt received. Shutting down server.")
|
||||
httpd.socket.close()
|
||||
if excqueue is not None:
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
|
|
@ -492,37 +462,11 @@ def main(argv=None):
|
|||
try:
|
||||
pelican, settings = get_instance(args)
|
||||
|
||||
readers = Readers(settings)
|
||||
reader_descs = sorted(
|
||||
{
|
||||
'%s (%s)' % (type(r).__name__, ', '.join(r.file_extensions))
|
||||
for r in readers.readers.values()
|
||||
if r.enabled
|
||||
}
|
||||
)
|
||||
|
||||
watchers = {'content': folder_watcher(pelican.path,
|
||||
readers.extensions,
|
||||
pelican.ignore_files),
|
||||
'theme': folder_watcher(pelican.theme,
|
||||
[''],
|
||||
pelican.ignore_files),
|
||||
'settings': file_watcher(args.settings)}
|
||||
|
||||
old_static = settings.get("STATIC_PATHS", [])
|
||||
for static_path in old_static:
|
||||
# use a prefix to avoid possible overriding of standard watchers
|
||||
# above
|
||||
watchers['[static]%s' % static_path] = folder_watcher(
|
||||
os.path.join(pelican.path, static_path),
|
||||
[''],
|
||||
pelican.ignore_files)
|
||||
|
||||
if args.autoreload and args.listen:
|
||||
excqueue = multiprocessing.Queue()
|
||||
p1 = multiprocessing.Process(
|
||||
target=autoreload,
|
||||
args=(watchers, args, old_static, reader_descs, excqueue))
|
||||
args=(args, excqueue))
|
||||
p2 = multiprocessing.Process(
|
||||
target=listen,
|
||||
args=(settings.get('BIND'), settings.get('PORT'),
|
||||
|
|
@ -532,26 +476,19 @@ def main(argv=None):
|
|||
exc = excqueue.get()
|
||||
p1.terminate()
|
||||
p2.terminate()
|
||||
logger.critical(exc)
|
||||
if exc is not None:
|
||||
logger.critical(exc)
|
||||
elif args.autoreload:
|
||||
print(' --- AutoReload Mode: Monitoring `content`, `theme` and'
|
||||
' `settings` for changes. ---')
|
||||
autoreload(watchers, args, old_static, reader_descs)
|
||||
autoreload(args)
|
||||
elif args.listen:
|
||||
listen(settings.get('BIND'), settings.get('PORT'),
|
||||
settings.get("OUTPUT_PATH"))
|
||||
else:
|
||||
if next(watchers['content']) is None:
|
||||
logger.warning(
|
||||
'No valid files found in content for '
|
||||
+ 'the active readers:\n'
|
||||
+ '\n'.join(reader_descs))
|
||||
|
||||
if next(watchers['theme']) is None:
|
||||
logger.warning('Empty theme folder. Using `basic` theme.')
|
||||
|
||||
watcher = FileSystemWatcher(args.settings, Readers, settings)
|
||||
watcher.check()
|
||||
pelican.run()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.warning('Keyboard interrupt received. Exiting.')
|
||||
except Exception as e:
|
||||
logger.critical('%s', e)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import pytz
|
|||
|
||||
from pelican import utils
|
||||
from pelican.generators import TemplatePagesGenerator
|
||||
from pelican.readers import Readers
|
||||
from pelican.settings import read_settings
|
||||
from pelican.tests.support import (LoggedTestCase, get_article,
|
||||
locale_available, unittest)
|
||||
|
|
@ -361,47 +362,91 @@ class TestUtils(LoggedTestCase):
|
|||
self.assertNotIn(a_arts[4], b_arts[5].translations)
|
||||
self.assertNotIn(a_arts[5], b_arts[4].translations)
|
||||
|
||||
def test_watchers(self):
|
||||
# Test if file changes are correctly detected
|
||||
# Make sure to handle not getting any files correctly.
|
||||
def test_filesystemwatcher(self):
|
||||
def create_file(name, content):
|
||||
with open(name, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
dirname = os.path.join(os.path.dirname(__file__), 'content')
|
||||
folder_watcher = utils.folder_watcher(dirname, ['rst'])
|
||||
# disable logger filter
|
||||
from pelican.utils import logger
|
||||
logger.disable_filter()
|
||||
|
||||
path = os.path.join(dirname, 'article_with_metadata.rst')
|
||||
file_watcher = utils.file_watcher(path)
|
||||
# create a temp "project" dir
|
||||
root = mkdtemp()
|
||||
content_path = os.path.join(root, 'content')
|
||||
static_path = os.path.join(root, 'content', 'static')
|
||||
config_file = os.path.join(root, 'config.py')
|
||||
theme_path = os.path.join(root, 'mytheme')
|
||||
|
||||
# first check returns True
|
||||
self.assertEqual(next(folder_watcher), True)
|
||||
self.assertEqual(next(file_watcher), True)
|
||||
# populate
|
||||
os.mkdir(content_path)
|
||||
os.mkdir(theme_path)
|
||||
create_file(config_file,
|
||||
'PATH = "content"\n'
|
||||
'THEME = "mytheme"\n'
|
||||
'STATIC_PATHS = ["static"]')
|
||||
|
||||
# next check without modification returns False
|
||||
self.assertEqual(next(folder_watcher), False)
|
||||
self.assertEqual(next(file_watcher), False)
|
||||
t = time.time() - 1000 # make sure it's in the "past"
|
||||
os.utime(config_file, (t, t))
|
||||
settings = read_settings(config_file)
|
||||
|
||||
# after modification, returns True
|
||||
t = time.time()
|
||||
os.utime(path, (t, t))
|
||||
self.assertEqual(next(folder_watcher), True)
|
||||
self.assertEqual(next(file_watcher), True)
|
||||
watcher = utils.FileSystemWatcher(config_file, Readers, settings)
|
||||
# should get a warning for static not not existing
|
||||
self.assertLogCountEqual(1, 'Watched path does not exist: .*static')
|
||||
|
||||
# file watcher with None or empty path should return None
|
||||
self.assertEqual(next(utils.file_watcher('')), None)
|
||||
self.assertEqual(next(utils.file_watcher(None)), None)
|
||||
# create it and update config
|
||||
os.mkdir(static_path)
|
||||
watcher.update_watchers(settings)
|
||||
# no new warning
|
||||
self.assertLogCountEqual(1, 'Watched path does not exist: .*static')
|
||||
|
||||
empty_path = os.path.join(os.path.dirname(__file__), 'empty')
|
||||
try:
|
||||
os.mkdir(empty_path)
|
||||
os.mkdir(os.path.join(empty_path, "empty_folder"))
|
||||
shutil.copy(__file__, empty_path)
|
||||
# get modified values
|
||||
modified = watcher.check()
|
||||
# empty theme and content should raise warnings
|
||||
self.assertLogCountEqual(1, 'No valid files found in content')
|
||||
self.assertLogCountEqual(1, 'Empty theme folder. Using `basic` theme')
|
||||
|
||||
# if no files of interest, returns None
|
||||
watcher = utils.folder_watcher(empty_path, ['rst'])
|
||||
self.assertEqual(next(watcher), None)
|
||||
except OSError:
|
||||
self.fail("OSError Exception in test_files_changed test")
|
||||
finally:
|
||||
shutil.rmtree(empty_path, True)
|
||||
self.assertIsNone(modified['content']) # empty
|
||||
self.assertIsNone(modified['theme']) # empty
|
||||
self.assertIsNone(modified['[static]static']) # empty
|
||||
self.assertTrue(modified['settings']) # modified, first time
|
||||
|
||||
# add a content, add file to theme and check again
|
||||
create_file(os.path.join(content_path, 'article.md'),
|
||||
'Title: test\n'
|
||||
'Date: 01-01-2020')
|
||||
|
||||
create_file(os.path.join(theme_path, 'dummy'),
|
||||
'test')
|
||||
|
||||
modified = watcher.check()
|
||||
# no new warning
|
||||
self.assertLogCountEqual(1, 'No valid files found in content')
|
||||
self.assertLogCountEqual(1, 'Empty theme folder. Using `basic` theme')
|
||||
|
||||
self.assertIsNone(modified['[static]static']) # empty
|
||||
self.assertFalse(modified['settings']) # not modified
|
||||
self.assertTrue(modified['theme']) # modified
|
||||
self.assertTrue(modified['content']) # modified
|
||||
|
||||
# change config, remove static path
|
||||
create_file(config_file,
|
||||
'PATH = "content"\n'
|
||||
'THEME = "mytheme"\n'
|
||||
'STATIC_PATHS = []')
|
||||
|
||||
settings = read_settings(config_file)
|
||||
watcher.update_watchers(settings)
|
||||
|
||||
modified = watcher.check()
|
||||
self.assertNotIn('[static]static', modified) # should be gone
|
||||
self.assertTrue(modified['settings']) # modified
|
||||
self.assertFalse(modified['content']) # not modified
|
||||
self.assertFalse(modified['theme']) # not modified
|
||||
|
||||
# cleanup
|
||||
logger.enable_filter()
|
||||
shutil.rmtree(root)
|
||||
|
||||
def test_clean_output_dir(self):
|
||||
retention = ()
|
||||
|
|
|
|||
193
pelican/utils.py
193
pelican/utils.py
|
|
@ -735,60 +735,167 @@ def order_content(content_list, order_by='slug'):
|
|||
return content_list
|
||||
|
||||
|
||||
def folder_watcher(path, extensions, ignores=[]):
|
||||
'''Generator for monitoring a folder for modifications.
|
||||
class FileSystemWatcher:
|
||||
def __init__(self, settings_file, reader_class, settings=None):
|
||||
self.watchers = {
|
||||
'settings': FileSystemWatcher.file_watcher(settings_file)
|
||||
}
|
||||
|
||||
Returns a boolean indicating if files are changed since last check.
|
||||
Returns None if there are no matching files in the folder'''
|
||||
self.settings = None
|
||||
self.reader_class = reader_class
|
||||
self._extensions = None
|
||||
self._content_path = None
|
||||
self._theme_path = None
|
||||
self._ignore_files = None
|
||||
|
||||
def file_times(path):
|
||||
'''Return `mtime` for each file in path'''
|
||||
if settings is not None:
|
||||
self.update_watchers(settings)
|
||||
|
||||
for root, dirs, files in os.walk(path, followlinks=True):
|
||||
dirs[:] = [x for x in dirs if not x.startswith(os.curdir)]
|
||||
def update_watchers(self, settings):
|
||||
new_extensions = set(self.reader_class(settings).extensions)
|
||||
new_content_path = settings.get('PATH', '')
|
||||
new_theme_path = settings.get('THEME', '')
|
||||
new_ignore_files = set(settings.get('IGNORE_FILES', []))
|
||||
|
||||
for f in files:
|
||||
valid_extension = f.endswith(tuple(extensions))
|
||||
file_ignored = any(
|
||||
fnmatch.fnmatch(f, ignore) for ignore in ignores
|
||||
)
|
||||
if valid_extension and not file_ignored:
|
||||
try:
|
||||
yield os.stat(os.path.join(root, f)).st_mtime
|
||||
except OSError as e:
|
||||
logger.warning('Caught Exception: %s', e)
|
||||
extensions_changed = new_extensions != self._extensions
|
||||
content_changed = new_content_path != self._content_path
|
||||
theme_changed = new_theme_path != self._theme_path
|
||||
ignore_changed = new_ignore_files != self._ignore_files
|
||||
|
||||
LAST_MTIME = 0
|
||||
while True:
|
||||
try:
|
||||
mtime = max(file_times(path))
|
||||
if mtime > LAST_MTIME:
|
||||
LAST_MTIME = mtime
|
||||
yield True
|
||||
except ValueError:
|
||||
yield None
|
||||
# Refresh content watcher if related settings changed
|
||||
if extensions_changed or content_changed or ignore_changed:
|
||||
self.add_watcher('content',
|
||||
new_content_path,
|
||||
new_extensions,
|
||||
new_ignore_files)
|
||||
|
||||
# Refresh theme watcher if related settings changed
|
||||
if theme_changed or ignore_changed:
|
||||
self.add_watcher('theme',
|
||||
new_theme_path,
|
||||
[''],
|
||||
new_ignore_files)
|
||||
|
||||
# Watch STATIC_PATHS
|
||||
old_static_watchers = set(key
|
||||
for key in self.watchers
|
||||
if key.startswith('[static]'))
|
||||
|
||||
for path in settings.get('STATIC_PATHS', []):
|
||||
key = '[static]{}'.format(path)
|
||||
if ignore_changed or (key not in self.watchers):
|
||||
self.add_watcher(
|
||||
key,
|
||||
os.path.join(new_content_path, path),
|
||||
[''],
|
||||
new_ignore_files)
|
||||
if key in old_static_watchers:
|
||||
old_static_watchers.remove(key)
|
||||
|
||||
# cleanup removed static watchers
|
||||
for key in old_static_watchers:
|
||||
del self.watchers[key]
|
||||
|
||||
# update values
|
||||
self.settings = settings
|
||||
self._extensions = new_extensions
|
||||
self._content_path = new_content_path
|
||||
self._theme_path = new_theme_path
|
||||
self._ignore_files = new_ignore_files
|
||||
|
||||
def check(self):
|
||||
'''return a key:watcher_status dict for all watchers'''
|
||||
result = {key: next(watcher) for key, watcher in self.watchers.items()}
|
||||
|
||||
# Various warnings
|
||||
if result.get('content') is None:
|
||||
reader_descs = sorted(
|
||||
{
|
||||
'%s (%s)' % (type(r).__name__, ', '.join(r.file_extensions))
|
||||
for r in self.reader_class(self.settings).readers.values()
|
||||
if r.enabled
|
||||
}
|
||||
)
|
||||
logger.warning(
|
||||
'No valid files found in content for the active readers:\n'
|
||||
+ '\n'.join(reader_descs))
|
||||
|
||||
if result.get('theme') is None:
|
||||
logger.warning('Empty theme folder. Using `basic` theme.')
|
||||
|
||||
return result
|
||||
|
||||
def add_watcher(self, key, path, extensions=[''], ignores=[]):
|
||||
watcher = self.get_watcher(path, extensions, ignores)
|
||||
if watcher is not None:
|
||||
self.watchers[key] = watcher
|
||||
|
||||
def get_watcher(self, path, extensions=[''], ignores=[]):
|
||||
'''return a watcher depending on path type (file or folder)'''
|
||||
if not os.path.exists(path):
|
||||
logger.warning("Watched path does not exist: %s", path)
|
||||
return None
|
||||
|
||||
if os.path.isdir(path):
|
||||
return self.folder_watcher(path, extensions, ignores)
|
||||
else:
|
||||
yield False
|
||||
return self.file_watcher(path)
|
||||
|
||||
@staticmethod
|
||||
def folder_watcher(path, extensions, ignores=[]):
|
||||
'''Generator for monitoring a folder for modifications.
|
||||
|
||||
def file_watcher(path):
|
||||
'''Generator for monitoring a file for modifications'''
|
||||
LAST_MTIME = 0
|
||||
while True:
|
||||
if path:
|
||||
Returns a boolean indicating if files are changed since last check.
|
||||
Returns None if there are no matching files in the folder'''
|
||||
|
||||
def file_times(path):
|
||||
'''Return `mtime` for each file in path'''
|
||||
|
||||
for root, dirs, files in os.walk(path, followlinks=True):
|
||||
dirs[:] = [x for x in dirs if not x.startswith(os.curdir)]
|
||||
|
||||
for f in files:
|
||||
valid_extension = f.endswith(tuple(extensions))
|
||||
file_ignored = any(
|
||||
fnmatch.fnmatch(f, ignore) for ignore in ignores
|
||||
)
|
||||
if valid_extension and not file_ignored:
|
||||
try:
|
||||
yield os.stat(os.path.join(root, f)).st_mtime
|
||||
except OSError as e:
|
||||
logger.warning('Caught Exception: %s', e)
|
||||
|
||||
LAST_MTIME = 0
|
||||
while True:
|
||||
try:
|
||||
mtime = os.stat(path).st_mtime
|
||||
except OSError as e:
|
||||
logger.warning('Caught Exception: %s', e)
|
||||
continue
|
||||
|
||||
if mtime > LAST_MTIME:
|
||||
LAST_MTIME = mtime
|
||||
yield True
|
||||
mtime = max(file_times(path))
|
||||
if mtime > LAST_MTIME:
|
||||
LAST_MTIME = mtime
|
||||
yield True
|
||||
except ValueError:
|
||||
yield None
|
||||
else:
|
||||
yield False
|
||||
else:
|
||||
yield None
|
||||
|
||||
@staticmethod
|
||||
def file_watcher(path):
|
||||
'''Generator for monitoring a file for modifications'''
|
||||
LAST_MTIME = 0
|
||||
while True:
|
||||
if path:
|
||||
try:
|
||||
mtime = os.stat(path).st_mtime
|
||||
except OSError as e:
|
||||
logger.warning('Caught Exception: %s', e)
|
||||
continue
|
||||
|
||||
if mtime > LAST_MTIME:
|
||||
LAST_MTIME = mtime
|
||||
yield True
|
||||
else:
|
||||
yield False
|
||||
else:
|
||||
yield None
|
||||
|
||||
|
||||
def set_date_tzinfo(d, tz_name=None):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue