forked from github/pelican
Refactor file/folder watchers and autoreload
Combined file and folder watchers under a class and refactored common watcher related code from __init__.py to the class. This simplifies the main and autoreload functions in __init__ as well as fix the problem with crashes related to multiprocessing on systems where default spawn mode is "spawn" instead of "fork".
This commit is contained in:
Deniz Turgut
parent
e6df353302
commit
48d842faa7
3 changed files with 273 additions and 184 deletions
|
|
@ -24,8 +24,7 @@ from pelican.plugins._utils import load_plugins
|
|||
from pelican.readers import Readers
|
||||
from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer
|
||||
from pelican.settings import read_settings
|
||||
from pelican.utils import (clean_output_dir, file_watcher,
|
||||
folder_watcher, maybe_pluralize)
|
||||
from pelican.utils import (FileSystemWatcher, clean_output_dir, maybe_pluralize)
|
||||
from pelican.writers import Writer
|
||||
|
||||
try:
|
||||
|
|
@ -381,65 +380,36 @@ def get_instance(args):
|
|||
return cls(settings), settings
|
||||
|
||||
|
||||
def autoreload(watchers, args, old_static, reader_descs, excqueue=None):
|
||||
def autoreload(args, excqueue=None):
|
||||
print(' --- AutoReload Mode: Monitoring `content`, `theme` and'
|
||||
' `settings` for changes. ---')
|
||||
pelican, settings = get_instance(args)
|
||||
watcher = FileSystemWatcher(args.settings, Readers, settings)
|
||||
sleep = False
|
||||
while True:
|
||||
try:
|
||||
# Check source dir for changed files ending with the given
|
||||
# extension in the settings. In the theme dir is no such
|
||||
# restriction; all files are recursively checked if they
|
||||
# have changed, no matter what extension the filenames
|
||||
# have.
|
||||
modified = {k: next(v) for k, v in watchers.items()}
|
||||
# Don't sleep first time, but sleep afterwards to reduce cpu load
|
||||
if sleep:
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
sleep = True
|
||||
|
||||
modified = watcher.check()
|
||||
|
||||
if modified['settings']:
|
||||
pelican, settings = get_instance(args)
|
||||
|
||||
# Adjust static watchers if there are any changes
|
||||
new_static = settings.get("STATIC_PATHS", [])
|
||||
|
||||
# Added static paths
|
||||
# Add new watchers and set them as modified
|
||||
new_watchers = set(new_static).difference(old_static)
|
||||
for static_path in new_watchers:
|
||||
static_key = '[static]%s' % static_path
|
||||
watchers[static_key] = folder_watcher(
|
||||
os.path.join(pelican.path, static_path),
|
||||
[''],
|
||||
pelican.ignore_files)
|
||||
modified[static_key] = next(watchers[static_key])
|
||||
|
||||
# Removed static paths
|
||||
# Remove watchers and modified values
|
||||
old_watchers = set(old_static).difference(new_static)
|
||||
for static_path in old_watchers:
|
||||
static_key = '[static]%s' % static_path
|
||||
watchers.pop(static_key)
|
||||
modified.pop(static_key)
|
||||
|
||||
# Replace old_static with the new one
|
||||
old_static = new_static
|
||||
watcher.update_watchers(settings)
|
||||
|
||||
if any(modified.values()):
|
||||
print('\n-> Modified: {}. re-generating...'.format(
|
||||
', '.join(k for k, v in modified.items() if v)))
|
||||
|
||||
if modified['content'] is None:
|
||||
logger.warning(
|
||||
'No valid files found in content for '
|
||||
+ 'the active readers:\n'
|
||||
+ '\n'.join(reader_descs))
|
||||
|
||||
if modified['theme'] is None:
|
||||
logger.warning('Empty theme folder. Using `basic` '
|
||||
'theme.')
|
||||
|
||||
pelican.run()
|
||||
|
||||
except KeyboardInterrupt as e:
|
||||
logger.warning("Keyboard interrupt, quitting.")
|
||||
except KeyboardInterrupt:
|
||||
if excqueue is not None:
|
||||
excqueue.put(traceback.format_exception_only(type(e), e)[-1])
|
||||
return
|
||||
excqueue.put(None)
|
||||
return
|
||||
raise
|
||||
|
||||
except Exception as e:
|
||||
if (args.verbosity == logging.DEBUG):
|
||||
|
|
@ -449,10 +419,8 @@ def autoreload(watchers, args, old_static, reader_descs, excqueue=None):
|
|||
else:
|
||||
raise
|
||||
logger.warning(
|
||||
'Caught exception "%s". Reloading.', e)
|
||||
|
||||
finally:
|
||||
time.sleep(.5) # sleep to avoid cpu load
|
||||
'Caught exception:\n"%s".', e,
|
||||
exc_info=settings.get('DEBUG', False))
|
||||
|
||||
|
||||
def listen(server, port, output, excqueue=None):
|
||||
|
|
@ -476,8 +444,10 @@ def listen(server, port, output, excqueue=None):
|
|||
return
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nKeyboard interrupt received. Shutting down server.")
|
||||
httpd.socket.close()
|
||||
if excqueue is not None:
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
|
|
@ -492,37 +462,11 @@ def main(argv=None):
|
|||
try:
|
||||
pelican, settings = get_instance(args)
|
||||
|
||||
readers = Readers(settings)
|
||||
reader_descs = sorted(
|
||||
{
|
||||
'%s (%s)' % (type(r).__name__, ', '.join(r.file_extensions))
|
||||
for r in readers.readers.values()
|
||||
if r.enabled
|
||||
}
|
||||
)
|
||||
|
||||
watchers = {'content': folder_watcher(pelican.path,
|
||||
readers.extensions,
|
||||
pelican.ignore_files),
|
||||
'theme': folder_watcher(pelican.theme,
|
||||
[''],
|
||||
pelican.ignore_files),
|
||||
'settings': file_watcher(args.settings)}
|
||||
|
||||
old_static = settings.get("STATIC_PATHS", [])
|
||||
for static_path in old_static:
|
||||
# use a prefix to avoid possible overriding of standard watchers
|
||||
# above
|
||||
watchers['[static]%s' % static_path] = folder_watcher(
|
||||
os.path.join(pelican.path, static_path),
|
||||
[''],
|
||||
pelican.ignore_files)
|
||||
|
||||
if args.autoreload and args.listen:
|
||||
excqueue = multiprocessing.Queue()
|
||||
p1 = multiprocessing.Process(
|
||||
target=autoreload,
|
||||
args=(watchers, args, old_static, reader_descs, excqueue))
|
||||
args=(args, excqueue))
|
||||
p2 = multiprocessing.Process(
|
||||
target=listen,
|
||||
args=(settings.get('BIND'), settings.get('PORT'),
|
||||
|
|
@ -532,26 +476,19 @@ def main(argv=None):
|
|||
exc = excqueue.get()
|
||||
p1.terminate()
|
||||
p2.terminate()
|
||||
logger.critical(exc)
|
||||
if exc is not None:
|
||||
logger.critical(exc)
|
||||
elif args.autoreload:
|
||||
print(' --- AutoReload Mode: Monitoring `content`, `theme` and'
|
||||
' `settings` for changes. ---')
|
||||
autoreload(watchers, args, old_static, reader_descs)
|
||||
autoreload(args)
|
||||
elif args.listen:
|
||||
listen(settings.get('BIND'), settings.get('PORT'),
|
||||
settings.get("OUTPUT_PATH"))
|
||||
else:
|
||||
if next(watchers['content']) is None:
|
||||
logger.warning(
|
||||
'No valid files found in content for '
|
||||
+ 'the active readers:\n'
|
||||
+ '\n'.join(reader_descs))
|
||||
|
||||
if next(watchers['theme']) is None:
|
||||
logger.warning('Empty theme folder. Using `basic` theme.')
|
||||
|
||||
watcher = FileSystemWatcher(args.settings, Readers, settings)
|
||||
watcher.check()
|
||||
pelican.run()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.warning('Keyboard interrupt received. Exiting.')
|
||||
except Exception as e:
|
||||
logger.critical('%s', e)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue