From 5ccbbcc7d16931a27b4153d7b4d20df6f5c93925 Mon Sep 17 00:00:00 2001 From: Deniz Turgut Date: Fri, 19 Apr 2013 13:35:20 -0400 Subject: [PATCH 1/2] add feedback to user and refactor 'autoreload' code --- pelican/__init__.py | 70 ++++++++++++++++++++++------------- pelican/tests/test_pelican.py | 6 +-- pelican/tests/test_utils.py | 29 +++++++++------ pelican/utils.py | 68 ++++++++++++++++------------------ 4 files changed, 97 insertions(+), 76 deletions(-) diff --git a/pelican/__init__.py b/pelican/__init__.py index 707fa8f0..4e6cc72a 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -16,8 +16,7 @@ from pelican.generators import (ArticlesGenerator, PagesGenerator, SourceFileGenerator, TemplatePagesGenerator) from pelican.log import init from pelican.settings import read_settings -from pelican.utils import (clean_output_dir, files_changed, file_changed, - NoFilesError) +from pelican.utils import clean_output_dir, folder_watcher, file_watcher from pelican.writers import Writer __major__ = 3 @@ -148,6 +147,7 @@ class Pelican(object): def run(self): """Run the generators and return""" + start_time = time.time() context = self.settings.copy() context['filenames'] = {} # share the dict between all the generators @@ -181,6 +181,14 @@ class Pelican(object): signals.finalized.send(self) + articles_generator = next(g for g in generators if isinstance(g, ArticlesGenerator)) + pages_generator = next(g for g in generators if isinstance(g, PagesGenerator)) + + print('Done: Processed {} articles and {} pages in {:.2f} seconds.'.format( + len(articles_generator.articles) + len(articles_generator.translations), + len(pages_generator.pages) + len(pages_generator.translations), + time.time() - start_time)) + def get_generator_classes(self): generators = [StaticGenerator, ArticlesGenerator, PagesGenerator] @@ -295,9 +303,19 @@ def main(): init(args.verbosity) pelican = get_instance(args) + watchers = {'content': folder_watcher(pelican.path, + pelican.markup, + pelican.ignore_files), + 'theme': folder_watcher(pelican.theme, + [''], + pelican.ignore_files), + 'settings': file_watcher(args.settings)} + try: if args.autoreload: - files_found_error = True + print(' --- AutoReload Mode: Monitoring `content`, `theme` and `settings`' + ' for changes. ---') + while True: try: # Check source dir for changed files ending with the given @@ -305,45 +323,45 @@ def main(): # restriction; all files are recursively checked if they # have changed, no matter what extension the filenames # have. - if (files_changed( - pelican.path, - pelican.markup, - pelican.ignore_files) - or files_changed( - pelican.theme, - [''], - pelican.ignore_files - )): - if not files_found_error: - files_found_error = True - pelican.run() + modified = {k: next(v) for k, v in watchers.items()} - # reload also if settings.py changed - if file_changed(args.settings): - logger.info('%s changed, re-generating' % - args.settings) + if modified['settings']: pelican = get_instance(args) + + if any(modified.values()): + print('\n-> Modified: {}. re-generating...'.format( + ', '.join(k for k, v in modified.items() if v))) + + if modified['content'] is None: + logger.warning('No valid files found in content.') + + if modified['theme'] is None: + logger.warning('Empty theme folder. Using `basic` theme.') + pelican.run() time.sleep(.5) # sleep to avoid cpu load + except KeyboardInterrupt: logger.warning("Keyboard interrupt, quitting.") break - except NoFilesError: - if files_found_error: - logger.warning("No valid files found in content. " - "Nothing to generate.") - files_found_error = False - time.sleep(1) # sleep to avoid cpu load + except Exception as e: if (args.verbosity == logging.DEBUG): logger.critical(e.args) raise logger.warning( 'Caught exception "{0}". Reloading.'.format(e)) - continue + else: + if next(watchers['content']) is None: + logger.warning('No valid files found in content.') + + if next(watchers['theme']) is None: + logger.warning('Empty theme folder. Using `basic` theme.') + pelican.run() + except Exception as e: logger.critical(e) diff --git a/pelican/tests/test_pelican.py b/pelican/tests/test_pelican.py index 4baa4418..d6c0d8e9 100644 --- a/pelican/tests/test_pelican.py +++ b/pelican/tests/test_pelican.py @@ -10,7 +10,7 @@ import logging from pelican import Pelican from pelican.settings import read_settings -from pelican.tests.support import LoggedTestCase +from pelican.tests.support import LoggedTestCase, mute CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) SAMPLES_PATH = os.path.abspath(os.path.join( @@ -73,7 +73,7 @@ class TestPelican(LoggedTestCase): 'LOCALE': locale.normalize('en_US'), }) pelican = Pelican(settings=settings) - pelican.run() + mute(True)(pelican.run)() dcmp = dircmp(self.temp_path, os.path.join(OUTPUT_PATH, 'basic')) self.assertFilesEqual(recursiveDiff(dcmp)) self.assertLogCountEqual( @@ -89,6 +89,6 @@ class TestPelican(LoggedTestCase): 'LOCALE': locale.normalize('en_US'), }) pelican = Pelican(settings=settings) - pelican.run() + mute(True)(pelican.run)() dcmp = dircmp(self.temp_path, os.path.join(OUTPUT_PATH, 'custom')) self.assertFilesEqual(recursiveDiff(dcmp)) diff --git a/pelican/tests/test_utils.py b/pelican/tests/test_utils.py index e40046bd..f89c2907 100644 --- a/pelican/tests/test_utils.py +++ b/pelican/tests/test_utils.py @@ -10,7 +10,6 @@ from sys import platform from pelican import utils from .support import get_article, LoggedTestCase, locale_available, unittest -from pelican.utils import NoFilesError class TestUtils(LoggedTestCase): @@ -127,31 +126,39 @@ class TestUtils(LoggedTestCase): self.assertNotIn(en_article3, trans) self.assertNotIn(fr_article3, index) - def test_files_changed(self): + def test_watchers(self): # Test if file changes are correctly detected # Make sure to handle not getting any files correctly. dirname = os.path.join(os.path.dirname(__file__), 'content') + folder_watcher = utils.folder_watcher(dirname, ['rst']) + path = os.path.join(dirname, 'article_with_metadata.rst') - changed = utils.files_changed(dirname, 'rst') - self.assertEqual(changed, True) + file_watcher = utils.file_watcher(path) - changed = utils.files_changed(dirname, 'rst') - self.assertEqual(changed, False) + # first check returns True + self.assertEqual(next(folder_watcher), True) + self.assertEqual(next(file_watcher), True) + # next check without modification returns False + self.assertEqual(next(folder_watcher), False) + self.assertEqual(next(file_watcher), False) + + # after modification, returns True t = time.time() os.utime(path, (t, t)) - changed = utils.files_changed(dirname, 'rst') - self.assertEqual(changed, True) - self.assertAlmostEqual(utils.LAST_MTIME, t, delta=1) + self.assertEqual(next(folder_watcher), True) + self.assertEqual(next(file_watcher), True) empty_path = os.path.join(os.path.dirname(__file__), 'empty') try: os.mkdir(empty_path) os.mkdir(os.path.join(empty_path, "empty_folder")) shutil.copy(__file__, empty_path) - with self.assertRaises(NoFilesError): - utils.files_changed(empty_path, 'rst') + + # if no files of interest, returns None + watcher = utils.folder_watcher(empty_path, ['rst']) + self.assertEqual(next(watcher), None) except OSError: self.fail("OSError Exception in test_files_changed test") finally: diff --git a/pelican/utils.py b/pelican/utils.py index aa807de2..c6f108f9 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -11,7 +11,7 @@ import logging import errno import locale import fnmatch -from collections import defaultdict, Hashable +from collections import Hashable from functools import partial from codecs import open, BOM_UTF8 @@ -81,10 +81,6 @@ def python_2_unicode_compatible(klass): return klass -class NoFilesError(Exception): - pass - - class memoized(object): """Function decorator to cache return values. @@ -433,46 +429,46 @@ def process_translations(content_list): return index, translations -LAST_MTIME = 0 +def folder_watcher(path, extensions, ignores=[]): + '''Generator for monitoring a folder for modifications. - -def files_changed(path, extensions, ignores=[]): - """Return True if the files have changed since the last check""" + Returns a boolean indicating if files are changed since last check. + Returns None if there are no matching files in the folder''' def file_times(path): - """Return the last time files have been modified""" + '''Return `mtime` for each file in path''' + for root, dirs, files in os.walk(path): - dirs[:] = [x for x in dirs if x[0] != os.curdir] + dirs[:] = [x for x in dirs if not x.startswith(os.curdir)] + for f in files: - if any(f.endswith(ext) for ext in extensions) \ - and not any(fnmatch.fnmatch(f, ignore) - for ignore in ignores): + if (f.endswith(tuple(extensions)) and + not any(fnmatch.fnmatch(f, ignore) for ignore in ignores)): yield os.stat(os.path.join(root, f)).st_mtime - global LAST_MTIME - try: - mtime = max(file_times(path)) + LAST_MTIME = 0 + while True: + try: + mtime = max(file_times(path)) + if mtime > LAST_MTIME: + LAST_MTIME = mtime + yield True + except ValueError: + yield None + else: + yield False + + +def file_watcher(path): + '''Generator for monitoring a file for modifications''' + LAST_MTIME = 0 + while True: + mtime = os.stat(path).st_mtime if mtime > LAST_MTIME: LAST_MTIME = mtime - return True - except ValueError: - raise NoFilesError("No files with the given extension(s) found.") - return False - - -FILENAMES_MTIMES = defaultdict(int) - - -def file_changed(path): - mtime = os.stat(path).st_mtime - if FILENAMES_MTIMES[path] == 0: - FILENAMES_MTIMES[path] = mtime - return False - else: - if mtime > FILENAMES_MTIMES[path]: - FILENAMES_MTIMES[path] = mtime - return True - return False + yield True + else: + yield False def set_date_tzinfo(d, tz_name=None): From e86f4eedcf83066e4e16d82ceec43bf2deb43b8e Mon Sep 17 00:00:00 2001 From: Deniz Turgut Date: Fri, 19 Apr 2013 23:35:09 -0400 Subject: [PATCH 2/2] bugfix and tests for no settings path (no -s option) --- pelican/tests/test_utils.py | 4 ++++ pelican/utils.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/pelican/tests/test_utils.py b/pelican/tests/test_utils.py index f89c2907..18babca7 100644 --- a/pelican/tests/test_utils.py +++ b/pelican/tests/test_utils.py @@ -150,6 +150,10 @@ class TestUtils(LoggedTestCase): self.assertEqual(next(folder_watcher), True) self.assertEqual(next(file_watcher), True) + # file watcher with None or empty path should return None + self.assertEqual(next(utils.file_watcher('')), None) + self.assertEqual(next(utils.file_watcher(None)), None) + empty_path = os.path.join(os.path.dirname(__file__), 'empty') try: os.mkdir(empty_path) diff --git a/pelican/utils.py b/pelican/utils.py index c6f108f9..39a3f8f4 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -463,12 +463,15 @@ def file_watcher(path): '''Generator for monitoring a file for modifications''' LAST_MTIME = 0 while True: - mtime = os.stat(path).st_mtime - if mtime > LAST_MTIME: - LAST_MTIME = mtime - yield True + if path: + mtime = os.stat(path).st_mtime + if mtime > LAST_MTIME: + LAST_MTIME = mtime + yield True + else: + yield False else: - yield False + yield None def set_date_tzinfo(d, tz_name=None):