mirror of
https://github.com/getpelican/pelican.git
synced 2025-10-15 20:28:56 +02:00
Use watchfiles as a file watching backend
This doesn't use polling unless absolutely necessarily, making it more efficient. It also reduces the amount of first-party code required, and simplifies working out which files are being watched.
This commit is contained in:
Jake Howard
Deniz Turgut
parent
6ed7395812
commit
61ca47c519
4 changed files with 32 additions and 260 deletions
|
|
@ -27,7 +27,7 @@ from pelican.plugins._utils import get_plugin_name, load_plugins
|
|||
from pelican.readers import Readers
|
||||
from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer
|
||||
from pelican.settings import read_settings
|
||||
from pelican.utils import (FileSystemWatcher, clean_output_dir, maybe_pluralize)
|
||||
from pelican.utils import (wait_for_changes, clean_output_dir, maybe_pluralize)
|
||||
from pelican.writers import Writer
|
||||
|
||||
try:
|
||||
|
|
@ -452,26 +452,19 @@ def autoreload(args, excqueue=None):
|
|||
console.print(' --- AutoReload Mode: Monitoring `content`, `theme` and'
|
||||
' `settings` for changes. ---')
|
||||
pelican, settings = get_instance(args)
|
||||
watcher = FileSystemWatcher(args.settings, Readers, settings)
|
||||
sleep = False
|
||||
settings_file = os.path.abspath(args.settings)
|
||||
while True:
|
||||
try:
|
||||
# Don't sleep first time, but sleep afterwards to reduce cpu load
|
||||
if sleep:
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
sleep = True
|
||||
changed_files = wait_for_changes(args.settings, Readers, settings)
|
||||
|
||||
modified = watcher.check()
|
||||
changed_files = {c[1] for c in changed_files}
|
||||
|
||||
if modified['settings']:
|
||||
if settings_file in changed_files:
|
||||
pelican, settings = get_instance(args)
|
||||
watcher.update_watchers(settings)
|
||||
|
||||
if any(modified.values()):
|
||||
console.print('\n-> Modified: {}. re-generating...'.format(
|
||||
', '.join(k for k, v in modified.items() if v)))
|
||||
pelican.run()
|
||||
console.print('\n-> Modified: {}. re-generating...'.format(
|
||||
', '.join(changed_files)))
|
||||
pelican.run()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
if excqueue is not None:
|
||||
|
|
@ -558,8 +551,6 @@ def main(argv=None):
|
|||
listen(settings.get('BIND'), settings.get('PORT'),
|
||||
settings.get("OUTPUT_PATH"))
|
||||
else:
|
||||
watcher = FileSystemWatcher(args.settings, Readers, settings)
|
||||
watcher.check()
|
||||
with console.status("Generating..."):
|
||||
pelican.run()
|
||||
except KeyboardInterrupt:
|
||||
|
|
|
|||
|
|
@ -412,92 +412,6 @@ class TestUtils(LoggedTestCase):
|
|||
self.assertNotIn(a_arts[4], b_arts[5].translations)
|
||||
self.assertNotIn(a_arts[5], b_arts[4].translations)
|
||||
|
||||
def test_filesystemwatcher(self):
|
||||
def create_file(name, content):
|
||||
with open(name, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
# disable logger filter
|
||||
from pelican.utils import logger
|
||||
logger.disable_filter()
|
||||
|
||||
# create a temp "project" dir
|
||||
root = mkdtemp()
|
||||
content_path = os.path.join(root, 'content')
|
||||
static_path = os.path.join(root, 'content', 'static')
|
||||
config_file = os.path.join(root, 'config.py')
|
||||
theme_path = os.path.join(root, 'mytheme')
|
||||
|
||||
# populate
|
||||
os.mkdir(content_path)
|
||||
os.mkdir(theme_path)
|
||||
create_file(config_file,
|
||||
'PATH = "content"\n'
|
||||
'THEME = "mytheme"\n'
|
||||
'STATIC_PATHS = ["static"]')
|
||||
|
||||
t = time.time() - 1000 # make sure it's in the "past"
|
||||
os.utime(config_file, (t, t))
|
||||
settings = read_settings(config_file)
|
||||
|
||||
watcher = utils.FileSystemWatcher(config_file, Readers, settings)
|
||||
# should get a warning for static not not existing
|
||||
self.assertLogCountEqual(1, 'Watched path does not exist: .*static')
|
||||
|
||||
# create it and update config
|
||||
os.mkdir(static_path)
|
||||
watcher.update_watchers(settings)
|
||||
# no new warning
|
||||
self.assertLogCountEqual(1, 'Watched path does not exist: .*static')
|
||||
|
||||
# get modified values
|
||||
modified = watcher.check()
|
||||
# empty theme and content should raise warnings
|
||||
self.assertLogCountEqual(1, 'No valid files found in content')
|
||||
self.assertLogCountEqual(1, 'Empty theme folder. Using `basic` theme')
|
||||
|
||||
self.assertIsNone(modified['content']) # empty
|
||||
self.assertIsNone(modified['theme']) # empty
|
||||
self.assertIsNone(modified['[static]static']) # empty
|
||||
self.assertTrue(modified['settings']) # modified, first time
|
||||
|
||||
# add a content, add file to theme and check again
|
||||
create_file(os.path.join(content_path, 'article.md'),
|
||||
'Title: test\n'
|
||||
'Date: 01-01-2020')
|
||||
|
||||
create_file(os.path.join(theme_path, 'dummy'),
|
||||
'test')
|
||||
|
||||
modified = watcher.check()
|
||||
# no new warning
|
||||
self.assertLogCountEqual(1, 'No valid files found in content')
|
||||
self.assertLogCountEqual(1, 'Empty theme folder. Using `basic` theme')
|
||||
|
||||
self.assertIsNone(modified['[static]static']) # empty
|
||||
self.assertFalse(modified['settings']) # not modified
|
||||
self.assertTrue(modified['theme']) # modified
|
||||
self.assertTrue(modified['content']) # modified
|
||||
|
||||
# change config, remove static path
|
||||
create_file(config_file,
|
||||
'PATH = "content"\n'
|
||||
'THEME = "mytheme"\n'
|
||||
'STATIC_PATHS = []')
|
||||
|
||||
settings = read_settings(config_file)
|
||||
watcher.update_watchers(settings)
|
||||
|
||||
modified = watcher.check()
|
||||
self.assertNotIn('[static]static', modified) # should be gone
|
||||
self.assertTrue(modified['settings']) # modified
|
||||
self.assertFalse(modified['content']) # not modified
|
||||
self.assertFalse(modified['theme']) # not modified
|
||||
|
||||
# cleanup
|
||||
logger.enable_filter()
|
||||
shutil.rmtree(root)
|
||||
|
||||
def test_clean_output_dir(self):
|
||||
retention = ()
|
||||
test_directory = os.path.join(self.temp_output,
|
||||
|
|
|
|||
180
pelican/utils.py
180
pelican/utils.py
|
|
@ -24,6 +24,8 @@ except ModuleNotFoundError:
|
|||
from backports.zoneinfo import ZoneInfo
|
||||
from markupsafe import Markup
|
||||
|
||||
import watchfiles
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -755,167 +757,31 @@ def order_content(content_list, order_by='slug'):
|
|||
return content_list
|
||||
|
||||
|
||||
class FileSystemWatcher:
|
||||
def __init__(self, settings_file, reader_class, settings=None):
|
||||
self.watchers = {
|
||||
'settings': FileSystemWatcher.file_watcher(settings_file)
|
||||
}
|
||||
def wait_for_changes(settings_file, reader_class, settings):
|
||||
new_extensions = set(reader_class(settings).extensions)
|
||||
content_path = settings.get('PATH', '')
|
||||
theme_path = settings.get('THEME', '')
|
||||
ignore_files = set(settings.get('IGNORE_FILES', []))
|
||||
|
||||
self.settings = None
|
||||
self.reader_class = reader_class
|
||||
self._extensions = None
|
||||
self._content_path = None
|
||||
self._theme_path = None
|
||||
self._ignore_files = None
|
||||
watching_paths = [
|
||||
settings_file,
|
||||
theme_path,
|
||||
content_path,
|
||||
]
|
||||
|
||||
if settings is not None:
|
||||
self.update_watchers(settings)
|
||||
watching_paths.extend(
|
||||
os.path.join(content_path, path) for path in settings.get('STATIC_PATHS', [])
|
||||
)
|
||||
|
||||
def update_watchers(self, settings):
|
||||
new_extensions = set(self.reader_class(settings).extensions)
|
||||
new_content_path = settings.get('PATH', '')
|
||||
new_theme_path = settings.get('THEME', '')
|
||||
new_ignore_files = set(settings.get('IGNORE_FILES', []))
|
||||
watching_paths = [os.path.abspath(p) for p in watching_paths if p and os.path.exists(p)]
|
||||
|
||||
extensions_changed = new_extensions != self._extensions
|
||||
content_changed = new_content_path != self._content_path
|
||||
theme_changed = new_theme_path != self._theme_path
|
||||
ignore_changed = new_ignore_files != self._ignore_files
|
||||
|
||||
# Refresh content watcher if related settings changed
|
||||
if extensions_changed or content_changed or ignore_changed:
|
||||
self.add_watcher('content',
|
||||
new_content_path,
|
||||
new_extensions,
|
||||
new_ignore_files)
|
||||
|
||||
# Refresh theme watcher if related settings changed
|
||||
if theme_changed or ignore_changed:
|
||||
self.add_watcher('theme',
|
||||
new_theme_path,
|
||||
[''],
|
||||
new_ignore_files)
|
||||
|
||||
# Watch STATIC_PATHS
|
||||
old_static_watchers = set(key
|
||||
for key in self.watchers
|
||||
if key.startswith('[static]'))
|
||||
|
||||
for path in settings.get('STATIC_PATHS', []):
|
||||
key = '[static]{}'.format(path)
|
||||
if ignore_changed or (key not in self.watchers):
|
||||
self.add_watcher(
|
||||
key,
|
||||
os.path.join(new_content_path, path),
|
||||
[''],
|
||||
new_ignore_files)
|
||||
if key in old_static_watchers:
|
||||
old_static_watchers.remove(key)
|
||||
|
||||
# cleanup removed static watchers
|
||||
for key in old_static_watchers:
|
||||
del self.watchers[key]
|
||||
|
||||
# update values
|
||||
self.settings = settings
|
||||
self._extensions = new_extensions
|
||||
self._content_path = new_content_path
|
||||
self._theme_path = new_theme_path
|
||||
self._ignore_files = new_ignore_files
|
||||
|
||||
def check(self):
|
||||
'''return a key:watcher_status dict for all watchers'''
|
||||
result = {key: next(watcher) for key, watcher in self.watchers.items()}
|
||||
|
||||
# Various warnings
|
||||
if result.get('content') is None:
|
||||
reader_descs = sorted(
|
||||
{
|
||||
' | %s (%s)' % (type(r).__name__, ', '.join(r.file_extensions))
|
||||
for r in self.reader_class(self.settings).readers.values()
|
||||
if r.enabled
|
||||
}
|
||||
)
|
||||
logger.warning(
|
||||
'No valid files found in content for the active readers:\n'
|
||||
+ '\n'.join(reader_descs))
|
||||
|
||||
if result.get('theme') is None:
|
||||
logger.warning('Empty theme folder. Using `basic` theme.')
|
||||
|
||||
return result
|
||||
|
||||
def add_watcher(self, key, path, extensions=[''], ignores=[]):
|
||||
watcher = self.get_watcher(path, extensions, ignores)
|
||||
if watcher is not None:
|
||||
self.watchers[key] = watcher
|
||||
|
||||
def get_watcher(self, path, extensions=[''], ignores=[]):
|
||||
'''return a watcher depending on path type (file or folder)'''
|
||||
if not os.path.exists(path):
|
||||
logger.warning("Watched path does not exist: %s", path)
|
||||
return None
|
||||
|
||||
if os.path.isdir(path):
|
||||
return self.folder_watcher(path, extensions, ignores)
|
||||
else:
|
||||
return self.file_watcher(path)
|
||||
|
||||
@staticmethod
|
||||
def folder_watcher(path, extensions, ignores=[]):
|
||||
'''Generator for monitoring a folder for modifications.
|
||||
|
||||
Returns a boolean indicating if files are changed since last check.
|
||||
Returns None if there are no matching files in the folder'''
|
||||
|
||||
def file_times(path):
|
||||
'''Return `mtime` for each file in path'''
|
||||
|
||||
for root, dirs, files in os.walk(path, followlinks=True):
|
||||
dirs[:] = [x for x in dirs if not x.startswith(os.curdir)]
|
||||
|
||||
for f in files:
|
||||
valid_extension = f.endswith(tuple(extensions))
|
||||
file_ignored = any(
|
||||
fnmatch.fnmatch(f, ignore) for ignore in ignores
|
||||
)
|
||||
if valid_extension and not file_ignored:
|
||||
try:
|
||||
yield os.stat(os.path.join(root, f)).st_mtime
|
||||
except OSError as e:
|
||||
logger.warning('Caught Exception: %s', e)
|
||||
|
||||
LAST_MTIME = 0
|
||||
while True:
|
||||
try:
|
||||
mtime = max(file_times(path))
|
||||
if mtime > LAST_MTIME:
|
||||
LAST_MTIME = mtime
|
||||
yield True
|
||||
except ValueError:
|
||||
yield None
|
||||
else:
|
||||
yield False
|
||||
|
||||
@staticmethod
|
||||
def file_watcher(path):
|
||||
'''Generator for monitoring a file for modifications'''
|
||||
LAST_MTIME = 0
|
||||
while True:
|
||||
if path:
|
||||
try:
|
||||
mtime = os.stat(path).st_mtime
|
||||
except OSError as e:
|
||||
logger.warning('Caught Exception: %s', e)
|
||||
continue
|
||||
|
||||
if mtime > LAST_MTIME:
|
||||
LAST_MTIME = mtime
|
||||
yield True
|
||||
else:
|
||||
yield False
|
||||
else:
|
||||
yield None
|
||||
return next(watchfiles.watch(
|
||||
*watching_paths,
|
||||
watch_filter=watchfiles.DefaultFilter(
|
||||
ignore_entity_patterns=[fnmatch.translate(pattern) for pattern in ignore_files]
|
||||
),
|
||||
rust_timeout=0
|
||||
))
|
||||
|
||||
|
||||
def set_date_tzinfo(d, tz_name=None):
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ rich = ">=10.1"
|
|||
unidecode = ">=1.1"
|
||||
markdown = {version = ">=3.1", optional = true}
|
||||
backports-zoneinfo = {version = "^0.2.1", python = "<3.9"}
|
||||
watchfiles = "^0.19.0"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
BeautifulSoup4 = "^4.9"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue