From d8172318366f8147c1bee07f0f1cabf3d7256d5d Mon Sep 17 00:00:00 2001
From: MinchinWeb
Date: Thu, 23 Apr 2020 12:49:44 -0600
Subject: [PATCH 01/88] Allow generators to deal with settings that are
`pathlib.Path`s
---
pelican/generators.py | 38 ++++++++++++++++++++++----------------
1 file changed, 22 insertions(+), 16 deletions(-)
diff --git a/pelican/generators.py b/pelican/generators.py
index 63e20a0a..424e9c22 100644
--- a/pelican/generators.py
+++ b/pelican/generators.py
@@ -18,7 +18,6 @@ from pelican.readers import Readers
from pelican.utils import (DateFormatter, copy, mkdir_p, order_content,
posixize_path, process_translations)
-
logger = logging.getLogger(__name__)
@@ -322,8 +321,9 @@ class ArticlesGenerator(CachingGenerator):
all_articles = list(self.articles)
for article in self.articles:
all_articles.extend(article.translations)
- order_content(all_articles,
- order_by=self.settings['ARTICLE_ORDER_BY'])
+ order_content(
+ all_articles, order_by=self.settings['ARTICLE_ORDER_BY']
+ )
if self.settings.get('FEED_ALL_ATOM'):
writer.write_feed(
@@ -352,7 +352,7 @@ class ArticlesGenerator(CachingGenerator):
self.settings['CATEGORY_FEED_ATOM'].format(slug=cat.slug),
self.settings.get(
'CATEGORY_FEED_ATOM_URL',
- self.settings['CATEGORY_FEED_ATOM']).format(
+ str(self.settings['CATEGORY_FEED_ATOM'])).format(
slug=cat.slug
),
feed_title=cat.name
@@ -365,7 +365,7 @@ class ArticlesGenerator(CachingGenerator):
self.settings['CATEGORY_FEED_RSS'].format(slug=cat.slug),
self.settings.get(
'CATEGORY_FEED_RSS_URL',
- self.settings['CATEGORY_FEED_RSS']).format(
+ str(self.settings['CATEGORY_FEED_RSS'])).format(
slug=cat.slug
),
feed_title=cat.name,
@@ -380,8 +380,9 @@ class ArticlesGenerator(CachingGenerator):
self.settings['AUTHOR_FEED_ATOM'].format(slug=auth.slug),
self.settings.get(
'AUTHOR_FEED_ATOM_URL',
- self.settings['AUTHOR_FEED_ATOM']
- ).format(slug=auth.slug),
+ str(self.settings['AUTHOR_FEED_ATOM'])).format(
+ slug=auth.slug
+ ),
feed_title=auth.name
)
@@ -392,8 +393,9 @@ class ArticlesGenerator(CachingGenerator):
self.settings['AUTHOR_FEED_RSS'].format(slug=auth.slug),
self.settings.get(
'AUTHOR_FEED_RSS_URL',
- self.settings['AUTHOR_FEED_RSS']
- ).format(slug=auth.slug),
+ str(self.settings['AUTHOR_FEED_RSS'])).format(
+ slug=auth.slug
+ ),
feed_title=auth.name,
feed_type='rss'
)
@@ -408,8 +410,9 @@ class ArticlesGenerator(CachingGenerator):
self.settings['TAG_FEED_ATOM'].format(slug=tag.slug),
self.settings.get(
'TAG_FEED_ATOM_URL',
- self.settings['TAG_FEED_ATOM']
- ).format(slug=tag.slug),
+ str(self.settings['TAG_FEED_ATOM'])).format(
+ slug=tag.slug
+ ),
feed_title=tag.name
)
@@ -420,8 +423,9 @@ class ArticlesGenerator(CachingGenerator):
self.settings['TAG_FEED_RSS'].format(slug=tag.slug),
self.settings.get(
'TAG_FEED_RSS_URL',
- self.settings['TAG_FEED_RSS']
- ).format(slug=tag.slug),
+ str(self.settings['TAG_FEED_RSS'])).format(
+ slug=tag.slug
+ ),
feed_title=tag.name,
feed_type='rss'
)
@@ -443,7 +447,8 @@ class ArticlesGenerator(CachingGenerator):
.format(lang=lang),
self.settings.get(
'TRANSLATION_FEED_ATOM_URL',
- self.settings['TRANSLATION_FEED_ATOM']
+ str(
+ self.settings['TRANSLATION_FEED_ATOM'])
).format(lang=lang),
)
if self.settings.get('TRANSLATION_FEED_RSS'):
@@ -454,8 +459,9 @@ class ArticlesGenerator(CachingGenerator):
.format(lang=lang),
self.settings.get(
'TRANSLATION_FEED_RSS_URL',
- self.settings['TRANSLATION_FEED_RSS']
- ).format(lang=lang),
+ str(self.settings['TRANSLATION_FEED_RSS'])).format(
+ lang=lang
+ ),
feed_type='rss'
)
From cfba3d72beef60de10311a75f83ebb259269fed8 Mon Sep 17 00:00:00 2001
From: MinchinWeb
Date: Thu, 23 Apr 2020 13:47:10 -0600
Subject: [PATCH 02/88] fix testing failures
when settings could be pathlib.Path
---
pelican/contents.py | 2 +-
pelican/generators.py | 52 +++++++++++++++++++++++-------------------
pelican/settings.py | 4 ++--
pelican/urlwrappers.py | 3 +++
pelican/utils.py | 34 ++++++++++++++++++---------
pelican/writers.py | 2 +-
6 files changed, 58 insertions(+), 39 deletions(-)
diff --git a/pelican/contents.py b/pelican/contents.py
index 2bb2e3a0..2e29d84e 100644
--- a/pelican/contents.py
+++ b/pelican/contents.py
@@ -215,7 +215,7 @@ class Content:
if not klass:
klass = self.__class__.__name__
fq_key = ('{}_{}'.format(klass, key)).upper()
- return self.settings[fq_key].format(**self.url_format)
+ return str(self.settings[fq_key]).format(**self.url_format)
def get_url_setting(self, key):
if hasattr(self, 'override_' + key):
diff --git a/pelican/generators.py b/pelican/generators.py
index 424e9c22..d92e8ff8 100644
--- a/pelican/generators.py
+++ b/pelican/generators.py
@@ -349,12 +349,12 @@ class ArticlesGenerator(CachingGenerator):
writer.write_feed(
arts,
self.context,
- self.settings['CATEGORY_FEED_ATOM'].format(slug=cat.slug),
+ str(self.settings['CATEGORY_FEED_ATOM']).format(slug=cat.slug),
self.settings.get(
'CATEGORY_FEED_ATOM_URL',
- str(self.settings['CATEGORY_FEED_ATOM'])).format(
+ str(self.settings['CATEGORY_FEED_ATOM']).format(
slug=cat.slug
- ),
+ )),
feed_title=cat.name
)
@@ -362,12 +362,12 @@ class ArticlesGenerator(CachingGenerator):
writer.write_feed(
arts,
self.context,
- self.settings['CATEGORY_FEED_RSS'].format(slug=cat.slug),
+ str(self.settings['CATEGORY_FEED_RSS']).format(slug=cat.slug),
self.settings.get(
'CATEGORY_FEED_RSS_URL',
- str(self.settings['CATEGORY_FEED_RSS'])).format(
+ str(self.settings['CATEGORY_FEED_RSS']).format(
slug=cat.slug
- ),
+ )),
feed_title=cat.name,
feed_type='rss'
)
@@ -377,12 +377,12 @@ class ArticlesGenerator(CachingGenerator):
writer.write_feed(
arts,
self.context,
- self.settings['AUTHOR_FEED_ATOM'].format(slug=auth.slug),
+ str(self.settings['AUTHOR_FEED_ATOM']).format(slug=auth.slug),
self.settings.get(
'AUTHOR_FEED_ATOM_URL',
- str(self.settings['AUTHOR_FEED_ATOM'])).format(
+ str(self.settings['AUTHOR_FEED_ATOM']).format(
slug=auth.slug
- ),
+ )),
feed_title=auth.name
)
@@ -390,12 +390,12 @@ class ArticlesGenerator(CachingGenerator):
writer.write_feed(
arts,
self.context,
- self.settings['AUTHOR_FEED_RSS'].format(slug=auth.slug),
+ str(self.settings['AUTHOR_FEED_RSS']).format(slug=auth.slug),
self.settings.get(
'AUTHOR_FEED_RSS_URL',
- str(self.settings['AUTHOR_FEED_RSS'])).format(
+ str(self.settings['AUTHOR_FEED_RSS']).format(
slug=auth.slug
- ),
+ )),
feed_title=auth.name,
feed_type='rss'
)
@@ -407,12 +407,12 @@ class ArticlesGenerator(CachingGenerator):
writer.write_feed(
arts,
self.context,
- self.settings['TAG_FEED_ATOM'].format(slug=tag.slug),
+ str(self.settings['TAG_FEED_ATOM']).format(slug=tag.slug),
self.settings.get(
'TAG_FEED_ATOM_URL',
- str(self.settings['TAG_FEED_ATOM'])).format(
+ str(self.settings['TAG_FEED_ATOM']).format(
slug=tag.slug
- ),
+ )),
feed_title=tag.name
)
@@ -420,12 +420,12 @@ class ArticlesGenerator(CachingGenerator):
writer.write_feed(
arts,
self.context,
- self.settings['TAG_FEED_RSS'].format(slug=tag.slug),
+ str(self.settings['TAG_FEED_RSS']).format(slug=tag.slug),
self.settings.get(
'TAG_FEED_RSS_URL',
- str(self.settings['TAG_FEED_RSS'])).format(
+ str(self.settings['TAG_FEED_RSS']).format(
slug=tag.slug
- ),
+ )),
feed_title=tag.name,
feed_type='rss'
)
@@ -443,27 +443,31 @@ class ArticlesGenerator(CachingGenerator):
writer.write_feed(
items,
self.context,
- self.settings['TRANSLATION_FEED_ATOM']
- .format(lang=lang),
+ str(
+ self.settings['TRANSLATION_FEED_ATOM']
+ ).format(lang=lang),
self.settings.get(
'TRANSLATION_FEED_ATOM_URL',
str(
- self.settings['TRANSLATION_FEED_ATOM'])
+ self.settings['TRANSLATION_FEED_ATOM']
).format(lang=lang),
)
+ )
if self.settings.get('TRANSLATION_FEED_RSS'):
writer.write_feed(
items,
self.context,
- self.settings['TRANSLATION_FEED_RSS']
- .format(lang=lang),
+ str(
+ self.settings['TRANSLATION_FEED_RSS']
+ ).format(lang=lang),
self.settings.get(
'TRANSLATION_FEED_RSS_URL',
- str(self.settings['TRANSLATION_FEED_RSS'])).format(
+ str(self.settings['TRANSLATION_FEED_RSS']).format(
lang=lang
),
feed_type='rss'
)
+ )
def generate_articles(self, write):
"""Generate the articles."""
diff --git a/pelican/settings.py b/pelican/settings.py
index 7b333de8..ddb6748d 100644
--- a/pelican/settings.py
+++ b/pelican/settings.py
@@ -406,7 +406,7 @@ def handle_deprecated_settings(settings):
for key in ['TRANSLATION_FEED_ATOM',
'TRANSLATION_FEED_RSS'
]:
- if settings.get(key) and '%s' in settings[key]:
+ if settings.get(key) and not isinstance(settings[key], Path) and '%s' in settings[key]:
logger.warning('%%s usage in %s is deprecated, use {lang} '
'instead.', key)
try:
@@ -423,7 +423,7 @@ def handle_deprecated_settings(settings):
'TAG_FEED_ATOM',
'TAG_FEED_RSS',
]:
- if settings.get(key) and '%s' in settings[key]:
+ if settings.get(key) and not isinstance(settings[key], Path) and '%s' in settings[key]:
logger.warning('%%s usage in %s is deprecated, use {slug} '
'instead.', key)
try:
diff --git a/pelican/urlwrappers.py b/pelican/urlwrappers.py
index efe09fbc..e00b914c 100644
--- a/pelican/urlwrappers.py
+++ b/pelican/urlwrappers.py
@@ -1,6 +1,7 @@
import functools
import logging
import os
+import pathlib
from pelican.utils import slugify
@@ -110,6 +111,8 @@ class URLWrapper:
"""
setting = "{}_{}".format(self.__class__.__name__.upper(), key)
value = self.settings[setting]
+ if isinstance(value, pathlib.Path):
+ value = str(value)
if not isinstance(value, str):
logger.warning('%s is set to %s', setting, value)
return value
diff --git a/pelican/utils.py b/pelican/utils.py
index e82117d3..a3ece8ce 100644
--- a/pelican/utils.py
+++ b/pelican/utils.py
@@ -3,6 +3,7 @@ import fnmatch
import locale
import logging
import os
+import pathlib
import re
import shutil
import sys
@@ -921,17 +922,28 @@ def split_all(path):
>>> split_all(os.path.join('a', 'b', 'c'))
['a', 'b', 'c']
"""
- components = []
- path = path.lstrip('/')
- while path:
- head, tail = os.path.split(path)
- if tail:
- components.insert(0, tail)
- elif head == path:
- components.insert(0, head)
- break
- path = head
- return components
+ if isinstance(path, str):
+ components = []
+ path = path.lstrip('/')
+ while path:
+ head, tail = os.path.split(path)
+ if tail:
+ components.insert(0, tail)
+ elif head == path:
+ components.insert(0, head)
+ break
+ path = head
+ return components
+ elif isinstance(path, pathlib.Path):
+ return path.parts
+ elif path is None:
+ return None
+ else:
+ raise TypeError(
+ '"path" was {}, must be string, None, or pathlib.Path'.format(
+ type(path)
+ )
+ )
def is_selected_for_writing(settings, path):
diff --git a/pelican/writers.py b/pelican/writers.py
index 9b27a748..379af1f4 100644
--- a/pelican/writers.py
+++ b/pelican/writers.py
@@ -29,7 +29,7 @@ class Writer:
self.urljoiner = posix_join
else:
self.urljoiner = lambda base, url: urljoin(
- base if base.endswith('/') else base + '/', url)
+ base if base.endswith('/') else base + '/', str(url))
def _create_new_feed(self, feed_type, feed_title, context):
feed_class = Rss201rev2Feed if feed_type == 'rss' else Atom1Feed
From a13371670970661c7d3f673e9c634df83f7a95bf Mon Sep 17 00:00:00 2001
From: MinchinWeb
Date: Thu, 21 May 2020 21:43:06 -0600
Subject: [PATCH 03/88] flake8 fixes
---
pelican/generators.py | 3 +--
pelican/settings.py | 11 +++++++++--
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/pelican/generators.py b/pelican/generators.py
index d92e8ff8..ecc06851 100644
--- a/pelican/generators.py
+++ b/pelican/generators.py
@@ -462,11 +462,10 @@ class ArticlesGenerator(CachingGenerator):
).format(lang=lang),
self.settings.get(
'TRANSLATION_FEED_RSS_URL',
- str(self.settings['TRANSLATION_FEED_RSS']).format(
+ str(self.settings['TRANSLATION_FEED_RSS'])).format(
lang=lang
),
feed_type='rss'
- )
)
def generate_articles(self, write):
diff --git a/pelican/settings.py b/pelican/settings.py
index ddb6748d..a5e39161 100644
--- a/pelican/settings.py
+++ b/pelican/settings.py
@@ -6,6 +6,7 @@ import logging
import os
import re
from os.path import isabs
+from pathlib import Path
from pelican.log import LimitFilter
@@ -406,7 +407,10 @@ def handle_deprecated_settings(settings):
for key in ['TRANSLATION_FEED_ATOM',
'TRANSLATION_FEED_RSS'
]:
- if settings.get(key) and not isinstance(settings[key], Path) and '%s' in settings[key]:
+ if (
+ settings.get(key) and not isinstance(settings[key], Path)
+ and '%s' in settings[key]
+ ):
logger.warning('%%s usage in %s is deprecated, use {lang} '
'instead.', key)
try:
@@ -423,7 +427,10 @@ def handle_deprecated_settings(settings):
'TAG_FEED_ATOM',
'TAG_FEED_RSS',
]:
- if settings.get(key) and not isinstance(settings[key], Path) and '%s' in settings[key]:
+ if (
+ settings.get(key) and not isinstance(settings[key], Path)
+ and '%s' in settings[key]
+ ):
logger.warning('%%s usage in %s is deprecated, use {slug} '
'instead.', key)
try:
From b10c7c699b49b1701692c0d0edc7691ac71c3c8f Mon Sep 17 00:00:00 2001
From: Ryan de Kleer
Date: Thu, 15 Sep 2022 16:51:34 -0700
Subject: [PATCH 04/88] Fix false-positive in content gen. test failures
Assert equal dirs by return value of diff subprocess, rather than its output.
This prevents tests from failing when file contents are the same but the
file modes are different.
Fix #3042
---
pelican/tests/support.py | 17 ++++++++++++
pelican/tests/test_pelican.py | 51 +++++++++++++++++------------------
2 files changed, 42 insertions(+), 26 deletions(-)
diff --git a/pelican/tests/support.py b/pelican/tests/support.py
index 55ddf625..8a394395 100644
--- a/pelican/tests/support.py
+++ b/pelican/tests/support.py
@@ -218,6 +218,23 @@ class LogCountHandler(BufferingHandler):
])
+def diff_subproc(first, second):
+ """
+ Return a subprocess that runs a diff on the two paths.
+
+ Check results with::
+
+ >>> out_stream, err_stream = proc.communicate()
+ >>> didCheckFail = proc.returnCode != 0
+ """
+ return subprocess.Popen(
+ ['git', '--no-pager', 'diff', '--no-ext-diff', '--exit-code',
+ '-w', first, second],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+
+
class LoggedTestCase(unittest.TestCase):
"""A test case that captures log messages."""
diff --git a/pelican/tests/test_pelican.py b/pelican/tests/test_pelican.py
index 389dbb3d..adba32e0 100644
--- a/pelican/tests/test_pelican.py
+++ b/pelican/tests/test_pelican.py
@@ -3,6 +3,7 @@ import logging
import os
import subprocess
import sys
+import unittest
from collections.abc import Sequence
from shutil import rmtree
from tempfile import mkdtemp
@@ -10,8 +11,12 @@ from tempfile import mkdtemp
from pelican import Pelican
from pelican.generators import StaticGenerator
from pelican.settings import read_settings
-from pelican.tests.support import (LoggedTestCase, locale_available,
- mute, unittest)
+from pelican.tests.support import (
+ LoggedTestCase,
+ diff_subproc,
+ locale_available,
+ mute
+)
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
SAMPLES_PATH = os.path.abspath(os.path.join(
@@ -54,28 +59,19 @@ class TestPelican(LoggedTestCase):
locale.setlocale(locale.LC_ALL, self.old_locale)
super().tearDown()
- def assertDirsEqual(self, left_path, right_path):
- out, err = subprocess.Popen(
- ['git', '--no-pager', 'diff', '--no-ext-diff', '--exit-code',
- '-w', left_path, right_path],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE
- ).communicate()
+ def assertDirsEqual(self, left_path, right_path, msg=None):
+ """
+ Check if the files are the same (ignoring whitespace) below both paths.
+ """
+ proc = diff_subproc(left_path, right_path)
- def ignorable_git_crlf_errors(line):
- # Work around for running tests on Windows
- for msg in [
- "LF will be replaced by CRLF",
- "CRLF will be replaced by LF",
- "The file will have its original line endings"]:
- if msg in line:
- return True
- return False
- if err:
- err = '\n'.join([line for line in err.decode('utf8').splitlines()
- if not ignorable_git_crlf_errors(line)])
- assert not out, out
- assert not err, err
+ out, err = proc.communicate()
+ if proc.returncode != 0:
+ msg = self._formatMessage(
+ msg,
+ "%s and %s differ:\n%s" % (left_path, right_path, err)
+ )
+ raise self.failureException(msg)
def test_order_of_generators(self):
# StaticGenerator must run last, so it can identify files that
@@ -104,7 +100,8 @@ class TestPelican(LoggedTestCase):
pelican = Pelican(settings=settings)
mute(True)(pelican.run)()
self.assertDirsEqual(
- self.temp_path, os.path.join(OUTPUT_PATH, 'basic'))
+ self.temp_path, os.path.join(OUTPUT_PATH, 'basic')
+ )
self.assertLogCountEqual(
count=1,
msg="Unable to find.*skipping url replacement",
@@ -121,7 +118,8 @@ class TestPelican(LoggedTestCase):
pelican = Pelican(settings=settings)
mute(True)(pelican.run)()
self.assertDirsEqual(
- self.temp_path, os.path.join(OUTPUT_PATH, 'custom'))
+ self.temp_path, os.path.join(OUTPUT_PATH, 'custom')
+ )
@unittest.skipUnless(locale_available('fr_FR.UTF-8') or
locale_available('French'), 'French locale needed')
@@ -141,7 +139,8 @@ class TestPelican(LoggedTestCase):
pelican = Pelican(settings=settings)
mute(True)(pelican.run)()
self.assertDirsEqual(
- self.temp_path, os.path.join(OUTPUT_PATH, 'custom_locale'))
+ self.temp_path, os.path.join(OUTPUT_PATH, 'custom_locale')
+ )
def test_theme_static_paths_copy(self):
# the same thing with a specified set of settings should work
From bb682973fb9e71e378bc74b9e7130508721f3a44 Mon Sep 17 00:00:00 2001
From: "Martin (mart-e)"
Date: Sat, 6 May 2023 08:40:29 +0200
Subject: [PATCH 05/88] Don't specify unlimited feed size by default
Having a feed with hundreds of articles, making a very large file, is
rarely expected.
Set a high fallback value of 100 so it does not change for small sites.
Still allow to have infinite feed by setting FEED_MAX_ITEM = None
---
docs/settings.rst | 6 +++---
pelican/settings.py | 2 +-
pelican/writers.py | 8 +++-----
3 files changed, 7 insertions(+), 9 deletions(-)
diff --git a/docs/settings.rst b/docs/settings.rst
index e51c6a12..0c0353a3 100644
--- a/docs/settings.rst
+++ b/docs/settings.rst
@@ -998,10 +998,10 @@ the ``TAG_FEED_ATOM`` and ``TAG_FEED_RSS`` settings:
placeholder. If not set, ``TAG_FEED_RSS`` is used both for save location and
URL.
-.. data:: FEED_MAX_ITEMS
+.. data:: FEED_MAX_ITEMS = 100
- Maximum number of items allowed in a feed. Feed item quantity is
- unrestricted by default.
+ Maximum number of items allowed in a feed. Setting to ``None`` will cause the
+ feed to contains every article. 100 if not specified.
.. data:: RSS_FEED_SUMMARY_ONLY = True
diff --git a/pelican/settings.py b/pelican/settings.py
index 5b495e86..f38b46f0 100644
--- a/pelican/settings.py
+++ b/pelican/settings.py
@@ -40,7 +40,7 @@ DEFAULT_CONFIG = {
'AUTHOR_FEED_ATOM': 'feeds/{slug}.atom.xml',
'AUTHOR_FEED_RSS': 'feeds/{slug}.rss.xml',
'TRANSLATION_FEED_ATOM': 'feeds/all-{lang}.atom.xml',
- 'FEED_MAX_ITEMS': '',
+ 'FEED_MAX_ITEMS': 100,
'RSS_FEED_SUMMARY_ONLY': True,
'SITEURL': '',
'SITENAME': 'A Pelican Blog',
diff --git a/pelican/writers.py b/pelican/writers.py
index 73ee4b33..f0280269 100644
--- a/pelican/writers.py
+++ b/pelican/writers.py
@@ -143,11 +143,9 @@ class Writer:
feed = self._create_new_feed(feed_type, feed_title, context)
- max_items = len(elements)
- if self.settings['FEED_MAX_ITEMS']:
- max_items = min(self.settings['FEED_MAX_ITEMS'], max_items)
- for i in range(max_items):
- self._add_item_to_the_feed(feed, elements[i])
+ # FEED_MAX_ITEMS = None means [:None] to get every element
+ for element in elements[:self.settings['FEED_MAX_ITEMS']]:
+ self._add_item_to_the_feed(feed, element)
signals.feed_generated.send(context, feed=feed)
if path:
From 5214248344ba75ec1e0ea804bcfcc25bc4b533e2 Mon Sep 17 00:00:00 2001
From: DJ Ramones <50655786+djramones@users.noreply.github.com>
Date: Sun, 18 Jun 2023 11:07:39 +0800
Subject: [PATCH 06/88] Implement period_archives common context variable
Also, set default patterns for time-period *_ARCHIVE_URL settings.
---
docs/settings.rst | 25 +++---
docs/themes.rst | 61 ++++++++++++++
pelican/generators.py | 135 ++++++++++++++++++-------------
pelican/settings.py | 6 +-
pelican/tests/test_generators.py | 97 ++++++++++++++++++++++
5 files changed, 255 insertions(+), 69 deletions(-)
diff --git a/docs/settings.rst b/docs/settings.rst
index e51c6a12..259b53f5 100644
--- a/docs/settings.rst
+++ b/docs/settings.rst
@@ -572,33 +572,36 @@ posts for the month at ``posts/2011/Aug/index.html``.
This way a reader can remove a portion of your URL and automatically arrive
at an appropriate archive of posts, without having to specify a page name.
-.. data:: YEAR_ARCHIVE_URL = ''
-
- The URL to use for per-year archives of your posts. Used only if you have
- the ``{url}`` placeholder in ``PAGINATION_PATTERNS``.
-
.. data:: YEAR_ARCHIVE_SAVE_AS = ''
The location to save per-year archives of your posts.
-.. data:: MONTH_ARCHIVE_URL = ''
+.. data:: YEAR_ARCHIVE_URL = 'posts/{date:%Y}/'
- The URL to use for per-month archives of your posts. Used only if you have
- the ``{url}`` placeholder in ``PAGINATION_PATTERNS``.
+ The URL to use for per-year archives of your posts. This default value
+ matches a ``YEAR_ARCHIVE_SAVE_AS`` setting of
+ ``posts/{date:%Y}/index.html``.
.. data:: MONTH_ARCHIVE_SAVE_AS = ''
The location to save per-month archives of your posts.
-.. data:: DAY_ARCHIVE_URL = ''
+.. data:: MONTH_ARCHIVE_URL = 'posts/{date:%Y}/{date:%b}/'
- The URL to use for per-day archives of your posts. Used only if you have the
- ``{url}`` placeholder in ``PAGINATION_PATTERNS``.
+ The URL to use for per-month archives of your posts. This default value
+ matches a ``MONTH_ARCHIVE_SAVE_AS`` setting of
+ ``posts/{date:%Y}/{date:%b}/index.html``.
.. data:: DAY_ARCHIVE_SAVE_AS = ''
The location to save per-day archives of your posts.
+.. data:: DAY_ARCHIVE_URL = 'posts/{date:%Y}/{date:%b}/{date:%d}/'
+
+ The URL to use for per-day archives of your posts. This default value
+ matches a ``DAY_ARCHIVE_SAVE_AS`` setting of
+ ``posts/{date:%Y}/{date:%b}/{date:%d}/index.html``.
+
``DIRECT_TEMPLATES`` work a bit differently than noted above. Only the
``_SAVE_AS`` settings are available, but it is available for any direct
template.
diff --git a/docs/themes.rst b/docs/themes.rst
index fe6337d6..8e46b716 100644
--- a/docs/themes.rst
+++ b/docs/themes.rst
@@ -71,6 +71,8 @@ All templates will receive the variables defined in your settings file, as long
as they are in all-caps. You can access them directly.
+.. _common_variables:
+
Common Variables
----------------
@@ -92,6 +94,10 @@ dates The same list of articles, but ordered by date,
ascending.
hidden_articles The list of hidden articles
drafts The list of draft articles
+period_archives A dictionary containing elements related to
+ time-period archives (if enabled). See the section
+ :ref:`Listing and Linking to Period Archives
+ ` for details.
authors A list of (author, articles) tuples, containing all
the authors and corresponding articles (values)
categories A list of (category, articles) tuples, containing
@@ -348,6 +354,61 @@ period_archives.html template
`_.
+.. _period_archives_variable:
+
+Listing and Linking to Period Archives
+""""""""""""""""""""""""""""""""""""""
+
+The ``period_archives`` variable can be used to generate a list of links to
+the set of period archives that Pelican generates. As a :ref:`common variable
+`, it is available for use in any template, so you
+can implement such an index in a custom direct template, or in a sidebar
+visible across different site pages.
+
+``period_archives`` is a dict that may contain ``year``, ``month``, and/or
+``day`` keys, depending on which ``*_ARCHIVE_SAVE_AS`` settings are enabled.
+The corresponding value is a list of dicts, where each dict in turn represents
+a time period, with the following keys and values:
+
+=================== ===================================================
+Key Value
+=================== ===================================================
+period The same tuple as described in
+ ``period_archives.html``, e.g.
+ ``(2023, 'June', 18)``.
+period_num The same tuple as described in
+ ``period_archives.html``, e.g. ``(2023, 6, 18)``.
+url The URL to the period archive page, e.g.
+ ``posts/2023/06/18/``. This is controlled by the
+ corresponding ``*_ARCHIVE_URL`` setting.
+save_as The path to the save location of the period archive
+ page file, e.g. ``posts/2023/06/18/index.html``.
+ This is used internally by Pelican and is usually
+ not relevant to themes.
+articles A list of :ref:`Article ` objects
+ that fall under the time period.
+dates Same list as ``articles``, but ordered by date.
+=================== ===================================================
+
+Here is an example of how ``period_archives`` can be used in a template:
+
+.. code-block:: html+jinja
+
+
\n'
source = fmtstr % post.get('source_url')
caption = post.get('caption')
- players = '\n'.join(player.get('embed_code')
- for player in post.get('player'))
+ players = [
+ # If embed_code is False, couldn't get the video
+ player.get('embed_code') or None
+ for player in post.get('player')]
+ # If there are no embeddable players, say so, once
+ if len(players) > 0 and all(
+ player is None for player in players):
+ players = "
(This video isn't available anymore.)
\n"
+ else:
+ players = '\n'.join(players)
content = source + caption + players
elif type == 'answer':
title = post.get('question')
From b6a9a8333b285e9781d290187c69b5667f428579 Mon Sep 17 00:00:00 2001
From: Deniz Turgut
Date: Sat, 28 Oct 2023 14:49:55 +0300
Subject: [PATCH 23/88] skip tests that require git if git is not installed
and minor tweaks to subprocess handling
---
pelican/tests/support.py | 3 ++-
pelican/tests/test_pelican.py | 9 +++++++--
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/pelican/tests/support.py b/pelican/tests/support.py
index 8a394395..720e4d0e 100644
--- a/pelican/tests/support.py
+++ b/pelican/tests/support.py
@@ -231,7 +231,8 @@ def diff_subproc(first, second):
['git', '--no-pager', 'diff', '--no-ext-diff', '--exit-code',
'-w', first, second],
stdout=subprocess.PIPE,
- stderr=subprocess.PIPE
+ stderr=subprocess.PIPE,
+ text=True,
)
diff --git a/pelican/tests/test_pelican.py b/pelican/tests/test_pelican.py
index adba32e0..885c2138 100644
--- a/pelican/tests/test_pelican.py
+++ b/pelican/tests/test_pelican.py
@@ -15,7 +15,8 @@ from pelican.tests.support import (
LoggedTestCase,
diff_subproc,
locale_available,
- mute
+ mute,
+ skipIfNoExecutable,
)
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -69,7 +70,8 @@ class TestPelican(LoggedTestCase):
if proc.returncode != 0:
msg = self._formatMessage(
msg,
- "%s and %s differ:\n%s" % (left_path, right_path, err)
+ "%s and %s differ:\nstdout:\n%s\nstderr\n%s" %
+ (left_path, right_path, out, err)
)
raise self.failureException(msg)
@@ -88,6 +90,7 @@ class TestPelican(LoggedTestCase):
generator_classes, Sequence,
"_get_generator_classes() must return a Sequence to preserve order")
+ @skipIfNoExecutable(['git', '--version'])
def test_basic_generation_works(self):
# when running pelican without settings, it should pick up the default
# ones and generate correct output without raising any exception
@@ -107,6 +110,7 @@ class TestPelican(LoggedTestCase):
msg="Unable to find.*skipping url replacement",
level=logging.WARNING)
+ @skipIfNoExecutable(['git', '--version'])
def test_custom_generation_works(self):
# the same thing with a specified set of settings should work
settings = read_settings(path=SAMPLE_CONFIG, override={
@@ -121,6 +125,7 @@ class TestPelican(LoggedTestCase):
self.temp_path, os.path.join(OUTPUT_PATH, 'custom')
)
+ @skipIfNoExecutable(['git', '--version'])
@unittest.skipUnless(locale_available('fr_FR.UTF-8') or
locale_available('French'), 'French locale needed')
def test_custom_locale_generation_works(self):
From dc427ad9d6e460debe0a667cfd41d6d9ca4133d1 Mon Sep 17 00:00:00 2001
From: Gullumluvl <7593801+Gullumluvl@users.noreply.github.com>
Date: Sat, 28 Oct 2023 14:24:16 +0200
Subject: [PATCH 24/88] Strip HTML tags from SITENAME inside title tags. Fixes
#3147 (#3149)
---
pelican/themes/notmyidea/templates/author.html | 2 +-
.../themes/notmyidea/templates/authors.html | 2 +-
pelican/themes/notmyidea/templates/base.html | 6 +++---
.../themes/notmyidea/templates/categories.html | 2 +-
.../themes/notmyidea/templates/category.html | 2 +-
pelican/themes/notmyidea/templates/tag.html | 2 +-
pelican/themes/notmyidea/templates/tags.html | 2 +-
pelican/themes/simple/templates/archives.html | 2 +-
pelican/themes/simple/templates/article.html | 2 +-
pelican/themes/simple/templates/author.html | 2 +-
pelican/themes/simple/templates/authors.html | 2 +-
pelican/themes/simple/templates/base.html | 18 +++++++++---------
.../themes/simple/templates/categories.html | 2 +-
pelican/themes/simple/templates/category.html | 2 +-
pelican/themes/simple/templates/page.html | 2 +-
.../simple/templates/period_archives.html | 2 +-
pelican/themes/simple/templates/tag.html | 2 +-
pelican/themes/simple/templates/tags.html | 2 +-
18 files changed, 28 insertions(+), 28 deletions(-)
diff --git a/pelican/themes/notmyidea/templates/author.html b/pelican/themes/notmyidea/templates/author.html
index 0b372902..536ac50d 100644
--- a/pelican/themes/notmyidea/templates/author.html
+++ b/pelican/themes/notmyidea/templates/author.html
@@ -1,2 +1,2 @@
{% extends "index.html" %}
-{% block title %}{{ SITENAME }} - {{ author }}{% endblock %}
+{% block title %}{{ SITENAME|striptags }} - {{ author }}{% endblock %}
diff --git a/pelican/themes/notmyidea/templates/authors.html b/pelican/themes/notmyidea/templates/authors.html
index e61a332f..b9f87e22 100644
--- a/pelican/themes/notmyidea/templates/authors.html
+++ b/pelican/themes/notmyidea/templates/authors.html
@@ -1,6 +1,6 @@
{% extends "base.html" %}
-{% block title %}{{ SITENAME }} - Authors{% endblock %}
+{% block title %}{{ SITENAME|striptags }} - Authors{% endblock %}
{% block content %}
diff --git a/pelican/themes/notmyidea/templates/base.html b/pelican/themes/notmyidea/templates/base.html
index 2b302899..8483f268 100644
--- a/pelican/themes/notmyidea/templates/base.html
+++ b/pelican/themes/notmyidea/templates/base.html
@@ -5,13 +5,13 @@
- {% block title %}{{ SITENAME }}{%endblock%}
+ {% block title %}{{ SITENAME|striptags }}{%endblock%}
{% if FEED_ALL_ATOM %}
-
+
{% endif %}
{% if FEED_ALL_RSS %}
-
+
{% endif %}
{% block extra_head %}{% endblock extra_head %}
{% endblock head %}
diff --git a/pelican/themes/notmyidea/templates/categories.html b/pelican/themes/notmyidea/templates/categories.html
index 07f6290a..7c5951c7 100644
--- a/pelican/themes/notmyidea/templates/categories.html
+++ b/pelican/themes/notmyidea/templates/categories.html
@@ -1,6 +1,6 @@
{% extends "base.html" %}
-{% block title %}{{ SITENAME }} - Categories{% endblock %}
+{% block title %}{{ SITENAME|striptags }} - Categories{% endblock %}
{% block content %}
diff --git a/pelican/themes/notmyidea/templates/category.html b/pelican/themes/notmyidea/templates/category.html
index 56f8e93e..ff14ed76 100644
--- a/pelican/themes/notmyidea/templates/category.html
+++ b/pelican/themes/notmyidea/templates/category.html
@@ -1,2 +1,2 @@
{% extends "index.html" %}
-{% block title %}{{ SITENAME }} - {{ category }}{% endblock %}
+{% block title %}{{ SITENAME|striptags }} - {{ category }}{% endblock %}
diff --git a/pelican/themes/notmyidea/templates/tag.html b/pelican/themes/notmyidea/templates/tag.html
index 68cdcba6..1e32857b 100644
--- a/pelican/themes/notmyidea/templates/tag.html
+++ b/pelican/themes/notmyidea/templates/tag.html
@@ -1,2 +1,2 @@
{% extends "index.html" %}
-{% block title %}{{ SITENAME }} - {{ tag }}{% endblock %}
+{% block title %}{{ SITENAME|striptags }} - {{ tag }}{% endblock %}
diff --git a/pelican/themes/notmyidea/templates/tags.html b/pelican/themes/notmyidea/templates/tags.html
index fb099557..a1729321 100644
--- a/pelican/themes/notmyidea/templates/tags.html
+++ b/pelican/themes/notmyidea/templates/tags.html
@@ -1,6 +1,6 @@
{% extends "base.html" %}
-{% block title %}{{ SITENAME }} - Tags{% endblock %}
+{% block title %}{{ SITENAME|striptags }} - Tags{% endblock %}
{% block content %}
diff --git a/pelican/themes/simple/templates/archives.html b/pelican/themes/simple/templates/archives.html
index cd129507..b7754c45 100644
--- a/pelican/themes/simple/templates/archives.html
+++ b/pelican/themes/simple/templates/archives.html
@@ -1,6 +1,6 @@
{% extends "base.html" %}
-{% block title %}{{ SITENAME }} - Archives{% endblock %}
+{% block title %}{{ SITENAME|striptags }} - Archives{% endblock %}
{% block content %}
\n',
'2016-08-14-the-slug',
- '2016-08-14 16:37:35', 'testy', ['video'], ['interviews'],
+ '2016-08-14 16:37:35+0000', 'testy', ['video'], ['interviews'],
'published', 'article', 'html')],
posts,
posts)
diff --git a/pelican/tools/pelican_import.py b/pelican/tools/pelican_import.py
index 16ce6305..44568161 100755
--- a/pelican/tools/pelican_import.py
+++ b/pelican/tools/pelican_import.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
import argparse
+import datetime
import logging
import os
import re
@@ -416,10 +417,12 @@ def tumblr2fields(api_key, blogname):
slug = post.get('slug') or slugify(title, regex_subs=subs)
tags = post.get('tags')
timestamp = post.get('timestamp')
- date = SafeDatetime.fromtimestamp(int(timestamp)).strftime(
- "%Y-%m-%d %H:%M:%S")
- slug = SafeDatetime.fromtimestamp(int(timestamp)).strftime(
- "%Y-%m-%d-") + slug
+ date = SafeDatetime.fromtimestamp(
+ int(timestamp), tz=datetime.timezone.utc
+ ).strftime("%Y-%m-%d %H:%M:%S%z")
+ slug = SafeDatetime.fromtimestamp(
+ int(timestamp), tz=datetime.timezone.utc
+ ).strftime("%Y-%m-%d-") + slug
format = post.get('format')
content = post.get('body')
type = post.get('type')
From 11c13ceae1c72bd786a1b09657de2926eb6ae267 Mon Sep 17 00:00:00 2001
From: Deniz Turgut
Date: Sat, 28 Oct 2023 16:31:05 +0300
Subject: [PATCH 26/88] use a tempfile for intermediate html file for pandoc in
importer
---
pelican/tools/pelican_import.py | 64 ++++++++++++++++-----------------
1 file changed, 31 insertions(+), 33 deletions(-)
diff --git a/pelican/tools/pelican_import.py b/pelican/tools/pelican_import.py
index 44568161..95e196ba 100755
--- a/pelican/tools/pelican_import.py
+++ b/pelican/tools/pelican_import.py
@@ -7,6 +7,7 @@ import os
import re
import subprocess
import sys
+import tempfile
import time
from collections import defaultdict
from html import unescape
@@ -785,9 +786,8 @@ def fields2pelican(
print(out_filename)
if in_markup in ('html', 'wp-html'):
- html_filename = os.path.join(output_path, filename + '.html')
-
- with open(html_filename, 'w', encoding='utf-8') as fp:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ html_filename = os.path.join(tmpdir, 'pandoc-input.html')
# Replace newlines with paragraphs wrapped with
so
# HTML is valid before conversion
if in_markup == 'wp-html':
@@ -796,41 +796,39 @@ def fields2pelican(
paragraphs = content.splitlines()
paragraphs = ['
{}
'.format(p) for p in paragraphs]
new_content = ''.join(paragraphs)
+ with open(html_filename, 'w', encoding='utf-8') as fp:
+ fp.write(new_content)
- fp.write(new_content)
+ if pandoc_version < (2,):
+ parse_raw = '--parse-raw' if not strip_raw else ''
+ wrap_none = '--wrap=none' \
+ if pandoc_version >= (1, 16) else '--no-wrap'
+ cmd = ('pandoc --normalize {0} --from=html'
+ ' --to={1} {2} -o "{3}" "{4}"')
+ cmd = cmd.format(parse_raw,
+ out_markup if out_markup != 'markdown' else "gfm",
+ wrap_none,
+ out_filename, html_filename)
+ else:
+ from_arg = '-f html+raw_html' if not strip_raw else '-f html'
+ cmd = ('pandoc {0} --to={1}-smart --wrap=none -o "{2}" "{3}"')
+ cmd = cmd.format(from_arg,
+ out_markup if out_markup != 'markdown' else "gfm",
+ out_filename, html_filename)
- if pandoc_version < (2,):
- parse_raw = '--parse-raw' if not strip_raw else ''
- wrap_none = '--wrap=none' \
- if pandoc_version >= (1, 16) else '--no-wrap'
- cmd = ('pandoc --normalize {0} --from=html'
- ' --to={1} {2} -o "{3}" "{4}"')
- cmd = cmd.format(parse_raw,
- out_markup if out_markup != 'markdown' else "gfm",
- wrap_none,
- out_filename, html_filename)
- else:
- from_arg = '-f html+raw_html' if not strip_raw else '-f html'
- cmd = ('pandoc {0} --to={1}-smart --wrap=none -o "{2}" "{3}"')
- cmd = cmd.format(from_arg,
- out_markup if out_markup != 'markdown' else "gfm",
- out_filename, html_filename)
+ try:
+ rc = subprocess.call(cmd, shell=True)
+ if rc < 0:
+ error = 'Child was terminated by signal %d' % -rc
+ exit(error)
- try:
- rc = subprocess.call(cmd, shell=True)
- if rc < 0:
- error = 'Child was terminated by signal %d' % -rc
+ elif rc > 0:
+ error = 'Please, check your Pandoc installation.'
+ exit(error)
+ except OSError as e:
+ error = 'Pandoc execution failed: %s' % e
exit(error)
- elif rc > 0:
- error = 'Please, check your Pandoc installation.'
- exit(error)
- except OSError as e:
- error = 'Pandoc execution failed: %s' % e
- exit(error)
-
- os.remove(html_filename)
-
with open(out_filename, encoding='utf-8') as fs:
content = fs.read()
if out_markup == 'markdown':
From 61ca47c5194f33ad28e96cf965ba9f582d6d7909 Mon Sep 17 00:00:00 2001
From: Jake Howard
Date: Wed, 21 Jun 2023 22:01:38 +0100
Subject: [PATCH 27/88] Use watchfiles as a file watching backend
This doesn't use polling unless absolutely necessarily, making it more efficient. It also reduces the amount of first-party code required, and simplifies working out which files are being watched.
---
pelican/__init__.py | 25 ++---
pelican/tests/test_utils.py | 86 -----------------
pelican/utils.py | 180 +++++-------------------------------
pyproject.toml | 1 +
4 files changed, 32 insertions(+), 260 deletions(-)
diff --git a/pelican/__init__.py b/pelican/__init__.py
index f0af3429..a4f5b38e 100644
--- a/pelican/__init__.py
+++ b/pelican/__init__.py
@@ -27,7 +27,7 @@ from pelican.plugins._utils import get_plugin_name, load_plugins
from pelican.readers import Readers
from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer
from pelican.settings import read_settings
-from pelican.utils import (FileSystemWatcher, clean_output_dir, maybe_pluralize)
+from pelican.utils import (wait_for_changes, clean_output_dir, maybe_pluralize)
from pelican.writers import Writer
try:
@@ -452,26 +452,19 @@ def autoreload(args, excqueue=None):
console.print(' --- AutoReload Mode: Monitoring `content`, `theme` and'
' `settings` for changes. ---')
pelican, settings = get_instance(args)
- watcher = FileSystemWatcher(args.settings, Readers, settings)
- sleep = False
+ settings_file = os.path.abspath(args.settings)
while True:
try:
- # Don't sleep first time, but sleep afterwards to reduce cpu load
- if sleep:
- time.sleep(0.5)
- else:
- sleep = True
+ changed_files = wait_for_changes(args.settings, Readers, settings)
- modified = watcher.check()
+ changed_files = {c[1] for c in changed_files}
- if modified['settings']:
+ if settings_file in changed_files:
pelican, settings = get_instance(args)
- watcher.update_watchers(settings)
- if any(modified.values()):
- console.print('\n-> Modified: {}. re-generating...'.format(
- ', '.join(k for k, v in modified.items() if v)))
- pelican.run()
+ console.print('\n-> Modified: {}. re-generating...'.format(
+ ', '.join(changed_files)))
+ pelican.run()
except KeyboardInterrupt:
if excqueue is not None:
@@ -558,8 +551,6 @@ def main(argv=None):
listen(settings.get('BIND'), settings.get('PORT'),
settings.get("OUTPUT_PATH"))
else:
- watcher = FileSystemWatcher(args.settings, Readers, settings)
- watcher.check()
with console.status("Generating..."):
pelican.run()
except KeyboardInterrupt:
diff --git a/pelican/tests/test_utils.py b/pelican/tests/test_utils.py
index e1758726..7ff1af7a 100644
--- a/pelican/tests/test_utils.py
+++ b/pelican/tests/test_utils.py
@@ -412,92 +412,6 @@ class TestUtils(LoggedTestCase):
self.assertNotIn(a_arts[4], b_arts[5].translations)
self.assertNotIn(a_arts[5], b_arts[4].translations)
- def test_filesystemwatcher(self):
- def create_file(name, content):
- with open(name, 'w') as f:
- f.write(content)
-
- # disable logger filter
- from pelican.utils import logger
- logger.disable_filter()
-
- # create a temp "project" dir
- root = mkdtemp()
- content_path = os.path.join(root, 'content')
- static_path = os.path.join(root, 'content', 'static')
- config_file = os.path.join(root, 'config.py')
- theme_path = os.path.join(root, 'mytheme')
-
- # populate
- os.mkdir(content_path)
- os.mkdir(theme_path)
- create_file(config_file,
- 'PATH = "content"\n'
- 'THEME = "mytheme"\n'
- 'STATIC_PATHS = ["static"]')
-
- t = time.time() - 1000 # make sure it's in the "past"
- os.utime(config_file, (t, t))
- settings = read_settings(config_file)
-
- watcher = utils.FileSystemWatcher(config_file, Readers, settings)
- # should get a warning for static not not existing
- self.assertLogCountEqual(1, 'Watched path does not exist: .*static')
-
- # create it and update config
- os.mkdir(static_path)
- watcher.update_watchers(settings)
- # no new warning
- self.assertLogCountEqual(1, 'Watched path does not exist: .*static')
-
- # get modified values
- modified = watcher.check()
- # empty theme and content should raise warnings
- self.assertLogCountEqual(1, 'No valid files found in content')
- self.assertLogCountEqual(1, 'Empty theme folder. Using `basic` theme')
-
- self.assertIsNone(modified['content']) # empty
- self.assertIsNone(modified['theme']) # empty
- self.assertIsNone(modified['[static]static']) # empty
- self.assertTrue(modified['settings']) # modified, first time
-
- # add a content, add file to theme and check again
- create_file(os.path.join(content_path, 'article.md'),
- 'Title: test\n'
- 'Date: 01-01-2020')
-
- create_file(os.path.join(theme_path, 'dummy'),
- 'test')
-
- modified = watcher.check()
- # no new warning
- self.assertLogCountEqual(1, 'No valid files found in content')
- self.assertLogCountEqual(1, 'Empty theme folder. Using `basic` theme')
-
- self.assertIsNone(modified['[static]static']) # empty
- self.assertFalse(modified['settings']) # not modified
- self.assertTrue(modified['theme']) # modified
- self.assertTrue(modified['content']) # modified
-
- # change config, remove static path
- create_file(config_file,
- 'PATH = "content"\n'
- 'THEME = "mytheme"\n'
- 'STATIC_PATHS = []')
-
- settings = read_settings(config_file)
- watcher.update_watchers(settings)
-
- modified = watcher.check()
- self.assertNotIn('[static]static', modified) # should be gone
- self.assertTrue(modified['settings']) # modified
- self.assertFalse(modified['content']) # not modified
- self.assertFalse(modified['theme']) # not modified
-
- # cleanup
- logger.enable_filter()
- shutil.rmtree(root)
-
def test_clean_output_dir(self):
retention = ()
test_directory = os.path.join(self.temp_output,
diff --git a/pelican/utils.py b/pelican/utils.py
index d8cf15b4..4832e0c1 100644
--- a/pelican/utils.py
+++ b/pelican/utils.py
@@ -24,6 +24,8 @@ except ModuleNotFoundError:
from backports.zoneinfo import ZoneInfo
from markupsafe import Markup
+import watchfiles
+
logger = logging.getLogger(__name__)
@@ -755,167 +757,31 @@ def order_content(content_list, order_by='slug'):
return content_list
-class FileSystemWatcher:
- def __init__(self, settings_file, reader_class, settings=None):
- self.watchers = {
- 'settings': FileSystemWatcher.file_watcher(settings_file)
- }
+def wait_for_changes(settings_file, reader_class, settings):
+ new_extensions = set(reader_class(settings).extensions)
+ content_path = settings.get('PATH', '')
+ theme_path = settings.get('THEME', '')
+ ignore_files = set(settings.get('IGNORE_FILES', []))
- self.settings = None
- self.reader_class = reader_class
- self._extensions = None
- self._content_path = None
- self._theme_path = None
- self._ignore_files = None
+ watching_paths = [
+ settings_file,
+ theme_path,
+ content_path,
+ ]
- if settings is not None:
- self.update_watchers(settings)
+ watching_paths.extend(
+ os.path.join(content_path, path) for path in settings.get('STATIC_PATHS', [])
+ )
- def update_watchers(self, settings):
- new_extensions = set(self.reader_class(settings).extensions)
- new_content_path = settings.get('PATH', '')
- new_theme_path = settings.get('THEME', '')
- new_ignore_files = set(settings.get('IGNORE_FILES', []))
+ watching_paths = [os.path.abspath(p) for p in watching_paths if p and os.path.exists(p)]
- extensions_changed = new_extensions != self._extensions
- content_changed = new_content_path != self._content_path
- theme_changed = new_theme_path != self._theme_path
- ignore_changed = new_ignore_files != self._ignore_files
-
- # Refresh content watcher if related settings changed
- if extensions_changed or content_changed or ignore_changed:
- self.add_watcher('content',
- new_content_path,
- new_extensions,
- new_ignore_files)
-
- # Refresh theme watcher if related settings changed
- if theme_changed or ignore_changed:
- self.add_watcher('theme',
- new_theme_path,
- [''],
- new_ignore_files)
-
- # Watch STATIC_PATHS
- old_static_watchers = set(key
- for key in self.watchers
- if key.startswith('[static]'))
-
- for path in settings.get('STATIC_PATHS', []):
- key = '[static]{}'.format(path)
- if ignore_changed or (key not in self.watchers):
- self.add_watcher(
- key,
- os.path.join(new_content_path, path),
- [''],
- new_ignore_files)
- if key in old_static_watchers:
- old_static_watchers.remove(key)
-
- # cleanup removed static watchers
- for key in old_static_watchers:
- del self.watchers[key]
-
- # update values
- self.settings = settings
- self._extensions = new_extensions
- self._content_path = new_content_path
- self._theme_path = new_theme_path
- self._ignore_files = new_ignore_files
-
- def check(self):
- '''return a key:watcher_status dict for all watchers'''
- result = {key: next(watcher) for key, watcher in self.watchers.items()}
-
- # Various warnings
- if result.get('content') is None:
- reader_descs = sorted(
- {
- ' | %s (%s)' % (type(r).__name__, ', '.join(r.file_extensions))
- for r in self.reader_class(self.settings).readers.values()
- if r.enabled
- }
- )
- logger.warning(
- 'No valid files found in content for the active readers:\n'
- + '\n'.join(reader_descs))
-
- if result.get('theme') is None:
- logger.warning('Empty theme folder. Using `basic` theme.')
-
- return result
-
- def add_watcher(self, key, path, extensions=[''], ignores=[]):
- watcher = self.get_watcher(path, extensions, ignores)
- if watcher is not None:
- self.watchers[key] = watcher
-
- def get_watcher(self, path, extensions=[''], ignores=[]):
- '''return a watcher depending on path type (file or folder)'''
- if not os.path.exists(path):
- logger.warning("Watched path does not exist: %s", path)
- return None
-
- if os.path.isdir(path):
- return self.folder_watcher(path, extensions, ignores)
- else:
- return self.file_watcher(path)
-
- @staticmethod
- def folder_watcher(path, extensions, ignores=[]):
- '''Generator for monitoring a folder for modifications.
-
- Returns a boolean indicating if files are changed since last check.
- Returns None if there are no matching files in the folder'''
-
- def file_times(path):
- '''Return `mtime` for each file in path'''
-
- for root, dirs, files in os.walk(path, followlinks=True):
- dirs[:] = [x for x in dirs if not x.startswith(os.curdir)]
-
- for f in files:
- valid_extension = f.endswith(tuple(extensions))
- file_ignored = any(
- fnmatch.fnmatch(f, ignore) for ignore in ignores
- )
- if valid_extension and not file_ignored:
- try:
- yield os.stat(os.path.join(root, f)).st_mtime
- except OSError as e:
- logger.warning('Caught Exception: %s', e)
-
- LAST_MTIME = 0
- while True:
- try:
- mtime = max(file_times(path))
- if mtime > LAST_MTIME:
- LAST_MTIME = mtime
- yield True
- except ValueError:
- yield None
- else:
- yield False
-
- @staticmethod
- def file_watcher(path):
- '''Generator for monitoring a file for modifications'''
- LAST_MTIME = 0
- while True:
- if path:
- try:
- mtime = os.stat(path).st_mtime
- except OSError as e:
- logger.warning('Caught Exception: %s', e)
- continue
-
- if mtime > LAST_MTIME:
- LAST_MTIME = mtime
- yield True
- else:
- yield False
- else:
- yield None
+ return next(watchfiles.watch(
+ *watching_paths,
+ watch_filter=watchfiles.DefaultFilter(
+ ignore_entity_patterns=[fnmatch.translate(pattern) for pattern in ignore_files]
+ ),
+ rust_timeout=0
+ ))
def set_date_tzinfo(d, tz_name=None):
diff --git a/pyproject.toml b/pyproject.toml
index 826c1179..788b8dcb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,6 +41,7 @@ rich = ">=10.1"
unidecode = ">=1.1"
markdown = {version = ">=3.1", optional = true}
backports-zoneinfo = {version = "^0.2.1", python = "<3.9"}
+watchfiles = "^0.19.0"
[tool.poetry.dev-dependencies]
BeautifulSoup4 = "^4.9"
From 7643e0e92b901d71ee1f4996e019682982abb3df Mon Sep 17 00:00:00 2001
From: Jake Howard
Date: Fri, 14 Jul 2023 16:50:43 +0100
Subject: [PATCH 28/88] Make sure the package depends on `watchfiles`
---
setup.py | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/setup.py b/setup.py
index 18eedb00..4ffee0cb 100755
--- a/setup.py
+++ b/setup.py
@@ -8,9 +8,18 @@ from setuptools import find_packages, setup
version = "4.8.0"
-requires = ['feedgenerator >= 1.9', 'jinja2 >= 2.7', 'pygments',
- 'docutils>=0.15', 'blinker', 'unidecode', 'python-dateutil',
- 'rich', 'backports-zoneinfo[tzdata] >= 0.2; python_version<"3.9"']
+requires = [
+ 'feedgenerator >= 1.9',
+ 'jinja2 >= 2.7',
+ 'pygments',
+ 'docutils>=0.15',
+ 'blinker',
+ 'unidecode',
+ 'python-dateutil',
+ 'rich',
+ 'backports-zoneinfo[tzdata] >= 0.2; python_version<"3.9"',
+ 'watchfiles'
+]
entry_points = {
'console_scripts': [
From 5519efef2e24a3fef506a1e19222fc49053e39a6 Mon Sep 17 00:00:00 2001
From: Jake Howard
Date: Tue, 15 Aug 2023 17:45:50 +0100
Subject: [PATCH 29/88] Log watching files which don't exist
---
pelican/utils.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/pelican/utils.py b/pelican/utils.py
index 4832e0c1..d8a19e4b 100644
--- a/pelican/utils.py
+++ b/pelican/utils.py
@@ -773,7 +773,11 @@ def wait_for_changes(settings_file, reader_class, settings):
os.path.join(content_path, path) for path in settings.get('STATIC_PATHS', [])
)
- watching_paths = [os.path.abspath(p) for p in watching_paths if p and os.path.exists(p)]
+ watching_paths = [os.path.abspath(p) for p in watching_paths if p]
+
+ for path in watching_paths:
+ if not os.path.exists(path):
+ logger.warning("Unable to watch path '%s' as it does not exist.", path)
return next(watchfiles.watch(
*watching_paths,
From b388057d664daacb7a318e3334fdb975841b0962 Mon Sep 17 00:00:00 2001
From: Jake Howard
Date: Tue, 15 Aug 2023 17:47:04 +0100
Subject: [PATCH 30/88] Remove unused extensions list
---
pelican/utils.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/pelican/utils.py b/pelican/utils.py
index d8a19e4b..23c6fe1c 100644
--- a/pelican/utils.py
+++ b/pelican/utils.py
@@ -758,7 +758,6 @@ def order_content(content_list, order_by='slug'):
def wait_for_changes(settings_file, reader_class, settings):
- new_extensions = set(reader_class(settings).extensions)
content_path = settings.get('PATH', '')
theme_path = settings.get('THEME', '')
ignore_files = set(settings.get('IGNORE_FILES', []))
From 631ac1bdb39d3ebe3d7dd94aa4e33a187ddb52c5 Mon Sep 17 00:00:00 2001
From: Jake Howard
Date: Tue, 15 Aug 2023 17:49:58 +0100
Subject: [PATCH 31/88] Cleanup imports
---
pelican/__init__.py | 2 +-
pelican/tests/test_utils.py | 2 --
pelican/utils.py | 8 ++++----
3 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/pelican/__init__.py b/pelican/__init__.py
index a4f5b38e..e0526b1f 100644
--- a/pelican/__init__.py
+++ b/pelican/__init__.py
@@ -27,7 +27,7 @@ from pelican.plugins._utils import get_plugin_name, load_plugins
from pelican.readers import Readers
from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer
from pelican.settings import read_settings
-from pelican.utils import (wait_for_changes, clean_output_dir, maybe_pluralize)
+from pelican.utils import clean_output_dir, maybe_pluralize, wait_for_changes
from pelican.writers import Writer
try:
diff --git a/pelican/tests/test_utils.py b/pelican/tests/test_utils.py
index 7ff1af7a..d8296285 100644
--- a/pelican/tests/test_utils.py
+++ b/pelican/tests/test_utils.py
@@ -2,7 +2,6 @@ import locale
import logging
import os
import shutil
-import time
from datetime import timezone
from sys import platform
from tempfile import mkdtemp
@@ -14,7 +13,6 @@ except ModuleNotFoundError:
from pelican import utils
from pelican.generators import TemplatePagesGenerator
-from pelican.readers import Readers
from pelican.settings import read_settings
from pelican.tests.support import (LoggedTestCase, get_article,
locale_available, unittest)
diff --git a/pelican/utils.py b/pelican/utils.py
index 23c6fe1c..1225e479 100644
--- a/pelican/utils.py
+++ b/pelican/utils.py
@@ -760,7 +760,9 @@ def order_content(content_list, order_by='slug'):
def wait_for_changes(settings_file, reader_class, settings):
content_path = settings.get('PATH', '')
theme_path = settings.get('THEME', '')
- ignore_files = set(settings.get('IGNORE_FILES', []))
+ ignore_files = set(
+ fnmatch.translate(pattern) for pattern in settings.get('IGNORE_FILES', [])
+ )
watching_paths = [
settings_file,
@@ -780,9 +782,7 @@ def wait_for_changes(settings_file, reader_class, settings):
return next(watchfiles.watch(
*watching_paths,
- watch_filter=watchfiles.DefaultFilter(
- ignore_entity_patterns=[fnmatch.translate(pattern) for pattern in ignore_files]
- ),
+ watch_filter=watchfiles.DefaultFilter(ignore_entity_patterns=ignore_files),
rust_timeout=0
))
From b289dcea82bac461cf879be5935e64a1ff192622 Mon Sep 17 00:00:00 2001
From: Deniz Turgut
Date: Sat, 28 Oct 2023 17:30:45 +0300
Subject: [PATCH 32/88] don't watch not existing paths
---
pelican/utils.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/pelican/utils.py b/pelican/utils.py
index 1225e479..84a18deb 100644
--- a/pelican/utils.py
+++ b/pelican/utils.py
@@ -764,21 +764,25 @@ def wait_for_changes(settings_file, reader_class, settings):
fnmatch.translate(pattern) for pattern in settings.get('IGNORE_FILES', [])
)
- watching_paths = [
+ candidate_paths = [
settings_file,
theme_path,
content_path,
]
- watching_paths.extend(
+ candidate_paths.extend(
os.path.join(content_path, path) for path in settings.get('STATIC_PATHS', [])
)
- watching_paths = [os.path.abspath(p) for p in watching_paths if p]
-
- for path in watching_paths:
+ watching_paths = []
+ for path in candidate_paths:
+ if not path:
+ continue
+ path = os.path.abspath(path)
if not os.path.exists(path):
logger.warning("Unable to watch path '%s' as it does not exist.", path)
+ else:
+ watching_paths.append(path)
return next(watchfiles.watch(
*watching_paths,
From 43e513f218e66ccdf128e5de5a4db279f47ff063 Mon Sep 17 00:00:00 2001
From: Deniz Turgut
Date: Sat, 28 Oct 2023 17:37:56 +0300
Subject: [PATCH 33/88] run pelican first before waiting for changes
---
pelican/__init__.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pelican/__init__.py b/pelican/__init__.py
index e0526b1f..fcdda8a4 100644
--- a/pelican/__init__.py
+++ b/pelican/__init__.py
@@ -455,8 +455,9 @@ def autoreload(args, excqueue=None):
settings_file = os.path.abspath(args.settings)
while True:
try:
- changed_files = wait_for_changes(args.settings, Readers, settings)
+ pelican.run()
+ changed_files = wait_for_changes(args.settings, Readers, settings)
changed_files = {c[1] for c in changed_files}
if settings_file in changed_files:
@@ -464,7 +465,6 @@ def autoreload(args, excqueue=None):
console.print('\n-> Modified: {}. re-generating...'.format(
', '.join(changed_files)))
- pelican.run()
except KeyboardInterrupt:
if excqueue is not None:
From f342dc309758a7f1197f2797b8965653f538b68a Mon Sep 17 00:00:00 2001
From: Chris Rose
Date: Sat, 28 Oct 2023 08:00:27 -0700
Subject: [PATCH 34/88] Add macOS testing for 3.11/3.12
---
.github/workflows/main.yml | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c0ffd9c6..6f146631 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -27,6 +27,10 @@ jobs:
python: "3.12"
- os: macos
python: "3.10"
+ - os: macos
+ python: "3.11"
+ - os: macos
+ python: "3.12"
- os: windows
python: "3.10"
From 7dfc799f255815de1665e257a93987598aab95f4 Mon Sep 17 00:00:00 2001
From: Chris Rose
Date: Sat, 28 Oct 2023 10:44:39 -0700
Subject: [PATCH 35/88] Use ruff in pre-commit
---
.pre-commit-config.yaml | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4c0c85b0..f68521e5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -13,13 +13,11 @@ repos:
- id: end-of-file-fixer
- id: forbid-new-submodules
- id: trailing-whitespace
- - repo: https://github.com/PyCQA/flake8
- rev: 3.9.2
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.1.0
hooks:
- - id: flake8
- name: Flake8 on commit diff
- description: This hook limits Flake8 checks to changed lines of code.
- entry: bash
- args: [-c, 'git diff HEAD | flake8 --diff --max-line-length=88']
+ - id: ruff
+ - id: ruff-format
+ args: ["--check"]
exclude: ^pelican/tests/output/
From 19c797af5e08dd7ccc38f939c5fc9c9bbe862e54 Mon Sep 17 00:00:00 2001
From: Chris Rose
Date: Sat, 28 Oct 2023 10:50:17 -0700
Subject: [PATCH 36/88] Add support to verify windows, too
---
.github/workflows/main.yml | 28 ++++++++--------------------
1 file changed, 8 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 6f146631..ba3aef55 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -9,37 +9,25 @@ env:
jobs:
test:
- name: Test - ${{ matrix.config.python }} - ${{ matrix.config.os }}
- runs-on: ${{ matrix.config.os }}-latest
+ name: Test - ${{ matrix.python }} - ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}-latest
strategy:
matrix:
- config:
+ os: [ubuntu, macos, windows]
+ python: ["3.10", "3.11", "3.12"]
+ include:
- os: ubuntu
python: "3.8"
- os: ubuntu
python: "3.9"
- - os: ubuntu
- python: "3.10"
- - os: ubuntu
- python: "3.11"
- - os: ubuntu
- python: "3.12"
- - os: macos
- python: "3.10"
- - os: macos
- python: "3.11"
- - os: macos
- python: "3.12"
- - os: windows
- python: "3.10"
steps:
- uses: actions/checkout@v3
- - name: Set up Python ${{ matrix.config.python }}
+ - name: Set up Python ${{ matrix.python }}
uses: actions/setup-python@v4
with:
- python-version: ${{ matrix.config.python }}
+ python-version: ${{ matrix.python }}
cache: "pip"
cache-dependency-path: "**/requirements/*"
- name: Install locale (Linux)
@@ -58,7 +46,7 @@ jobs:
echo "===== PANDOC ====="
pandoc --version | head -2
- name: Run tests
- run: tox -e py${{ matrix.config.python }}
+ run: tox -e py${{ matrix.python }}
lint:
name: Lint
From 58fd8553850a161f1656599f4d0d40f43eefbf82 Mon Sep 17 00:00:00 2001
From: Chris Rose
Date: Sat, 28 Oct 2023 10:54:09 -0700
Subject: [PATCH 37/88] inv task now uses ruff
---
pyproject.toml | 1 +
tasks.py | 15 +++++++++++----
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 826c1179..b3eaa0d1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,6 +57,7 @@ pytest = "^7.1"
pytest-cov = "^4.0"
pytest-sugar = "^0.9.5"
pytest-xdist = "^2.0"
+ruff = "^0.1.3"
tox = {version = "^3.13", optional = true}
flake8 = "^3.8"
flake8-import-order = "^0.18.1"
diff --git a/tasks.py b/tasks.py
index 148899c7..d41e2955 100644
--- a/tasks.py
+++ b/tasks.py
@@ -66,13 +66,20 @@ def isort(c, check=False, diff=False):
@task
-def flake8(c):
- c.run(f"git diff HEAD | {VENV_BIN}/flake8 --diff --max-line-length=88", pty=PTY)
+def ruff(c, fix=False, diff=False):
+ """Run Ruff to ensure code meets project standards."""
+ diff_flag, fix_flag = "", ""
+ if fix:
+ fix_flag = "--fix"
+ if diff:
+ diff_flag = "--diff"
+ c.run(f"{VENV_BIN}/ruff check {diff_flag} {fix_flag} .", pty=PTY)
@task
-def lint(c):
- flake8(c)
+def lint(c, fix=False, diff=False):
+ """Check code style via linting tools."""
+ ruff(c, fix=fix, diff=diff)
@task
From 6cf6a1ffe97b23074cf4e8c223fb6174d4092ae4 Mon Sep 17 00:00:00 2001
From: Chris Rose
Date: Sat, 28 Oct 2023 10:58:41 -0700
Subject: [PATCH 38/88] Ruff lint fixes
---
pelican/tools/pelican_themes.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/pelican/tools/pelican_themes.py b/pelican/tools/pelican_themes.py
index 96d07c1f..b8bf1be2 100755
--- a/pelican/tools/pelican_themes.py
+++ b/pelican/tools/pelican_themes.py
@@ -10,7 +10,7 @@ def err(msg, die=None):
"""Print an error message and exits if an exit code is given"""
sys.stderr.write(msg + '\n')
if die:
- sys.exit(die if type(die) is int else 1)
+ sys.exit(die if isinstance(die, int) else 1)
try:
@@ -135,16 +135,16 @@ def themes():
def list_themes(v=False):
"""Display the list of the themes"""
- for t, l in themes():
+ for theme_path, link_target in themes():
if not v:
- t = os.path.basename(t)
- if l:
+ theme_path = os.path.basename(theme_path)
+ if link_target:
if v:
- print(t + (" (symbolic link to `" + l + "')"))
+ print(theme_path + (" (symbolic link to `" + link_target + "')"))
else:
- print(t + '@')
+ print(theme_path + '@')
else:
- print(t)
+ print(theme_path)
def remove(theme_name, v=False):
From 29b10ef6e640f017757fd0afeb67dcd607cf2e75 Mon Sep 17 00:00:00 2001
From: Chris Rose
Date: Sat, 28 Oct 2023 11:02:06 -0700
Subject: [PATCH 39/88] Use poetry directly in lints
---
.github/workflows/main.yml | 16 ++++++++++------
tox.ini | 14 --------------
2 files changed, 10 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c0ffd9c6..b59c5316 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -62,16 +62,20 @@ jobs:
steps:
- uses: actions/checkout@v3
+ - name: Install Poetry
+ run: pipx install poetry
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.9"
- cache: "pip"
- cache-dependency-path: "**/requirements/*"
- - name: Install tox
- run: python -m pip install -U pip tox
- - name: Check
- run: tox -e flake8
+ cache: "poetry"
+ cache-dependency-path: "pyproject.toml"
+ - name: Install dependencies
+ run: |
+ poetry env use "3.9"
+ poetry install --no-interaction
+ - name: Run linters
+ run: poetry run invoke lint --diff
docs:
name: Build docs
diff --git a/tox.ini b/tox.ini
index c31044ca..361c52dd 100644
--- a/tox.ini
+++ b/tox.ini
@@ -30,17 +30,3 @@ filterwarnings =
default::DeprecationWarning
error:.*:Warning:pelican
addopts = -n auto -r a
-
-[flake8]
-application-import-names = pelican
-import-order-style = cryptography
-max-line-length = 88
-
-[testenv:flake8]
-basepython = python3.9
-skip_install = true
-deps =
- -rrequirements/style.pip
-commands =
- flake8 --version
- flake8 pelican
From 33d6712e8b1283354b305ea73ac0ee3331092dfc Mon Sep 17 00:00:00 2001
From: Chris Rose
Date: Sat, 28 Oct 2023 11:18:24 -0700
Subject: [PATCH 40/88] Don't install pelican's dependencies to lint
---
.github/workflows/main.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b59c5316..b477cecb 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -73,7 +73,7 @@ jobs:
- name: Install dependencies
run: |
poetry env use "3.9"
- poetry install --no-interaction
+ poetry install --no-interaction --no-root
- name: Run linters
run: poetry run invoke lint --diff
From b8d5919cd24edc0aeb322f5c1eb036810ae6b38e Mon Sep 17 00:00:00 2001
From: Deniz Turgut
Date: Sat, 28 Oct 2023 22:11:11 +0300
Subject: [PATCH 41/88] expand period tests to be more specific
---
pelican/generators.py | 2 +-
pelican/tests/test_generators.py | 62 ++++++++++++++++++++++++--------
2 files changed, 48 insertions(+), 16 deletions(-)
diff --git a/pelican/generators.py b/pelican/generators.py
index 7ab99263..d874d97c 100644
--- a/pelican/generators.py
+++ b/pelican/generators.py
@@ -484,7 +484,7 @@ class ArticlesGenerator(CachingGenerator):
except PelicanTemplateNotFound:
template = self.get_template('archives')
- for granularity in list(self.period_archives.keys()):
+ for granularity in self.period_archives:
for period in self.period_archives[granularity]:
context = self.context.copy()
diff --git a/pelican/tests/test_generators.py b/pelican/tests/test_generators.py
index a6fe9731..ac271c1c 100644
--- a/pelican/tests/test_generators.py
+++ b/pelican/tests/test_generators.py
@@ -431,11 +431,12 @@ class TestArticlesGenerator(unittest.TestCase):
path=CONTENT_DIR, theme=settings['THEME'], output_path=None)
generator.generate_context()
period_archives = generator.context['period_archives']
- self.assertEqual(len(period_archives.items()), 1)
- self.assertIn('year', period_archives.keys())
- archive_years = [p['period'][0] for p in period_archives['year']]
- self.assertIn(1970, archive_years)
- self.assertIn(2014, archive_years)
+ abbreviated_archives = {
+ granularity: {period['period'] for period in periods}
+ for granularity, periods in period_archives.items()
+ }
+ expected = {'year': {(1970,), (2010,), (2012,), (2014,)}}
+ self.assertEqual(expected, abbreviated_archives)
# Month archives enabled:
settings['MONTH_ARCHIVE_SAVE_AS'] = \
@@ -448,11 +449,22 @@ class TestArticlesGenerator(unittest.TestCase):
path=CONTENT_DIR, theme=settings['THEME'], output_path=None)
generator.generate_context()
period_archives = generator.context['period_archives']
- self.assertEqual(len(period_archives.items()), 2)
- self.assertIn('month', period_archives.keys())
- month_archives_tuples = [p['period'] for p in period_archives['month']]
- self.assertIn((1970, 'January'), month_archives_tuples)
- self.assertIn((2014, 'February'), month_archives_tuples)
+ abbreviated_archives = {
+ granularity: {period['period'] for period in periods}
+ for granularity, periods in period_archives.items()
+ }
+ expected = {
+ 'year': {(1970,), (2010,), (2012,), (2014,)},
+ 'month': {
+ (1970, 'January'),
+ (2010, 'December'),
+ (2012, 'December'),
+ (2012, 'November'),
+ (2012, 'October'),
+ (2014, 'February'),
+ },
+ }
+ self.assertEqual(expected, abbreviated_archives)
# Day archives enabled:
settings['DAY_ARCHIVE_SAVE_AS'] = \
@@ -465,11 +477,31 @@ class TestArticlesGenerator(unittest.TestCase):
path=CONTENT_DIR, theme=settings['THEME'], output_path=None)
generator.generate_context()
period_archives = generator.context['period_archives']
- self.assertEqual(len(period_archives.items()), 3)
- self.assertIn('day', period_archives.keys())
- day_archives_tuples = [p['period'] for p in period_archives['day']]
- self.assertIn((1970, 'January', 1), day_archives_tuples)
- self.assertIn((2014, 'February', 9), day_archives_tuples)
+ abbreviated_archives = {
+ granularity: {period['period'] for period in periods}
+ for granularity, periods in period_archives.items()
+ }
+ expected = {
+ 'year': {(1970,), (2010,), (2012,), (2014,)},
+ 'month': {
+ (1970, 'January'),
+ (2010, 'December'),
+ (2012, 'December'),
+ (2012, 'November'),
+ (2012, 'October'),
+ (2014, 'February'),
+ },
+ 'day': {
+ (1970, 'January', 1),
+ (2010, 'December', 2),
+ (2012, 'December', 20),
+ (2012, 'November', 29),
+ (2012, 'October', 30),
+ (2012, 'October', 31),
+ (2014, 'February', 9),
+ },
+ }
+ self.assertEqual(expected, abbreviated_archives)
# Further item values tests
filtered_archives = [
From b812f2ad1c5feb010d5d33bda247c126dadd1b4b Mon Sep 17 00:00:00 2001
From: Yasser Tahiri
Date: Sat, 28 Oct 2023 21:06:24 +0100
Subject: [PATCH 42/88] =?UTF-8?q?chore:=20Simplify=20boolean=20`if`=20expr?=
=?UTF-8?q?ession=20=E2=9C=A8=20(#2944)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
pelican/tools/pelican_themes.py | 21 ++++++++++-----------
pelican/writers.py | 22 ++++++++++++----------
tasks.py | 6 +++---
3 files changed, 25 insertions(+), 24 deletions(-)
diff --git a/pelican/tools/pelican_themes.py b/pelican/tools/pelican_themes.py
index 96d07c1f..47f1a625 100755
--- a/pelican/tools/pelican_themes.py
+++ b/pelican/tools/pelican_themes.py
@@ -183,7 +183,7 @@ def install(path, v=False, u=False):
exists = os.path.exists(theme_path)
if exists and not u:
err(path + ' : already exists')
- elif exists and u:
+ elif exists:
remove(theme_name, v)
install(path, v)
else:
@@ -245,15 +245,14 @@ def clean(v=False):
c = 0
for path in os.listdir(_THEMES_PATH):
path = os.path.join(_THEMES_PATH, path)
- if os.path.islink(path):
- if is_broken_link(path):
- if v:
- print('Removing {}'.format(path))
- try:
- os.remove(path)
- except OSError:
- print('Error: cannot remove {}'.format(path))
- else:
- c += 1
+ if os.path.islink(path) and is_broken_link(path):
+ if v:
+ print('Removing {}'.format(path))
+ try:
+ os.remove(path)
+ except OSError:
+ print('Error: cannot remove {}'.format(path))
+ else:
+ c += 1
print("\nRemoved {} broken links".format(c))
diff --git a/pelican/writers.py b/pelican/writers.py
index b08da1f3..afc8e4b7 100644
--- a/pelican/writers.py
+++ b/pelican/writers.py
@@ -37,13 +37,12 @@ class Writer:
feed_title = context['SITENAME'] + ' - ' + feed_title
else:
feed_title = context['SITENAME']
- feed = feed_class(
+ return feed_class(
title=Markup(feed_title).striptags(),
link=(self.site_url + '/'),
feed_url=self.feed_url,
description=context.get('SITESUBTITLE', ''),
subtitle=context.get('SITESUBTITLE', None))
- return feed
def _add_item_to_the_feed(self, feed, item):
title = Markup(item.title).striptags()
@@ -71,7 +70,7 @@ class Writer:
if description == content:
description = None
- categories = list()
+ categories = []
if hasattr(item, 'category'):
categories.append(item.category)
if hasattr(item, 'tags'):
@@ -83,13 +82,17 @@ class Writer:
unique_id=get_tag_uri(link, item.date),
description=description,
content=content,
- categories=categories if categories else None,
+ categories=categories or None,
author_name=getattr(item, 'author', ''),
pubdate=set_date_tzinfo(
- item.date, self.settings.get('TIMEZONE', None)),
+ item.date, self.settings.get('TIMEZONE', None)
+ ),
updateddate=set_date_tzinfo(
item.modified, self.settings.get('TIMEZONE', None)
- ) if hasattr(item, 'modified') else None)
+ )
+ if hasattr(item, 'modified')
+ else None,
+ )
def _open_w(self, filename, encoding, override=False):
"""Open a file to write some content to it.
@@ -101,9 +104,8 @@ class Writer:
if override:
raise RuntimeError('File %s is set to be overridden twice'
% filename)
- else:
- logger.info('Skipping %s', filename)
- filename = os.devnull
+ logger.info('Skipping %s', filename)
+ filename = os.devnull
elif filename in self._written_files:
if override:
logger.info('Overwriting %s', filename)
@@ -139,7 +141,7 @@ class Writer:
'SITEURL', path_to_url(get_relative_path(path)))
self.feed_domain = context.get('FEED_DOMAIN')
- self.feed_url = self.urljoiner(self.feed_domain, url if url else path)
+ self.feed_url = self.urljoiner(self.feed_domain, url or path)
feed = self._create_new_feed(feed_type, feed_title, context)
diff --git a/tasks.py b/tasks.py
index 148899c7..e4268ec6 100644
--- a/tasks.py
+++ b/tasks.py
@@ -8,7 +8,7 @@ PKG_NAME = "pelican"
PKG_PATH = Path(PKG_NAME)
DOCS_PORT = os.environ.get("DOCS_PORT", 8000)
BIN_DIR = "bin" if os.name != "nt" else "Scripts"
-PTY = True if os.name != "nt" else False
+PTY = os.name != "nt"
ACTIVE_VENV = os.environ.get("VIRTUAL_ENV", None)
VENV_HOME = Path(os.environ.get("WORKON_HOME", "~/virtualenvs"))
VENV_PATH = Path(ACTIVE_VENV) if ACTIVE_VENV else (VENV_HOME / PKG_NAME)
@@ -16,8 +16,8 @@ VENV = str(VENV_PATH.expanduser())
VENV_BIN = Path(VENV) / Path(BIN_DIR)
TOOLS = ["poetry", "pre-commit", "psutil"]
-POETRY = which("poetry") if which("poetry") else (VENV_BIN / "poetry")
-PRECOMMIT = which("pre-commit") if which("pre-commit") else (VENV_BIN / "pre-commit")
+POETRY = which("poetry") or VENV_BIN / "poetry"
+PRECOMMIT = which("pre-commit") or VENV_BIN / "pre-commit"
@task
From 8a7e01646b4fa962f6225fa5d4cac4693876d7bb Mon Sep 17 00:00:00 2001
From: Will Thong
Date: Sat, 28 Oct 2023 21:11:44 +0100
Subject: [PATCH 43/88] Add rel='nofollow' to all external hardcoded links in
templates (#3162)
---
pelican/tests/output/basic/a-markdown-powered-article.html | 4 ++--
pelican/tests/output/basic/archives.html | 4 ++--
pelican/tests/output/basic/article-1.html | 4 ++--
pelican/tests/output/basic/article-2.html | 4 ++--
pelican/tests/output/basic/article-3.html | 4 ++--
pelican/tests/output/basic/author/alexis-metaireau.html | 4 ++--
pelican/tests/output/basic/authors.html | 4 ++--
pelican/tests/output/basic/categories.html | 4 ++--
pelican/tests/output/basic/category/bar.html | 4 ++--
pelican/tests/output/basic/category/cat1.html | 4 ++--
pelican/tests/output/basic/category/misc.html | 4 ++--
pelican/tests/output/basic/category/yeah.html | 4 ++--
.../output/basic/drafts/a-draft-article-without-date.html | 4 ++--
pelican/tests/output/basic/drafts/a-draft-article.html | 4 ++--
pelican/tests/output/basic/filename_metadata-example.html | 4 ++--
pelican/tests/output/basic/index.html | 4 ++--
pelican/tests/output/basic/oh-yeah-fr.html | 4 ++--
pelican/tests/output/basic/oh-yeah.html | 4 ++--
pelican/tests/output/basic/override/index.html | 4 ++--
.../tests/output/basic/pages/this-is-a-test-hidden-page.html | 4 ++--
pelican/tests/output/basic/pages/this-is-a-test-page.html | 4 ++--
pelican/tests/output/basic/second-article-fr.html | 4 ++--
pelican/tests/output/basic/second-article.html | 4 ++--
pelican/tests/output/basic/tag/bar.html | 4 ++--
pelican/tests/output/basic/tag/baz.html | 4 ++--
pelican/tests/output/basic/tag/foo.html | 4 ++--
pelican/tests/output/basic/tag/foobar.html | 4 ++--
pelican/tests/output/basic/tag/oh.html | 4 ++--
pelican/tests/output/basic/tag/yeah.html | 4 ++--
pelican/tests/output/basic/tags.html | 4 ++--
pelican/tests/output/basic/this-is-a-super-article.html | 4 ++--
pelican/tests/output/basic/unbelievable.html | 4 ++--
pelican/tests/output/custom/a-markdown-powered-article.html | 4 ++--
pelican/tests/output/custom/archives.html | 4 ++--
pelican/tests/output/custom/article-1.html | 4 ++--
pelican/tests/output/custom/article-2.html | 4 ++--
pelican/tests/output/custom/article-3.html | 4 ++--
pelican/tests/output/custom/author/alexis-metaireau.html | 4 ++--
pelican/tests/output/custom/author/alexis-metaireau2.html | 4 ++--
pelican/tests/output/custom/author/alexis-metaireau3.html | 4 ++--
pelican/tests/output/custom/authors.html | 4 ++--
pelican/tests/output/custom/categories.html | 4 ++--
pelican/tests/output/custom/category/bar.html | 4 ++--
pelican/tests/output/custom/category/cat1.html | 4 ++--
pelican/tests/output/custom/category/misc.html | 4 ++--
pelican/tests/output/custom/category/yeah.html | 4 ++--
.../output/custom/drafts/a-draft-article-without-date.html | 4 ++--
pelican/tests/output/custom/drafts/a-draft-article.html | 4 ++--
pelican/tests/output/custom/filename_metadata-example.html | 4 ++--
pelican/tests/output/custom/index.html | 4 ++--
pelican/tests/output/custom/index2.html | 4 ++--
pelican/tests/output/custom/index3.html | 4 ++--
pelican/tests/output/custom/jinja2_template.html | 4 ++--
pelican/tests/output/custom/oh-yeah-fr.html | 4 ++--
pelican/tests/output/custom/oh-yeah.html | 4 ++--
pelican/tests/output/custom/override/index.html | 4 ++--
.../tests/output/custom/pages/this-is-a-test-hidden-page.html | 4 ++--
pelican/tests/output/custom/pages/this-is-a-test-page.html | 4 ++--
pelican/tests/output/custom/second-article-fr.html | 4 ++--
pelican/tests/output/custom/second-article.html | 4 ++--
pelican/tests/output/custom/tag/bar.html | 4 ++--
pelican/tests/output/custom/tag/baz.html | 4 ++--
pelican/tests/output/custom/tag/foo.html | 4 ++--
pelican/tests/output/custom/tag/foobar.html | 4 ++--
pelican/tests/output/custom/tag/oh.html | 4 ++--
pelican/tests/output/custom/tag/yeah.html | 4 ++--
pelican/tests/output/custom/tags.html | 4 ++--
pelican/tests/output/custom/this-is-a-super-article.html | 4 ++--
pelican/tests/output/custom/unbelievable.html | 4 ++--
pelican/tests/output/custom_locale/archives.html | 4 ++--
.../tests/output/custom_locale/author/alexis-metaireau.html | 4 ++--
.../tests/output/custom_locale/author/alexis-metaireau2.html | 4 ++--
.../tests/output/custom_locale/author/alexis-metaireau3.html | 4 ++--
pelican/tests/output/custom_locale/authors.html | 4 ++--
pelican/tests/output/custom_locale/categories.html | 4 ++--
pelican/tests/output/custom_locale/category/bar.html | 4 ++--
pelican/tests/output/custom_locale/category/cat1.html | 4 ++--
pelican/tests/output/custom_locale/category/misc.html | 4 ++--
pelican/tests/output/custom_locale/category/yeah.html | 4 ++--
.../custom_locale/drafts/a-draft-article-without-date.html | 4 ++--
.../tests/output/custom_locale/drafts/a-draft-article.html | 4 ++--
pelican/tests/output/custom_locale/index.html | 4 ++--
pelican/tests/output/custom_locale/index2.html | 4 ++--
pelican/tests/output/custom_locale/index3.html | 4 ++--
pelican/tests/output/custom_locale/jinja2_template.html | 4 ++--
pelican/tests/output/custom_locale/oh-yeah-fr.html | 4 ++--
pelican/tests/output/custom_locale/override/index.html | 4 ++--
.../custom_locale/pages/this-is-a-test-hidden-page.html | 4 ++--
.../tests/output/custom_locale/pages/this-is-a-test-page.html | 4 ++--
.../posts/2010/décembre/02/this-is-a-super-article/index.html | 4 ++--
.../posts/2010/octobre/15/unbelievable/index.html | 4 ++--
.../custom_locale/posts/2010/octobre/20/oh-yeah/index.html | 4 ++--
.../posts/2011/avril/20/a-markdown-powered-article/index.html | 4 ++--
.../custom_locale/posts/2011/février/17/article-1/index.html | 4 ++--
.../custom_locale/posts/2011/février/17/article-2/index.html | 4 ++--
.../custom_locale/posts/2011/février/17/article-3/index.html | 4 ++--
.../posts/2012/février/29/second-article/index.html | 4 ++--
.../2012/novembre/30/filename_metadata-example/index.html | 4 ++--
pelican/tests/output/custom_locale/second-article-fr.html | 4 ++--
pelican/tests/output/custom_locale/tag/bar.html | 4 ++--
pelican/tests/output/custom_locale/tag/baz.html | 4 ++--
pelican/tests/output/custom_locale/tag/foo.html | 4 ++--
pelican/tests/output/custom_locale/tag/foobar.html | 4 ++--
pelican/tests/output/custom_locale/tag/oh.html | 4 ++--
pelican/tests/output/custom_locale/tag/yeah.html | 4 ++--
pelican/tests/output/custom_locale/tags.html | 4 ++--
pelican/themes/notmyidea/templates/base.html | 4 ++--
pelican/themes/notmyidea/templates/twitter.html | 2 +-
pelican/themes/simple/templates/base.html | 4 ++--
109 files changed, 217 insertions(+), 217 deletions(-)
diff --git a/pelican/tests/output/basic/a-markdown-powered-article.html b/pelican/tests/output/basic/a-markdown-powered-article.html
index ca9b62eb..0098ccac 100644
--- a/pelican/tests/output/basic/a-markdown-powered-article.html
+++ b/pelican/tests/output/basic/a-markdown-powered-article.html
@@ -58,10 +58,10 @@
diff --git a/pelican/tests/output/basic/archives.html b/pelican/tests/output/basic/archives.html
index 93c1d5be..e3a6c7df 100644
--- a/pelican/tests/output/basic/archives.html
+++ b/pelican/tests/output/basic/archives.html
@@ -60,10 +60,10 @@