From f19de98b9e6495bf42154ef6dd763b6b8c486fde Mon Sep 17 00:00:00 2001 From: boxydog Date: Mon, 17 Jun 2024 08:11:21 -0500 Subject: [PATCH 1/5] Log warnings about files that would have been processed by disabled readers --- pelican/__init__.py | 5 +++-- pelican/generators.py | 27 +++++++++++++++++++++++- pelican/readers.py | 39 ++++++++++++++++++++++++++++++----- pelican/tests/test_pelican.py | 24 ++++++++++++++++++++- pelican/tests/test_readers.py | 15 +++++++++++++- pelican/tests/test_utils.py | 7 +++++++ pelican/utils.py | 15 +++++++++++--- 7 files changed, 119 insertions(+), 13 deletions(-) diff --git a/pelican/__init__.py b/pelican/__init__.py index 20d17706..96849bae 100644 --- a/pelican/__init__.py +++ b/pelican/__init__.py @@ -30,7 +30,6 @@ from pelican.generators import ( ) from pelican.plugins import signals from pelican.plugins._utils import get_plugin_name, load_plugins -from pelican.readers import Readers from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer from pelican.settings import read_settings from pelican.utils import clean_output_dir, maybe_pluralize, wait_for_changes @@ -126,6 +125,8 @@ class Pelican: for p in generators: if hasattr(p, "generate_context"): p.generate_context() + if hasattr(p, "check_disabled_readers"): + p.check_disabled_readers() # for plugins that create/edit the summary logger.debug("Signal all_generators_finalized.send()") @@ -573,7 +574,7 @@ def autoreload(args, excqueue=None): try: pelican.run() - changed_files = wait_for_changes(args.settings, Readers, settings) + changed_files = wait_for_changes(args.settings, settings) changed_files = {c[1] for c in changed_files} if settings_file in changed_files: diff --git a/pelican/generators.py b/pelican/generators.py index 73b51713..548c494f 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -7,6 +7,7 @@ from collections import defaultdict from functools import partial from itertools import chain, groupby from operator import attrgetter +from typing import List, Optional, Set from jinja2 import ( BaseLoader, @@ -156,7 +157,9 @@ class Generator: return False - def get_files(self, paths, exclude=None, extensions=None): + def get_files( + self, paths, exclude: Optional[List[str]] = None, extensions=None + ) -> Set[str]: """Return a list of files to use, based on rules :param paths: the list pf paths to search (relative to self.path) @@ -250,6 +253,13 @@ class Generator: # return the name of the class for logging purposes return self.__class__.__name__ + def _check_disabled_readers(self, paths, exclude: Optional[List[str]]) -> None: + """Log warnings for files that would have been processed by disabled readers.""" + for fil in self.get_files( + paths, exclude=exclude, extensions=self.readers.disabled_extensions + ): + self.readers.check_file(fil) + class CachingGenerator(Generator, FileStampDataCacher): """Subclass of Generator and FileStampDataCacher classes @@ -643,6 +653,11 @@ class ArticlesGenerator(CachingGenerator): self.generate_authors(write) self.generate_drafts(write) + def check_disabled_readers(self) -> None: + self._check_disabled_readers( + self.settings["ARTICLE_PATHS"], exclude=self.settings["ARTICLE_EXCLUDES"] + ) + def generate_context(self): """Add the articles into the shared context""" @@ -849,6 +864,11 @@ class PagesGenerator(CachingGenerator): super().__init__(*args, **kwargs) signals.page_generator_init.send(self) + def check_disabled_readers(self) -> None: + self._check_disabled_readers( + self.settings["PAGE_PATHS"], exclude=self.settings["PAGE_EXCLUDES"] + ) + def generate_context(self): all_pages = [] hidden_pages = [] @@ -953,6 +973,11 @@ class StaticGenerator(Generator): self.fallback_to_symlinks = False signals.static_generator_init.send(self) + def check_disabled_readers(self) -> None: + self._check_disabled_readers( + self.settings["STATIC_PATHS"], exclude=self.settings["STATIC_EXCLUDES"] + ) + def generate_context(self): self.staticfiles = [] linked_files = set(self.context["static_links"]) diff --git a/pelican/readers.py b/pelican/readers.py index 422f39fc..ee7e3466 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -17,7 +17,7 @@ from pelican import rstdirectives # NOQA from pelican.cache import FileStampDataCacher from pelican.contents import Author, Category, Page, Tag from pelican.plugins import signals -from pelican.utils import get_date, pelican_open, posixize_path +from pelican.utils import file_suffix, get_date, pelican_open, posixize_path try: from markdown import Markdown @@ -125,6 +125,10 @@ class BaseReader: metadata = {} return content, metadata + def disabled_message(self) -> str: + """Message about why this plugin was disabled.""" + return "" + class _FieldBodyTranslator(HTMLTranslator): def __init__(self, document): @@ -347,6 +351,12 @@ class MarkdownReader(BaseReader): metadata = {} return content, metadata + def disabled_message(self) -> str: + return ( + "Could not import markdown.Markdown. " + "Have you installed the markdown package?" + ) + class HTMLReader(BaseReader): """Parses HTML files as input, looking for meta, title, and body tags""" @@ -508,17 +518,23 @@ class Readers(FileStampDataCacher): def __init__(self, settings=None, cache_name=""): self.settings = settings or {} self.readers = {} + self.disabled_readers = {} + # extension => reader for readers that are enabled self.reader_classes = {} + # extension => reader for readers that are not enabled + disabled_reader_classes = {} for cls in [BaseReader] + BaseReader.__subclasses__(): if not cls.enabled: logger.debug( "Missing dependencies for %s", ", ".join(cls.file_extensions) ) - continue for ext in cls.file_extensions: - self.reader_classes[ext] = cls + if cls.enabled: + self.reader_classes[ext] = cls + else: + disabled_reader_classes[ext] = cls if self.settings["READERS"]: self.reader_classes.update(self.settings["READERS"]) @@ -531,6 +547,9 @@ class Readers(FileStampDataCacher): self.readers[fmt] = reader_class(self.settings) + for fmt, reader_class in disabled_reader_classes.items(): + self.disabled_readers[fmt] = reader_class(self.settings) + # set up caching cache_this_level = ( cache_name != "" and self.settings["CONTENT_CACHING_LAYER"] == "reader" @@ -541,8 +560,13 @@ class Readers(FileStampDataCacher): @property def extensions(self): + """File extensions that will be processed by a reader.""" return self.readers.keys() + @property + def disabled_extensions(self): + return self.disabled_readers.keys() + def read_file( self, base_path, @@ -562,8 +586,7 @@ class Readers(FileStampDataCacher): logger.debug("Read file %s -> %s", source_path, content_class.__name__) if not fmt: - _, ext = os.path.splitext(os.path.basename(path)) - fmt = ext[1:] + fmt = file_suffix(path) if fmt not in self.readers: raise TypeError("Pelican does not know how to parse %s", path) @@ -654,6 +677,12 @@ class Readers(FileStampDataCacher): context=context, ) + def check_file(self, source_path: str) -> None: + """Log a warning if a file is processed by a disabled reader.""" + reader = self.disabled_readers.get(file_suffix(source_path), None) + if reader: + logger.warning(f"{source_path}: {reader.disabled_message()}") + def find_empty_alt(content, path): """Find images with empty alt diff --git a/pelican/tests/test_pelican.py b/pelican/tests/test_pelican.py index add5f576..e243be61 100644 --- a/pelican/tests/test_pelican.py +++ b/pelican/tests/test_pelican.py @@ -9,10 +9,11 @@ import unittest from collections.abc import Sequence from shutil import rmtree from tempfile import TemporaryDirectory, mkdtemp -from unittest.mock import patch +from unittest.mock import PropertyMock, patch from rich.console import Console +import pelican.readers from pelican import Pelican, __version__, main from pelican.generators import StaticGenerator from pelican.settings import read_settings @@ -303,3 +304,24 @@ class TestPelican(LoggedTestCase): main(["-o", temp_dir, "pelican/tests/simple_content"]) self.assertIn("Processed 1 article", out.getvalue()) self.assertEqual("", err.getvalue()) + + def test_main_on_content_markdown_disabled(self): + """Invoke main on simple_content directory.""" + with patch.object( + pelican.readers.MarkdownReader, "enabled", new_callable=PropertyMock + ) as attr_mock: + attr_mock.return_value = False + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + with TemporaryDirectory() as temp_dir: + # Don't highlight anything. + # See https://rich.readthedocs.io/en/stable/highlighting.html + with patch("pelican.console", new=Console(highlight=False)): + main(["-o", temp_dir, "pelican/tests/simple_content"]) + self.assertIn("Processed 0 articles", out.getvalue()) + self.assertLogCountEqual( + 1, + ".*article_with_md_extension.md: " + "Could not import markdown.Markdown. " + "Have you installed the markdown package?", + ) diff --git a/pelican/tests/test_readers.py b/pelican/tests/test_readers.py index ec366fa8..68938a83 100644 --- a/pelican/tests/test_readers.py +++ b/pelican/tests/test_readers.py @@ -1,5 +1,5 @@ import os -from unittest.mock import patch +from unittest.mock import PropertyMock, patch from pelican import readers from pelican.tests.support import get_settings, unittest @@ -32,6 +32,19 @@ class ReaderTest(unittest.TestCase): else: self.fail(f"Expected {key} to have value {value}, but was not in Dict") + def test_markdown_disabled(self): + with patch.object( + readers.MarkdownReader, "enabled", new_callable=PropertyMock + ) as attr_mock: + attr_mock.return_value = False + readrs = readers.Readers(settings=get_settings()) + self.assertEqual( + set(readers.MarkdownReader.file_extensions), + readrs.disabled_readers.keys(), + ) + for val in readrs.disabled_readers.values(): + self.assertEqual(readers.MarkdownReader, val.__class__) + class TestAssertDictHasSubset(ReaderTest): def setUp(self): diff --git a/pelican/tests/test_utils.py b/pelican/tests/test_utils.py index f7f11ffb..0da59dd4 100644 --- a/pelican/tests/test_utils.py +++ b/pelican/tests/test_utils.py @@ -966,3 +966,10 @@ class TestMemoized(unittest.TestCase): container.get.cache.clear() self.assertEqual("bar", container.get("bar")) get_mock.assert_called_once_with("bar") + + +class TestStringUtils(unittest.TestCase): + def test_file_suffix(self): + self.assertEqual("", utils.file_suffix("")) + self.assertEqual("", utils.file_suffix("foo")) + self.assertEqual("md", utils.file_suffix("foo.md")) diff --git a/pelican/utils.py b/pelican/utils.py index a29fdf81..b780ab97 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -29,6 +29,7 @@ from typing import ( ) import dateutil.parser +from watchfiles import Change try: from zoneinfo import ZoneInfo @@ -39,7 +40,6 @@ from markupsafe import Markup if TYPE_CHECKING: from pelican.contents import Content - from pelican.readers import Readers from pelican.settings import Settings logger = logging.getLogger(__name__) @@ -797,9 +797,8 @@ def order_content( def wait_for_changes( settings_file: str, - reader_class: type[Readers], settings: Settings, -): +) -> set[tuple[Change, str]]: content_path = settings.get("PATH", "") theme_path = settings.get("THEME", "") ignore_files = { @@ -924,3 +923,13 @@ def temporary_locale( locale.setlocale(lc_category, temp_locale) yield locale.setlocale(lc_category, orig_locale) + + +def file_suffix(path: str) -> str: + """Return the suffix of a filename in a path.""" + _, ext = os.path.splitext(os.path.basename(path)) + ret = "" + if len(ext) > 1: + # drop the ".", e.g., "exe", not ".exe" + ret = ext[1:] + return ret From 6a501917283c664c3724b1b8b46a96d4f1874d30 Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Tue, 25 Jun 2024 10:49:07 +0200 Subject: [PATCH 2/5] Tweak Markdown-not-installed console warnings Adding single-quotation marks should cause 'markdown' to be highlighted in green, presumably via Rich. --- pelican/readers.py | 4 ++-- pelican/tests/test_pelican.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pelican/readers.py b/pelican/readers.py index ee7e3466..3d0e8d58 100644 --- a/pelican/readers.py +++ b/pelican/readers.py @@ -353,8 +353,8 @@ class MarkdownReader(BaseReader): def disabled_message(self) -> str: return ( - "Could not import markdown.Markdown. " - "Have you installed the markdown package?" + "Could not import 'markdown.Markdown'. " + "Have you installed the 'markdown' package?" ) diff --git a/pelican/tests/test_pelican.py b/pelican/tests/test_pelican.py index e243be61..a43afa3e 100644 --- a/pelican/tests/test_pelican.py +++ b/pelican/tests/test_pelican.py @@ -322,6 +322,6 @@ class TestPelican(LoggedTestCase): self.assertLogCountEqual( 1, ".*article_with_md_extension.md: " - "Could not import markdown.Markdown. " - "Have you installed the markdown package?", + "Could not import 'markdown.Markdown'. " + "Have you installed the 'markdown' package?", ) From bdd4e45628c901cad3c95f281c28d52a2bb676b4 Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Tue, 25 Jun 2024 10:54:24 +0200 Subject: [PATCH 3/5] Enforce 75% code coverage in CI via Tox --- tasks.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks.py b/tasks.py index 3a1722eb..cf9851ed 100644 --- a/tasks.py +++ b/tasks.py @@ -49,7 +49,7 @@ def coverage(c): """Generate code coverage of running the test suite.""" c.run( f"{VENV_BIN}/pytest --cov=pelican --cov-report term-missing " - "--cov-fail-under 74", + "--cov-fail-under 75", pty=PTY, ) c.run(f"{VENV_BIN}/coverage html", pty=PTY) diff --git a/tox.ini b/tox.ini index 61e8908a..7b8598ac 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ deps = commands = {envpython} --version - pytest -s --cov=pelican pelican + pytest -s --cov=pelican --cov-fail-under 75 pelican [testenv:docs] basepython = python3.11 From 28e54106f2fc3f6c837cef62f2498fd75ae7e91e Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Tue, 25 Jun 2024 11:00:41 +0200 Subject: [PATCH 4/5] Update Ruff linter --- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bfdd6149..9ae674ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # ruff version should match the one in pyproject.toml - rev: v0.4.6 + rev: v0.4.10 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/pyproject.toml b/pyproject.toml index dd256a44..1a2e1474 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ dev = [ "tox>=4.11.3", "invoke>=2.2.0", # ruff version should match the one in .pre-commit-config.yaml - "ruff==0.4.6", + "ruff==0.4.10", "tomli>=2.0.1; python_version < \"3.11\"", ] From 36ebe91af78823860e4dc6c790f3f28507457016 Mon Sep 17 00:00:00 2001 From: Justin Mayer Date: Tue, 25 Jun 2024 11:33:28 +0200 Subject: [PATCH 5/5] Rename default branch to `main` --- .github/workflows/main.yml | 2 +- CONTRIBUTING.rst | 4 ++-- README.rst | 4 ++-- docs/settings.rst | 2 +- docs/themes.rst | 4 ++-- docs/tips.rst | 16 ++++++++-------- pelican/settings.py | 2 +- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4c0127df..e07caa72 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -110,7 +110,7 @@ jobs: environment: Deployment needs: [test, lint, docs, build] runs-on: ubuntu-latest - if: github.ref=='refs/heads/master' && github.event_name!='pull_request' && github.repository == 'getpelican/pelican' + if: github.ref=='refs/heads/main' && github.event_name!='pull_request' && github.repository == 'getpelican/pelican' permissions: contents: write diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 4faace91..5ac5f0ad 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -24,7 +24,7 @@ Before you ask for help, please make sure you do the following: 3. Try reproducing the issue in a clean environment, ensuring you are using: -* latest Pelican release (or an up-to-date Git clone of Pelican master) +* latest Pelican release (or an up-to-date Git clone of Pelican ``main`` branch) * latest releases of libraries used by Pelican * no plugins or only those related to the issue @@ -87,7 +87,7 @@ Using Git and GitHub -------------------- * `Create a new branch`_ specific to your change (as opposed to making - your commits in the master branch). + your commits in the ``main`` branch). * **Don't put multiple unrelated fixes/features in the same branch / pull request.** For example, if you're working on a new feature and find a bugfix that doesn't *require* your new feature, **make a new distinct branch and pull diff --git a/README.rst b/README.rst index 3f708242..0e3577ec 100644 --- a/README.rst +++ b/README.rst @@ -64,8 +64,8 @@ Why the name “Pelican”? .. _`Pelican's internals`: https://docs.getpelican.com/en/latest/internals.html .. _`hosted on GitHub`: https://github.com/getpelican/pelican -.. |build-status| image:: https://img.shields.io/github/actions/workflow/status/getpelican/pelican/main.yml?branch=master - :target: https://github.com/getpelican/pelican/actions/workflows/main.yml?query=branch%3Amaster +.. |build-status| image:: https://img.shields.io/github/actions/workflow/status/getpelican/pelican/main.yml?branch=main + :target: https://github.com/getpelican/pelican/actions/workflows/main.yml?query=branch%3Amain :alt: GitHub Actions CI: continuous integration status .. |pypi-version| image:: https://img.shields.io/pypi/v/pelican.svg :target: https://pypi.org/project/pelican/ diff --git a/docs/settings.rst b/docs/settings.rst index 4ae608c6..7269c0bd 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -28,7 +28,7 @@ Environment variables can also be used here but must be escaped appropriately:: Settings are configured in the form of a Python module (a file). There is an `example settings file -`_ +`_ available for reference. To see a list of current settings in your environment, including both default diff --git a/docs/themes.rst b/docs/themes.rst index 2e01ec8e..ace5dcb9 100644 --- a/docs/themes.rst +++ b/docs/themes.rst @@ -17,7 +17,7 @@ To generate its HTML output, Pelican uses the `Jinja `_ templating engine due to its flexibility and straightforward syntax. If you want to create your own theme, feel free to take inspiration from the `"simple" theme -`_. +`_. To generate your site using a theme you have created (or downloaded manually and then modified), you can specify that theme via the ``-t`` flag:: @@ -368,7 +368,7 @@ period_num A tuple of the form (``year``, ``month``, ``day``), You can see an example of how to use `period` in the `"simple" theme period_archives.html template -`_. +`_. .. _period_archives_variable: diff --git a/docs/tips.rst b/docs/tips.rst index 1398ccd9..e5574c7c 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -89,18 +89,18 @@ Publishing a User Site to GitHub Pages from a Branch ---------------------------------------------------- To publish a Pelican site in the form of User Pages, you need to *push* the -content of the ``output`` dir generated by Pelican to the ``master`` branch of +content of the ``output`` dir generated by Pelican to the ``main`` branch of your ``.github.io`` repository on GitHub. Again, you can take advantage of ``ghp-import``:: $ pelican content -o output -s pelicanconf.py $ ghp-import output -b gh-pages - $ git push git@github.com:elemoine/elemoine.github.io.git gh-pages:master + $ git push git@github.com:elemoine/elemoine.github.io.git gh-pages:main The ``git push`` command pushes the local ``gh-pages`` branch (freshly updated by the ``ghp-import`` command) to the ``elemoine.github.io`` repository's -``master`` branch on GitHub. +``main`` branch on GitHub. .. note:: @@ -116,10 +116,10 @@ inside the ``Pelican`` folder you can run:: $ pelican content -o .. -s pelicanconf.py -Now you can push the whole project ``.github.io`` to the master +Now you can push the whole project ``.github.io`` to the main branch of your GitHub repository:: - $ git push origin master + $ git push origin main (assuming origin is set to your remote repository). @@ -127,7 +127,7 @@ Publishing to GitHub Pages Using a Custom GitHub Actions Workflow ----------------------------------------------------------------- Pelican-powered sites can be published to GitHub Pages via a `custom workflow -`_. +`_. To use it: 1. Enable GitHub Pages in your repo: go to **Settings → Pages** and choose @@ -144,7 +144,7 @@ To use it: workflow_dispatch: jobs: deploy: - uses: "getpelican/pelican/.github/workflows/github_pages.yml@master" + uses: "getpelican/pelican/.github/workflows/github_pages.yml@main" permissions: contents: "read" pages: "write" @@ -152,7 +152,7 @@ To use it: with: settings: "publishconf.py" - You may want to replace the ``@master`` with the ID of a specific commit in + You may want to replace the ``@main`` with the ID of a specific commit in this repo in order to pin the version of the reusable workflow that you're using: ``uses: getpelican/pelican/.github/workflows/github_pages.yml@``. If you do this you might want to get Dependabot to send you automated pull diff --git a/pelican/settings.py b/pelican/settings.py index 46b761a5..66d6beeb 100644 --- a/pelican/settings.py +++ b/pelican/settings.py @@ -347,7 +347,7 @@ def handle_deprecated_settings(settings: Settings) -> Settings: "FILES_TO_COPY", "STATIC_PATHS and EXTRA_PATH_METADATA", "https://github.com/getpelican/pelican/" - "blob/master/docs/settings.rst#path-metadata", + "blob/main/docs/settings.rst#path-metadata", ), ]: if old in settings: