From fc522349865e03ddf6a1e007093727451a8cb5b8 Mon Sep 17 00:00:00 2001 From: Deniz Turgut Date: Sat, 10 Dec 2016 01:58:38 -0500 Subject: [PATCH 1/2] Fix generator caching With generator level caching, cached files weren't added to the collections. Only new files were processed. This fixes the issue. --- pelican/generators.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pelican/generators.py b/pelican/generators.py index a7a7fb25..88752392 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -533,7 +533,7 @@ class ArticlesGenerator(CachingGenerator): continue if article_or_draft.status.lower() == "published": - all_articles.append(article_or_draft) + pass elif article_or_draft.status.lower() == "draft": article_or_draft = self.readers.read_file( base_path=self.path, path=f, content_class=Draft, @@ -542,8 +542,6 @@ class ArticlesGenerator(CachingGenerator): preread_sender=self, context_signal=signals.article_generator_context, context_sender=self) - self.add_source_path(article_or_draft) - all_drafts.append(article_or_draft) else: logger.error( "Unknown status '%s' for file %s, skipping it.", @@ -553,6 +551,10 @@ class ArticlesGenerator(CachingGenerator): self.cache_data(f, article_or_draft) + if article_or_draft.status.lower() == "published": + all_articles.append(article_or_draft) + else: + all_drafts.append(article_or_draft) self.add_source_path(article_or_draft) self.articles, self.translations = process_translations( @@ -636,11 +638,7 @@ class PagesGenerator(CachingGenerator): self._add_failed_source_path(f) continue - if page.status.lower() == "published": - all_pages.append(page) - elif page.status.lower() == "hidden": - hidden_pages.append(page) - else: + if page.status.lower() not in ("published", "hidden"): logger.error( "Unknown status '%s' for file %s, skipping it.", page.status, f) @@ -649,6 +647,10 @@ class PagesGenerator(CachingGenerator): self.cache_data(f, page) + if page.status.lower() == "published": + all_pages.append(page) + elif page.status.lower() == "hidden": + hidden_pages.append(page) self.add_source_path(page) self.pages, self.translations = process_translations( From 1c09741280d6f3c448883e332ae5ac9e00c2ad96 Mon Sep 17 00:00:00 2001 From: Deniz Turgut Date: Sat, 10 Dec 2016 05:18:35 -0500 Subject: [PATCH 2/2] Add tests to make sure cached content is properly added to generators --- pelican/tests/test_cache.py | 94 +++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/pelican/tests/test_cache.py b/pelican/tests/test_cache.py index e8ea4967..1f26deec 100644 --- a/pelican/tests/test_cache.py +++ b/pelican/tests/test_cache.py @@ -36,6 +36,100 @@ class TestCache(unittest.TestCase): settings['CACHE_PATH'] = self.temp_cache return settings + def test_generator_caching(self): + """Test that cached and uncached content is same in generator level""" + settings = self._get_cache_enabled_settings() + settings['CONTENT_CACHING_LAYER'] = 'generator' + settings['PAGE_PATHS'] = ['TestPages'] + settings['DEFAULT_DATE'] = (1970, 1, 1) + settings['READERS'] = {'asc': None} + + def sorted_titles(items): + return sorted(item.title for item in items) + + # Articles + generator = ArticlesGenerator( + context=settings.copy(), settings=settings, + path=CONTENT_DIR, theme=settings['THEME'], output_path=None) + generator.generate_context() + uncached_articles = sorted_titles(generator.articles) + uncached_drafts = sorted_titles(generator.drafts) + + generator = ArticlesGenerator( + context=settings.copy(), settings=settings, + path=CONTENT_DIR, theme=settings['THEME'], output_path=None) + generator.generate_context() + cached_articles = sorted_titles(generator.articles) + cached_drafts = sorted_titles(generator.drafts) + + self.assertEqual(uncached_articles, cached_articles) + self.assertEqual(uncached_drafts, cached_drafts) + + # Pages + generator = PagesGenerator( + context=settings.copy(), settings=settings, + path=CUR_DIR, theme=settings['THEME'], output_path=None) + generator.generate_context() + uncached_pages = sorted_titles(generator.pages) + uncached_hidden_pages = sorted_titles(generator.hidden_pages) + + generator = PagesGenerator( + context=settings.copy(), settings=settings, + path=CUR_DIR, theme=settings['THEME'], output_path=None) + generator.generate_context() + cached_pages = sorted_titles(generator.pages) + cached_hidden_pages = sorted_titles(generator.hidden_pages) + + self.assertEqual(uncached_pages, cached_pages) + self.assertEqual(uncached_hidden_pages, cached_hidden_pages) + + def test_reader_caching(self): + """Test that cached and uncached content is same in reader level""" + settings = self._get_cache_enabled_settings() + settings['CONTENT_CACHING_LAYER'] = 'reader' + settings['PAGE_PATHS'] = ['TestPages'] + settings['DEFAULT_DATE'] = (1970, 1, 1) + settings['READERS'] = {'asc': None} + + def sorted_titles(items): + return sorted(item.title for item in items) + + # Articles + generator = ArticlesGenerator( + context=settings.copy(), settings=settings, + path=CONTENT_DIR, theme=settings['THEME'], output_path=None) + generator.generate_context() + uncached_articles = sorted_titles(generator.articles) + uncached_drafts = sorted_titles(generator.drafts) + + generator = ArticlesGenerator( + context=settings.copy(), settings=settings, + path=CONTENT_DIR, theme=settings['THEME'], output_path=None) + generator.generate_context() + cached_articles = sorted_titles(generator.articles) + cached_drafts = sorted_titles(generator.drafts) + + self.assertEqual(uncached_articles, cached_articles) + self.assertEqual(uncached_drafts, cached_drafts) + + # Pages + generator = PagesGenerator( + context=settings.copy(), settings=settings, + path=CUR_DIR, theme=settings['THEME'], output_path=None) + generator.generate_context() + uncached_pages = sorted_titles(generator.pages) + uncached_hidden_pages = sorted_titles(generator.hidden_pages) + + generator = PagesGenerator( + context=settings.copy(), settings=settings, + path=CUR_DIR, theme=settings['THEME'], output_path=None) + generator.generate_context() + cached_pages = sorted_titles(generator.pages) + cached_hidden_pages = sorted_titles(generator.hidden_pages) + + self.assertEqual(uncached_pages, cached_pages) + self.assertEqual(uncached_hidden_pages, cached_hidden_pages) + @unittest.skipUnless(MagicMock, 'Needs Mock module') def test_article_object_caching(self): """Test Article objects caching at the generator level"""