From e97e9b5ae5fa5494f136521afb3594cf3e65fc83 Mon Sep 17 00:00:00 2001 From: Antoine Brenner Date: Tue, 15 Apr 2014 00:04:40 +0200 Subject: [PATCH 1/2] Fix unittest issue related to python2/python3 differences Under python 2, with non-ascii locales, u"{:%b}".format(date) can raise UnicodeDecodeError because u"{:%b}".format(date) will call date.__format__(u"%b"), which will return a byte string and not a unicode string. eg: locale.setlocale(locale.LC_ALL, 'ja_JP.utf8') date.__format__(u"%b") == '12\xe6\x9c\x88' # True This commit catches UnicodeDecodeError and calls date.__format__() with byte strings instead of characters, since it to work with character strings --- pelican/generators.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pelican/generators.py b/pelican/generators.py index 7c2dbbf2..1b584d3f 100644 --- a/pelican/generators.py +++ b/pelican/generators.py @@ -309,7 +309,20 @@ class ArticlesGenerator(Generator): # format string syntax can be used for specifying the # period archive dates date = archive[0].date - save_as = save_as_fmt.format(date=date) + # Under python 2, with non-ascii locales, u"{:%b}".format(date) might raise UnicodeDecodeError + # because u"{:%b}".format(date) will call date.__format__(u"%b"), which will return a byte string + # and not a unicode string. + # eg: + # locale.setlocale(locale.LC_ALL, 'ja_JP.utf8') + # date.__format__(u"%b") == '12\xe6\x9c\x88' # True + try: + save_as = save_as_fmt.format(date=date) + except UnicodeDecodeError: + # Python2 only: + # Let date.__format__() work with byte strings instead of characters since it fails to work with characters + bytes_save_as_fmt = save_as_fmt.encode('utf8') + bytes_save_as = bytes_save_as_fmt.format(date=date) + save_as = unicode(bytes_save_as,'utf8') context = self.context.copy() if key == period_date_key['year']: From 7277c95fb588463d035decefcf86f59d41a4e7c0 Mon Sep 17 00:00:00 2001 From: Antoine Brenner Date: Tue, 15 Apr 2014 16:36:29 +0200 Subject: [PATCH 2/2] Make sure locale is what we want before/after the tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The locale is a global state, and it was not properly reset to whatever it was before the unitttest possibly changed it. This is now fixed. Not restoring the locale led to weird issues: depending on the order chosen by "python -m unittest discover" to run the unit tests, some tests would apparently randomly fail due to the locale not being what was expected. For example, test_period_in_timeperiod_archive would call mock('posts/1970/ 1月/index.html',...) instead of expected mock('posts/1970/Jan/index.html',...) and fail. --- pelican/tests/test_contents.py | 6 ++++++ pelican/tests/test_generators.py | 11 +++++++++++ pelican/tests/test_importer.py | 11 +++++++++++ pelican/tests/test_paginator.py | 8 +++++++- pelican/tests/test_pelican.py | 2 +- pelican/tests/test_settings.py | 5 +++++ pelican/tests/test_utils.py | 3 +++ 7 files changed, 44 insertions(+), 2 deletions(-) diff --git a/pelican/tests/test_contents.py b/pelican/tests/test_contents.py index 27d2a897..3c0f8d75 100644 --- a/pelican/tests/test_contents.py +++ b/pelican/tests/test_contents.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals, absolute_import import six from datetime import datetime from sys import platform +import locale from pelican.tests.support import unittest, get_settings @@ -22,6 +23,8 @@ class TestPage(unittest.TestCase): def setUp(self): super(TestPage, self).setUp() + self.old_locale = locale.setlocale(locale.LC_ALL) + locale.setlocale(locale.LC_ALL, str('C')) self.page_kwargs = { 'content': TEST_CONTENT, 'context': { @@ -35,6 +38,9 @@ class TestPage(unittest.TestCase): 'source_path': '/path/to/file/foo.ext' } + def tearDown(self): + locale.setlocale(locale.LC_ALL, self.old_locale) + def test_use_args(self): # Creating a page with arguments passed to the constructor should use # them to initialise object's attributes. diff --git a/pelican/tests/test_generators.py b/pelican/tests/test_generators.py index a500f87a..ff487c3e 100644 --- a/pelican/tests/test_generators.py +++ b/pelican/tests/test_generators.py @@ -14,6 +14,7 @@ from pelican.generators import (Generator, ArticlesGenerator, PagesGenerator, TemplatePagesGenerator) from pelican.writers import Writer from pelican.tests.support import unittest, get_settings +import locale CUR_DIR = os.path.dirname(__file__) CONTENT_DIR = os.path.join(CUR_DIR, 'content') @@ -21,11 +22,17 @@ CONTENT_DIR = os.path.join(CUR_DIR, 'content') class TestGenerator(unittest.TestCase): def setUp(self): + self.old_locale = locale.setlocale(locale.LC_ALL) + locale.setlocale(locale.LC_ALL, str('C')) self.settings = get_settings() self.settings['READERS'] = {'asc': None} self.generator = Generator(self.settings.copy(), self.settings, CUR_DIR, self.settings['THEME'], None) + def tearDown(self): + locale.setlocale(locale.LC_ALL, self.old_locale) + + def test_include_path(self): filename = os.path.join(CUR_DIR, 'content', 'article.rst') include_path = self.generator._include_path @@ -373,10 +380,14 @@ class TestTemplatePagesGenerator(unittest.TestCase): def setUp(self): self.temp_content = mkdtemp(prefix='pelicantests.') self.temp_output = mkdtemp(prefix='pelicantests.') + self.old_locale = locale.setlocale(locale.LC_ALL) + locale.setlocale(locale.LC_ALL, str('C')) + def tearDown(self): rmtree(self.temp_content) rmtree(self.temp_output) + locale.setlocale(locale.LC_ALL, self.old_locale) def test_generate_output(self): diff --git a/pelican/tests/test_importer.py b/pelican/tests/test_importer.py index 8412c75b..65193bf5 100644 --- a/pelican/tests/test_importer.py +++ b/pelican/tests/test_importer.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals, print_function import os import re +import locale from pelican.tools.pelican_import import wp2fields, fields2pelican, decode_wp_content, build_header, build_markdown_header, get_attachments, download_attachments from pelican.tests.support import (unittest, temporary_folder, mute, skipIfNoExecutable) @@ -30,9 +31,14 @@ except ImportError: class TestWordpressXmlImporter(unittest.TestCase): def setUp(self): + self.old_locale = locale.setlocale(locale.LC_ALL) + locale.setlocale(locale.LC_ALL, str('C')) self.posts = list(wp2fields(WORDPRESS_XML_SAMPLE)) self.custposts = list(wp2fields(WORDPRESS_XML_SAMPLE, True)) + def tearDown(self): + locale.setlocale(locale.LC_ALL, self.old_locale) + def test_ignore_empty_posts(self): self.assertTrue(self.posts) for title, content, fname, date, author, categ, tags, kind, format in self.posts: @@ -261,8 +267,13 @@ class TestBuildHeader(unittest.TestCase): @unittest.skipUnless(BeautifulSoup, 'Needs BeautifulSoup module') class TestWordpressXMLAttachements(unittest.TestCase): def setUp(self): + self.old_locale = locale.setlocale(locale.LC_ALL) + locale.setlocale(locale.LC_ALL, str('C')) self.attachments = get_attachments(WORDPRESS_XML_SAMPLE) + def tearDown(self): + locale.setlocale(locale.LC_ALL, self.old_locale) + def test_recognise_attachments(self): self.assertTrue(self.attachments) self.assertTrue(len(self.attachments.keys()) == 3) diff --git a/pelican/tests/test_paginator.py b/pelican/tests/test_paginator.py index f454d47d..108dc791 100644 --- a/pelican/tests/test_paginator.py +++ b/pelican/tests/test_paginator.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals, absolute_import import six +import locale from pelican.tests.support import unittest, get_settings @@ -16,6 +17,8 @@ TEST_SUMMARY = generate_lorem_ipsum(n=1, html=False) class TestPage(unittest.TestCase): def setUp(self): super(TestPage, self).setUp() + self.old_locale = locale.setlocale(locale.LC_ALL) + locale.setlocale(locale.LC_ALL, str('C')) self.page_kwargs = { 'content': TEST_CONTENT, 'context': { @@ -29,6 +32,9 @@ class TestPage(unittest.TestCase): 'source_path': '/path/to/file/foo.ext' } + def tearDown(self): + locale.setlocale(locale.LC_ALL, self.old_locale) + def test_save_as_preservation(self): settings = get_settings() # fix up pagination rules @@ -47,4 +53,4 @@ class TestPage(unittest.TestCase): object_list = [Article(**self.page_kwargs), Article(**self.page_kwargs)] paginator = Paginator('foobar.foo', object_list, settings) page = paginator.page(1) - self.assertEqual(page.save_as, 'foobar.foo') \ No newline at end of file + self.assertEqual(page.save_as, 'foobar.foo') diff --git a/pelican/tests/test_pelican.py b/pelican/tests/test_pelican.py index 15876095..974986cd 100644 --- a/pelican/tests/test_pelican.py +++ b/pelican/tests/test_pelican.py @@ -44,8 +44,8 @@ class TestPelican(LoggedTestCase): super(TestPelican, self).setUp() self.temp_path = mkdtemp(prefix='pelicantests.') self.temp_cache = mkdtemp(prefix='pelican_cache.') - self.old_locale = locale.setlocale(locale.LC_ALL) self.maxDiff = None + self.old_locale = locale.setlocale(locale.LC_ALL) locale.setlocale(locale.LC_ALL, str('C')) def tearDown(self): diff --git a/pelican/tests/test_settings.py b/pelican/tests/test_settings.py index 7907a551..930e0fea 100644 --- a/pelican/tests/test_settings.py +++ b/pelican/tests/test_settings.py @@ -16,10 +16,15 @@ class TestSettingsConfiguration(unittest.TestCase): optimizations. """ def setUp(self): + self.old_locale = locale.setlocale(locale.LC_ALL) + locale.setlocale(locale.LC_ALL, str('C')) self.PATH = abspath(dirname(__file__)) default_conf = join(self.PATH, 'default_conf.py') self.settings = read_settings(default_conf) + def tearDown(self): + locale.setlocale(locale.LC_ALL, self.old_locale) + def test_overwrite_existing_settings(self): self.assertEqual(self.settings.get('SITENAME'), "Alexis' log") self.assertEqual(self.settings.get('SITEURL'), diff --git a/pelican/tests/test_utils.py b/pelican/tests/test_utils.py index 9047593f..02398336 100644 --- a/pelican/tests/test_utils.py +++ b/pelican/tests/test_utils.py @@ -354,9 +354,12 @@ class TestCopy(unittest.TestCase): def setUp(self): self.root_dir = mkdtemp(prefix='pelicantests.') + self.old_locale = locale.setlocale(locale.LC_ALL) + locale.setlocale(locale.LC_ALL, str('C')) def tearDown(self): shutil.rmtree(self.root_dir) + locale.setlocale(locale.LC_ALL, self.old_locale) def _create_file(self, *path): with open(os.path.join(self.root_dir, *path), 'w') as f: