Merge pull request #1309 from al-tonio/date_format_japanese_fix_python2

Fix for locale related issues while running the tests.

pelican/generators.py

@@ -309,7 +309,20 @@ class ArticlesGenerator(Generator):
                 # format string syntax can be used for specifying the
                 # period archive dates
                 date = archive[0].date
-                save_as = save_as_fmt.format(date=date)
+                # Under python 2, with non-ascii locales, u"{:%b}".format(date) might raise UnicodeDecodeError
+                # because u"{:%b}".format(date) will call date.__format__(u"%b"), which will return a byte string
+                # and not a unicode string.
+                # eg:
+                # locale.setlocale(locale.LC_ALL, 'ja_JP.utf8')
+                # date.__format__(u"%b") == '12\xe6\x9c\x88' # True
+                try:
+                    save_as = save_as_fmt.format(date=date)
+                except UnicodeDecodeError:
+                    # Python2 only:
+                    # Let date.__format__() work with byte strings instead of characters since it fails to work with characters
+                    bytes_save_as_fmt = save_as_fmt.encode('utf8')
+                    bytes_save_as     = bytes_save_as_fmt.format(date=date)
+                    save_as           = unicode(bytes_save_as,'utf8')
                 context = self.context.copy()
 
                 if key == period_date_key['year']:

pelican/tests/test_contents.py

@@ -4,6 +4,7 @@ from __future__ import unicode_literals, absolute_import
 import six
 from datetime import datetime
 from sys import platform
+import locale
 
 from pelican.tests.support import unittest, get_settings
 
@@ -22,6 +23,8 @@ class TestPage(unittest.TestCase):
 
     def setUp(self):
         super(TestPage, self).setUp()
+        self.old_locale = locale.setlocale(locale.LC_ALL)
+        locale.setlocale(locale.LC_ALL, str('C'))
         self.page_kwargs = {
             'content': TEST_CONTENT,
             'context': {
@@ -35,6 +38,9 @@ class TestPage(unittest.TestCase):
             'source_path': '/path/to/file/foo.ext'
         }
 
+    def tearDown(self):
+        locale.setlocale(locale.LC_ALL, self.old_locale)
+
     def test_use_args(self):
         # Creating a page with arguments passed to the constructor should use
         # them to initialise object's attributes.

pelican/tests/test_generators.py

@@ -14,6 +14,7 @@ from pelican.generators import (Generator, ArticlesGenerator, PagesGenerator,
                                 TemplatePagesGenerator)
 from pelican.writers import Writer
 from pelican.tests.support import unittest, get_settings
+import locale
 
 CUR_DIR = os.path.dirname(__file__)
 CONTENT_DIR = os.path.join(CUR_DIR, 'content')
@@ -21,11 +22,17 @@ CONTENT_DIR = os.path.join(CUR_DIR, 'content')
 
 class TestGenerator(unittest.TestCase):
     def setUp(self):
+        self.old_locale = locale.setlocale(locale.LC_ALL)
+        locale.setlocale(locale.LC_ALL, str('C'))
         self.settings = get_settings()
         self.settings['READERS'] = {'asc': None}
         self.generator = Generator(self.settings.copy(), self.settings,
                                    CUR_DIR, self.settings['THEME'], None)
 
+    def tearDown(self):
+        locale.setlocale(locale.LC_ALL, self.old_locale)
+
+
     def test_include_path(self):
         filename = os.path.join(CUR_DIR, 'content', 'article.rst')
         include_path = self.generator._include_path
@@ -373,10 +380,14 @@ class TestTemplatePagesGenerator(unittest.TestCase):
     def setUp(self):
         self.temp_content = mkdtemp(prefix='pelicantests.')
         self.temp_output = mkdtemp(prefix='pelicantests.')
+        self.old_locale = locale.setlocale(locale.LC_ALL)
+        locale.setlocale(locale.LC_ALL, str('C'))
+
 
     def tearDown(self):
         rmtree(self.temp_content)
         rmtree(self.temp_output)
+        locale.setlocale(locale.LC_ALL, self.old_locale)
 
     def test_generate_output(self):
 

pelican/tests/test_importer.py

@@ -4,6 +4,7 @@ from __future__ import unicode_literals, print_function
 import os
 import re
 
+import locale
 from pelican.tools.pelican_import import wp2fields, fields2pelican, decode_wp_content, build_header, build_markdown_header, get_attachments, download_attachments
 from pelican.tests.support import (unittest, temporary_folder, mute,
                                    skipIfNoExecutable)
@@ -30,9 +31,14 @@ except ImportError:
 class TestWordpressXmlImporter(unittest.TestCase):
 
     def setUp(self):
+        self.old_locale = locale.setlocale(locale.LC_ALL)
+        locale.setlocale(locale.LC_ALL, str('C'))
         self.posts = list(wp2fields(WORDPRESS_XML_SAMPLE))
         self.custposts = list(wp2fields(WORDPRESS_XML_SAMPLE, True))
 
+    def tearDown(self):
+        locale.setlocale(locale.LC_ALL, self.old_locale)
+
     def test_ignore_empty_posts(self):
         self.assertTrue(self.posts)
         for title, content, fname, date, author, categ, tags, kind, format in self.posts:
@@ -261,8 +267,13 @@ class TestBuildHeader(unittest.TestCase):
 @unittest.skipUnless(BeautifulSoup, 'Needs BeautifulSoup module')
 class TestWordpressXMLAttachements(unittest.TestCase):
     def setUp(self):
+        self.old_locale = locale.setlocale(locale.LC_ALL)
+        locale.setlocale(locale.LC_ALL, str('C'))
         self.attachments = get_attachments(WORDPRESS_XML_SAMPLE)
 
+    def tearDown(self):
+        locale.setlocale(locale.LC_ALL, self.old_locale)
+
     def test_recognise_attachments(self):
         self.assertTrue(self.attachments)
         self.assertTrue(len(self.attachments.keys()) == 3)

pelican/tests/test_paginator.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, absolute_import
 import six
+import locale
 
 from pelican.tests.support import unittest, get_settings
 
@@ -16,6 +17,8 @@ TEST_SUMMARY = generate_lorem_ipsum(n=1, html=False)
 class TestPage(unittest.TestCase):
     def setUp(self):
         super(TestPage, self).setUp()
+        self.old_locale = locale.setlocale(locale.LC_ALL)
+        locale.setlocale(locale.LC_ALL, str('C'))
         self.page_kwargs = {
             'content': TEST_CONTENT,
             'context': {
@@ -29,6 +32,9 @@ class TestPage(unittest.TestCase):
             'source_path': '/path/to/file/foo.ext'
         }
 
+    def tearDown(self):
+        locale.setlocale(locale.LC_ALL, self.old_locale)
+
     def test_save_as_preservation(self):
         settings = get_settings()
         # fix up pagination rules
@@ -47,4 +53,4 @@ class TestPage(unittest.TestCase):
         object_list = [Article(**self.page_kwargs), Article(**self.page_kwargs)]
         paginator = Paginator('foobar.foo', object_list, settings)
         page = paginator.page(1)
-        self.assertEqual(page.save_as, 'foobar.foo')
+        self.assertEqual(page.save_as, 'foobar.foo')

pelican/tests/test_pelican.py

@@ -44,8 +44,8 @@ class TestPelican(LoggedTestCase):
         super(TestPelican, self).setUp()
         self.temp_path = mkdtemp(prefix='pelicantests.')
         self.temp_cache = mkdtemp(prefix='pelican_cache.')
-        self.old_locale = locale.setlocale(locale.LC_ALL)
         self.maxDiff = None
+        self.old_locale = locale.setlocale(locale.LC_ALL)
         locale.setlocale(locale.LC_ALL, str('C'))
 
     def tearDown(self):

pelican/tests/test_settings.py

@@ -16,10 +16,15 @@ class TestSettingsConfiguration(unittest.TestCase):
     optimizations.
     """
     def setUp(self):
+        self.old_locale = locale.setlocale(locale.LC_ALL)
+        locale.setlocale(locale.LC_ALL, str('C'))
         self.PATH = abspath(dirname(__file__))
         default_conf = join(self.PATH, 'default_conf.py')
         self.settings = read_settings(default_conf)
 
+    def tearDown(self):
+        locale.setlocale(locale.LC_ALL, self.old_locale)
+
     def test_overwrite_existing_settings(self):
         self.assertEqual(self.settings.get('SITENAME'), "Alexis' log")
         self.assertEqual(self.settings.get('SITEURL'),

pelican/tests/test_utils.py

@@ -354,9 +354,12 @@ class TestCopy(unittest.TestCase):
 
     def setUp(self):
         self.root_dir = mkdtemp(prefix='pelicantests.')
+        self.old_locale = locale.setlocale(locale.LC_ALL)
+        locale.setlocale(locale.LC_ALL, str('C'))
 
     def tearDown(self):
         shutil.rmtree(self.root_dir)
+        locale.setlocale(locale.LC_ALL, self.old_locale)
 
     def _create_file(self, *path):
         with open(os.path.join(self.root_dir, *path), 'w') as f: