Expose use_unicode setting of slugify in settings and use it

pelican/contents.py

@@ -92,16 +92,17 @@ class Content(object):
         if not hasattr(self, 'slug'):
             if (settings['SLUGIFY_SOURCE'] == 'title' and
                     hasattr(self, 'title')):
-                self.slug = slugify(
-                    self.title,
-                    regex_subs=settings.get('SLUG_REGEX_SUBSTITUTIONS', []))
+                value = self.title
             elif (settings['SLUGIFY_SOURCE'] == 'basename' and
                     source_path is not None):
-                basename = os.path.basename(
-                    os.path.splitext(source_path)[0])
+                value = os.path.basename(os.path.splitext(source_path)[0])
+            else:
+                value = None
+            if value is not None:
                 self.slug = slugify(
-                    basename,
-                    regex_subs=settings.get('SLUG_REGEX_SUBSTITUTIONS', []))
+                    value,
+                    regex_subs=settings.get('SLUG_REGEX_SUBSTITUTIONS', []),
+                    use_unicode=settings['SLUGIFY_USE_UNICODE'])
 
         self.source_path = source_path
         self.relative_source_path = self.get_relative_source_path()

pelican/settings.py

@@ -155,6 +155,7 @@ DEFAULT_CONFIG = {
     ],
     'INTRASITE_LINK_REGEX': '[{|](?P<what>.*?)[|}]',
     'SLUGIFY_SOURCE': 'title',
+    'SLUGIFY_USE_UNICODE': False,
     'CACHE_CONTENT': False,
     'CONTENT_CACHING_LAYER': 'reader',
     'CACHE_PATH': 'cache',

pelican/tests/test_contents.py

@@ -135,6 +135,17 @@ class TestPage(LoggedTestCase):
         page = Page(**page_kwargs)
         self.assertEqual(page.slug, 'foo')
 
+        # test slug from unicode title
+        # slug doesn't use unicode
+        settings['SLUGIFY_SOURCE'] = "title"
+        page_kwargs['metadata']['title'] = '指導書'
+        page = Page(**page_kwargs)
+        self.assertEqual(page.slug, 'zhi-dao-shu')
+        # slug uses unicode
+        settings['SLUGIFY_USE_UNICODE'] = True
+        page = Page(**page_kwargs)
+        self.assertEqual(page.slug, '指導書')
+
     def test_defaultlang(self):
         # If no lang is given, default to the default one.
         page = Page(**self.page_kwargs)

pelican/urlwrappers.py

@@ -34,15 +34,14 @@ class URLWrapper(object):
         if self._slug is None:
             class_key = '{}_REGEX_SUBSTITUTIONS'.format(
                 self.__class__.__name__.upper())
-            if class_key in self.settings:
-                self._slug = slugify(
-                    self.name,
-                    regex_subs=self.settings[class_key])
-            else:
-                self._slug = slugify(
-                    self.name,
-                    regex_subs=self.settings.get(
-                        'SLUG_REGEX_SUBSTITUTIONS', []))
+            regex_subs = self.settings.get(
+                class_key,
+                self.settings.get('SLUG_REGEX_SUBSTITUTIONS', []))
+            self._slug = slugify(
+                self.name,
+                regex_subs=regex_subs,
+                use_unicode=self.settings.get('SLUGIFY_USE_UNICODE', False)
+            )
         return self._slug
 
     @slug.setter
@@ -61,8 +60,13 @@ class URLWrapper(object):
         return hash(self.slug)
 
     def _normalize_key(self, key):
-        subs = self.settings.get('SLUG_REGEX_SUBSTITUTIONS', [])
-        return slugify(key, regex_subs=subs)
+        class_key = '{}_REGEX_SUBSTITUTIONS'.format(
+            self.__class__.__name__.upper())
+        regex_subs = self.settings.get(
+            class_key,
+            self.settings.get('SLUG_REGEX_SUBSTITUTIONS', []))
+        use_unicode = self.settings.get('SLUGIFY_USE_UNICODE', False)
+        return slugify(key, regex_subs=regex_subs, use_unicode=use_unicode)
 
     def __eq__(self, other):
         if isinstance(other, self.__class__):