Merge df2472a871 into a740f8aa88

pelican/contents.py

@@ -95,11 +95,13 @@ class Content(object):
         if not hasattr(self, 'slug'):
             if settings['SLUGIFY_SOURCE'] == 'title' and hasattr(self, 'title'):
                 self.slug = slugify(self.title,
-                                settings.get('SLUG_SUBSTITUTIONS', ()))
+                                settings.get('SLUG_SUBSTITUTIONS', ()),
+                                settings['ALLOW_NON_ASCII_IN_SLUG'])
             elif settings['SLUGIFY_SOURCE'] == 'basename' and source_path != None:
                 basename = os.path.basename(os.path.splitext(source_path)[0])
                 self.slug = slugify(basename,
-                                settings.get('SLUG_SUBSTITUTIONS', ()))
+                                settings.get('SLUG_SUBSTITUTIONS', ()),
+                                settings['ALLOW_NON_ASCII_IN_SLUG'])
 
         self.source_path = source_path
 
@@ -175,11 +177,13 @@ class Content(object):
             'date': getattr(self, 'date', SafeDatetime.now()),
             'author': slugify(
                 getattr(self, 'author', ''),
-                slug_substitutions
+                slug_substitutions,
+                self.settings['ALLOW_NON_ASCII_IN_SLUG']
             ),
             'category': slugify(
                 getattr(self, 'category', default_category),
-                slug_substitutions
+                slug_substitutions,
+                self.settings['ALLOW_NON_ASCII_IN_SLUG']
             )
         })
         return metadata

pelican/settings.py

@@ -125,6 +125,7 @@ DEFAULT_CONFIG = {
     'TEMPLATE_PAGES': {},
     'IGNORE_FILES': ['.#*'],
     'SLUG_SUBSTITUTIONS': (),
+    'ALLOW_NON_ASCII_IN_SLUG': False,
     'INTRASITE_LINK_REGEX': '[{|](?P<what>.*?)[|}]',
     'SLUGIFY_SOURCE': 'title',
     'CACHE_CONTENT': True,

pelican/tests/test_utils.py

@@ -118,6 +118,8 @@ class TestUtils(LoggedTestCase):
 
         for value, expected in samples:
             self.assertEqual(utils.slugify(value), expected)
+            # nothing will be changed if allow_non_ascii is True.
+            self.assertEqual(utils.slugify(value, allow_non_ascii=True), value)
 
     def test_slugify_substitute(self):
 

pelican/urlwrappers.py

@@ -17,7 +17,9 @@ class URLWrapper(object):
         # but are here for clarity
         self.settings = settings
         self._name = name
-        self.slug = slugify(name, self.settings.get('SLUG_SUBSTITUTIONS', ()))
+        self.slug = slugify(name,
+                self.settings.get('SLUG_SUBSTITUTIONS', ()),
+                self.settings['ALLOW_NON_ASCII_IN_SLUG'])
         self.name = name
 
     @property
@@ -27,7 +29,9 @@ class URLWrapper(object):
     @name.setter
     def name(self, name):
         self._name = name
-        self.slug = slugify(name, self.settings.get('SLUG_SUBSTITUTIONS', ()))
+        self.slug = slugify(name,
+                self.settings.get('SLUG_SUBSTITUTIONS', ()),
+                self.settings['ALLOW_NON_ASCII_IN_SLUG'])
 
     def as_dict(self):
         d = self.__dict__
@@ -42,7 +46,7 @@ class URLWrapper(object):
 
     def _normalize_key(self, key):
         subs = self.settings.get('SLUG_SUBSTITUTIONS', ())
-        return six.text_type(slugify(key, subs))
+        return six.text_type(slugify(key, subs, self.settings['ALLOW_NON_ASCII_IN_SLUG']))
 
     def __eq__(self, other):
         return self._key() == self._normalize_key(other)

pelican/utils.py

@@ -240,7 +240,7 @@ def pelican_open(filename):
     yield content
 
 
-def slugify(value, substitutions=()):
+def slugify(value, substitutions=(), allow_non_ascii=False):
     """
     Normalizes string, converts to lowercase, removes non-alpha characters,
     and converts spaces to hyphens.
@@ -249,24 +249,27 @@ def slugify(value, substitutions=()):
     """
     # TODO Maybe steal again from current Django 1.5dev
     value = Markup(value).striptags()
-    # value must be unicode per se
-    import unicodedata
-    from unidecode import unidecode
-    # unidecode returns str in Py2 and 3, so in Py2 we have to make
-    # it unicode again
-    value = unidecode(value)
-    if isinstance(value, six.binary_type):
-        value = value.decode('ascii')
-    # still unicode
-    value = unicodedata.normalize('NFKD', value).lower()
+
     for src, dst in substitutions:
         value = value.replace(src.lower(), dst.lower())
-    value = re.sub('[^\w\s-]', '', value).strip()
-    value = re.sub('[-\s]+', '-', value)
-    # we want only ASCII chars
-    value = value.encode('ascii', 'ignore')
+
+    if not allow_non_ascii:
+        # value must be unicode per se
+        from unidecode import unidecode
+        # unidecode returns str in Py2 and 3, so in Py2 we have to make
+        # it unicode again
+        value = unidecode(value)
+        if isinstance(value, six.binary_type):
+            value = value.decode('ascii')
+        # still unicode
+        import unicodedata
+        value = unicodedata.normalize('NFKD', value).lower()
+        value = re.sub('[^\w\s-]', '', value).strip()
+        value = re.sub('[-\s]+', '-', value)
+        # we want only ASCII chars
+        value = value.encode('ascii', 'ignore')
     # but Pelican should generally use only unicode
-    return value.decode('ascii')
+    return unicode(value)
 
 
 def copy(source, destination):