Improve word count behavior when generating summary

Improve _HTMLWordTruncator by using more than one unicode block in
_word_regex, making word count function behave properly with CJK,
Cyrillic, and more Latin characters when generating summary.

pelican/tests/test_utils.py

@@ -218,6 +218,19 @@ class TestUtils(LoggedTestCase):
             utils.truncate_html_words('word ' * 100, 20),
             'word ' * 20 + '…')
 
+        # Plain text with Unicode content.
+        self.assertEqual(
+            utils.truncate_html_words(
+                '我愿意这样，朋友——我独自远行，不但没有你，\
+                 并且再没有别的影在黑暗里。', 12
+            ),
+            '我愿意这样，朋友——我独自远行' + ' …')
+        self.assertEqual(
+            utils.truncate_html_words(
+                'Ты мелькнула, ты предстала, Снова сердце задрожало,', 3
+            ),
+            'Ты мелькнула, ты' + ' …')
+
         # Words enclosed or intervaled by HTML tags.
         self.assertEqual(
             utils.truncate_html_words('<p>' + 'word ' * 100 + '</p>', 20),

pelican/utils.py

@@ -133,6 +133,7 @@ class memoized:
     (not reevaluated).
 
     """
+
     def __init__(self, func):
         self.func = func
         self.cache = {}
@@ -408,7 +409,21 @@ def posixize_path(rel_path):
 
 class _HTMLWordTruncator(HTMLParser):
 
-    _word_regex = re.compile(r"\w[\w'-]*", re.U)
+    _word_regex = re.compile(r"(({SBC})({SBC}|-|')*)|{DBC}".format(
+        # SBC means Latin-like characters. A word contains a few characters.
+        #         ASCII |Extended Latin | Cyrillic
+        SBC="[0-9a-zA-Z]|[\u00C0-\u024f]|[\u0400-\u04FF]",
+        # DBC means CJK-like characters. An character can stand for a word.
+        DBC=("([\u4E00-\u9FFF])|"          # CJK Unified Ideographs
+             "([\u3400-\u4DBF])|"          # CJK Unified Ideographs Extension A
+             "([\uF900-\uFAFF])|"          # CJK Compatibility Ideographs
+             "([\U00020000-\U0002A6DF])|"  # CJK Unified Ideographs Extension B
+             "([\U0002F800-\U0002FA1F])|"  # CJK Compatibility Ideographs Supplement
+             "([\u3040-\u30FF])|"          # Hiragana and Katakana
+             "([\u1100-\u11FF])|"          # Hangul Jamo
+             "([\uAC00-\uD7FF])|"          # Hangul Compatibility Jamo
+             "([\u3130-\u318F])"           # Hangul Syllables
+             )), re.UNICODE)
     _word_prefix_regex = re.compile(r'\w', re.U)
     _singlets = ('br', 'col', 'link', 'base', 'img', 'param', 'area',
                  'hr', 'input')
@@ -818,8 +833,8 @@ class FileSystemWatcher:
                 }
             )
             logger.warning(
-                    'No valid files found in content for the active readers:\n'
-                    + '\n'.join(reader_descs))
+                'No valid files found in content for the active readers:\n'
+                + '\n'.join(reader_descs))
 
         if result.get('theme') is None:
             logger.warning('Empty theme folder. Using `basic` theme.')