Anchor the generic Spider user agent

Signed-off-by: Dhruv Maniya <dhruvmaniya1998@gmail.com>

code/robots.py

@@ -34,6 +34,10 @@ default_values = {
 }
 default_value = "Unclear at this time."
 
+# These agents publish a complete User-Agent value rather than a token that
+# should match within a longer header.
+exact_match_agents = {"Spider"}
+
 def consolidate(existing_content, name: str, field: str, value: str) -> str:
     # New entry
     if name not in existing_content:
@@ -170,7 +174,10 @@ def json_to_table(robots_json):
 def list_to_pcre(lst):
     # Python re is not 100% identical to PCRE which is used by Apache, but it
     # should probably be close enough in the real world for re.escape to work.
-    formatted = "|".join(map(re.escape, lst))
+    formatted = "|".join(
+        f"^{re.escape(agent)}$" if agent in exact_match_agents else re.escape(agent)
+        for agent in lst
+    )
     return f"({formatted})"
 
 

code/tests.py

@@ -2,9 +2,22 @@
 """To run these tests just execute this script."""
 
 import json
+import re
 import unittest
 
-from robots import json_to_txt, json_to_table, json_to_htaccess, json_to_nginx, json_to_haproxy, json_to_caddy, json_to_lighttpd, consolidate, default_values, default_value
+from robots import (
+    consolidate,
+    default_value,
+    default_values,
+    json_to_caddy,
+    json_to_haproxy,
+    json_to_htaccess,
+    json_to_lighttpd,
+    json_to_nginx,
+    json_to_table,
+    json_to_txt,
+    list_to_pcre,
+)
 
 class RobotsUnittestExtensions:
     def loadJson(self, pathname):
@@ -50,6 +63,17 @@ class TestHtaccessGeneration(unittest.TestCase, RobotsUnittestExtensions):
         robots_htaccess = json_to_htaccess(self.robots_dict)
         self.assertEqualsFile("test_files/.htaccess", robots_htaccess)
 
+
+class TestUserAgentPatternGeneration(unittest.TestCase):
+    def test_spider_matches_only_the_complete_user_agent(self):
+        pattern = re.compile(list_to_pcre(["Spider", "ExampleBot"]), re.IGNORECASE)
+
+        self.assertIsNotNone(pattern.search("Spider"))
+        self.assertIsNotNone(pattern.search("spider"))
+        self.assertIsNone(pattern.search("Baiduspider"))
+        self.assertIsNone(pattern.search("OurCompanyName Test Spider"))
+        self.assertIsNotNone(pattern.search("Mozilla/5.0 ExampleBot/1.0"))
+
 class TestNginxConfigGeneration(unittest.TestCase, RobotsUnittestExtensions):
     maxDiff = 8192