Resolve agent names case-insensitively when ingesting knownagents.com

robots.txt user-agent matching is case-insensitive, so two entries whose
names differ only in case are the same crawler. updated_robots_json keyed
off the scraped name, so when knownagents.com changed the capitalisation
of a name the ingest added a second entry instead of updating the first.
consolidate() also looks the name up by exact key, so the curated operator
and respect values were left behind on the old key rather than carried over.

robots.json currently carries three such pairs, and each one emits a
duplicate User-agent line in every generated file.

This resolves an incoming name against the keys already present before
using it, so an existing entry is updated in place. No generated file
changes as a result.

code/robots.py

@@ -34,6 +34,25 @@ default_values = {
 }
 default_value = "Unclear at this time."
 
+def existing_key(existing_content, name: str) -> str:
+    """Return the key robots.json already uses for this agent, ignoring case.
+
+    robots.txt user-agent matching is case-insensitive, so two entries whose
+    names differ only in case are the same crawler. knownagents.com has changed
+    the capitalisation of a name before now, and keying off the scraped name
+    added a second entry rather than updating the first, which both duplicated
+    the User-agent line in every generated file and left the curated operator
+    and respect values behind on the old key.
+    """
+    if name in existing_content:
+        return name
+    lowered = name.lower()
+    for key in existing_content:
+        if key.lower() == lowered:
+            return key
+    return name
+
+
 def consolidate(existing_content, name: str, field: str, value: str) -> str:
     # New entry
     if name not in existing_content:
@@ -77,6 +96,7 @@ def updated_robots_json(soup):
         for agent in section.find_all("a", href=True):
             name = agent.find("div", {"class": "agent-name"}).get_text().strip()
             name = clean_robot_name(name)
+            name = existing_key(existing_content, name)
 
             desc_tag = agent.find("div", {"class": "description"})
             if desc_tag is not None:

code/tests.py

@@ -7,6 +7,7 @@ import unittest
 
 from robots import (
     consolidate,
+    existing_key,
     default_value,
     default_values,
     json_to_caddy,
@@ -203,6 +204,19 @@ class TestConsolidate(unittest.TestCase, RobotsUnittestExtensions):
         self.assertEqual("Rosie is the robot maid from The Jetsons, an American animated sitcom", 
                          consolidate(existing, "rosie", "description", "Rosie is the robot maid from The Jetsons, an American animated sitcom"))
 
+class TestExistingKey(unittest.TestCase):
+    def test_exact_match_wins(self):
+        existing = {"Rosie": {}, "rosie": {}}
+        self.assertEqual("rosie", existing_key(existing, "rosie"))
+
+    def test_matches_ignoring_case(self):
+        existing = {"Rosie": {"operator": "George Jetson"}}
+        self.assertEqual("Rosie", existing_key(existing, "rosie"))
+
+    def test_unknown_name_is_returned_unchanged(self):
+        self.assertEqual("rosie", existing_key({}, "rosie"))
+
+
 if __name__ == "__main__":
     import os
     os.chdir(os.path.dirname(__file__))