diff --git a/code/robots.py b/code/robots.py index e63d2bd..8b72388 100755 --- a/code/robots.py +++ b/code/robots.py @@ -178,27 +178,27 @@ def list_to_pcre(robots_json): patterns = [] # regular old substring matches: - formatted = "|".join( + formatted = "|".join([ f"{re.escape(agent)}" for agent, config in robots_json.items() if not config.get("sends_full_name", False) and not config.get("has_name_and_version", False) - ) + ]) patterns.extend( f"({formatted})" ) # agents that just send their names pokemon-style - exact_agents = "|".join( + exact_agents = "|".join([ f"^{re.escape(agent)}$" for agent, config in robots_json.items() if config.get("sends_full_name", False) - ) + ]) patterns.extend( f"^({exact_agents})$" ) # agents who use Name/1.1.3-style elements - versioned_agents = "|".join( + versioned_agents = "|".join([ f"{re.escape(agent)}/[0-9.]+" for agent, config in robots_json.items() if config.get("has_name_and_version", False) - ) + ]) patterns.extend( versioned_agents ) return f"({'|'.join(patterns)})"