From c4b366740e97e106fda86d3a9e8d3a42a8d81ab2 Mon Sep 17 00:00:00 2001 From: girst Date: Tue, 4 Aug 2026 14:13:52 +0200 Subject: [PATCH] do not needlessly escape hypens partially fixes #225. --- code/robots.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/code/robots.py b/code/robots.py index 2f2c444..b22e935 100755 --- a/code/robots.py +++ b/code/robots.py @@ -175,10 +175,16 @@ def json_to_table(robots_json): def list_to_pcre(robots_json): # Python re is not 100% identical to PCRE which is used by Apache, but it # should probably be close enough in the real world for re.escape to work. - exact_agents = "|".join(map(re.escape, robots_json)) + # We additionally un-escape '-' since it only requires escaping within + # character classes (which are also escaped and prevented here). + def escape(pattern): + return re.escape(pattern).replace("\\-", "-") + + exact_agents = "|".join(map(escape, robots_json)) patterns = [f"^({exact_agents})$"] + patterns.extend( - f"{re.escape(agent)}/[0-9.]+" + f"{escape(agent)}/[0-9.]+" for agent, config in robots_json.items() if config.get("has_name_and_version", False) )