remote/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2026-09-20 23:14:05 +02:00
Code Issues Projects Releases Packages Wiki Activity

do not needlessly escape hypens

Browse source 
partially fixes #225.
This commit is contained in:
girst 2026-08-04 14:13:52 +02:00
commit c4b366740e
1 changed files with 8 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

10
code/robots.py
View file

@ -175,10 +175,16 @@ def json_to_table(robots_json):
def list_to_pcre(robots_json):
# Python re is not 100% identical to PCRE which is used by Apache, but it
# should probably be close enough in the real world for re.escape to work.
exact_agents = "|".join(map(re.escape, robots_json))
# We additionally un-escape '-' since it only requires escaping within
# character classes (which are also escaped and prevented here).
def escape(pattern):
return re.escape(pattern).replace("\\-", "-")
exact_agents = "|".join(map(escape, robots_json))
patterns = [f"^({exact_agents})$"]
patterns.extend(
f"{re.escape(agent)}/[0-9.]+"
f"{escape(agent)}/[0-9.]+"
for agent, config in robots_json.items()
if config.get("has_name_and_version", False)
)