remote/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2026-09-24 19:04:16 +02:00
Code Issues Projects Releases Packages Wiki Activity

robots.py: list_to_pcre don't overwrite patterns

Browse source
This commit is contained in:
guest20 2026-08-04 14:46:01 +02:00 • committed by GitHub
commit 9d3a286796
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 4 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

7
code/robots.py
View file

@ -191,15 +191,16 @@ def list_to_pcre(robots_json):
for agent, config in robots_json.items()
if config.get("sends_full_name", False)
)
patterns = [f"^({exact_agents})$"]
patterns.extend( f"^({exact_agents})$" )
# agents who use Name/1.1.3-style elements
patterns.extend(
versioned_agents = "|".join(
f"{re.escape(agent)}/[0-9.]+"
for agent, config in robots_json.items()
if config.get("has_name_and_version", False)
)
patterns.extend( versioned_agents )
return f"({'|'.join(patterns)})"