remote/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2026-08-31 13:24:05 +02:00
Code Issues Projects Releases Packages Wiki Activity

robots.py: list_to_pcre join a list of patterns isntead

Browse source
This commit is contained in:
guest20 2026-08-04 14:56:33 +02:00 • committed by GitHub
commit d98c220be8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

12
code/robots.py
View file

@ -178,23 +178,23 @@ def list_to_pcre(robots_json):
patterns = []
# regular old substring matches:
formatted = "|".join([
formatted = [
f"{re.escape(agent)}"
for agent, config in robots_json.items()
if not config.get("sends_full_name", False) and not config.get("has_name_and_version", False)
])
patterns.extend( f"({formatted})" )
]
patterns.extend( formatted )
# agents that just send their names pokemon-style
exact_agents = "|".join([
exact_agents = [
f"^{re.escape(agent)}$"
for agent, config in robots_json.items()
if config.get("sends_full_name", False)
])
patterns.extend( f"^({exact_agents})$" )
patterns.extend(exact_agents)
# agents who use Name/1.1.3-style elements
versioned_agents = "|".join([
versioned_agents = [
f"{re.escape(agent)}/[0-9.]+"
for agent, config in robots_json.items()
if config.get("has_name_and_version", False)