remote/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2026-09-27 20:34:16 +02:00
Code Issues Projects Releases Packages Wiki Activity

robots.py: list_to_prcre - joining strings joins charaacters

Browse source 
A|m|a|z|o|n|b|o|t|||a|n|t|h|r|o|p|i|c
This commit is contained in:
guest20 2026-08-04 14:48:57 +02:00 • committed by GitHub
commit eed7c67963
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

12
code/robots.py
View file

@ -178,27 +178,27 @@ def list_to_pcre(robots_json):
patterns = []
# regular old substring matches:
formatted = "|".join(
formatted = "|".join([
f"{re.escape(agent)}"
for agent, config in robots_json.items()
if not config.get("sends_full_name", False) and not config.get("has_name_and_version", False)
)
])
patterns.extend( f"({formatted})" )
# agents that just send their names pokemon-style
exact_agents = "|".join(
exact_agents = "|".join([
f"^{re.escape(agent)}$"
for agent, config in robots_json.items()
if config.get("sends_full_name", False)
)
])
patterns.extend( f"^({exact_agents})$" )
# agents who use Name/1.1.3-style elements
versioned_agents = "|".join(
versioned_agents = "|".join([
f"{re.escape(agent)}/[0-9.]+"
for agent, config in robots_json.items()
if config.get("has_name_and_version", False)
)
])
patterns.extend( versioned_agents )
return f"({'|'.join(patterns)})"