remote/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2026-09-03 23:04:02 +02:00
Code Issues Projects Releases Packages Wiki Activity

robots.py: list_to_pcre gets support for sends_full_name

Browse source
This commit is contained in:
guest20 2026-08-04 14:36:25 +02:00 • committed by GitHub
commit eaf22ce5bb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 19 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

20
code/robots.py
View file

@ -175,13 +175,31 @@ def json_to_table(robots_json):
def list_to_pcre(robots_json):
# Python re is not 100% identical to PCRE which is used by Apache, but it
# should probably be close enough in the real world for re.escape to work.
exact_agents = "|".join(map(re.escape, robots_json))
patterns = []
# regular old substring matches:
formatted = "|".join(
f"{re.escape(agent)}"
for agent, config in robots_json.items()
if not config.get("sends_full_name", False) and not config.get("has_name_and_version", False)
)
patterns.extend( f"({formatted})" )
# agents that just send their names pokemon-style
exact_agents = "|".join(
f"^{re.escape(agent)}$"
for agent, config in robots_json.items()
if config.get("sends_full_name", False)
)
patterns = [f"^({exact_agents})$"]
# agents who use Name/1.1.3-style elements
patterns.extend(
f"{re.escape(agent)}/[0-9.]+"
for agent, config in robots_json.items()
if config.get("has_name_and_version", False)
)
return f"({'|'.join(patterns)})"