remote/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2026-08-29 20:34:04 +02:00
Code Issues Projects Releases Packages Wiki Activity

further minify regexps

Browse source 
- nginx and lighttpd allow single quoted strings, so use python's
  built-in methods to escape quotes
- only apache requires parentheses on the outside (and only for obscure
  reasons)
- caddy expects double quoted strings. at least escape inner quotes
  directly (re.escape does not do this since python 3.7), should any
  appear
This commit is contained in:
girst 2026-08-04 14:44:41 +02:00
commit a94b2c93e5
1 changed files with 17 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

25
code/robots.py
View file

@ -176,9 +176,13 @@ def list_to_pcre(robots_json):
# Python re is not 100% identical to PCRE which is used by Apache, but it
# should probably be close enough in the real world for re.escape to work.
# We additionally un-escape '-' since it only requires escaping within
# character classes (which are also escaped and prevented here).
# character classes (which are also escaped and prevented here) and '/'
# since this is not used as the regexp delimeter in any server software.
def escape(pattern):
return re.escape(pattern).replace("\\-", "-")
pattern = re.escape(pattern)
for c in "-/":
pattern = pattern.replace(fr"\{c}", c)
return pattern
exact_agents = "|".join(map(escape, robots_json))
patterns = [f"^({exact_agents})$"]
@ -188,40 +192,45 @@ def list_to_pcre(robots_json):
for agent, config in robots_json.items()
if config.get("has_name_and_version", False)
)
return f"({'|'.join(patterns)})"
return '|'.join(patterns)
def json_to_htaccess(robot_json):
# Creates a .htaccess filter file. It uses a regular expression to filter out
# User agents that contain any of the blocked values.
# The regular expression is wrapped in parenthesis, so a leading [-!=<>] does
# not accidentally change which comparison type is used.
htaccess = "RewriteEngine On\n"
htaccess += f"RewriteCond %{{HTTP_USER_AGENT}} {list_to_pcre(robot_json)} [NC]\n"
htaccess += f"RewriteCond %{{HTTP_USER_AGENT}} ({list_to_pcre(robot_json)}) [NC]\n"
htaccess += "RewriteRule !^/?robots\\.txt$ - [F]\n"
return htaccess
def json_to_nginx(robot_json):
# Creates an Nginx config file. This config snippet can be included in
# nginx server{} blocks to block AI bots.
config = f"set $block 0;\n\nif ($http_user_agent ~ \"{list_to_pcre(robot_json)}\") {{\n set $block 1;\n}}\n\nif ($request_uri = \"/robots.txt\") {{\n set $block 0;\n}}\n\nif ($block) {{\n return 403;\n}}"
config = f"set $block 0;\n\nif ($http_user_agent ~ {list_to_pcre(robot_json)!r}) {{\n set $block 1;\n}}\n\nif ($request_uri = '/robots.txt') {{\n set $block 0;\n}}\n\nif ($block) {{\n return 403;\n}}"
return config
def json_to_lighttpd(robot_json):
# Creates an Lighttpd config file. This config snippet can be included in
# Lighttpd configuration global or in $HTTP conditionals to block AI bots.
config = f"$HTTP[\"url\"] != \"/robots.txt\" {{ $HTTP[\"user-agent\"] =~ \"{list_to_pcre(robot_json)}\" {{ url.access-deny = ( \"\" ) }} }}"
config = f"$HTTP['url'] != '/robots.txt' {{ $HTTP['user-agent'] =~ {list_to_pcre(robot_json)!r} {{ url.access-deny = ( '' ) }} }}"
return config
def json_to_caddy(robot_json):
# single quotes (as returned by repr) are not valid string delimeters, so we
# must manually quote it end ensure no unescaped quotes are inside.
escaped_quotes = list_to_pcre(robot_json).replace('"', '\\"')
caddyfile = "@aibots {\n "
caddyfile += f' header_regexp User-Agent "{list_to_pcre(robot_json)}"'
caddyfile += f' header_regexp User-Agent "{escaped_quotes}"'
caddyfile += "\n}"
return caddyfile
def json_to_haproxy(robots_json):
# Creates a source file for HAProxy. Follow instructions in the README to implement it.
txt = "\n".join(f"{k}" for k in robots_json.keys())
txt = "\n".join(robots_json.keys())
return txt