remote/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2026-06-15 13:06:54 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add generation and output of lighttpd configuration fragment.

Browse source
This commit is contained in:
Flynn Marquardt 2026-02-13 19:18:46 +01:00
commit 70e83e6056
1 changed files with 11 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

11
code/robots.py
View file

@ -188,6 +188,13 @@ def json_to_nginx(robot_json):
return config
def json_to_lighttpd(robot_json):
# Creates an Lighttpd config file. This config snippet can be included in
# Lighttpd configuration global or in $HTTP conditionals to block AI bots.
config = f"$HTTP[\"url\"] != \"/robots.txt\" {{ $HTTP[\"user-agent\"] =~ \"{list_to_pcre(robot_json.keys())}\" {{ url.access-deny = ( \"\" ) }} }}"
return config
def json_to_caddy(robot_json):
caddyfile = "@aibots {\n "
caddyfile += f' header_regexp User-Agent "{list_to_pcre(robot_json.keys())}"'
@ -230,6 +237,10 @@ def conversions():
file_name="./nginx-block-ai-bots.conf",
converter=json_to_nginx,
)
update_file_if_changed(
file_name="./lighttpd-block-ai-bots.conf",
converter=json_to_lighttpd,
)
update_file_if_changed(
file_name="./Caddyfile",
converter=json_to_caddy,