From 70e83e60564a7beaa5b056d3cabf0d8546cc481e Mon Sep 17 00:00:00 2001 From: Flynn Marquardt Date: Fri, 13 Feb 2026 19:18:46 +0100 Subject: [PATCH 1/7] Add generation and output of lighttpd configuration fragment. --- code/robots.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/code/robots.py b/code/robots.py index a2fb7ec..21b6cc7 100755 --- a/code/robots.py +++ b/code/robots.py @@ -188,6 +188,13 @@ def json_to_nginx(robot_json): return config +def json_to_lighttpd(robot_json): + # Creates an Lighttpd config file. This config snippet can be included in + # Lighttpd configuration global or in $HTTP conditionals to block AI bots. + config = f"$HTTP[\"url\"] != \"/robots.txt\" {{ $HTTP[\"user-agent\"] =~ \"{list_to_pcre(robot_json.keys())}\" {{ url.access-deny = ( \"\" ) }} }}" + return config + + def json_to_caddy(robot_json): caddyfile = "@aibots {\n " caddyfile += f' header_regexp User-Agent "{list_to_pcre(robot_json.keys())}"' @@ -230,6 +237,10 @@ def conversions(): file_name="./nginx-block-ai-bots.conf", converter=json_to_nginx, ) + update_file_if_changed( + file_name="./lighttpd-block-ai-bots.conf", + converter=json_to_lighttpd, + ) update_file_if_changed( file_name="./Caddyfile", converter=json_to_caddy, From deae8eed2d4e9c41b38034dda0150e178afee179 Mon Sep 17 00:00:00 2001 From: Flynn Marquardt Date: Fri, 13 Feb 2026 19:20:12 +0100 Subject: [PATCH 2/7] Update with lighttpd instructions. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 586e9f7..b239da5 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ This repository provides the following files: - `nginx-block-ai-bots.conf` - `Caddyfile` - `haproxy-block-ai-bots.txt` +- `lighttpd-block-ai-bots.conf` `robots.txt` implements the Robots Exclusion Protocol ([RFC 9309](https://www.rfc-editor.org/rfc/rfc9309.html)). @@ -35,6 +36,7 @@ Note that, as stated in the [httpd documentation](https://httpd.apache.org/docs/ ``` (Note that the path of the `haproxy-block-ai-bots.txt` may be different in your environment.) +`lighttpd-block-ai-bots.conf` can be included with `include "fragments/lighttpd-block-ai-bots.conf"` in your lighttpd configuratio either global or in conditional section. [Bing uses the data it crawls for AI and training, you may opt out by adding a `meta` tag to the `head` of your site.](./docs/additional-steps/bing.md) From 55d404c115c56b1b70cb5e76e33be84a75781a81 Mon Sep 17 00:00:00 2001 From: Flynn Marquardt Date: Fri, 13 Feb 2026 19:21:31 +0100 Subject: [PATCH 3/7] Fix typos. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b239da5..7ac0618 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Note that, as stated in the [httpd documentation](https://httpd.apache.org/docs/ ``` (Note that the path of the `haproxy-block-ai-bots.txt` may be different in your environment.) -`lighttpd-block-ai-bots.conf` can be included with `include "fragments/lighttpd-block-ai-bots.conf"` in your lighttpd configuratio either global or in conditional section. +`lighttpd-block-ai-bots.conf` can be included with `include "fragments/lighttpd-block-ai-bots.conf"` in your lighttpd configuration either global or in any conditional section. [Bing uses the data it crawls for AI and training, you may opt out by adding a `meta` tag to the `head` of your site.](./docs/additional-steps/bing.md) From 5f0dd97ccbb9973744a98f082f0017ec207bf39b Mon Sep 17 00:00:00 2001 From: Flynn Marquardt Date: Fri, 13 Feb 2026 19:23:19 +0100 Subject: [PATCH 4/7] Add lighttpd sample file. --- lighttpd-block-ai-bots.conf | 1 + 1 file changed, 1 insertion(+) create mode 100644 lighttpd-block-ai-bots.conf diff --git a/lighttpd-block-ai-bots.conf b/lighttpd-block-ai-bots.conf new file mode 100644 index 0000000..24ddb2d --- /dev/null +++ b/lighttpd-block-ai-bots.conf @@ -0,0 +1 @@ +$HTTP["url"] != "/robots.txt" { $HTTP["user-agent"] =~ "(AddSearchBot|AI2Bot|AI2Bot\-DeepResearchEval|Ai2Bot\-Dolma|aiHitBot|amazon\-kendra|Amazonbot|AmazonBuyForMe|Amzn\-SearchBot|Amzn\-User|Andibot|Anomura|anthropic\-ai|Applebot|Applebot\-Extended|atlassian\-bot|Awario|AzureAI\-SearchBot|bedrockbot|bigsur\.ai|Bravebot|Brightbot\ 1\.0|BuddyBot|Bytespider|CCBot|Channel3Bot|ChatGLM\-Spider|ChatGPT\ Agent|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|Cloudflare\-AutoRAG|CloudVertexBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawl4AI|Crawlspace|Datenbank\ Crawler|DeepSeekBot|Devin|Diffbot|DuckAssistBot|Echobot\ Bot|EchoboxBot|FacebookBot|facebookexternalhit|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Gemini\-Deep\-Research|Google\-CloudVertexBot|Google\-Extended|Google\-Firebase|Google\-NotebookLM|GoogleAgent\-Mariner|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iAskBot|iaskspider|iaskspider/2\.0|IbouBot|ICC\-Crawler|ImagesiftBot|imageSpider|img2dataset|ISSCyberRiskCrawler|kagi\-fetcher|Kangaroo\ Bot|KlaviyoAIBot|KunatoCrawler|laion\-huggingface\-processor|LAIONDownloader|LCC|LinerBot|Linguee\ Bot|LinkupBot|Manus\-User|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|meta\-webindexer|MistralAI\-User|MistralAI\-User/1\.0|MyCentralAIScraperBot|netEstate\ Imprint\ Crawler|NotebookLM|NovaAct|OAI\-SearchBot|omgili|omgilibot|OpenAI|Operator|PanguBot|Panscient|panscient\.com|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|Poggio\-Citations|Poseidon\ Research\ Crawler|QualifiedBot|QuillBot|quillbot\.com|SBIntuitionsBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|ShapBot|Sidetrade\ indexer\ bot|Spider|TavilyBot|TerraCotta|Thinkbot|TikTokSpider|Timpibot|TwinAgent|VelenPublicWebCrawler|WARDBot|Webzio\-Extended|webzio\-extended|wpbot|WRTNBot|YaK|YandexAdditional|YandexAdditionalBot|YouBot|ZanistaBot)" { url.access-deny = ( "" ) } } \ No newline at end of file From 742f9b9b5ecf1c55f43d9efbac9e0e1080c0d6cd Mon Sep 17 00:00:00 2001 From: Flynn Marquardt Date: Sun, 15 Feb 2026 11:17:54 +0100 Subject: [PATCH 5/7] Add lighttpd test. --- code/tests.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/code/tests.py b/code/tests.py index 434406f..978c32b 100755 --- a/code/tests.py +++ b/code/tests.py @@ -4,7 +4,7 @@ import json import unittest -from robots import json_to_txt, json_to_table, json_to_htaccess, json_to_nginx, json_to_haproxy, json_to_caddy +from robots import json_to_txt, json_to_table, json_to_htaccess, json_to_nginx, json_to_haproxy, json_to_caddy, json_to_lighttpd class RobotsUnittestExtensions: def loadJson(self, pathname): @@ -86,6 +86,16 @@ class TestCaddyfileGeneration(unittest.TestCase, RobotsUnittestExtensions): robots_caddyfile = json_to_caddy(self.robots_dict) self.assertEqualsFile("test_files/Caddyfile", robots_caddyfile) +class TestLighttpdConfigGeneration(unittest.TestCase, RobotsUnittestExtensions): + maxDiff = 8192 + + def setUp(self): + self.robots_dict = self.loadJson("test_files/robots.json") + + def test_lighttpd_generation(self): + robots_lighttpd = json_to_lighttpd(self.robots_dict) + self.assertEqualsFile("test_files/lighttpd-block-ai-bots.conf", robots_lighttpd) + if __name__ == "__main__": import os From 790bff181777c6c21528f1c07b3c325c1121aeed Mon Sep 17 00:00:00 2001 From: Flynn Marquardt Date: Sun, 15 Feb 2026 11:18:35 +0100 Subject: [PATCH 6/7] Add reference file for lighttpd test. --- code/test_files/lighttpd-block-ai-bots.conf | 1 + 1 file changed, 1 insertion(+) create mode 100644 code/test_files/lighttpd-block-ai-bots.conf diff --git a/code/test_files/lighttpd-block-ai-bots.conf b/code/test_files/lighttpd-block-ai-bots.conf new file mode 100644 index 0000000..a5a4bac --- /dev/null +++ b/code/test_files/lighttpd-block-ai-bots.conf @@ -0,0 +1 @@ +$HTTP["url"] != "/robots.txt" { $HTTP["user-agent"] =~ "(AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash)" { url.access-deny = ( "" ) } } \ No newline at end of file From 793b0454f28aaf1c62d0ae72684fd466aa63ba39 Mon Sep 17 00:00:00 2001 From: Flynn Marquardt Date: Sun, 15 Feb 2026 11:20:34 +0100 Subject: [PATCH 7/7] Use adverb. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ac0618..ed17862 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Note that, as stated in the [httpd documentation](https://httpd.apache.org/docs/ ``` (Note that the path of the `haproxy-block-ai-bots.txt` may be different in your environment.) -`lighttpd-block-ai-bots.conf` can be included with `include "fragments/lighttpd-block-ai-bots.conf"` in your lighttpd configuration either global or in any conditional section. +`lighttpd-block-ai-bots.conf` can be included with `include "fragments/lighttpd-block-ai-bots.conf"` in your lighttpd configuration either globally or in any conditional section. [Bing uses the data it crawls for AI and training, you may opt out by adding a `meta` tag to the `head` of your site.](./docs/additional-steps/bing.md)