ai.robots.txt/code/test_files/Caddyfile

@aibots {
        header_regexp User-Agent "\b(AI2Bot|Ai2Bot-Dolma|Amazonbot|anthropic-ai|Applebot|Applebot-Extended|Bytespider|CCBot|ChatGPT-User|Claude-Web|ClaudeBot|cohere-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google-Extended|GoogleOther|GoogleOther-Image|GoogleOther-Video|GPTBot|iaskspider/2\.0|ICC-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta-ExternalAgent|Meta-ExternalFetcher|OAI-SearchBot|omgili|omgilibot|Perplexity-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash)\b"
}
	`@aibots {`
	header_regexp User-Agent "\b(AI2Bot\|Ai2Bot-Dolma\|Amazonbot\|anthropic-ai\|Applebot\|Applebot-Extended\|Bytespider\|CCBot\|ChatGPT-User\|Claude-Web\|ClaudeBot\|cohere-ai\|Diffbot\|FacebookBot\|facebookexternalhit\|FriendlyCrawler\|Google-Extended\|GoogleOther\|GoogleOther-Image\|GoogleOther-Video\|GPTBot\|iaskspider/2\.0\|ICC-Crawler\|ImagesiftBot\|img2dataset\|ISSCyberRiskCrawler\|Kangaroo\ Bot\|Meta-ExternalAgent\|Meta-ExternalFetcher\|OAI-SearchBot\|omgili\|omgilibot\|Perplexity-User\|PerplexityBot\|PetalBot\|Scrapy\|Sidetrade\ indexer\ bot\|Timpibot\|VelenPublicWebCrawler\|Webzio-Extended\|YouBot\|crawler\.with\.dots\|star\\\*crawler\|Is\ this\ a\ crawler\?\|a\[mazing\]\{42\}\(robot\)\|2\^32\$\|curl\\|sudo\ bash)\b"
	`}`