github/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-12-29 12:18:33 +01:00
Code Issues Projects Releases Packages Wiki Activity

Added Test and Docu

Browse source
This commit is contained in:
Julian Beittel 2025-04-23 09:41:18 +02:00
commit 1e92364c13
5 changed files with 19 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

56
Caddyfile
View file

@ -1,57 +1,3 @@
@aibots {
header User-Agent "*AI2Bot*"
header User-Agent "*Ai2Bot-Dolma*"
header User-Agent "*aiHitBot*"
header User-Agent "*Amazonbot*"
header User-Agent "*anthropic-ai*"
header User-Agent "*Applebot*"
header User-Agent "*Applebot-Extended*"
header User-Agent "*Brightbot 1.0*"
header User-Agent "*Bytespider*"
header User-Agent "*CCBot*"
header User-Agent "*ChatGPT-User*"
header User-Agent "*Claude-Web*"
header User-Agent "*ClaudeBot*"
header User-Agent "*cohere-ai*"
header User-Agent "*cohere-training-data-crawler*"
header User-Agent "*Cotoyogi*"
header User-Agent "*Crawlspace*"
header User-Agent "*Diffbot*"
header User-Agent "*DuckAssistBot*"
header User-Agent "*FacebookBot*"
header User-Agent "*Factset_spyderbot*"
header User-Agent "*FirecrawlAgent*"
header User-Agent "*FriendlyCrawler*"
header User-Agent "*Google-Extended*"
header User-Agent "*GoogleOther*"
header User-Agent "*GoogleOther-Image*"
header User-Agent "*GoogleOther-Video*"
header User-Agent "*GPTBot*"
header User-Agent "*iaskspider/2.0*"
header User-Agent "*ICC-Crawler*"
header User-Agent "*ImagesiftBot*"
header User-Agent "*img2dataset*"
header User-Agent "*imgproxy*"
header User-Agent "*ISSCyberRiskCrawler*"
header User-Agent "*Kangaroo Bot*"
header User-Agent "*Meta-ExternalAgent*"
header User-Agent "*Meta-ExternalFetcher*"
header User-Agent "*NovaAct*"
header User-Agent "*OAI-SearchBot*"
header User-Agent "*omgili*"
header User-Agent "*omgilibot*"
header User-Agent "*Operator*"
header User-Agent "*PanguBot*"
header User-Agent "*Perplexity-User*"
header User-Agent "*PerplexityBot*"
header User-Agent "*PetalBot*"
header User-Agent "*Scrapy*"
header User-Agent "*SemrushBot-OCOB*"
header User-Agent "*SemrushBot-SWA*"
header User-Agent "*Sidetrade indexer bot*"
header User-Agent "*TikTokSpider*"
header User-Agent "*Timpibot*"
header User-Agent "*VelenPublicWebCrawler*"
header User-Agent "*Webzio-Extended*"
header User-Agent "*YouBot*"
header_regexp User-Agent "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|imgproxy|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot)"
}