ai.robots.txt/servers/apache.conf

# for apache2.conf or .htaccess; intended to block via user agent string
RewriteEngine On
RewriteCond %{HTTP_USER_AGENT} (AdsBot-Google|Amazonbot|anthropic-ai|Applebot|Applebot-Extended|AwarioRssBot|AwarioSmartBot|Bytespider|CCBot|ChatGPT-User|ClaudeBot|Claude-Web|cohere-ai|DataForSeoBot|Diffbot|FacebookBot|FriendlyCrawler|Google-Extended|GoogleOther|GPTBot|img2dataset|ImagesiftBot|magpie-crawler|Meltwater|omgili|omgilibot|peer39_crawler|peer39_crawler/1.0|PerplexityBot|PiplBot|scoop.it|Seekr|YouBot) [NC]
RewriteRule .* - [F,L]
chore: create apache.conf.txt 2024-04-08 12:26:04 -07:00			`# for apache2.conf or .htaccess; intended to block via user agent string`
			`RewriteEngine On`
chore: propagate applebot-extended out 2024-06-12 18:53:26 -07:00			`RewriteCond %{HTTP_USER_AGENT} (AdsBot-Google\|Amazonbot\|anthropic-ai\|Applebot\|Applebot-Extended\|AwarioRssBot\|AwarioSmartBot\|Bytespider\|CCBot\|ChatGPT-User\|ClaudeBot\|Claude-Web\|cohere-ai\|DataForSeoBot\|Diffbot\|FacebookBot\|FriendlyCrawler\|Google-Extended\|GoogleOther\|GPTBot\|img2dataset\|ImagesiftBot\|magpie-crawler\|Meltwater\|omgili\|omgilibot\|peer39_crawler\|peer39_crawler/1.0\|PerplexityBot\|PiplBot\|scoop.it\|Seekr\|YouBot) [NC]`
chore: create apache.conf.txt 2024-04-08 12:26:04 -07:00			`RewriteRule .* - [F,L]`