mirror of
https://github.com/ai-robots-txt/ai.robots.txt.git
synced 2026-09-28 04:44:29 +02:00
fix: configure exact user-agent matching
Signed-off-by: Dhruv Maniya <dhruvmaniya1998@gmail.com>
This commit is contained in:
Dhruv Maniya
parent
af31418609
commit
49669178ef
11 changed files with 43 additions and 27 deletions
|
|
@ -34,10 +34,6 @@ default_values = {
|
|||
}
|
||||
default_value = "Unclear at this time."
|
||||
|
||||
# These agents publish a complete User-Agent value rather than a token that
|
||||
# should match within a longer header.
|
||||
exact_match_agents = {"Spider"}
|
||||
|
||||
def consolidate(existing_content, name: str, field: str, value: str) -> str:
|
||||
# New entry
|
||||
if name not in existing_content:
|
||||
|
|
@ -96,7 +92,7 @@ def updated_robots_json(soup):
|
|||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
existing_content[name] = {
|
||||
robot = {
|
||||
"operator": consolidate(existing_content, name, "operator", operator),
|
||||
"respect": consolidate(existing_content, name, "respect", default_value),
|
||||
"function": consolidate(existing_content, name, "function", f"{category}"),
|
||||
|
|
@ -110,6 +106,11 @@ def updated_robots_json(soup):
|
|||
),
|
||||
),
|
||||
}
|
||||
if "has_name_and_version" in existing_content.get(name, {}):
|
||||
robot["has_name_and_version"] = existing_content[name][
|
||||
"has_name_and_version"
|
||||
]
|
||||
existing_content[name] = robot
|
||||
|
||||
print(f"Total: {len(existing_content)}")
|
||||
sorted_keys = sorted(existing_content, key=lambda k: k.lower())
|
||||
|
|
@ -171,41 +172,44 @@ def json_to_table(robots_json):
|
|||
return table
|
||||
|
||||
|
||||
def list_to_pcre(lst):
|
||||
def list_to_pcre(robots_json):
|
||||
# Python re is not 100% identical to PCRE which is used by Apache, but it
|
||||
# should probably be close enough in the real world for re.escape to work.
|
||||
formatted = "|".join(
|
||||
f"^{re.escape(agent)}$" if agent in exact_match_agents else re.escape(agent)
|
||||
for agent in lst
|
||||
exact_agents = "|".join(map(re.escape, robots_json))
|
||||
patterns = [f"^({exact_agents})$"]
|
||||
patterns.extend(
|
||||
f"{re.escape(agent)}/[0-9.]+"
|
||||
for agent, config in robots_json.items()
|
||||
if config.get("has_name_and_version", False)
|
||||
)
|
||||
return f"({formatted})"
|
||||
return f"({'|'.join(patterns)})"
|
||||
|
||||
|
||||
def json_to_htaccess(robot_json):
|
||||
# Creates a .htaccess filter file. It uses a regular expression to filter out
|
||||
# User agents that contain any of the blocked values.
|
||||
htaccess = "RewriteEngine On\n"
|
||||
htaccess += f"RewriteCond %{{HTTP_USER_AGENT}} {list_to_pcre(robot_json.keys())} [NC]\n"
|
||||
htaccess += f"RewriteCond %{{HTTP_USER_AGENT}} {list_to_pcre(robot_json)} [NC]\n"
|
||||
htaccess += "RewriteRule !^/?robots\\.txt$ - [F]\n"
|
||||
return htaccess
|
||||
|
||||
def json_to_nginx(robot_json):
|
||||
# Creates an Nginx config file. This config snippet can be included in
|
||||
# nginx server{} blocks to block AI bots.
|
||||
config = f"set $block 0;\n\nif ($http_user_agent ~* \"{list_to_pcre(robot_json.keys())}\") {{\n set $block 1;\n}}\n\nif ($request_uri = \"/robots.txt\") {{\n set $block 0;\n}}\n\nif ($block) {{\n return 403;\n}}"
|
||||
config = f"set $block 0;\n\nif ($http_user_agent ~* \"{list_to_pcre(robot_json)}\") {{\n set $block 1;\n}}\n\nif ($request_uri = \"/robots.txt\") {{\n set $block 0;\n}}\n\nif ($block) {{\n return 403;\n}}"
|
||||
return config
|
||||
|
||||
|
||||
def json_to_lighttpd(robot_json):
|
||||
# Creates an Lighttpd config file. This config snippet can be included in
|
||||
# Lighttpd configuration global or in $HTTP conditionals to block AI bots.
|
||||
config = f"$HTTP[\"url\"] != \"/robots.txt\" {{ $HTTP[\"user-agent\"] =~ \"{list_to_pcre(robot_json.keys())}\" {{ url.access-deny = ( \"\" ) }} }}"
|
||||
config = f"$HTTP[\"url\"] != \"/robots.txt\" {{ $HTTP[\"user-agent\"] =~ \"{list_to_pcre(robot_json)}\" {{ url.access-deny = ( \"\" ) }} }}"
|
||||
return config
|
||||
|
||||
|
||||
def json_to_caddy(robot_json):
|
||||
caddyfile = "@aibots {\n "
|
||||
caddyfile += f' header_regexp User-Agent "{list_to_pcre(robot_json.keys())}"'
|
||||
caddyfile += f' header_regexp User-Agent "{list_to_pcre(robot_json)}"'
|
||||
caddyfile += "\n}"
|
||||
return caddyfile
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
RewriteEngine On
|
||||
RewriteCond %{HTTP_USER_AGENT} (AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash) [NC]
|
||||
RewriteCond %{HTTP_USER_AGENT} (^(AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash)$) [NC]
|
||||
RewriteRule !^/?robots\.txt$ - [F]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
@aibots {
|
||||
header_regexp User-Agent "(AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash)"
|
||||
header_regexp User-Agent "(^(AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash)$)"
|
||||
}
|
||||
|
|
@ -1 +1 @@
|
|||
$HTTP["url"] != "/robots.txt" { $HTTP["user-agent"] =~ "(AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash)" { url.access-deny = ( "" ) } }
|
||||
$HTTP["url"] != "/robots.txt" { $HTTP["user-agent"] =~ "(^(AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash)$)" { url.access-deny = ( "" ) } }
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
set $block 0;
|
||||
|
||||
if ($http_user_agent ~* "(AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash)") {
|
||||
if ($http_user_agent ~* "(^(AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash)$)") {
|
||||
set $block 1;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -65,14 +65,25 @@ class TestHtaccessGeneration(unittest.TestCase, RobotsUnittestExtensions):
|
|||
|
||||
|
||||
class TestUserAgentPatternGeneration(unittest.TestCase):
|
||||
def test_spider_matches_only_the_complete_user_agent(self):
|
||||
pattern = re.compile(list_to_pcre(["Spider", "ExampleBot"]), re.IGNORECASE)
|
||||
def test_agents_match_only_the_complete_user_agent_by_default(self):
|
||||
pattern = re.compile(
|
||||
list_to_pcre({"Spider": {}, "ExampleBot": {}}), re.IGNORECASE
|
||||
)
|
||||
|
||||
self.assertIsNotNone(pattern.search("Spider"))
|
||||
self.assertIsNotNone(pattern.search("spider"))
|
||||
self.assertIsNone(pattern.search("Baiduspider"))
|
||||
self.assertIsNone(pattern.search("OurCompanyName Test Spider"))
|
||||
self.assertIsNotNone(pattern.search("Mozilla/5.0 ExampleBot/1.0"))
|
||||
self.assertIsNone(pattern.search("Mozilla/5.0 ExampleBot/1.0"))
|
||||
|
||||
def test_name_and_version_agents_match_versioned_tokens(self):
|
||||
pattern = re.compile(
|
||||
list_to_pcre({"Code": {"has_name_and_version": True}}), re.IGNORECASE
|
||||
)
|
||||
|
||||
self.assertIsNotNone(pattern.search("Code"))
|
||||
self.assertIsNotNone(pattern.search("Mozilla/5.0 Code/1.2.3"))
|
||||
self.assertIsNone(pattern.search("https://codeberg.org/example"))
|
||||
|
||||
class TestNginxConfigGeneration(unittest.TestCase, RobotsUnittestExtensions):
|
||||
maxDiff = 8192
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue