remote/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2026-10-07 18:07:00 +02:00
Code Issues Projects Releases Packages Wiki Activity

Update from Dark Visitors

Browse source
This commit is contained in:
dark-visitors 2026-05-13 02:24:20 +00:00
commit aea7db9e34
1 changed files with 56 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

56
robots.json
View file

@ -167,6 +167,13 @@
"frequency": "Unclear at this time.",
"description": "Brave search has a crawler to discover new pages and index their content."
},
"Brightbot": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Data Providers",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/brightbot"
},
"Brightbot 1.0": {
"operator": "https://brightdata.com/brightbot",
"respect": "Unclear at this time.",
@ -223,6 +230,13 @@
"frequency": "Only when prompted by a user.",
"description": "Used by plugins in ChatGPT to answer queries based on user input."
},
"Claude-Code": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Coding Agents",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/claude-code"
},
"Claude-SearchBot": {
"operator": "[Anthropic](https://www.anthropic.com)",
"respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)",
@ -265,6 +279,13 @@
"frequency": "Unclear at this time.",
"description": "CloudVertexBot is a Google-operated crawler available to site owners to request targeted crawls of their own sites for AI training purposes on the Vertex AI platform. More info can be found at https://darkvisitors.com/agents/agents/cloudvertexbot"
},
"Code": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Coding Agents",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/code"
},
"cohere-ai": {
"operator": "[Cohere](https://cohere.com)",
"respect": "Unclear at this time.",
@ -426,6 +447,13 @@
"frequency": "Unclear at this time.",
"description": "Supports Google's Firebase AI products."
},
"Google-Gemini-CLI": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Coding Agents",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/google-gemini-cli"
},
"Google-NotebookLM": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
@ -468,6 +496,13 @@
"frequency": "No information.",
"description": "Data is used to train current and future models, removed paywalled data, PII and data that violates the company's policies."
},
"HenkBot": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Data Providers",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/henkbot"
},
"iAskBot": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
@ -727,6 +762,13 @@
"frequency": "Unclear at this time.",
"description": "The purpose of this bot is unclear at this time but it is a member of OpenAI's suite of crawlers."
},
"opencode": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Coding Agents",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/opencode"
},
"Operator": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
@ -874,6 +916,13 @@
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/tavilybot"
},
"Terra Cotta": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Data Providers",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/terra-cotta"
},
"TerraCotta": {
"operator": "[Ceramic AI](https://ceramic.ai/)",
"respect": "[Yes](https://github.com/CeramicTeam/CeramicTerracotta)",
@ -902,6 +951,13 @@
"frequency": "No information.",
"description": "Makes data available for training AI models."
},
"Trae": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Coding Agents",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/trae"
},
"TwinAgent": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",