remote/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2026-06-04 08:06:55 +02:00
Code Issues Projects Releases Packages Wiki Activity

Update from Dark Visitors

Browse source
This commit is contained in:
dark-visitors 2026-03-06 01:21:01 +00:00
commit 3de8865f00
1 changed files with 21 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

21
robots.json
View file

@ -90,6 +90,20 @@
"frequency": "No information provided.",
"description": "Scrapes data to train LLMs and AI products offered by Anthropic."
},
"ApifyBot": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "Undocumented AI Agents",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/apifybot"
},
"ApifyWebsiteContentCrawler": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Data Scrapers",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/apifywebsitecontentcrawler"
},
"Applebot": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
@ -328,6 +342,13 @@
"frequency": "Unclear at this time.",
"description": "Supports company's AI-powered social and email management products."
},
"ExaBot": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Search Crawlers",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/exabot"
},
"FacebookBot": {
"operator": "Meta/Facebook",
"respect": "[Yes](https://developers.facebook.com/docs/sharing/bot/)",