github/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-12-29 12:18:33 +01:00
Code Issues Projects Releases Packages Wiki Activity

Update from Dark Visitors

Browse source
This commit is contained in:
dark-visitors 2025-09-10 00:53:56 +00:00
commit dd391bf960
1 changed files with 8 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

16
robots.json
View file

@ -391,13 +391,6 @@
"frequency": "Unclear at this time.",
"description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher"
},
"meta-webindexer": {
"operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)",
"respect": "Unclear at this time.",
"function": "AI Assistants",
"frequency": "Unhinged, more than 1 per second.",
"description": "As per their documentation, \"The Meta-WebIndexer crawler navigates the web to improve Meta AI search result quality for users. In doing so, Meta analyzes online content to enhance the relevance and accuracy of Meta AI. Allowing Meta-WebIndexer in your robots.txt file helps us cite and link to your content in Meta AI's responses.\""
},
"Meta-ExternalFetcher": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
@ -405,6 +398,13 @@
"frequency": "Unclear at this time.",
"description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher"
},
"meta-webindexer": {
"operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)",
"respect": "Unclear at this time.",
"function": "AI Assistants",
"frequency": "Unhinged, more than 1 per second.",
"description": "As per their documentation, \"The Meta-WebIndexer crawler navigates the web to improve Meta AI search result quality for users. In doing so, Meta analyzes online content to enhance the relevance and accuracy of Meta AI. Allowing Meta-WebIndexer in your robots.txt file helps us cite and link to your content in Meta AI's responses.\""
},
"MistralAI-User": {
"operator": "Mistral",
"respect": "Unclear at this time.",
@ -671,4 +671,4 @@
"frequency": "No information.",
"description": "Retrieves data used for You.com web search engine and LLMs."
}
}
}