From dd391bf9601cb57c3129c45d1ea7883bd983e146 Mon Sep 17 00:00:00 2001 From: dark-visitors Date: Wed, 10 Sep 2025 00:53:56 +0000 Subject: [PATCH] Update from Dark Visitors --- robots.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/robots.json b/robots.json index 5faef5a..6267d89 100644 --- a/robots.json +++ b/robots.json @@ -391,13 +391,6 @@ "frequency": "Unclear at this time.", "description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher" }, - "meta-webindexer": { - "operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)", - "respect": "Unclear at this time.", - "function": "AI Assistants", - "frequency": "Unhinged, more than 1 per second.", - "description": "As per their documentation, \"The Meta-WebIndexer crawler navigates the web to improve Meta AI search result quality for users. In doing so, Meta analyzes online content to enhance the relevance and accuracy of Meta AI. Allowing Meta-WebIndexer in your robots.txt file helps us cite and link to your content in Meta AI's responses.\"" - }, "Meta-ExternalFetcher": { "operator": "Unclear at this time.", "respect": "Unclear at this time.", @@ -405,6 +398,13 @@ "frequency": "Unclear at this time.", "description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher" }, + "meta-webindexer": { + "operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)", + "respect": "Unclear at this time.", + "function": "AI Assistants", + "frequency": "Unhinged, more than 1 per second.", + "description": "As per their documentation, \"The Meta-WebIndexer crawler navigates the web to improve Meta AI search result quality for users. In doing so, Meta analyzes online content to enhance the relevance and accuracy of Meta AI. Allowing Meta-WebIndexer in your robots.txt file helps us cite and link to your content in Meta AI's responses.\"" + }, "MistralAI-User": { "operator": "Mistral", "respect": "Unclear at this time.", @@ -671,4 +671,4 @@ "frequency": "No information.", "description": "Retrieves data used for You.com web search engine and LLMs." } -} +} \ No newline at end of file