github/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-12-29 12:18:33 +01:00
Code Issues Projects Releases Packages Wiki Activity

Update from Dark Visitors

Browse source
This commit is contained in:
dark-visitors 2025-10-24 00:53:30 +00:00
commit 97e19445ce
1 changed files with 12 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

26
robots.json
View file

@ -27,13 +27,6 @@
"frequency": "No information provided.",
"description": "Scrapes data for AI systems."
},
"Amazonbot": {
"operator": "Amazon",
"respect": "Yes",
"function": "Service improvement and enabling answers for Alexa users.",
"frequency": "No information provided.",
"description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses."
},
"amazon-kendra-": {
"operator": "Amazon",
"respect": "Yes",
@ -41,6 +34,13 @@
"frequency": "No information provided.",
"description": "Amazon Kendra is a highly accurate intelligent search service that enables your users to search unstructured data using natural language. It returns specific answers to questions, giving users an experience that's close to interacting with a human expert. It is highly scalable and capable of meeting performance demands, tightly integrated with other AWS services such as Amazon S3 and Amazon Lex, and offers enterprise-grade security."
},
"Amazonbot": {
"operator": "Amazon",
"respect": "Yes",
"function": "Service improvement and enabling answers for Alexa users.",
"frequency": "No information provided.",
"description": "Includes references to crawled website when surfacing answers via Alexa; does not clearly outline other uses."
},
"Andibot": {
"operator": "[Andi](https://andisearch.com/)",
"respect": "Unclear at this time",
@ -371,11 +371,11 @@
"respect": "No"
},
"IbouBot": {
"operator": "Ibou",
"respect": "Yes",
"function": "Search result generation.",
"frequency": "Unclear at this time.",
"description": "Ibou.io operates a crawler service named IbouBot which fuels and updates their graph representation of the World Wide Web. This database and all the metrics are used to provide a search engine."
"operator": "Ibou",
"respect": "Yes",
"function": "Search result generation.",
"frequency": "Unclear at this time.",
"description": "Ibou.io operates a crawler service named IbouBot which fuels and updates their graph representation of the World Wide Web. This database and all the metrics are used to provide a search engine."
},
"ICC-Crawler": {
"operator": "[NICT](https://nict.go.jp)",
@ -727,4 +727,4 @@
"frequency": "No information.",
"description": "Retrieves data used for You.com web search engine and LLMs."
}
}
}