remote/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2026-10-07 18:07:00 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #271 from lyrenth/correct-aiwebindex-metadata

Browse source 
Correct AIWebIndex metadata
This commit is contained in:
Glyn Normington 2026-08-24 15:37:53 +01:00 • committed by GitHub
commit d8e26238a7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 5 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

10
robots.json
View file

@ -42,11 +42,11 @@
"description": "Scrapes data for AI systems."
},
"AIWebIndex": {
"operator": "Lyrenth that builds an AI-readable index of web content for AI systems",
"respect": "Unclear at this time.",
"function": "AI Data Providers",
"frequency": "Unclear at this time.",
"description": "AIWebIndex is a web crawler operated by Lyrenth that builds an AI-readable index of web content for AI systems. More info can be found at https://knownagents.com/agents/aiwebindex"
"operator": "[Lyrenth](https://lyrenth.com)",
"respect": "[Yes](https://lyrenth.com/crawler-policy)",
"function": "AI Search Crawlers",
"frequency": "At most one request per domain every 2 seconds, and slower where robots.txt sets a longer Crawl-delay.",
"description": "Builds an index of public pages and serves them to AI agents as extracted, readable text with attribution and a link back to the source. Does not train foundation models on crawled content. Identity can be checked three ways: published IP ranges at https://lyrenth.com/bot/ip-ranges.json, forward-confirmed reverse DNS under lyrenth.com, and Web Bot Auth signatures (RFC 9421). Full policy at https://lyrenth.com/crawler-policy"
},
"amazon-kendra": {
"operator": "Amazon",