github/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-12-29 12:18:33 +01:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #168 from ai-robots-txt/google-firebase-shapbot

Browse source 
chore: add Google-Firebase and ShapBot
This commit is contained in:
Glyn Normington 2025-08-27 11:13:04 +01:00 • committed by GitHub
commit 1677278c5a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 16 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

18
robots.json
View file

@ -279,8 +279,15 @@
"frequency": "No information.",
"description": "Used to train Gemini and Vertex AI generative APIs. Does not impact a site's inclusion or ranking in Google Search."
},
"Google-Firebase": {
"operator": "Google",
"respect": "Unclear at this time.",
"function": "Used as part of AI apps developed by users of Google's Firebase AI products.",
"frequency": "Unclear at this time.",
"description": "Supports Google's Firebase AI products."
},
"GoogleAgent-Mariner": {
"operator": "Unclear at this time.",
"operator": "Google",
"respect": "Unclear at this time.",
"function": "AI Agents",
"frequency": "Unclear at this time.",
@ -566,6 +573,13 @@
"frequency": "Roughly once every 10 seconds.",
"description": "Data collected is used for the SEO Writing Assistant tool to check if URL is accessible."
},
"ShapBot": {
"operator": "[Parallel](https://parallel.ai)",
"respect": "[Yes](https://docs.parallel.ai/features/crawler)",
"function": "Collects data for Parallel's web APIs.",
"frequency": "Unclear at this time.",
"description": "ShapBot helps discover and index websites for Parallel's web APIs."
},
"Sidetrade indexer bot": {
"description": "AI product training.",
"frequency": "No information.",
@ -650,4 +664,4 @@
"frequency": "No information.",
"description": "Retrieves data used for You.com web search engine and LLMs."
}
}
}