github/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-12-29 12:18:33 +01:00
Code Issues Projects Releases Packages Wiki Activity

Update from Dark Visitors

Browse source
This commit is contained in:
dark-visitors 2025-12-05 01:00:44 +00:00
commit f1c752ef12
1 changed files with 14 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

14
robots.json
View file

@ -153,6 +153,13 @@
"frequency": "Monthly at present.", "frequency": "Monthly at present.",
"description": "Web archive going back to 2008. [Cited in thousands of research papers per year](https://commoncrawl.org/research-papers)." "description": "Web archive going back to 2008. [Cited in thousands of research papers per year](https://commoncrawl.org/research-papers)."
}, },
"Channel3Bot": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Search Crawlers",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/channel3bot"
},
"ChatGLM-Spider": { "ChatGLM-Spider": {
"operator": "Unclear at this time.", "operator": "Unclear at this time.",
"respect": "Unclear at this time.", "respect": "Unclear at this time.",
@ -811,6 +818,13 @@
"frequency": "No information.", "frequency": "No information.",
"description": "Makes data available for training AI models." "description": "Makes data available for training AI models."
}, },
"TwinAgent": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "AI Agents",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/twinagent"
},
"VelenPublicWebCrawler": { "VelenPublicWebCrawler": {
"operator": "[Velen Crawler](https://velen.io)", "operator": "[Velen Crawler](https://velen.io)",
"respect": "[Yes](https://velen.io)", "respect": "[Yes](https://velen.io)",