github/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-12-29 12:18:33 +01:00
Code Issues Projects Releases Packages Wiki Activity

Update from Dark Visitors

Browse source
This commit is contained in:
dark-visitors 2025-12-04 01:00:27 +00:00
commit 45b071b29f
1 changed files with 7 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

7
robots.json
View file

@ -237,6 +237,13 @@
"frequency": "No information provided.", "frequency": "No information provided.",
"description": "Scrapes data for AI training in Japanese language." "description": "Scrapes data for AI training in Japanese language."
}, },
"Crawl4AI": {
"operator": "Unclear at this time.",
"respect": "Unclear at this time.",
"function": "Undocumented AI Agents",
"frequency": "Unclear at this time.",
"description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/crawl4ai"
},
"Crawlspace": { "Crawlspace": {
"operator": "[Crawlspace](https://crawlspace.dev)", "operator": "[Crawlspace](https://crawlspace.dev)",
"respect": "[Yes](https://news.ycombinator.com/item?id=42756654)", "respect": "[Yes](https://news.ycombinator.com/item?id=42756654)",