From 45b071b29f38a69400804f6af1c5147f5cfca52a Mon Sep 17 00:00:00 2001 From: dark-visitors Date: Thu, 4 Dec 2025 01:00:27 +0000 Subject: [PATCH] Update from Dark Visitors --- robots.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/robots.json b/robots.json index ce32e85..46ac417 100644 --- a/robots.json +++ b/robots.json @@ -237,6 +237,13 @@ "frequency": "No information provided.", "description": "Scrapes data for AI training in Japanese language." }, + "Crawl4AI": { + "operator": "Unclear at this time.", + "respect": "Unclear at this time.", + "function": "Undocumented AI Agents", + "frequency": "Unclear at this time.", + "description": "Description unavailable from darkvisitors.com More info can be found at https://darkvisitors.com/agents/agents/crawl4ai" + }, "Crawlspace": { "operator": "[Crawlspace](https://crawlspace.dev)", "respect": "[Yes](https://news.ycombinator.com/item?id=42756654)",