diff --git a/code/robots.py b/code/robots.py index 715bb44..d72a2de 100755 --- a/code/robots.py +++ b/code/robots.py @@ -10,7 +10,12 @@ from pathlib import Path def load_robots_json(): """Load the robots.json contents into a dictionary.""" - return json.loads(Path("./robots.json").read_text(encoding="utf-8")) + bots = json.loads(Path("./robots.json").read_text(encoding="utf-8")) + for ro in bots: + if "description" in ro: + # urls in the json have "agents" twice, so let's tidy that up here... + ro["description"] = ro["description"].replace('https://darkvisitors.com/agents/agents/', 'https://darkvisitors.com/agents/') + return bots def get_agent_soup():