remote/ai.robots.txt
1
0
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2026-09-28 04:44:29 +02:00
Code Issues Projects Releases Packages Wiki Activity

robots.py: div class=description

Browse source
This commit is contained in:
guest20 2026-06-17 13:20:29 +02:00 • committed by GitHub
commit 25415e342c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 3 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

10
code/robots.py
View file

@ -56,13 +56,9 @@ def updated_robots_json(soup):
name = agent.find("div", {"class": "agent-name"}).get_text().strip()
name = clean_robot_name(name)
# This line below occasionally throws this error: AttributeError: 'NoneType' object has no attribute 'get_text'
#desc = agent.find("p").get_text().strip()
# Attempting a different way to handle to avoid errors:
p_tag = agent.find("p")
if p_tag is not None:
desc = p_tag.get_text().strip()
desc_tag = agent.find("div", {"class": "description"})
if desc_tag is not None:
desc = desc_tag.get_text().strip()
else:
desc = "Description unavailable from darkvisitors.com"