mirror of
https://github.com/ai-robots-txt/ai.robots.txt.git
synced 2026-11-21 09:29:38 +01:00
More name changes
This commit is contained in:
parent
0c5de5b90b
commit
c43c5afd07
2 changed files with 5 additions and 5 deletions
4
.github/workflows/ai_robots_update.yml
vendored
4
.github/workflows/ai_robots_update.yml
vendored
|
|
@ -14,9 +14,9 @@ jobs:
|
|||
fetch-depth: 0
|
||||
- run: |
|
||||
pip install beautifulsoup4 requests
|
||||
git config --global user.name "knownagents"
|
||||
git config --global user.name "Known Agents"
|
||||
git config --global user.email "knownagents@users.noreply.github.com"
|
||||
echo "Updating robots.json with data from knownagents.com ..."
|
||||
echo "Updating robots.json with data from Known Agents ..."
|
||||
python code/robots.py --update
|
||||
echo "Updating generated files from robots.json ..."
|
||||
python code/robots.py --convert
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ def load_robots_json():
|
|||
return json.loads(Path("./robots.json").read_text(encoding="utf-8"))
|
||||
|
||||
def get_agent_soup():
|
||||
"""Retrieve current known agents from knownagents.com"""
|
||||
"""Retrieve current agent data from Known Agents."""
|
||||
session = requests.Session()
|
||||
try:
|
||||
response = session.get("https://knownagents.com/agents")
|
||||
|
|
@ -67,7 +67,7 @@ def get_agent_soup():
|
|||
|
||||
|
||||
def updated_robots_json(soup):
|
||||
"""Update AI scraper information with data from knownagents."""
|
||||
"""Update AI scraper information with data from Known Agents."""
|
||||
existing_content = load_robots_json()
|
||||
|
||||
for section in soup.find_all("div", {"class": "agent-links-section"}):
|
||||
|
|
@ -265,7 +265,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument(
|
||||
"--update",
|
||||
action="store_true",
|
||||
help="Update the robots.json file with data from knownagents.com/agents",
|
||||
help="Update the robots.json file with data from Known Agents",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--convert",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue