ai.robots.txt/.github/workflows/ai_robots_update.yml

name: Updates for AI robots files
on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:

jobs:
  known-agents:
    runs-on: ubuntu-latest
    name: known-agents
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - run: |
          pip install beautifulsoup4 requests
          git config --global user.name "Known Agents"
          git config --global user.email "knownagents@users.noreply.github.com"
          echo "Updating robots.json with data from knownagents.com ..."
          python code/robots.py --update
          echo "Updating generated files from robots.json ..."
          python code/robots.py --convert
          echo "... done."
          git --no-pager diff
          git add -A
          if ! git diff --cached --quiet; then
            git commit -m "Update from knownagents.com"
            git rebase origin/main
            git push
          else
            echo "No changes to commit."
          fi
        shell: bash