Webmaster Broken Link Checker Agent

import requests, json, time

LINKS = []  # fetched from sitemap

def check_link(url):
    try:
        resp = requests.head(url, timeout=5, allow_redirects=True)
        return {"url": url, "status": resp.status_code, "final_url": resp.url}
    except:
        return {"url": url, "status": "error", "final_url": None}

def categorize_links(results):
    ok = [r for r in results if r["status"] == 200]
    broken = [r for r in results if r["status"] == 404]
    redirect = [r for r in results if r["status"] in (301, 302)]
    loop = [r for r in results if r["status"] == 302 and r.get("loop")]
    return {"ok": ok, "broken": broken, "redirect": redirect, "loop": loop}

results = [check_link(u) for u in LINKS]
print(json.dumps(categorize_links(results), indent=2))