AI crawler traffic to a warehouse

import datetime as dt, os, requests

BASE = "https://api.rankraft.ai"
HEADERS = {"X-API-Key": os.environ["RANKRAFT_API_KEY"]}
SITE = os.environ["RANKRAFT_SITE_ID"]
TZ = "Asia/Karachi"  # your reporting time zone

day = (dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)).date().isoformat()

def fetch(path, query):
    rows, cursor = [], None
    while True:
        body = {**query, **({"cursor": cursor} if cursor else {})}
        res = requests.post(f"{BASE}{path}", json=body, headers=HEADERS, timeout=60)
        if res.status_code == 429:
            import time; time.sleep(int(res.headers.get("Retry-After", "60"))); continue
        res.raise_for_status()
        page = res.json()
        rows += page["data"]
        cursor = page["info"]["next_cursor"]
        if not cursor:
            return rows

common = {"site_id": SITE, "start_date": day, "end_date": day, "timezone": TZ, "limit": 500}

bots = fetch("/v1/reports/bots", {**common,
    "group_by": ["date", "bot_name", "page_type"],
    "metrics": ["hits", "pages", "verified_share", "error_hits"]})

referrals = fetch("/v1/reports/referrals", {**common,
    "group_by": ["date", "referral_source", "path"],
    "metrics": ["visits"]})

# load(bots, "rankraft_bot_daily"); load(referrals, "rankraft_referral_daily")