Auto Updates

async def has_sitemap_changed(
    sitemap_url: str, 
    last_check_time: datetime | None
) -> tuple[bool, datetime | None]:
    sitemap_info = await parse_sitemap(sitemap_url)

    if not sitemap_info:
        return (True, None)  # Can't parse - assume changed

    if not sitemap_info.newest_lastmod:
        return (True, None)  # No timestamp - assume changed

    if not last_check_time:
        return (True, sitemap_info.newest_lastmod)  # First check

    # Compare timestamps
    newest = sitemap_info.newest_lastmod
    last_check = last_check_time

    if newest.tzinfo:
        newest = newest.replace(tzinfo=None)
    if last_check.tzinfo:
        last_check = last_check.replace(tzinfo=None)

    has_changed = newest > last_check

    return (has_changed, sitemap_info.newest_lastmod)

@app.post("/internal/cron/recrawl")
async def trigger_recrawl(
    background_tasks: BackgroundTasks, 
    x_cron_secret: str = Header(None)
):
    if not settings.cron_secret or x_cron_secret != settings.cron_secret:
        raise HTTPException(status_code=401, detail="Unauthorized")

    background_tasks.add_task(run_recrawl_in_background)
    return {"status": "triggered", "message": "Recrawl started in background"}

async def run_recrawl_in_background():
    try:
        print("[RECRAWL] Starting background recrawl...")
        results = await recrawl_due_sites()
        print(f"[RECRAWL] Completed: {results}")
    except Exception as e:
        print(f"[RECRAWL] Error: {e}")

@app.post("/internal/hooks/site-changed")
async def trigger_site_recrawl(
    request: Request,
    base_url: str = Body(...),
    webhook_secret: str = Body(None)
):
    try:
        client = get_supabase_client()
        if not client:
            raise HTTPException(status_code=503, detail="Database unavailable")

        result = client.table("crawl_sites") \
            .select("*") \
            .eq("base_url", base_url) \
            .execute()

        if not result.data:
            raise HTTPException(status_code=404, detail="Site not enrolled")

        site = result.data[0]

        if site.get("webhook_secret") and webhook_secret != site.get("webhook_secret"):
            raise HTTPException(status_code=401, detail="Invalid webhook secret")

        # Schedule immediate recrawl
        now = datetime.now(timezone.utc)
        client.table("crawl_sites").update({
            "next_crawl_at": now.isoformat(),
            "updated_at": now.isoformat()
        }).eq("base_url", base_url).execute()

        return {
            "status": "scheduled", 
            "base_url": base_url, 
            "next_crawl_at": now.isoformat()
        }

    except HTTPException:
        raise
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

MIN_INTERVAL_MINUTES = 60        # 1 hour minimum
MAX_INTERVAL_MINUTES = 10080     # 7 days maximum
EWMA_ALPHA = 0.5                 # Moderate adaptation rate

def compute_next_crawl(
    site_recrawl_interval: int,
    avg_change_interval: float | None,
    content_changed: bool,
    last_changed_at: datetime | None,
    now: datetime,
    adaptive_enabled: bool = False
) -> tuple[datetime, float | None, datetime | None]:
    if not adaptive_enabled:
        interval_minutes = site_recrawl_interval
        return (
            now + timedelta(minutes=interval_minutes),
            avg_change_interval,
            last_changed_at if not content_changed else now
        )

    # Track average change interval using EWMA
    if avg_change_interval is None:
        avg_change_interval = float(site_recrawl_interval)

    if content_changed and last_changed_at:
        minutes_since_change = (now - last_changed_at).total_seconds() / 60
        avg_change_interval = (
            EWMA_ALPHA * minutes_since_change +
            (1 - EWMA_ALPHA) * avg_change_interval
        )

    # Use learned interval, clamped to min/max
    effective_interval = max(
        MIN_INTERVAL_MINUTES,
        min(MAX_INTERVAL_MINUTES, avg_change_interval)
    )

    return (
        now + timedelta(minutes=effective_interval),
        avg_change_interval,
        now if content_changed else last_changed_at
    )