pigway-frame-flow/tests/test_upload_retries.py

from datetime import UTC, datetime, timedelta

from pigway_frame_flow.errors import ErrorCode, FlowError
from pigway_frame_flow.models import JobInput
from pigway_frame_flow.scheduler.engine import Scheduler


def failed_job(app, service, monkeypatch):
    scheduler = app.state.scheduler
    job = scheduler.save_job(
        JobInput(
            name="Retry",
            schedule={"type": "interval", "interval_minutes": 120},
            source={"type": "template", "template": "quote"},
        )
    )
    now = datetime.now(UTC)
    service.db.execute(
        "UPDATE jobs SET next_run=? WHERE id=?",
        ((now - timedelta(seconds=1)).isoformat(), job["id"]),
    )
    driver = service.registry.get("living-room")
    upload = driver.upload

    def fail(*a):
        raise FlowError(ErrorCode.UPLOAD_FAILED, "Rejected", 502)

    monkeypatch.setattr(driver, "upload", fail)
    scheduler.tick(now)
    service.process_one()
    scheduler.tick(now + timedelta(seconds=1))
    return scheduler, job, now, driver, upload


def test_five_backoffs_then_one_scheduled_failure_regenerates(
    app, service, monkeypatch, client
):
    scheduler, job, now, driver, upload = failed_job(app, service, monkeypatch)
    policy = scheduler.upload_retries
    plan = policy.load(job["id"])
    asset = plan["asset_id"]
    path = service.assets.path(asset)
    original = path.read_bytes()
    assert client.get("/api/v1/assets").json() == []
    assert (
        service.db.one("SELECT kind FROM assets WHERE id=?", (asset,))["kind"]
        == "cache"
    )
    times = [datetime.fromisoformat(x) for x in plan["times"]]
    gaps = [(times[0] - now).total_seconds()] + [
        (b - a).total_seconds() for a, b in zip(times, times[1:], strict=False)
    ]
    assert gaps == sorted(gaps) and len(times) == 5
    assert times[-1] < datetime.fromisoformat(plan["deadline"])
    for number, when in enumerate(times, 1):
        scheduler.tick(when + timedelta(milliseconds=1))
        attempt = policy.load(job["id"])
        assert attempt["count"] == number
        op = service.operation(attempt["active"])
        assert (
            service.db.one(
                "SELECT payload FROM operations WHERE id=?", (op["operation_id"],)
            )["payload"].find(asset)
            >= 0
        )
        service.process_one()
        assert path.read_bytes() == original
    assert len(service.db.all("SELECT * FROM job_runs")) == 1
    # The plan survives a process restart, without resetting the five-attempt cap.
    scheduler = Scheduler(service)
    deadline = datetime.fromisoformat(plan["deadline"])
    scheduler.tick(deadline + timedelta(milliseconds=1))
    planned = scheduler.upload_retries.load(job["id"])
    assert planned["phase"] == "scheduled_running"
    service.process_one()
    scheduler.tick(deadline + timedelta(seconds=1))
    replacement = scheduler.upload_retries.load(job["id"])
    assert replacement["phase"] == "replacement"
    assert not path.exists()
    assert service.db.one("SELECT id FROM assets WHERE id=?", (asset,)) is None
    monkeypatch.setattr(driver, "upload", upload)
    service.process_one()
    service.sync()
    op = service.operation(replacement["active"])
    assert op["status"] == "COMPLETED"
    assert op["result"]["rendered_asset_id"] != asset
    assert len(service.db.all("SELECT * FROM job_runs")) == 2


def test_retry_success_cancels_rest_and_keeps_schedule(app, service, monkeypatch):
    scheduler, job, now, driver, upload = failed_job(app, service, monkeypatch)
    policy = scheduler.upload_retries
    plan = policy.load(job["id"])
    deadline = service.db.one("SELECT next_run FROM jobs WHERE id=?", (job["id"],))[
        "next_run"
    ]
    monkeypatch.setattr(driver, "upload", upload)
    when = datetime.fromisoformat(plan["times"][0]) + timedelta(milliseconds=1)
    scheduler.tick(when)
    operation = policy.load(job["id"])["active"]
    service.process_one()
    service.sync()
    scheduler.tick(when + timedelta(seconds=1))
    assert policy.load(job["id"]) is None
    assert service.operation(operation)["status"] == "COMPLETED"
    assert (
        service.db.one("SELECT next_run FROM jobs WHERE id=?", (job["id"],))["next_run"]
        == deadline
    )
    assert (
        service.db.one("SELECT kind FROM assets WHERE id=?", (plan["asset_id"],))[
            "kind"
        ]
        == "generated"
    )
    assert driver.client.uploads == 1


def test_expired_queued_retry_never_uploads(app, service, monkeypatch):
    scheduler, job, now, driver, upload = failed_job(app, service, monkeypatch)
    policy = scheduler.upload_retries
    plan = policy.load(job["id"])
    scheduler.tick(datetime.fromisoformat(plan["times"][0]) + timedelta(milliseconds=1))
    plan = policy.load(job["id"])
    plan["deadline"] = (datetime.now(UTC) - timedelta(seconds=1)).isoformat()
    policy.save(job["id"], plan)

    def unexpected(*a):
        raise AssertionError("Expired retry must never upload")

    monkeypatch.setattr(driver, "upload", unexpected)
    service.process_one()
    operation = service.operation(plan["active"])
    assert operation["status"] == "FAILED"
    assert operation["error"]["details"]["reason"] == "RETRY_WINDOW_CLOSED"
    assert service.assets.path(plan["asset_id"]).exists()


def test_lost_response_is_reconciled_before_retry_without_duplicate(
    app, service, monkeypatch
):
    scheduler, job, now, driver, upload = failed_job(app, service, monkeypatch)
    policy = scheduler.upload_retries
    plan = policy.load(job["id"])
    asset = service.assets.info(plan["asset_id"])
    monkeypatch.setattr(driver, "upload", upload)
    # The original request reached the cloud, but its response was lost.
    driver.upload(service.assets.path(plan["asset_id"]), asset["name"])
    service.update(plan["root"], "UNKNOWN")
    scheduler.tick(datetime.fromisoformat(plan["times"][0]) + timedelta(milliseconds=1))
    scheduler.tick(datetime.fromisoformat(plan["times"][0]) + timedelta(seconds=1))
    assert service.operation(plan["root"])["status"] == "COMPLETED"
    assert policy.load(job["id"]) is None
    assert driver.client.uploads == 1