feat: recover failed scheduled uploads with bounded persistent retries [skip ci]

pigway_frame_flow/scheduler/engine.py

@@ -18,6 +18,7 @@ from pigway_frame_flow.models import (
     PlaylistInput,
     RestoreContent,
 )
+from pigway_frame_flow.scheduler.upload_retries import UploadRetries
 from pigway_frame_flow.storage.database import encode, now_iso
 
 
@@ -60,6 +61,7 @@ class Scheduler:
         self.thread = None
         self.webhook_thread = None
         self.tick_lock = threading.Lock()
+        self.upload_retries = UploadRetries(service)
 
     def jobs(self):
         rows = [
@@ -147,6 +149,7 @@ class Scheduler:
     def tick(self, current=None):
         current = current or datetime.now(UTC)
         with self.tick_lock:
+            self.upload_retries.tick(current)
             for row in self.db.all(
                 "SELECT * FROM jobs WHERE enabled=1 AND next_run<=?",
                 (current.isoformat(),),
@@ -183,6 +186,10 @@ class Scheduler:
                                 ),
                             ).model_dump(mode="json")
                         )
+                        if action == "display":
+                            payload = self.upload_retries.scheduled_payload(
+                                row["id"], payload
+                            )
                         operation_id = self.service.submit(
                             action,
                             payload,
@@ -191,6 +198,10 @@ class Scheduler:
                             "schedule-" + uuid4().hex,
                             "job-" + row["id"] + "-" + slot,
                         )
+                        if action == "display":
+                            self.upload_retries.scheduled_submitted(
+                                row["id"], operation_id
+                            )
                     following = next_run(job.schedule, current)
                     enabled = bool(following)
                     updated = job.model_copy(update={"enabled": enabled}).model_dump(

pigway_frame_flow/scheduler/upload_retries.py

@@ -0,0 +1,246 @@
+"""Durable, job-local upload retries; scheduled execution remains authoritative."""
+
+import json
+from datetime import datetime, timedelta
+
+from pigway_frame_flow.errors import FlowError
+from pigway_frame_flow.storage.database import encode
+
+
+class UploadRetries:
+    def __init__(self, service):
+        self.service, self.db = service, service.db
+
+    def load(self, job_id):
+        row = self.db.one(
+            "SELECT payload FROM app_config WHERE key=?", ("upload-retry:" + job_id,)
+        )
+        return json.loads(row["payload"]) if row else None
+
+    def save(self, job_id, plan):
+        if plan is None:
+            self.db.execute(
+                "DELETE FROM app_config WHERE key=?", ("upload-retry:" + job_id,)
+            )
+        else:
+            self.db.execute(
+                "INSERT OR REPLACE INTO app_config VALUES(?,?)",
+                ("upload-retry:" + job_id, encode(plan)),
+            )
+
+    @staticmethod
+    def reuse(payload, asset_id):
+        return {**payload, "content": {"type": "image", "asset_id": asset_id}}
+
+    def scheduled_payload(self, job_id, payload):
+        plan = self.load(job_id)
+        if plan and plan["phase"] in ("extra", "scheduled", "scheduled_running"):
+            return self.reuse(payload, plan["asset_id"])
+        return payload
+
+    def scheduled_submitted(self, job_id, operation_id):
+        plan = self.load(job_id)
+        if plan and plan["phase"] in ("extra", "scheduled", "scheduled_running"):
+            plan.update(phase="scheduled_running", active=operation_id)
+            self.save(job_id, plan)
+
+    def close_unknown(self, operation):
+        if operation["status"] == "UNKNOWN":
+            raw = self.db.one(
+                "SELECT * FROM operations WHERE id=?", (operation["operation_id"],)
+            )
+            driver = self.service.registry.get(operation["device_id"])
+            self.service.upload_reconciliation.check(
+                raw, driver, driver.device(refresh=True)
+            )
+            self.service.reconcile(operation["operation_id"], acknowledge=True)
+            return self.service.operation(operation["operation_id"])
+        return operation
+
+    def discard(self, asset_id):
+        row = self.db.one("SELECT kind FROM assets WHERE id=?", (asset_id,))
+        meta = self.db.one(
+            "SELECT starred FROM asset_metadata WHERE asset_id=?", (asset_id,)
+        )
+        # Only disposable generated staging files are owned by this recovery.
+        if row and row["kind"] == "cache" and not (meta and meta["starred"]):
+            self.service.assets.path(asset_id).unlink(missing_ok=True)
+            with self.db.transaction() as tx:
+                for table, field in (
+                    ("assets", "id"),
+                    ("asset_sources", "asset_id"),
+                    ("asset_metadata", "asset_id"),
+                ):
+                    tx.execute(f"DELETE FROM {table} WHERE {field}=?", (asset_id,))
+
+    def tick(self, current):
+        for job in self.db.all("SELECT * FROM jobs WHERE enabled=1"):
+            try:
+                self.check(job, current)
+            except FlowError:
+                # A read failure leaves the durable plan in place for a later check.
+                continue
+
+    def check(self, job, current):
+        config = json.loads(job["payload"])
+        if config["source"]["type"] not in (
+            "photo_source",
+            "auto_collage",
+            "template",
+            "provider",
+        ):
+            return
+        plan = self.load(job["id"])
+        if plan and plan["source"] != config["source"]:
+            self.save(job["id"], None)
+            plan = None
+        if not plan:
+            if not job["last_result"] or not job["next_run"]:
+                return
+            operation = self.service.operation(job["last_result"])
+            asset_id = (operation["result"] or {}).get("rendered_asset_id")
+            if operation["status"] not in ("FAILED", "UNKNOWN") or not asset_id:
+                return
+            deadline = datetime.fromisoformat(job["next_run"])
+            remaining = (deadline - current).total_seconds()
+            if remaining <= 0:
+                times = []
+            else:
+                # Five increasing delays occupy the first half of the available
+                # window, leaving time for upload/sync before the next schedule.
+                unit = remaining / 62
+                times = [
+                    (current + timedelta(seconds=unit * n)).isoformat()
+                    for n in (1, 3, 7, 15, 31)
+                ]
+            plan = dict(
+                root=operation["operation_id"],
+                active=operation["operation_id"],
+                asset_id=asset_id,
+                source=config["source"],
+                times=times,
+                count=0,
+                deadline=deadline.isoformat(),
+                phase="extra",
+            )
+            self.save(job["id"], plan)
+        operation = self.service.operation(plan["active"])
+        deadline = min(
+            datetime.fromisoformat(plan["deadline"]),
+            datetime.fromisoformat(job["next_run"]),
+        )
+        if (
+            plan["phase"] == "extra"
+            and current >= deadline
+            and operation["status"] != "COMPLETED"
+        ):
+            if operation["status"] == "SYNCING":
+                self.service.update(
+                    operation["operation_id"],
+                    "UNKNOWN",
+                    operation["result"],
+                    operation["error"],
+                )
+                operation = self.service.operation(operation["operation_id"])
+            operation = self.close_unknown(operation)
+            if operation["status"] == "QUEUED":
+                self.service.update(
+                    operation["operation_id"],
+                    "FAILED",
+                    error={
+                        "code": "CONFLICT",
+                        "message": "Upload retry window closed",
+                        "request_id": operation["request_id"],
+                        "details": {"reason": "RETRY_WINDOW_CLOSED"},
+                    },
+                )
+            if operation["status"] != "COMPLETED":
+                plan["phase"] = "scheduled"
+                self.save(job["id"], plan)
+                return
+        if operation["status"] == "COMPLETED":
+            self.save(job["id"], None)
+            # Extra retries are not job_runs and do not move the next plan time.
+            self.db.execute(
+                "UPDATE jobs SET last_result=? WHERE id=?",
+                (operation["operation_id"], job["id"]),
+            )
+            return
+        if operation["status"] not in ("FAILED", "UNKNOWN"):
+            return
+        if plan["phase"] == "replacement":
+            # A failed replacement gets its own bounded recovery window.
+            self.db.execute(
+                "UPDATE jobs SET last_result=? WHERE id=?",
+                (operation["operation_id"], job["id"]),
+            )
+            self.save(job["id"], None)
+            return
+        if plan["phase"] == "scheduled_running":
+            operation = self.close_unknown(operation)
+            if operation["status"] == "COMPLETED":
+                return
+            self.discard(plan["asset_id"])
+            payload = self.payload(config)
+            key = "replacement-" + plan["root"]
+            op_id = self.service.submit(
+                "display", payload, config["device_id"], "scheduler-retry", key, key
+            )
+            plan.update(phase="replacement", active=op_id)
+            self.save(job["id"], plan)
+            self.db.execute(
+                "UPDATE jobs SET last_result=? WHERE id=?", (op_id, job["id"])
+            )
+            self.db.event(
+                "upload.regenerated",
+                {
+                    "job_id": job["id"],
+                    "operation_id": op_id,
+                    "failed_asset_id": plan["asset_id"],
+                },
+            )
+            return
+        if current >= deadline or plan["count"] >= len(plan["times"]):
+            plan["phase"] = "scheduled"
+            self.save(job["id"], plan)
+            return
+        if current < datetime.fromisoformat(plan["times"][plan["count"]]):
+            return
+        operation = self.close_unknown(operation)
+        if operation["status"] == "COMPLETED":
+            return
+        number = plan["count"] + 1
+        key = f"upload-retry-{plan['root']}-{number}"
+        op_id = self.service.submit(
+            "display",
+            self.reuse(self.payload(config), plan["asset_id"]),
+            config["device_id"],
+            "scheduler-retry",
+            key,
+            key,
+        )
+        plan.update(active=op_id, count=number)
+        self.save(job["id"], plan)
+        self.db.execute("UPDATE jobs SET last_result=? WHERE id=?", (op_id, job["id"]))
+        self.db.event(
+            "upload.retry",
+            {
+                "job_id": job["id"],
+                "operation_id": op_id,
+                "attempt": number,
+                "deadline": deadline.isoformat(),
+            },
+        )
+
+    @staticmethod
+    def payload(config):
+        return {
+            "device_id": config["device_id"],
+            "content": config["source"],
+            "options": {
+                "duration": config["display_duration"],
+                "restore_previous": config["restore_previous"],
+                "priority": config["priority"],
+                "cleanup_unstarred": config.get("cleanup_unstarred", False),
+            },
+        }