feat: recover failed scheduled uploads with bounded persistent retries [skip ci]

pigway_frame_flow/scheduler/engine.py

@@ -18,6 +18,7 @@ from pigway_frame_flow.models import (
     PlaylistInput,
     RestoreContent,
 )
+from pigway_frame_flow.scheduler.upload_retries import UploadRetries
 from pigway_frame_flow.storage.database import encode, now_iso
 
 
@@ -60,6 +61,7 @@ class Scheduler:
         self.thread = None
         self.webhook_thread = None
         self.tick_lock = threading.Lock()
+        self.upload_retries = UploadRetries(service)
 
     def jobs(self):
         rows = [
@@ -147,6 +149,7 @@ class Scheduler:
     def tick(self, current=None):
         current = current or datetime.now(UTC)
         with self.tick_lock:
+            self.upload_retries.tick(current)
             for row in self.db.all(
                 "SELECT * FROM jobs WHERE enabled=1 AND next_run<=?",
                 (current.isoformat(),),
@@ -183,6 +186,10 @@ class Scheduler:
                                 ),
                             ).model_dump(mode="json")
                         )
+                        if action == "display":
+                            payload = self.upload_retries.scheduled_payload(
+                                row["id"], payload
+                            )
                         operation_id = self.service.submit(
                             action,
                             payload,
@@ -191,6 +198,10 @@ class Scheduler:
                             "schedule-" + uuid4().hex,
                             "job-" + row["id"] + "-" + slot,
                         )
+                        if action == "display":
+                            self.upload_retries.scheduled_submitted(
+                                row["id"], operation_id
+                            )
                     following = next_run(job.schedule, current)
                     enabled = bool(following)
                     updated = job.model_copy(update={"enabled": enabled}).model_dump(