Add documented KoKonna failure policy, bounded retries and durable recovery escalation [skip ci]

pigway_frame_flow/display/service.py

@@ -6,6 +6,7 @@ from datetime import UTC, datetime, timedelta
 from uuid import uuid4
 
 from pigway_frame_flow.display.cloud_library import CloudLibrary
+from pigway_frame_flow.display.recovery import RecoveryMonitor
 from pigway_frame_flow.display.retention import Retention
 from pigway_frame_flow.display.upload_reconciliation import UploadReconciliation
 from pigway_frame_flow.drivers.base import DriverRegistry
@@ -50,6 +51,7 @@ class DisplayService:
         self.lock = threading.RLock()
         self.retention = Retention(self)
         self.cloud_library = CloudLibrary(self)
+        self.recovery = RecoveryMonitor(self)
         self.upload_reconciliation = UploadReconciliation(self)
         self.expiration_checked: dict[str, float] = {}
         self.unknown_checked: dict[str, float] = {}
@@ -165,6 +167,7 @@ class DisplayService:
             request_id=op["request_id"],
             client_id=op["client_id"],
             action=op["action"],
+            recovery=self.recovery.info(id),
         )
 
     def update(self, id, status, result=None, error=None):
@@ -184,6 +187,8 @@ class DisplayService:
                 id,
             ),
         )
+        if status in ("COMPLETED", "FAILED"):
+            self.recovery.resolve(id)
 
     def blocked(self, device_id):
         return (
@@ -702,21 +707,34 @@ class DisplayService:
         # Late acknowledgements only resolve a known display target. Never replay
         # writes or infer success from gallery presence alone.
         for op in self.db.all(
-            "SELECT * FROM operations WHERE status='UNKNOWN' AND action='display' AND write_started=1"
+            "SELECT * FROM operations WHERE status='UNKNOWN' AND action!='render-ai' AND write_started=1"
         ):
             expected = json.loads(op["result"] or "{}").get("image_id")
             intent = json.loads(op["result"] or "{}").get("upload_intent")
-            if not expected and not intent:
-                continue
             checked = time.monotonic()
             if checked - self.unknown_checked.get(op["id"], float("-inf")) < 60:
                 continue
             self.unknown_checked[op["id"]] = checked
+            if op["action"] != "display" or (not expected and not intent):
+                self.recovery.observe(
+                    op,
+                    "missing_upload_evidence"
+                    if op["action"] == "display"
+                    else "manual_reconciliation_required",
+                )
+                continue
             try:
                 driver = self.registry.get(op["device_id"])
                 device = driver.device(refresh=True)
                 if intent:
                     expected = self.upload_reconciliation.check(op, driver, device)
+                    if (
+                        self.db.one(
+                            "SELECT status FROM operations WHERE id=?", (op["id"],)
+                        )["status"]
+                        == "FAILED"
+                    ):
+                        continue
                 if (
                     expected
                     and device.current_content.id == expected
@@ -727,8 +745,18 @@ class DisplayService:
                     self.db.history(op, "COMPLETED")
                     self.db.event("operation.completed", {"operation_id": op["id"]})
                     self.unknown_checked.pop(op["id"], None)
+                elif expected:
+                    self.recovery.observe(
+                        op,
+                        "waiting_device_sync"
+                        if device.current_content.id == expected
+                        else "target_mismatch",
+                    )
             except FlowError:
-                pass
+                self.recovery.observe(op, "read_unavailable")
+            except Exception:
+                # Bad vendor metadata for one operation must not stop the worker.
+                self.recovery.observe(op, "reconciliation_error")
         for op in self.db.all("SELECT * FROM operations WHERE status='SYNCING'"):
             try:
                 device = self.registry.get(op["device_id"]).device()