Add documented KoKonna failure policy, bounded retries and durable recovery escalation [skip ci]

pigway_frame_flow/display/recovery.py

@@ -0,0 +1,73 @@
+"""Durable, read-only recovery progress for writes with uncertain outcomes."""
+
+from datetime import UTC, datetime
+from uuid import uuid4
+
+from pigway_frame_flow.storage.database import encode, now_iso
+
+
+class RecoveryMonitor:
+    def __init__(self, service):
+        self.service, self.db = service, service.db
+
+    def observe(self, op, reason):
+        timestamp = now_iso()
+        age = (
+            datetime.now(UTC) - datetime.fromisoformat(op["created_at"])
+        ).total_seconds()
+        review = age >= self.service.settings.sync_timeout + 300
+        with self.db.transaction() as tx:
+            previous = tx.execute(
+                "SELECT * FROM operation_recovery WHERE operation_id=?", (op["id"],)
+            ).fetchone()
+            # Once escalated, wall-clock corrections must not clear the alert.
+            review = review or bool(previous and previous["state"] == "manual_review")
+            state = "manual_review" if review else "checking"
+            tx.execute(
+                "INSERT INTO operation_recovery VALUES(?,?,?,?,?,1) ON CONFLICT(operation_id) DO UPDATE SET state=excluded.state,reason=excluded.reason,last_checked_at=excluded.last_checked_at,checks=operation_recovery.checks+1",
+                (op["id"], state, reason, timestamp, timestamp),
+            )
+            if review and (not previous or previous["state"] != "manual_review"):
+                tx.execute(
+                    "INSERT INTO events VALUES(?,?,?,?)",
+                    (
+                        uuid4().hex,
+                        "operation.blocked",
+                        encode(
+                            {
+                                "operation_id": op["id"],
+                                "device_id": op["device_id"],
+                                "reason": reason,
+                                "requires_manual_review": True,
+                            }
+                        ),
+                        timestamp,
+                    ),
+                )
+                tx.execute(
+                    "INSERT INTO history(client_id,request_id,operation_id,device_id,action,result,timestamp) VALUES(?,?,?,?,?,?,?)",
+                    (
+                        op["client_id"],
+                        op["request_id"],
+                        op["id"],
+                        op["device_id"],
+                        op["action"],
+                        "MANUAL_REVIEW_REQUIRED",
+                        timestamp,
+                    ),
+                )
+
+    def resolve(self, operation_id):
+        self.db.execute(
+            "UPDATE operation_recovery SET state='resolved' WHERE operation_id=? AND state!='resolved'",
+            (operation_id,),
+        )
+
+    def info(self, operation_id):
+        row = self.db.one(
+            "SELECT state,reason,first_checked_at,last_checked_at,checks FROM operation_recovery WHERE operation_id=?",
+            (operation_id,),
+        )
+        if row:
+            row["requires_manual_review"] = row["state"] == "manual_review"
+        return row

pigway_frame_flow/display/service.py

@@ -6,6 +6,7 @@ from datetime import UTC, datetime, timedelta
 from uuid import uuid4
 
 from pigway_frame_flow.display.cloud_library import CloudLibrary
+from pigway_frame_flow.display.recovery import RecoveryMonitor
 from pigway_frame_flow.display.retention import Retention
 from pigway_frame_flow.display.upload_reconciliation import UploadReconciliation
 from pigway_frame_flow.drivers.base import DriverRegistry
@@ -50,6 +51,7 @@ class DisplayService:
         self.lock = threading.RLock()
         self.retention = Retention(self)
         self.cloud_library = CloudLibrary(self)
+        self.recovery = RecoveryMonitor(self)
         self.upload_reconciliation = UploadReconciliation(self)
         self.expiration_checked: dict[str, float] = {}
         self.unknown_checked: dict[str, float] = {}
@@ -165,6 +167,7 @@ class DisplayService:
             request_id=op["request_id"],
             client_id=op["client_id"],
             action=op["action"],
+            recovery=self.recovery.info(id),
         )
 
     def update(self, id, status, result=None, error=None):
@@ -184,6 +187,8 @@ class DisplayService:
                 id,
             ),
         )
+        if status in ("COMPLETED", "FAILED"):
+            self.recovery.resolve(id)
 
     def blocked(self, device_id):
         return (
@@ -702,21 +707,34 @@ class DisplayService:
         # Late acknowledgements only resolve a known display target. Never replay
         # writes or infer success from gallery presence alone.
         for op in self.db.all(
-            "SELECT * FROM operations WHERE status='UNKNOWN' AND action='display' AND write_started=1"
+            "SELECT * FROM operations WHERE status='UNKNOWN' AND action!='render-ai' AND write_started=1"
         ):
             expected = json.loads(op["result"] or "{}").get("image_id")
             intent = json.loads(op["result"] or "{}").get("upload_intent")
-            if not expected and not intent:
-                continue
             checked = time.monotonic()
             if checked - self.unknown_checked.get(op["id"], float("-inf")) < 60:
                 continue
             self.unknown_checked[op["id"]] = checked
+            if op["action"] != "display" or (not expected and not intent):
+                self.recovery.observe(
+                    op,
+                    "missing_upload_evidence"
+                    if op["action"] == "display"
+                    else "manual_reconciliation_required",
+                )
+                continue
             try:
                 driver = self.registry.get(op["device_id"])
                 device = driver.device(refresh=True)
                 if intent:
                     expected = self.upload_reconciliation.check(op, driver, device)
+                    if (
+                        self.db.one(
+                            "SELECT status FROM operations WHERE id=?", (op["id"],)
+                        )["status"]
+                        == "FAILED"
+                    ):
+                        continue
                 if (
                     expected
                     and device.current_content.id == expected
@@ -727,8 +745,18 @@ class DisplayService:
                     self.db.history(op, "COMPLETED")
                     self.db.event("operation.completed", {"operation_id": op["id"]})
                     self.unknown_checked.pop(op["id"], None)
+                elif expected:
+                    self.recovery.observe(
+                        op,
+                        "waiting_device_sync"
+                        if device.current_content.id == expected
+                        else "target_mismatch",
+                    )
             except FlowError:
-                pass
+                self.recovery.observe(op, "read_unavailable")
+            except Exception:
+                # Bad vendor metadata for one operation must not stop the worker.
+                self.recovery.observe(op, "reconciliation_error")
         for op in self.db.all("SELECT * FROM operations WHERE status='SYNCING'"):
             try:
                 device = self.registry.get(op["device_id"]).device()

pigway_frame_flow/display/upload_reconciliation.py

@@ -59,6 +59,12 @@ class UploadReconciliation:
                     candidates.append(image)
             # Ambiguous matches never release a write lock.
             if len(candidates) != 1:
+                service.recovery.observe(
+                    op,
+                    "gallery_match_missing"
+                    if not candidates
+                    else "gallery_match_ambiguous",
+                )
                 return None
             expected = candidates[0].image_id
             data = driver.download(expected)
@@ -66,6 +72,7 @@ class UploadReconciliation:
                 len(data) != intent["size"]
                 or hashlib.sha256(data).hexdigest() != intent["sha256"]
             ):
+                service.recovery.observe(op, "content_mismatch")
                 return None
             result["image_id"] = expected
             result["upload_confirmed"] = True