fix: defer automatic uploads while device synchronization is pending

pigway_frame_flow/display/recovery.py

@@ -74,6 +74,51 @@ class RecoveryMonitor:
             row["requires_manual_review"] = row["state"] == "manual_review"
         return row
 
+    def observe_device(self, op, *, device=None, error=None):
+        row = self.db.one("SELECT result FROM operations WHERE id=?", (op["id"],))
+        result = json.loads(row["result"] or "{}")
+        observation = {"checked_at": now_iso()}
+        if device is not None:
+            observation.update(
+                image_id=device.current_content.id,
+                synced=device.current_content.synced,
+                online=device.status == "online",
+                heartbeat=device.last_seen_at,
+            )
+        else:
+            observation["read_error"] = {
+                "code": error.code,
+                **{
+                    k: v
+                    for k, v in error.details.items()
+                    if k
+                    in ("transport_error", "http_status", "attempts", "retry_exhausted")
+                },
+            }
+        samples = result.get("sync_observations", [])
+        if samples:
+            old = samples[-1]
+            changed = {k: v for k, v in old.items() if k != "checked_at"} != {
+                k: v for k, v in observation.items() if k != "checked_at"
+            }
+            if (
+                not changed
+                and (
+                    datetime.now(UTC) - datetime.fromisoformat(old["checked_at"])
+                ).total_seconds()
+                < 60
+            ):
+                return observation
+        samples.append(observation)
+        result["sync_observations"] = (
+            [samples[0], *samples[-11:]] if len(samples) > 12 else samples
+        )
+        # Evidence must not reset updated_at: that is the sync timeout's clock.
+        self.db.execute(
+            "UPDATE operations SET result=? WHERE id=?", (encode(result), op["id"])
+        )
+        return observation
+
     def close_superseded_displays(self):
         for op in self.db.all(
             "SELECT * FROM operations WHERE action='display' AND write_started=1 "

pigway_frame_flow/display/service.py

@@ -174,7 +174,7 @@ class DisplayService:
         if result:
             row = self.db.one("SELECT result FROM operations WHERE id=?", (id,))
             previous = json.loads(row["result"]) if row and row["result"] else {}
-            for key in ("photo_attempts", "upload_intent"):
+            for key in ("photo_attempts", "upload_intent", "sync_observations"):
                 if key in previous:
                     result = {**result, key: previous[key]}
         self.db.execute(
@@ -519,6 +519,23 @@ class DisplayService:
                     continue
                 payload = json.loads(op["payload"])
                 if op["action"] == "display":
+                    if op["client_id"] in (
+                        "scheduler",
+                        "scheduler-retry",
+                    ) and self.db.one(
+                        "SELECT id FROM operations WHERE device_id=? AND action='display' "
+                        "AND status='SYNCING' AND write_started=1 AND updated_at>? LIMIT 1",
+                        (
+                            op["device_id"],
+                            (
+                                datetime.now(UTC)
+                                - timedelta(seconds=self.settings.sync_timeout)
+                            ).isoformat(),
+                        ),
+                    ):
+                        # Upload activates a new image upstream. Give the prior
+                        # image its existing confirmation window before replacing it.
+                        continue
                     priority = payload["options"]["priority"]
                     top = self.top(op["device_id"])
                     if top and priority < top["priority"]:
@@ -823,6 +840,7 @@ class DisplayService:
             try:
                 driver = self.registry.get(op["device_id"])
                 device = driver.device(refresh=True)
+                self.recovery.observe_device(op, device=device)
                 if intent:
                     expected = self.upload_reconciliation.check(op, driver, device)
                     if (
@@ -850,7 +868,8 @@ class DisplayService:
                         if device.current_content.id == expected
                         else "target_mismatch",
                     )
-            except FlowError:
+            except FlowError as exc:
+                self.recovery.observe_device(op, error=exc)
                 self.recovery.observe(op, "read_unavailable")
             except Exception:
                 # Bad vendor metadata for one operation must not stop the worker.
@@ -858,6 +877,7 @@ class DisplayService:
         for op in self.db.all("SELECT * FROM operations WHERE status='SYNCING'"):
             try:
                 device = self.registry.get(op["device_id"]).device()
+                observation = self.recovery.observe_device(op, device=device)
                 target = json.loads(op["result"])["image_id"]
                 if (
                     device.current_content.id == target
@@ -878,11 +898,11 @@ class DisplayService:
                             "code": E.WRITE_RESULT_UNKNOWN,
                             "message": "Frame refresh was not confirmed",
                             "request_id": op["request_id"],
-                            "details": {},
+                            "details": {"last_device_observation": observation},
                         },
                     )
-            except FlowError:
-                pass
+            except FlowError as exc:
+                self.recovery.observe_device(op, error=exc)
 
         self.recovery.close_superseded_displays()