way/pigway-frame-flow
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 29 Wiki Activity

feat: recover failed scheduled uploads with bounded persistent retries [skip ci]

Browse Source
This commit is contained in:
渡口浪人
2026-10-18 08:42:05 +08:00
parent 41c6897151
commit 7c929cb955
17 changed files with 528 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pigway_frame_flow/display/cloud_library.py
+14 -3
View File
@@ -13,18 +13,29 @@ class CloudLibrary:
def register(self, device_id, image_id, asset_id, operation_id):
operation = self.db.one(
"SELECT payload FROM operations WHERE id=?", (operation_id,)
"SELECT payload,client_id FROM operations WHERE id=?", (operation_id,)
)
payload = json.loads(operation["payload"])
content = payload.get("content", {})
job = self.db.one(
"SELECT job_id FROM job_runs WHERE operation_id=?", (operation_id,)
)
if not job and operation["client_id"] == "scheduler-retry":
job = self.db.one(
"SELECT job_id FROM asset_metadata WHERE asset_id=? AND job_id IS NOT NULL",
(asset_id,),
)
job_config = (
self.db.one("SELECT payload FROM jobs WHERE id=?", (job["job_id"],))
if job
else None
)
source = json.loads(job_config["payload"])["source"] if job_config else content
job_id = (
job["job_id"]
if job
and content.get("type") in ("photo_source", "auto_collage")
and content.get("item_id") is None
and source.get("type") in ("photo_source", "auto_collage")
and source.get("item_id") is None
else None
)
self.db.execute(
pigway_frame_flow/display/service.py
+59 -8
View File
@@ -115,7 +115,7 @@ class DisplayService:
)
return existing["operation_id"]
if (
action not in ("render", "render-ai", "local-upload")
action not in ("display", "render", "render-ai", "local-upload")
and tx.execute(
"SELECT id FROM operations WHERE device_id=? AND status='UNKNOWN' AND write_started=1 AND action!='render-ai' LIMIT 1",
(device_id,),
@@ -187,6 +187,13 @@ class DisplayService:
id,
),
)
if status == "COMPLETED":
row = self.db.one("SELECT result FROM operations WHERE id=?", (id,))
asset_id = json.loads(row["result"] or "{}").get("rendered_asset_id")
if asset_id and self.db.one(
"SELECT id FROM assets WHERE id=?", (asset_id,)
):
self.assets.promote(asset_id)
if status in ("COMPLETED", "FAILED"):
self.recovery.resolve(id)
@@ -226,12 +233,17 @@ class DisplayService:
(device_id, image_id, image_id, device_id, device_id),
)
def content_asset(self, content, device_id=None, operation_id=None):
def content_asset(
self, content, device_id=None, operation_id=None, *, staging=False
):
if isinstance(content, AutoCollageContent):
from pigway_frame_flow.extensions.auto_collage import prepare
return self.content_asset(
prepare(self, content, device_id), device_id, operation_id
prepare(self, content, device_id),
device_id,
operation_id,
staging=staging,
)
if isinstance(content, ProviderContent):
values = {**content.data, **self.providers.get(content.provider_id)}
@@ -244,11 +256,14 @@ class DisplayService:
),
device_id,
operation_id,
staging=staging,
)
if isinstance(content, TemplateContent):
values = self.renderer.resolve(content.template, content.data)
data = self.renderer.render(content.template, values, content.profile)
asset = self.assets.put(data, content.template + ".jpg", "generated")
asset = self.assets.put(
data, content.template + ".jpg", "cache" if staging else "generated"
)
source = {
"type": "template",
"template": content.template,
@@ -377,7 +392,7 @@ class DisplayService:
"UPDATE operations SET result=? WHERE id=?",
(encode({"photo_attempts": attempts}), operation_id),
)
return self.assets.put(rendered, name)
return self.assets.put(rendered, name, "cache" if staging else "images")
raise FlowError(
E.INVALID_REQUEST, "No decodable photo found within 5 candidates"
)
@@ -390,7 +405,9 @@ class DisplayService:
else:
raise FlowError(E.INVALID_REQUEST, "Content requires renderable image data")
return self.assets.put(
convert(data, content.fit, content.profile), content.name
convert(data, content.fit, content.profile),
content.name,
"cache" if staging else "images",
)
def current_content(self, device_id):
@@ -446,10 +463,38 @@ class DisplayService:
yield from rows
cursor_time, cursor_id = rows[-1]["created_at"], rows[-1]["id"]
def upload_retry_expired(self, op):
if op["client_id"] != "scheduler-retry" or not op["request_id"].startswith(
"upload-retry-"
):
return False
root = op["request_id"].split("-")[-2]
for row in self.db.all(
"SELECT payload FROM app_config WHERE key LIKE 'upload-retry:%'"
):
plan = json.loads(row["payload"])
if plan["root"] == root:
return now_iso() >= plan["deadline"]
return True
def process_one(self):
with self.lock:
for op in self.queued_operations():
if self.upload_retry_expired(op):
self.update(
op["id"],
"FAILED",
error={
"code": E.CONFLICT,
"message": "Upload retry window closed",
"request_id": op["request_id"],
"details": {"reason": "RETRY_WINDOW_CLOSED"},
},
)
self.db.history(op, "RETRY_WINDOW_CLOSED")
continue
if op["action"] not in (
"display",
"render",
"render-ai",
"local-upload",
@@ -632,6 +677,13 @@ class DisplayService:
self.update(id, "SYNCING", {"image_id": target})
self.db.history(op, "ACCEPTED")
return
# A new upload is independent of a previous failed write. Read back its
# outcome before closing the old attempt; never replay that operation.
for pending in self.db.all(
"SELECT id FROM operations WHERE device_id=? AND status='UNKNOWN' AND write_started=1 AND action='display'",
(device_id,),
):
self.reconcile(pending["id"], acknowledge=True)
request = DisplayRequest.model_validate(payload)
content = request.content
self.update(id, "RENDERING")
@@ -648,7 +700,7 @@ class DisplayService:
)
image_id = matches[0].image_id
else:
asset = self.content_asset(content, device_id, id)
asset = self.content_asset(content, device_id, id, staging=True)
self.assets.record_origin(asset["asset_id"], op)
image_id = None
self.db.execute(
@@ -683,7 +735,6 @@ class DisplayService:
driver.display(image_id)
result = {"image_id": image_id}
if asset:
self.assets.promote(asset["asset_id"])
result["rendered_asset_id"] = asset["asset_id"]
source = self.db.one(
"SELECT payload FROM asset_sources WHERE asset_id=?",
pigway_frame_flow/display/upload_reconciliation.py
-1
View File
@@ -77,7 +77,6 @@ class UploadReconciliation:
result["image_id"] = expected
result["upload_confirmed"] = True
service.update(op["id"], "UNKNOWN", result, json.loads(op["error"] or "{}"))
service.assets.promote(result["rendered_asset_id"])
source = service.db.one(
"SELECT payload FROM asset_sources WHERE asset_id=?",
(result["rendered_asset_id"],),