Add documented KoKonna failure policy, bounded retries and durable recovery escalation [skip ci]
This commit is contained in:
渡口浪人
@@ -124,7 +124,7 @@ def create_app(settings=None, start_worker=True):
|
||||
app = FastAPI(
|
||||
title="Pigway Frame Flow API",
|
||||
docs_url=None,
|
||||
version="0.1.3",
|
||||
version="0.1.4",
|
||||
lifespan=lifespan,
|
||||
responses=error_responses,
|
||||
)
|
||||
@@ -1000,7 +1000,7 @@ def create_app(settings=None, start_worker=True):
|
||||
)
|
||||
def settings_view():
|
||||
return SettingsView(
|
||||
version="0.1.3",
|
||||
version="0.1.4",
|
||||
mock=settings.mock,
|
||||
write_enabled=settings.write_enabled,
|
||||
api_key_configured=bool(settings.api_key)
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Durable, read-only recovery progress for writes with uncertain outcomes."""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from pigway_frame_flow.storage.database import encode, now_iso
|
||||
|
||||
|
||||
class RecoveryMonitor:
|
||||
def __init__(self, service):
|
||||
self.service, self.db = service, service.db
|
||||
|
||||
def observe(self, op, reason):
|
||||
timestamp = now_iso()
|
||||
age = (
|
||||
datetime.now(UTC) - datetime.fromisoformat(op["created_at"])
|
||||
).total_seconds()
|
||||
review = age >= self.service.settings.sync_timeout + 300
|
||||
with self.db.transaction() as tx:
|
||||
previous = tx.execute(
|
||||
"SELECT * FROM operation_recovery WHERE operation_id=?", (op["id"],)
|
||||
).fetchone()
|
||||
# Once escalated, wall-clock corrections must not clear the alert.
|
||||
review = review or bool(previous and previous["state"] == "manual_review")
|
||||
state = "manual_review" if review else "checking"
|
||||
tx.execute(
|
||||
"INSERT INTO operation_recovery VALUES(?,?,?,?,?,1) ON CONFLICT(operation_id) DO UPDATE SET state=excluded.state,reason=excluded.reason,last_checked_at=excluded.last_checked_at,checks=operation_recovery.checks+1",
|
||||
(op["id"], state, reason, timestamp, timestamp),
|
||||
)
|
||||
if review and (not previous or previous["state"] != "manual_review"):
|
||||
tx.execute(
|
||||
"INSERT INTO events VALUES(?,?,?,?)",
|
||||
(
|
||||
uuid4().hex,
|
||||
"operation.blocked",
|
||||
encode(
|
||||
{
|
||||
"operation_id": op["id"],
|
||||
"device_id": op["device_id"],
|
||||
"reason": reason,
|
||||
"requires_manual_review": True,
|
||||
}
|
||||
),
|
||||
timestamp,
|
||||
),
|
||||
)
|
||||
tx.execute(
|
||||
"INSERT INTO history(client_id,request_id,operation_id,device_id,action,result,timestamp) VALUES(?,?,?,?,?,?,?)",
|
||||
(
|
||||
op["client_id"],
|
||||
op["request_id"],
|
||||
op["id"],
|
||||
op["device_id"],
|
||||
op["action"],
|
||||
"MANUAL_REVIEW_REQUIRED",
|
||||
timestamp,
|
||||
),
|
||||
)
|
||||
|
||||
def resolve(self, operation_id):
|
||||
self.db.execute(
|
||||
"UPDATE operation_recovery SET state='resolved' WHERE operation_id=? AND state!='resolved'",
|
||||
(operation_id,),
|
||||
)
|
||||
|
||||
def info(self, operation_id):
|
||||
row = self.db.one(
|
||||
"SELECT state,reason,first_checked_at,last_checked_at,checks FROM operation_recovery WHERE operation_id=?",
|
||||
(operation_id,),
|
||||
)
|
||||
if row:
|
||||
row["requires_manual_review"] = row["state"] == "manual_review"
|
||||
return row
|
||||
@@ -6,6 +6,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
from pigway_frame_flow.display.cloud_library import CloudLibrary
|
||||
from pigway_frame_flow.display.recovery import RecoveryMonitor
|
||||
from pigway_frame_flow.display.retention import Retention
|
||||
from pigway_frame_flow.display.upload_reconciliation import UploadReconciliation
|
||||
from pigway_frame_flow.drivers.base import DriverRegistry
|
||||
@@ -50,6 +51,7 @@ class DisplayService:
|
||||
self.lock = threading.RLock()
|
||||
self.retention = Retention(self)
|
||||
self.cloud_library = CloudLibrary(self)
|
||||
self.recovery = RecoveryMonitor(self)
|
||||
self.upload_reconciliation = UploadReconciliation(self)
|
||||
self.expiration_checked: dict[str, float] = {}
|
||||
self.unknown_checked: dict[str, float] = {}
|
||||
@@ -165,6 +167,7 @@ class DisplayService:
|
||||
request_id=op["request_id"],
|
||||
client_id=op["client_id"],
|
||||
action=op["action"],
|
||||
recovery=self.recovery.info(id),
|
||||
)
|
||||
|
||||
def update(self, id, status, result=None, error=None):
|
||||
@@ -184,6 +187,8 @@ class DisplayService:
|
||||
id,
|
||||
),
|
||||
)
|
||||
if status in ("COMPLETED", "FAILED"):
|
||||
self.recovery.resolve(id)
|
||||
|
||||
def blocked(self, device_id):
|
||||
return (
|
||||
@@ -702,21 +707,34 @@ class DisplayService:
|
||||
# Late acknowledgements only resolve a known display target. Never replay
|
||||
# writes or infer success from gallery presence alone.
|
||||
for op in self.db.all(
|
||||
"SELECT * FROM operations WHERE status='UNKNOWN' AND action='display' AND write_started=1"
|
||||
"SELECT * FROM operations WHERE status='UNKNOWN' AND action!='render-ai' AND write_started=1"
|
||||
):
|
||||
expected = json.loads(op["result"] or "{}").get("image_id")
|
||||
intent = json.loads(op["result"] or "{}").get("upload_intent")
|
||||
if not expected and not intent:
|
||||
continue
|
||||
checked = time.monotonic()
|
||||
if checked - self.unknown_checked.get(op["id"], float("-inf")) < 60:
|
||||
continue
|
||||
self.unknown_checked[op["id"]] = checked
|
||||
if op["action"] != "display" or (not expected and not intent):
|
||||
self.recovery.observe(
|
||||
op,
|
||||
"missing_upload_evidence"
|
||||
if op["action"] == "display"
|
||||
else "manual_reconciliation_required",
|
||||
)
|
||||
continue
|
||||
try:
|
||||
driver = self.registry.get(op["device_id"])
|
||||
device = driver.device(refresh=True)
|
||||
if intent:
|
||||
expected = self.upload_reconciliation.check(op, driver, device)
|
||||
if (
|
||||
self.db.one(
|
||||
"SELECT status FROM operations WHERE id=?", (op["id"],)
|
||||
)["status"]
|
||||
== "FAILED"
|
||||
):
|
||||
continue
|
||||
if (
|
||||
expected
|
||||
and device.current_content.id == expected
|
||||
@@ -727,8 +745,18 @@ class DisplayService:
|
||||
self.db.history(op, "COMPLETED")
|
||||
self.db.event("operation.completed", {"operation_id": op["id"]})
|
||||
self.unknown_checked.pop(op["id"], None)
|
||||
elif expected:
|
||||
self.recovery.observe(
|
||||
op,
|
||||
"waiting_device_sync"
|
||||
if device.current_content.id == expected
|
||||
else "target_mismatch",
|
||||
)
|
||||
except FlowError:
|
||||
pass
|
||||
self.recovery.observe(op, "read_unavailable")
|
||||
except Exception:
|
||||
# Bad vendor metadata for one operation must not stop the worker.
|
||||
self.recovery.observe(op, "reconciliation_error")
|
||||
for op in self.db.all("SELECT * FROM operations WHERE status='SYNCING'"):
|
||||
try:
|
||||
device = self.registry.get(op["device_id"]).device()
|
||||
|
||||
@@ -59,6 +59,12 @@ class UploadReconciliation:
|
||||
candidates.append(image)
|
||||
# Ambiguous matches never release a write lock.
|
||||
if len(candidates) != 1:
|
||||
service.recovery.observe(
|
||||
op,
|
||||
"gallery_match_missing"
|
||||
if not candidates
|
||||
else "gallery_match_ambiguous",
|
||||
)
|
||||
return None
|
||||
expected = candidates[0].image_id
|
||||
data = driver.download(expected)
|
||||
@@ -66,6 +72,7 @@ class UploadReconciliation:
|
||||
len(data) != intent["size"]
|
||||
or hashlib.sha256(data).hexdigest() != intent["sha256"]
|
||||
):
|
||||
service.recovery.observe(op, "content_mismatch")
|
||||
return None
|
||||
result["image_id"] = expected
|
||||
result["upload_confirmed"] = True
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Hardened adapter over the MIT-0 guooscar 1.2.0 device/upload methods.
|
||||
|
||||
No UI, scheduler, renderer, environment lookup, or constructor cloud calls.
|
||||
Never retry a write; a timeout may have occurred after cloud acceptance.
|
||||
Retry only read-only requests or writes proven unsent/rejected; never replay ambiguous writes.
|
||||
"""
|
||||
|
||||
import threading
|
||||
@@ -19,28 +19,85 @@ BASE_URL = "https://api.kokonna.art/openapi"
|
||||
|
||||
|
||||
class SafeTransport:
|
||||
MAX_ATTEMPTS = 3
|
||||
MAX_RETRY_DELAY = 60
|
||||
|
||||
def __init__(self, key, session=None):
|
||||
self.key = key
|
||||
self.session = session or requests.Session()
|
||||
self.calls: deque[float] = deque()
|
||||
self.lock = threading.Lock()
|
||||
self.cooldown_until = 0.0
|
||||
|
||||
def request(self, endpoint, payload=None, write=False, binary=False):
|
||||
if not self.key:
|
||||
raise FlowError(E.CONFIG_REQUIRED, "KoKonna configuration required", 503)
|
||||
for attempt in range(1, self.MAX_ATTEMPTS + 1):
|
||||
try:
|
||||
return self._request_once(endpoint, payload, write, binary)
|
||||
except FlowError as exc:
|
||||
exc.details["attempts"] = attempt
|
||||
delay = max(
|
||||
2 ** (attempt - 1), exc.details.get("retry_after_seconds", 0)
|
||||
)
|
||||
if (
|
||||
not exc.details.get("retryable")
|
||||
or attempt == self.MAX_ATTEMPTS
|
||||
or delay > self.MAX_RETRY_DELAY
|
||||
):
|
||||
if exc.details.get("retryable"):
|
||||
exc.details["retry_exhausted"] = attempt == self.MAX_ATTEMPTS
|
||||
exc.details["retry_deferred"] = delay > self.MAX_RETRY_DELAY
|
||||
raise
|
||||
time.sleep(delay)
|
||||
raise AssertionError("Retry loop exhausted")
|
||||
|
||||
def _reserve(self):
|
||||
with self.lock:
|
||||
current = time.monotonic()
|
||||
while self.calls and current - self.calls[0] >= 60:
|
||||
self.calls.popleft()
|
||||
# Reserve capacity beneath the official 20/minute ceiling.
|
||||
delay = max(0.0, self.cooldown_until - current)
|
||||
if len(self.calls) >= 18:
|
||||
raise FlowError(E.RATE_LIMITED, "Device request budget exhausted", 429)
|
||||
delay = max(delay, 60 - (current - self.calls[0]))
|
||||
if delay:
|
||||
raise FlowError(
|
||||
E.RATE_LIMITED,
|
||||
"Device request budget exhausted",
|
||||
429,
|
||||
details={
|
||||
"retryable": True,
|
||||
"retry_after_seconds": delay,
|
||||
"write_outcome": "not_sent",
|
||||
"failure_kind": "local_rate_limit",
|
||||
},
|
||||
)
|
||||
self.calls.append(current)
|
||||
url = BASE_URL + endpoint
|
||||
|
||||
@staticmethod
|
||||
def _retry_after(value):
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
try:
|
||||
seconds = float(value)
|
||||
except (ValueError, TypeError):
|
||||
try:
|
||||
seconds = (
|
||||
parsedate_to_datetime(value) - datetime.now(UTC)
|
||||
).total_seconds()
|
||||
except (ValueError, TypeError, OverflowError):
|
||||
return 60.0
|
||||
import math
|
||||
|
||||
return max(0.0, seconds) if math.isfinite(seconds) else 60.0
|
||||
|
||||
def _request_once(self, endpoint, payload, write, binary):
|
||||
if not self.key:
|
||||
raise FlowError(E.CONFIG_REQUIRED, "KoKonna configuration required", 503)
|
||||
self._reserve()
|
||||
try:
|
||||
response = self.session.request(
|
||||
"GET" if binary else "POST",
|
||||
url,
|
||||
BASE_URL + endpoint,
|
||||
headers={}
|
||||
if binary
|
||||
else {
|
||||
@@ -52,8 +109,7 @@ class SafeTransport:
|
||||
allow_redirects=False,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
# Persist only fixed categories: exception text and download URLs may
|
||||
# contain credentials. Unknown writes must still never be replayed.
|
||||
# Never retain exception text or a URL: binary URLs contain the key.
|
||||
category = next(
|
||||
(
|
||||
label
|
||||
@@ -68,50 +124,124 @@ class SafeTransport:
|
||||
),
|
||||
"request_error",
|
||||
)
|
||||
not_sent = isinstance(exc, requests.ConnectTimeout)
|
||||
uncertain = write and not not_sent
|
||||
raise FlowError(
|
||||
E.WRITE_RESULT_UNKNOWN if write else E.CLOUD_UNAVAILABLE,
|
||||
E.WRITE_RESULT_UNKNOWN if uncertain else E.CLOUD_UNAVAILABLE,
|
||||
"Cloud write outcome is unknown"
|
||||
if write
|
||||
if uncertain
|
||||
else "Device cloud is unavailable",
|
||||
502,
|
||||
details={"transport_error": category},
|
||||
details={
|
||||
"transport_error": category,
|
||||
"retryable": (not write and category != "tls_error") or not_sent,
|
||||
"write_outcome": "unknown"
|
||||
if uncertain
|
||||
else ("not_sent" if write else "read_only"),
|
||||
},
|
||||
) from None
|
||||
if response.status_code != 200:
|
||||
code = (
|
||||
E.RATE_LIMITED
|
||||
if response.status_code == 429
|
||||
else (
|
||||
E.WRITE_RESULT_UNKNOWN
|
||||
if write and response.status_code >= 500
|
||||
else E.CLOUD_UNAVAILABLE
|
||||
)
|
||||
)
|
||||
raise FlowError(
|
||||
code,
|
||||
"Device cloud request failed",
|
||||
429 if code == E.RATE_LIMITED else 502,
|
||||
details={"http_status": response.status_code},
|
||||
)
|
||||
if binary:
|
||||
if len(response.content) > 20 * 1024 * 1024:
|
||||
raise FlowError(E.INVALID_REQUEST, "Cloud image exceeds size limit")
|
||||
return response.content
|
||||
try:
|
||||
value = response.json()
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError()
|
||||
# Cloud errors may echo credentials: never return raw messages.
|
||||
if value.get("success") is False or (
|
||||
"message" in value and "id" not in value
|
||||
value = None
|
||||
if not binary or response.status_code != 200:
|
||||
try:
|
||||
value = response.json()
|
||||
except ValueError:
|
||||
pass
|
||||
message = value.get("message") if isinstance(value, dict) else None
|
||||
status = response.status_code
|
||||
# Only exact documented business failures establish rejection. Generic
|
||||
# 5xx responses may arrive after a write; those remain UNKNOWN.
|
||||
business = None
|
||||
if status in (404, 500) and isinstance(message, str):
|
||||
if message == "image not found":
|
||||
business = (
|
||||
E.NOT_FOUND,
|
||||
"Cloud image not found",
|
||||
404,
|
||||
"image_not_found",
|
||||
)
|
||||
elif status == 500 and message == "imageId is empty":
|
||||
business = (
|
||||
E.INVALID_REQUEST,
|
||||
"Cloud image ID is invalid",
|
||||
400,
|
||||
"invalid_image_id",
|
||||
)
|
||||
elif status == 500 and message.startswith("can not find robot "):
|
||||
business = (
|
||||
E.CONFIG_REQUIRED,
|
||||
"Device key does not identify a frame",
|
||||
503,
|
||||
"device_key_invalid",
|
||||
)
|
||||
elif status == 404 and message.startswith("can not find method "):
|
||||
business = (
|
||||
E.UNSUPPORTED_CAPABILITY,
|
||||
"Cloud method not found",
|
||||
422,
|
||||
"method_not_found",
|
||||
)
|
||||
if business:
|
||||
code, safe_message, local_status, kind = business
|
||||
raise FlowError(
|
||||
code,
|
||||
safe_message,
|
||||
local_status,
|
||||
details={
|
||||
"http_status": status,
|
||||
"failure_kind": kind,
|
||||
"retryable": False,
|
||||
"write_outcome": "rejected" if write else "read_only",
|
||||
},
|
||||
)
|
||||
if status != 200:
|
||||
limited = status == 429
|
||||
rejected = 400 <= status < 500
|
||||
uncertain = write and not rejected
|
||||
details = {
|
||||
"http_status": status,
|
||||
"retryable": limited or (not write and status >= 500),
|
||||
"write_outcome": "unknown"
|
||||
if uncertain
|
||||
else ("rejected" if write else "read_only"),
|
||||
}
|
||||
if limited:
|
||||
delay = self._retry_after(response.headers.get("Retry-After"))
|
||||
details["retry_after_seconds"] = delay
|
||||
with self.lock:
|
||||
self.cooldown_until = max(
|
||||
self.cooldown_until, time.monotonic() + delay
|
||||
)
|
||||
raise FlowError(
|
||||
E.RATE_LIMITED
|
||||
if limited
|
||||
else (E.WRITE_RESULT_UNKNOWN if uncertain else E.CLOUD_UNAVAILABLE),
|
||||
"Device cloud request failed",
|
||||
429 if limited else 502,
|
||||
details=details,
|
||||
)
|
||||
if binary:
|
||||
if len(response.content) > 20 * 1024 * 1024:
|
||||
raise FlowError(E.INVALID_REQUEST, "Cloud image exceeds size limit")
|
||||
return response.content
|
||||
if (
|
||||
not isinstance(value, dict)
|
||||
or value.get("success") is False
|
||||
or ("message" in value and "id" not in value)
|
||||
):
|
||||
raise ValueError()
|
||||
raise FlowError(
|
||||
E.WRITE_RESULT_UNKNOWN if write else E.CLOUD_UNAVAILABLE,
|
||||
"Invalid device cloud response",
|
||||
502,
|
||||
details={
|
||||
"failure_kind": "invalid_response",
|
||||
"retryable": not write,
|
||||
"write_outcome": "unknown" if write else "read_only",
|
||||
},
|
||||
)
|
||||
return value
|
||||
except ValueError:
|
||||
raise FlowError(
|
||||
E.WRITE_RESULT_UNKNOWN if write else E.CLOUD_UNAVAILABLE,
|
||||
"Invalid device cloud response",
|
||||
502,
|
||||
) from None
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
|
||||
class KokonnaClient:
|
||||
@@ -136,7 +266,7 @@ class KokonnaClient:
|
||||
def upload(self, path, name):
|
||||
self.guard()
|
||||
value = guooscar_protocol.upload(self.transport, path.read_bytes(), name)
|
||||
if not isinstance(value.get("id"), int) or value["id"] <= 0:
|
||||
if type(value.get("id")) is not int or value["id"] <= 0:
|
||||
raise FlowError(E.WRITE_RESULT_UNKNOWN, "Upload ID not confirmed", 502)
|
||||
return value["id"]
|
||||
|
||||
|
||||
@@ -138,6 +138,7 @@ def install_webhooks(app, service):
|
||||
allowed = {
|
||||
"operation.completed",
|
||||
"operation.failed",
|
||||
"operation.blocked",
|
||||
"display.changed",
|
||||
"device.online",
|
||||
"device.offline",
|
||||
|
||||
@@ -177,7 +177,17 @@ class OperationResult(BaseModel):
|
||||
deferred: bool = False
|
||||
|
||||
|
||||
class RecoveryState(BaseModel):
|
||||
state: Literal["checking", "manual_review", "resolved"]
|
||||
reason: str
|
||||
first_checked_at: str
|
||||
last_checked_at: str
|
||||
checks: int
|
||||
requires_manual_review: bool
|
||||
|
||||
|
||||
class Operation(BaseModel):
|
||||
recovery: RecoveryState | None = None
|
||||
operation_id: str
|
||||
status: OperationStatus
|
||||
device_id: str
|
||||
@@ -336,7 +346,7 @@ class SettingsView(BaseModel):
|
||||
|
||||
class Health(BaseModel):
|
||||
status: str
|
||||
version: str = "0.1.3"
|
||||
version: str = "0.1.4"
|
||||
|
||||
|
||||
class Readiness(BaseModel):
|
||||
@@ -366,7 +376,7 @@ class RestoreRequest(StrictModel):
|
||||
|
||||
class Version(BaseModel):
|
||||
product: str = "Pigway Frame Flow"
|
||||
version: str = "0.1.3"
|
||||
version: str = "0.1.4"
|
||||
api_version: str = "v1"
|
||||
commit: str = "development"
|
||||
build_time: str = "unknown"
|
||||
|
||||
@@ -31,6 +31,7 @@ class Database:
|
||||
CREATE TABLE IF NOT EXISTS image_sources (device_id TEXT, image_id INTEGER, asset_id TEXT, operation_id TEXT, payload TEXT NOT NULL, PRIMARY KEY(device_id,image_id));
|
||||
CREATE TABLE IF NOT EXISTS app_config (key TEXT PRIMARY KEY, payload TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS playlists (id TEXT PRIMARY KEY, payload TEXT, position INTEGER, next_run TEXT);
|
||||
CREATE TABLE IF NOT EXISTS operation_recovery (operation_id TEXT PRIMARY KEY, state TEXT NOT NULL, reason TEXT NOT NULL, first_checked_at TEXT NOT NULL, last_checked_at TEXT NOT NULL, checks INTEGER NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS operations (
|
||||
id TEXT PRIMARY KEY, status TEXT NOT NULL, action TEXT NOT NULL,
|
||||
payload TEXT NOT NULL, device_id TEXT NOT NULL, client_id TEXT NOT NULL,
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="color-scheme" content="light dark"><title>Pigway Frame Flow</title><link rel="stylesheet" href="/ui/style.css"></head>
|
||||
<body><aside class="sidebar"><a class="brand" href="/"><span class="brand-icon">F<span>↗</span></span><span>Frame Flow<small>Pigway · Smart Frame Automation</small></span></a><nav id="nav"></nav><div class="sidebar-bottom"><small>Version 0.1.3</small></div></aside><div class="workspace"><header><div><span class="eyebrow">PIGWAY FRAME FLOW</span><h1 id="page-title"></h1></div><div class="toolbar"><select id="device" aria-label="Device"></select><button id="lang" class="icon-button" aria-label="Change language">EN</button><button id="theme" class="icon-button" aria-label="Change theme">◐</button></div></header><main id="main"></main><footer id="footer"></footer></div><div id="toast" role="status" aria-live="polite"></div><dialog id="modal"><div id="modal-content"></div></dialog><script src="/ui/i18n.js"></script><script src="/ui/features.js"></script><script src="/ui/extensions.js"></script><script src="/ui/app.js"></script></body></html>
|
||||
<body><aside class="sidebar"><a class="brand" href="/"><span class="brand-icon">F<span>↗</span></span><span>Frame Flow<small>Pigway · Smart Frame Automation</small></span></a><nav id="nav"></nav><div class="sidebar-bottom"><small>Version 0.1.4</small></div></aside><div class="workspace"><header><div><span class="eyebrow">PIGWAY FRAME FLOW</span><h1 id="page-title"></h1></div><div class="toolbar"><select id="device" aria-label="Device"></select><button id="lang" class="icon-button" aria-label="Change language">EN</button><button id="theme" class="icon-button" aria-label="Change theme">◐</button></div></header><main id="main"></main><footer id="footer"></footer></div><div id="toast" role="status" aria-live="polite"></div><dialog id="modal"><div id="modal-content"></div></dialog><script src="/ui/i18n.js"></script><script src="/ui/features.js"></script><script src="/ui/extensions.js"></script><script src="/ui/app.js"></script></body></html>
|
||||
|
||||
Reference in New Issue
Block a user