way/pigway-device-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

fix: unify active alert and RGB behavior

Browse Source
This commit is contained in:
way
2026-09-26 22:35:23 +08:00
parent d3ccca4b37
commit 34f334dd23
4 changed files with 97 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/pigway_pi_control.py
+84 -37
View File
@@ -262,7 +262,7 @@ LED_BRIGHT=cf("led","brightness",10,int)
WARNING_BREATHE_SECONDS=cf("led","warning_breathe_seconds",3.0)
CRITICAL_BREATHE_SECONDS=cf("led","critical_breathe_seconds",1.5)
EMERGENCY_FLASH_MS=cf("led","emergency_flash_ms",300,int)
RGB_UPDATE_MS=cf("led","rgb_update_ms",40,int)
RGB_UPDATE_MS=cf("led","rgb_update_ms",80,int)
RGB_WARNING=(cf("led","warning_r",255,int),cf("led","warning_g",255,int),cf("led","warning_b",0,int))
RGB_CRITICAL=(cf("led","critical_r",255,int),cf("led","critical_g",72,int),cf("led","critical_b",0,int))
RGB_EMERGENCY=(cf("led","emergency_r",255,int),cf("led","emergency_g",0,int),cf("led","emergency_b",0,int))
@@ -317,28 +317,76 @@ def alert(title,l2="",l3="",l4="",sev=1,color=(20,16,0),blink=0):
return {"title":title[:21],"l2":l2[:21],"l3":l3[:21],"l4":l4[:21],
"sev":sev,"color":color,"blink":blink}
class AlertManager:
def __init__(self):
self._active={}
def set_alert(self,alert_id,severity,title,l2="",l3="",l4="",metadata=None):
now=time.monotonic()
display={"id":alert_id,"title":title[:21],"l2":l2[:21],"l3":l3[:21],"l4":l4[:21],
"sev":severity,"metadata":metadata or {}}
current=self._active.get(alert_id)
if current is None:
display["active"]=True; display["first_seen"]=now; display["last_changed"]=now
self._active[alert_id]=display
log("WARN","ALERT_ACTIVE",alert=alert_id,severity=severity,detail=l2)
else:
changed=any(current.get(k)!=display.get(k) for k in ("sev","title","l2","l3","l4","metadata"))
current.update(display)
if changed: current["last_changed"]=now
def clear_alert(self,alert_id):
if alert_id in self._active:
self._active.pop(alert_id)
log("INFO","ALERT_RECOVERED",alert=alert_id)
def active_alerts(self):
return sorted(self._active.values(),key=lambda a:(-a["sev"],a["first_seen"],a["id"]))
def transition(key,active,on_event,off_event,**fields):
old=event_state.get(key,False)
if active!=old:
event_state[key]=active
log("WARN" if active else "INFO",on_event if active else off_event,**fields)
def make_alerts(cpu,temp,mem,disk,sample_elapsed=0.0):
def update_active_alerts(cpu,temp,mem,disk,sample_elapsed=0.0):
global wifi_weak_since,wifi_weak_active
global cpu_hi,mem_hi
a=[]
cpu_hi=cpu_hi+sample_elapsed if cpu>=CPU_HIGH else 0 if cpu<CPU_RECOVER else cpu_hi
mem_hi=mem_hi+sample_elapsed if mem>=MEM_HIGH else 0 if mem<MEM_RECOVER else mem_hi
if cpu_hi>=CPU_HIGH_SEC:a.append(alert("CPU LOAD HIGH",f"LOAD {cpu:.1f}%",f"TEMP {temp:.1f}C","FOR >=30 SEC",1,(24,18,0)))
if temp>=TEMP_CRIT:a.append(alert("TEMP CRITICAL",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING",3,(32,0,0),2))
elif temp>=TEMP_HIGH:a.append(alert("TEMP HIGH",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING",1,(24,0,0)))
if mem_hi>=MEM_HIGH_SEC:a.append(alert("MEMORY HIGH",f"USED {mem:.1f}%","CHECK PROCESS","",1,(18,0,24)))
if disk>=DISK_CRIT:a.append(alert("DISK CRITICAL",f"USED {disk:.1f}%","FREE < 5%","CHECK STORAGE",2,(28,6,0),2))
elif disk>=DISK_HIGH:a.append(alert("DISK SPACE LOW",f"USED {disk:.1f}%","FREE < 10%","CHECK STORAGE",1,(28,8,0)))
if not link_ok:a.append(alert("NETWORK DOWN","NO LAN / WIFI","NO IP ADDRESS","",2,(0,8,28),1))
elif not net_ok:a.append(alert("INTERNET DOWN","LOCAL LINK OK",ip,"",2,(0,8,28),1))
if cpu_hi>=CPU_HIGH_SEC: alerts.set_alert("CPU_HIGH",1,"CPU LOAD HIGH",f"LOAD {cpu:.1f}%",f"TEMP {temp:.1f}C","CHECK LOAD")
else: alerts.clear_alert("CPU_HIGH")
if temp>=TEMP_CRIT:
alerts.set_alert("TEMP_CRITICAL",3,"TEMP CRITICAL",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING")
alerts.clear_alert("TEMP_HIGH")
elif temp>=TEMP_HIGH:
alerts.set_alert("TEMP_HIGH",1,"TEMP HIGH",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING")
alerts.clear_alert("TEMP_CRITICAL")
else:
alerts.clear_alert("TEMP_HIGH"); alerts.clear_alert("TEMP_CRITICAL")
if mem_hi>=MEM_HIGH_SEC: alerts.set_alert("MEMORY_HIGH",1,"MEMORY HIGH",f"USED {mem:.1f}%","CHECK PROCESS","")
else: alerts.clear_alert("MEMORY_HIGH")
if disk>=DISK_CRIT:
alerts.set_alert("DISK_CRITICAL",2,"DISK CRITICAL",f"USED {disk:.1f}%","FREE < 5%","CHECK STORAGE")
alerts.clear_alert("DISK_HIGH")
elif disk>=DISK_HIGH:
alerts.set_alert("DISK_HIGH",1,"DISK SPACE LOW",f"USED {disk:.1f}%","FREE < 10%","CHECK STORAGE")
alerts.clear_alert("DISK_CRITICAL")
else:
alerts.clear_alert("DISK_HIGH"); alerts.clear_alert("DISK_CRITICAL")
if not link_ok:
alerts.set_alert("NETWORK_DOWN",2,"NETWORK DOWN","NO LAN / WIFI","NO IP ADDRESS","")
alerts.clear_alert("INTERNET_DOWN")
elif not net_ok:
alerts.set_alert("INTERNET_DOWN",2,"INTERNET DOWN","LOCAL LINK OK",ip,"")
alerts.clear_alert("NETWORK_DOWN")
else:
alerts.clear_alert("NETWORK_DOWN"); alerts.clear_alert("INTERNET_DOWN")
for w,ok in zip(watches,watch_state):
if not ok:a.append(alert("SERVICE DOWN" if w[0]=="service" else "PROCESS DOWN",w[1],"CHECK / RESTART","",2,(0,24,24),1))
alert_id=("SERVICE_DOWN" if w[0]=="service" else "PROCESS_DOWN")+"|"+w[1]
if not ok:
alerts.set_alert(alert_id,2,"SERVICE DOWN" if w[0]=="service" else "PROCESS DOWN",w[1],"CHECK / RESTART","CRITICAL",{"target":w[2]})
else: alerts.clear_alert(alert_id)
if net_kind=="WIF" and str(net_metric).lstrip("-").isdigit():
rssi=float(net_metric)
if not wifi_weak_active:
@@ -349,17 +397,24 @@ def make_alerts(cpu,temp,mem,disk,sample_elapsed=0.0):
elif rssi >= WIFI_RECOVER:
wifi_weak_active=False; wifi_weak_since=None
if wifi_weak_active:
a.append(alert("WIFI SIGNAL WEAK",f"{rssi:.0f} dBm",f"IF {net_iface}","CHECK SIGNAL",1,(0,0,28),1))
alerts.set_alert("WIFI_WEAK",1,"WIFI SIGNAL WEAK",f"RSSI {rssi:.0f} dBm",f"IF {net_iface}","CHECK SIGNAL")
else: alerts.clear_alert("WIFI_WEAK")
else:
wifi_weak_since=None; wifi_weak_active=False
wifi_weak_since=None; wifi_weak_active=False; alerts.clear_alert("WIFI_WEAK")
if health.get("under_voltage_now"):
a.append(alert("POWER LOW","UNDERVOLT",health.get("raw",""),"CHECK 5V",3,(28,18,0),2))
if health.get("throttled_now") or health.get("freq_capped_now"):
a.append(alert("CPU THROTTLED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"TEMP / POWER",2,(28,18,0),1))
if fs_readonly:
a.append(alert("FILESYSTEM RO","ROOT READ ONLY","CHECK STORAGE","",3,(28,6,0),2))
a.sort(key=lambda x:x["sev"],reverse=True)
return a
alerts.set_alert("POWER_UNDERVOLT",2,"POWER LOW","UNDERVOLTAGE","CHECK POWER","CRITICAL",{"raw":health.get("raw","")})
else: alerts.clear_alert("POWER_UNDERVOLT")
if health.get("throttled_now"):
alerts.set_alert("THROTTLED",2,"CPU THROTTLED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"CHECK POWER")
else: alerts.clear_alert("THROTTLED")
if health.get("freq_capped_now"):
alerts.set_alert("FREQ_CAPPED",2,"FREQ CAPPED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"CHECK POWER")
else: alerts.clear_alert("FREQ_CAPPED")
if health.get("soft_temp_limit_now"):
alerts.set_alert("SOFT_TEMP_LIMIT",2,"SOFT TEMP LIMIT","CPU TEMP LIMITED",health.get("raw",""),"CHECK COOLING")
else: alerts.clear_alert("SOFT_TEMP_LIMIT")
if fs_readonly: alerts.set_alert("ROOT_READONLY",3,"FILESYSTEM RO","ROOT READ ONLY","CHECK STORAGE","EMERGENCY")
else: alerts.clear_alert("ROOT_READONLY")
def rgb_mode_for(a):
if not a: return "OFF"
@@ -370,7 +425,7 @@ def rgb_mode_for(a):
def breathe(level_seconds,elapsed):
# A full sine-derived ease in/out cycle gives continuous light transitions.
phase=(elapsed/max(0.1,level_seconds))%1.0
return 0.5-0.5*math.cos(2.0*math.pi*phase)
return 0.05+0.95*(0.5-0.5*math.cos(2.0*math.pi*phase))
def rgb_engine(mode,elapsed):
if mode=="OFF":
@@ -426,8 +481,7 @@ def render_alert(a,idx,total):
def alert_key(a):
# Sensor values change without changing the identity of a fault.
return a["title"]+("|"+a["l2"] if a["title"] in ("SERVICE DOWN","PROCESS DOWN") else "")
return a["id"]
def find_live_alert(key, live):
for a in live:
@@ -442,15 +496,15 @@ led_alert=None
def led_worker():
previous_error=None
current_mode="OFF"
mode_started=time.monotonic()
animation_started=time.monotonic()
while not led_stop.is_set():
try:
new_mode=rgb_mode_for(led_alert)
if new_mode!=current_mode:
log("INFO","RGB_MODE",old=current_mode,new=new_mode)
current_mode=new_mode; mode_started=time.monotonic()
current_mode=new_mode
# Keep the verified five-write MCU RGB transaction contiguous.
with bus_lock: rgb_engine(current_mode,time.monotonic()-mode_started)
with bus_lock: rgb_engine(current_mode,time.monotonic()-animation_started)
if previous_error is not None: log("INFO","RGB_RECOVERED")
previous_error=None
except Exception as e:
@@ -481,9 +535,9 @@ last_sensor=last_render=0.0
last_net=last_watch=last_health=0.0
health={}; fs_readonly=False
agent={"last_sensor_ok":0.0,"last_network_ok":0.0,"last_watch_ok":0.0,"last_health_ok":0.0,"loop_errors":0,"i2c_errors":0}
alerts=AlertManager()
event_state={}
history_logged=set()
prev_alert_keys=set()
latest=(0.0,0.0,0.0,0.0)
active=[]
@@ -544,16 +598,9 @@ try:
agent["last_health_ok"]=now; last_health=now
c,t,m,d=latest
active=make_alerts(c,t,m,d,sample_elapsed)
update_active_alerts(c,t,m,d,sample_elapsed)
active=alerts.active_alerts()
led_alert=max(active,key=lambda a:a["sev"],default=None)
new_alert_keys={alert_key(a) for a in active}
for a in active:
k=alert_key(a)
if k not in prev_alert_keys:
log("WARN","ALERT_ACTIVE",alert=a["title"],detail=a["l2"],severity=a["sev"])
for k in prev_alert_keys-new_alert_keys:
log("INFO","ALERT_RECOVERED",alert=k)
prev_alert_keys=new_alert_keys
# If the currently displayed alert truly disappeared, skip it immediately.
if cycle_pages[cycle_pos]!="HOME":