From 34f334dd235906b94ba640382b8054f6aa9ef17b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=A1=E5=8F=A3=E6=B5=AA=E4=BA=BA?= Date: Sat, 26 Sep 2026 22:35:23 +0800 Subject: [PATCH] fix: unify active alert and RGB behavior --- README.md | 14 ++-- SHA256SUMS | 6 +- app/pigway_pi_control.py | 121 +++++++++++++++++++++++----------- config/pigway-pi-control.conf | 2 +- 4 files changed, 97 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 0fc1188..736a81e 100644 --- a/README.md +++ b/README.md @@ -67,11 +67,15 @@ CHECK STORAGE - 主循环异常计数 - I²C相关异常计数基础字段 -### 4. RGB 告警引擎 +### 4. Active Alert、OLED 与 RGB -RGB 与 OLED 页面完全解耦，只使用所有 active alert 的最高 severity。Warning 为黄色 3 秒呼吸，Critical 为橙红色 1.5 秒呼吸，Emergency 为纯红快速频闪；无 active alert 时 OFF。动画不写循环日志，模式切换仅记录一次 `RGB_MODE`。 +每个当前故障都由统一的 Active Alert 管理，包含稳定 ID、severity、OLED 显示数据、首次发现时间、最后变化时间和元数据。监控只更新 Alert；OLED 为每个 active alert 保留独立轮播页，RGB 只读取最高 severity，Journal 只记录状态转换。历史 power flags 只进入 diagnose 和一次性 Journal 记录，绝不产生 OLED/RGB 告警。 -### 5. 结构化事件日志 +### 5. RGB 告警引擎 + +RGB 与 OLED 页面完全解耦，只使用所有 active alert 的最高 severity。Warning 为黄色 3 秒呼吸，Critical 为橙红色 1.5 秒呼吸，Emergency 为纯红快速频闪；无 active alert 时 OFF。呼吸最低亮度为 5%，默认每 80ms 更新。动画不写循环日志，模式切换仅记录一次 `RGB_MODE`。 + +### 6. 结构化事件日志 日志继续交给 systemd journal，不额外制造长期 `.log` 文件。 @@ -108,7 +112,7 @@ journalctl -u pigway-pi-control -b --no-pager journalctl -u pigway-pi-control --no-pager | grep 'event=POWER' ``` -### 6. 诊断命令 +### 7. 诊断命令 ```bash sudo /usr/local/sbin/pigway-pi-control --diagnose @@ -120,7 +124,7 @@ sudo /usr/local/sbin/pigway-pi-control --diagnose RC2 保留 v3.2.0 的 RGB MCU 寄存器写入顺序；独立闪烁时钟使用所有有效告警的最高 severity，不受 OLED 页面及网络/watch 检查等待影响。服务恢复后自动切换到下一有效告警，没有告警时 OFF。 -### 7. 关机收尾 +### 8. 关机收尾 收到 systemd SIGTERM / 系统关机时： diff --git a/SHA256SUMS b/SHA256SUMS index 540855b..28d8f7c 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -1,7 +1,7 @@ -55fe2917b92a896fb346fb3c9c515fafcf47d7ea095c0d92dfea66a7b3edfc19 README.md +e3dea3247111f46f87a8b62b020124639e4e7a1911d6c21534694d3c32e41e15 README.md 72c46162c33f9587c6ccb01ad53fbaaf4ecec8cd21014b909e87e8707e253e36 app/oled_font_5x7.bin -75138ce957a501dc34bf2d886476b92e897d16d42c06ccbffda27524f164cce4 app/pigway_pi_control.py -fbd8829929bc755cef6beeded475ac5eaf1a5b9152b4294bb8d169e224e3bc9c config/pigway-pi-control.conf +388d1f7c3a329af6580945227ae4c601f97ce30f2788ccadc14b81644f300e25 app/pigway_pi_control.py +16649f17170d8aea354387e2d37a83398ccb661e4f7666a3d0e47f18f189ea8d config/pigway-pi-control.conf 4c2816da08378f2927c746b70ecfa7d5312e1730c08727cd89e8c2801e707054 install.sh 81aea0a89b298b2512fcb9bf6418618ba7a66474d425390cf95a0128c4093474 systemd/pigway-pi-control.service a3d74973cafbabaf58a660a8c83d28347d1c06d9fd2dab3ecfce5b7607f6fee2 uninstall.sh diff --git a/app/pigway_pi_control.py b/app/pigway_pi_control.py index cc7ade8..48f8d0d 100755 --- a/app/pigway_pi_control.py +++ b/app/pigway_pi_control.py @@ -262,7 +262,7 @@ LED_BRIGHT=cf("led","brightness",10,int) WARNING_BREATHE_SECONDS=cf("led","warning_breathe_seconds",3.0) CRITICAL_BREATHE_SECONDS=cf("led","critical_breathe_seconds",1.5) EMERGENCY_FLASH_MS=cf("led","emergency_flash_ms",300,int) -RGB_UPDATE_MS=cf("led","rgb_update_ms",40,int) +RGB_UPDATE_MS=cf("led","rgb_update_ms",80,int) RGB_WARNING=(cf("led","warning_r",255,int),cf("led","warning_g",255,int),cf("led","warning_b",0,int)) RGB_CRITICAL=(cf("led","critical_r",255,int),cf("led","critical_g",72,int),cf("led","critical_b",0,int)) RGB_EMERGENCY=(cf("led","emergency_r",255,int),cf("led","emergency_g",0,int),cf("led","emergency_b",0,int)) @@ -317,28 +317,76 @@ def alert(title,l2="",l3="",l4="",sev=1,color=(20,16,0),blink=0): return {"title":title[:21],"l2":l2[:21],"l3":l3[:21],"l4":l4[:21], "sev":sev,"color":color,"blink":blink} +class AlertManager: + def __init__(self): + self._active={} + + def set_alert(self,alert_id,severity,title,l2="",l3="",l4="",metadata=None): + now=time.monotonic() + display={"id":alert_id,"title":title[:21],"l2":l2[:21],"l3":l3[:21],"l4":l4[:21], + "sev":severity,"metadata":metadata or {}} + current=self._active.get(alert_id) + if current is None: + display["active"]=True; display["first_seen"]=now; display["last_changed"]=now + self._active[alert_id]=display + log("WARN","ALERT_ACTIVE",alert=alert_id,severity=severity,detail=l2) + else: + changed=any(current.get(k)!=display.get(k) for k in ("sev","title","l2","l3","l4","metadata")) + current.update(display) + if changed: current["last_changed"]=now + + def clear_alert(self,alert_id): + if alert_id in self._active: + self._active.pop(alert_id) + log("INFO","ALERT_RECOVERED",alert=alert_id) + + def active_alerts(self): + return sorted(self._active.values(),key=lambda a:(-a["sev"],a["first_seen"],a["id"])) + def transition(key,active,on_event,off_event,**fields): old=event_state.get(key,False) if active!=old: event_state[key]=active log("WARN" if active else "INFO",on_event if active else off_event,**fields) -def make_alerts(cpu,temp,mem,disk,sample_elapsed=0.0): +def update_active_alerts(cpu,temp,mem,disk,sample_elapsed=0.0): global wifi_weak_since,wifi_weak_active global cpu_hi,mem_hi - a=[] cpu_hi=cpu_hi+sample_elapsed if cpu>=CPU_HIGH else 0 if cpu=MEM_HIGH else 0 if mem=CPU_HIGH_SEC:a.append(alert("CPU LOAD HIGH",f"LOAD {cpu:.1f}%",f"TEMP {temp:.1f}C","FOR >=30 SEC",1,(24,18,0))) - if temp>=TEMP_CRIT:a.append(alert("TEMP CRITICAL",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING",3,(32,0,0),2)) - elif temp>=TEMP_HIGH:a.append(alert("TEMP HIGH",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING",1,(24,0,0))) - if mem_hi>=MEM_HIGH_SEC:a.append(alert("MEMORY HIGH",f"USED {mem:.1f}%","CHECK PROCESS","",1,(18,0,24))) - if disk>=DISK_CRIT:a.append(alert("DISK CRITICAL",f"USED {disk:.1f}%","FREE < 5%","CHECK STORAGE",2,(28,6,0),2)) - elif disk>=DISK_HIGH:a.append(alert("DISK SPACE LOW",f"USED {disk:.1f}%","FREE < 10%","CHECK STORAGE",1,(28,8,0))) - if not link_ok:a.append(alert("NETWORK DOWN","NO LAN / WIFI","NO IP ADDRESS","",2,(0,8,28),1)) - elif not net_ok:a.append(alert("INTERNET DOWN","LOCAL LINK OK",ip,"",2,(0,8,28),1)) + if cpu_hi>=CPU_HIGH_SEC: alerts.set_alert("CPU_HIGH",1,"CPU LOAD HIGH",f"LOAD {cpu:.1f}%",f"TEMP {temp:.1f}C","CHECK LOAD") + else: alerts.clear_alert("CPU_HIGH") + if temp>=TEMP_CRIT: + alerts.set_alert("TEMP_CRITICAL",3,"TEMP CRITICAL",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING") + alerts.clear_alert("TEMP_HIGH") + elif temp>=TEMP_HIGH: + alerts.set_alert("TEMP_HIGH",1,"TEMP HIGH",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING") + alerts.clear_alert("TEMP_CRITICAL") + else: + alerts.clear_alert("TEMP_HIGH"); alerts.clear_alert("TEMP_CRITICAL") + if mem_hi>=MEM_HIGH_SEC: alerts.set_alert("MEMORY_HIGH",1,"MEMORY HIGH",f"USED {mem:.1f}%","CHECK PROCESS","") + else: alerts.clear_alert("MEMORY_HIGH") + if disk>=DISK_CRIT: + alerts.set_alert("DISK_CRITICAL",2,"DISK CRITICAL",f"USED {disk:.1f}%","FREE < 5%","CHECK STORAGE") + alerts.clear_alert("DISK_HIGH") + elif disk>=DISK_HIGH: + alerts.set_alert("DISK_HIGH",1,"DISK SPACE LOW",f"USED {disk:.1f}%","FREE < 10%","CHECK STORAGE") + alerts.clear_alert("DISK_CRITICAL") + else: + alerts.clear_alert("DISK_HIGH"); alerts.clear_alert("DISK_CRITICAL") + if not link_ok: + alerts.set_alert("NETWORK_DOWN",2,"NETWORK DOWN","NO LAN / WIFI","NO IP ADDRESS","") + alerts.clear_alert("INTERNET_DOWN") + elif not net_ok: + alerts.set_alert("INTERNET_DOWN",2,"INTERNET DOWN","LOCAL LINK OK",ip,"") + alerts.clear_alert("NETWORK_DOWN") + else: + alerts.clear_alert("NETWORK_DOWN"); alerts.clear_alert("INTERNET_DOWN") for w,ok in zip(watches,watch_state): - if not ok:a.append(alert("SERVICE DOWN" if w[0]=="service" else "PROCESS DOWN",w[1],"CHECK / RESTART","",2,(0,24,24),1)) + alert_id=("SERVICE_DOWN" if w[0]=="service" else "PROCESS_DOWN")+"|"+w[1] + if not ok: + alerts.set_alert(alert_id,2,"SERVICE DOWN" if w[0]=="service" else "PROCESS DOWN",w[1],"CHECK / RESTART","CRITICAL",{"target":w[2]}) + else: alerts.clear_alert(alert_id) if net_kind=="WIF" and str(net_metric).lstrip("-").isdigit(): rssi=float(net_metric) if not wifi_weak_active: @@ -349,17 +397,24 @@ def make_alerts(cpu,temp,mem,disk,sample_elapsed=0.0): elif rssi >= WIFI_RECOVER: wifi_weak_active=False; wifi_weak_since=None if wifi_weak_active: - a.append(alert("WIFI SIGNAL WEAK",f"{rssi:.0f} dBm",f"IF {net_iface}","CHECK SIGNAL",1,(0,0,28),1)) + alerts.set_alert("WIFI_WEAK",1,"WIFI SIGNAL WEAK",f"RSSI {rssi:.0f} dBm",f"IF {net_iface}","CHECK SIGNAL") + else: alerts.clear_alert("WIFI_WEAK") else: - wifi_weak_since=None; wifi_weak_active=False + wifi_weak_since=None; wifi_weak_active=False; alerts.clear_alert("WIFI_WEAK") if health.get("under_voltage_now"): - a.append(alert("POWER LOW","UNDERVOLT",health.get("raw",""),"CHECK 5V",3,(28,18,0),2)) - if health.get("throttled_now") or health.get("freq_capped_now"): - a.append(alert("CPU THROTTLED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"TEMP / POWER",2,(28,18,0),1)) - if fs_readonly: - a.append(alert("FILESYSTEM RO","ROOT READ ONLY","CHECK STORAGE","",3,(28,6,0),2)) - a.sort(key=lambda x:x["sev"],reverse=True) - return a + alerts.set_alert("POWER_UNDERVOLT",2,"POWER LOW","UNDERVOLTAGE","CHECK POWER","CRITICAL",{"raw":health.get("raw","")}) + else: alerts.clear_alert("POWER_UNDERVOLT") + if health.get("throttled_now"): + alerts.set_alert("THROTTLED",2,"CPU THROTTLED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"CHECK POWER") + else: alerts.clear_alert("THROTTLED") + if health.get("freq_capped_now"): + alerts.set_alert("FREQ_CAPPED",2,"FREQ CAPPED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"CHECK POWER") + else: alerts.clear_alert("FREQ_CAPPED") + if health.get("soft_temp_limit_now"): + alerts.set_alert("SOFT_TEMP_LIMIT",2,"SOFT TEMP LIMIT","CPU TEMP LIMITED",health.get("raw",""),"CHECK COOLING") + else: alerts.clear_alert("SOFT_TEMP_LIMIT") + if fs_readonly: alerts.set_alert("ROOT_READONLY",3,"FILESYSTEM RO","ROOT READ ONLY","CHECK STORAGE","EMERGENCY") + else: alerts.clear_alert("ROOT_READONLY") def rgb_mode_for(a): if not a: return "OFF" @@ -370,7 +425,7 @@ def rgb_mode_for(a): def breathe(level_seconds,elapsed): # A full sine-derived ease in/out cycle gives continuous light transitions. phase=(elapsed/max(0.1,level_seconds))%1.0 - return 0.5-0.5*math.cos(2.0*math.pi*phase) + return 0.05+0.95*(0.5-0.5*math.cos(2.0*math.pi*phase)) def rgb_engine(mode,elapsed): if mode=="OFF": @@ -426,8 +481,7 @@ def render_alert(a,idx,total): def alert_key(a): - # Sensor values change without changing the identity of a fault. - return a["title"]+("|"+a["l2"] if a["title"] in ("SERVICE DOWN","PROCESS DOWN") else "") + return a["id"] def find_live_alert(key, live): for a in live: @@ -442,15 +496,15 @@ led_alert=None def led_worker(): previous_error=None current_mode="OFF" - mode_started=time.monotonic() + animation_started=time.monotonic() while not led_stop.is_set(): try: new_mode=rgb_mode_for(led_alert) if new_mode!=current_mode: log("INFO","RGB_MODE",old=current_mode,new=new_mode) - current_mode=new_mode; mode_started=time.monotonic() + current_mode=new_mode # Keep the verified five-write MCU RGB transaction contiguous. - with bus_lock: rgb_engine(current_mode,time.monotonic()-mode_started) + with bus_lock: rgb_engine(current_mode,time.monotonic()-animation_started) if previous_error is not None: log("INFO","RGB_RECOVERED") previous_error=None except Exception as e: @@ -481,9 +535,9 @@ last_sensor=last_render=0.0 last_net=last_watch=last_health=0.0 health={}; fs_readonly=False agent={"last_sensor_ok":0.0,"last_network_ok":0.0,"last_watch_ok":0.0,"last_health_ok":0.0,"loop_errors":0,"i2c_errors":0} +alerts=AlertManager() event_state={} history_logged=set() -prev_alert_keys=set() latest=(0.0,0.0,0.0,0.0) active=[] @@ -544,16 +598,9 @@ try: agent["last_health_ok"]=now; last_health=now c,t,m,d=latest - active=make_alerts(c,t,m,d,sample_elapsed) + update_active_alerts(c,t,m,d,sample_elapsed) + active=alerts.active_alerts() led_alert=max(active,key=lambda a:a["sev"],default=None) - new_alert_keys={alert_key(a) for a in active} - for a in active: - k=alert_key(a) - if k not in prev_alert_keys: - log("WARN","ALERT_ACTIVE",alert=a["title"],detail=a["l2"],severity=a["sev"]) - for k in prev_alert_keys-new_alert_keys: - log("INFO","ALERT_RECOVERED",alert=k) - prev_alert_keys=new_alert_keys # If the currently displayed alert truly disappeared, skip it immediately. if cycle_pages[cycle_pos]!="HOME": diff --git a/config/pigway-pi-control.conf b/config/pigway-pi-control.conf index d13d20c..17546d1 100644 --- a/config/pigway-pi-control.conf +++ b/config/pigway-pi-control.conf @@ -154,7 +154,7 @@ brightness = 10 warning_breathe_seconds = 3.0 critical_breathe_seconds = 1.5 emergency_flash_ms = 300 -rgb_update_ms = 40 +rgb_update_ms = 80 # RGB 基色（0~255）。安全默认值严格不含蓝色，避免紫色/蓝色残留。 warning_r = 255