way/pigway-device-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

fix: unify active alert and RGB behavior

Browse Source
This commit is contained in:
way
2026-09-26 22:35:23 +08:00
parent d3ccca4b37
commit 34f334dd23
4 changed files with 97 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+9 -5
View File
@@ -67,11 +67,15 @@ CHECK STORAGE
- 主循环异常计数
- I²C相关异常计数基础字段
### 4. RGB 告警引擎
### 4. Active Alert、OLED 与 RGB
RGB 与 OLED 页面完全解耦，只使用所有 active alert 的最高 severity。Warning 为黄色 3 秒呼吸，Critical 为橙红色 1.5 秒呼吸，Emergency 为纯红快速频闪；无 active alert 时 OFF。动画不写循环日志，模式切换仅记录一次 `RGB_MODE`。
每个当前故障都由统一的 Active Alert 管理，包含稳定 ID、severity、OLED 显示数据、首次发现时间、最后变化时间和元数据。监控只更新 Alert；OLED 为每个 active alert 保留独立轮播页，RGB 只读取最高 severity，Journal 只记录状态转换。历史 power flags 只进入 diagnose 和一次性 Journal 记录，绝不产生 OLED/RGB 告警。
### 5. 结构化事件日志
### 5. RGB 告警引擎
RGB 与 OLED 页面完全解耦，只使用所有 active alert 的最高 severity。Warning 为黄色 3 秒呼吸，Critical 为橙红色 1.5 秒呼吸，Emergency 为纯红快速频闪；无 active alert 时 OFF。呼吸最低亮度为 5%，默认每 80ms 更新。动画不写循环日志，模式切换仅记录一次 `RGB_MODE`。
### 6. 结构化事件日志
日志继续交给 systemd journal，不额外制造长期 `.log` 文件。
@@ -108,7 +112,7 @@ journalctl -u pigway-pi-control -b --no-pager
journalctl -u pigway-pi-control --no-pager | grep 'event=POWER'
```
### 6. 诊断命令
### 7. 诊断命令
```bash
sudo /usr/local/sbin/pigway-pi-control --diagnose
@@ -120,7 +124,7 @@ sudo /usr/local/sbin/pigway-pi-control --diagnose
RC2 保留 v3.2.0 的 RGB MCU 寄存器写入顺序；独立闪烁时钟使用所有有效告警的最高 severity，不受 OLED 页面及网络/watch 检查等待影响。服务恢复后自动切换到下一有效告警，没有告警时 OFF。
### 7. 关机收尾
### 8. 关机收尾
收到 systemd SIGTERM / 系统关机时：
SHA256SUMS
+3 -3
View File
@@ -1,7 +1,7 @@
55fe2917b92a896fb346fb3c9c515fafcf47d7ea095c0d92dfea66a7b3edfc19 README.md
e3dea3247111f46f87a8b62b020124639e4e7a1911d6c21534694d3c32e41e15 README.md
72c46162c33f9587c6ccb01ad53fbaaf4ecec8cd21014b909e87e8707e253e36 app/oled_font_5x7.bin
75138ce957a501dc34bf2d886476b92e897d16d42c06ccbffda27524f164cce4 app/pigway_pi_control.py
fbd8829929bc755cef6beeded475ac5eaf1a5b9152b4294bb8d169e224e3bc9c config/pigway-pi-control.conf
388d1f7c3a329af6580945227ae4c601f97ce30f2788ccadc14b81644f300e25 app/pigway_pi_control.py
16649f17170d8aea354387e2d37a83398ccb661e4f7666a3d0e47f18f189ea8d config/pigway-pi-control.conf
4c2816da08378f2927c746b70ecfa7d5312e1730c08727cd89e8c2801e707054 install.sh
81aea0a89b298b2512fcb9bf6418618ba7a66474d425390cf95a0128c4093474 systemd/pigway-pi-control.service
a3d74973cafbabaf58a660a8c83d28347d1c06d9fd2dab3ecfce5b7607f6fee2 uninstall.sh
app/pigway_pi_control.py
+84 -37
View File
@@ -262,7 +262,7 @@ LED_BRIGHT=cf("led","brightness",10,int)
WARNING_BREATHE_SECONDS=cf("led","warning_breathe_seconds",3.0)
CRITICAL_BREATHE_SECONDS=cf("led","critical_breathe_seconds",1.5)
EMERGENCY_FLASH_MS=cf("led","emergency_flash_ms",300,int)
RGB_UPDATE_MS=cf("led","rgb_update_ms",40,int)
RGB_UPDATE_MS=cf("led","rgb_update_ms",80,int)
RGB_WARNING=(cf("led","warning_r",255,int),cf("led","warning_g",255,int),cf("led","warning_b",0,int))
RGB_CRITICAL=(cf("led","critical_r",255,int),cf("led","critical_g",72,int),cf("led","critical_b",0,int))
RGB_EMERGENCY=(cf("led","emergency_r",255,int),cf("led","emergency_g",0,int),cf("led","emergency_b",0,int))
@@ -317,28 +317,76 @@ def alert(title,l2="",l3="",l4="",sev=1,color=(20,16,0),blink=0):
return {"title":title[:21],"l2":l2[:21],"l3":l3[:21],"l4":l4[:21],
"sev":sev,"color":color,"blink":blink}
class AlertManager:
def __init__(self):
self._active={}
def set_alert(self,alert_id,severity,title,l2="",l3="",l4="",metadata=None):
now=time.monotonic()
display={"id":alert_id,"title":title[:21],"l2":l2[:21],"l3":l3[:21],"l4":l4[:21],
"sev":severity,"metadata":metadata or {}}
current=self._active.get(alert_id)
if current is None:
display["active"]=True; display["first_seen"]=now; display["last_changed"]=now
self._active[alert_id]=display
log("WARN","ALERT_ACTIVE",alert=alert_id,severity=severity,detail=l2)
else:
changed=any(current.get(k)!=display.get(k) for k in ("sev","title","l2","l3","l4","metadata"))
current.update(display)
if changed: current["last_changed"]=now
def clear_alert(self,alert_id):
if alert_id in self._active:
self._active.pop(alert_id)
log("INFO","ALERT_RECOVERED",alert=alert_id)
def active_alerts(self):
return sorted(self._active.values(),key=lambda a:(-a["sev"],a["first_seen"],a["id"]))
def transition(key,active,on_event,off_event,**fields):
old=event_state.get(key,False)
if active!=old:
event_state[key]=active
log("WARN" if active else "INFO",on_event if active else off_event,**fields)
def make_alerts(cpu,temp,mem,disk,sample_elapsed=0.0):
def update_active_alerts(cpu,temp,mem,disk,sample_elapsed=0.0):
global wifi_weak_since,wifi_weak_active
global cpu_hi,mem_hi
a=[]
cpu_hi=cpu_hi+sample_elapsed if cpu>=CPU_HIGH else 0 if cpu<CPU_RECOVER else cpu_hi
mem_hi=mem_hi+sample_elapsed if mem>=MEM_HIGH else 0 if mem<MEM_RECOVER else mem_hi
if cpu_hi>=CPU_HIGH_SEC:a.append(alert("CPU LOAD HIGH",f"LOAD {cpu:.1f}%",f"TEMP {temp:.1f}C","FOR >=30 SEC",1,(24,18,0)))
if temp>=TEMP_CRIT:a.append(alert("TEMP CRITICAL",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING",3,(32,0,0),2))
elif temp>=TEMP_HIGH:a.append(alert("TEMP HIGH",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING",1,(24,0,0)))
if mem_hi>=MEM_HIGH_SEC:a.append(alert("MEMORY HIGH",f"USED {mem:.1f}%","CHECK PROCESS","",1,(18,0,24)))
if disk>=DISK_CRIT:a.append(alert("DISK CRITICAL",f"USED {disk:.1f}%","FREE < 5%","CHECK STORAGE",2,(28,6,0),2))
elif disk>=DISK_HIGH:a.append(alert("DISK SPACE LOW",f"USED {disk:.1f}%","FREE < 10%","CHECK STORAGE",1,(28,8,0)))
if not link_ok:a.append(alert("NETWORK DOWN","NO LAN / WIFI","NO IP ADDRESS","",2,(0,8,28),1))
elif not net_ok:a.append(alert("INTERNET DOWN","LOCAL LINK OK",ip,"",2,(0,8,28),1))
if cpu_hi>=CPU_HIGH_SEC: alerts.set_alert("CPU_HIGH",1,"CPU LOAD HIGH",f"LOAD {cpu:.1f}%",f"TEMP {temp:.1f}C","CHECK LOAD")
else: alerts.clear_alert("CPU_HIGH")
if temp>=TEMP_CRIT:
alerts.set_alert("TEMP_CRITICAL",3,"TEMP CRITICAL",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING")
alerts.clear_alert("TEMP_HIGH")
elif temp>=TEMP_HIGH:
alerts.set_alert("TEMP_HIGH",1,"TEMP HIGH",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING")
alerts.clear_alert("TEMP_CRITICAL")
else:
alerts.clear_alert("TEMP_HIGH"); alerts.clear_alert("TEMP_CRITICAL")
if mem_hi>=MEM_HIGH_SEC: alerts.set_alert("MEMORY_HIGH",1,"MEMORY HIGH",f"USED {mem:.1f}%","CHECK PROCESS","")
else: alerts.clear_alert("MEMORY_HIGH")
if disk>=DISK_CRIT:
alerts.set_alert("DISK_CRITICAL",2,"DISK CRITICAL",f"USED {disk:.1f}%","FREE < 5%","CHECK STORAGE")
alerts.clear_alert("DISK_HIGH")
elif disk>=DISK_HIGH:
alerts.set_alert("DISK_HIGH",1,"DISK SPACE LOW",f"USED {disk:.1f}%","FREE < 10%","CHECK STORAGE")
alerts.clear_alert("DISK_CRITICAL")
else:
alerts.clear_alert("DISK_HIGH"); alerts.clear_alert("DISK_CRITICAL")
if not link_ok:
alerts.set_alert("NETWORK_DOWN",2,"NETWORK DOWN","NO LAN / WIFI","NO IP ADDRESS","")
alerts.clear_alert("INTERNET_DOWN")
elif not net_ok:
alerts.set_alert("INTERNET_DOWN",2,"INTERNET DOWN","LOCAL LINK OK",ip,"")
alerts.clear_alert("NETWORK_DOWN")
else:
alerts.clear_alert("NETWORK_DOWN"); alerts.clear_alert("INTERNET_DOWN")
for w,ok in zip(watches,watch_state):
if not ok:a.append(alert("SERVICE DOWN" if w[0]=="service" else "PROCESS DOWN",w[1],"CHECK / RESTART","",2,(0,24,24),1))
alert_id=("SERVICE_DOWN" if w[0]=="service" else "PROCESS_DOWN")+"|"+w[1]
if not ok:
alerts.set_alert(alert_id,2,"SERVICE DOWN" if w[0]=="service" else "PROCESS DOWN",w[1],"CHECK / RESTART","CRITICAL",{"target":w[2]})
else: alerts.clear_alert(alert_id)
if net_kind=="WIF" and str(net_metric).lstrip("-").isdigit():
rssi=float(net_metric)
if not wifi_weak_active:
@@ -349,17 +397,24 @@ def make_alerts(cpu,temp,mem,disk,sample_elapsed=0.0):
elif rssi >= WIFI_RECOVER:
wifi_weak_active=False; wifi_weak_since=None
if wifi_weak_active:
a.append(alert("WIFI SIGNAL WEAK",f"{rssi:.0f} dBm",f"IF {net_iface}","CHECK SIGNAL",1,(0,0,28),1))
alerts.set_alert("WIFI_WEAK",1,"WIFI SIGNAL WEAK",f"RSSI {rssi:.0f} dBm",f"IF {net_iface}","CHECK SIGNAL")
else: alerts.clear_alert("WIFI_WEAK")
else:
wifi_weak_since=None; wifi_weak_active=False
wifi_weak_since=None; wifi_weak_active=False; alerts.clear_alert("WIFI_WEAK")
if health.get("under_voltage_now"):
a.append(alert("POWER LOW","UNDERVOLT",health.get("raw",""),"CHECK 5V",3,(28,18,0),2))
if health.get("throttled_now") or health.get("freq_capped_now"):
a.append(alert("CPU THROTTLED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"TEMP / POWER",2,(28,18,0),1))
if fs_readonly:
a.append(alert("FILESYSTEM RO","ROOT READ ONLY","CHECK STORAGE","",3,(28,6,0),2))
a.sort(key=lambda x:x["sev"],reverse=True)
return a
alerts.set_alert("POWER_UNDERVOLT",2,"POWER LOW","UNDERVOLTAGE","CHECK POWER","CRITICAL",{"raw":health.get("raw","")})
else: alerts.clear_alert("POWER_UNDERVOLT")
if health.get("throttled_now"):
alerts.set_alert("THROTTLED",2,"CPU THROTTLED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"CHECK POWER")
else: alerts.clear_alert("THROTTLED")
if health.get("freq_capped_now"):
alerts.set_alert("FREQ_CAPPED",2,"FREQ CAPPED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"CHECK POWER")
else: alerts.clear_alert("FREQ_CAPPED")
if health.get("soft_temp_limit_now"):
alerts.set_alert("SOFT_TEMP_LIMIT",2,"SOFT TEMP LIMIT","CPU TEMP LIMITED",health.get("raw",""),"CHECK COOLING")
else: alerts.clear_alert("SOFT_TEMP_LIMIT")
if fs_readonly: alerts.set_alert("ROOT_READONLY",3,"FILESYSTEM RO","ROOT READ ONLY","CHECK STORAGE","EMERGENCY")
else: alerts.clear_alert("ROOT_READONLY")
def rgb_mode_for(a):
if not a: return "OFF"
@@ -370,7 +425,7 @@ def rgb_mode_for(a):
def breathe(level_seconds,elapsed):
# A full sine-derived ease in/out cycle gives continuous light transitions.
phase=(elapsed/max(0.1,level_seconds))%1.0
return 0.5-0.5*math.cos(2.0*math.pi*phase)
return 0.05+0.95*(0.5-0.5*math.cos(2.0*math.pi*phase))
def rgb_engine(mode,elapsed):
if mode=="OFF":
@@ -426,8 +481,7 @@ def render_alert(a,idx,total):
def alert_key(a):
# Sensor values change without changing the identity of a fault.
return a["title"]+("|"+a["l2"] if a["title"] in ("SERVICE DOWN","PROCESS DOWN") else "")
return a["id"]
def find_live_alert(key, live):
for a in live:
@@ -442,15 +496,15 @@ led_alert=None
def led_worker():
previous_error=None
current_mode="OFF"
mode_started=time.monotonic()
animation_started=time.monotonic()
while not led_stop.is_set():
try:
new_mode=rgb_mode_for(led_alert)
if new_mode!=current_mode:
log("INFO","RGB_MODE",old=current_mode,new=new_mode)
current_mode=new_mode; mode_started=time.monotonic()
current_mode=new_mode
# Keep the verified five-write MCU RGB transaction contiguous.
with bus_lock: rgb_engine(current_mode,time.monotonic()-mode_started)
with bus_lock: rgb_engine(current_mode,time.monotonic()-animation_started)
if previous_error is not None: log("INFO","RGB_RECOVERED")
previous_error=None
except Exception as e:
@@ -481,9 +535,9 @@ last_sensor=last_render=0.0
last_net=last_watch=last_health=0.0
health={}; fs_readonly=False
agent={"last_sensor_ok":0.0,"last_network_ok":0.0,"last_watch_ok":0.0,"last_health_ok":0.0,"loop_errors":0,"i2c_errors":0}
alerts=AlertManager()
event_state={}
history_logged=set()
prev_alert_keys=set()
latest=(0.0,0.0,0.0,0.0)
active=[]
@@ -544,16 +598,9 @@ try:
agent["last_health_ok"]=now; last_health=now
c,t,m,d=latest
active=make_alerts(c,t,m,d,sample_elapsed)
update_active_alerts(c,t,m,d,sample_elapsed)
active=alerts.active_alerts()
led_alert=max(active,key=lambda a:a["sev"],default=None)
new_alert_keys={alert_key(a) for a in active}
for a in active:
k=alert_key(a)
if k not in prev_alert_keys:
log("WARN","ALERT_ACTIVE",alert=a["title"],detail=a["l2"],severity=a["sev"])
for k in prev_alert_keys-new_alert_keys:
log("INFO","ALERT_RECOVERED",alert=k)
prev_alert_keys=new_alert_keys
# If the currently displayed alert truly disappeared, skip it immediately.
if cycle_pages[cycle_pos]!="HOME":
config/pigway-pi-control.conf
+1 -1
View File
@@ -154,7 +154,7 @@ brightness = 10
warning_breathe_seconds = 3.0
critical_breathe_seconds = 1.5
emergency_flash_ms = 300
rgb_update_ms = 40
rgb_update_ms = 80
# RGB 基色（0~255）。安全默认值严格不含蓝色，避免紫色/蓝色残留。
warning_r = 255