feat: add P0 system health monitoring

app/pigway_pi_control.py

@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
-import re
-import os,time,socket,subprocess,configparser,signal
+import os,time,socket,subprocess,configparser,signal,re,sys,json
 from pathlib import Path
-APP_VERSION="3.2.0"
+APP_VERSION="3.3.0"
 from smbus2 import SMBus
 from PIL import Image
 from shutil import disk_usage
@@ -15,6 +14,47 @@ def stop(*_):
     global running; running=False
 signal.signal(signal.SIGTERM,stop); signal.signal(signal.SIGINT,stop)
 
+def log(level,event,**fields):
+    parts=[f"level={level}",f"event={event}"]
+    for k,v in fields.items(): parts.append(f"{k}={str(v).replace(chr(32),'_')}")
+    print(" ".join(parts),flush=True)
+
+def uptime_seconds():
+    try:return int(float(Path("/proc/uptime").read_text().split()[0]))
+    except Exception:return 0
+
+def load_average():
+    try:
+        x=Path("/proc/loadavg").read_text().split();return (float(x[0]),float(x[1]),float(x[2]))
+    except Exception:return (0.0,0.0,0.0)
+
+def cpu_freq_mhz():
+    for q in ("/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq","/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq"):
+        try:return int(Path(q).read_text().strip())/1000.0
+        except Exception:pass
+    return 0.0
+
+def root_readonly():
+    try:
+        for line in Path("/proc/mounts").read_text().splitlines():
+            a=line.split()
+            if len(a)>=4 and a[1]=="/":return "ro" in a[3].split(",")
+    except Exception:pass
+    return False
+
+def throttled_state():
+    raw=_run(["vcgencmd","get_throttled"])
+    m=re.search(r"0x([0-9a-fA-F]+)",raw);v=int(m.group(1),16) if m else 0
+    return {"raw":f"0x{v:x}","under_voltage_now":bool(v&1),"freq_capped_now":bool(v&2),"throttled_now":bool(v&4),"soft_temp_now":bool(v&8),"under_voltage_occurred":bool(v&(1<<16)),"freq_capped_occurred":bool(v&(1<<17)),"throttled_occurred":bool(v&(1<<18)),"soft_temp_occurred":bool(v&(1<<19))}
+
+def diagnose():
+    print(f"Pigway Pi Control {APP_VERSION} diagnostic")
+    print("uptime_seconds=",uptime_seconds());print("load_average=",load_average());print("cpu_freq_mhz=",round(cpu_freq_mhz()))
+    print("root_readonly=",root_readonly());print("throttled=",json.dumps(throttled_state()))
+    print("network=",primary_network());print("internet=",internet())
+    out=_run(["i2cdetect","-y","1"],4) if False else ""
+    return 0
+
 bus=SMBus(1)
 FONT_PATH="/usr/local/share/pigway-pi-control/oled_font_5x7.bin"
 FONT5=Path(FONT_PATH).read_bytes()
@@ -135,8 +175,8 @@ def mem_pct():
     return 100*(vals["MemTotal"]-vals["MemAvailable"])/vals["MemTotal"]
 def disk_pct():
     d=disk_usage("/"); return 100*d.used/d.total
-def _run(args):
-    try: return subprocess.check_output(args,text=True,stderr=subprocess.DEVNULL,timeout=2).strip()
+def _run(args,timeout=2):
+    try: return subprocess.check_output(args,text=True,stderr=subprocess.DEVNULL,timeout=timeout).strip()
     except Exception: return ""
 
 def primary_network():
@@ -194,6 +234,7 @@ OLED_REFRESH=cf("timing","oled_refresh_interval",1.0)
 PAGE_INTERVAL=cf("timing","page_interval",10.0)
 NETWORK_INTERVAL=cf("timing","network_interval",5.0)
 SERVICE_INTERVAL=cf("timing","service_interval",5.0)
+HEALTH_INTERVAL=cf("timing","health_interval",5.0)
 
 CPU_HIGH=cf("alerts","cpu_high",90.0)
 CPU_RECOVER=cf("alerts","cpu_recover",75.0)
@@ -232,6 +273,12 @@ def alert(title,l2="",l3="",l4="",sev=1,color=(20,16,0),blink=0):
     return {"title":title[:21],"l2":l2[:21],"l3":l3[:21],"l4":l4[:21],
             "sev":sev,"color":color,"blink":blink}
 
+def transition(key,active,on_event,off_event,**fields):
+    old=event_state.get(key,False)
+    if active!=old:
+        event_state[key]=active
+        log("WARN" if active else "INFO",on_event if active else off_event,**fields)
+
 def make_alerts(cpu,temp,mem,disk):
     global wifi_weak_since,wifi_weak_active
     global cpu_hi,mem_hi
@@ -247,7 +294,7 @@ def make_alerts(cpu,temp,mem,disk):
     if not link_ok:a.append(alert("NETWORK DOWN","NO LAN / WIFI","NO IP ADDRESS","",2,(0,8,28),1))
     elif not net_ok:a.append(alert("INTERNET DOWN","LOCAL LINK OK",ip,"",2,(0,8,28),1))
     for w,ok in zip(watches,watch_state):
-        if not ok:a.append(alert("SERVICE DOWN",w[1],"CHECK / RESTART","",2,(0,24,24),1))
+        if not ok:a.append(alert("SERVICE DOWN" if w[0]=="service" else "PROCESS DOWN",w[1],"CHECK / RESTART","",2,(0,24,24),1))
     if net_kind=="WIF" and str(net_metric).lstrip("-").isdigit():
         rssi=float(net_metric)
         if not wifi_weak_active:
@@ -261,6 +308,12 @@ def make_alerts(cpu,temp,mem,disk):
             a.append(alert("WIFI SIGNAL WEAK",f"{rssi:.0f} dBm",f"IF {net_iface}","CHECK SIGNAL",1,(0,0,28),1))
     else:
         wifi_weak_since=None; wifi_weak_active=False
+    if health.get("under_voltage_now"):
+        a.append(alert("POWER LOW","UNDERVOLT",health.get("raw",""),"CHECK 5V",3,(28,18,0),2))
+    if health.get("throttled_now") or health.get("freq_capped_now"):
+        a.append(alert("CPU THROTTLED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"TEMP / POWER",2,(28,18,0),1))
+    if fs_readonly:
+        a.append(alert("FILESYSTEM RO","ROOT READ ONLY","CHECK STORAGE","",3,(28,6,0),2))
     a.sort(key=lambda x:x["sev"],reverse=True)
     return a
 
@@ -314,10 +367,20 @@ def render_alert(a,idx,total):
                (a["l2"],""),(a["l3"],""),(a["l4"],"")])
 
 
-oled_init(); rgb_off(); apply_dark_mode()
+if "--diagnose" in sys.argv:
+    raise SystemExit(diagnose())
+
+oled_init(); rgb_off(); apply_dark_mode(); log("INFO","START",version=APP_VERSION,uptime=uptime_seconds())
 watch_state=[check_watch(w) for w in watches]
 last_sensor=last_render=0.0
-last_net=last_watch=time.monotonic()
+last_net=last_watch=last_health=time.monotonic()
+health=throttled_state(); fs_readonly=root_readonly()
+agent={"last_sensor_ok":0.0,"last_network_ok":0.0,"last_watch_ok":0.0,"last_health_ok":0.0,"loop_errors":0,"i2c_errors":0}
+event_state={}
+prev_net_identity=None
+prev_watch_map={w[1]:ok for w,ok in zip(watches,watch_state)}
+history_logged=set()
+prev_alert_keys=set()
 latest=(0.0,0.0,0.0,0.0)
 active=[]
 
@@ -345,18 +408,45 @@ while running:
         # Independent clocks: update data/state only.
         if now-last_sensor >= SENSOR_INTERVAL:
             c=cpu_pct(); t=temp_c(); m=mem_pct(); d=disk_pct()
-            latest=(c,t,m,d); update_fan(t); last_sensor=now
+            latest=(c,t,m,d); update_fan(t); last_sensor=now; agent["last_sensor_ok"]=now
 
         if now-last_net >= NETWORK_INTERVAL:
+            old_id=(net_iface,ip)
             net_iface,net_kind,net_metric,ip_label,ip=primary_network(); link_ok=ip!="NO IP"; net_ok=internet() if link_ok else False
-            last_net=now
+            new_id=(net_iface,ip)
+            if old_id!=("","NO IP") and new_id!=old_id:
+                log("INFO","NET_SWITCH",old_if=old_id[0],old_ip=old_id[1],new_if=net_iface,new_ip=ip)
+            transition("network_down",not link_ok,"NETWORK_DOWN","NETWORK_RECOVERED",iface=net_iface,ip=ip)
+            transition("internet_down",link_ok and not net_ok,"INTERNET_DOWN","INTERNET_RECOVERED",iface=net_iface,ip=ip)
+            last_net=now; agent["last_network_ok"]=now
 
         if now-last_watch >= SERVICE_INTERVAL:
             watch_state=[check_watch(w) for w in watches]
-            last_watch=now
+            for w,ok in zip(watches,watch_state):
+                transition("watch:"+w[1],not ok,"WATCH_DOWN","WATCH_RECOVERED",name=w[1],target=w[2])
+            last_watch=now; agent["last_watch_ok"]=now
+
+        if now-last_health >= HEALTH_INTERVAL:
+            health=throttled_state(); fs_readonly=root_readonly()
+            transition("undervoltage",health.get("under_voltage_now",False),"POWER_LOW","POWER_RECOVERED",raw=health.get("raw"))
+            transition("throttled",health.get("throttled_now",False) or health.get("freq_capped_now",False),"THROTTLED","THROTTLE_RECOVERED",raw=health.get("raw"),freq_mhz=f"{cpu_freq_mhz():.0f}")
+            transition("root_ro",fs_readonly,"FILESYSTEM_RO","FILESYSTEM_RW")
+            for k in ("under_voltage_occurred","freq_capped_occurred","throttled_occurred","soft_temp_occurred"):
+                if health.get(k) and k not in history_logged:
+                    history_logged.add(k); log("WARN","HISTORY_FLAG",flag=k,raw=health.get("raw"))
+            la=load_average()
+            agent["last_health_ok"]=now; last_health=now
 
         c,t,m,d=latest
         active=make_alerts(c,t,m,d)
+        new_alert_keys={alert_key(a) for a in active}
+        for a in active:
+            k=alert_key(a)
+            if k not in prev_alert_keys:
+                log("WARN","ALERT_ACTIVE",alert=a["title"],detail=a["l2"],severity=a["sev"])
+        for k in prev_alert_keys-new_alert_keys:
+            log("INFO","ALERT_RECOVERED",alert=k)
+        prev_alert_keys=new_alert_keys
 
         # If the currently displayed alert truly disappeared, skip it immediately.
         if cycle_pages[cycle_pos]!="HOME":
@@ -408,6 +498,12 @@ while running:
 
         time.sleep(0.20)
     except Exception as e:
-        print("loop error:",e,flush=True); time.sleep(1)
+        agent["loop_errors"]+=1; log("ERROR","LOOP_ERROR",count=agent["loop_errors"],error=repr(e)); time.sleep(1)
 
-rgb_off(); mcu(0x08,0x00); bus.close()
+log("INFO","STOP",reason="signal")
+try:
+    rgb_off(); mcu(0x08,0x00)
+    oled_show([("",""),("",""),("",""),("","")]); cmd(0xAE)
+except Exception as e: log("ERROR","SHUTDOWN_CLEANUP",error=repr(e))
+try: bus.close()
+except Exception: pass