#!/usr/bin/env python3 import time,socket,subprocess,configparser,signal,re,errno,threading,argparse from pathlib import Path APP_VERSION="3.3.0" from smbus2 import SMBus from PIL import Image from shutil import disk_usage OLED=0x3C; MCU=0x0D; W,H=128,32 CFG="/etc/pigway-pi-control.conf" running=True def stop(*_): global running; running=False signal.signal(signal.SIGTERM,stop); signal.signal(signal.SIGINT,stop) def log(level,event,**fields): parts=[f"level={level}",f"event={event}"] for k,v in fields.items(): parts.append(f"{k}={str(v).replace(chr(32),'_')}") print(" ".join(parts),flush=True) def uptime_seconds(): try:return int(float(Path("/proc/uptime").read_text().split()[0])) except Exception:return 0 def load_average(): try: x=Path("/proc/loadavg").read_text().split();return (float(x[0]),float(x[1]),float(x[2])) except Exception:return (0.0,0.0,0.0) def cpu_freq_mhz(): for q in ("/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq","/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq"): try:return int(Path(q).read_text().strip())/1000.0 except Exception:pass return 0.0 def root_readonly(): try: for line in Path("/proc/mounts").read_text().splitlines(): a=line.split() if len(a)>=4 and a[1]=="/":return "ro" in a[3].split(",") except Exception:pass return False def throttled_state(): raw=_run(["vcgencmd","get_throttled"]) m=re.search(r"0x([0-9a-fA-F]+)",raw);v=int(m.group(1),16) if m else 0 if not m: return {"raw":"UNKNOWN","error":"vcgencmd get_throttled unavailable or invalid"} return {"raw":f"0x{v:x}","under_voltage_now":bool(v&1),"freq_capped_now":bool(v&2),"throttled_now":bool(v&4),"soft_temp_limit_now":bool(v&8),"under_voltage_occurred":bool(v&(1<<16)),"freq_capped_occurred":bool(v&(1<<17)),"throttled_occurred":bool(v&(1<<18)),"soft_temp_limit_occurred":bool(v&(1<<19))} bus=None bus_lock=threading.RLock() FONT_PATH="/usr/local/share/pigway-pi-control/oled_font_5x7.bin" FONT5=b"" agent={"loop_errors":0,"i2c_errors":0} def bus_write(method,*args): with bus_lock: try: return getattr(bus,method)(*args) except OSError: agent["i2c_errors"]+=1 raise fan_level=0 prev_cpu=None page=0; page_since=time.monotonic() cpu_hi=mem_hi=0 net_ok=link_ok=True net_iface=""; net_kind="NET"; net_metric="DOWN"; ip_label="IP4"; ip="NO IP" wifi_weak_since=None; wifi_weak_active=False last_net=last_watch=0 watch_state=[] def cmd(c): bus_write("write_byte_data",OLED,0x00,c) def oled_init(): # Seller-native physical 128x32 configuration. for c in [0xAE,0xD5,0x80,0xA8,0x1F,0xD3,0x00,0x40,0x8D,0x14, 0x20,0x00,0xA1,0xC8,0xDA,0x02,0x81,0x7F,0xD9,0xF1, 0xDB,0x40,0xA4,0xA6,0xAF]: cmd(c) def draw_text_5x7(pix,x,y,text): for ch in text: code=ord(ch) if code<0 or code>255: code=ord("?") off=code*5 if off+5>len(FONT5): code=ord("?"); off=code*5 for cx in range(5): col=FONT5[off+cx] for cy in range(7): if col & (1<W-6: break def oled_show(lines_lr, title_zh=False): im=Image.new("1",(W,H),0); pix=im.load() for row,(left,right) in enumerate(lines_lr[:4]): y=row*8 draw_text_5x7(pix,0,y,left) if right: draw_text_5x7(pix,max(0,W-len(right)*6),y,right) data=[] for pg in range(4): for x in range(W): v=0 for bit in range(8): if pix[x,pg*8+bit]: v|=1<=3 => all three LEDs mcu(0x01,state[0]); mcu(0x02,state[1]); mcu(0x03,state[2]) _last_rgb=state def set_pi_led(name,off=True): base=Path("/sys/class/leds")/name try: if off: (base/"trigger").write_text("none") (base/"brightness").write_text("0") except Exception as e: print(f"board LED {name}: {e}",flush=True) def apply_dark_mode(): if DARK_PWR: set_pi_led("PWR",True) if DARK_ACT: set_pi_led("ACT",True) FAN_REG=[0x00,0x04,0x06,0x08,0x09,0x01] FAN_NAME=["OFF","L1","L2","L3","L4","MAX"] def fan_target(t): if tfan_level: fan_level=want elif want=1000 and n%1000==0 else str(n) except Exception: pass return iface,kind,metric,label,(ipval or "NO IP") def get_ip(): return primary_network()[4] def internet(): try: s=socket.create_connection(("1.1.1.1",53),0.5);s.close();return True except: return False def load_cfg(): c=configparser.ConfigParser(); c.read(CFG) watches=[] if c.has_section("services"): for label,target in c["services"].items(): target=target.strip() if target and not target.startswith("#"): watches.append(("service",label.upper(),target)) if c.has_section("processes"): for label,target in c["processes"].items(): target=target.strip() if target and not target.startswith("#"): watches.append(("process",label.upper(),target)) return c,watches cfg,watches=load_cfg() def cf(section,key,default,cast=float): try: return cast(cfg.get(section,key)) except: return default SENSOR_INTERVAL=cf("timing","sensor_interval",1.0) OLED_REFRESH=cf("timing","oled_refresh_interval",1.0) PAGE_INTERVAL=cf("timing","page_interval",10.0) NETWORK_INTERVAL=cf("timing","network_interval",5.0) SERVICE_INTERVAL=cf("timing","service_interval",5.0) HEALTH_INTERVAL=cf("timing","health_interval",5.0) CPU_HIGH=cf("alerts","cpu_high",90.0) CPU_RECOVER=cf("alerts","cpu_recover",75.0) CPU_HIGH_SEC=cf("alerts","cpu_high_seconds",30,int) MEM_HIGH=cf("alerts","memory_high",90.0) MEM_RECOVER=cf("alerts","memory_recover",85.0) MEM_HIGH_SEC=cf("alerts","memory_high_seconds",30,int) TEMP_HIGH=cf("alerts","temp_high",68.0) TEMP_CRIT=cf("alerts","temp_critical",75.0) DISK_HIGH=cf("alerts","disk_high",90.0) DISK_CRIT=cf("alerts","disk_critical",95.0) WIFI_WEAK=cf("alerts","wifi_weak",-75.0) WIFI_RECOVER=cf("alerts","wifi_recover",-68.0) WIFI_WEAK_SECONDS=cf("alerts","wifi_weak_seconds",30.0) FAN_T=[cf("fan","start",50.0),cf("fan","level2",53.0),cf("fan","level3",56.0), cf("fan","level4",59.0),cf("fan","max",62.0)] FAN_HYST=cf("fan","hysteresis",1.5) ROW_Y=[cf("display","row1_y",0,int),cf("display","row2_y",8,int),cf("display","row3_y",16,int),cf("display","row4_y",24,int)] DARK_PWR=cf("dark_mode","pi_pwr_led",True,lambda x: str(x).strip().lower() in ("off","0","false","no")) DARK_ACT=cf("dark_mode","pi_act_led",True,lambda x: str(x).strip().lower() in ("off","0","false","no")) LED_BRIGHT=cf("led","brightness",10,int) LED_WARN_ON=cf("led","warning_on_ms",300,int); LED_WARN_OFF=cf("led","warning_off_ms",2700,int) LED_CRIT_ON=cf("led","critical_on_ms",400,int); LED_CRIT_OFF=cf("led","critical_off_ms",1100,int) LED_EMERG_ON=cf("led","emergency_on_ms",400,int); LED_EMERG_OFF=cf("led","emergency_off_ms",400,int) def check_watch(w): typ,label,target=w args=["systemctl","is-active","--quiet",target] if typ=="service" else ["pgrep","-f","--",target] try: r=subprocess.run(args,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=2) return r.returncode==0 except (OSError,subprocess.TimeoutExpired): return False def diagnose(): print("Agent:") print(f"version={APP_VERSION} uptime_seconds={uptime_seconds()} config={CFG}") print("loop_errors=UNAVAILABLE i2c_errors=UNAVAILABLE scope=separate_diagnostic_process") print("System:") print(f"load_average={load_average()} cpu_freq_mhz={cpu_freq_mhz():.0f} root_readonly={root_readonly()}") print("Power:") power=throttled_state() print("throttled="+power["raw"]) for key in ("under_voltage_now","freq_capped_now","throttled_now","soft_temp_limit_now", "under_voltage_occurred","freq_capped_occurred","throttled_occurred","soft_temp_limit_occurred"): print(f"{key}={power.get(key,'UNKNOWN')}") if "error" in power: print("error="+power["error"]) iface,kind,metric,label,address=primary_network() print("Network:") print(f"primary_interface={iface or 'NONE'} kind={kind} metric={metric} ip_label={label} IP={address} internet={internet()}") print("I2C:") # SMBus receive-byte performs a read only; no register selection or display commands. try: with SMBus(1) as probe: for address,name in ((MCU,"MCU"),(OLED,"OLED")): try: probe.read_byte(address) print(f"0x{address:02X} {name}=PRESENT") except OSError as e: status="MISSING" if e.errno in (errno.ENXIO,getattr(errno,"EREMOTEIO",121)) else "ERROR" print(f"0x{address:02X} {name}={status} error={e}") except OSError as e: for address,name in ((MCU,"MCU"),(OLED,"OLED")): print(f"0x{address:02X} {name}=ERROR error={e}") print("Watches:") for w in watches: print(f"watch {w[1]}={'UP' if check_watch(w) else 'DOWN'} type={w[0]} target={w[2]}") return 0 def alert(title,l2="",l3="",l4="",sev=1,color=(20,16,0),blink=0): return {"title":title[:21],"l2":l2[:21],"l3":l3[:21],"l4":l4[:21], "sev":sev,"color":color,"blink":blink} def transition(key,active,on_event,off_event,**fields): old=event_state.get(key,False) if active!=old: event_state[key]=active log("WARN" if active else "INFO",on_event if active else off_event,**fields) def make_alerts(cpu,temp,mem,disk,sample_elapsed=0.0): global wifi_weak_since,wifi_weak_active global cpu_hi,mem_hi a=[] cpu_hi=cpu_hi+sample_elapsed if cpu>=CPU_HIGH else 0 if cpu=MEM_HIGH else 0 if mem=CPU_HIGH_SEC:a.append(alert("CPU LOAD HIGH",f"LOAD {cpu:.1f}%",f"TEMP {temp:.1f}C","FOR >=30 SEC",1,(24,18,0))) if temp>=TEMP_CRIT:a.append(alert("TEMP CRITICAL",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING",3,(32,0,0),2)) elif temp>=TEMP_HIGH:a.append(alert("TEMP HIGH",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING",1,(24,0,0))) if mem_hi>=MEM_HIGH_SEC:a.append(alert("MEMORY HIGH",f"USED {mem:.1f}%","CHECK PROCESS","",1,(18,0,24))) if disk>=DISK_CRIT:a.append(alert("DISK CRITICAL",f"USED {disk:.1f}%","FREE < 5%","CHECK STORAGE",2,(28,6,0),2)) elif disk>=DISK_HIGH:a.append(alert("DISK SPACE LOW",f"USED {disk:.1f}%","FREE < 10%","CHECK STORAGE",1,(28,8,0))) if not link_ok:a.append(alert("NETWORK DOWN","NO LAN / WIFI","NO IP ADDRESS","",2,(0,8,28),1)) elif not net_ok:a.append(alert("INTERNET DOWN","LOCAL LINK OK",ip,"",2,(0,8,28),1)) for w,ok in zip(watches,watch_state): if not ok:a.append(alert("SERVICE DOWN" if w[0]=="service" else "PROCESS DOWN",w[1],"CHECK / RESTART","",2,(0,24,24),1)) if net_kind=="WIF" and str(net_metric).lstrip("-").isdigit(): rssi=float(net_metric) if not wifi_weak_active: if rssi <= WIFI_WEAK: if wifi_weak_since is None: wifi_weak_since=time.monotonic() if time.monotonic()-wifi_weak_since >= WIFI_WEAK_SECONDS: wifi_weak_active=True else: wifi_weak_since=None elif rssi >= WIFI_RECOVER: wifi_weak_active=False; wifi_weak_since=None if wifi_weak_active: a.append(alert("WIFI SIGNAL WEAK",f"{rssi:.0f} dBm",f"IF {net_iface}","CHECK SIGNAL",1,(0,0,28),1)) else: wifi_weak_since=None; wifi_weak_active=False if health.get("under_voltage_now"): a.append(alert("POWER LOW","UNDERVOLT",health.get("raw",""),"CHECK 5V",3,(28,18,0),2)) if health.get("throttled_now") or health.get("freq_capped_now"): a.append(alert("CPU THROTTLED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"TEMP / POWER",2,(28,18,0),1)) if fs_readonly: a.append(alert("FILESYSTEM RO","ROOT READ ONLY","CHECK STORAGE","",3,(28,6,0),2)) a.sort(key=lambda x:x["sev"],reverse=True) return a def led_for(a): # Color = fault object. Blink frequency = severity. All alerts blink. if not a: rgb_off(); return if a["sev"]>=3: on_ms,off_ms=LED_EMERG_ON,LED_EMERG_OFF elif a["sev"]>=2: on_ms,off_ms=LED_CRIT_ON,LED_CRIT_OFF else: on_ms,off_ms=LED_WARN_ON,LED_WARN_OFF cycle=max(1,on_ms+off_ms) phase=int(time.monotonic()*1000)%cycle rgb(*a["color"]) if phase18 else ip left=[ f"CPU {cpu:.1f}%", f"TMP {temp:.1f}C", f"DSK {disk:.1f}%", ip_label+" "+display_ip, ] right=[ f"MEM {mem:.1f}%", f"FAN {FAN_NAME[fan_level]}", f"{net_kind} {net_metric}", "", ] for row in range(4): y=row*8 draw_text_5x7(pix,0,y,left[row]) if right[row]: draw_text_5x7(pix,72,y,right[row]) data=[] for pg in range(4): for x in range(W): v=0 for bit in range(8): if pix[x,pg*8+bit]: v|=1<= SENSOR_INTERVAL: sample_elapsed=now-last_sensor if last_sensor else 0.0 c=cpu_pct(); t=temp_c(); m=mem_pct(); d=disk_pct() latest=(c,t,m,d); update_fan(t); last_sensor=now; agent["last_sensor_ok"]=now if now-last_net >= NETWORK_INTERVAL: old_id=(net_iface,ip) net_iface,net_kind,net_metric,ip_label,ip=primary_network(); link_ok=ip!="NO IP"; net_ok=internet() if link_ok else False new_id=(net_iface,ip) if new_id!=old_id: log("INFO","NET_SWITCH",old_if=old_id[0],old_ip=old_id[1],new_if=net_iface,new_ip=ip) transition("network_down",not link_ok,"NETWORK_DOWN","NETWORK_RECOVERED",iface=net_iface,ip=ip) transition("internet_down",link_ok and not net_ok,"INTERNET_DOWN","INTERNET_RECOVERED",iface=net_iface,ip=ip) last_net=now; agent["last_network_ok"]=now if now-last_watch >= SERVICE_INTERVAL: watch_state=[check_watch(w) for w in watches] for w,ok in zip(watches,watch_state): transition("watch:"+w[0]+":"+w[1],not ok,"WATCH_DOWN","WATCH_RECOVERED",name=w[1],target=w[2]) last_watch=now; agent["last_watch_ok"]=now if now-last_health >= HEALTH_INTERVAL: power_sample=throttled_state(); fs_readonly=root_readonly() transition("power_check", "error" in power_sample,"POWER_CHECK_ERROR","POWER_CHECK_RECOVERED") # An unavailable reading is not evidence that an existing fault recovered. if "error" not in power_sample: health=power_sample transition("undervoltage",health.get("under_voltage_now",False),"POWER_LOW","POWER_RECOVERED",raw=health.get("raw")) transition("throttled",health.get("throttled_now",False) or health.get("freq_capped_now",False),"THROTTLED","THROTTLE_RECOVERED",raw=health.get("raw"),freq_mhz=f"{cpu_freq_mhz():.0f}") transition("soft_temp",health.get("soft_temp_limit_now",False),"SOFT_TEMP_LIMIT","SOFT_TEMP_RECOVERED",raw=health.get("raw")) transition("root_ro",fs_readonly,"FILESYSTEM_RO","FILESYSTEM_RW") for k in ("under_voltage_occurred","freq_capped_occurred","throttled_occurred","soft_temp_limit_occurred"): if health.get(k) and k not in history_logged: history_logged.add(k); log("WARN","HISTORY_FLAG",flag=k,raw=health.get("raw")) agent["last_health_ok"]=now; last_health=now c,t,m,d=latest active=make_alerts(c,t,m,d,sample_elapsed) led_alert=max(active,key=lambda a:a["sev"],default=None) new_alert_keys={alert_key(a) for a in active} for a in active: k=alert_key(a) if k not in prev_alert_keys: log("WARN","ALERT_ACTIVE",alert=a["title"],detail=a["l2"],severity=a["sev"]) for k in prev_alert_keys-new_alert_keys: log("INFO","ALERT_RECOVERED",alert=k) prev_alert_keys=new_alert_keys # If the currently displayed alert truly disappeared, skip it immediately. if cycle_pages[cycle_pos]!="HOME": key=cycle_pages[cycle_pos] if find_live_alert(key,active) is None: cycle_pos += 1 if cycle_pos>=len(cycle_pages): # End of cycle: capture a fresh snapshot. cycle_pages=["HOME"]+[alert_key(a) for a in active] cycle_pos=0 page_started=now # ONLY this timer performs normal page changes. if now-page_started >= PAGE_INTERVAL: cycle_pos += 1 if cycle_pos>=len(cycle_pages): # A complete cycle ended. Snapshot all currently active alerts. cycle_pages=["HOME"]+[alert_key(a) for a in active] cycle_pos=0 page_started=now # Special case: while idle on HOME with no alert pages queued, begin a new # alert cycle immediately when the first alert appears. HOME keeps its # current 10-second slot; alerts follow it and cannot be kicked out by refresh. if cycle_pages==["HOME"] and active: cycle_pages=["HOME"]+[alert_key(a) for a in active] # Render CURRENT page only. 1-second refresh does not affect cycle_pos. if now-last_render >= OLED_REFRESH: current=cycle_pages[cycle_pos] if current=="HOME": render_home(c,t,m,d) else: a=find_live_alert(current,active) if a: # Number refers to this cycle's alert slot, all using same 6x8 font. render_alert(a,cycle_pos,len(cycle_pages)-1) else: render_home(c,t,m,d) last_render=now time.sleep(0.20) except Exception as e: agent["loop_errors"]+=1; log("ERROR","LOOP_ERROR",count=agent["loop_errors"],error=repr(e)); time.sleep(1) finally: cleanup()