#!/usr/bin/env python3 import time,socket,subprocess,configparser,signal,re,errno,threading,argparse,json,os,platform from pathlib import Path APP_VERSION="3.7.0" from smbus2 import SMBus from PIL import Image from shutil import disk_usage OLED=0x3C; MCU=0x0D; W,H=128,32 CFG="/etc/pigway-pi-control.conf" STATUS_PATH="/run/pigway-pi-control/status.json" running=True def stop(*_): global running; running=False signal.signal(signal.SIGTERM,stop); signal.signal(signal.SIGINT,stop) def log(level,event,**fields): parts=[f"level={level}",f"event={event}"] for k,v in fields.items(): parts.append(f"{k}={str(v).replace(chr(32),'_')}") print(" ".join(parts),flush=True) def uptime_seconds(): try:return int(float(Path("/proc/uptime").read_text().split()[0])) except Exception:return 0 def load_average(): try: x=Path("/proc/loadavg").read_text().split();return (float(x[0]),float(x[1]),float(x[2])) except Exception:return (0.0,0.0,0.0) def memory_usage(): try: values={} for line in Path("/proc/meminfo").read_text().splitlines(): key,value=line.split(":",1); values[key]=int(value.split()[0])*1024 total=values["MemTotal"]; available=values.get("MemAvailable",values.get("MemFree",0)) used=max(0,total-available) return total,used,available,(used*100.0/total if total else 0.0) except Exception:return 0,0,0,0.0 def wifi_percent(metric): try:return max(0,min(100,round((float(metric)+100.0)*2.0))) except Exception:return None def device_details(): info={} try: for line in Path("/proc/cpuinfo").read_text().splitlines(): if ":" in line: key,value=(x.strip() for x in line.split(":",1)) if key in ("Serial","Revision","Hardware","Model"): info[key.lower()]=value except Exception: pass try: info["model"]=Path("/proc/device-tree/model").read_bytes().rstrip(b"\0").decode() except Exception: pass try: os_release={} for line in Path("/etc/os-release").read_text().splitlines(): if "=" in line: key,value=line.split("=",1); os_release[key]=value.strip().strip('"') info["os"]=os_release.get("PRETTY_NAME",platform.platform()) except Exception: info["os"]=platform.platform() info.update({"hostname":socket.gethostname(),"kernel":platform.release(), "architecture":platform.machine(),"cpu_count":os.cpu_count() or 0}) return info def cpu_freq_mhz(): for q in ("/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq","/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq"): try:return int(Path(q).read_text().strip())/1000.0 except Exception:pass return 0.0 def root_readonly(): try: for line in Path("/proc/mounts").read_text().splitlines(): a=line.split() if len(a)>=4 and a[1]=="/":return "ro" in a[3].split(",") except Exception:pass return False def throttled_state(): raw=_run(["vcgencmd","get_throttled"]) m=re.search(r"0x([0-9a-fA-F]+)",raw);v=int(m.group(1),16) if m else 0 if not m: return {"raw":"UNKNOWN","error":"vcgencmd get_throttled unavailable or invalid"} return {"raw":f"0x{v:x}","under_voltage_now":bool(v&1),"freq_capped_now":bool(v&2),"throttled_now":bool(v&4),"soft_temp_limit_now":bool(v&8),"under_voltage_occurred":bool(v&(1<<16)),"freq_capped_occurred":bool(v&(1<<17)),"throttled_occurred":bool(v&(1<<18)),"soft_temp_limit_occurred":bool(v&(1<<19))} bus=None bus_lock=threading.RLock() FONT_PATH="/usr/local/share/pigway-pi-control/oled_font_5x7.bin" FONT5=b"" agent={"loop_errors":0,"i2c_errors":0} def bus_write(method,*args): with bus_lock: try: return getattr(bus,method)(*args) except OSError: agent["i2c_errors"]+=1 raise fan_level=0 prev_cpu=None page=0; page_since=time.monotonic() cpu_hi=mem_hi=0 net_ok=link_ok=True net_iface=""; net_kind="NET"; net_metric="DOWN"; ip_label="IP4"; ip="NO IP" wifi_weak_since=None; wifi_weak_active=False last_net=last_watch=0 watch_state=[] def cmd(c): bus_write("write_byte_data",OLED,0x00,c) def oled_init(): # Seller-native physical 128x32 configuration. for c in [0xAE,0xD5,0x80,0xA8,0x1F,0xD3,0x00,0x40,0x8D,0x14, 0x20,0x00,0xA1,0xC8,0xDA,0x02,0x81,0x7F,0xD9,0xF1, 0xDB,0x40,0xA4,0xA6,0xAF]: cmd(c) def draw_text_5x7(pix,x,y,text): for ch in text: code=ord(ch) if code<0 or code>255: code=ord("?") off=code*5 if off+5>len(FONT5): code=ord("?"); off=code*5 for cx in range(5): col=FONT5[off+cx] for cy in range(7): if col & (1<W-6: break def oled_show(lines_lr, title_zh=False): im=Image.new("1",(W,H),0); pix=im.load() for row,(left,right) in enumerate(lines_lr[:4]): y=row*8 draw_text_5x7(pix,0,y,left) if right: draw_text_5x7(pix,max(0,W-len(right)*6),y,right) data=[] for pg in range(4): for x in range(W): v=0 for bit in range(8): if pix[x,pg*8+bit]: v|=1<fan_level: fan_level=want elif want=1000 and n%1000==0 else str(n) except Exception: pass return iface,kind,metric,label,(ipval or "NO IP") def get_ip(): return primary_network()[4] def internet(): try: s=socket.create_connection(("1.1.1.1",53),0.5);s.close();return True except: return False def load_cfg(): c=configparser.ConfigParser(); c.read(CFG) watches=[] if c.has_section("services"): for label,target in c["services"].items(): target=target.strip() paused=c.getboolean("service_monitor_disabled",label,fallback=False) if target and not target.startswith("#") and not paused: watches.append(("service",label.upper(),target)) if c.has_section("processes"): for label,target in c["processes"].items(): target=target.strip() if target and not target.startswith("#"): watches.append(("process",label.upper(),target)) return c,watches cfg,watches=load_cfg() def cf(section,key,default,cast=float): try: return cast(cfg.get(section,key)) except: return default DEVICE_ID=cf("device","identifier","auto",str).strip() if not DEVICE_ID or DEVICE_ID.lower()=="auto": DEVICE_ID=socket.gethostname() DEVICE_NAME=cf("device","name",DEVICE_ID,str).strip() or DEVICE_ID if DEVICE_NAME.lower()=="auto": DEVICE_NAME=DEVICE_ID DEVICE_INFO=device_details() SENSOR_INTERVAL=cf("timing","sensor_interval",1.0) OLED_REFRESH=cf("timing","oled_refresh_interval",1.0) PAGE_INTERVAL=cf("timing","page_interval",10.0) NETWORK_INTERVAL=cf("timing","network_interval",5.0) SERVICE_INTERVAL=cf("timing","service_interval",5.0) HEALTH_INTERVAL=cf("timing","health_interval",1.0) CPU_HIGH=cf("alerts","cpu_high",90.0) CPU_RECOVER=cf("alerts","cpu_recover",75.0) CPU_HIGH_SEC=cf("alerts","cpu_high_seconds",30,int) MEM_HIGH=cf("alerts","memory_high",90.0) MEM_RECOVER=cf("alerts","memory_recover",85.0) MEM_HIGH_SEC=cf("alerts","memory_high_seconds",30,int) TEMP_HIGH=cf("alerts","temp_high",68.0) TEMP_CRIT=cf("alerts","temp_critical",75.0) DISK_HIGH=cf("alerts","disk_high",90.0) DISK_CRIT=cf("alerts","disk_critical",95.0) WIFI_WEAK=cf("alerts","wifi_weak",-75.0) WIFI_RECOVER=cf("alerts","wifi_recover",-68.0) WIFI_WEAK_SECONDS=cf("alerts","wifi_weak_seconds",30.0) FAN_T=[cf("fan","start",50.0),cf("fan","level2",53.0),cf("fan","level3",56.0), cf("fan","level4",59.0),cf("fan","max",62.0)] FAN_HYST=cf("fan","hysteresis",1.5) ROW_Y=[cf("display","row1_y",0,int),cf("display","row2_y",8,int),cf("display","row3_y",16,int),cf("display","row4_y",24,int)] DARK_PWR=cf("dark_mode","pi_pwr_led",True,lambda x: str(x).strip().lower() in ("off","0","false","no")) DARK_ACT=cf("dark_mode","pi_act_led",True,lambda x: str(x).strip().lower() in ("off","0","false","no")) RGB_WRITE_DELAY_MS=max(1,cf("led","write_delay_ms",10,int)) WARNING_FLASH=(max(50,cf("led","warning_flash_on_ms",1500,int)),max(50,cf("led","warning_flash_off_ms",1500,int))) CRITICAL_FLASH=(max(50,cf("led","critical_flash_on_ms",750,int)),max(50,cf("led","critical_flash_off_ms",750,int))) EMERGENCY_FLASH=(max(50,cf("led","emergency_flash_on_ms",300,int)),max(50,cf("led","emergency_flash_off_ms",300,int))) NORMAL_MODE=cf("led","normal_mode","off",str).strip().lower() NORMAL_MODES={"off","solid","flash","flow","breathe","marquee","rainbow","colorful"} if NORMAL_MODE not in NORMAL_MODES: NORMAL_MODE="off" NORMAL_COLOR=tuple(max(0,min(255,cf("led",f"normal_{channel}",value,int))) for channel,value in zip(("r","g","b"),(0,0,0))) NORMAL_FLASH=(max(50,cf("led","normal_flash_on_ms",1000,int)), max(50,cf("led","normal_flash_off_ms",1000,int))) NORMAL_EFFECT_SPEED=max(1,min(3,cf("led","normal_effect_speed",2,int))) NORMAL_EFFECT_COLOR=cf("led","normal_effect_color","green",str).strip().lower() RGB_EFFECT_COLORS={"red":0,"green":1,"blue":2,"yellow":3,"purple":4,"cyan":5,"white":6} if NORMAL_EFFECT_COLOR not in RGB_EFFECT_COLORS: NORMAL_EFFECT_COLOR="green" RGB_EFFECTS={"flow":0,"breathe":1,"marquee":2,"rainbow":3,"colorful":4} NORMAL_RGB_MODE={"off":"OFF","solid":"NORMAL_SOLID","flash":"NORMAL_FLASH", **{name:f"NORMAL_{name.upper()}" for name in RGB_EFFECTS}}[NORMAL_MODE] def rgb_cfg(name,default): return tuple(max(0,min(255,cf("led",f"{name}_{channel}",value,int))) for channel,value in zip(("r","g","b"),default)) RGB_GROUP_VALUES={ "CPU":rgb_cfg("cpu",(255,0,0)), "POWER":rgb_cfg("power",(255,40,0)), "MEMORY":rgb_cfg("memory",(255,0,255)), "STORAGE":rgb_cfg("storage",(255,255,255)), "NETWORK":rgb_cfg("network",(0,0,255)), "SERVICE":rgb_cfg("service",(0,255,96)), } RGB_OBJECT_GROUPS={ "ROOT_READONLY":"STORAGE", "TEMP_CRITICAL":"CPU","TEMP_HIGH":"CPU","SOFT_TEMP_LIMIT":"CPU", "THROTTLED":"CPU","FREQ_CAPPED":"CPU","CPU_HIGH":"CPU", "POWER_UNDERVOLT":"POWER", "MEMORY_HIGH":"MEMORY", "DISK_CRITICAL":"STORAGE","DISK_HIGH":"STORAGE", "NETWORK_DOWN":"NETWORK","INTERNET_DOWN":"NETWORK","WIFI_WEAK":"NETWORK", "SERVICE_DOWN":"SERVICE","PROCESS_DOWN":"SERVICE", } def check_watch(w): typ,label,target=w args=["systemctl","is-active","--quiet",target] if typ=="service" else ["pgrep","-f","--",target] try: r=subprocess.run(args,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=2) return r.returncode==0 except (OSError,subprocess.TimeoutExpired): return False def diagnose(): print("Agent:") print(f"version={APP_VERSION} uptime_seconds={uptime_seconds()} config={CFG}") print(f"machine_id={DEVICE_ID} machine_name={DEVICE_NAME} hostname={socket.gethostname()}") print("loop_errors=UNAVAILABLE i2c_errors=UNAVAILABLE scope=separate_diagnostic_process") print("System:") print(f"load_average={load_average()} cpu_freq_mhz={cpu_freq_mhz():.0f} root_readonly={root_readonly()}") print("Power:") power=throttled_state() print("throttled="+power["raw"]) for key in ("under_voltage_now","freq_capped_now","throttled_now","soft_temp_limit_now", "under_voltage_occurred","freq_capped_occurred","throttled_occurred","soft_temp_limit_occurred"): print(f"{key}={power.get(key,'UNKNOWN')}") if "error" in power: print("error="+power["error"]) iface,kind,metric,label,address=primary_network() print("Network:") print(f"primary_interface={iface or 'NONE'} kind={kind} metric={metric} signal_percent={wifi_percent(metric) if kind=='WIF' else 'N/A'} ip_label={label} IP={address} internet={internet()}") print("I2C:") # SMBus receive-byte performs a read only; no register selection or display commands. try: with SMBus(1) as probe: for address,name in ((MCU,"MCU"),(OLED,"OLED")): try: probe.read_byte(address) print(f"0x{address:02X} {name}=PRESENT") except OSError as e: status="MISSING" if e.errno in (errno.ENXIO,getattr(errno,"EREMOTEIO",121)) else "ERROR" print(f"0x{address:02X} {name}={status} error={e}") except OSError as e: for address,name in ((MCU,"MCU"),(OLED,"OLED")): print(f"0x{address:02X} {name}=ERROR error={e}") print("Watches:") for w in watches: print(f"watch {w[1]}={'UP' if check_watch(w) else 'DOWN'} type={w[0]} target={w[2]}") return 0 DISPLAY_PRIORITIES={ "ROOT_READONLY":100, "TEMP_CRITICAL":90, "SOFT_TEMP_LIMIT":85, "POWER_UNDERVOLT":80, "THROTTLED":75, "FREQ_CAPPED":70, "NETWORK_DOWN":65, "INTERNET_DOWN":65, "SERVICE_DOWN":60, "PROCESS_DOWN":60, "DISK_CRITICAL":55, "TEMP_HIGH":50, "CPU_HIGH":45, "MEMORY_HIGH":45, "DISK_HIGH":40, "WIFI_WEAK":30, "NORMAL_HOME":0, } def display_priority(request_id): return DISPLAY_PRIORITIES[request_id.split("|",1)[0]] def transition(key,active,on_event,off_event,**fields): old=event_state.get(key,False) if active!=old: event_state[key]=active log("WARN" if active else "INFO",on_event if active else off_event,**fields) def update_active_alerts(cpu,temp,mem,disk,sample_elapsed=0.0): global wifi_weak_since,wifi_weak_active global cpu_hi,mem_hi cpu_hi=cpu_hi+sample_elapsed if cpu>=CPU_HIGH else 0 if cpu=MEM_HIGH else 0 if mem=CPU_HIGH_SEC: display.request("CPU_HIGH",1,"CPU LOAD HIGH",f"LOAD {cpu:.1f}%",f"TEMP {temp:.1f}C","CHECK LOAD") else: display.clear("CPU_HIGH") if temp>=TEMP_CRIT: display.request("TEMP_CRITICAL",3,"TEMP CRITICAL",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING") display.clear("TEMP_HIGH") elif temp>=TEMP_HIGH: display.request("TEMP_HIGH",1,"TEMP HIGH",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING") display.clear("TEMP_CRITICAL") else: display.clear("TEMP_HIGH"); display.clear("TEMP_CRITICAL") if mem_hi>=MEM_HIGH_SEC: display.request("MEMORY_HIGH",1,"MEMORY HIGH",f"USED {mem:.1f}%","CHECK PROCESS","") else: display.clear("MEMORY_HIGH") if disk>=DISK_CRIT: display.request("DISK_CRITICAL",2,"DISK CRITICAL",f"USED {disk:.1f}%","FREE < 5%","CHECK STORAGE") display.clear("DISK_HIGH") elif disk>=DISK_HIGH: display.request("DISK_HIGH",1,"DISK SPACE LOW",f"USED {disk:.1f}%","FREE < 10%","CHECK STORAGE") display.clear("DISK_CRITICAL") else: display.clear("DISK_HIGH"); display.clear("DISK_CRITICAL") if not link_ok: display.request("NETWORK_DOWN",2,"NETWORK DOWN","NO LAN / WIFI","NO IP ADDRESS","") display.clear("INTERNET_DOWN") elif not net_ok: display.request("INTERNET_DOWN",2,"INTERNET DOWN","LOCAL LINK OK",ip,"") display.clear("NETWORK_DOWN") else: display.clear("NETWORK_DOWN"); display.clear("INTERNET_DOWN") for w,ok in zip(watches,watch_state): alert_id=("SERVICE_DOWN" if w[0]=="service" else "PROCESS_DOWN")+"|"+w[1] if not ok: display.request(alert_id,2,"SERVICE DOWN" if w[0]=="service" else "PROCESS DOWN",w[1],"CHECK / RESTART","CRITICAL",{"target":w[2]}) else: display.clear(alert_id) if net_kind=="WIF" and str(net_metric).lstrip("-").isdigit(): rssi=float(net_metric) if not wifi_weak_active: if rssi <= WIFI_WEAK: if wifi_weak_since is None: wifi_weak_since=time.monotonic() if time.monotonic()-wifi_weak_since >= WIFI_WEAK_SECONDS: wifi_weak_active=True else: wifi_weak_since=None elif rssi >= WIFI_RECOVER: wifi_weak_active=False; wifi_weak_since=None if wifi_weak_active: display.request("WIFI_WEAK",1,"WIFI SIGNAL WEAK",f"RSSI {rssi:.0f} dBm",f"IF {net_iface}","CHECK SIGNAL") else: display.clear("WIFI_WEAK") else: wifi_weak_since=None; wifi_weak_active=False; display.clear("WIFI_WEAK") if health.get("under_voltage_now"): display.request("POWER_UNDERVOLT",2,"POWER LOW","UNDERVOLTAGE","CHECK POWER","CRITICAL",{"raw":health.get("raw","")}) else: display.clear("POWER_UNDERVOLT") if health.get("throttled_now"): display.request("THROTTLED",2,"CPU THROTTLED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"CHECK POWER") else: display.clear("THROTTLED") if health.get("freq_capped_now"): display.request("FREQ_CAPPED",2,"FREQ CAPPED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"CHECK POWER") else: display.clear("FREQ_CAPPED") if health.get("soft_temp_limit_now"): display.request("SOFT_TEMP_LIMIT",2,"SOFT TEMP LIMIT","CPU TEMP LIMITED",health.get("raw",""),"CHECK COOLING") else: display.clear("SOFT_TEMP_LIMIT") if fs_readonly: display.request("ROOT_READONLY",3,"FILESYSTEM RO","ROOT READ ONLY","CHECK STORAGE","EMERGENCY") else: display.clear("ROOT_READONLY") def rgb_mode_for(request_id,severity): if severity<=0: return "OFF" alert_type=request_id.split("|",1)[0] color=RGB_OBJECT_GROUPS[alert_type] level="EMERGENCY" if severity>=3 else "CRITICAL" if severity>=2 else "WARNING" return f"{color}_{level}_FLASH" def rgb_flash(mode,on): global _last_rgb color=mode.split("_",1)[0] channels=RGB_GROUP_VALUES[color] state=(mode,bool(on)) if _last_rgb==state: return if not on: mcu(0x07,0x00) else: mcu(0x00,0xff); time.sleep(RGB_WRITE_DELAY_MS/1000.0) mcu(0x01,channels[0]); time.sleep(RGB_WRITE_DELAY_MS/1000.0) mcu(0x02,channels[1]); time.sleep(RGB_WRITE_DELAY_MS/1000.0) mcu(0x03,channels[2]) _last_rgb=state def rgb_static(channels,state_name): global _last_rgb state=(state_name,channels) if _last_rgb==state: return mcu(0x07,0x00); time.sleep(RGB_WRITE_DELAY_MS/1000.0) mcu(0x00,0xff); time.sleep(RGB_WRITE_DELAY_MS/1000.0) mcu(0x01,channels[0]); time.sleep(RGB_WRITE_DELAY_MS/1000.0) mcu(0x02,channels[1]); time.sleep(RGB_WRITE_DELAY_MS/1000.0) mcu(0x03,channels[2]) _last_rgb=state def rgb_normal_effect(mode): global _last_rgb effect_name=mode.removeprefix("NORMAL_").lower() state=("effect",effect_name,NORMAL_EFFECT_SPEED,NORMAL_EFFECT_COLOR) if _last_rgb==state: return # Hardware-calibrated order: speed and color are parameters; the final # effect register write commits and starts the selected MCU animation. mcu(0x05,NORMAL_EFFECT_SPEED); time.sleep(RGB_WRITE_DELAY_MS/1000.0) mcu(0x06,RGB_EFFECT_COLORS[NORMAL_EFFECT_COLOR]); time.sleep(RGB_WRITE_DELAY_MS/1000.0) mcu(0x04,RGB_EFFECTS[effect_name]) _last_rgb=state def render_home(cpu,temp,mem,disk): # 128x32 fixed columns. Both columns are LEFT-aligned. # Seller 5x7 font has 6px character pitch; right column starts at x=72. im=Image.new("1",(W,H),0); pix=im.load() display_ip=ip[:7]+".."+ip[-7:] if ip_label=="IP6" and len(ip)>18 else ip left=[ f"CPU {cpu:.1f}%", f"TMP {temp:.1f}C", f"DSK {disk:.1f}%", ip_label+" "+display_ip, ] signal=wifi_percent(net_metric) if net_kind=="WIF" else None right=[ f"MEM {mem:.1f}%", f"FAN {FAN_NAME[fan_level]}", f"WIF {signal}%" if signal is not None else f"{net_kind} {net_metric}", "", ] for row in range(4): y=row*8 draw_text_5x7(pix,0,y,left[row]) if right[row]: draw_text_5x7(pix,72,y,right[row]) data=[] for pg in range(4): for x in range(W): v=0 for bit in range(8): if pix[x,pg*8+bit]: v|=1<1 and time.monotonic()-self.page_started>=PAGE_INTERVAL: pos=pages.index(self.current_oled_page_id) self._show_page_locked(pages[(pos+1)%len(pages)]) return request=self.requests[self.current_oled_page_id] if request["id"]=="NORMAL_HOME": render_home(*request["metadata"]["home"]) else: alert_ids=[a["id"] for a in alerts] render_alert(request,alert_ids.index(request["id"])+1,len(alerts)) def owner_snapshot(self): with self.lock: return dict(self.requests[self.current_owner_id]) def status_snapshot(self): with self.lock: alerts=self._ordered_alerts_locked() return { "owner":self.current_owner_id, "oled_page":self.current_oled_page_id, "rgb_mode":self.current_rgb_mode, "alerts":[{k:a[k] for k in ("id","priority","severity","title","l2","l3","l4","first_seen","last_changed")} for a in alerts], } def _rgb_worker(self): previous_error=None; applied_mode="OFF"; applied_flash=False flash_on=None; animation_started=time.monotonic() while not self.stop_event.is_set(): try: with self.lock: wanted_mode=self.current_rgb_mode wanted_flash=wanted_mode.endswith("_FLASH") if wanted_mode!=applied_mode: with bus_lock: if wanted_mode=="OFF": rgb_off() elif wanted_mode=="NORMAL_SOLID": rgb_static(NORMAL_COLOR,"normal_solid") elif wanted_mode=="NORMAL_FLASH": rgb_static(NORMAL_COLOR,"normal_flash_on") elif wanted_mode.startswith("NORMAL_"): rgb_normal_effect(wanted_mode) elif wanted_flash: rgb_off() rgb_flash(wanted_mode,True) else: raise ValueError(f"unsupported RGB mode: {wanted_mode}") log("INFO","RGB_MODE",old=applied_mode,new=wanted_mode) applied_mode=wanted_mode; animation_started=time.monotonic() flash_on=True if wanted_mode.endswith("_FLASH") else None applied_flash=applied_mode.endswith("_FLASH") if applied_flash: elapsed_ms=(time.monotonic()-animation_started)*1000.0 if applied_mode=="NORMAL_FLASH": on_ms,off_ms=NORMAL_FLASH elif "_EMERGENCY_" in applied_mode: on_ms,off_ms=EMERGENCY_FLASH elif "_CRITICAL_" in applied_mode: on_ms,off_ms=CRITICAL_FLASH else: on_ms,off_ms=WARNING_FLASH on=elapsed_ms%(on_ms+off_ms)= SENSOR_INTERVAL: sample_elapsed=now-last_sensor if last_sensor else 0.0 c=cpu_pct(); t=temp_c(); m=mem_pct(); d=disk_pct() latest=(c,t,m,d); update_fan(t); last_sensor=now; agent["last_sensor_ok"]=now if now-last_net >= NETWORK_INTERVAL: old_id=(net_iface,ip) net_iface,net_kind,net_metric,ip_label,ip=primary_network(); link_ok=ip!="NO IP"; net_ok=internet() if link_ok else False new_id=(net_iface,ip) if new_id!=old_id: log("INFO","NET_SWITCH",old_if=old_id[0],old_ip=old_id[1],new_if=net_iface,new_ip=ip) transition("network_down",not link_ok,"NETWORK_DOWN","NETWORK_RECOVERED",iface=net_iface,ip=ip) transition("internet_down",link_ok and not net_ok,"INTERNET_DOWN","INTERNET_RECOVERED",iface=net_iface,ip=ip) last_net=now; agent["last_network_ok"]=now if now-last_watch >= SERVICE_INTERVAL: watch_state=[check_watch(w) for w in watches] for w,ok in zip(watches,watch_state): transition("watch:"+w[0]+":"+w[1],not ok,"WATCH_DOWN","WATCH_RECOVERED",name=w[1],target=w[2]) last_watch=now; agent["last_watch_ok"]=now if now-last_health >= HEALTH_INTERVAL: power_sample=throttled_state(); fs_readonly=root_readonly() transition("power_check", "error" in power_sample,"POWER_CHECK_ERROR","POWER_CHECK_RECOVERED") # An unavailable reading is not evidence that an existing fault recovered. if "error" not in power_sample: health=power_sample transition("undervoltage",health.get("under_voltage_now",False),"POWER_LOW","POWER_RECOVERED",raw=health.get("raw")) transition("throttled",health.get("throttled_now",False) or health.get("freq_capped_now",False),"THROTTLED","THROTTLE_RECOVERED",raw=health.get("raw"),freq_mhz=f"{cpu_freq_mhz():.0f}") transition("soft_temp",health.get("soft_temp_limit_now",False),"SOFT_TEMP_LIMIT","SOFT_TEMP_RECOVERED",raw=health.get("raw")) transition("root_ro",fs_readonly,"FILESYSTEM_RO","FILESYSTEM_RW") for k in ("under_voltage_occurred","freq_capped_occurred","throttled_occurred","soft_temp_limit_occurred"): if health.get(k) and k not in history_logged: history_logged.add(k); log("WARN","HISTORY_FLAG",flag=k,raw=health.get("raw")) agent["last_health_ok"]=now; last_health=now c,t,m,d=latest display.update_home(c,t,m,d) display.begin_update() try: update_active_alerts(c,t,m,d,sample_elapsed) finally: display.end_update() if now-last_render >= OLED_REFRESH: display.refresh() last_render=now if now-last_status >= 1.0: write_status(c,t,m,d); last_status=now time.sleep(0.20) except Exception as e: agent["loop_errors"]+=1; log("ERROR","LOOP_ERROR",count=agent["loop_errors"],error=repr(e)); time.sleep(1) finally: cleanup()