777 lines
33 KiB
Python
Executable File
777 lines
33 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import time,socket,subprocess,configparser,signal,re,errno,threading,argparse,json
|
|
from pathlib import Path
|
|
APP_VERSION="3.6.0"
|
|
from smbus2 import SMBus
|
|
from PIL import Image
|
|
from shutil import disk_usage
|
|
|
|
OLED=0x3C; MCU=0x0D; W,H=128,32
|
|
CFG="/etc/pigway-pi-control.conf"
|
|
STATUS_PATH="/run/pigway-pi-control/status.json"
|
|
running=True
|
|
|
|
def stop(*_):
|
|
global running; running=False
|
|
signal.signal(signal.SIGTERM,stop); signal.signal(signal.SIGINT,stop)
|
|
|
|
def log(level,event,**fields):
|
|
parts=[f"level={level}",f"event={event}"]
|
|
for k,v in fields.items(): parts.append(f"{k}={str(v).replace(chr(32),'_')}")
|
|
print(" ".join(parts),flush=True)
|
|
|
|
def uptime_seconds():
|
|
try:return int(float(Path("/proc/uptime").read_text().split()[0]))
|
|
except Exception:return 0
|
|
|
|
def load_average():
|
|
try:
|
|
x=Path("/proc/loadavg").read_text().split();return (float(x[0]),float(x[1]),float(x[2]))
|
|
except Exception:return (0.0,0.0,0.0)
|
|
|
|
def cpu_freq_mhz():
|
|
for q in ("/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq","/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_cur_freq"):
|
|
try:return int(Path(q).read_text().strip())/1000.0
|
|
except Exception:pass
|
|
return 0.0
|
|
|
|
def root_readonly():
|
|
try:
|
|
for line in Path("/proc/mounts").read_text().splitlines():
|
|
a=line.split()
|
|
if len(a)>=4 and a[1]=="/":return "ro" in a[3].split(",")
|
|
except Exception:pass
|
|
return False
|
|
|
|
def throttled_state():
|
|
raw=_run(["vcgencmd","get_throttled"])
|
|
m=re.search(r"0x([0-9a-fA-F]+)",raw);v=int(m.group(1),16) if m else 0
|
|
if not m: return {"raw":"UNKNOWN","error":"vcgencmd get_throttled unavailable or invalid"}
|
|
return {"raw":f"0x{v:x}","under_voltage_now":bool(v&1),"freq_capped_now":bool(v&2),"throttled_now":bool(v&4),"soft_temp_limit_now":bool(v&8),"under_voltage_occurred":bool(v&(1<<16)),"freq_capped_occurred":bool(v&(1<<17)),"throttled_occurred":bool(v&(1<<18)),"soft_temp_limit_occurred":bool(v&(1<<19))}
|
|
|
|
bus=None
|
|
bus_lock=threading.RLock()
|
|
FONT_PATH="/usr/local/share/pigway-pi-control/oled_font_5x7.bin"
|
|
FONT5=b""
|
|
agent={"loop_errors":0,"i2c_errors":0}
|
|
|
|
def bus_write(method,*args):
|
|
with bus_lock:
|
|
try: return getattr(bus,method)(*args)
|
|
except OSError:
|
|
agent["i2c_errors"]+=1
|
|
raise
|
|
|
|
fan_level=0
|
|
prev_cpu=None
|
|
page=0; page_since=time.monotonic()
|
|
cpu_hi=mem_hi=0
|
|
net_ok=link_ok=True
|
|
net_iface=""; net_kind="NET"; net_metric="DOWN"; ip_label="IP4"; ip="NO IP"
|
|
wifi_weak_since=None; wifi_weak_active=False
|
|
last_net=last_watch=0
|
|
watch_state=[]
|
|
|
|
def cmd(c): bus_write("write_byte_data",OLED,0x00,c)
|
|
def oled_init():
|
|
# Seller-native physical 128x32 configuration.
|
|
for c in [0xAE,0xD5,0x80,0xA8,0x1F,0xD3,0x00,0x40,0x8D,0x14,
|
|
0x20,0x00,0xA1,0xC8,0xDA,0x02,0x81,0x7F,0xD9,0xF1,
|
|
0xDB,0x40,0xA4,0xA6,0xAF]:
|
|
cmd(c)
|
|
|
|
def draw_text_5x7(pix,x,y,text):
|
|
for ch in text:
|
|
code=ord(ch)
|
|
if code<0 or code>255: code=ord("?")
|
|
off=code*5
|
|
if off+5>len(FONT5): code=ord("?"); off=code*5
|
|
for cx in range(5):
|
|
col=FONT5[off+cx]
|
|
for cy in range(7):
|
|
if col & (1<<cy):
|
|
xx=x+cx; yy=y+cy
|
|
if 0<=xx<W and 0<=yy<H: pix[xx,yy]=1
|
|
x+=6
|
|
if x>W-6: break
|
|
|
|
def oled_show(lines_lr, title_zh=False):
|
|
im=Image.new("1",(W,H),0); pix=im.load()
|
|
for row,(left,right) in enumerate(lines_lr[:4]):
|
|
y=row*8
|
|
draw_text_5x7(pix,0,y,left)
|
|
if right:
|
|
draw_text_5x7(pix,max(0,W-len(right)*6),y,right)
|
|
data=[]
|
|
for pg in range(4):
|
|
for x in range(W):
|
|
v=0
|
|
for bit in range(8):
|
|
if pix[x,pg*8+bit]: v|=1<<bit
|
|
data.append(v)
|
|
cmd(0x21);cmd(0);cmd(127);cmd(0x22);cmd(0);cmd(3)
|
|
for i in range(0,len(data),16):
|
|
bus_write("write_i2c_block_data",OLED,0x40,data[i:i+16])
|
|
|
|
|
|
def mcu(reg,val): bus_write("write_byte_data",MCU,reg,val)
|
|
_last_rgb=None
|
|
def rgb_off():
|
|
global _last_rgb
|
|
if _last_rgb != ("off",):
|
|
mcu(0x07,0x00) # seller closeRGB(): true OFF
|
|
_last_rgb=("off",)
|
|
|
|
def set_pi_led(name,off=True):
|
|
base=Path("/sys/class/leds")/name
|
|
try:
|
|
if off:
|
|
(base/"trigger").write_text("none")
|
|
(base/"brightness").write_text("0")
|
|
except Exception as e:
|
|
print(f"board LED {name}: {e}",flush=True)
|
|
|
|
def apply_dark_mode():
|
|
if DARK_PWR: set_pi_led("PWR",True)
|
|
if DARK_ACT: set_pi_led("ACT",True)
|
|
|
|
FAN_REG=[0x00,0x04,0x06,0x08,0x09,0x01]
|
|
FAN_NAME=["OFF","L1","L2","L3","L4","MAX"]
|
|
def fan_target(t):
|
|
if t<FAN_T[0]: return 0
|
|
if t<FAN_T[1]: return 1
|
|
if t<FAN_T[2]: return 2
|
|
if t<FAN_T[3]: return 3
|
|
if t<FAN_T[4]: return 4
|
|
return 5
|
|
def update_fan(t):
|
|
global fan_level
|
|
want=fan_target(t); old=fan_level
|
|
if want>fan_level: fan_level=want
|
|
elif want<fan_level:
|
|
boundary = FAN_T[max(0,fan_level-1)] - FAN_HYST
|
|
if t < boundary: fan_level-=1
|
|
if old!=fan_level: mcu(0x08,FAN_REG[fan_level])
|
|
|
|
def cpu_pct():
|
|
global prev_cpu
|
|
p=list(map(int,Path("/proc/stat").read_text().splitlines()[0].split()[1:9]))
|
|
idle=p[3]+p[4]; total=sum(p)
|
|
if prev_cpu is None: prev_cpu=(idle,total); return 0.0
|
|
di=idle-prev_cpu[0]; dt=total-prev_cpu[1]; prev_cpu=(idle,total)
|
|
return 100.0*(dt-di)/dt if dt else 0.0
|
|
def temp_c(): return int(Path("/sys/class/thermal/thermal_zone0/temp").read_text())/1000.0
|
|
def mem_pct():
|
|
vals={}
|
|
for l in Path("/proc/meminfo").read_text().splitlines():
|
|
if ":" in l:
|
|
k,v=l.split(":",1); vals[k]=int(v.split()[0])
|
|
return 100*(vals["MemTotal"]-vals["MemAvailable"])/vals["MemTotal"]
|
|
def disk_pct():
|
|
d=disk_usage("/"); return 100*d.used/d.total
|
|
def _run(args,timeout=2):
|
|
try: return subprocess.check_output(args,text=True,stderr=subprocess.DEVNULL,timeout=timeout).strip()
|
|
except Exception: return ""
|
|
|
|
def primary_network():
|
|
out=_run(["ip","-4","route","get","1.1.1.1"])
|
|
dev=re.search(r"\bdev\s+(\S+)",out); src=re.search(r"\bsrc\s+(\S+)",out); label="IP4"
|
|
if not dev:
|
|
out=_run(["ip","-6","route","get","2606:4700:4700::1111"])
|
|
dev=re.search(r"\bdev\s+(\S+)",out); src=re.search(r"\bsrc\s+(\S+)",out); label="IP6"
|
|
if not dev: return "","NET","DOWN","IP4","NO IP"
|
|
iface=dev.group(1); ipval=src.group(1) if src else ""
|
|
if Path(f"/sys/class/net/{iface}/wireless").exists():
|
|
kind="WIF"; metric="--"
|
|
try:
|
|
for line in Path("/proc/net/wireless").read_text().splitlines():
|
|
if line.lstrip().startswith(iface+":"):
|
|
metric=str(int(float(line.split()[3].rstrip(".")))); break
|
|
except Exception: pass
|
|
else:
|
|
kind="ETH"; metric="UP"
|
|
try:
|
|
n=int(Path(f"/sys/class/net/{iface}/speed").read_text().strip())
|
|
metric=f"{n//1000}G" if n>=1000 and n%1000==0 else str(n)
|
|
except Exception: pass
|
|
return iface,kind,metric,label,(ipval or "NO IP")
|
|
|
|
def get_ip():
|
|
return primary_network()[4]
|
|
|
|
def internet():
|
|
try:
|
|
s=socket.create_connection(("1.1.1.1",53),0.5);s.close();return True
|
|
except: return False
|
|
|
|
def load_cfg():
|
|
c=configparser.ConfigParser(); c.read(CFG)
|
|
watches=[]
|
|
if c.has_section("services"):
|
|
for label,target in c["services"].items():
|
|
target=target.strip()
|
|
if target and not target.startswith("#"): watches.append(("service",label.upper(),target))
|
|
if c.has_section("processes"):
|
|
for label,target in c["processes"].items():
|
|
target=target.strip()
|
|
if target and not target.startswith("#"): watches.append(("process",label.upper(),target))
|
|
return c,watches
|
|
cfg,watches=load_cfg()
|
|
|
|
def cf(section,key,default,cast=float):
|
|
try: return cast(cfg.get(section,key))
|
|
except: return default
|
|
|
|
SENSOR_INTERVAL=cf("timing","sensor_interval",1.0)
|
|
OLED_REFRESH=cf("timing","oled_refresh_interval",1.0)
|
|
PAGE_INTERVAL=cf("timing","page_interval",10.0)
|
|
NETWORK_INTERVAL=cf("timing","network_interval",5.0)
|
|
SERVICE_INTERVAL=cf("timing","service_interval",5.0)
|
|
HEALTH_INTERVAL=cf("timing","health_interval",1.0)
|
|
|
|
CPU_HIGH=cf("alerts","cpu_high",90.0)
|
|
CPU_RECOVER=cf("alerts","cpu_recover",75.0)
|
|
CPU_HIGH_SEC=cf("alerts","cpu_high_seconds",30,int)
|
|
MEM_HIGH=cf("alerts","memory_high",90.0)
|
|
MEM_RECOVER=cf("alerts","memory_recover",85.0)
|
|
MEM_HIGH_SEC=cf("alerts","memory_high_seconds",30,int)
|
|
TEMP_HIGH=cf("alerts","temp_high",68.0)
|
|
TEMP_CRIT=cf("alerts","temp_critical",75.0)
|
|
DISK_HIGH=cf("alerts","disk_high",90.0)
|
|
DISK_CRIT=cf("alerts","disk_critical",95.0)
|
|
WIFI_WEAK=cf("alerts","wifi_weak",-75.0)
|
|
WIFI_RECOVER=cf("alerts","wifi_recover",-68.0)
|
|
WIFI_WEAK_SECONDS=cf("alerts","wifi_weak_seconds",30.0)
|
|
|
|
FAN_T=[cf("fan","start",50.0),cf("fan","level2",53.0),cf("fan","level3",56.0),
|
|
cf("fan","level4",59.0),cf("fan","max",62.0)]
|
|
FAN_HYST=cf("fan","hysteresis",1.5)
|
|
ROW_Y=[cf("display","row1_y",0,int),cf("display","row2_y",8,int),cf("display","row3_y",16,int),cf("display","row4_y",24,int)]
|
|
DARK_PWR=cf("dark_mode","pi_pwr_led",True,lambda x: str(x).strip().lower() in ("off","0","false","no"))
|
|
DARK_ACT=cf("dark_mode","pi_act_led",True,lambda x: str(x).strip().lower() in ("off","0","false","no"))
|
|
RGB_WRITE_DELAY_MS=max(1,cf("led","write_delay_ms",10,int))
|
|
WARNING_FLASH=(max(50,cf("led","warning_flash_on_ms",1500,int)),max(50,cf("led","warning_flash_off_ms",1500,int)))
|
|
CRITICAL_FLASH=(max(50,cf("led","critical_flash_on_ms",750,int)),max(50,cf("led","critical_flash_off_ms",750,int)))
|
|
EMERGENCY_FLASH=(max(50,cf("led","emergency_flash_on_ms",300,int)),max(50,cf("led","emergency_flash_off_ms",300,int)))
|
|
|
|
def rgb_cfg(name,default):
|
|
return tuple(max(0,min(255,cf("led",f"{name}_{channel}",value,int)))
|
|
for channel,value in zip(("r","g","b"),default))
|
|
|
|
RGB_GROUP_VALUES={
|
|
"CPU":rgb_cfg("cpu",(255,0,0)),
|
|
"POWER":rgb_cfg("power",(255,40,0)),
|
|
"MEMORY":rgb_cfg("memory",(255,0,255)),
|
|
"STORAGE":rgb_cfg("storage",(255,255,255)),
|
|
"NETWORK":rgb_cfg("network",(0,0,255)),
|
|
"SERVICE":rgb_cfg("service",(0,255,96)),
|
|
}
|
|
RGB_OBJECT_GROUPS={
|
|
"ROOT_READONLY":"STORAGE",
|
|
"TEMP_CRITICAL":"CPU","TEMP_HIGH":"CPU","SOFT_TEMP_LIMIT":"CPU",
|
|
"THROTTLED":"CPU","FREQ_CAPPED":"CPU","CPU_HIGH":"CPU",
|
|
"POWER_UNDERVOLT":"POWER",
|
|
"MEMORY_HIGH":"MEMORY",
|
|
"DISK_CRITICAL":"STORAGE","DISK_HIGH":"STORAGE",
|
|
"NETWORK_DOWN":"NETWORK","INTERNET_DOWN":"NETWORK","WIFI_WEAK":"NETWORK",
|
|
"SERVICE_DOWN":"SERVICE","PROCESS_DOWN":"SERVICE",
|
|
}
|
|
|
|
def check_watch(w):
|
|
typ,label,target=w
|
|
args=["systemctl","is-active","--quiet",target] if typ=="service" else ["pgrep","-f","--",target]
|
|
try:
|
|
r=subprocess.run(args,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,timeout=2)
|
|
return r.returncode==0
|
|
except (OSError,subprocess.TimeoutExpired):
|
|
return False
|
|
|
|
|
|
def diagnose():
|
|
print("Agent:")
|
|
print(f"version={APP_VERSION} uptime_seconds={uptime_seconds()} config={CFG}")
|
|
print("loop_errors=UNAVAILABLE i2c_errors=UNAVAILABLE scope=separate_diagnostic_process")
|
|
print("System:")
|
|
print(f"load_average={load_average()} cpu_freq_mhz={cpu_freq_mhz():.0f} root_readonly={root_readonly()}")
|
|
print("Power:")
|
|
power=throttled_state()
|
|
print("throttled="+power["raw"])
|
|
for key in ("under_voltage_now","freq_capped_now","throttled_now","soft_temp_limit_now",
|
|
"under_voltage_occurred","freq_capped_occurred","throttled_occurred","soft_temp_limit_occurred"):
|
|
print(f"{key}={power.get(key,'UNKNOWN')}")
|
|
if "error" in power: print("error="+power["error"])
|
|
iface,kind,metric,label,address=primary_network()
|
|
print("Network:")
|
|
print(f"primary_interface={iface or 'NONE'} kind={kind} metric={metric} ip_label={label} IP={address} internet={internet()}")
|
|
print("I2C:")
|
|
# SMBus receive-byte performs a read only; no register selection or display commands.
|
|
try:
|
|
with SMBus(1) as probe:
|
|
for address,name in ((MCU,"MCU"),(OLED,"OLED")):
|
|
try:
|
|
probe.read_byte(address)
|
|
print(f"0x{address:02X} {name}=PRESENT")
|
|
except OSError as e:
|
|
status="MISSING" if e.errno in (errno.ENXIO,getattr(errno,"EREMOTEIO",121)) else "ERROR"
|
|
print(f"0x{address:02X} {name}={status} error={e}")
|
|
except OSError as e:
|
|
for address,name in ((MCU,"MCU"),(OLED,"OLED")):
|
|
print(f"0x{address:02X} {name}=ERROR error={e}")
|
|
print("Watches:")
|
|
for w in watches:
|
|
print(f"watch {w[1]}={'UP' if check_watch(w) else 'DOWN'} type={w[0]} target={w[2]}")
|
|
return 0
|
|
|
|
|
|
DISPLAY_PRIORITIES={
|
|
"ROOT_READONLY":100,
|
|
"TEMP_CRITICAL":90,
|
|
"SOFT_TEMP_LIMIT":85,
|
|
"POWER_UNDERVOLT":80,
|
|
"THROTTLED":75,
|
|
"FREQ_CAPPED":70,
|
|
"NETWORK_DOWN":65,
|
|
"INTERNET_DOWN":65,
|
|
"SERVICE_DOWN":60,
|
|
"PROCESS_DOWN":60,
|
|
"DISK_CRITICAL":55,
|
|
"TEMP_HIGH":50,
|
|
"CPU_HIGH":45,
|
|
"MEMORY_HIGH":45,
|
|
"DISK_HIGH":40,
|
|
"WIFI_WEAK":30,
|
|
"NORMAL_HOME":0,
|
|
}
|
|
|
|
def display_priority(request_id):
|
|
return DISPLAY_PRIORITIES[request_id.split("|",1)[0]]
|
|
|
|
def transition(key,active,on_event,off_event,**fields):
|
|
old=event_state.get(key,False)
|
|
if active!=old:
|
|
event_state[key]=active
|
|
log("WARN" if active else "INFO",on_event if active else off_event,**fields)
|
|
|
|
def update_active_alerts(cpu,temp,mem,disk,sample_elapsed=0.0):
|
|
global wifi_weak_since,wifi_weak_active
|
|
global cpu_hi,mem_hi
|
|
cpu_hi=cpu_hi+sample_elapsed if cpu>=CPU_HIGH else 0 if cpu<CPU_RECOVER else cpu_hi
|
|
mem_hi=mem_hi+sample_elapsed if mem>=MEM_HIGH else 0 if mem<MEM_RECOVER else mem_hi
|
|
if cpu_hi>=CPU_HIGH_SEC: display.request("CPU_HIGH",1,"CPU LOAD HIGH",f"LOAD {cpu:.1f}%",f"TEMP {temp:.1f}C","CHECK LOAD")
|
|
else: display.clear("CPU_HIGH")
|
|
if temp>=TEMP_CRIT:
|
|
display.request("TEMP_CRITICAL",3,"TEMP CRITICAL",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING")
|
|
display.clear("TEMP_HIGH")
|
|
elif temp>=TEMP_HIGH:
|
|
display.request("TEMP_HIGH",1,"TEMP HIGH",f"TEMP {temp:.1f}C",f"FAN {FAN_NAME[fan_level]}","CHECK COOLING")
|
|
display.clear("TEMP_CRITICAL")
|
|
else:
|
|
display.clear("TEMP_HIGH"); display.clear("TEMP_CRITICAL")
|
|
if mem_hi>=MEM_HIGH_SEC: display.request("MEMORY_HIGH",1,"MEMORY HIGH",f"USED {mem:.1f}%","CHECK PROCESS","")
|
|
else: display.clear("MEMORY_HIGH")
|
|
if disk>=DISK_CRIT:
|
|
display.request("DISK_CRITICAL",2,"DISK CRITICAL",f"USED {disk:.1f}%","FREE < 5%","CHECK STORAGE")
|
|
display.clear("DISK_HIGH")
|
|
elif disk>=DISK_HIGH:
|
|
display.request("DISK_HIGH",1,"DISK SPACE LOW",f"USED {disk:.1f}%","FREE < 10%","CHECK STORAGE")
|
|
display.clear("DISK_CRITICAL")
|
|
else:
|
|
display.clear("DISK_HIGH"); display.clear("DISK_CRITICAL")
|
|
if not link_ok:
|
|
display.request("NETWORK_DOWN",2,"NETWORK DOWN","NO LAN / WIFI","NO IP ADDRESS","")
|
|
display.clear("INTERNET_DOWN")
|
|
elif not net_ok:
|
|
display.request("INTERNET_DOWN",2,"INTERNET DOWN","LOCAL LINK OK",ip,"")
|
|
display.clear("NETWORK_DOWN")
|
|
else:
|
|
display.clear("NETWORK_DOWN"); display.clear("INTERNET_DOWN")
|
|
for w,ok in zip(watches,watch_state):
|
|
alert_id=("SERVICE_DOWN" if w[0]=="service" else "PROCESS_DOWN")+"|"+w[1]
|
|
if not ok:
|
|
display.request(alert_id,2,"SERVICE DOWN" if w[0]=="service" else "PROCESS DOWN",w[1],"CHECK / RESTART","CRITICAL",{"target":w[2]})
|
|
else: display.clear(alert_id)
|
|
if net_kind=="WIF" and str(net_metric).lstrip("-").isdigit():
|
|
rssi=float(net_metric)
|
|
if not wifi_weak_active:
|
|
if rssi <= WIFI_WEAK:
|
|
if wifi_weak_since is None: wifi_weak_since=time.monotonic()
|
|
if time.monotonic()-wifi_weak_since >= WIFI_WEAK_SECONDS: wifi_weak_active=True
|
|
else: wifi_weak_since=None
|
|
elif rssi >= WIFI_RECOVER:
|
|
wifi_weak_active=False; wifi_weak_since=None
|
|
if wifi_weak_active:
|
|
display.request("WIFI_WEAK",1,"WIFI SIGNAL WEAK",f"RSSI {rssi:.0f} dBm",f"IF {net_iface}","CHECK SIGNAL")
|
|
else: display.clear("WIFI_WEAK")
|
|
else:
|
|
wifi_weak_since=None; wifi_weak_active=False; display.clear("WIFI_WEAK")
|
|
if health.get("under_voltage_now"):
|
|
display.request("POWER_UNDERVOLT",2,"POWER LOW","UNDERVOLTAGE","CHECK POWER","CRITICAL",{"raw":health.get("raw","")})
|
|
else: display.clear("POWER_UNDERVOLT")
|
|
if health.get("throttled_now"):
|
|
display.request("THROTTLED",2,"CPU THROTTLED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"CHECK POWER")
|
|
else: display.clear("THROTTLED")
|
|
if health.get("freq_capped_now"):
|
|
display.request("FREQ_CAPPED",2,"FREQ CAPPED",f"FREQ {cpu_freq_mhz():.0f}M",health.get("raw",""),"CHECK POWER")
|
|
else: display.clear("FREQ_CAPPED")
|
|
if health.get("soft_temp_limit_now"):
|
|
display.request("SOFT_TEMP_LIMIT",2,"SOFT TEMP LIMIT","CPU TEMP LIMITED",health.get("raw",""),"CHECK COOLING")
|
|
else: display.clear("SOFT_TEMP_LIMIT")
|
|
if fs_readonly: display.request("ROOT_READONLY",3,"FILESYSTEM RO","ROOT READ ONLY","CHECK STORAGE","EMERGENCY")
|
|
else: display.clear("ROOT_READONLY")
|
|
|
|
def rgb_mode_for(request_id,severity):
|
|
if severity<=0: return "OFF"
|
|
alert_type=request_id.split("|",1)[0]
|
|
color=RGB_OBJECT_GROUPS[alert_type]
|
|
level="EMERGENCY" if severity>=3 else "CRITICAL" if severity>=2 else "WARNING"
|
|
return f"{color}_{level}_FLASH"
|
|
|
|
def rgb_flash(mode,on):
|
|
global _last_rgb
|
|
color=mode.split("_",1)[0]
|
|
channels=RGB_GROUP_VALUES[color]
|
|
state=(mode,bool(on))
|
|
if _last_rgb==state: return
|
|
if not on:
|
|
mcu(0x07,0x00)
|
|
else:
|
|
mcu(0x00,0xff); time.sleep(RGB_WRITE_DELAY_MS/1000.0)
|
|
mcu(0x01,channels[0]); time.sleep(RGB_WRITE_DELAY_MS/1000.0)
|
|
mcu(0x02,channels[1]); time.sleep(RGB_WRITE_DELAY_MS/1000.0)
|
|
mcu(0x03,channels[2])
|
|
_last_rgb=state
|
|
|
|
def render_home(cpu,temp,mem,disk):
|
|
# 128x32 fixed columns. Both columns are LEFT-aligned.
|
|
# Seller 5x7 font has 6px character pitch; right column starts at x=72.
|
|
im=Image.new("1",(W,H),0); pix=im.load()
|
|
display_ip=ip[:7]+".."+ip[-7:] if ip_label=="IP6" and len(ip)>18 else ip
|
|
left=[
|
|
f"CPU {cpu:.1f}%",
|
|
f"TMP {temp:.1f}C",
|
|
f"DSK {disk:.1f}%",
|
|
ip_label+" "+display_ip,
|
|
]
|
|
right=[
|
|
f"MEM {mem:.1f}%",
|
|
f"FAN {FAN_NAME[fan_level]}",
|
|
f"{net_kind} {net_metric}",
|
|
"",
|
|
]
|
|
for row in range(4):
|
|
y=row*8
|
|
draw_text_5x7(pix,0,y,left[row])
|
|
if right[row]:
|
|
draw_text_5x7(pix,72,y,right[row])
|
|
data=[]
|
|
for pg in range(4):
|
|
for x in range(W):
|
|
v=0
|
|
for bit in range(8):
|
|
if pix[x,pg*8+bit]: v|=1<<bit
|
|
data.append(v)
|
|
cmd(0x21);cmd(0);cmd(127);cmd(0x22);cmd(0);cmd(3)
|
|
for i in range(0,len(data),16):
|
|
bus_write("write_i2c_block_data",OLED,0x40,data[i:i+16])
|
|
|
|
|
|
def render_alert(a,index,total):
|
|
oled_show([(a["title"],f"{index}/{total}"),
|
|
(a["l2"],""),(a["l3"],""),(a["l4"],"")])
|
|
|
|
|
|
class DisplayController:
|
|
def __init__(self):
|
|
now=time.monotonic()
|
|
self.lock=threading.RLock()
|
|
self.requests={"NORMAL_HOME":{
|
|
"id":"NORMAL_HOME","priority":0,"severity":0,"rgb_mode":"OFF",
|
|
"title":"","l2":"","l3":"","l4":"","active":True,
|
|
"first_seen":now,"last_changed":now,"metadata":{"home":(0.0,0.0,0.0,0.0)},
|
|
}}
|
|
self.current_owner_id="NORMAL_HOME"
|
|
self.current_priority=0
|
|
self.current_rgb_mode="OFF"
|
|
self.owner_since=now
|
|
self.current_oled_page_id="NORMAL_HOME"
|
|
self.page_started=now
|
|
self.batch_depth=0
|
|
self.stop_event=threading.Event()
|
|
self.rgb_thread=threading.Thread(target=self._rgb_worker,name="rgb",daemon=True)
|
|
|
|
def start(self):
|
|
oled_init(); self.refresh(); rgb_off(); self.rgb_thread.start()
|
|
|
|
def update_home(self,cpu,temp,mem,disk):
|
|
with self.lock:
|
|
request=self.requests["NORMAL_HOME"]
|
|
values=(cpu,temp,mem,disk)
|
|
if request["metadata"].get("home")!=values:
|
|
request["metadata"]={"home":values}; request["last_changed"]=time.monotonic()
|
|
|
|
def request(self,request_id,severity,title,l2="",l3="",l4="",metadata=None):
|
|
now=time.monotonic(); priority=display_priority(request_id)
|
|
request={"id":request_id,"priority":priority,"severity":severity,
|
|
"rgb_mode":rgb_mode_for(request_id,severity),"title":title[:21],
|
|
"l2":l2[:21],"l3":l3[:21],"l4":l4[:21],"active":True,
|
|
"last_changed":now,"metadata":metadata or {}}
|
|
with self.lock:
|
|
current=self.requests.get(request_id)
|
|
if current is None:
|
|
request["first_seen"]=now; self.requests[request_id]=request
|
|
log("WARN","ALERT_ACTIVE",alert=request_id,severity=severity,detail=l2)
|
|
else:
|
|
request["first_seen"]=current["first_seen"]
|
|
changed=any(current.get(k)!=request.get(k) for k in
|
|
("priority","severity","rgb_mode","title","l2","l3","l4","metadata"))
|
|
if not changed: request["last_changed"]=current["last_changed"]
|
|
self.requests[request_id]=request
|
|
if self.batch_depth==0:
|
|
self._arbitrate_locked()
|
|
self._sync_rgb_locked()
|
|
|
|
def clear(self,request_id):
|
|
if request_id=="NORMAL_HOME": return
|
|
with self.lock:
|
|
if self.requests.pop(request_id,None) is not None:
|
|
log("INFO","ALERT_RECOVERED",alert=request_id)
|
|
if self.batch_depth==0:
|
|
self._arbitrate_locked()
|
|
if self.current_oled_page_id not in self.requests:
|
|
self._show_page_locked(self.current_owner_id)
|
|
else: self._sync_rgb_locked()
|
|
|
|
def begin_update(self):
|
|
with self.lock: self.batch_depth+=1
|
|
|
|
def end_update(self):
|
|
with self.lock:
|
|
if self.batch_depth<=0: raise RuntimeError("display update batch imbalance")
|
|
self.batch_depth-=1
|
|
if self.batch_depth==0:
|
|
self._arbitrate_locked()
|
|
if self.current_oled_page_id not in self.requests:
|
|
self._show_page_locked(self.current_owner_id)
|
|
else: self._sync_rgb_locked()
|
|
|
|
def _arbitrate_locked(self):
|
|
current=self.requests.get(self.current_owner_id)
|
|
ordered=sorted(self.requests.values(),key=lambda r:(-r["priority"],r["first_seen"],r["id"]))
|
|
candidate=ordered[0]
|
|
if current is not None and candidate["priority"]<=current["priority"]: return
|
|
self._change_owner_locked(candidate)
|
|
|
|
def _change_owner_locked(self,request):
|
|
old=self.current_owner_id
|
|
if request["id"]==old: return
|
|
self.current_owner_id=request["id"]
|
|
self.current_priority=request["priority"]
|
|
self.current_rgb_mode=request["rgb_mode"]
|
|
self.owner_since=time.monotonic()
|
|
log("INFO","DISPLAY_OWNER",old=old,new=request["id"],priority=request["priority"])
|
|
self._show_page_locked(request["id"])
|
|
|
|
def _ordered_alerts_locked(self):
|
|
return sorted((r for r in self.requests.values() if r["id"]!="NORMAL_HOME"),
|
|
key=lambda r:(-r["priority"],r["first_seen"],r["id"]))
|
|
|
|
def _show_page_locked(self,page_id):
|
|
alerts=self._ordered_alerts_locked()
|
|
request=self.requests.get(page_id,self.requests["NORMAL_HOME"])
|
|
self.current_oled_page_id=request["id"]
|
|
self.page_started=time.monotonic()
|
|
self._sync_rgb_locked()
|
|
if request["id"]=="NORMAL_HOME":
|
|
render_home(*request["metadata"]["home"])
|
|
else:
|
|
alert_ids=[a["id"] for a in alerts]
|
|
render_alert(request,alert_ids.index(request["id"])+1,len(alerts))
|
|
|
|
def _sync_rgb_locked(self):
|
|
# Alert pages show that alert's object color. HOME keeps the highest-
|
|
# priority active owner visible instead of briefly switching RGB off.
|
|
rgb_request=self.requests.get(self.current_oled_page_id)
|
|
if self.current_oled_page_id=="NORMAL_HOME":
|
|
rgb_request=self.requests.get(self.current_owner_id,self.requests["NORMAL_HOME"])
|
|
if rgb_request is None:
|
|
rgb_request=self.requests["NORMAL_HOME"]
|
|
self.current_rgb_mode=rgb_request["rgb_mode"]
|
|
|
|
def refresh(self):
|
|
with self.lock:
|
|
alerts=self._ordered_alerts_locked()
|
|
pages=["NORMAL_HOME"]+[a["id"] for a in alerts]
|
|
if self.current_oled_page_id not in pages:
|
|
self._show_page_locked(self.current_owner_id)
|
|
return
|
|
if len(pages)>1 and time.monotonic()-self.page_started>=PAGE_INTERVAL:
|
|
pos=pages.index(self.current_oled_page_id)
|
|
self._show_page_locked(pages[(pos+1)%len(pages)])
|
|
return
|
|
request=self.requests[self.current_oled_page_id]
|
|
if request["id"]=="NORMAL_HOME": render_home(*request["metadata"]["home"])
|
|
else:
|
|
alert_ids=[a["id"] for a in alerts]
|
|
render_alert(request,alert_ids.index(request["id"])+1,len(alerts))
|
|
|
|
def owner_snapshot(self):
|
|
with self.lock: return dict(self.requests[self.current_owner_id])
|
|
|
|
def status_snapshot(self):
|
|
with self.lock:
|
|
alerts=self._ordered_alerts_locked()
|
|
return {
|
|
"owner":self.current_owner_id,
|
|
"oled_page":self.current_oled_page_id,
|
|
"rgb_mode":self.current_rgb_mode,
|
|
"alerts":[{k:a[k] for k in ("id","priority","severity","title","l2","l3","l4","first_seen","last_changed")}
|
|
for a in alerts],
|
|
}
|
|
|
|
def _rgb_worker(self):
|
|
previous_error=None; applied_mode="OFF"; applied_flash=False; animation_started=time.monotonic()
|
|
while not self.stop_event.is_set():
|
|
try:
|
|
with self.lock: wanted_mode=self.current_rgb_mode
|
|
wanted_flash=wanted_mode.endswith("_FLASH")
|
|
if wanted_mode!=applied_mode:
|
|
with bus_lock:
|
|
if wanted_mode=="OFF": rgb_off()
|
|
elif wanted_flash: rgb_flash(wanted_mode,True)
|
|
else: raise ValueError(f"unsupported RGB mode: {wanted_mode}")
|
|
log("INFO","RGB_MODE",old=applied_mode,new=wanted_mode)
|
|
applied_mode=wanted_mode; animation_started=time.monotonic()
|
|
applied_flash=applied_mode.endswith("_FLASH")
|
|
if applied_flash:
|
|
elapsed_ms=(time.monotonic()-animation_started)*1000.0
|
|
if "_EMERGENCY_" in applied_mode: on_ms,off_ms=EMERGENCY_FLASH
|
|
elif "_CRITICAL_" in applied_mode: on_ms,off_ms=CRITICAL_FLASH
|
|
else: on_ms,off_ms=WARNING_FLASH
|
|
on=elapsed_ms%(on_ms+off_ms)<on_ms
|
|
with bus_lock: rgb_flash(applied_mode,on)
|
|
if previous_error is not None: log("INFO","RGB_RECOVERED")
|
|
previous_error=None
|
|
except Exception as e:
|
|
if repr(e)!=previous_error: log("ERROR","RGB_ERROR",error=repr(e))
|
|
previous_error=repr(e)
|
|
delay=0.04 if applied_flash else 0.25
|
|
self.stop_event.wait(delay)
|
|
|
|
def shutdown(self):
|
|
self.stop_event.set()
|
|
if self.rgb_thread.is_alive(): self.rgb_thread.join()
|
|
for name,action in (("rgb",rgb_off),("oled_clear",lambda:oled_show([("","")]*4)),
|
|
("oled_off",lambda:cmd(0xAE))):
|
|
try: action()
|
|
except Exception as e: log("ERROR","SHUTDOWN_CLEANUP",step=name,error=repr(e))
|
|
|
|
def cleanup():
|
|
log("INFO","STOP",loop_errors=agent["loop_errors"],i2c_errors=agent["i2c_errors"])
|
|
if bus is None: return
|
|
display.shutdown()
|
|
for name,action in (("fan",lambda:mcu(0x08,0x00)),("bus_close",bus.close)):
|
|
try: action()
|
|
except Exception as e: log("ERROR","SHUTDOWN_CLEANUP",step=name,error=repr(e))
|
|
try: Path(STATUS_PATH).unlink(missing_ok=True)
|
|
except Exception as e: log("ERROR","SHUTDOWN_CLEANUP",step="status_file",error=repr(e))
|
|
|
|
def write_status(cpu,temp,mem,disk):
|
|
display_state=display.status_snapshot()
|
|
state={
|
|
"version":APP_VERSION,"timestamp":int(time.time()),"uptime_seconds":uptime_seconds(),
|
|
"system":{"load_average":load_average(),"cpu_percent":round(cpu,1),
|
|
"cpu_freq_mhz":round(cpu_freq_mhz()),"temperature_c":round(temp,1),
|
|
"memory_percent":round(mem,1),"disk_percent":round(disk,1),
|
|
"root_readonly":bool(fs_readonly),"fan_level":fan_level,"fan_name":FAN_NAME[fan_level]},
|
|
"power":health,
|
|
"network":{"interface":net_iface,"kind":net_kind,"metric":net_metric,
|
|
"ip_label":ip_label,"ip":ip,"link":bool(link_ok),"internet":bool(net_ok)},
|
|
"watches":[{"type":w[0],"name":w[1],"target":w[2],"up":bool(ok)}
|
|
for w,ok in zip(watches,watch_state)],
|
|
"display":display_state,
|
|
"agent":{"loop_errors":agent["loop_errors"],"i2c_errors":agent["i2c_errors"]},
|
|
}
|
|
path=Path(STATUS_PATH); path.parent.mkdir(mode=0o755,parents=True,exist_ok=True)
|
|
temporary=path.with_suffix(".tmp")
|
|
temporary.write_text(json.dumps(state,ensure_ascii=False,separators=(",",":")))
|
|
temporary.chmod(0o644); temporary.replace(path)
|
|
|
|
# Dispatch only after all shared checks and configuration have been initialized.
|
|
parser=argparse.ArgumentParser()
|
|
parser.add_argument("--diagnose",action="store_true")
|
|
args=parser.parse_args()
|
|
if args.diagnose: raise SystemExit(diagnose())
|
|
|
|
watch_state=[]
|
|
last_sensor=last_render=last_status=0.0
|
|
last_net=last_watch=last_health=0.0
|
|
health={}; fs_readonly=False
|
|
agent={"last_sensor_ok":0.0,"last_network_ok":0.0,"last_watch_ok":0.0,"last_health_ok":0.0,"loop_errors":0,"i2c_errors":0}
|
|
event_state={}
|
|
history_logged=set()
|
|
latest=(0.0,0.0,0.0,0.0)
|
|
display=DisplayController()
|
|
try:
|
|
bus=SMBus(1)
|
|
FONT5=Path(FONT_PATH).read_bytes()
|
|
display.start(); apply_dark_mode()
|
|
log("INFO","START",version=APP_VERSION,uptime=uptime_seconds())
|
|
while running:
|
|
try:
|
|
now=time.monotonic()
|
|
sample_elapsed=0.0
|
|
|
|
|
|
# Independent clocks: update data/state only.
|
|
if now-last_sensor >= SENSOR_INTERVAL:
|
|
sample_elapsed=now-last_sensor if last_sensor else 0.0
|
|
c=cpu_pct(); t=temp_c(); m=mem_pct(); d=disk_pct()
|
|
latest=(c,t,m,d); update_fan(t); last_sensor=now; agent["last_sensor_ok"]=now
|
|
|
|
if now-last_net >= NETWORK_INTERVAL:
|
|
old_id=(net_iface,ip)
|
|
net_iface,net_kind,net_metric,ip_label,ip=primary_network(); link_ok=ip!="NO IP"; net_ok=internet() if link_ok else False
|
|
new_id=(net_iface,ip)
|
|
if new_id!=old_id:
|
|
log("INFO","NET_SWITCH",old_if=old_id[0],old_ip=old_id[1],new_if=net_iface,new_ip=ip)
|
|
transition("network_down",not link_ok,"NETWORK_DOWN","NETWORK_RECOVERED",iface=net_iface,ip=ip)
|
|
transition("internet_down",link_ok and not net_ok,"INTERNET_DOWN","INTERNET_RECOVERED",iface=net_iface,ip=ip)
|
|
last_net=now; agent["last_network_ok"]=now
|
|
|
|
if now-last_watch >= SERVICE_INTERVAL:
|
|
watch_state=[check_watch(w) for w in watches]
|
|
for w,ok in zip(watches,watch_state):
|
|
transition("watch:"+w[0]+":"+w[1],not ok,"WATCH_DOWN","WATCH_RECOVERED",name=w[1],target=w[2])
|
|
last_watch=now; agent["last_watch_ok"]=now
|
|
|
|
if now-last_health >= HEALTH_INTERVAL:
|
|
power_sample=throttled_state(); fs_readonly=root_readonly()
|
|
transition("power_check", "error" in power_sample,"POWER_CHECK_ERROR","POWER_CHECK_RECOVERED")
|
|
# An unavailable reading is not evidence that an existing fault recovered.
|
|
if "error" not in power_sample: health=power_sample
|
|
transition("undervoltage",health.get("under_voltage_now",False),"POWER_LOW","POWER_RECOVERED",raw=health.get("raw"))
|
|
transition("throttled",health.get("throttled_now",False) or health.get("freq_capped_now",False),"THROTTLED","THROTTLE_RECOVERED",raw=health.get("raw"),freq_mhz=f"{cpu_freq_mhz():.0f}")
|
|
transition("soft_temp",health.get("soft_temp_limit_now",False),"SOFT_TEMP_LIMIT","SOFT_TEMP_RECOVERED",raw=health.get("raw"))
|
|
transition("root_ro",fs_readonly,"FILESYSTEM_RO","FILESYSTEM_RW")
|
|
for k in ("under_voltage_occurred","freq_capped_occurred","throttled_occurred","soft_temp_limit_occurred"):
|
|
if health.get(k) and k not in history_logged:
|
|
history_logged.add(k); log("WARN","HISTORY_FLAG",flag=k,raw=health.get("raw"))
|
|
agent["last_health_ok"]=now; last_health=now
|
|
|
|
c,t,m,d=latest
|
|
display.update_home(c,t,m,d)
|
|
display.begin_update()
|
|
try: update_active_alerts(c,t,m,d,sample_elapsed)
|
|
finally: display.end_update()
|
|
if now-last_render >= OLED_REFRESH:
|
|
display.refresh()
|
|
last_render=now
|
|
if now-last_status >= 1.0:
|
|
write_status(c,t,m,d); last_status=now
|
|
|
|
time.sleep(0.20)
|
|
except Exception as e:
|
|
agent["loop_errors"]+=1; log("ERROR","LOOP_ERROR",count=agent["loop_errors"],error=repr(e)); time.sleep(1)
|
|
finally:
|
|
cleanup()
|