feat: add manual disk discovery and weekly capacity cache

app/pigway_pi_control.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python3
 import time,socket,subprocess,configparser,signal,re,argparse,json,os,platform
 from pathlib import Path
-APP_VERSION="3.10.0"
+APP_VERSION="3.11.2"
 import sys
 sys.path.append("/usr/local/share/pigway-pi-control")
 from plugin_api import Publisher,registered,discover
 from network_rates import NetworkRates
-from storage_cache import StorageCache
+from storage_cache import StorageCache,StorageInventory
 storage_cache=StorageCache("/run/pigway-pi-control/storage-cache.json")
 
 CFG="/etc/pigway-pi-control.conf"
@@ -199,6 +199,7 @@ def load_cfg():
             if target and not target.startswith("#"): watches.append(("process",label.upper(),target))
     return c,watches
 cfg,watches=load_cfg()
+storage_inventory=StorageInventory(cache_path=Path(CFG).with_suffix(".storage.json"),capacity_path="/run/pigway-pi-control/storage-inventory-usage.json")
 
 def cf(section,key,default,cast=float):
     try: return cast(cfg.get(section,key))
@@ -427,15 +428,17 @@ def write_status(cpu,temp,mem,disk):
     display_state=alert_manager.status_snapshot()
     memory_total,memory_used,memory_available,memory_calculated=memory_usage()
     storage=storage_cache.sample()
+    disks=storage_inventory.sample(cfg)
     disk_total,disk_used,disk_free=(storage[k] for k in ("total","used","free")) if storage else (None,None,None)
     signal=wifi_percent(net_metric) if net_kind=="WIF" else None
     state={
         "version":APP_VERSION,"timestamp":int(time.time()),
         "collection":{"interval_seconds":min(1.0,SENSOR_INTERVAL,NETWORK_INTERVAL,SERVICE_INTERVAL,HEALTH_INTERVAL),
           "refresh":{
+            "storage_inventory":{"interval_seconds":604800,"observed_at":storage_inventory.observed_at,"next_update_at":storage_inventory.next_update_at,"error":storage_inventory.error},
             "system":{"interval_seconds":SENSOR_INTERVAL,"observed_at":sensor_observed_at,"next_update_at":sensor_observed_at+SENSOR_INTERVAL if sensor_observed_at else None},
             "network":{"interval_seconds":NETWORK_INTERVAL,"observed_at":network_observed_at,"next_update_at":network_observed_at+NETWORK_INTERVAL if network_observed_at else None},
-            "storage":{"interval_seconds":86400,"observed_at":storage_cache.observed_at,"next_update_at":storage_cache.next_update_at,"error":storage_cache.error},
+            "storage":{"interval_seconds":604800,"observed_at":storage_cache.observed_at,"next_update_at":storage_cache.next_update_at,"error":storage_cache.error},
             "display":{"interval_seconds":1,"next_update_at":time.time()+1},
             "alerts":{"interval_seconds":1,"next_update_at":time.time()+1}}},"uptime_seconds":uptime_seconds(),
         "machine":{"id":DEVICE_ID,"name":DEVICE_NAME,"hostname":socket.gethostname()},
@@ -446,6 +449,9 @@ def write_status(cpu,temp,mem,disk):
                   "memory_total_bytes":memory_total,"memory_used_bytes":memory_used,
                   "memory_available_bytes":memory_available,
                   "storage_observed_at":storage_cache.observed_at,
+                  "storage_inventory":disks,"storage_inventory_observed_at":storage_inventory.observed_at,
+                  "storage_inventory_error":storage_inventory.error,
+                  "storage_scanned_at":storage_inventory.scanned_at,"storage_discovery_mode":"manual",
                   "disk_percent":round(disk,1) if disk is not None else None,"disk_total_bytes":disk_total,
                   "disk_used_bytes":disk_used,"disk_free_bytes":disk_free,
                   "root_readonly":bool(fs_readonly)},

app/pigway_pi_control_api.py

@@ -7,14 +7,15 @@ from concurrent.futures import ThreadPoolExecutor,as_completed
 from http import HTTPStatus
 from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer
 from pathlib import Path
+STORAGE_SCAN_LOCK=threading.Lock()
 from urllib.parse import parse_qs,urlencode,urlparse
 from urllib.request import Request,urlopen
 
-APP_VERSION="3.7.2"
+APP_VERSION="3.11.2"
 CFG=Path("/etc/pigway-pi-control.conf")
 STATUS=Path("/run/pigway-pi-control/status.json")
 TOKEN=Path("/etc/pigway-pi-control-api.token")
-ALLOWED_SECTIONS={"device","services","processes","alerts","timing","dark_mode","api"}
+ALLOWED_SECTIONS={"device","services","processes","alerts","timing","dark_mode","api","storage_names","storage_visible"}
 NAME_RE=re.compile(r"^[A-Za-z0-9_.-]{1,64}$")
 UNIT_RE=re.compile(r"^[A-Za-z0-9_.@:-]{1,128}\.service$")
 RGB_KEY_RE=re.compile(r"^(cpu|power|memory|storage|network|service|normal)_[rgb]$")
@@ -74,9 +75,16 @@ def api_setting(key,default,cast=str):
 
 def config_json():
     cfg=load_cfg()
-    return {section:{k:v for k,v in cfg.items(section) if not (section=="timing" and k in {"oled_refresh_interval","page_interval"})} for section in cfg.sections() if section in ALLOWED_SECTIONS}
+    return {section:{k:v for k,v in cfg.items(section,raw=True) if not (section=="timing" and k in {"oled_refresh_interval","page_interval"})} for section in cfg.sections() if section in ALLOWED_SECTIONS}
 
 def validate_value(section,key,value,existing):
+    if section in {"storage_names","storage_visible"}:
+        if not re.fullmatch(r"[a-f0-9]{32}",key):raise ValueError("invalid storage identifier")
+        if section=="storage_visible":
+            if value.lower() not in {"true","false"}:raise ValueError("visibility must be true or false")
+        elif len(value)>64 or any(ord(c)<32 for c in value):
+            raise ValueError("storage name must be at most 64 characters without control characters")
+        return
     if section=="api" and key=="enabled":
         if value.lower() not in {"true","false"}:raise ValueError("enabled must be true or false")
         return
@@ -388,6 +396,16 @@ class Handler(BaseHTTPRequestHandler):
 
     def do_PUT(self):
         path=urlparse(self.path).path
+        if path=="/api/v1/storage/scan":
+            if not self.authorized():self.send_json(401,{"error":"bearer token required"});return
+            try:
+                from storage_cache import StorageInventory
+                with STORAGE_SCAN_LOCK:
+                    result=StorageInventory(cache_path=CFG.with_suffix(".storage.json")).scan()
+                self.send_json(200,result)
+            except Exception:
+                self.send_json(503,{"error":"storage scan failed; previous inventory retained"})
+            return
         if path not in {"/api/v1/plugins","/api/v1/plugin/config","/api/v1/config","/api/v1/services","/api/v1/services/control","/api/v1/processes"}: self.send_json(404,{"error":"not found"}); return
         if not self.authorized(): self.send_json(401,{"error":"bearer token required"}); return
         try:

app/storage_cache.py

@@ -4,6 +4,159 @@ import time
 from pathlib import Path
 from shutil import disk_usage
 
+class StorageInventory:
+    """Read host block devices without mounting, probing SMART, or writing disks."""
+
+    def __init__(self, clock=time.time, runner=None, mounts_path="/proc/1/mountinfo", usage=None, cache_path=None, capacity_path=None):
+        import subprocess
+        self.clock = clock
+        self.runner = runner or (lambda: subprocess.check_output(
+            ["lsblk", "--json", "--bytes", "--paths", "--output",
+             "NAME,KNAME,TYPE,SIZE,FSTYPE,UUID,LABEL,MODEL,SERIAL,MAJ:MIN"],
+            text=True, timeout=5, stderr=subprocess.DEVNULL))
+        self.mounts_path = Path(mounts_path)
+        self.usage = usage or (lambda mount: disk_usage("/proc/1/root" + mount))
+        self.next_update_at = 0
+        self.observed_at = None
+        self.error = False
+        self.devices = []
+        self.cache_path = Path(cache_path) if cache_path else None
+        self.scanned_at = None
+        self.capacity_path = Path(capacity_path) if capacity_path else None
+        if self.capacity_path:
+            try:
+                saved=json.loads(self.capacity_path.read_text())
+                at=saved['observed_at']
+                if isinstance(at,(int,float)) and 0 <= self.clock()-at < 604800 and isinstance(saved['devices'],list):
+                    self.devices=saved['devices'];self.observed_at=at
+                    self.scanned_at=saved['scanned_at'];self.next_update_at=at+604800
+            except (OSError,ValueError,KeyError,TypeError):pass
+
+    def scan(self):
+        if self.cache_path is None:
+            raise ValueError("storage inventory cache path required")
+        try:
+            previous = json.loads(self.cache_path.read_text())
+            if 0 <= self.clock() - previous["scanned_at"] < 60:
+                return {"accepted": True, "scanned_at": previous["scanned_at"], "cooldown": True}
+        except (OSError, ValueError, KeyError, TypeError):
+            pass
+        data = json.loads(self.runner())
+        self.discover(data)
+        document = {"scanned_at": self.clock(), "inventory": data}
+        temporary = self.cache_path.with_suffix(".tmp")
+        with open(temporary,"w",opener=lambda path, flags: __import__("os").open(path, flags, 0o600)) as file:
+            json.dump(document, file)
+        temporary.replace(self.cache_path)
+        return {"accepted": True, "scanned_at": document["scanned_at"], "cooldown": False}
+
+    def sample(self, config=None):
+        data = None
+        changed = False
+        if self.cache_path is not None:
+            try:
+                document = json.loads(self.cache_path.read_text())
+                changed = document["scanned_at"] != self.scanned_at
+                data = document["inventory"]
+                self.scanned_at = document["scanned_at"]
+            except FileNotFoundError:
+                pass
+            except (OSError, ValueError, KeyError, TypeError):
+                self.error = True
+        if data is not None and (changed or self.clock() >= self.next_update_at):
+            try:
+                self.devices = self.discover(data)
+                self.observed_at = self.clock()
+                self.error = False
+            except (OSError, ValueError, TypeError, KeyError, __import__("subprocess").SubprocessError):
+                self.error = True
+            self.next_update_at = self.clock() + (3600 if self.error else 604800)
+            if self.capacity_path and not self.error:
+                try:
+                    self.capacity_path.write_text(json.dumps(dict(devices=self.devices,observed_at=self.observed_at,scanned_at=self.scanned_at)))
+                except OSError:pass
+        names = dict(config.items("storage_names", raw=True)) if config is not None and config.has_section("storage_names") else {}
+        visible = dict(config.items("storage_visible", raw=True)) if config is not None and config.has_section("storage_visible") else {}
+        result = []
+        for device in self.devices:
+            item = dict(device)
+            item["name"] = names.get(item["id"], "auto")
+            if item["name"].lower() == "auto":
+                item["name"] = item["label"] or item["mountpoint"] or item["device"]
+            item["visible"] = visible.get(item["id"], "true").lower() != "false"
+            if self.error:
+                item["state"] = "unknown"
+            result.append(item)
+        known = {d["id"] for d in result}
+        for identifier in sorted((names.keys() | visible.keys()) - known):
+            result.append(dict(id=identifier, name=names.get(identifier, identifier),
+                               visible=visible.get(identifier, "true").lower() != "false",
+                               state="unknown" if self.error else "missing", device="", mountpoint="",
+                               label="", capacity_bytes=None, used_bytes=None, free_bytes=None,
+                               used_percent=None))
+        return result
+
+    def discover(self, data):
+        import hashlib
+        import re
+        mounts, sources = {}, {}
+        for line in self.mounts_path.read_text().splitlines():
+            left, right = line.split(" - ", 1)
+            fields = left.split()
+            # Bind mounts and Btrfs subvolumes must not duplicate filesystem capacity.
+            mount = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4])
+            row = (fields[3] != "/", len(mount), mount)
+            mounts.setdefault(fields[2], []).append(row)
+            source = right.split()[1]
+            if source.startswith("/dev/"):
+                sources.setdefault(str(Path(source).resolve()), []).append(row)
+        devices = data["blockdevices"]
+        if not isinstance(devices, list):
+            raise ValueError("invalid block device inventory")
+        result, seen = [], set()
+
+        def walk(device, serial="", parent=""):
+            kind = device.get("type")
+            if kind in {"loop", "rom", "zram"}:
+                return
+            serial = str(device.get("serial") or serial).strip()
+            children = device.get("children") or []
+            uuid = str(device.get("uuid") or "").strip()
+            fs = str(device.get("fstype") or "")
+            # A parent disk with partitions/RAID/LVM is represented by its usable leaves.
+            if not children and kind not in {"loop", "rom"} and fs not in {"swap", "linux_raid_member", "LVM2_member", "crypto_LUKS"}:
+                partition = re.search(r"(\d+)$", str(device.get("kname") or device.get("name") or ""))
+                suffix = ":" + partition[1] if kind == "part" and partition else ""
+                source = "uuid:" + uuid if uuid else "serial:" + serial + suffix if serial else "device:" + str(device.get("name"))
+                identifier = hashlib.sha256(source.encode()).hexdigest()[:32]
+                if identifier not in seen:
+                    seen.add(identifier)
+                    candidates = mounts.get(device.get("maj:min"), []) or sources.get(str(Path(device["name"]).resolve()), [])
+                    mount = sorted(candidates)[0][2] if candidates else ""
+                    item = dict(id=identifier, device=str(device.get("name") or ""),
+                                label=str(device.get("label") or ""), uuid=uuid,
+                                stable_identity=bool(uuid or serial), model=str(device.get("model") or "").strip(),
+                                mountpoint=mount, filesystem=fs, state="mounted" if mount else "unmounted",
+                                capacity_bytes=int(device.get("size") or 0), used_bytes=None,
+                                free_bytes=None, used_percent=None)
+                    if mount:
+                        try:
+                            usage = self.usage(mount)
+                            if usage.total <= 0:
+                                raise ValueError("invalid filesystem capacity")
+                            item.update(capacity_bytes=usage.total, used_bytes=usage.used,
+                                        free_bytes=usage.free, used_percent=round(100 * usage.used / usage.total, 1))
+                        except (OSError, ValueError):
+                            item["state"] = "unknown"
+                    result.append(item)
+            for child in children:
+                walk(child, serial, str(device.get("name") or parent))
+
+        for device in devices:
+            walk(device)
+        return sorted(result, key=lambda d: (d["mountpoint"] != "/", d["mountpoint"], d["device"]))
+
+
 
 class StorageCache:
     def __init__(self, path, reader=disk_usage, clock=time.time):
@@ -23,7 +176,7 @@ class StorageCache:
                 at = data['observed_at']
                 if isinstance(at, (int, float)) and 0 <= self.clock() - at < 90000:
                     self.values, self.observed_at = values, at
-                    if not self.error:self.next_update_at = at + 86400
+                    if not self.error:self.next_update_at = at + 604800
         except (OSError, ValueError, KeyError, TypeError):
             pass
 
@@ -42,7 +195,7 @@ class StorageCache:
             return self.values
         self.values = dict(total=usage.total, used=usage.used, free=usage.free)
         self.observed_at = now
-        self.next_update_at = now + 86400
+        self.next_update_at = now + 604800
         self.error = False
         self.persist()
         return self.values