Add manually gated first production bootstrap with fail-closed residue checks
This commit is contained in:
Pigway
@@ -381,6 +381,232 @@ class Deploy:
|
||||
)
|
||||
return config
|
||||
|
||||
def container_optional(self):
|
||||
result = self.run("container", "ls", "-a", "--format", "{{.Names}}")
|
||||
if self.env["PROD_CONTAINER"] not in result.stdout.decode().splitlines():
|
||||
return None
|
||||
return self.inspect(self.env["PROD_CONTAINER"])
|
||||
|
||||
def host_paths(self, paths):
|
||||
# Inspect the daemon host, never the job container workspace. Read-only root.
|
||||
code = "import json,sys;from pathlib import Path;print(json.dumps([{'exists':p.exists() or p.is_symlink() or any(a.is_symlink() for a in p.parents),'empty':p.is_dir() and not p.is_symlink() and not any(a.is_symlink() for a in p.parents) and not any(p.iterdir())} for p in [Path('/host'+v) for v in json.load(sys.stdin)]]))"
|
||||
return json.loads(
|
||||
self.run(
|
||||
"run",
|
||||
"--rm",
|
||||
"-i",
|
||||
"--user",
|
||||
"0:0",
|
||||
"--network",
|
||||
"none",
|
||||
"--mount",
|
||||
"type=bind,source=/,target=/host,readonly",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
self.old_image,
|
||||
"-c",
|
||||
code,
|
||||
data=json.dumps(paths).encode(),
|
||||
).stdout
|
||||
)
|
||||
|
||||
def bootstrap_check(self):
|
||||
self.settings()
|
||||
if self.env["PROD_MOCK"] != "true" or self.env["PROD_WRITE_ENABLED"] != "false":
|
||||
raise ValueError("Bootstrap requires mock=true and write=false")
|
||||
if self.env.get("PROD_PHOTOS_DIR"):
|
||||
raise ValueError("First bootstrap must not attach Photos")
|
||||
paths = [self.env[k] for k in ("PROD_DATA_DIR", "PROD_CONFIG_DIR")]
|
||||
if any(".." in Path(p).parts or p.endswith("/") for p in paths) or any(
|
||||
Path(a) in Path(b).parents
|
||||
for a, b in ((paths[0], paths[1]), (paths[1], paths[0]))
|
||||
):
|
||||
raise ValueError("Unsafe or overlapping bootstrap directories")
|
||||
old = self.container_optional()
|
||||
if old:
|
||||
labels = old["Config"].get("Labels") or {}
|
||||
state = old["State"]
|
||||
if (
|
||||
state.get("Status") != "created"
|
||||
or state.get("Running")
|
||||
or state.get("StartedAt") != "0001-01-01T00:00:00Z"
|
||||
or labels.get("com.docker.compose.project") != self.env["PROD_PROJECT"]
|
||||
or labels.get("com.docker.compose.service") != "pigway-frame-flow"
|
||||
):
|
||||
raise ValueError(
|
||||
"Bootstrap refused: existing container is not a never-started owned residue"
|
||||
)
|
||||
mounts = old["Mounts"]
|
||||
if (
|
||||
len(mounts) != 2
|
||||
or {m["Destination"] for m in mounts} != {"/data", "/config"}
|
||||
or any(m["Type"] != "bind" for m in mounts)
|
||||
):
|
||||
raise ValueError("Bootstrap refused: unknown persistent mount")
|
||||
if any(v["exists"] for v in self.host_paths([m["Source"] for m in mounts])):
|
||||
raise ValueError("Bootstrap refused: old mount has existing data/path")
|
||||
if any(v["exists"] and not v["empty"] for v in self.host_paths(paths)):
|
||||
raise ValueError("Bootstrap refused: target directory nonempty or unsafe")
|
||||
volume = self.run("volume", "ls", "--format", "{{.Name}}")
|
||||
if self.env["PROD_STATE_VOLUME"] in volume.stdout.decode().splitlines():
|
||||
raise ValueError(
|
||||
"Bootstrap refused: existing deployment journal requires review"
|
||||
)
|
||||
return old
|
||||
|
||||
def bootstrap_config(self):
|
||||
config = yaml.safe_load((ROOT / "docker-compose.yml").read_text())
|
||||
config["name"] = self.env["PROD_PROJECT"]
|
||||
service = config["services"]["pigway-frame-flow"]
|
||||
service.pop("build")
|
||||
service.pop("env_file")
|
||||
service["container_name"] = self.env["PROD_CONTAINER"]
|
||||
service["environment"] = {}
|
||||
service["ports"] = [
|
||||
f"{self.env['PROD_WEB_BIND']}:{self.env['PROD_WEB_PORT']}:18787"
|
||||
]
|
||||
service["labels"] = {"frameflow.bootstrap.owner": self.owner}
|
||||
service["volumes"] = [
|
||||
{
|
||||
"type": "bind",
|
||||
"source": self.env[key],
|
||||
"target": target,
|
||||
"bind": {"create_host_path": False},
|
||||
}
|
||||
for key, target in (
|
||||
("PROD_DATA_DIR", "/data"),
|
||||
("PROD_CONFIG_DIR", "/config"),
|
||||
)
|
||||
]
|
||||
return config
|
||||
|
||||
def bootstrap_cleanup(self):
|
||||
old = self.container_optional()
|
||||
if old:
|
||||
if (old["Config"].get("Labels") or {}).get(
|
||||
"frameflow.bootstrap.owner"
|
||||
) != self.owner:
|
||||
raise ValueError("Refusing cleanup of another container")
|
||||
self.run("rm", "-f", old["Id"])
|
||||
if self.state and self.state.get("network_created"):
|
||||
name = self.env["PROD_PROJECT"] + "_default"
|
||||
result = self.run("network", "inspect", name, check=False)
|
||||
if result.returncode == 0:
|
||||
network = json.loads(result.stdout)[0]
|
||||
if (
|
||||
network.get("Containers")
|
||||
or (network.get("Labels") or {}).get("com.docker.compose.project")
|
||||
!= self.env["PROD_PROJECT"]
|
||||
):
|
||||
raise ValueError("Refusing cleanup of occupied/unknown network")
|
||||
self.run("network", "rm", network["Id"])
|
||||
|
||||
def bootstrap(self):
|
||||
self.settings()
|
||||
if self.env["PROD_MOCK"] != "true" or self.env["PROD_WRITE_ENABLED"] != "false":
|
||||
raise ValueError("Bootstrap requires mock=true and write=false")
|
||||
self.secret_config()
|
||||
# Build before host inspection so helpers use this verified repository image.
|
||||
candidate = self.build(self.bootstrap_config())
|
||||
self.old_image = candidate["services"]["pigway-frame-flow"]["image"]
|
||||
self.lock()
|
||||
safe_unlock = True
|
||||
try:
|
||||
residue = self.bootstrap_check()
|
||||
network_name = self.env["PROD_PROJECT"] + "_default"
|
||||
networks = (
|
||||
self.run("network", "ls", "--format", "{{.Name}}")
|
||||
.stdout.decode()
|
||||
.splitlines()
|
||||
)
|
||||
if network_name in networks:
|
||||
network = json.loads(
|
||||
self.run("network", "inspect", network_name).stdout
|
||||
)[0]
|
||||
if (
|
||||
set(network.get("Containers") or {})
|
||||
- ({residue["Id"]} if residue else set())
|
||||
or (network.get("Labels") or {}).get("com.docker.compose.project")
|
||||
!= self.env["PROD_PROJECT"]
|
||||
):
|
||||
raise ValueError(
|
||||
"Bootstrap network identity/occupancy requires review"
|
||||
)
|
||||
self.state = {
|
||||
"owner": self.owner,
|
||||
"kind": "bootstrap",
|
||||
"status": "prepared",
|
||||
"candidate": candidate,
|
||||
"network_created": network_name not in networks,
|
||||
}
|
||||
self.run(
|
||||
"volume",
|
||||
"create",
|
||||
"--label",
|
||||
"frameflow.deploy.project=" + self.env["PROD_PROJECT"],
|
||||
self.env["PROD_STATE_VOLUME"],
|
||||
)
|
||||
self.state_io(self.state)
|
||||
if residue:
|
||||
self.run("rm", residue["Id"])
|
||||
# -v creates only the two explicit daemon-host paths. No parent is mounted writable.
|
||||
self.run(
|
||||
"run",
|
||||
"--rm",
|
||||
"--user",
|
||||
"0:0",
|
||||
"--network",
|
||||
"none",
|
||||
"-v",
|
||||
self.env["PROD_DATA_DIR"] + ":/data",
|
||||
"-v",
|
||||
self.env["PROD_CONFIG_DIR"] + ":/config",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
self.old_image,
|
||||
"-c",
|
||||
"import os;from pathlib import Path;paths=[Path('/data'),Path('/config')];assert all(not any(p.iterdir()) for p in paths);[(os.chown(p,10333,10333),os.chmod(p,0o750)) for p in paths]",
|
||||
)
|
||||
self.activate(candidate)
|
||||
self.verify(candidate)
|
||||
self.preflight()
|
||||
self.run("restart", self.env["PROD_CONTAINER"])
|
||||
self.verify(candidate)
|
||||
self.preflight()
|
||||
info = self.inspect(self.env["PROD_CONTAINER"])
|
||||
for _ in range(60):
|
||||
if info["State"].get("Health", {}).get("Status") == "healthy":
|
||||
break
|
||||
time.sleep(2)
|
||||
info = self.inspect(self.env["PROD_CONTAINER"])
|
||||
else:
|
||||
raise RuntimeError("Bootstrap health timeout")
|
||||
expected_image = json.loads(
|
||||
self.run(
|
||||
"image",
|
||||
"inspect",
|
||||
candidate["services"]["pigway-frame-flow"]["image"],
|
||||
).stdout
|
||||
)[0]["Id"]
|
||||
if info["Image"] != expected_image:
|
||||
raise RuntimeError("Bootstrap image mismatch")
|
||||
self.state["status"] = "success"
|
||||
self.state_io(self.state)
|
||||
print("BOOTSTRAP VERIFIED", flush=True)
|
||||
except BaseException:
|
||||
print("BOOTSTRAP FAILED", flush=True)
|
||||
self.diagnostics()
|
||||
if self.state:
|
||||
safe_unlock = False
|
||||
self.bootstrap_cleanup()
|
||||
self.state["status"] = "failed"
|
||||
self.state_io(self.state)
|
||||
safe_unlock = True
|
||||
raise
|
||||
finally:
|
||||
if safe_unlock:
|
||||
self.unlock()
|
||||
|
||||
def diagnostics(self):
|
||||
try:
|
||||
self._diagnostics()
|
||||
@@ -393,6 +619,19 @@ class Deploy:
|
||||
result = self.run("inspect", self.env["PROD_CONTAINER"], check=False)
|
||||
if result.returncode == 0:
|
||||
info = json.loads(result.stdout)[0]
|
||||
self.secret_values.extend(
|
||||
value
|
||||
for key, value in (
|
||||
entry.split("=", 1)
|
||||
for entry in info.get("Config", {}).get("Env", [])
|
||||
if "=" in entry
|
||||
)
|
||||
if value
|
||||
and any(
|
||||
word in key
|
||||
for word in ("KEY", "TOKEN", "PASSWORD", "SECRET", "SESSION")
|
||||
)
|
||||
)
|
||||
state = info["State"]
|
||||
print(
|
||||
"Scoped production diagnostic:",
|
||||
@@ -510,6 +749,16 @@ class Deploy:
|
||||
]:
|
||||
raise ValueError("Recovery state volume ownership mismatch")
|
||||
state = json.loads(self.state_io())
|
||||
if state.get("kind") == "bootstrap":
|
||||
self.state = state
|
||||
if state["owner"] != self.owner:
|
||||
raise ValueError("Bootstrap journal owner mismatch")
|
||||
if state["status"] == "prepared":
|
||||
self.bootstrap_cleanup()
|
||||
state["status"] = "failed"
|
||||
self.state_io(state)
|
||||
recovered = True
|
||||
return
|
||||
previous = state["previous"]
|
||||
service = previous["services"]["pigway-frame-flow"]
|
||||
if (
|
||||
@@ -561,6 +810,15 @@ def main():
|
||||
if any(not tool.env.get(key) for key in REQUIRED):
|
||||
print("PLAN ONLY: configure required Variables:", ", ".join(REQUIRED))
|
||||
return
|
||||
if tool.container_optional() is None:
|
||||
print("PLAN ONLY: production not bootstrapped / bootstrap required")
|
||||
return
|
||||
info = tool.inspect(tool.env["PROD_CONTAINER"])
|
||||
if info["State"].get("Status") == "created":
|
||||
print(
|
||||
"PLAN ONLY: production not bootstrapped; created residue requires bootstrap safety checks"
|
||||
)
|
||||
return
|
||||
baseline = tool.preflight()
|
||||
tool.secret_config()
|
||||
tool.verify(baseline)
|
||||
@@ -576,7 +834,22 @@ def main():
|
||||
raise SystemExit(143)
|
||||
|
||||
signal.signal(signal.SIGTERM, cancel)
|
||||
if action == "deploy":
|
||||
if action == "bootstrap":
|
||||
if tool.env.get("GITHUB_EVENT_NAME") != "workflow_dispatch":
|
||||
raise SystemExit("Bootstrap requires explicit workflow_dispatch")
|
||||
try:
|
||||
tool.bootstrap()
|
||||
except BaseException:
|
||||
print(
|
||||
"BOOTSTRAP FAILED: inspect redacted diagnostics; data/config retained",
|
||||
flush=True,
|
||||
)
|
||||
raise
|
||||
elif action == "deploy":
|
||||
info = tool.container_optional()
|
||||
if info is None or info["State"].get("Status") == "created":
|
||||
print("production not bootstrapped / bootstrap required")
|
||||
return
|
||||
tool.deploy()
|
||||
elif action == "rollback":
|
||||
tool.recover(manual=True)
|
||||
|
||||
Reference in New Issue
Block a user