Add manually gated first production bootstrap with fail-closed residue checks
This commit is contained in:
Pigway
@@ -0,0 +1,323 @@
|
||||
"""First production bootstrap safety and failure paths, mocked Docker only."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from test_production_deploy import existing
|
||||
from test_production_deploy import tool as deploy_fixture
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tool():
|
||||
return deploy_fixture.__wrapped__()
|
||||
|
||||
|
||||
def residue(tool):
|
||||
value = existing(tool)
|
||||
value.update(Id="residue-id")
|
||||
value["State"] = {
|
||||
"Status": "created",
|
||||
"Running": False,
|
||||
"StartedAt": "0001-01-01T00:00:00Z",
|
||||
}
|
||||
for mount in value["Mounts"]:
|
||||
mount["Source"] = "/workspace/way/pigway-frame-flow" + mount["Destination"]
|
||||
return value
|
||||
|
||||
|
||||
def prepare(tool, monkeypatch, old=None):
|
||||
tool.env["PROD_MOCK"] = "true"
|
||||
monkeypatch.setattr(tool, "container_optional", lambda: old)
|
||||
monkeypatch.setattr(
|
||||
tool,
|
||||
"host_paths",
|
||||
lambda paths: [{"exists": False, "empty": False} for _ in paths],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
tool, "run", lambda *args, **kw: SimpleNamespace(stdout=b"", returncode=0)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["absent", "residue"])
|
||||
def test_bootstrap_accepts_only_empty_first_install(tool, monkeypatch, kind):
|
||||
old = residue(tool) if kind == "residue" else None
|
||||
prepare(tool, monkeypatch, old)
|
||||
assert tool.bootstrap_check() == old
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kind",
|
||||
[
|
||||
"running",
|
||||
"stopped",
|
||||
"old-path",
|
||||
"extra-mount",
|
||||
"identity",
|
||||
"target-nonempty",
|
||||
"target-symlink",
|
||||
"journal",
|
||||
],
|
||||
)
|
||||
def test_bootstrap_fail_closed(tool, monkeypatch, kind):
|
||||
old = residue(tool)
|
||||
prepare(tool, monkeypatch, old)
|
||||
if kind == "running":
|
||||
old["State"].update(Status="running", Running=True)
|
||||
elif kind == "stopped":
|
||||
old["State"].update(Status="exited", StartedAt="2026-10-01T00:00:00Z")
|
||||
elif kind == "extra-mount":
|
||||
old["Mounts"].append({"Type": "volume", "Destination": "/unknown"})
|
||||
elif kind == "identity":
|
||||
old["Config"]["Labels"] = {}
|
||||
elif kind == "journal":
|
||||
monkeypatch.setattr(
|
||||
tool,
|
||||
"run",
|
||||
lambda *a, **kw: SimpleNamespace(
|
||||
stdout=tool.env["PROD_STATE_VOLUME"].encode()
|
||||
),
|
||||
)
|
||||
else:
|
||||
monkeypatch.setattr(
|
||||
tool,
|
||||
"host_paths",
|
||||
lambda paths: [
|
||||
{
|
||||
"exists": kind == "old-path"
|
||||
or not paths[0].startswith("/workspace"),
|
||||
"empty": False,
|
||||
}
|
||||
for _ in paths
|
||||
],
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
tool.bootstrap_check()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mock,writes", [("false", "false"), ("true", "true")])
|
||||
def test_bootstrap_requires_mock_and_no_writes(tool, monkeypatch, mock, writes):
|
||||
prepare(tool, monkeypatch)
|
||||
tool.env.update(PROD_MOCK=mock, PROD_WRITE_ENABLED=writes)
|
||||
with pytest.raises(ValueError, match="mock=true"):
|
||||
tool.bootstrap_check()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", [None, "ready", "smoke", "commit"])
|
||||
def test_bootstrap_verifies_restart_and_preserves_data_on_failure(
|
||||
tool, monkeypatch, failure
|
||||
):
|
||||
prepare(tool, monkeypatch)
|
||||
candidate = tool.bootstrap_config()
|
||||
candidate["services"]["pigway-frame-flow"]["image"] = "release-image"
|
||||
calls, saved, verifies = [], [], []
|
||||
monkeypatch.setattr(tool, "secret_config", lambda: {})
|
||||
monkeypatch.setattr(tool, "build", lambda config: candidate)
|
||||
monkeypatch.setattr(tool, "bootstrap_check", lambda: None)
|
||||
monkeypatch.setattr(tool, "lock", lambda: calls.append(("lock",)))
|
||||
monkeypatch.setattr(tool, "unlock", lambda: calls.append(("unlock",)))
|
||||
monkeypatch.setattr(tool, "state_io", lambda state: saved.append(dict(state)))
|
||||
monkeypatch.setattr(
|
||||
tool, "activate", lambda config: calls.append(("activate", config))
|
||||
)
|
||||
monkeypatch.setattr(tool, "preflight", lambda: calls.append(("preflight",)))
|
||||
monkeypatch.setattr(tool, "diagnostics", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
tool,
|
||||
"inspect",
|
||||
lambda _: {"Image": "sha256:new", "State": {"Health": {"Status": "healthy"}}},
|
||||
)
|
||||
|
||||
def run(*args, **kw):
|
||||
calls.append(args)
|
||||
output = b""
|
||||
if args[:2] == ("image", "inspect"):
|
||||
output = b'[{"Id":"sha256:new"}]'
|
||||
return SimpleNamespace(
|
||||
stdout=output, returncode=1 if args[:2] == ("network", "inspect") else 0
|
||||
)
|
||||
|
||||
monkeypatch.setattr(tool, "run", run)
|
||||
|
||||
def verify(config):
|
||||
verifies.append(config)
|
||||
if failure:
|
||||
raise RuntimeError(failure)
|
||||
|
||||
monkeypatch.setattr(tool, "verify", verify)
|
||||
if failure:
|
||||
with pytest.raises(RuntimeError, match=failure):
|
||||
tool.bootstrap()
|
||||
assert saved[-1]["status"] == "failed"
|
||||
else:
|
||||
tool.bootstrap()
|
||||
assert len(verifies) == 2
|
||||
assert ("restart", tool.env["PROD_CONTAINER"]) in calls
|
||||
assert saved[-1]["status"] == "success"
|
||||
assert not any("down" in c or "prune" in c for c in calls)
|
||||
assert not any(c[0] in ("rm", "volume") and "remove" in c for c in calls)
|
||||
|
||||
|
||||
def test_bootstrap_config_is_adoptable_by_preflight(tool, monkeypatch):
|
||||
prepare(tool, monkeypatch)
|
||||
config = tool.bootstrap_config()
|
||||
runtime = existing(tool)
|
||||
monkeypatch.setattr(tool, "inspect", lambda _: runtime)
|
||||
monkeypatch.setattr(tool, "run", lambda *a, **kw: SimpleNamespace(returncode=1))
|
||||
adopted = tool.preflight()
|
||||
service = config["services"]["pigway-frame-flow"]
|
||||
other = adopted["services"]["pigway-frame-flow"]
|
||||
for key in ("volumes", "ports", "container_name", "restart", "user"):
|
||||
assert service[key] == other[key]
|
||||
assert config["name"] == adopted["name"]
|
||||
|
||||
|
||||
def test_cleanup_only_removes_owned_container_and_empty_owned_network(
|
||||
tool, monkeypatch
|
||||
):
|
||||
prepare(tool, monkeypatch)
|
||||
old = residue(tool)
|
||||
old["Config"]["Labels"]["frameflow.bootstrap.owner"] = tool.owner
|
||||
monkeypatch.setattr(tool, "container_optional", lambda: old)
|
||||
tool.state = {"network_created": True}
|
||||
calls = []
|
||||
|
||||
def run(*args, **kw):
|
||||
calls.append(args)
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps(
|
||||
[
|
||||
{
|
||||
"Id": "network-id",
|
||||
"Containers": {},
|
||||
"Labels": {
|
||||
"com.docker.compose.project": tool.env["PROD_PROJECT"]
|
||||
},
|
||||
}
|
||||
]
|
||||
).encode(),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(tool, "run", run)
|
||||
tool.bootstrap_cleanup()
|
||||
assert calls == [
|
||||
("rm", "-f", "residue-id"),
|
||||
("network", "inspect", "production-frame_default"),
|
||||
("network", "rm", "network-id"),
|
||||
]
|
||||
old["Config"]["Labels"].pop("frameflow.bootstrap.owner")
|
||||
with pytest.raises(ValueError):
|
||||
tool.bootstrap_cleanup()
|
||||
|
||||
|
||||
def test_diagnostics_masks_all_secrets(tool, monkeypatch, capsys):
|
||||
tool.secret_values = ["test-secret"]
|
||||
monkeypatch.setattr(
|
||||
tool,
|
||||
"run",
|
||||
lambda *a, **kw: SimpleNamespace(
|
||||
returncode=1, stdout=b"test-secret https://private.invalid/key", stderr=b""
|
||||
),
|
||||
)
|
||||
tool.diagnostics()
|
||||
output = capsys.readouterr().out
|
||||
assert "test-secret" not in output and "private.invalid" not in output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["ready", "smoke", "commit"])
|
||||
def test_actual_verify_rejects_ready_smoke_and_commit_failure(
|
||||
tool, monkeypatch, failure
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
tool.verify.__func__.__globals__["time"], "sleep", lambda _: None
|
||||
)
|
||||
config = {
|
||||
"services": {"pigway-frame-flow": {"environment": {"BUILD_COMMIT": "expected"}}}
|
||||
}
|
||||
|
||||
def run(*args, **kw):
|
||||
if "-i" in args and failure == "smoke":
|
||||
raise RuntimeError("smoke failed")
|
||||
if "-i" not in args and failure == "ready":
|
||||
return SimpleNamespace(returncode=1)
|
||||
return SimpleNamespace(returncode=0, stdout=b'{"commit":"wrong"}')
|
||||
|
||||
monkeypatch.setattr(tool, "run", run)
|
||||
with pytest.raises(RuntimeError):
|
||||
tool.verify(config)
|
||||
|
||||
|
||||
def test_workflow_bootstrap_manual_only_and_ci_namespace_separate(tool):
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
root = Path(__file__).parents[1]
|
||||
workflow = yaml.safe_load((root / ".github/workflows/verify.yml").read_text())
|
||||
inputs = workflow[True]["workflow_dispatch"]["inputs"]["deploy_action"]
|
||||
assert inputs["options"] == ["plan", "bootstrap", "deploy", "rollback"]
|
||||
assert inputs["default"] == "plan"
|
||||
assert workflow["jobs"]["deploy"]["needs"] == "verify"
|
||||
assert "success()" in workflow["jobs"]["deploy"]["if"]
|
||||
tool.env["PROD_PROJECT"] = "ffci-invalid"
|
||||
with pytest.raises(ValueError):
|
||||
tool.settings()
|
||||
|
||||
|
||||
def test_automatic_push_without_production_never_creates_it(tool, monkeypatch, capsys):
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"bootstrap_main", Path(__file__).parents[1] / "scripts/production-deploy.py"
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
tool.env.update(PROD_DEPLOY_ENABLED="true", GITHUB_EVENT_NAME="push")
|
||||
monkeypatch.setattr(module, "Deploy", lambda: tool)
|
||||
monkeypatch.setattr(tool, "container_optional", lambda: None)
|
||||
monkeypatch.setattr(tool, "deploy", lambda: pytest.fail("Production touched"))
|
||||
monkeypatch.setattr(tool, "bootstrap", lambda: pytest.fail("Production touched"))
|
||||
module.main()
|
||||
assert "bootstrap required" in capsys.readouterr().out
|
||||
tool.env["CD_ACTION"] = "bootstrap"
|
||||
with pytest.raises(SystemExit, match="workflow_dispatch"):
|
||||
module.main()
|
||||
|
||||
|
||||
def test_bootstrap_recovery_cleans_only_owned_prepared_transaction(tool, monkeypatch):
|
||||
prepare(tool, monkeypatch)
|
||||
calls = []
|
||||
tool.state = {
|
||||
"owner": tool.owner,
|
||||
"kind": "bootstrap",
|
||||
"status": "prepared",
|
||||
"network_created": True,
|
||||
}
|
||||
|
||||
def run(*args, **kw):
|
||||
calls.append(args)
|
||||
value = [{"Labels": {"frameflow.deploy.project": tool.env["PROD_PROJECT"]}}]
|
||||
if args[0] == "inspect":
|
||||
value = [
|
||||
{
|
||||
"Id": "lock-id",
|
||||
"Image": "built-image",
|
||||
"Config": {"Labels": {"frameflow.deploy.owner": tool.owner}},
|
||||
}
|
||||
]
|
||||
return SimpleNamespace(returncode=0, stdout=json.dumps(value).encode())
|
||||
|
||||
monkeypatch.setattr(tool, "run", run)
|
||||
monkeypatch.setattr(tool, "bootstrap_cleanup", lambda: calls.append(("cleanup",)))
|
||||
|
||||
def state_io(payload=None):
|
||||
if payload is not None:
|
||||
tool.state = payload
|
||||
return json.dumps(tool.state).encode()
|
||||
|
||||
monkeypatch.setattr(tool, "state_io", state_io)
|
||||
tool.recover()
|
||||
assert tool.state["status"] == "failed"
|
||||
assert ("cleanup",) in calls and ("rm", "lock-id") in calls
|
||||
Reference in New Issue
Block a user