From 2e8251abef7fea34eaa1ba7531fa99c50e61cea5 Mon Sep 17 00:00:00 2001 From: Pigway Date: Wed, 7 Oct 2026 23:15:31 +0800 Subject: [PATCH] Add opt-in production CD with daemon lock, verified rollback and safe planning --- .github/workflows/verify.yml | 58 +++- README.md | 2 + docs/deployment/GITEA_CD.md | 169 +++++++++ scripts/production-deploy.py | 588 ++++++++++++++++++++++++++++++++ tests/test_production_deploy.py | 245 +++++++++++++ 5 files changed, 1061 insertions(+), 1 deletion(-) create mode 100644 docs/deployment/GITEA_CD.md create mode 100644 scripts/production-deploy.py create mode 100644 tests/test_production_deploy.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 2ed0681..63330b4 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1,5 +1,14 @@ name: Verify Frame Flow -on: [push, pull_request, workflow_dispatch] +on: + push: + pull_request: + workflow_dispatch: + inputs: + deploy_action: + description: Production action (deploy requires explicit enabled Variable) + type: choice + options: [plan, deploy, rollback] + default: plan jobs: verify: runs-on: ubuntu-latest @@ -30,3 +39,50 @@ jobs: if [ -n "${FRAME_FLOW_CI_COMPOSE:-}" ] && [ -f "$FRAME_FLOW_CI_COMPOSE" ]; then python scripts/ci-compose-verify.py --cleanup fi + + deploy: + needs: verify + if: success() && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + concurrency: + group: frame-flow-production + cancel-in-progress: false + env: + CD_ACTION: ${{ github.event.inputs.deploy_action }} + PROD_DEPLOY_ENABLED: ${{ vars.PROD_DEPLOY_ENABLED }} + PROD_PROJECT: ${{ vars.PROD_PROJECT }} + PROD_CONTAINER: ${{ vars.PROD_CONTAINER }} + PROD_IMAGE_REPO: ${{ vars.PROD_IMAGE_REPO }} + PROD_DATA_DIR: ${{ vars.PROD_DATA_DIR }} + PROD_CONFIG_DIR: ${{ vars.PROD_CONFIG_DIR }} + PROD_PHOTOS_DIR: ${{ vars.PROD_PHOTOS_DIR }} + PROD_WEB_BIND: ${{ vars.PROD_WEB_BIND }} + PROD_WEB_PORT: ${{ vars.PROD_WEB_PORT }} + PROD_STATE_VOLUME: ${{ vars.PROD_STATE_VOLUME }} + PROD_MOCK: ${{ vars.PROD_MOCK }} + PROD_WRITE_ENABLED: ${{ vars.PROD_WRITE_ENABLED }} + CI_PIP_INDEX_URL: ${{ vars.CI_PIP_INDEX_URL }} + CI_PIP_FALLBACK_INDEX_URL: ${{ vars.CI_PIP_FALLBACK_INDEX_URL }} + DEBIAN_MIRROR: ${{ vars.DEBIAN_MIRROR }} + DEBIAN_SECURITY_MIRROR: ${{ vars.DEBIAN_SECURITY_MIRROR }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.14' + - name: Install deploy tooling + run: python scripts/ci-install.py + - name: Plan or deploy production (default is plan) + env: + PRODUCTION_ENV: ${{ secrets.PRODUCTION_ENV }} + run: | + . ./deploy/build-cn.env + python scripts/production-deploy.py + - name: Recover interrupted production deployment + if: always() + env: + CD_ACTION: recover + run: | + if [ -n "${FRAME_FLOW_DEPLOY_OWNER:-}" ]; then + python scripts/production-deploy.py + fi diff --git a/README.md b/README.md index 9d0f58f..8539bfc 100644 --- a/README.md +++ b/README.md @@ -338,3 +338,5 @@ APT only and are loaded later in the Docker verification step. If dependency installation fails, the always() cleanup step still executes. It skips the Python/PyYAML cleanup script when no CI Compose manifest exists, as no Docker verification resources have been created at that point. + +Production CD: [safe trial, configuration, and rollback](docs/deployment/GITEA_CD.md). diff --git a/docs/deployment/GITEA_CD.md b/docs/deployment/GITEA_CD.md new file mode 100644 index 0000000..831a58b --- /dev/null +++ b/docs/deployment/GITEA_CD.md @@ -0,0 +1,169 @@ +# Gitea Actions production CD v1 + +## Audit and adoption boundary + +Production `docker-compose.yml` is unchanged: AMD64 image, UID/GID 10333, +`unless-stopped`, read-only rootfs, `/tmp` tmpfs, dropped capabilities, limited +logs, `/data` and `/config` bind mounts, default private bridge, and configurable +host bind/port. SQLite/images/generated/cache live under `/data`; accounts, +vendor sessions and configuration live under `/config`. Neither is disposable. +The optional photos mount is read-only and has supplementary group access. + +Historical repository receipts mention `/volume2/docker/pigway-frame-flow/data`, +`/volume2/docker/pigway-frame-flow/config`, `/volume1/photo`, port 18787, and +container/project `pigway-frame-flow`. These are **confirmation candidates**, not +script defaults. The deployment script accepts only explicitly configured paths +that match the existing running container on the Runner's Docker daemon. +It rejects different projects, ports, unknown mounts, host/custom networking, +missing photos confirmation, or different user/restart policy. It does not +create NAS host directories or migrate/adopt an unrelated container. + +CI uses `ffci-*` projects, per-run image tags and named volumes, no host ports. +Its cleanup does not refer to any production resource. Production uses fixed +configured names and existing bind mounts, with `create_host_path: false`. + +## Gitea configuration + +Repository Settings → Actions → Variables (ordinary configuration): + +| Variable | Value to explicitly confirm | +| --- | --- | +| `PROD_DEPLOY_ENABLED` | Initially `false`; change to `true` only after plan acceptance | +| `PROD_PROJECT` | Existing Compose project label; candidate `pigway-frame-flow` | +| `PROD_CONTAINER` | Existing container name; candidate `pigway-frame-flow` | +| `PROD_IMAGE_REPO` | Fixed local image repository; suggested `pigway-frame-flow` | +| `PROD_DATA_DIR` | Existing absolute NAS host data path; no default | +| `PROD_CONFIG_DIR` | Existing absolute NAS host config path; no default | +| `PROD_PHOTOS_DIR` | Existing read-only photos source if present; otherwise empty | +| `PROD_WEB_BIND` | Exact existing Docker HostIp (`0.0.0.0` or NAS LAN IP etc.) | +| `PROD_WEB_PORT` | Exact existing mapped port | +| `PROD_STATE_VOLUME` | Dedicated new deployment journal volume, e.g. `pigway-frame-flow-cd-state` | +| `PROD_MOCK` | Explicit `true`/`false`, consistent with intended production mode | +| `PROD_WRITE_ENABLED` | Explicit `true`/`false`; preserve owner's existing write policy | + +The state volume must be separate from application/CI storage. An existing +volume without the matching `frameflow.deploy.project` label is rejected. +Supplementary groups are read from the existing container; no host ACL changes. +Keep existing `DEBIAN_MIRROR`, `DEBIAN_SECURITY_MIRROR`, `CI_PIP_INDEX_URL`, and +`CI_PIP_FALLBACK_INDEX_URL` settings from Verify; CI/production builds share the +Dockerfile, pinned base, lockfile, and BuildKit caches. + +Repository Settings → Actions → Secrets: + +`PRODUCTION_ENV`: multiline **unquoted** `KEY=value` entries, without shell +expansion. At least: + +```dotenv +FRAME_FLOW_TOKEN= +KOKONNA_API_KEY= +``` + +Optional additional runtime secret variables can be included. No key values are +committed, passed to image builds, or printed. Nonsecret mock/write flags use +Variables. Other existing runtime environment values are retained. The script +sets commit/build time and internal data/config paths explicitly. Dollar signs +in secret values are escaped before Compose interpolation. Snapshots containing +old runtime secrets are stored only in a private journal file (0600, directory +0700) on the dedicated Docker volume; Docker administrators can access it. +Never upload that journal or a full Docker inspect as a public CI artifact. + +## First safe trial + +1. On NAS, inspect only the necessary fields (use existing Docker/sudo access): + ```sh + docker inspect pigway-frame-flow --format '{{index .Config.Labels "com.docker.compose.project"}}' + docker inspect pigway-frame-flow --format '{{json .Mounts}}' + docker inspect pigway-frame-flow --format '{{json .HostConfig.PortBindings}} {{.HostConfig.NetworkMode}}' + ``` + Confirm the Variables against these actual values, including photos. +2. Configure the Secret privately. Keep `PROD_DEPLOY_ENABLED=false`. +3. Manually run **Verify Frame Flow** on `main`, `deploy_action=plan`. + Verify must pass first. Plan checks resource identity, paths/port, required + Secret keys, current `/ready`, smoke endpoints and commit. It performs no + build, stop, restart, tagging, lock creation, volume creation, or data writes. + If no production Variables exist, it reports configuration missing and does + nothing; that is an unconfigured plan, not production acceptance. +4. Review `PLAN ONLY` output and arrange a maintenance window for the first real + deploy/restart. Set `PROD_DEPLOY_ENABLED=true`, manually run `deploy_action=deploy`. +5. After `DEPLOY VERIFIED`, subsequent pushes to `main` run Verify → deploy. + Pushes to other branches and pull requests never deploy. A manual run defaults + to plan, even when automatic deployment is enabled. + +The script intentionally supports updates to an existing healthy deployment, +not first-time bootstrap. No real production deployment or rollback drill has +been performed during implementation; those are first-run acceptance steps. + +## Deployment transaction and rollback + +1. Verify job must succeed completely, including isolated Docker tests/cleanup. +2. Deploy job checks out the same revision and prepares tools. An atomic + `docker create` reservation named `-deploy-lock` serializes + production changes on the daemon. Job `concurrency` is also declared, but + correctness does not depend on Gitea implementing it. A competing deploy + fails safely instead of changing production. No other job may remove a lock + whose owner label differs from its own. +3. Re-read the current baseline under the lock; confirm ready/smoke. Record exact + image ID, runtime Compose/environment, version/commit. Pin the old image with + an immutable rollback tag so normal dangling-image cleanup cannot remove it. +4. Refuse a stale checked-out `main` revision that no longer equals remote main. + Build an immutable `release--` tag locally, without a registry + and without changing `latest` or CI image tags. +5. Persist a `prepared` journal with old/new runtime configs. Create an additive + SQLite online backup under `/data/deploy-backups/.db`. No data/config + directory or file is removed, replaced or recursively chowned. +6. `compose up -d --no-build --pull never` replaces only the configured service. + Check `/ready`, the unchanged nine-endpoint read-only smoke test, and exact + `/api/v1/version` commit. Only all checks passing produce `DEPLOY VERIFIED`. +7. Any activation/verification failure runs `up` again with the saved exact old + image ID and old runtime environment/mounts, then repeats ready/smoke/commit + verification. A failed deploy remains a failed workflow even if rollback + succeeds. If rollback fails, the journal/lock are retained, failure is explicit, + and redacted scoped diagnostics are emitted. +8. An `always()` step uses the recorded owner to recover an interrupted prepared + transaction and release only its own lock. Abrupt host loss or termination of + the entire runner can prevent both handlers: inspect the journal/lock before + manually recovering; never automatically force-unlock an unknown owner. + +Rollback is an **image/config rollback**, not an automatic database rewind. +User data accumulated during deployment remains. The SQLite backup is for +operator-reviewed recovery; destructive or backward-incompatible future schema +changes must not be auto-deployed without a separate migration/backup plan. +Baseline artifacts and additive backups are retained for the operator; there is +no automated pruning of production rollback history or data. + +After enabling CD, use its workflow/script as the production lifecycle owner. +The old NAS checkout's Compose/.env files are not rewritten remotely by CD and +may describe an older deployment. Do not concurrently recreate production from +that old checkout. The repository's standalone Compose remains usable for an +independent deployment; the private journal records the CD runtime configuration. + +## Actual commit and manual rollback + +```sh +curl -fsS http://NAS_LAN_IP:PORT/api/v1/version +# Check `commit`, `version`, and `build_time` against successful deployment output. +docker inspect PRODUCTION_CONTAINER --format '{{.Image}}' +``` + +For a manual rollback: keep CD enabled and run **Verify Frame Flow** on `main` +with `deploy_action=rollback`. After Verify passes, the script locks production, +loads the last journal's `previous` image/config and verifies recovery. This is +last-baseline rollback, not an arbitrary image selector. Never use `down -v`, +volume pruning, or delete data/config to roll back. + +If a failed run retained a lock, inspect only its owner label: + +```sh +docker inspect PRODUCTION_CONTAINER-deploy-lock --format '{{index .Config.Labels "frameflow.deploy.owner"}}' +``` + +With the documented Variables configured in a trusted Docker-enabled shell: + +```sh +FRAME_FLOW_DEPLOY_OWNER= CD_ACTION=recover python scripts/production-deploy.py +``` + +`recover` retries that owner's journaled rollback and retains the lock again if +recovery fails. If the very first attempt stopped before a journal was written, +confirm that the original image is still running and healthy before an operator +removes the stale lock. Do not delete a live lock or any persistence volume. diff --git a/scripts/production-deploy.py b/scripts/production-deploy.py new file mode 100644 index 0000000..dad1af4 --- /dev/null +++ b/scripts/production-deploy.py @@ -0,0 +1,588 @@ +"""Opt-in production deployment; host paths verified against the running container.""" + +import json +import os +import re +import signal +import subprocess +import tempfile +import time +from datetime import UTC, datetime +from pathlib import Path +from uuid import uuid4 + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +REQUIRED = ( + "PROD_PROJECT", + "PROD_CONTAINER", + "PROD_IMAGE_REPO", + "PROD_DATA_DIR", + "PROD_CONFIG_DIR", + "PROD_WEB_BIND", + "PROD_WEB_PORT", + "PROD_STATE_VOLUME", + "PROD_MOCK", + "PROD_WRITE_ENABLED", +) + + +class Deploy: + def __init__(self): + self.env = os.environ.copy() + self.owner = self.env.get("FRAME_FLOW_DEPLOY_OWNER") or uuid4().hex + if not re.fullmatch(r"[0-9a-f]{32}", self.owner): + raise ValueError("Invalid deployment owner") + self.lock_id = None + self.state = None + self.old_image = None + self.secret_values = [] + self.docker = self.env.get("DOCKER_BIN", "docker") + + def run(self, *args, data=None, check=True): + result = subprocess.run( + [self.docker, *args], + input=data, + capture_output=True, + env=self.env | {"DOCKER_BUILDKIT": "1"}, + ) + if check and result.returncode: + # Never print Docker inspect/config/env output or secret-bearing commands. + raise RuntimeError( + f"Docker {args[0]} failed (exit {result.returncode}); inspect scoped container logs privately" + ) + return result + + def inspect(self, name): + return json.loads(self.run("inspect", name).stdout)[0] + + def settings(self): + missing = [key for key in REQUIRED if not self.env.get(key)] + if missing: + raise ValueError("Missing Variables: " + ", ".join(missing)) + for key in ( + "PROD_PROJECT", + "PROD_CONTAINER", + "PROD_IMAGE_REPO", + "PROD_STATE_VOLUME", + ): + if not re.fullmatch(r"[a-z][a-z0-9-]{2,80}", self.env[key]) or self.env[ + key + ].startswith("ffci-"): + raise ValueError("Unsafe production resource name: " + key) + for key in ("PROD_DATA_DIR", "PROD_CONFIG_DIR"): + if not self.env[key].startswith("/") or self.env[key] == "/": + raise ValueError("Absolute existing NAS path required: " + key) + if self.env["PROD_DATA_DIR"] == self.env["PROD_CONFIG_DIR"]: + raise ValueError("data/config must use distinct directories") + if any( + self.env[key] not in ("true", "false") + for key in ("PROD_MOCK", "PROD_WRITE_ENABLED") + ): + raise ValueError("Explicit true/false required for mock and writes") + if not 1 <= int(self.env["PROD_WEB_PORT"]) <= 65535: + raise ValueError("Invalid production port") + + def preflight(self, require_running=True): + self.settings() + old = self.inspect(self.env["PROD_CONTAINER"]) + labels = old["Config"].get("Labels") or {} + if ( + labels.get("com.docker.compose.project") != self.env["PROD_PROJECT"] + or labels.get("com.docker.compose.service") != "pigway-frame-flow" + ): + raise ValueError( + "Existing container does not belong to the configured production project/service" + ) + if require_running and not old["State"]["Running"]: + raise ValueError("Existing production container must be running") + mounts = {m["Destination"]: m for m in old["Mounts"]} + for target, key in (("/data", "PROD_DATA_DIR"), ("/config", "PROD_CONFIG_DIR")): + m = mounts.get(target, {}) + if ( + m.get("Type") != "bind" + or m.get("Source") != self.env[key] + or not m.get("RW") + ): + raise ValueError("Production mount mismatch: " + key) + photos = self.env.get("PROD_PHOTOS_DIR") + if "/photos" in mounts and ( + not photos + or mounts["/photos"]["Source"] != photos + or mounts["/photos"]["RW"] + ): + raise ValueError( + "Confirm PROD_PHOTOS_DIR for the existing read-only photo mount" + ) + if photos and "/photos" not in mounts: + raise ValueError( + "Configured photos directory does not match current container" + ) + if set(mounts) != {"/data", "/config"} | ({"/photos"} if photos else set()): + raise ValueError("Unrecognized persistent mount; review before adopting CD") + bindings = old["HostConfig"].get("PortBindings", {}).get("18787/tcp") + if bindings != [ + {"HostIp": self.env["PROD_WEB_BIND"], "HostPort": self.env["PROD_WEB_PORT"]} + ]: + raise ValueError( + "Port/bind mismatch; CD must not change production networking" + ) + if ( + old["HostConfig"].get("NetworkMode") + != self.env["PROD_PROJECT"] + "_default" + ): + raise ValueError("Custom production network requires explicit review") + config = yaml.safe_load((ROOT / "docker-compose.yml").read_text()) + config["name"] = self.env["PROD_PROJECT"] + service = config["services"]["pigway-frame-flow"] + service.pop("build") + service.pop("env_file") + service["container_name"] = self.env["PROD_CONTAINER"] + service["image"] = old["Image"] + service["environment"] = dict( + value.split("=", 1) for value in old["Config"]["Env"] if "=" in value + ) + self.secret_values.extend( + value + for key, value in service["environment"].items() + if value + and any( + word in key + for word in ("KEY", "TOKEN", "PASSWORD", "SECRET", "SESSION") + ) + ) + service["ports"] = [ + f"{self.env['PROD_WEB_BIND']}:{self.env['PROD_WEB_PORT']}:18787" + ] + service["volumes"] = [ + { + "type": "bind", + "source": self.env[key], + "target": target, + "bind": {"create_host_path": False}, + } + for key, target in ( + ("PROD_DATA_DIR", "/data"), + ("PROD_CONFIG_DIR", "/config"), + ) + ] + service["group_add"] = old["HostConfig"].get("GroupAdd") or [] + if photos: + service["volumes"].append( + { + "type": "bind", + "source": photos, + "target": "/photos", + "read_only": True, + "bind": {"create_host_path": False}, + } + ) + # Adoption must preserve runtime constraints, not silently change them. + if ( + old["Config"]["User"] != service["user"] + or old["HostConfig"]["RestartPolicy"]["Name"] != service["restart"] + ): + raise ValueError("Production user/restart policy differs from repository") + self.old_image = old["Image"] + volume = self.run( + "volume", "inspect", self.env["PROD_STATE_VOLUME"], check=False + ) + if ( + volume.returncode == 0 + and (json.loads(volume.stdout)[0].get("Labels") or {}).get( + "frameflow.deploy.project" + ) + != self.env["PROD_PROJECT"] + ): + raise ValueError("State volume is not owned by this deployment project") + return config + + def lock(self): + result = self.run( + "create", + "--name", + self.env["PROD_CONTAINER"] + "-deploy-lock", + "--label", + "frameflow.deploy.owner=" + self.owner, + "--network", + "none", + "--entrypoint", + "/bin/true", + self.old_image, + ) + self.lock_id = result.stdout.decode().strip() + + def unlock(self): + if self.lock_id: + self.run("rm", self.lock_id) + self.lock_id = None + + def state_io(self, payload=None): + script = ( + "import json,os,sys;from pathlib import Path;p=Path('/state/deploy.json');" + "os.umask(0o077);os.chmod('/state',0o700);v=json.load(sys.stdin);q=p.with_suffix('.tmp');q.write_text(json.dumps(v));q.replace(p)" + if payload is not None + else "from pathlib import Path;print(Path('/state/deploy.json').read_text())" + ) + return self.run( + "run", + "--rm", + "-i", + "--user", + "0:0", + "--network", + "none", + "--mount", + f"type=volume,source={self.env['PROD_STATE_VOLUME']},target=/state", + "--entrypoint", + "python", + self.old_image, + "-c", + script, + data=json.dumps(payload).encode() if payload is not None else None, + ).stdout + + def activate(self, config): + with tempfile.NamedTemporaryFile(mode="w", suffix=".yml") as file: + yaml.safe_dump(json.loads(json.dumps(config).replace("$", "$$")), file) + file.flush() + self.run( + "compose", + "-p", + self.env["PROD_PROJECT"], + "-f", + file.name, + "config", + "--quiet", + ) + self.run( + "compose", + "-p", + self.env["PROD_PROJECT"], + "-f", + file.name, + "up", + "-d", + "--no-build", + "--pull", + "never", + "pigway-frame-flow", + ) + + def verify(self, config): + container = self.env["PROD_CONTAINER"] + code = "import urllib.request;urllib.request.urlopen('http://127.0.0.1:18787/ready',timeout=3)" + for _ in range(60): + if ( + self.run( + "exec", container, "python", "-c", code, check=False + ).returncode + == 0 + ): + break + time.sleep(2) + else: + raise RuntimeError("Production /ready timeout") + self.run( + "exec", + "-e", + "FRAME_FLOW_URL=http://127.0.0.1:18787", + "-i", + container, + "python", + "-", + data=(ROOT / "scripts/smoke-test.py").read_bytes(), + ) + expected = config["services"]["pigway-frame-flow"]["environment"].get( + "BUILD_COMMIT" + ) + value = json.loads( + self.run( + "exec", + container, + "python", + "-c", + "import json,urllib.request;print(json.dumps(json.load(urllib.request.urlopen('http://127.0.0.1:18787/api/v1/version'))))", + ).stdout + ) + if value["commit"] != expected: + raise RuntimeError("Running commit mismatch") + print( + "Production /ready and all smoke endpoints: HTTP 200; commit", + value["commit"], + flush=True, + ) + return value + + def rollback(self, previous): + self.activate(previous) + self.verify(previous) + print("ROLLBACK VERIFIED", flush=True) + + def secret_config(self): + secret = self.env.get("PRODUCTION_ENV", "") + proposed = dict( + line.split("=", 1) + for line in secret.splitlines() + if line.strip() and not line.startswith("#") and "=" in line + ) + for key in ("FRAME_FLOW_TOKEN", "KOKONNA_API_KEY"): + if key not in proposed: + raise ValueError("Missing PRODUCTION_ENV key: " + key) + if self.env["PROD_MOCK"] == "false" and not proposed["KOKONNA_API_KEY"]: + raise ValueError("KoKonna API key required for non-mock production") + if not proposed["FRAME_FLOW_TOKEN"]: + raise ValueError("FRAME_FLOW_TOKEN required for smoke checks") + self.secret_values.extend(v for v in proposed.values() if v) + return proposed + + def build(self, config): + commit = ( + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT) + .decode() + .strip() + ) + main = ( + subprocess.check_output( + ["git", "ls-remote", "origin", "refs/heads/main"], cwd=ROOT + ) + .decode() + .split()[0] + ) + if commit != main: + raise ValueError("Stale main revision; refusing automatic downgrade") + image = ( + self.env["PROD_IMAGE_REPO"] + ":release-" + commit + "-" + self.owner[:8] + ) + command = ["build", "--pull", "-t", image] + now = datetime.now(UTC).isoformat() + for key, value in { + "DEBIAN_MIRROR": self.env.get("DEBIAN_MIRROR") or "deb.debian.org", + "DEBIAN_SECURITY_MIRROR": self.env.get("DEBIAN_SECURITY_MIRROR") + or "deb.debian.org", + "BUILD_COMMIT": commit, + "BUILD_TIME": now, + }.items(): + command += ["--build-arg", key + "=" + value] + self.run(*command, str(ROOT)) + service = config["services"]["pigway-frame-flow"] + service["image"] = image + proposed = self.secret_config() + service["environment"].update(proposed) + service["environment"].update( + KOKONNA_MOCK=self.env["PROD_MOCK"], + KOKONNA_WRITE_ENABLED=self.env["PROD_WRITE_ENABLED"], + BUILD_COMMIT=commit, + BUILD_TIME=now, + DATA_DIR="/data", + CONFIG_DIR="/config", + TZ="Asia/Shanghai", + ) + return config + + def diagnostics(self): + try: + self._diagnostics() + except Exception: + print( + "Scoped diagnostics unavailable; rollback still attempted", flush=True + ) + + def _diagnostics(self): + result = self.run("inspect", self.env["PROD_CONTAINER"], check=False) + if result.returncode == 0: + info = json.loads(result.stdout)[0] + state = info["State"] + print( + "Scoped production diagnostic:", + json.dumps( + { + "image": info["Image"], + "status": state.get("Status"), + "exit_code": state.get("ExitCode"), + "health": state.get("Health", {}).get("Status"), + } + ), + flush=True, + ) + logs = self.run("logs", "--tail", "60", self.env["PROD_CONTAINER"], check=False) + text = (logs.stdout + logs.stderr).decode(errors="replace") + for value in sorted(self.secret_values, key=len, reverse=True): + text = text.replace(value, "[REDACTED]") + text = re.sub(r"https?://[^\s\"'<>]+", "[REDACTED-URL]", text) + print("Scoped container logs (redacted):", text, flush=True) + + def deploy(self): + previous = self.preflight() + self.lock() + touched = False + safe_unlock = True + try: + # Refresh baseline under the daemon-level lock. + previous = self.preflight() + version = self.verify(previous) + pin = self.env["PROD_IMAGE_REPO"] + ":rollback-" + self.owner + self.run("image", "tag", self.old_image, pin) + candidate = self.build(json.loads(json.dumps(previous))) + self.state = { + "owner": self.owner, + "status": "prepared", + "previous": previous, + "candidate": candidate, + "previous_image": self.old_image, + "previous_version": version, + "rollback_tag": pin, + } + self.run( + "volume", + "create", + "--label", + "frameflow.deploy.project=" + self.env["PROD_PROJECT"], + self.env["PROD_STATE_VOLUME"], + ) + self.state_io(self.state) + # Online backup is additive; rollback never restores/deletes user data. + backup = "/data/deploy-backups/" + self.owner + ".db" + self.run( + "exec", + self.env["PROD_CONTAINER"], + "python", + "-c", + "import sqlite3;from pathlib import Path;p=Path('" + + backup + + "');p.parent.mkdir(exist_ok=True);assert Path('/data/frameflow.db').is_file();sqlite3.connect('/data/frameflow.db').backup(sqlite3.connect(p))", + ) + touched = True + self.activate(candidate) + self.verify(candidate) + self.state["status"] = "success" + self.state_io(self.state) + print("DEPLOY VERIFIED", flush=True) + except BaseException as exc: + print("DEPLOY FAILED:", type(exc).__name__, flush=True) + self.diagnostics() + if touched: + safe_unlock = False + try: + self.rollback(previous) + except BaseException: + print( + "ROLLBACK FAILED: deployment lock retained; always() recovery will retry", + flush=True, + ) + self.diagnostics() + raise + safe_unlock = True + self.state["status"] = "rolled_back" + self.state_io(self.state) + raise + finally: + if safe_unlock: + self.unlock() + + def recover(self, manual=False): + self.settings() + lock = self.run( + "inspect", self.env["PROD_CONTAINER"] + "-deploy-lock", check=False + ) + if manual: + self.preflight(require_running=False) + self.lock() + elif lock.returncode == 0: + info = json.loads(lock.stdout)[0] + if (info["Config"].get("Labels") or {}).get( + "frameflow.deploy.owner" + ) != self.owner: + return + self.lock_id = info["Id"] + self.old_image = info["Image"] + else: + return + recovered = False + try: + self.run("volume", "inspect", self.env["PROD_STATE_VOLUME"]) + volume = json.loads( + self.run("volume", "inspect", self.env["PROD_STATE_VOLUME"]).stdout + )[0] + if (volume.get("Labels") or {}).get("frameflow.deploy.project") != self.env[ + "PROD_PROJECT" + ]: + raise ValueError("Recovery state volume ownership mismatch") + state = json.loads(self.state_io()) + previous = state["previous"] + service = previous["services"]["pigway-frame-flow"] + if ( + previous["name"] != self.env["PROD_PROJECT"] + or service["container_name"] != self.env["PROD_CONTAINER"] + ): + raise ValueError("Rollback journal resource mismatch") + sources = {m["target"]: m["source"] for m in service["volumes"]} + if ( + sources.get("/data") != self.env["PROD_DATA_DIR"] + or sources.get("/config") != self.env["PROD_CONFIG_DIR"] + ): + raise ValueError("Rollback journal persistence mismatch") + if manual: + state["owner"] = self.owner + state["status"] = "prepared" + self.state_io(state) + if manual or ( + state["owner"] == self.owner + and state["status"] not in ("success", "rolled_back") + ): + self.rollback(state["previous"]) + state["status"] = "rolled_back" + self.state_io(state) + recovered = True + except BaseException: + print("RECOVERY FAILED: production lock retained", flush=True) + self.diagnostics() + raise + finally: + if recovered: + self.unlock() + + +def main(): + tool = Deploy() + action = tool.env.get("CD_ACTION") or ( + "deploy" if tool.env.get("PROD_DEPLOY_ENABLED") == "true" else "plan" + ) + if action == "recover": + if tool.env.get("FRAME_FLOW_DEPLOY_OWNER"): + tool.recover() + return + if action != "plan" and tool.env.get("PROD_DEPLOY_ENABLED") != "true": + raise SystemExit( + "Production CD disabled; set PROD_DEPLOY_ENABLED=true after safe trial" + ) + if action == "plan": + if any(not tool.env.get(key) for key in REQUIRED): + print("PLAN ONLY: configure required Variables:", ", ".join(REQUIRED)) + return + baseline = tool.preflight() + tool.secret_config() + tool.verify(baseline) + print( + "PLAN ONLY: production resource identity/mounts/port checked; no resources modified" + ) + return + if tool.env.get("GITHUB_ENV"): + with Path(tool.env["GITHUB_ENV"]).open("a") as file: + file.write("FRAME_FLOW_DEPLOY_OWNER=" + tool.owner + "\n") + + def cancel(*_): + raise SystemExit(143) + + signal.signal(signal.SIGTERM, cancel) + if action == "deploy": + tool.deploy() + elif action == "rollback": + tool.recover(manual=True) + else: + raise SystemExit("Unknown CD_ACTION") + + +if __name__ == "__main__": + main() diff --git a/tests/test_production_deploy.py b/tests/test_production_deploy.py new file mode 100644 index 0000000..ed473ac --- /dev/null +++ b/tests/test_production_deploy.py @@ -0,0 +1,245 @@ +"""Deployment gates, isolation checks, and rollback use mocked Docker only.""" + +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture +def tool(): + spec = importlib.util.spec_from_file_location( + "production_deploy", Path(__file__).parents[1] / "scripts/production-deploy.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + instance = module.Deploy() + instance.env = { + "PROD_PROJECT": "production-frame", + "PROD_CONTAINER": "production-frame", + "PROD_IMAGE_REPO": "pigway-frame-flow", + "PROD_DATA_DIR": "/confirmed/data", + "PROD_CONFIG_DIR": "/confirmed/config", + "PROD_WEB_BIND": "192.0.2.1", + "PROD_WEB_PORT": "18787", + "PROD_STATE_VOLUME": "production-frame-cd", + "PROD_MOCK": "false", + "PROD_WRITE_ENABLED": "false", + } + return instance + + +def existing(tool): + return { + "Image": "sha256:previous", + "State": {"Running": True}, + "Config": { + "User": "10333:10333", + "Env": ["BUILD_COMMIT=previous", "FRAME_FLOW_TOKEN=fixture-token"], + "Labels": { + "com.docker.compose.project": tool.env["PROD_PROJECT"], + "com.docker.compose.service": "pigway-frame-flow", + }, + }, + "Mounts": [ + {"Type": "bind", "Destination": target, "Source": tool.env[key], "RW": True} + for target, key in [ + ("/data", "PROD_DATA_DIR"), + ("/config", "PROD_CONFIG_DIR"), + ] + ], + "HostConfig": { + "PortBindings": { + "18787/tcp": [ + {"HostIp": tool.env["PROD_WEB_BIND"], "HostPort": "18787"} + ] + }, + "NetworkMode": "production-frame_default", + "RestartPolicy": {"Name": "unless-stopped"}, + }, + } + + +def test_preflight_preserves_paths_and_has_no_write_commands(tool, monkeypatch): + monkeypatch.setattr(tool, "inspect", lambda _: existing(tool)) + calls = [] + monkeypatch.setattr( + tool, + "run", + lambda *args, **kwargs: calls.append(args) or SimpleNamespace(returncode=1), + ) + config = tool.preflight() + service = config["services"]["pigway-frame-flow"] + assert service["image"] == "sha256:previous" + assert all(m["bind"]["create_host_path"] is False for m in service["volumes"]) + assert service["environment"]["FRAME_FLOW_TOKEN"] == "fixture-token" + assert calls == [("volume", "inspect", "production-frame-cd")] + + +@pytest.mark.parametrize("change", ["path", "project", "port", "network"]) +def test_preflight_rejects_wrong_host_resources(tool, monkeypatch, change): + old = existing(tool) + if change == "path": + old["Mounts"][0]["Source"] = "/unrelated/data" + elif change == "project": + old["Config"]["Labels"]["com.docker.compose.project"] = "ffci-fixture" + elif change == "port": + old["HostConfig"]["PortBindings"]["18787/tcp"][0]["HostPort"] = "9999" + else: + old["HostConfig"]["NetworkMode"] = "host" + monkeypatch.setattr(tool, "inspect", lambda _: old) + with pytest.raises(ValueError): + tool.preflight() + + +@pytest.mark.parametrize("failure", [None, "candidate", "rollback"]) +def test_deploy_rolls_back_exact_baseline_without_deleting_data( + tool, monkeypatch, failure +): + old = {"services": {"pigway-frame-flow": {"image": "sha256:previous"}}} + new = {"services": {"pigway-frame-flow": {"image": "candidate"}}} + activated, calls, saved = [], [], [] + monkeypatch.setattr(tool, "preflight", lambda: old) + monkeypatch.setattr(tool, "lock", lambda: calls.append("lock")) + monkeypatch.setattr(tool, "unlock", lambda: calls.append("unlock")) + monkeypatch.setattr(tool, "run", lambda *args, **kw: calls.append(args)) + monkeypatch.setattr(tool, "build", lambda _: new) + monkeypatch.setattr(tool, "diagnostics", lambda: None) + monkeypatch.setattr(tool, "activate", lambda c: activated.append(c)) + monkeypatch.setattr( + tool, "state_io", lambda state: saved.append(json.loads(json.dumps(state))) + ) + verifies = [] + + def verify(config): + verifies.append(config) + if config == new and failure: + raise RuntimeError("candidate not ready") + if config == old and failure == "rollback" and len(verifies) > 1: + raise RuntimeError("rollback not ready") + return {"commit": "previous", "version": "0.1.0"} + + monkeypatch.setattr(tool, "verify", verify) + if failure: + with pytest.raises(RuntimeError): + tool.deploy() + assert activated == [new, old] + assert saved[0]["previous"] == old + assert ("unlock" in calls) is (failure != "rollback") + else: + tool.deploy() + assert activated == [new] + assert saved[-1]["status"] == "success" + assert not any( + isinstance(c, tuple) and ("down" in c or "prune" in c) for c in calls + ) + + +def test_activate_escapes_dollar_secrets_and_never_down(tool, monkeypatch): + calls = [] + + def run(*args, **kwargs): + calls.append(args) + if "config" in args: + path = Path(args[args.index("-f") + 1]) + assert "$$literal" in path.read_text() + + monkeypatch.setattr(tool, "run", run) + tool.activate( + { + "services": { + "pigway-frame-flow": {"environment": {"FRAME_FLOW_TOKEN": "$literal"}} + } + } + ) + assert len(calls) == 2 and "up" in calls[1] and "--no-build" in calls[1] + + +def test_workflow_deploy_is_verify_gated_and_manual_plan_default(tool): + import yaml + + workflow = yaml.safe_load( + (Path(__file__).parents[1] / ".github/workflows/verify.yml").read_text() + ) + job = workflow["jobs"]["deploy"] + assert job["needs"] == "verify" and "success()" in job["if"] + assert "refs/heads/main" in job["if"] and "pull_request" not in job["if"] + assert job["env"]["PROD_DEPLOY_ENABLED"] == "${{ vars.PROD_DEPLOY_ENABLED }}" + assert ( + workflow[True]["workflow_dispatch"]["inputs"]["deploy_action"]["default"] + == "plan" + ) + assert job["steps"][-1]["if"] == "always()" + + +def test_recovery_handles_missing_or_crashed_production_container(tool, monkeypatch): + previous = { + "name": tool.env["PROD_PROJECT"], + "services": { + "pigway-frame-flow": { + "container_name": tool.env["PROD_CONTAINER"], + "image": "sha256:old", + "volumes": [ + {"source": tool.env["PROD_DATA_DIR"], "target": "/data"}, + {"source": tool.env["PROD_CONFIG_DIR"], "target": "/config"}, + ], + } + }, + } + state = {"owner": tool.owner, "status": "prepared", "previous": previous} + calls = [] + + def run(*args, **kwargs): + calls.append(args) + if args[0] == "inspect": + assert args[1].endswith("-deploy-lock") + return SimpleNamespace( + returncode=0, + stdout=json.dumps( + [ + { + "Id": "owned-lock", + "Image": "sha256:old", + "Config": { + "Labels": {"frameflow.deploy.owner": tool.owner} + }, + } + ] + ).encode(), + ) + return SimpleNamespace( + returncode=0, + stdout=json.dumps( + [{"Labels": {"frameflow.deploy.project": tool.env["PROD_PROJECT"]}}] + ).encode(), + ) + + monkeypatch.setattr(tool, "run", run) + + def state_io(payload=None): + if payload is not None: + state.update(payload) + return json.dumps(state).encode() + + monkeypatch.setattr(tool, "state_io", state_io) + monkeypatch.setattr( + tool, "rollback", lambda config: calls.append(("rollback", config)) + ) + tool.recover() + assert ("rollback", previous) in calls + assert ("rm", "owned-lock") in calls + assert state["status"] == "rolled_back" + + +def test_lock_conflict_never_activates_production(tool, monkeypatch): + monkeypatch.setattr(tool, "preflight", lambda: {}) + monkeypatch.setattr(tool, "activate", lambda _: pytest.fail("Production touched")) + + def fail(*args, **kwargs): + raise RuntimeError("Lock already reserved") + + monkeypatch.setattr(tool, "run", fail) + with pytest.raises(RuntimeError): + tool.deploy()