Add opt-in production CD with daemon lock, verified rollback and safe planning

.github/workflows/verify.yml

@@ -1,5 +1,14 @@
 name: Verify Frame Flow
-on: [push, pull_request, workflow_dispatch]
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      deploy_action:
+        description: Production action (deploy requires explicit enabled Variable)
+        type: choice
+        options: [plan, deploy, rollback]
+        default: plan
 jobs:
   verify:
     runs-on: ubuntu-latest
@@ -30,3 +39,50 @@ jobs:
           if [ -n "${FRAME_FLOW_CI_COMPOSE:-}" ] && [ -f "$FRAME_FLOW_CI_COMPOSE" ]; then
             python scripts/ci-compose-verify.py --cleanup
           fi
+
+  deploy:
+    needs: verify
+    if: success() && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
+    runs-on: ubuntu-latest
+    concurrency:
+      group: frame-flow-production
+      cancel-in-progress: false
+    env:
+      CD_ACTION: ${{ github.event.inputs.deploy_action }}
+      PROD_DEPLOY_ENABLED: ${{ vars.PROD_DEPLOY_ENABLED }}
+      PROD_PROJECT: ${{ vars.PROD_PROJECT }}
+      PROD_CONTAINER: ${{ vars.PROD_CONTAINER }}
+      PROD_IMAGE_REPO: ${{ vars.PROD_IMAGE_REPO }}
+      PROD_DATA_DIR: ${{ vars.PROD_DATA_DIR }}
+      PROD_CONFIG_DIR: ${{ vars.PROD_CONFIG_DIR }}
+      PROD_PHOTOS_DIR: ${{ vars.PROD_PHOTOS_DIR }}
+      PROD_WEB_BIND: ${{ vars.PROD_WEB_BIND }}
+      PROD_WEB_PORT: ${{ vars.PROD_WEB_PORT }}
+      PROD_STATE_VOLUME: ${{ vars.PROD_STATE_VOLUME }}
+      PROD_MOCK: ${{ vars.PROD_MOCK }}
+      PROD_WRITE_ENABLED: ${{ vars.PROD_WRITE_ENABLED }}
+      CI_PIP_INDEX_URL: ${{ vars.CI_PIP_INDEX_URL }}
+      CI_PIP_FALLBACK_INDEX_URL: ${{ vars.CI_PIP_FALLBACK_INDEX_URL }}
+      DEBIAN_MIRROR: ${{ vars.DEBIAN_MIRROR }}
+      DEBIAN_SECURITY_MIRROR: ${{ vars.DEBIAN_SECURITY_MIRROR }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.14'
+      - name: Install deploy tooling
+        run: python scripts/ci-install.py
+      - name: Plan or deploy production (default is plan)
+        env:
+          PRODUCTION_ENV: ${{ secrets.PRODUCTION_ENV }}
+        run: |
+          . ./deploy/build-cn.env
+          python scripts/production-deploy.py
+      - name: Recover interrupted production deployment
+        if: always()
+        env:
+          CD_ACTION: recover
+        run: |
+          if [ -n "${FRAME_FLOW_DEPLOY_OWNER:-}" ]; then
+            python scripts/production-deploy.py
+          fi

README.md

@@ -338,3 +338,5 @@ APT only and are loaded later in the Docker verification step.
 If dependency installation fails, the always() cleanup step still executes.
 It skips the Python/PyYAML cleanup script when no CI Compose manifest exists,
 as no Docker verification resources have been created at that point.
+
+Production CD: [safe trial, configuration, and rollback](docs/deployment/GITEA_CD.md).

docs/deployment/GITEA_CD.md

@@ -0,0 +1,169 @@
+# Gitea Actions production CD v1
+
+## Audit and adoption boundary
+
+Production `docker-compose.yml` is unchanged: AMD64 image, UID/GID 10333,
+`unless-stopped`, read-only rootfs, `/tmp` tmpfs, dropped capabilities, limited
+logs, `/data` and `/config` bind mounts, default private bridge, and configurable
+host bind/port. SQLite/images/generated/cache live under `/data`; accounts,
+vendor sessions and configuration live under `/config`. Neither is disposable.
+The optional photos mount is read-only and has supplementary group access.
+
+Historical repository receipts mention `/volume2/docker/pigway-frame-flow/data`,
+`/volume2/docker/pigway-frame-flow/config`, `/volume1/photo`, port 18787, and
+container/project `pigway-frame-flow`. These are **confirmation candidates**, not
+script defaults. The deployment script accepts only explicitly configured paths
+that match the existing running container on the Runner's Docker daemon.
+It rejects different projects, ports, unknown mounts, host/custom networking,
+missing photos confirmation, or different user/restart policy. It does not
+create NAS host directories or migrate/adopt an unrelated container.
+
+CI uses `ffci-*` projects, per-run image tags and named volumes, no host ports.
+Its cleanup does not refer to any production resource. Production uses fixed
+configured names and existing bind mounts, with `create_host_path: false`.
+
+## Gitea configuration
+
+Repository Settings → Actions → Variables (ordinary configuration):
+
+| Variable | Value to explicitly confirm |
+| --- | --- |
+| `PROD_DEPLOY_ENABLED` | Initially `false`; change to `true` only after plan acceptance |
+| `PROD_PROJECT` | Existing Compose project label; candidate `pigway-frame-flow` |
+| `PROD_CONTAINER` | Existing container name; candidate `pigway-frame-flow` |
+| `PROD_IMAGE_REPO` | Fixed local image repository; suggested `pigway-frame-flow` |
+| `PROD_DATA_DIR` | Existing absolute NAS host data path; no default |
+| `PROD_CONFIG_DIR` | Existing absolute NAS host config path; no default |
+| `PROD_PHOTOS_DIR` | Existing read-only photos source if present; otherwise empty |
+| `PROD_WEB_BIND` | Exact existing Docker HostIp (`0.0.0.0` or NAS LAN IP etc.) |
+| `PROD_WEB_PORT` | Exact existing mapped port |
+| `PROD_STATE_VOLUME` | Dedicated new deployment journal volume, e.g. `pigway-frame-flow-cd-state` |
+| `PROD_MOCK` | Explicit `true`/`false`, consistent with intended production mode |
+| `PROD_WRITE_ENABLED` | Explicit `true`/`false`; preserve owner's existing write policy |
+
+The state volume must be separate from application/CI storage. An existing
+volume without the matching `frameflow.deploy.project` label is rejected.
+Supplementary groups are read from the existing container; no host ACL changes.
+Keep existing `DEBIAN_MIRROR`, `DEBIAN_SECURITY_MIRROR`, `CI_PIP_INDEX_URL`, and
+`CI_PIP_FALLBACK_INDEX_URL` settings from Verify; CI/production builds share the
+Dockerfile, pinned base, lockfile, and BuildKit caches.
+
+Repository Settings → Actions → Secrets:
+
+`PRODUCTION_ENV`: multiline **unquoted** `KEY=value` entries, without shell
+expansion. At least:
+
+```dotenv
+FRAME_FLOW_TOKEN=<existing Frame Flow administrator token>
+KOKONNA_API_KEY=<existing KoKonna key, or empty when mock>
+```
+
+Optional additional runtime secret variables can be included. No key values are
+committed, passed to image builds, or printed. Nonsecret mock/write flags use
+Variables. Other existing runtime environment values are retained. The script
+sets commit/build time and internal data/config paths explicitly. Dollar signs
+in secret values are escaped before Compose interpolation. Snapshots containing
+old runtime secrets are stored only in a private journal file (0600, directory
+0700) on the dedicated Docker volume; Docker administrators can access it.
+Never upload that journal or a full Docker inspect as a public CI artifact.
+
+## First safe trial
+
+1. On NAS, inspect only the necessary fields (use existing Docker/sudo access):
+   ```sh
+   docker inspect pigway-frame-flow --format '{{index .Config.Labels "com.docker.compose.project"}}'
+   docker inspect pigway-frame-flow --format '{{json .Mounts}}'
+   docker inspect pigway-frame-flow --format '{{json .HostConfig.PortBindings}} {{.HostConfig.NetworkMode}}'
+   ```
+   Confirm the Variables against these actual values, including photos.
+2. Configure the Secret privately. Keep `PROD_DEPLOY_ENABLED=false`.
+3. Manually run **Verify Frame Flow** on `main`, `deploy_action=plan`.
+   Verify must pass first. Plan checks resource identity, paths/port, required
+   Secret keys, current `/ready`, smoke endpoints and commit. It performs no
+   build, stop, restart, tagging, lock creation, volume creation, or data writes.
+   If no production Variables exist, it reports configuration missing and does
+   nothing; that is an unconfigured plan, not production acceptance.
+4. Review `PLAN ONLY` output and arrange a maintenance window for the first real
+   deploy/restart. Set `PROD_DEPLOY_ENABLED=true`, manually run `deploy_action=deploy`.
+5. After `DEPLOY VERIFIED`, subsequent pushes to `main` run Verify → deploy.
+   Pushes to other branches and pull requests never deploy. A manual run defaults
+   to plan, even when automatic deployment is enabled.
+
+The script intentionally supports updates to an existing healthy deployment,
+not first-time bootstrap. No real production deployment or rollback drill has
+been performed during implementation; those are first-run acceptance steps.
+
+## Deployment transaction and rollback
+
+1. Verify job must succeed completely, including isolated Docker tests/cleanup.
+2. Deploy job checks out the same revision and prepares tools. An atomic
+   `docker create` reservation named `<PROD_CONTAINER>-deploy-lock` serializes
+   production changes on the daemon. Job `concurrency` is also declared, but
+   correctness does not depend on Gitea implementing it. A competing deploy
+   fails safely instead of changing production. No other job may remove a lock
+   whose owner label differs from its own.
+3. Re-read the current baseline under the lock; confirm ready/smoke. Record exact
+   image ID, runtime Compose/environment, version/commit. Pin the old image with
+   an immutable rollback tag so normal dangling-image cleanup cannot remove it.
+4. Refuse a stale checked-out `main` revision that no longer equals remote main.
+   Build an immutable `release-<commit>-<owner>` tag locally, without a registry
+   and without changing `latest` or CI image tags.
+5. Persist a `prepared` journal with old/new runtime configs. Create an additive
+   SQLite online backup under `/data/deploy-backups/<owner>.db`. No data/config
+   directory or file is removed, replaced or recursively chowned.
+6. `compose up -d --no-build --pull never` replaces only the configured service.
+   Check `/ready`, the unchanged nine-endpoint read-only smoke test, and exact
+   `/api/v1/version` commit. Only all checks passing produce `DEPLOY VERIFIED`.
+7. Any activation/verification failure runs `up` again with the saved exact old
+   image ID and old runtime environment/mounts, then repeats ready/smoke/commit
+   verification. A failed deploy remains a failed workflow even if rollback
+   succeeds. If rollback fails, the journal/lock are retained, failure is explicit,
+   and redacted scoped diagnostics are emitted.
+8. An `always()` step uses the recorded owner to recover an interrupted prepared
+   transaction and release only its own lock. Abrupt host loss or termination of
+   the entire runner can prevent both handlers: inspect the journal/lock before
+   manually recovering; never automatically force-unlock an unknown owner.
+
+Rollback is an **image/config rollback**, not an automatic database rewind.
+User data accumulated during deployment remains. The SQLite backup is for
+operator-reviewed recovery; destructive or backward-incompatible future schema
+changes must not be auto-deployed without a separate migration/backup plan.
+Baseline artifacts and additive backups are retained for the operator; there is
+no automated pruning of production rollback history or data.
+
+After enabling CD, use its workflow/script as the production lifecycle owner.
+The old NAS checkout's Compose/.env files are not rewritten remotely by CD and
+may describe an older deployment. Do not concurrently recreate production from
+that old checkout. The repository's standalone Compose remains usable for an
+independent deployment; the private journal records the CD runtime configuration.
+
+## Actual commit and manual rollback
+
+```sh
+curl -fsS http://NAS_LAN_IP:PORT/api/v1/version
+# Check `commit`, `version`, and `build_time` against successful deployment output.
+docker inspect PRODUCTION_CONTAINER --format '{{.Image}}'
+```
+
+For a manual rollback: keep CD enabled and run **Verify Frame Flow** on `main`
+with `deploy_action=rollback`. After Verify passes, the script locks production,
+loads the last journal's `previous` image/config and verifies recovery. This is
+last-baseline rollback, not an arbitrary image selector. Never use `down -v`,
+volume pruning, or delete data/config to roll back.
+
+If a failed run retained a lock, inspect only its owner label:
+
+```sh
+docker inspect PRODUCTION_CONTAINER-deploy-lock --format '{{index .Config.Labels "frameflow.deploy.owner"}}'
+```
+
+With the documented Variables configured in a trusted Docker-enabled shell:
+
+```sh
+FRAME_FLOW_DEPLOY_OWNER=<confirmed failed owner> CD_ACTION=recover python scripts/production-deploy.py
+```
+
+`recover` retries that owner's journaled rollback and retains the lock again if
+recovery fails. If the very first attempt stopped before a journal was written,
+confirm that the original image is still running and healthy before an operator
+removes the stale lock. Do not delete a live lock or any persistence volume.

scripts/production-deploy.py

@@ -0,0 +1,588 @@
+"""Opt-in production deployment; host paths verified against the running container."""
+
+import json
+import os
+import re
+import signal
+import subprocess
+import tempfile
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+from uuid import uuid4
+
+import yaml
+
+ROOT = Path(__file__).resolve().parents[1]
+REQUIRED = (
+    "PROD_PROJECT",
+    "PROD_CONTAINER",
+    "PROD_IMAGE_REPO",
+    "PROD_DATA_DIR",
+    "PROD_CONFIG_DIR",
+    "PROD_WEB_BIND",
+    "PROD_WEB_PORT",
+    "PROD_STATE_VOLUME",
+    "PROD_MOCK",
+    "PROD_WRITE_ENABLED",
+)
+
+
+class Deploy:
+    def __init__(self):
+        self.env = os.environ.copy()
+        self.owner = self.env.get("FRAME_FLOW_DEPLOY_OWNER") or uuid4().hex
+        if not re.fullmatch(r"[0-9a-f]{32}", self.owner):
+            raise ValueError("Invalid deployment owner")
+        self.lock_id = None
+        self.state = None
+        self.old_image = None
+        self.secret_values = []
+        self.docker = self.env.get("DOCKER_BIN", "docker")
+
+    def run(self, *args, data=None, check=True):
+        result = subprocess.run(
+            [self.docker, *args],
+            input=data,
+            capture_output=True,
+            env=self.env | {"DOCKER_BUILDKIT": "1"},
+        )
+        if check and result.returncode:
+            # Never print Docker inspect/config/env output or secret-bearing commands.
+            raise RuntimeError(
+                f"Docker {args[0]} failed (exit {result.returncode}); inspect scoped container logs privately"
+            )
+        return result
+
+    def inspect(self, name):
+        return json.loads(self.run("inspect", name).stdout)[0]
+
+    def settings(self):
+        missing = [key for key in REQUIRED if not self.env.get(key)]
+        if missing:
+            raise ValueError("Missing Variables: " + ", ".join(missing))
+        for key in (
+            "PROD_PROJECT",
+            "PROD_CONTAINER",
+            "PROD_IMAGE_REPO",
+            "PROD_STATE_VOLUME",
+        ):
+            if not re.fullmatch(r"[a-z][a-z0-9-]{2,80}", self.env[key]) or self.env[
+                key
+            ].startswith("ffci-"):
+                raise ValueError("Unsafe production resource name: " + key)
+        for key in ("PROD_DATA_DIR", "PROD_CONFIG_DIR"):
+            if not self.env[key].startswith("/") or self.env[key] == "/":
+                raise ValueError("Absolute existing NAS path required: " + key)
+        if self.env["PROD_DATA_DIR"] == self.env["PROD_CONFIG_DIR"]:
+            raise ValueError("data/config must use distinct directories")
+        if any(
+            self.env[key] not in ("true", "false")
+            for key in ("PROD_MOCK", "PROD_WRITE_ENABLED")
+        ):
+            raise ValueError("Explicit true/false required for mock and writes")
+        if not 1 <= int(self.env["PROD_WEB_PORT"]) <= 65535:
+            raise ValueError("Invalid production port")
+
+    def preflight(self, require_running=True):
+        self.settings()
+        old = self.inspect(self.env["PROD_CONTAINER"])
+        labels = old["Config"].get("Labels") or {}
+        if (
+            labels.get("com.docker.compose.project") != self.env["PROD_PROJECT"]
+            or labels.get("com.docker.compose.service") != "pigway-frame-flow"
+        ):
+            raise ValueError(
+                "Existing container does not belong to the configured production project/service"
+            )
+        if require_running and not old["State"]["Running"]:
+            raise ValueError("Existing production container must be running")
+        mounts = {m["Destination"]: m for m in old["Mounts"]}
+        for target, key in (("/data", "PROD_DATA_DIR"), ("/config", "PROD_CONFIG_DIR")):
+            m = mounts.get(target, {})
+            if (
+                m.get("Type") != "bind"
+                or m.get("Source") != self.env[key]
+                or not m.get("RW")
+            ):
+                raise ValueError("Production mount mismatch: " + key)
+        photos = self.env.get("PROD_PHOTOS_DIR")
+        if "/photos" in mounts and (
+            not photos
+            or mounts["/photos"]["Source"] != photos
+            or mounts["/photos"]["RW"]
+        ):
+            raise ValueError(
+                "Confirm PROD_PHOTOS_DIR for the existing read-only photo mount"
+            )
+        if photos and "/photos" not in mounts:
+            raise ValueError(
+                "Configured photos directory does not match current container"
+            )
+        if set(mounts) != {"/data", "/config"} | ({"/photos"} if photos else set()):
+            raise ValueError("Unrecognized persistent mount; review before adopting CD")
+        bindings = old["HostConfig"].get("PortBindings", {}).get("18787/tcp")
+        if bindings != [
+            {"HostIp": self.env["PROD_WEB_BIND"], "HostPort": self.env["PROD_WEB_PORT"]}
+        ]:
+            raise ValueError(
+                "Port/bind mismatch; CD must not change production networking"
+            )
+        if (
+            old["HostConfig"].get("NetworkMode")
+            != self.env["PROD_PROJECT"] + "_default"
+        ):
+            raise ValueError("Custom production network requires explicit review")
+        config = yaml.safe_load((ROOT / "docker-compose.yml").read_text())
+        config["name"] = self.env["PROD_PROJECT"]
+        service = config["services"]["pigway-frame-flow"]
+        service.pop("build")
+        service.pop("env_file")
+        service["container_name"] = self.env["PROD_CONTAINER"]
+        service["image"] = old["Image"]
+        service["environment"] = dict(
+            value.split("=", 1) for value in old["Config"]["Env"] if "=" in value
+        )
+        self.secret_values.extend(
+            value
+            for key, value in service["environment"].items()
+            if value
+            and any(
+                word in key
+                for word in ("KEY", "TOKEN", "PASSWORD", "SECRET", "SESSION")
+            )
+        )
+        service["ports"] = [
+            f"{self.env['PROD_WEB_BIND']}:{self.env['PROD_WEB_PORT']}:18787"
+        ]
+        service["volumes"] = [
+            {
+                "type": "bind",
+                "source": self.env[key],
+                "target": target,
+                "bind": {"create_host_path": False},
+            }
+            for key, target in (
+                ("PROD_DATA_DIR", "/data"),
+                ("PROD_CONFIG_DIR", "/config"),
+            )
+        ]
+        service["group_add"] = old["HostConfig"].get("GroupAdd") or []
+        if photos:
+            service["volumes"].append(
+                {
+                    "type": "bind",
+                    "source": photos,
+                    "target": "/photos",
+                    "read_only": True,
+                    "bind": {"create_host_path": False},
+                }
+            )
+        # Adoption must preserve runtime constraints, not silently change them.
+        if (
+            old["Config"]["User"] != service["user"]
+            or old["HostConfig"]["RestartPolicy"]["Name"] != service["restart"]
+        ):
+            raise ValueError("Production user/restart policy differs from repository")
+        self.old_image = old["Image"]
+        volume = self.run(
+            "volume", "inspect", self.env["PROD_STATE_VOLUME"], check=False
+        )
+        if (
+            volume.returncode == 0
+            and (json.loads(volume.stdout)[0].get("Labels") or {}).get(
+                "frameflow.deploy.project"
+            )
+            != self.env["PROD_PROJECT"]
+        ):
+            raise ValueError("State volume is not owned by this deployment project")
+        return config
+
+    def lock(self):
+        result = self.run(
+            "create",
+            "--name",
+            self.env["PROD_CONTAINER"] + "-deploy-lock",
+            "--label",
+            "frameflow.deploy.owner=" + self.owner,
+            "--network",
+            "none",
+            "--entrypoint",
+            "/bin/true",
+            self.old_image,
+        )
+        self.lock_id = result.stdout.decode().strip()
+
+    def unlock(self):
+        if self.lock_id:
+            self.run("rm", self.lock_id)
+            self.lock_id = None
+
+    def state_io(self, payload=None):
+        script = (
+            "import json,os,sys;from pathlib import Path;p=Path('/state/deploy.json');"
+            "os.umask(0o077);os.chmod('/state',0o700);v=json.load(sys.stdin);q=p.with_suffix('.tmp');q.write_text(json.dumps(v));q.replace(p)"
+            if payload is not None
+            else "from pathlib import Path;print(Path('/state/deploy.json').read_text())"
+        )
+        return self.run(
+            "run",
+            "--rm",
+            "-i",
+            "--user",
+            "0:0",
+            "--network",
+            "none",
+            "--mount",
+            f"type=volume,source={self.env['PROD_STATE_VOLUME']},target=/state",
+            "--entrypoint",
+            "python",
+            self.old_image,
+            "-c",
+            script,
+            data=json.dumps(payload).encode() if payload is not None else None,
+        ).stdout
+
+    def activate(self, config):
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".yml") as file:
+            yaml.safe_dump(json.loads(json.dumps(config).replace("$", "$$")), file)
+            file.flush()
+            self.run(
+                "compose",
+                "-p",
+                self.env["PROD_PROJECT"],
+                "-f",
+                file.name,
+                "config",
+                "--quiet",
+            )
+            self.run(
+                "compose",
+                "-p",
+                self.env["PROD_PROJECT"],
+                "-f",
+                file.name,
+                "up",
+                "-d",
+                "--no-build",
+                "--pull",
+                "never",
+                "pigway-frame-flow",
+            )
+
+    def verify(self, config):
+        container = self.env["PROD_CONTAINER"]
+        code = "import urllib.request;urllib.request.urlopen('http://127.0.0.1:18787/ready',timeout=3)"
+        for _ in range(60):
+            if (
+                self.run(
+                    "exec", container, "python", "-c", code, check=False
+                ).returncode
+                == 0
+            ):
+                break
+            time.sleep(2)
+        else:
+            raise RuntimeError("Production /ready timeout")
+        self.run(
+            "exec",
+            "-e",
+            "FRAME_FLOW_URL=http://127.0.0.1:18787",
+            "-i",
+            container,
+            "python",
+            "-",
+            data=(ROOT / "scripts/smoke-test.py").read_bytes(),
+        )
+        expected = config["services"]["pigway-frame-flow"]["environment"].get(
+            "BUILD_COMMIT"
+        )
+        value = json.loads(
+            self.run(
+                "exec",
+                container,
+                "python",
+                "-c",
+                "import json,urllib.request;print(json.dumps(json.load(urllib.request.urlopen('http://127.0.0.1:18787/api/v1/version'))))",
+            ).stdout
+        )
+        if value["commit"] != expected:
+            raise RuntimeError("Running commit mismatch")
+        print(
+            "Production /ready and all smoke endpoints: HTTP 200; commit",
+            value["commit"],
+            flush=True,
+        )
+        return value
+
+    def rollback(self, previous):
+        self.activate(previous)
+        self.verify(previous)
+        print("ROLLBACK VERIFIED", flush=True)
+
+    def secret_config(self):
+        secret = self.env.get("PRODUCTION_ENV", "")
+        proposed = dict(
+            line.split("=", 1)
+            for line in secret.splitlines()
+            if line.strip() and not line.startswith("#") and "=" in line
+        )
+        for key in ("FRAME_FLOW_TOKEN", "KOKONNA_API_KEY"):
+            if key not in proposed:
+                raise ValueError("Missing PRODUCTION_ENV key: " + key)
+        if self.env["PROD_MOCK"] == "false" and not proposed["KOKONNA_API_KEY"]:
+            raise ValueError("KoKonna API key required for non-mock production")
+        if not proposed["FRAME_FLOW_TOKEN"]:
+            raise ValueError("FRAME_FLOW_TOKEN required for smoke checks")
+        self.secret_values.extend(v for v in proposed.values() if v)
+        return proposed
+
+    def build(self, config):
+        commit = (
+            subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT)
+            .decode()
+            .strip()
+        )
+        main = (
+            subprocess.check_output(
+                ["git", "ls-remote", "origin", "refs/heads/main"], cwd=ROOT
+            )
+            .decode()
+            .split()[0]
+        )
+        if commit != main:
+            raise ValueError("Stale main revision; refusing automatic downgrade")
+        image = (
+            self.env["PROD_IMAGE_REPO"] + ":release-" + commit + "-" + self.owner[:8]
+        )
+        command = ["build", "--pull", "-t", image]
+        now = datetime.now(UTC).isoformat()
+        for key, value in {
+            "DEBIAN_MIRROR": self.env.get("DEBIAN_MIRROR") or "deb.debian.org",
+            "DEBIAN_SECURITY_MIRROR": self.env.get("DEBIAN_SECURITY_MIRROR")
+            or "deb.debian.org",
+            "BUILD_COMMIT": commit,
+            "BUILD_TIME": now,
+        }.items():
+            command += ["--build-arg", key + "=" + value]
+        self.run(*command, str(ROOT))
+        service = config["services"]["pigway-frame-flow"]
+        service["image"] = image
+        proposed = self.secret_config()
+        service["environment"].update(proposed)
+        service["environment"].update(
+            KOKONNA_MOCK=self.env["PROD_MOCK"],
+            KOKONNA_WRITE_ENABLED=self.env["PROD_WRITE_ENABLED"],
+            BUILD_COMMIT=commit,
+            BUILD_TIME=now,
+            DATA_DIR="/data",
+            CONFIG_DIR="/config",
+            TZ="Asia/Shanghai",
+        )
+        return config
+
+    def diagnostics(self):
+        try:
+            self._diagnostics()
+        except Exception:
+            print(
+                "Scoped diagnostics unavailable; rollback still attempted", flush=True
+            )
+
+    def _diagnostics(self):
+        result = self.run("inspect", self.env["PROD_CONTAINER"], check=False)
+        if result.returncode == 0:
+            info = json.loads(result.stdout)[0]
+            state = info["State"]
+            print(
+                "Scoped production diagnostic:",
+                json.dumps(
+                    {
+                        "image": info["Image"],
+                        "status": state.get("Status"),
+                        "exit_code": state.get("ExitCode"),
+                        "health": state.get("Health", {}).get("Status"),
+                    }
+                ),
+                flush=True,
+            )
+        logs = self.run("logs", "--tail", "60", self.env["PROD_CONTAINER"], check=False)
+        text = (logs.stdout + logs.stderr).decode(errors="replace")
+        for value in sorted(self.secret_values, key=len, reverse=True):
+            text = text.replace(value, "[REDACTED]")
+        text = re.sub(r"https?://[^\s\"'<>]+", "[REDACTED-URL]", text)
+        print("Scoped container logs (redacted):", text, flush=True)
+
+    def deploy(self):
+        previous = self.preflight()
+        self.lock()
+        touched = False
+        safe_unlock = True
+        try:
+            # Refresh baseline under the daemon-level lock.
+            previous = self.preflight()
+            version = self.verify(previous)
+            pin = self.env["PROD_IMAGE_REPO"] + ":rollback-" + self.owner
+            self.run("image", "tag", self.old_image, pin)
+            candidate = self.build(json.loads(json.dumps(previous)))
+            self.state = {
+                "owner": self.owner,
+                "status": "prepared",
+                "previous": previous,
+                "candidate": candidate,
+                "previous_image": self.old_image,
+                "previous_version": version,
+                "rollback_tag": pin,
+            }
+            self.run(
+                "volume",
+                "create",
+                "--label",
+                "frameflow.deploy.project=" + self.env["PROD_PROJECT"],
+                self.env["PROD_STATE_VOLUME"],
+            )
+            self.state_io(self.state)
+            # Online backup is additive; rollback never restores/deletes user data.
+            backup = "/data/deploy-backups/" + self.owner + ".db"
+            self.run(
+                "exec",
+                self.env["PROD_CONTAINER"],
+                "python",
+                "-c",
+                "import sqlite3;from pathlib import Path;p=Path('"
+                + backup
+                + "');p.parent.mkdir(exist_ok=True);assert Path('/data/frameflow.db').is_file();sqlite3.connect('/data/frameflow.db').backup(sqlite3.connect(p))",
+            )
+            touched = True
+            self.activate(candidate)
+            self.verify(candidate)
+            self.state["status"] = "success"
+            self.state_io(self.state)
+            print("DEPLOY VERIFIED", flush=True)
+        except BaseException as exc:
+            print("DEPLOY FAILED:", type(exc).__name__, flush=True)
+            self.diagnostics()
+            if touched:
+                safe_unlock = False
+                try:
+                    self.rollback(previous)
+                except BaseException:
+                    print(
+                        "ROLLBACK FAILED: deployment lock retained; always() recovery will retry",
+                        flush=True,
+                    )
+                    self.diagnostics()
+                    raise
+                safe_unlock = True
+                self.state["status"] = "rolled_back"
+                self.state_io(self.state)
+            raise
+        finally:
+            if safe_unlock:
+                self.unlock()
+
+    def recover(self, manual=False):
+        self.settings()
+        lock = self.run(
+            "inspect", self.env["PROD_CONTAINER"] + "-deploy-lock", check=False
+        )
+        if manual:
+            self.preflight(require_running=False)
+            self.lock()
+        elif lock.returncode == 0:
+            info = json.loads(lock.stdout)[0]
+            if (info["Config"].get("Labels") or {}).get(
+                "frameflow.deploy.owner"
+            ) != self.owner:
+                return
+            self.lock_id = info["Id"]
+            self.old_image = info["Image"]
+        else:
+            return
+        recovered = False
+        try:
+            self.run("volume", "inspect", self.env["PROD_STATE_VOLUME"])
+            volume = json.loads(
+                self.run("volume", "inspect", self.env["PROD_STATE_VOLUME"]).stdout
+            )[0]
+            if (volume.get("Labels") or {}).get("frameflow.deploy.project") != self.env[
+                "PROD_PROJECT"
+            ]:
+                raise ValueError("Recovery state volume ownership mismatch")
+            state = json.loads(self.state_io())
+            previous = state["previous"]
+            service = previous["services"]["pigway-frame-flow"]
+            if (
+                previous["name"] != self.env["PROD_PROJECT"]
+                or service["container_name"] != self.env["PROD_CONTAINER"]
+            ):
+                raise ValueError("Rollback journal resource mismatch")
+            sources = {m["target"]: m["source"] for m in service["volumes"]}
+            if (
+                sources.get("/data") != self.env["PROD_DATA_DIR"]
+                or sources.get("/config") != self.env["PROD_CONFIG_DIR"]
+            ):
+                raise ValueError("Rollback journal persistence mismatch")
+            if manual:
+                state["owner"] = self.owner
+                state["status"] = "prepared"
+                self.state_io(state)
+            if manual or (
+                state["owner"] == self.owner
+                and state["status"] not in ("success", "rolled_back")
+            ):
+                self.rollback(state["previous"])
+                state["status"] = "rolled_back"
+                self.state_io(state)
+            recovered = True
+        except BaseException:
+            print("RECOVERY FAILED: production lock retained", flush=True)
+            self.diagnostics()
+            raise
+        finally:
+            if recovered:
+                self.unlock()
+
+
+def main():
+    tool = Deploy()
+    action = tool.env.get("CD_ACTION") or (
+        "deploy" if tool.env.get("PROD_DEPLOY_ENABLED") == "true" else "plan"
+    )
+    if action == "recover":
+        if tool.env.get("FRAME_FLOW_DEPLOY_OWNER"):
+            tool.recover()
+        return
+    if action != "plan" and tool.env.get("PROD_DEPLOY_ENABLED") != "true":
+        raise SystemExit(
+            "Production CD disabled; set PROD_DEPLOY_ENABLED=true after safe trial"
+        )
+    if action == "plan":
+        if any(not tool.env.get(key) for key in REQUIRED):
+            print("PLAN ONLY: configure required Variables:", ", ".join(REQUIRED))
+            return
+        baseline = tool.preflight()
+        tool.secret_config()
+        tool.verify(baseline)
+        print(
+            "PLAN ONLY: production resource identity/mounts/port checked; no resources modified"
+        )
+        return
+    if tool.env.get("GITHUB_ENV"):
+        with Path(tool.env["GITHUB_ENV"]).open("a") as file:
+            file.write("FRAME_FLOW_DEPLOY_OWNER=" + tool.owner + "\n")
+
+    def cancel(*_):
+        raise SystemExit(143)
+
+    signal.signal(signal.SIGTERM, cancel)
+    if action == "deploy":
+        tool.deploy()
+    elif action == "rollback":
+        tool.recover(manual=True)
+    else:
+        raise SystemExit("Unknown CD_ACTION")
+
+
+if __name__ == "__main__":
+    main()

tests/test_production_deploy.py

@@ -0,0 +1,245 @@
+"""Deployment gates, isolation checks, and rollback use mocked Docker only."""
+
+import importlib.util
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+
+@pytest.fixture
+def tool():
+    spec = importlib.util.spec_from_file_location(
+        "production_deploy", Path(__file__).parents[1] / "scripts/production-deploy.py"
+    )
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    instance = module.Deploy()
+    instance.env = {
+        "PROD_PROJECT": "production-frame",
+        "PROD_CONTAINER": "production-frame",
+        "PROD_IMAGE_REPO": "pigway-frame-flow",
+        "PROD_DATA_DIR": "/confirmed/data",
+        "PROD_CONFIG_DIR": "/confirmed/config",
+        "PROD_WEB_BIND": "192.0.2.1",
+        "PROD_WEB_PORT": "18787",
+        "PROD_STATE_VOLUME": "production-frame-cd",
+        "PROD_MOCK": "false",
+        "PROD_WRITE_ENABLED": "false",
+    }
+    return instance
+
+
+def existing(tool):
+    return {
+        "Image": "sha256:previous",
+        "State": {"Running": True},
+        "Config": {
+            "User": "10333:10333",
+            "Env": ["BUILD_COMMIT=previous", "FRAME_FLOW_TOKEN=fixture-token"],
+            "Labels": {
+                "com.docker.compose.project": tool.env["PROD_PROJECT"],
+                "com.docker.compose.service": "pigway-frame-flow",
+            },
+        },
+        "Mounts": [
+            {"Type": "bind", "Destination": target, "Source": tool.env[key], "RW": True}
+            for target, key in [
+                ("/data", "PROD_DATA_DIR"),
+                ("/config", "PROD_CONFIG_DIR"),
+            ]
+        ],
+        "HostConfig": {
+            "PortBindings": {
+                "18787/tcp": [
+                    {"HostIp": tool.env["PROD_WEB_BIND"], "HostPort": "18787"}
+                ]
+            },
+            "NetworkMode": "production-frame_default",
+            "RestartPolicy": {"Name": "unless-stopped"},
+        },
+    }
+
+
+def test_preflight_preserves_paths_and_has_no_write_commands(tool, monkeypatch):
+    monkeypatch.setattr(tool, "inspect", lambda _: existing(tool))
+    calls = []
+    monkeypatch.setattr(
+        tool,
+        "run",
+        lambda *args, **kwargs: calls.append(args) or SimpleNamespace(returncode=1),
+    )
+    config = tool.preflight()
+    service = config["services"]["pigway-frame-flow"]
+    assert service["image"] == "sha256:previous"
+    assert all(m["bind"]["create_host_path"] is False for m in service["volumes"])
+    assert service["environment"]["FRAME_FLOW_TOKEN"] == "fixture-token"
+    assert calls == [("volume", "inspect", "production-frame-cd")]
+
+
+@pytest.mark.parametrize("change", ["path", "project", "port", "network"])
+def test_preflight_rejects_wrong_host_resources(tool, monkeypatch, change):
+    old = existing(tool)
+    if change == "path":
+        old["Mounts"][0]["Source"] = "/unrelated/data"
+    elif change == "project":
+        old["Config"]["Labels"]["com.docker.compose.project"] = "ffci-fixture"
+    elif change == "port":
+        old["HostConfig"]["PortBindings"]["18787/tcp"][0]["HostPort"] = "9999"
+    else:
+        old["HostConfig"]["NetworkMode"] = "host"
+    monkeypatch.setattr(tool, "inspect", lambda _: old)
+    with pytest.raises(ValueError):
+        tool.preflight()
+
+
+@pytest.mark.parametrize("failure", [None, "candidate", "rollback"])
+def test_deploy_rolls_back_exact_baseline_without_deleting_data(
+    tool, monkeypatch, failure
+):
+    old = {"services": {"pigway-frame-flow": {"image": "sha256:previous"}}}
+    new = {"services": {"pigway-frame-flow": {"image": "candidate"}}}
+    activated, calls, saved = [], [], []
+    monkeypatch.setattr(tool, "preflight", lambda: old)
+    monkeypatch.setattr(tool, "lock", lambda: calls.append("lock"))
+    monkeypatch.setattr(tool, "unlock", lambda: calls.append("unlock"))
+    monkeypatch.setattr(tool, "run", lambda *args, **kw: calls.append(args))
+    monkeypatch.setattr(tool, "build", lambda _: new)
+    monkeypatch.setattr(tool, "diagnostics", lambda: None)
+    monkeypatch.setattr(tool, "activate", lambda c: activated.append(c))
+    monkeypatch.setattr(
+        tool, "state_io", lambda state: saved.append(json.loads(json.dumps(state)))
+    )
+    verifies = []
+
+    def verify(config):
+        verifies.append(config)
+        if config == new and failure:
+            raise RuntimeError("candidate not ready")
+        if config == old and failure == "rollback" and len(verifies) > 1:
+            raise RuntimeError("rollback not ready")
+        return {"commit": "previous", "version": "0.1.0"}
+
+    monkeypatch.setattr(tool, "verify", verify)
+    if failure:
+        with pytest.raises(RuntimeError):
+            tool.deploy()
+        assert activated == [new, old]
+        assert saved[0]["previous"] == old
+        assert ("unlock" in calls) is (failure != "rollback")
+    else:
+        tool.deploy()
+        assert activated == [new]
+        assert saved[-1]["status"] == "success"
+    assert not any(
+        isinstance(c, tuple) and ("down" in c or "prune" in c) for c in calls
+    )
+
+
+def test_activate_escapes_dollar_secrets_and_never_down(tool, monkeypatch):
+    calls = []
+
+    def run(*args, **kwargs):
+        calls.append(args)
+        if "config" in args:
+            path = Path(args[args.index("-f") + 1])
+            assert "$$literal" in path.read_text()
+
+    monkeypatch.setattr(tool, "run", run)
+    tool.activate(
+        {
+            "services": {
+                "pigway-frame-flow": {"environment": {"FRAME_FLOW_TOKEN": "$literal"}}
+            }
+        }
+    )
+    assert len(calls) == 2 and "up" in calls[1] and "--no-build" in calls[1]
+
+
+def test_workflow_deploy_is_verify_gated_and_manual_plan_default(tool):
+    import yaml
+
+    workflow = yaml.safe_load(
+        (Path(__file__).parents[1] / ".github/workflows/verify.yml").read_text()
+    )
+    job = workflow["jobs"]["deploy"]
+    assert job["needs"] == "verify" and "success()" in job["if"]
+    assert "refs/heads/main" in job["if"] and "pull_request" not in job["if"]
+    assert job["env"]["PROD_DEPLOY_ENABLED"] == "${{ vars.PROD_DEPLOY_ENABLED }}"
+    assert (
+        workflow[True]["workflow_dispatch"]["inputs"]["deploy_action"]["default"]
+        == "plan"
+    )
+    assert job["steps"][-1]["if"] == "always()"
+
+
+def test_recovery_handles_missing_or_crashed_production_container(tool, monkeypatch):
+    previous = {
+        "name": tool.env["PROD_PROJECT"],
+        "services": {
+            "pigway-frame-flow": {
+                "container_name": tool.env["PROD_CONTAINER"],
+                "image": "sha256:old",
+                "volumes": [
+                    {"source": tool.env["PROD_DATA_DIR"], "target": "/data"},
+                    {"source": tool.env["PROD_CONFIG_DIR"], "target": "/config"},
+                ],
+            }
+        },
+    }
+    state = {"owner": tool.owner, "status": "prepared", "previous": previous}
+    calls = []
+
+    def run(*args, **kwargs):
+        calls.append(args)
+        if args[0] == "inspect":
+            assert args[1].endswith("-deploy-lock")
+            return SimpleNamespace(
+                returncode=0,
+                stdout=json.dumps(
+                    [
+                        {
+                            "Id": "owned-lock",
+                            "Image": "sha256:old",
+                            "Config": {
+                                "Labels": {"frameflow.deploy.owner": tool.owner}
+                            },
+                        }
+                    ]
+                ).encode(),
+            )
+        return SimpleNamespace(
+            returncode=0,
+            stdout=json.dumps(
+                [{"Labels": {"frameflow.deploy.project": tool.env["PROD_PROJECT"]}}]
+            ).encode(),
+        )
+
+    monkeypatch.setattr(tool, "run", run)
+
+    def state_io(payload=None):
+        if payload is not None:
+            state.update(payload)
+        return json.dumps(state).encode()
+
+    monkeypatch.setattr(tool, "state_io", state_io)
+    monkeypatch.setattr(
+        tool, "rollback", lambda config: calls.append(("rollback", config))
+    )
+    tool.recover()
+    assert ("rollback", previous) in calls
+    assert ("rm", "owned-lock") in calls
+    assert state["status"] == "rolled_back"
+
+
+def test_lock_conflict_never_activates_production(tool, monkeypatch):
+    monkeypatch.setattr(tool, "preflight", lambda: {})
+    monkeypatch.setattr(tool, "activate", lambda _: pytest.fail("Production touched"))
+
+    def fail(*args, **kwargs):
+        raise RuntimeError("Lock already reserved")
+
+    monkeypatch.setattr(tool, "run", fail)
+    with pytest.raises(RuntimeError):
+        tool.deploy()