298 lines
11 KiB
Python
298 lines
11 KiB
Python
"""Release a verified source bundle on the Ubuntu T7 host; never write to frames."""
|
|
|
|
import datetime
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import time
|
|
import tomllib
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(os.getenv("FRAME_FLOW_ROOT", "/srv/frameflow"))
|
|
APP = ROOT / "apps/pigway-frame-flow"
|
|
RECOVERY = ROOT / "recovery/frame-flow"
|
|
CONTAINER = "pigway-frame-flow"
|
|
ORIGIN = os.getenv("FRAME_FLOW_ORIGIN", "http://127.0.0.1:6005").rstrip("/")
|
|
|
|
|
|
def run(*args):
|
|
return subprocess.check_output(args, text=True, timeout=600).strip()
|
|
|
|
|
|
def inspect():
|
|
return json.loads(run("docker", "inspect", CONTAINER))[0]
|
|
|
|
|
|
def archive_staging(backup, ctx):
|
|
# These are retired Frame-only inputs, never shared console migration files.
|
|
names = [
|
|
"frame-schedule-ui",
|
|
"frame-photos-ui",
|
|
"frame-final-status.sh",
|
|
"frame-compose.candidate.private.json",
|
|
"frame-fresh-compose.private.json",
|
|
"frame-image.tar",
|
|
"frame-prepare.sh",
|
|
"frame-nfs-setup.sh",
|
|
"frame.unit",
|
|
"compose.ubuntu-photos-api.yml",
|
|
]
|
|
archive = backup / "retired-frame-staging"
|
|
archive.mkdir(mode=0o700)
|
|
moved = []
|
|
for name in names:
|
|
source = ctx.parent / name
|
|
if source.exists():
|
|
assert not source.is_symlink(), name
|
|
source.rename(archive / name)
|
|
moved.append(name)
|
|
for p in archive.rglob("*"):
|
|
if not p.is_symlink():
|
|
p.chmod(0o700 if p.is_dir() else 0o600)
|
|
# Remove only disposable aliases, preserving every recovery reference and image.
|
|
references = "\n".join(
|
|
p.read_text(errors="replace")
|
|
for p in RECOVERY.rglob("*")
|
|
if p.is_file() and p.suffix in {".json", ".txt"}
|
|
)
|
|
removed = []
|
|
for tag in [
|
|
"pigway-frame-flow:history-ui-local-base",
|
|
"pigway-frame-flow:final-audit-local-base",
|
|
"pigway-frame-flow:0.1.9-5d2171b9be3e",
|
|
]:
|
|
tags = run(
|
|
"docker", "image", "ls", "--format", "{{.Repository}}:{{.Tag}}"
|
|
).splitlines()
|
|
if (
|
|
tag in tags
|
|
and tag not in references
|
|
and not run("docker", "ps", "-aq", "--filter", "ancestor=" + tag)
|
|
):
|
|
run("docker", "image", "rm", tag)
|
|
removed.append(tag)
|
|
return {
|
|
"archived_frame_inputs": moved,
|
|
"removed_temporary_aliases": removed,
|
|
"shared_inputs_and_recovery_preserved": True,
|
|
}
|
|
|
|
|
|
def wait_ready():
|
|
for attempt in range(30):
|
|
try:
|
|
with urllib.request.urlopen(ORIGIN + "/ready", timeout=4) as r:
|
|
assert json.load(r)["status"] == "ready"
|
|
break
|
|
except Exception:
|
|
if attempt == 29:
|
|
raise
|
|
time.sleep(2)
|
|
|
|
|
|
def verify_runtime(commit):
|
|
c = inspect()
|
|
env = dict(s.split("=", 1) for s in c["Config"]["Env"])
|
|
assert env["KOKONNA_MOCK"] == "false"
|
|
assert env["KOKONNA_WRITE_ENABLED"] == "true"
|
|
assert env["BUILD_COMMIT"] == commit
|
|
assert c["State"]["Running"]
|
|
assert c["Config"]["User"] == "10333:10333"
|
|
assert c["HostConfig"]["ReadonlyRootfs"]
|
|
assert set(c["HostConfig"]["CapDrop"]) == {"ALL"}
|
|
assert {m["Destination"] for m in c["Mounts"]} == {"/data", "/config"}
|
|
for mount in c["Mounts"]:
|
|
assert Path(mount["Source"]).resolve() == APP / mount["Destination"][1:]
|
|
return c
|
|
|
|
|
|
def main():
|
|
assert os.geteuid() == 0, "Run with sudo on the Ubuntu Server"
|
|
expected_uuid = os.environ.get("FRAME_FLOW_MOUNT_UUID", "")
|
|
assert expected_uuid, "Set FRAME_FLOW_MOUNT_UUID to the verified storage UUID"
|
|
assert os.environ.get("FRAME_FLOW_ORIGIN"), (
|
|
"Set FRAME_FLOW_ORIGIN to the verified listener"
|
|
)
|
|
os.umask(0o077)
|
|
run("/usr/local/libexec/pigway-guard-mount")
|
|
assert run("findmnt", "-n", "-o", "UUID", str(ROOT)) == expected_uuid
|
|
with (RECOVERY / ".release.lock").open("a") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
release(lock)
|
|
|
|
|
|
def verify_prebuilt_image(image, commit, version, hashes):
|
|
expected = os.getenv("FRAME_FLOW_PREBUILT_IMAGE_ID", "")
|
|
if not expected:
|
|
return False
|
|
assert re.fullmatch(r"sha256:[0-9a-f]{64}", expected), "Require immutable image ID"
|
|
metadata = json.loads(run("docker", "image", "inspect", image))[0]
|
|
assert metadata["Id"] == expected, "Prebuilt image ID mismatch"
|
|
labels = metadata["Config"].get("Labels") or {}
|
|
assert labels.get("org.opencontainers.image.revision") == commit
|
|
assert labels.get("org.opencontainers.image.version") == version
|
|
project = {
|
|
n: h
|
|
for n, h in hashes.items()
|
|
if n.startswith(("pigway_frame_flow/", "vendor/"))
|
|
or n in {"requirements.lock", "LICENSE", "NOTICE.md"}
|
|
}
|
|
# Validate the artifact before it can access production config or data.
|
|
code = "import hashlib,json,pathlib,sys; h=json.load(sys.stdin); assert h and all(hashlib.sha256((pathlib.Path('/app')/n).read_bytes()).hexdigest()==v for n,v in h.items())"
|
|
subprocess.run(
|
|
[
|
|
"docker",
|
|
"run",
|
|
"--rm",
|
|
"-i",
|
|
"--network=none",
|
|
"--read-only",
|
|
"--cap-drop=ALL",
|
|
"--security-opt=no-new-privileges",
|
|
"--user=10333:10333",
|
|
"--entrypoint=python",
|
|
expected,
|
|
"-c",
|
|
code,
|
|
],
|
|
input=json.dumps(project),
|
|
text=True,
|
|
check=True,
|
|
timeout=60,
|
|
)
|
|
return True
|
|
|
|
|
|
def release(lock):
|
|
ctx = Path(__file__).resolve().parent.parent
|
|
assert ctx.parent == ROOT / "migration-inbox"
|
|
commit = (ctx / "RELEASE_COMMIT").read_text().strip()
|
|
assert re.fullmatch("[0-9a-f]{40}", commit)
|
|
hashes = json.loads((ctx / "RELEASE_HASHES.json").read_text())
|
|
for name, expected in hashes.items():
|
|
p = (ctx / name).resolve()
|
|
assert p.is_relative_to(ctx)
|
|
assert hashlib.sha256(p.read_bytes()).hexdigest() == expected, name
|
|
before = inspect()
|
|
old_env = dict(s.split("=", 1) for s in before["Config"]["Env"])
|
|
verify_runtime(old_env["BUILD_COMMIT"])
|
|
stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
backup = RECOVERY / ("release-" + stamp + "-" + commit[:12])
|
|
backup.mkdir(mode=0o700)
|
|
shutil.copy2(APP / "compose.json", backup / "compose.before.json")
|
|
shutil.copytree(APP / "config", backup / "config")
|
|
with sqlite3.connect(
|
|
"file:" + str(APP / "data/frameflow.db") + "?mode=ro", uri=True
|
|
) as src:
|
|
with sqlite3.connect(backup / "frameflow.before.db") as dst:
|
|
src.backup(dst)
|
|
assert dst.execute("pragma quick_check").fetchone()[0] == "ok"
|
|
version = tomllib.loads((ctx / "pyproject.toml").read_text())["project"]["version"]
|
|
assert re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version)
|
|
image = "pigway-frame-flow:" + version + "-" + commit[:12]
|
|
if not verify_prebuilt_image(image, commit, version, hashes):
|
|
# A source-only overlay would retain vulnerable packages from the old image.
|
|
run(
|
|
"docker",
|
|
"build",
|
|
"--pull",
|
|
"-f",
|
|
str(ctx / "Dockerfile"),
|
|
"--build-arg",
|
|
"BUILD_COMMIT=" + commit,
|
|
"--build-arg",
|
|
"BUILD_TIME=" + stamp,
|
|
"--build-arg",
|
|
"ALPINE_MIRROR=" + os.getenv("ALPINE_MIRROR", "dl-cdn.alpinelinux.org"),
|
|
"-t",
|
|
image,
|
|
str(ctx),
|
|
)
|
|
compose = json.loads((APP / "compose.json").read_text())
|
|
service = compose["services"][CONTAINER]
|
|
service["image"] = image
|
|
service["environment"]["BUILD_COMMIT"] = commit
|
|
service["environment"]["BUILD_TIME"] = stamp
|
|
changed = False
|
|
try:
|
|
changed = True
|
|
(APP / "compose.json").write_text(json.dumps(compose, indent=2) + "\n")
|
|
print("Restarting Frame Flow; checking " + ORIGIN + "/ready", flush=True)
|
|
run("systemctl", "restart", "pigway-frame-flow.service")
|
|
wait_ready()
|
|
current = verify_runtime(commit)
|
|
assert current["Image"] == run(
|
|
"docker", "image", "inspect", "--format", "{{.Id}}", image
|
|
)
|
|
# Verify all deployed project files, not just a version string or an image tag.
|
|
project_hashes = {
|
|
n: h
|
|
for n, h in hashes.items()
|
|
if n.startswith(("pigway_frame_flow/", "vendor/"))
|
|
or n in {"requirements.lock", "LICENSE", "NOTICE.md"}
|
|
}
|
|
code = 'import hashlib,json,pathlib,sys; h=json.load(sys.stdin); assert all(hashlib.sha256((pathlib.Path("/app")/n).read_bytes()).hexdigest()==v for n,v in h.items())'
|
|
subprocess.run(
|
|
["docker", "exec", "-i", CONTAINER, "python", "-c", code],
|
|
input=json.dumps(project_hashes),
|
|
text=True,
|
|
check=True,
|
|
timeout=60,
|
|
)
|
|
for name in [
|
|
"app.js",
|
|
"features.js",
|
|
"extensions.js",
|
|
"i18n.js",
|
|
"index.html",
|
|
"style.css",
|
|
]:
|
|
url = ORIGIN + "/" + ("" if name == "index.html" else "ui/" + name)
|
|
with urllib.request.urlopen(url, timeout=10) as response:
|
|
assert (
|
|
response.read()
|
|
== (ctx / "pigway_frame_flow/web" / name).read_bytes()
|
|
)
|
|
run("systemctl", "is-active", "pigway-frame-flow.service")
|
|
except BaseException:
|
|
print("Release verification failed; restoring previous Compose", flush=True)
|
|
if changed:
|
|
shutil.copy2(backup / "compose.before.json", APP / "compose.json")
|
|
run("systemctl", "restart", "pigway-frame-flow.service")
|
|
raise
|
|
shutil.copy2(APP / "compose.json", RECOVERY / "compose.json")
|
|
(RECOVERY / "image.txt").write_text(image + "\n")
|
|
shutil.copy2(ctx / "source.tar.gz", backup / "source.tar.gz")
|
|
cleanup = archive_staging(backup, ctx)
|
|
receipt = {
|
|
"result": "PASS",
|
|
"commit": commit,
|
|
"image": image,
|
|
"image_id": current["Image"],
|
|
"backup": str(backup),
|
|
"mock": False,
|
|
"write_enabled": True,
|
|
"checked_at": stamp,
|
|
"cleanup": cleanup,
|
|
}
|
|
output = APP / "release-verification.json"
|
|
output.write_text(json.dumps(receipt, indent=2) + "\n")
|
|
owner = ctx.stat()
|
|
os.chown(output, owner.st_uid, owner.st_gid)
|
|
shutil.copy2(output, backup / "release-verification.json")
|
|
public_receipt = ctx.parent / "frame-release-verification.json"
|
|
shutil.copy2(output, public_receipt)
|
|
os.chown(public_receipt, owner.st_uid, owner.st_gid)
|
|
shutil.rmtree(ctx)
|
|
print(json.dumps(receipt, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|