Add opt-in production CD with daemon lock, verified rollback and safe planning
This commit is contained in:
Pigway
@@ -0,0 +1,588 @@
|
||||
"""Opt-in production deployment; host paths verified against the running container."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REQUIRED = (
|
||||
"PROD_PROJECT",
|
||||
"PROD_CONTAINER",
|
||||
"PROD_IMAGE_REPO",
|
||||
"PROD_DATA_DIR",
|
||||
"PROD_CONFIG_DIR",
|
||||
"PROD_WEB_BIND",
|
||||
"PROD_WEB_PORT",
|
||||
"PROD_STATE_VOLUME",
|
||||
"PROD_MOCK",
|
||||
"PROD_WRITE_ENABLED",
|
||||
)
|
||||
|
||||
|
||||
class Deploy:
|
||||
def __init__(self):
|
||||
self.env = os.environ.copy()
|
||||
self.owner = self.env.get("FRAME_FLOW_DEPLOY_OWNER") or uuid4().hex
|
||||
if not re.fullmatch(r"[0-9a-f]{32}", self.owner):
|
||||
raise ValueError("Invalid deployment owner")
|
||||
self.lock_id = None
|
||||
self.state = None
|
||||
self.old_image = None
|
||||
self.secret_values = []
|
||||
self.docker = self.env.get("DOCKER_BIN", "docker")
|
||||
|
||||
def run(self, *args, data=None, check=True):
|
||||
result = subprocess.run(
|
||||
[self.docker, *args],
|
||||
input=data,
|
||||
capture_output=True,
|
||||
env=self.env | {"DOCKER_BUILDKIT": "1"},
|
||||
)
|
||||
if check and result.returncode:
|
||||
# Never print Docker inspect/config/env output or secret-bearing commands.
|
||||
raise RuntimeError(
|
||||
f"Docker {args[0]} failed (exit {result.returncode}); inspect scoped container logs privately"
|
||||
)
|
||||
return result
|
||||
|
||||
def inspect(self, name):
|
||||
return json.loads(self.run("inspect", name).stdout)[0]
|
||||
|
||||
def settings(self):
|
||||
missing = [key for key in REQUIRED if not self.env.get(key)]
|
||||
if missing:
|
||||
raise ValueError("Missing Variables: " + ", ".join(missing))
|
||||
for key in (
|
||||
"PROD_PROJECT",
|
||||
"PROD_CONTAINER",
|
||||
"PROD_IMAGE_REPO",
|
||||
"PROD_STATE_VOLUME",
|
||||
):
|
||||
if not re.fullmatch(r"[a-z][a-z0-9-]{2,80}", self.env[key]) or self.env[
|
||||
key
|
||||
].startswith("ffci-"):
|
||||
raise ValueError("Unsafe production resource name: " + key)
|
||||
for key in ("PROD_DATA_DIR", "PROD_CONFIG_DIR"):
|
||||
if not self.env[key].startswith("/") or self.env[key] == "/":
|
||||
raise ValueError("Absolute existing NAS path required: " + key)
|
||||
if self.env["PROD_DATA_DIR"] == self.env["PROD_CONFIG_DIR"]:
|
||||
raise ValueError("data/config must use distinct directories")
|
||||
if any(
|
||||
self.env[key] not in ("true", "false")
|
||||
for key in ("PROD_MOCK", "PROD_WRITE_ENABLED")
|
||||
):
|
||||
raise ValueError("Explicit true/false required for mock and writes")
|
||||
if not 1 <= int(self.env["PROD_WEB_PORT"]) <= 65535:
|
||||
raise ValueError("Invalid production port")
|
||||
|
||||
def preflight(self, require_running=True):
|
||||
self.settings()
|
||||
old = self.inspect(self.env["PROD_CONTAINER"])
|
||||
labels = old["Config"].get("Labels") or {}
|
||||
if (
|
||||
labels.get("com.docker.compose.project") != self.env["PROD_PROJECT"]
|
||||
or labels.get("com.docker.compose.service") != "pigway-frame-flow"
|
||||
):
|
||||
raise ValueError(
|
||||
"Existing container does not belong to the configured production project/service"
|
||||
)
|
||||
if require_running and not old["State"]["Running"]:
|
||||
raise ValueError("Existing production container must be running")
|
||||
mounts = {m["Destination"]: m for m in old["Mounts"]}
|
||||
for target, key in (("/data", "PROD_DATA_DIR"), ("/config", "PROD_CONFIG_DIR")):
|
||||
m = mounts.get(target, {})
|
||||
if (
|
||||
m.get("Type") != "bind"
|
||||
or m.get("Source") != self.env[key]
|
||||
or not m.get("RW")
|
||||
):
|
||||
raise ValueError("Production mount mismatch: " + key)
|
||||
photos = self.env.get("PROD_PHOTOS_DIR")
|
||||
if "/photos" in mounts and (
|
||||
not photos
|
||||
or mounts["/photos"]["Source"] != photos
|
||||
or mounts["/photos"]["RW"]
|
||||
):
|
||||
raise ValueError(
|
||||
"Confirm PROD_PHOTOS_DIR for the existing read-only photo mount"
|
||||
)
|
||||
if photos and "/photos" not in mounts:
|
||||
raise ValueError(
|
||||
"Configured photos directory does not match current container"
|
||||
)
|
||||
if set(mounts) != {"/data", "/config"} | ({"/photos"} if photos else set()):
|
||||
raise ValueError("Unrecognized persistent mount; review before adopting CD")
|
||||
bindings = old["HostConfig"].get("PortBindings", {}).get("18787/tcp")
|
||||
if bindings != [
|
||||
{"HostIp": self.env["PROD_WEB_BIND"], "HostPort": self.env["PROD_WEB_PORT"]}
|
||||
]:
|
||||
raise ValueError(
|
||||
"Port/bind mismatch; CD must not change production networking"
|
||||
)
|
||||
if (
|
||||
old["HostConfig"].get("NetworkMode")
|
||||
!= self.env["PROD_PROJECT"] + "_default"
|
||||
):
|
||||
raise ValueError("Custom production network requires explicit review")
|
||||
config = yaml.safe_load((ROOT / "docker-compose.yml").read_text())
|
||||
config["name"] = self.env["PROD_PROJECT"]
|
||||
service = config["services"]["pigway-frame-flow"]
|
||||
service.pop("build")
|
||||
service.pop("env_file")
|
||||
service["container_name"] = self.env["PROD_CONTAINER"]
|
||||
service["image"] = old["Image"]
|
||||
service["environment"] = dict(
|
||||
value.split("=", 1) for value in old["Config"]["Env"] if "=" in value
|
||||
)
|
||||
self.secret_values.extend(
|
||||
value
|
||||
for key, value in service["environment"].items()
|
||||
if value
|
||||
and any(
|
||||
word in key
|
||||
for word in ("KEY", "TOKEN", "PASSWORD", "SECRET", "SESSION")
|
||||
)
|
||||
)
|
||||
service["ports"] = [
|
||||
f"{self.env['PROD_WEB_BIND']}:{self.env['PROD_WEB_PORT']}:18787"
|
||||
]
|
||||
service["volumes"] = [
|
||||
{
|
||||
"type": "bind",
|
||||
"source": self.env[key],
|
||||
"target": target,
|
||||
"bind": {"create_host_path": False},
|
||||
}
|
||||
for key, target in (
|
||||
("PROD_DATA_DIR", "/data"),
|
||||
("PROD_CONFIG_DIR", "/config"),
|
||||
)
|
||||
]
|
||||
service["group_add"] = old["HostConfig"].get("GroupAdd") or []
|
||||
if photos:
|
||||
service["volumes"].append(
|
||||
{
|
||||
"type": "bind",
|
||||
"source": photos,
|
||||
"target": "/photos",
|
||||
"read_only": True,
|
||||
"bind": {"create_host_path": False},
|
||||
}
|
||||
)
|
||||
# Adoption must preserve runtime constraints, not silently change them.
|
||||
if (
|
||||
old["Config"]["User"] != service["user"]
|
||||
or old["HostConfig"]["RestartPolicy"]["Name"] != service["restart"]
|
||||
):
|
||||
raise ValueError("Production user/restart policy differs from repository")
|
||||
self.old_image = old["Image"]
|
||||
volume = self.run(
|
||||
"volume", "inspect", self.env["PROD_STATE_VOLUME"], check=False
|
||||
)
|
||||
if (
|
||||
volume.returncode == 0
|
||||
and (json.loads(volume.stdout)[0].get("Labels") or {}).get(
|
||||
"frameflow.deploy.project"
|
||||
)
|
||||
!= self.env["PROD_PROJECT"]
|
||||
):
|
||||
raise ValueError("State volume is not owned by this deployment project")
|
||||
return config
|
||||
|
||||
def lock(self):
|
||||
result = self.run(
|
||||
"create",
|
||||
"--name",
|
||||
self.env["PROD_CONTAINER"] + "-deploy-lock",
|
||||
"--label",
|
||||
"frameflow.deploy.owner=" + self.owner,
|
||||
"--network",
|
||||
"none",
|
||||
"--entrypoint",
|
||||
"/bin/true",
|
||||
self.old_image,
|
||||
)
|
||||
self.lock_id = result.stdout.decode().strip()
|
||||
|
||||
def unlock(self):
|
||||
if self.lock_id:
|
||||
self.run("rm", self.lock_id)
|
||||
self.lock_id = None
|
||||
|
||||
def state_io(self, payload=None):
|
||||
script = (
|
||||
"import json,os,sys;from pathlib import Path;p=Path('/state/deploy.json');"
|
||||
"os.umask(0o077);os.chmod('/state',0o700);v=json.load(sys.stdin);q=p.with_suffix('.tmp');q.write_text(json.dumps(v));q.replace(p)"
|
||||
if payload is not None
|
||||
else "from pathlib import Path;print(Path('/state/deploy.json').read_text())"
|
||||
)
|
||||
return self.run(
|
||||
"run",
|
||||
"--rm",
|
||||
"-i",
|
||||
"--user",
|
||||
"0:0",
|
||||
"--network",
|
||||
"none",
|
||||
"--mount",
|
||||
f"type=volume,source={self.env['PROD_STATE_VOLUME']},target=/state",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
self.old_image,
|
||||
"-c",
|
||||
script,
|
||||
data=json.dumps(payload).encode() if payload is not None else None,
|
||||
).stdout
|
||||
|
||||
def activate(self, config):
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".yml") as file:
|
||||
yaml.safe_dump(json.loads(json.dumps(config).replace("$", "$$")), file)
|
||||
file.flush()
|
||||
self.run(
|
||||
"compose",
|
||||
"-p",
|
||||
self.env["PROD_PROJECT"],
|
||||
"-f",
|
||||
file.name,
|
||||
"config",
|
||||
"--quiet",
|
||||
)
|
||||
self.run(
|
||||
"compose",
|
||||
"-p",
|
||||
self.env["PROD_PROJECT"],
|
||||
"-f",
|
||||
file.name,
|
||||
"up",
|
||||
"-d",
|
||||
"--no-build",
|
||||
"--pull",
|
||||
"never",
|
||||
"pigway-frame-flow",
|
||||
)
|
||||
|
||||
def verify(self, config):
|
||||
container = self.env["PROD_CONTAINER"]
|
||||
code = "import urllib.request;urllib.request.urlopen('http://127.0.0.1:18787/ready',timeout=3)"
|
||||
for _ in range(60):
|
||||
if (
|
||||
self.run(
|
||||
"exec", container, "python", "-c", code, check=False
|
||||
).returncode
|
||||
== 0
|
||||
):
|
||||
break
|
||||
time.sleep(2)
|
||||
else:
|
||||
raise RuntimeError("Production /ready timeout")
|
||||
self.run(
|
||||
"exec",
|
||||
"-e",
|
||||
"FRAME_FLOW_URL=http://127.0.0.1:18787",
|
||||
"-i",
|
||||
container,
|
||||
"python",
|
||||
"-",
|
||||
data=(ROOT / "scripts/smoke-test.py").read_bytes(),
|
||||
)
|
||||
expected = config["services"]["pigway-frame-flow"]["environment"].get(
|
||||
"BUILD_COMMIT"
|
||||
)
|
||||
value = json.loads(
|
||||
self.run(
|
||||
"exec",
|
||||
container,
|
||||
"python",
|
||||
"-c",
|
||||
"import json,urllib.request;print(json.dumps(json.load(urllib.request.urlopen('http://127.0.0.1:18787/api/v1/version'))))",
|
||||
).stdout
|
||||
)
|
||||
if value["commit"] != expected:
|
||||
raise RuntimeError("Running commit mismatch")
|
||||
print(
|
||||
"Production /ready and all smoke endpoints: HTTP 200; commit",
|
||||
value["commit"],
|
||||
flush=True,
|
||||
)
|
||||
return value
|
||||
|
||||
def rollback(self, previous):
|
||||
self.activate(previous)
|
||||
self.verify(previous)
|
||||
print("ROLLBACK VERIFIED", flush=True)
|
||||
|
||||
def secret_config(self):
|
||||
secret = self.env.get("PRODUCTION_ENV", "")
|
||||
proposed = dict(
|
||||
line.split("=", 1)
|
||||
for line in secret.splitlines()
|
||||
if line.strip() and not line.startswith("#") and "=" in line
|
||||
)
|
||||
for key in ("FRAME_FLOW_TOKEN", "KOKONNA_API_KEY"):
|
||||
if key not in proposed:
|
||||
raise ValueError("Missing PRODUCTION_ENV key: " + key)
|
||||
if self.env["PROD_MOCK"] == "false" and not proposed["KOKONNA_API_KEY"]:
|
||||
raise ValueError("KoKonna API key required for non-mock production")
|
||||
if not proposed["FRAME_FLOW_TOKEN"]:
|
||||
raise ValueError("FRAME_FLOW_TOKEN required for smoke checks")
|
||||
self.secret_values.extend(v for v in proposed.values() if v)
|
||||
return proposed
|
||||
|
||||
def build(self, config):
|
||||
commit = (
|
||||
subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT)
|
||||
.decode()
|
||||
.strip()
|
||||
)
|
||||
main = (
|
||||
subprocess.check_output(
|
||||
["git", "ls-remote", "origin", "refs/heads/main"], cwd=ROOT
|
||||
)
|
||||
.decode()
|
||||
.split()[0]
|
||||
)
|
||||
if commit != main:
|
||||
raise ValueError("Stale main revision; refusing automatic downgrade")
|
||||
image = (
|
||||
self.env["PROD_IMAGE_REPO"] + ":release-" + commit + "-" + self.owner[:8]
|
||||
)
|
||||
command = ["build", "--pull", "-t", image]
|
||||
now = datetime.now(UTC).isoformat()
|
||||
for key, value in {
|
||||
"DEBIAN_MIRROR": self.env.get("DEBIAN_MIRROR") or "deb.debian.org",
|
||||
"DEBIAN_SECURITY_MIRROR": self.env.get("DEBIAN_SECURITY_MIRROR")
|
||||
or "deb.debian.org",
|
||||
"BUILD_COMMIT": commit,
|
||||
"BUILD_TIME": now,
|
||||
}.items():
|
||||
command += ["--build-arg", key + "=" + value]
|
||||
self.run(*command, str(ROOT))
|
||||
service = config["services"]["pigway-frame-flow"]
|
||||
service["image"] = image
|
||||
proposed = self.secret_config()
|
||||
service["environment"].update(proposed)
|
||||
service["environment"].update(
|
||||
KOKONNA_MOCK=self.env["PROD_MOCK"],
|
||||
KOKONNA_WRITE_ENABLED=self.env["PROD_WRITE_ENABLED"],
|
||||
BUILD_COMMIT=commit,
|
||||
BUILD_TIME=now,
|
||||
DATA_DIR="/data",
|
||||
CONFIG_DIR="/config",
|
||||
TZ="Asia/Shanghai",
|
||||
)
|
||||
return config
|
||||
|
||||
def diagnostics(self):
|
||||
try:
|
||||
self._diagnostics()
|
||||
except Exception:
|
||||
print(
|
||||
"Scoped diagnostics unavailable; rollback still attempted", flush=True
|
||||
)
|
||||
|
||||
def _diagnostics(self):
|
||||
result = self.run("inspect", self.env["PROD_CONTAINER"], check=False)
|
||||
if result.returncode == 0:
|
||||
info = json.loads(result.stdout)[0]
|
||||
state = info["State"]
|
||||
print(
|
||||
"Scoped production diagnostic:",
|
||||
json.dumps(
|
||||
{
|
||||
"image": info["Image"],
|
||||
"status": state.get("Status"),
|
||||
"exit_code": state.get("ExitCode"),
|
||||
"health": state.get("Health", {}).get("Status"),
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
logs = self.run("logs", "--tail", "60", self.env["PROD_CONTAINER"], check=False)
|
||||
text = (logs.stdout + logs.stderr).decode(errors="replace")
|
||||
for value in sorted(self.secret_values, key=len, reverse=True):
|
||||
text = text.replace(value, "[REDACTED]")
|
||||
text = re.sub(r"https?://[^\s\"'<>]+", "[REDACTED-URL]", text)
|
||||
print("Scoped container logs (redacted):", text, flush=True)
|
||||
|
||||
def deploy(self):
|
||||
previous = self.preflight()
|
||||
self.lock()
|
||||
touched = False
|
||||
safe_unlock = True
|
||||
try:
|
||||
# Refresh baseline under the daemon-level lock.
|
||||
previous = self.preflight()
|
||||
version = self.verify(previous)
|
||||
pin = self.env["PROD_IMAGE_REPO"] + ":rollback-" + self.owner
|
||||
self.run("image", "tag", self.old_image, pin)
|
||||
candidate = self.build(json.loads(json.dumps(previous)))
|
||||
self.state = {
|
||||
"owner": self.owner,
|
||||
"status": "prepared",
|
||||
"previous": previous,
|
||||
"candidate": candidate,
|
||||
"previous_image": self.old_image,
|
||||
"previous_version": version,
|
||||
"rollback_tag": pin,
|
||||
}
|
||||
self.run(
|
||||
"volume",
|
||||
"create",
|
||||
"--label",
|
||||
"frameflow.deploy.project=" + self.env["PROD_PROJECT"],
|
||||
self.env["PROD_STATE_VOLUME"],
|
||||
)
|
||||
self.state_io(self.state)
|
||||
# Online backup is additive; rollback never restores/deletes user data.
|
||||
backup = "/data/deploy-backups/" + self.owner + ".db"
|
||||
self.run(
|
||||
"exec",
|
||||
self.env["PROD_CONTAINER"],
|
||||
"python",
|
||||
"-c",
|
||||
"import sqlite3;from pathlib import Path;p=Path('"
|
||||
+ backup
|
||||
+ "');p.parent.mkdir(exist_ok=True);assert Path('/data/frameflow.db').is_file();sqlite3.connect('/data/frameflow.db').backup(sqlite3.connect(p))",
|
||||
)
|
||||
touched = True
|
||||
self.activate(candidate)
|
||||
self.verify(candidate)
|
||||
self.state["status"] = "success"
|
||||
self.state_io(self.state)
|
||||
print("DEPLOY VERIFIED", flush=True)
|
||||
except BaseException as exc:
|
||||
print("DEPLOY FAILED:", type(exc).__name__, flush=True)
|
||||
self.diagnostics()
|
||||
if touched:
|
||||
safe_unlock = False
|
||||
try:
|
||||
self.rollback(previous)
|
||||
except BaseException:
|
||||
print(
|
||||
"ROLLBACK FAILED: deployment lock retained; always() recovery will retry",
|
||||
flush=True,
|
||||
)
|
||||
self.diagnostics()
|
||||
raise
|
||||
safe_unlock = True
|
||||
self.state["status"] = "rolled_back"
|
||||
self.state_io(self.state)
|
||||
raise
|
||||
finally:
|
||||
if safe_unlock:
|
||||
self.unlock()
|
||||
|
||||
def recover(self, manual=False):
|
||||
self.settings()
|
||||
lock = self.run(
|
||||
"inspect", self.env["PROD_CONTAINER"] + "-deploy-lock", check=False
|
||||
)
|
||||
if manual:
|
||||
self.preflight(require_running=False)
|
||||
self.lock()
|
||||
elif lock.returncode == 0:
|
||||
info = json.loads(lock.stdout)[0]
|
||||
if (info["Config"].get("Labels") or {}).get(
|
||||
"frameflow.deploy.owner"
|
||||
) != self.owner:
|
||||
return
|
||||
self.lock_id = info["Id"]
|
||||
self.old_image = info["Image"]
|
||||
else:
|
||||
return
|
||||
recovered = False
|
||||
try:
|
||||
self.run("volume", "inspect", self.env["PROD_STATE_VOLUME"])
|
||||
volume = json.loads(
|
||||
self.run("volume", "inspect", self.env["PROD_STATE_VOLUME"]).stdout
|
||||
)[0]
|
||||
if (volume.get("Labels") or {}).get("frameflow.deploy.project") != self.env[
|
||||
"PROD_PROJECT"
|
||||
]:
|
||||
raise ValueError("Recovery state volume ownership mismatch")
|
||||
state = json.loads(self.state_io())
|
||||
previous = state["previous"]
|
||||
service = previous["services"]["pigway-frame-flow"]
|
||||
if (
|
||||
previous["name"] != self.env["PROD_PROJECT"]
|
||||
or service["container_name"] != self.env["PROD_CONTAINER"]
|
||||
):
|
||||
raise ValueError("Rollback journal resource mismatch")
|
||||
sources = {m["target"]: m["source"] for m in service["volumes"]}
|
||||
if (
|
||||
sources.get("/data") != self.env["PROD_DATA_DIR"]
|
||||
or sources.get("/config") != self.env["PROD_CONFIG_DIR"]
|
||||
):
|
||||
raise ValueError("Rollback journal persistence mismatch")
|
||||
if manual:
|
||||
state["owner"] = self.owner
|
||||
state["status"] = "prepared"
|
||||
self.state_io(state)
|
||||
if manual or (
|
||||
state["owner"] == self.owner
|
||||
and state["status"] not in ("success", "rolled_back")
|
||||
):
|
||||
self.rollback(state["previous"])
|
||||
state["status"] = "rolled_back"
|
||||
self.state_io(state)
|
||||
recovered = True
|
||||
except BaseException:
|
||||
print("RECOVERY FAILED: production lock retained", flush=True)
|
||||
self.diagnostics()
|
||||
raise
|
||||
finally:
|
||||
if recovered:
|
||||
self.unlock()
|
||||
|
||||
|
||||
def main():
|
||||
tool = Deploy()
|
||||
action = tool.env.get("CD_ACTION") or (
|
||||
"deploy" if tool.env.get("PROD_DEPLOY_ENABLED") == "true" else "plan"
|
||||
)
|
||||
if action == "recover":
|
||||
if tool.env.get("FRAME_FLOW_DEPLOY_OWNER"):
|
||||
tool.recover()
|
||||
return
|
||||
if action != "plan" and tool.env.get("PROD_DEPLOY_ENABLED") != "true":
|
||||
raise SystemExit(
|
||||
"Production CD disabled; set PROD_DEPLOY_ENABLED=true after safe trial"
|
||||
)
|
||||
if action == "plan":
|
||||
if any(not tool.env.get(key) for key in REQUIRED):
|
||||
print("PLAN ONLY: configure required Variables:", ", ".join(REQUIRED))
|
||||
return
|
||||
baseline = tool.preflight()
|
||||
tool.secret_config()
|
||||
tool.verify(baseline)
|
||||
print(
|
||||
"PLAN ONLY: production resource identity/mounts/port checked; no resources modified"
|
||||
)
|
||||
return
|
||||
if tool.env.get("GITHUB_ENV"):
|
||||
with Path(tool.env["GITHUB_ENV"]).open("a") as file:
|
||||
file.write("FRAME_FLOW_DEPLOY_OWNER=" + tool.owner + "\n")
|
||||
|
||||
def cancel(*_):
|
||||
raise SystemExit(143)
|
||||
|
||||
signal.signal(signal.SIGTERM, cancel)
|
||||
if action == "deploy":
|
||||
tool.deploy()
|
||||
elif action == "rollback":
|
||||
tool.recover(manual=True)
|
||||
else:
|
||||
raise SystemExit("Unknown CD_ACTION")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user