Bound advisory network retries while keeping audits fail closed
This commit is contained in:
Pigway
@@ -28,7 +28,8 @@ jobs:
|
||||
- run: python -m ruff format --check pigway_frame_flow tests scripts
|
||||
- run: python -m mypy pigway_frame_flow
|
||||
- run: python -m pytest -q
|
||||
- run: python -m pip_audit -r requirements.lock --no-deps --disable-pip
|
||||
- name: Complete vulnerability audit with bounded network retries
|
||||
run: python scripts/ci-audit.py
|
||||
- name: Isolated Docker build, smoke and restart verification
|
||||
run: |
|
||||
. ./deploy/build-cn.env
|
||||
|
||||
@@ -321,3 +321,22 @@ Photos configuration or stored connection is missing. These checks do not prove
|
||||
session validity or scan the complete catalog: after restoring a source, also
|
||||
verify its read-only `/api/v1/photo-sources/{id}/images?limit=1` endpoint before
|
||||
running an automation task.
|
||||
|
||||
### Advisory-service network resilience
|
||||
|
||||
CI dependency installation uses the domestic `CI_PIP_INDEX_URL` first and the
|
||||
`CI_PIP_FALLBACK_INDEX_URL` sequentially on failure. These package indexes do not
|
||||
replace PyPI's vulnerability JSON API.
|
||||
|
||||
`python scripts/ci-audit.py` retains the pinned `requirements.lock`, PyPI advisory
|
||||
service and `--no-deps --disable-pip` checks. It uses a 60-second socket timeout,
|
||||
at most three attempts, and 5/10-second backoff only for network failures. Known
|
||||
vulnerabilities, malformed/incomplete reports and skipped dependencies fail
|
||||
immediately. Every attempt has a five-minute process limit. A timeout or failed
|
||||
final attempt still blocks deployment.
|
||||
|
||||
The advisory HTTP cache is reused within the job (including retries) and follows
|
||||
the audit client's normal HTTP freshness rules. It is not a permanent offline
|
||||
snapshot or a cross-job cache. Raw network exceptions are omitted because proxy
|
||||
URLs may contain credentials. Logs identify the advisory host and whether the
|
||||
complete audit succeeded.
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Retry transient advisory-service failures without accepting incomplete audits."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
NETWORK_ERRORS = (
|
||||
"ReadTimeout",
|
||||
"Read timed out",
|
||||
"ConnectTimeout",
|
||||
"ConnectionError",
|
||||
"ConnectionResetError",
|
||||
"NameResolutionError",
|
||||
"Temporary failure in name resolution",
|
||||
"MaxRetryError",
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
cache = Path(tempfile.gettempdir()) / "frame-flow-advisory-cache"
|
||||
cache.mkdir(mode=0o700, exist_ok=True)
|
||||
timeout = int(os.getenv("CI_AUDIT_TIMEOUT", "60"))
|
||||
attempts = int(os.getenv("CI_AUDIT_ATTEMPTS", "3"))
|
||||
if not 15 <= timeout <= 120 or not 1 <= attempts <= 3:
|
||||
raise SystemExit("Invalid audit timeout or attempt limit")
|
||||
print(
|
||||
"Advisory host: pypi.org; HTTP cache enabled; socket timeout:",
|
||||
timeout,
|
||||
flush=True,
|
||||
)
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"pip_audit",
|
||||
"-r",
|
||||
"requirements.lock",
|
||||
"--no-deps",
|
||||
"--disable-pip",
|
||||
"--vulnerability-service",
|
||||
"pypi",
|
||||
"--timeout",
|
||||
str(timeout),
|
||||
"--cache-dir",
|
||||
str(cache),
|
||||
"--format",
|
||||
"json",
|
||||
]
|
||||
for attempt in range(1, attempts + 1):
|
||||
result = subprocess.run(command, capture_output=True, text=True, timeout=300)
|
||||
try:
|
||||
report = json.loads(result.stdout)
|
||||
dependencies = report["dependencies"]
|
||||
complete = bool(dependencies) and all(
|
||||
isinstance(d.get("vulns"), list) and not d.get("skip_reason")
|
||||
for d in dependencies
|
||||
)
|
||||
except (ValueError, KeyError, TypeError, AttributeError):
|
||||
dependencies, complete = [], False
|
||||
vulnerabilities = [
|
||||
(d["name"], v["id"]) for d in dependencies for v in d.get("vulns", [])
|
||||
]
|
||||
if vulnerabilities:
|
||||
for name, advisory in vulnerabilities:
|
||||
print("Vulnerable dependency:", name, advisory, flush=True)
|
||||
raise SystemExit("Audit found vulnerabilities; deployment blocked")
|
||||
if result.returncode == 0 and complete:
|
||||
print(
|
||||
"Complete audit passed:",
|
||||
len(dependencies),
|
||||
"dependencies; no known vulnerabilities",
|
||||
flush=True,
|
||||
)
|
||||
return
|
||||
transient = any(marker in result.stderr for marker in NETWORK_ERRORS)
|
||||
# Do not print raw network exceptions: proxy URLs can contain credentials.
|
||||
print(
|
||||
"Audit attempt",
|
||||
attempt,
|
||||
"failed:",
|
||||
"network error" if transient else "incomplete/error result",
|
||||
flush=True,
|
||||
)
|
||||
if not transient or attempt == attempts:
|
||||
raise SystemExit("Audit unsuccessful; deployment blocked")
|
||||
time.sleep(5 * attempt)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Network retries must never turn findings or incomplete audits into success."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def audit(monkeypatch, tmp_path):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"ci_audit", Path(__file__).parents[1] / "scripts/ci-audit.py"
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
monkeypatch.setattr(module.tempfile, "gettempdir", lambda: str(tmp_path))
|
||||
monkeypatch.setattr(module.time, "sleep", lambda _: None)
|
||||
monkeypatch.delenv("CI_AUDIT_TIMEOUT", raising=False)
|
||||
monkeypatch.delenv("CI_AUDIT_ATTEMPTS", raising=False)
|
||||
return module
|
||||
|
||||
|
||||
def result(code=0, stderr="", vulns=None, dependencies=None):
|
||||
report = {
|
||||
"dependencies": dependencies
|
||||
if dependencies is not None
|
||||
else [{"name": "example", "vulns": vulns or []}]
|
||||
}
|
||||
return SimpleNamespace(returncode=code, stderr=stderr, stdout=json.dumps(report))
|
||||
|
||||
|
||||
def test_transient_retry_then_complete(audit, monkeypatch):
|
||||
replies = iter([result(1, "ReadTimeout"), result()])
|
||||
calls = []
|
||||
|
||||
def run(command, **kwargs):
|
||||
calls.append(command)
|
||||
return next(replies)
|
||||
|
||||
monkeypatch.setattr(audit.subprocess, "run", run)
|
||||
audit.main()
|
||||
assert len(calls) == 2
|
||||
assert calls[0] == calls[1]
|
||||
assert "--cache-dir" in calls[0] and "--disable-pip" in calls[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reply",
|
||||
[
|
||||
result(1, vulns=[{"id": "CVE-test"}]),
|
||||
result(dependencies=[]),
|
||||
result(dependencies=[{"name": "example", "skip_reason": "unavailable"}]),
|
||||
result(2),
|
||||
],
|
||||
)
|
||||
def test_no_retry_for_findings_or_incomplete(audit, monkeypatch, reply):
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
audit.subprocess, "run", lambda *a, **k: calls.append(a) or reply
|
||||
)
|
||||
with pytest.raises(SystemExit):
|
||||
audit.main()
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_network_exhaustion_blocks_and_redacts(audit, monkeypatch, capsys):
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
audit.subprocess,
|
||||
"run",
|
||||
lambda *a, **k: (
|
||||
calls.append(a)
|
||||
or result(1, "ConnectionError https://user:secret@example.invalid")
|
||||
),
|
||||
)
|
||||
with pytest.raises(SystemExit):
|
||||
audit.main()
|
||||
assert len(calls) == 3
|
||||
assert "secret" not in capsys.readouterr().out
|
||||
Reference in New Issue
Block a user