133 lines
3.9 KiB
Python
133 lines
3.9 KiB
Python
"""Install CI tooling with explicit, sequential indexes and secret-safe diagnostics."""
|
|
|
|
import ast
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from urllib.parse import urlsplit
|
|
|
|
PRIMARY = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
|
|
FALLBACK = "https://pypi.org/simple"
|
|
URL = re.compile(r"https?://[^\s\"'<>]+")
|
|
|
|
|
|
def host(url):
|
|
return urlsplit(url).hostname or "invalid"
|
|
|
|
|
|
def safe_output(text):
|
|
# Pip/proxy exceptions may contain credentials or signed query strings.
|
|
return URL.sub(lambda m: "https://" + host(m.group()) + "/[redacted-url]", text)
|
|
|
|
|
|
def install_env(index):
|
|
env = os.environ.copy()
|
|
for key in (
|
|
"PIP_EXTRA_INDEX_URL",
|
|
"PIP_NO_INDEX",
|
|
"PIP_FIND_LINKS",
|
|
"PIP_TRUSTED_HOST",
|
|
"PIP_NO_BUILD_ISOLATION",
|
|
):
|
|
env.pop(key, None)
|
|
env["PIP_CONFIG_FILE"] = os.devnull
|
|
env["PIP_INDEX_URL"] = index
|
|
return env
|
|
|
|
|
|
def diagnostics():
|
|
for key in ("PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "HTTPS_PROXY", "HTTP_PROXY"):
|
|
values = os.getenv(key, "").split()
|
|
print(
|
|
f"Inherited {key} hosts: {','.join(host(v) for v in values) or 'unset'}",
|
|
flush=True,
|
|
)
|
|
print("Inherited PIP_NO_INDEX set:", bool(os.getenv("PIP_NO_INDEX")), flush=True)
|
|
result = subprocess.run(
|
|
[sys.executable, "-m", "pip", "config", "list"],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
for line in result.stdout.splitlines():
|
|
key, separator, value = line.partition("=")
|
|
if separator and key.endswith(("index-url", "extra-index-url")):
|
|
try:
|
|
value = ast.literal_eval(value)
|
|
except (ValueError, SyntaxError):
|
|
value = ""
|
|
print(
|
|
"Inherited pip config",
|
|
key,
|
|
"hosts:",
|
|
",".join(host(v) for v in str(value).split()),
|
|
flush=True,
|
|
)
|
|
print(
|
|
"Effective CI pip config files: disabled; extra indexes/no-index disabled; build isolation retained",
|
|
flush=True,
|
|
)
|
|
|
|
|
|
def main():
|
|
locked = "--locked" in sys.argv[1:]
|
|
prefix = "DOCKER" if locked else "CI"
|
|
label = "Docker build" if locked else "CI Job"
|
|
diagnostics()
|
|
indexes = list(
|
|
dict.fromkeys(
|
|
[
|
|
os.getenv(f"{prefix}_PIP_INDEX_URL") or PRIMARY,
|
|
os.getenv(f"{prefix}_PIP_FALLBACK_INDEX_URL") or FALLBACK,
|
|
]
|
|
)
|
|
)
|
|
for index in indexes:
|
|
parsed = urlsplit(index)
|
|
if parsed.scheme != "https" or not parsed.hostname:
|
|
raise SystemExit("CI pip indexes must be valid HTTPS URLs")
|
|
for index in indexes:
|
|
print(f"Effective {label} pip index host:", host(index), flush=True)
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"pip",
|
|
"install",
|
|
"--disable-pip-version-check",
|
|
"--no-input",
|
|
"--retries",
|
|
"2",
|
|
"--timeout",
|
|
"30",
|
|
"--index-url",
|
|
index,
|
|
*(["-r", "requirements.lock"] if locked else ["-e", ".[test]"]),
|
|
],
|
|
env=install_env(index),
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
)
|
|
print(safe_output(result.stdout), flush=True)
|
|
if result.returncode == 0:
|
|
print(
|
|
"CI dependency installation succeeded; index host:",
|
|
host(index),
|
|
flush=True,
|
|
)
|
|
return
|
|
print(
|
|
"CI install failed at host",
|
|
host(index),
|
|
"exit",
|
|
result.returncode,
|
|
"; trying next configured index if available",
|
|
flush=True,
|
|
)
|
|
raise SystemExit("CI dependency installation failed on all configured indexes")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|