From f7e65de0a25fdcd081a36cce842563be7f82ddb9 Mon Sep 17 00:00:00 2001 From: Pigway Date: Wed, 7 Oct 2026 20:51:05 +0800 Subject: [PATCH] Make CI pip indexes explicit with safe diagnostics and sequential fallback --- .github/workflows/verify.yml | 10 ++- Dockerfile | 3 +- README.md | 28 ++++++++ scripts/ci-install.py | 130 +++++++++++++++++++++++++++++++++++ tests/test_ci_install.py | 85 +++++++++++++++++++++++ 5 files changed, 253 insertions(+), 3 deletions(-) create mode 100644 scripts/ci-install.py create mode 100644 tests/test_ci_install.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 0dc8100..2ed0681 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -4,6 +4,8 @@ jobs: verify: runs-on: ubuntu-latest env: + CI_PIP_INDEX_URL: ${{ vars.CI_PIP_INDEX_URL }} + CI_PIP_FALLBACK_INDEX_URL: ${{ vars.CI_PIP_FALLBACK_INDEX_URL }} DEBIAN_MIRROR: ${{ vars.DEBIAN_MIRROR }} DEBIAN_SECURITY_MIRROR: ${{ vars.DEBIAN_SECURITY_MIRROR }} steps: @@ -11,7 +13,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.14' - - run: python -m pip install -e '.[test]' + - name: Install CI dependencies with index diagnostics and fallback + run: python scripts/ci-install.py - run: python -m ruff check pigway_frame_flow tests scripts - run: python -m ruff format --check pigway_frame_flow tests scripts - run: python -m mypy pigway_frame_flow @@ -23,4 +26,7 @@ jobs: python scripts/ci-compose-verify.py - name: Cleanup isolated Docker resources if: always() - run: python scripts/ci-compose-verify.py --cleanup + run: | + if [ -n "${FRAME_FLOW_CI_COMPOSE:-}" ] && [ -f "$FRAME_FLOW_CI_COMPOSE" ]; then + python scripts/ci-compose-verify.py --cleanup + fi diff --git a/Dockerfile b/Dockerfile index a5b0f36..8332e5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,8 @@ RUN --mount=type=cache,id=frameflow-pip-py314-${TARGETARCH},target=/root/.cache/ if [ -f /build-wheels/SHA256SUMS ]; then \ (cd /build-wheels && sha256sum -c SHA256SUMS) \ && pip install --no-index --find-links=/build-wheels -r requirements.lock; \ - else pip install -r requirements.lock; fi + else python -c "import os; from urllib.parse import urlsplit; print('Docker build pip index host:', urlsplit(os.getenv('PIP_INDEX_URL', 'https://pypi.org/simple')).hostname, flush=True)" \ + && pip install -r requirements.lock; fi COPY pigway_frame_flow ./pigway_frame_flow COPY vendor ./vendor ARG BUILD_COMMIT=development diff --git a/README.md b/README.md index 056f09e..9d0f58f 100644 --- a/README.md +++ b/README.md @@ -310,3 +310,31 @@ fallback can retry. Abrupt host/daemon loss or a forcibly terminated entire job can prevent both cleanup paths; no CI script can guarantee cleanup in that case. No global prune or production resource removal is performed. Keep builder caches: only the run-specific image tag is removed. + +### CI Job pip indexes versus Docker build indexes + +The Job bootstrap runs `python scripts/ci-install.py` before Ruff/mypy/tests. +It needs only the Python standard library, so it runs before PyYAML or tooling +is installed. `CI_PIP_INDEX_URL` and `CI_PIP_FALLBACK_INDEX_URL` are optional +Gitea repository variables. Defaults are the TUNA HTTPS PyPI simple endpoint +and official `https://pypi.org/simple`. Each is attempted separately; the +fallback repeats the unchanged editable `.[test]` installation, including +PEP 517 build isolation and `setuptools>=75`. No simultaneous extra index is +used, no build dependency is removed, and both failures fail CI. + +Logs show inherited index/config/proxy hosts and the selected index host for +each attempt, with URL credentials and query strings removed. Bootstrap +disables inherited pip config files via `PIP_CONFIG_FILE=/dev/null`, explicitly +sets the selected index, and clears no-index/extra-index/find-links, insecure +trusted-host, and no-build-isolation environment settings. Constraints, TLS +certificate environment settings, and proxy settings remain in effect. + +These `CI_PIP_*` settings apply only to the Job's pip install. They are not +passed into Docker build arguments or persisted into the production image. +The Dockerfile keeps its own cached/offline dependency installation and logs +the Docker pip index host for online installs. Debian mirror variables affect +APT only and are loaded later in the Docker verification step. + +If dependency installation fails, the always() cleanup step still executes. +It skips the Python/PyYAML cleanup script when no CI Compose manifest exists, +as no Docker verification resources have been created at that point. diff --git a/scripts/ci-install.py b/scripts/ci-install.py new file mode 100644 index 0000000..e2ef856 --- /dev/null +++ b/scripts/ci-install.py @@ -0,0 +1,130 @@ +"""Install CI tooling with explicit, sequential indexes and secret-safe diagnostics.""" + +import ast +import os +import re +import subprocess +import sys +from urllib.parse import urlsplit + +PRIMARY = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" +FALLBACK = "https://pypi.org/simple" +URL = re.compile(r"https?://[^\s\"'<>]+") + + +def host(url): + return urlsplit(url).hostname or "invalid" + + +def safe_output(text): + # Pip/proxy exceptions may contain credentials or signed query strings. + return URL.sub(lambda m: "https://" + host(m.group()) + "/[redacted-url]", text) + + +def install_env(index): + env = os.environ.copy() + for key in ( + "PIP_EXTRA_INDEX_URL", + "PIP_NO_INDEX", + "PIP_FIND_LINKS", + "PIP_TRUSTED_HOST", + "PIP_NO_BUILD_ISOLATION", + ): + env.pop(key, None) + env["PIP_CONFIG_FILE"] = os.devnull + env["PIP_INDEX_URL"] = index + return env + + +def diagnostics(): + for key in ("PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "HTTPS_PROXY", "HTTP_PROXY"): + values = os.getenv(key, "").split() + print( + f"Inherited {key} hosts: {','.join(host(v) for v in values) or 'unset'}", + flush=True, + ) + print("Inherited PIP_NO_INDEX set:", bool(os.getenv("PIP_NO_INDEX")), flush=True) + result = subprocess.run( + [sys.executable, "-m", "pip", "config", "list"], + capture_output=True, + text=True, + ) + for line in result.stdout.splitlines(): + key, separator, value = line.partition("=") + if separator and key.endswith(("index-url", "extra-index-url")): + try: + value = ast.literal_eval(value) + except (ValueError, SyntaxError): + value = "" + print( + "Inherited pip config", + key, + "hosts:", + ",".join(host(v) for v in str(value).split()), + flush=True, + ) + print( + "Effective CI pip config files: disabled; extra indexes/no-index disabled; build isolation retained", + flush=True, + ) + + +def main(): + diagnostics() + indexes = list( + dict.fromkeys( + [ + os.getenv("CI_PIP_INDEX_URL") or PRIMARY, + os.getenv("CI_PIP_FALLBACK_INDEX_URL") or FALLBACK, + ] + ) + ) + for index in indexes: + parsed = urlsplit(index) + if parsed.scheme != "https" or not parsed.hostname: + raise SystemExit("CI pip indexes must be valid HTTPS URLs") + for index in indexes: + print("Effective CI Job pip index host:", host(index), flush=True) + result = subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--no-input", + "--retries", + "2", + "--timeout", + "30", + "--index-url", + index, + "-e", + ".[test]", + ], + env=install_env(index), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + print(safe_output(result.stdout), flush=True) + if result.returncode == 0: + print( + "CI dependency installation succeeded; index host:", + host(index), + flush=True, + ) + return + print( + "CI install failed at host", + host(index), + "exit", + result.returncode, + "; trying next configured index if available", + flush=True, + ) + raise SystemExit("CI dependency installation failed on all configured indexes") + + +if __name__ == "__main__": + main() diff --git a/tests/test_ci_install.py b/tests/test_ci_install.py new file mode 100644 index 0000000..c7ae036 --- /dev/null +++ b/tests/test_ci_install.py @@ -0,0 +1,85 @@ +"""Bootstrap must preserve build isolation and fall back without leaking secrets.""" + +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +@pytest.fixture +def installer(): + spec = importlib.util.spec_from_file_location( + "ci_install", Path(__file__).parents[1] / "scripts/ci-install.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_scrub_credentials_and_signed_urls(installer): + text = "https://user:fixture-secret@mirror.example/simple?token=fixture-token" + assert installer.safe_output(text) == "https://mirror.example/[redacted-url]" + + +def test_install_environment_overrides_inherited_indexes(installer, monkeypatch): + for key in ( + "PIP_NO_INDEX", + "PIP_EXTRA_INDEX_URL", + "PIP_FIND_LINKS", + "PIP_NO_BUILD_ISOLATION", + ): + monkeypatch.setenv(key, "fixture") + monkeypatch.setenv("PIP_CONSTRAINT", "supply-chain-constraints.txt") + env = installer.install_env("https://pypi.org/simple") + assert env["PIP_CONFIG_FILE"] == installer.os.devnull + assert env["PIP_INDEX_URL"] == "https://pypi.org/simple" + assert env["PIP_CONSTRAINT"] == "supply-chain-constraints.txt" + assert all( + key not in env + for key in ( + "PIP_NO_INDEX", + "PIP_EXTRA_INDEX_URL", + "PIP_FIND_LINKS", + "PIP_NO_BUILD_ISOLATION", + ) + ) + + +@pytest.mark.parametrize("fallback_succeeds", [True, False]) +def test_sequential_fallback_keeps_editable_and_isolation( + installer, monkeypatch, capsys, fallback_succeeds +): + calls = [] + monkeypatch.setenv( + "CI_PIP_INDEX_URL", "https://user:fixture-secret@mirror.example/simple" + ) + monkeypatch.setenv("CI_PIP_FALLBACK_INDEX_URL", "https://pypi.org/simple") + + def run(command, **kwargs): + if "config" in command: + return SimpleNamespace( + returncode=0, + stdout="global.index-url='https://user:fixture-secret@mirror.example/simple'\n", + ) + calls.append((command, kwargs["env"])) + return SimpleNamespace( + returncode=0 if len(calls) == 2 and fallback_succeeds else 1, + stdout="Could not fetch https://user:fixture-secret@mirror.example/simple?token=fixture-token\n", + ) + + monkeypatch.setattr(installer.subprocess, "run", run) + if fallback_succeeds: + installer.main() + else: + with pytest.raises(SystemExit): + installer.main() + assert len(calls) == 2 + assert calls[1][1]["PIP_INDEX_URL"] == "https://pypi.org/simple" + assert all( + command[-2:] == ["-e", ".[test]"] and "--no-build-isolation" not in command + for command, _ in calls + ) + output = capsys.readouterr().out + assert "fixture-secret" not in output and "fixture-token" not in output + assert "mirror.example" in output and "pypi.org" in output