Make CI pip indexes explicit with safe diagnostics and sequential fallback

.github/workflows/verify.yml

@@ -4,6 +4,8 @@ jobs:
   verify:
     runs-on: ubuntu-latest
     env:
+      CI_PIP_INDEX_URL: ${{ vars.CI_PIP_INDEX_URL }}
+      CI_PIP_FALLBACK_INDEX_URL: ${{ vars.CI_PIP_FALLBACK_INDEX_URL }}
       DEBIAN_MIRROR: ${{ vars.DEBIAN_MIRROR }}
       DEBIAN_SECURITY_MIRROR: ${{ vars.DEBIAN_SECURITY_MIRROR }}
     steps:
@@ -11,7 +13,8 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.14'
-      - run: python -m pip install -e '.[test]'
+      - name: Install CI dependencies with index diagnostics and fallback
+        run: python scripts/ci-install.py
       - run: python -m ruff check pigway_frame_flow tests scripts
       - run: python -m ruff format --check pigway_frame_flow tests scripts
       - run: python -m mypy pigway_frame_flow
@@ -23,4 +26,7 @@ jobs:
           python scripts/ci-compose-verify.py
       - name: Cleanup isolated Docker resources
         if: always()
-        run: python scripts/ci-compose-verify.py --cleanup
+        run: |
+          if [ -n "${FRAME_FLOW_CI_COMPOSE:-}" ] && [ -f "$FRAME_FLOW_CI_COMPOSE" ]; then
+            python scripts/ci-compose-verify.py --cleanup
+          fi

Dockerfile

@@ -31,7 +31,8 @@ RUN --mount=type=cache,id=frameflow-pip-py314-${TARGETARCH},target=/root/.cache/
     if [ -f /build-wheels/SHA256SUMS ]; then \
       (cd /build-wheels && sha256sum -c SHA256SUMS) \
       && pip install --no-index --find-links=/build-wheels -r requirements.lock; \
-    else pip install -r requirements.lock; fi
+    else python -c "import os; from urllib.parse import urlsplit; print('Docker build pip index host:', urlsplit(os.getenv('PIP_INDEX_URL', 'https://pypi.org/simple')).hostname, flush=True)" \
+      && pip install -r requirements.lock; fi
 COPY pigway_frame_flow ./pigway_frame_flow
 COPY vendor ./vendor
 ARG BUILD_COMMIT=development

README.md

@@ -310,3 +310,31 @@ fallback can retry. Abrupt host/daemon loss or a forcibly terminated entire job
 can prevent both cleanup paths; no CI script can guarantee cleanup in that case.
 No global prune or production resource removal is performed. Keep builder
 caches: only the run-specific image tag is removed.
+
+### CI Job pip indexes versus Docker build indexes
+
+The Job bootstrap runs `python scripts/ci-install.py` before Ruff/mypy/tests.
+It needs only the Python standard library, so it runs before PyYAML or tooling
+is installed. `CI_PIP_INDEX_URL` and `CI_PIP_FALLBACK_INDEX_URL` are optional
+Gitea repository variables. Defaults are the TUNA HTTPS PyPI simple endpoint
+and official `https://pypi.org/simple`. Each is attempted separately; the
+fallback repeats the unchanged editable `.[test]` installation, including
+PEP 517 build isolation and `setuptools>=75`. No simultaneous extra index is
+used, no build dependency is removed, and both failures fail CI.
+
+Logs show inherited index/config/proxy hosts and the selected index host for
+each attempt, with URL credentials and query strings removed. Bootstrap
+disables inherited pip config files via `PIP_CONFIG_FILE=/dev/null`, explicitly
+sets the selected index, and clears no-index/extra-index/find-links, insecure
+trusted-host, and no-build-isolation environment settings. Constraints, TLS
+certificate environment settings, and proxy settings remain in effect.
+
+These `CI_PIP_*` settings apply only to the Job's pip install. They are not
+passed into Docker build arguments or persisted into the production image.
+The Dockerfile keeps its own cached/offline dependency installation and logs
+the Docker pip index host for online installs. Debian mirror variables affect
+APT only and are loaded later in the Docker verification step.
+
+If dependency installation fails, the always() cleanup step still executes.
+It skips the Python/PyYAML cleanup script when no CI Compose manifest exists,
+as no Docker verification resources have been created at that point.

scripts/ci-install.py

@@ -0,0 +1,130 @@
+"""Install CI tooling with explicit, sequential indexes and secret-safe diagnostics."""
+
+import ast
+import os
+import re
+import subprocess
+import sys
+from urllib.parse import urlsplit
+
+PRIMARY = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
+FALLBACK = "https://pypi.org/simple"
+URL = re.compile(r"https?://[^\s\"'<>]+")
+
+
+def host(url):
+    return urlsplit(url).hostname or "invalid"
+
+
+def safe_output(text):
+    # Pip/proxy exceptions may contain credentials or signed query strings.
+    return URL.sub(lambda m: "https://" + host(m.group()) + "/[redacted-url]", text)
+
+
+def install_env(index):
+    env = os.environ.copy()
+    for key in (
+        "PIP_EXTRA_INDEX_URL",
+        "PIP_NO_INDEX",
+        "PIP_FIND_LINKS",
+        "PIP_TRUSTED_HOST",
+        "PIP_NO_BUILD_ISOLATION",
+    ):
+        env.pop(key, None)
+    env["PIP_CONFIG_FILE"] = os.devnull
+    env["PIP_INDEX_URL"] = index
+    return env
+
+
+def diagnostics():
+    for key in ("PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "HTTPS_PROXY", "HTTP_PROXY"):
+        values = os.getenv(key, "").split()
+        print(
+            f"Inherited {key} hosts: {','.join(host(v) for v in values) or 'unset'}",
+            flush=True,
+        )
+    print("Inherited PIP_NO_INDEX set:", bool(os.getenv("PIP_NO_INDEX")), flush=True)
+    result = subprocess.run(
+        [sys.executable, "-m", "pip", "config", "list"],
+        capture_output=True,
+        text=True,
+    )
+    for line in result.stdout.splitlines():
+        key, separator, value = line.partition("=")
+        if separator and key.endswith(("index-url", "extra-index-url")):
+            try:
+                value = ast.literal_eval(value)
+            except (ValueError, SyntaxError):
+                value = ""
+            print(
+                "Inherited pip config",
+                key,
+                "hosts:",
+                ",".join(host(v) for v in str(value).split()),
+                flush=True,
+            )
+    print(
+        "Effective CI pip config files: disabled; extra indexes/no-index disabled; build isolation retained",
+        flush=True,
+    )
+
+
+def main():
+    diagnostics()
+    indexes = list(
+        dict.fromkeys(
+            [
+                os.getenv("CI_PIP_INDEX_URL") or PRIMARY,
+                os.getenv("CI_PIP_FALLBACK_INDEX_URL") or FALLBACK,
+            ]
+        )
+    )
+    for index in indexes:
+        parsed = urlsplit(index)
+        if parsed.scheme != "https" or not parsed.hostname:
+            raise SystemExit("CI pip indexes must be valid HTTPS URLs")
+    for index in indexes:
+        print("Effective CI Job pip index host:", host(index), flush=True)
+        result = subprocess.run(
+            [
+                sys.executable,
+                "-m",
+                "pip",
+                "install",
+                "--disable-pip-version-check",
+                "--no-input",
+                "--retries",
+                "2",
+                "--timeout",
+                "30",
+                "--index-url",
+                index,
+                "-e",
+                ".[test]",
+            ],
+            env=install_env(index),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+        )
+        print(safe_output(result.stdout), flush=True)
+        if result.returncode == 0:
+            print(
+                "CI dependency installation succeeded; index host:",
+                host(index),
+                flush=True,
+            )
+            return
+        print(
+            "CI install failed at host",
+            host(index),
+            "exit",
+            result.returncode,
+            "; trying next configured index if available",
+            flush=True,
+        )
+    raise SystemExit("CI dependency installation failed on all configured indexes")
+
+
+if __name__ == "__main__":
+    main()

tests/test_ci_install.py

@@ -0,0 +1,85 @@
+"""Bootstrap must preserve build isolation and fall back without leaking secrets."""
+
+import importlib.util
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+
+@pytest.fixture
+def installer():
+    spec = importlib.util.spec_from_file_location(
+        "ci_install", Path(__file__).parents[1] / "scripts/ci-install.py"
+    )
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_scrub_credentials_and_signed_urls(installer):
+    text = "https://user:fixture-secret@mirror.example/simple?token=fixture-token"
+    assert installer.safe_output(text) == "https://mirror.example/[redacted-url]"
+
+
+def test_install_environment_overrides_inherited_indexes(installer, monkeypatch):
+    for key in (
+        "PIP_NO_INDEX",
+        "PIP_EXTRA_INDEX_URL",
+        "PIP_FIND_LINKS",
+        "PIP_NO_BUILD_ISOLATION",
+    ):
+        monkeypatch.setenv(key, "fixture")
+    monkeypatch.setenv("PIP_CONSTRAINT", "supply-chain-constraints.txt")
+    env = installer.install_env("https://pypi.org/simple")
+    assert env["PIP_CONFIG_FILE"] == installer.os.devnull
+    assert env["PIP_INDEX_URL"] == "https://pypi.org/simple"
+    assert env["PIP_CONSTRAINT"] == "supply-chain-constraints.txt"
+    assert all(
+        key not in env
+        for key in (
+            "PIP_NO_INDEX",
+            "PIP_EXTRA_INDEX_URL",
+            "PIP_FIND_LINKS",
+            "PIP_NO_BUILD_ISOLATION",
+        )
+    )
+
+
+@pytest.mark.parametrize("fallback_succeeds", [True, False])
+def test_sequential_fallback_keeps_editable_and_isolation(
+    installer, monkeypatch, capsys, fallback_succeeds
+):
+    calls = []
+    monkeypatch.setenv(
+        "CI_PIP_INDEX_URL", "https://user:fixture-secret@mirror.example/simple"
+    )
+    monkeypatch.setenv("CI_PIP_FALLBACK_INDEX_URL", "https://pypi.org/simple")
+
+    def run(command, **kwargs):
+        if "config" in command:
+            return SimpleNamespace(
+                returncode=0,
+                stdout="global.index-url='https://user:fixture-secret@mirror.example/simple'\n",
+            )
+        calls.append((command, kwargs["env"]))
+        return SimpleNamespace(
+            returncode=0 if len(calls) == 2 and fallback_succeeds else 1,
+            stdout="Could not fetch https://user:fixture-secret@mirror.example/simple?token=fixture-token\n",
+        )
+
+    monkeypatch.setattr(installer.subprocess, "run", run)
+    if fallback_succeeds:
+        installer.main()
+    else:
+        with pytest.raises(SystemExit):
+            installer.main()
+    assert len(calls) == 2
+    assert calls[1][1]["PIP_INDEX_URL"] == "https://pypi.org/simple"
+    assert all(
+        command[-2:] == ["-e", ".[test]"] and "--no-build-isolation" not in command
+        for command, _ in calls
+    )
+    output = capsys.readouterr().out
+    assert "fixture-secret" not in output and "fixture-token" not in output
+    assert "mirror.example" in output and "pypi.org" in output