diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index f29a4e8..aff60ea 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -13,6 +13,8 @@ jobs: verify: runs-on: ubuntu-latest env: + DOCKER_PIP_INDEX_URL: ${{ vars.DOCKER_PIP_INDEX_URL }} + DOCKER_PIP_FALLBACK_INDEX_URL: ${{ vars.DOCKER_PIP_FALLBACK_INDEX_URL }} CI_PIP_INDEX_URL: ${{ vars.CI_PIP_INDEX_URL }} CI_PIP_FALLBACK_INDEX_URL: ${{ vars.CI_PIP_FALLBACK_INDEX_URL }} DEBIAN_MIRROR: ${{ vars.DEBIAN_MIRROR }} @@ -64,6 +66,8 @@ jobs: PROD_STATE_VOLUME: ${{ vars.PROD_STATE_VOLUME }} PROD_MOCK: ${{ vars.PROD_MOCK }} PROD_WRITE_ENABLED: ${{ vars.PROD_WRITE_ENABLED }} + DOCKER_PIP_INDEX_URL: ${{ vars.DOCKER_PIP_INDEX_URL }} + DOCKER_PIP_FALLBACK_INDEX_URL: ${{ vars.DOCKER_PIP_FALLBACK_INDEX_URL }} CI_PIP_INDEX_URL: ${{ vars.CI_PIP_INDEX_URL }} CI_PIP_FALLBACK_INDEX_URL: ${{ vars.CI_PIP_FALLBACK_INDEX_URL }} DEBIAN_MIRROR: ${{ vars.DEBIAN_MIRROR }} diff --git a/Dockerfile b/Dockerfile index 8332e5f..3b9bc53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,13 +26,15 @@ RUN --mount=type=cache,id=frameflow-bookworm-apt-${TARGETARCH},target=/var/cache && mkdir -p /data/images /data/generated /data/cache /config \ && chown -R 10333:10333 /data /config COPY requirements.lock ./ +COPY scripts/ci-install.py /build-tools/install.py +ARG DOCKER_PIP_INDEX_URL=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple +ARG DOCKER_PIP_FALLBACK_INDEX_URL=https://pypi.org/simple RUN --mount=type=cache,id=frameflow-pip-py314-${TARGETARCH},target=/root/.cache/pip,sharing=locked \ --mount=type=bind,source=build-cache/wheels,target=/build-wheels,readonly \ if [ -f /build-wheels/SHA256SUMS ]; then \ (cd /build-wheels && sha256sum -c SHA256SUMS) \ && pip install --no-index --find-links=/build-wheels -r requirements.lock; \ - else python -c "import os; from urllib.parse import urlsplit; print('Docker build pip index host:', urlsplit(os.getenv('PIP_INDEX_URL', 'https://pypi.org/simple')).hostname, flush=True)" \ - && pip install -r requirements.lock; fi + else python /build-tools/install.py --locked; fi COPY pigway_frame_flow ./pigway_frame_flow COPY vendor ./vendor ARG BUILD_COMMIT=development diff --git a/docker-compose.yml b/docker-compose.yml index 57bc420..7977fc5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,8 @@ services: args: DEBIAN_MIRROR: ${DEBIAN_MIRROR:-deb.debian.org} DEBIAN_SECURITY_MIRROR: ${DEBIAN_SECURITY_MIRROR:-deb.debian.org} + DOCKER_PIP_INDEX_URL: ${DOCKER_PIP_INDEX_URL:-https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple} + DOCKER_PIP_FALLBACK_INDEX_URL: ${DOCKER_PIP_FALLBACK_INDEX_URL:-https://pypi.org/simple} BUILD_COMMIT: ${BUILD_COMMIT:-development} BUILD_TIME: ${BUILD_TIME:-unknown} restart: unless-stopped diff --git a/docs/EXTENSIONS.md b/docs/EXTENSIONS.md index 845d308..f5034fd 100644 --- a/docs/EXTENSIONS.md +++ b/docs/EXTENSIONS.md @@ -69,3 +69,20 @@ Publish a non-retained JSON message to the configured command topic: ``` Replies appear on `/result` with operation_id or the standard error object. Poll the REST Operation API for completion. QoS duplicate messages with the same request_id cannot repeat the operation; retained commands are rejected. Automatic broker reconnection never re-sends frame writes. No broker is started or required for ordinary NAS operation. + +### Docker dependency downloads + +Docker builds use `DOCKER_PIP_INDEX_URL` (default: Tsinghua PyPI mirror) and +`DOCKER_PIP_FALLBACK_INDEX_URL` (default: official PyPI). Configure these as +Gitea Variables for both Verify and production deployment. These public mirror +URLs must not contain credentials; build arguments are visible in build metadata. +CI tooling separately uses `CI_PIP_INDEX_URL` and `CI_PIP_FALLBACK_INDEX_URL`. +A failed mirror installation retries the same locked requirements against the +fallback, without extra indexes, version downgrades, or disabling build isolation. +Logs show index hosts and redact URL credentials/query parameters. + +APT/font layers remain cached when Python dependencies change. The Python layer +changes only when the lock, installer, base image, or mirror arguments change; +application source changes reuse it. BuildKit pip cache survives layer misses. +An explicitly supplied, checksum-verified offline wheel directory still takes +precedence and never falls back to the network. diff --git a/scripts/ci-install.py b/scripts/ci-install.py index e2ef856..b46edea 100644 --- a/scripts/ci-install.py +++ b/scripts/ci-install.py @@ -70,12 +70,15 @@ def diagnostics(): def main(): + locked = "--locked" in sys.argv[1:] + prefix = "DOCKER" if locked else "CI" + label = "Docker build" if locked else "CI Job" diagnostics() indexes = list( dict.fromkeys( [ - os.getenv("CI_PIP_INDEX_URL") or PRIMARY, - os.getenv("CI_PIP_FALLBACK_INDEX_URL") or FALLBACK, + os.getenv(f"{prefix}_PIP_INDEX_URL") or PRIMARY, + os.getenv(f"{prefix}_PIP_FALLBACK_INDEX_URL") or FALLBACK, ] ) ) @@ -84,7 +87,7 @@ def main(): if parsed.scheme != "https" or not parsed.hostname: raise SystemExit("CI pip indexes must be valid HTTPS URLs") for index in indexes: - print("Effective CI Job pip index host:", host(index), flush=True) + print(f"Effective {label} pip index host:", host(index), flush=True) result = subprocess.run( [ sys.executable, @@ -99,8 +102,7 @@ def main(): "30", "--index-url", index, - "-e", - ".[test]", + *(["-r", "requirements.lock"] if locked else ["-e", ".[test]"]), ], env=install_env(index), stdout=subprocess.PIPE, diff --git a/scripts/production-deploy.py b/scripts/production-deploy.py index 4705e1a..eaf5c31 100644 --- a/scripts/production-deploy.py +++ b/scripts/production-deploy.py @@ -364,6 +364,12 @@ class Deploy: command = ["build", "--pull", "-t", image] now = datetime.now(UTC).isoformat() for key, value in { + "DOCKER_PIP_INDEX_URL": self.env.get("DOCKER_PIP_INDEX_URL") + or "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple", + "DOCKER_PIP_FALLBACK_INDEX_URL": self.env.get( + "DOCKER_PIP_FALLBACK_INDEX_URL" + ) + or "https://pypi.org/simple", "DEBIAN_MIRROR": self.env.get("DEBIAN_MIRROR") or "deb.debian.org", "DEBIAN_SECURITY_MIRROR": self.env.get("DEBIAN_SECURITY_MIRROR") or "deb.debian.org", diff --git a/tests/test_ci_install.py b/tests/test_ci_install.py index c7ae036..ff17bb3 100644 --- a/tests/test_ci_install.py +++ b/tests/test_ci_install.py @@ -83,3 +83,27 @@ def test_sequential_fallback_keeps_editable_and_isolation( output = capsys.readouterr().out assert "fixture-secret" not in output and "fixture-token" not in output assert "mirror.example" in output and "pypi.org" in output + + +def test_docker_locked_install_has_independent_index_and_fallback( + installer, monkeypatch +): + monkeypatch.setattr(installer.sys, "argv", ["install.py", "--locked"]) + monkeypatch.setenv("CI_PIP_INDEX_URL", "https://ci.example/simple") + monkeypatch.setenv("DOCKER_PIP_INDEX_URL", "https://docker.example/simple") + monkeypatch.setenv("DOCKER_PIP_FALLBACK_INDEX_URL", "https://pypi.org/simple") + calls = [] + + def run(command, **kwargs): + if "config" in command: + return SimpleNamespace(returncode=0, stdout="") + calls.append((command, kwargs["env"])) + return SimpleNamespace(returncode=0 if len(calls) == 2 else 1, stdout="") + + monkeypatch.setattr(installer.subprocess, "run", run) + installer.main() + assert [env["PIP_INDEX_URL"] for _, env in calls] == [ + "https://docker.example/simple", + "https://pypi.org/simple", + ] + assert all(command[-2:] == ["-r", "requirements.lock"] for command, _ in calls)