fix(build): configure Docker pip mirrors with locked fallback [skip ci]
This commit is contained in:
@@ -13,6 +13,8 @@ jobs:
|
|||||||
verify:
|
verify:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
env:
|
env:
|
||||||
|
DOCKER_PIP_INDEX_URL: ${{ vars.DOCKER_PIP_INDEX_URL }}
|
||||||
|
DOCKER_PIP_FALLBACK_INDEX_URL: ${{ vars.DOCKER_PIP_FALLBACK_INDEX_URL }}
|
||||||
CI_PIP_INDEX_URL: ${{ vars.CI_PIP_INDEX_URL }}
|
CI_PIP_INDEX_URL: ${{ vars.CI_PIP_INDEX_URL }}
|
||||||
CI_PIP_FALLBACK_INDEX_URL: ${{ vars.CI_PIP_FALLBACK_INDEX_URL }}
|
CI_PIP_FALLBACK_INDEX_URL: ${{ vars.CI_PIP_FALLBACK_INDEX_URL }}
|
||||||
DEBIAN_MIRROR: ${{ vars.DEBIAN_MIRROR }}
|
DEBIAN_MIRROR: ${{ vars.DEBIAN_MIRROR }}
|
||||||
@@ -64,6 +66,8 @@ jobs:
|
|||||||
PROD_STATE_VOLUME: ${{ vars.PROD_STATE_VOLUME }}
|
PROD_STATE_VOLUME: ${{ vars.PROD_STATE_VOLUME }}
|
||||||
PROD_MOCK: ${{ vars.PROD_MOCK }}
|
PROD_MOCK: ${{ vars.PROD_MOCK }}
|
||||||
PROD_WRITE_ENABLED: ${{ vars.PROD_WRITE_ENABLED }}
|
PROD_WRITE_ENABLED: ${{ vars.PROD_WRITE_ENABLED }}
|
||||||
|
DOCKER_PIP_INDEX_URL: ${{ vars.DOCKER_PIP_INDEX_URL }}
|
||||||
|
DOCKER_PIP_FALLBACK_INDEX_URL: ${{ vars.DOCKER_PIP_FALLBACK_INDEX_URL }}
|
||||||
CI_PIP_INDEX_URL: ${{ vars.CI_PIP_INDEX_URL }}
|
CI_PIP_INDEX_URL: ${{ vars.CI_PIP_INDEX_URL }}
|
||||||
CI_PIP_FALLBACK_INDEX_URL: ${{ vars.CI_PIP_FALLBACK_INDEX_URL }}
|
CI_PIP_FALLBACK_INDEX_URL: ${{ vars.CI_PIP_FALLBACK_INDEX_URL }}
|
||||||
DEBIAN_MIRROR: ${{ vars.DEBIAN_MIRROR }}
|
DEBIAN_MIRROR: ${{ vars.DEBIAN_MIRROR }}
|
||||||
|
|||||||
+4
-2
@@ -26,13 +26,15 @@ RUN --mount=type=cache,id=frameflow-bookworm-apt-${TARGETARCH},target=/var/cache
|
|||||||
&& mkdir -p /data/images /data/generated /data/cache /config \
|
&& mkdir -p /data/images /data/generated /data/cache /config \
|
||||||
&& chown -R 10333:10333 /data /config
|
&& chown -R 10333:10333 /data /config
|
||||||
COPY requirements.lock ./
|
COPY requirements.lock ./
|
||||||
|
COPY scripts/ci-install.py /build-tools/install.py
|
||||||
|
ARG DOCKER_PIP_INDEX_URL=https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
|
||||||
|
ARG DOCKER_PIP_FALLBACK_INDEX_URL=https://pypi.org/simple
|
||||||
RUN --mount=type=cache,id=frameflow-pip-py314-${TARGETARCH},target=/root/.cache/pip,sharing=locked \
|
RUN --mount=type=cache,id=frameflow-pip-py314-${TARGETARCH},target=/root/.cache/pip,sharing=locked \
|
||||||
--mount=type=bind,source=build-cache/wheels,target=/build-wheels,readonly \
|
--mount=type=bind,source=build-cache/wheels,target=/build-wheels,readonly \
|
||||||
if [ -f /build-wheels/SHA256SUMS ]; then \
|
if [ -f /build-wheels/SHA256SUMS ]; then \
|
||||||
(cd /build-wheels && sha256sum -c SHA256SUMS) \
|
(cd /build-wheels && sha256sum -c SHA256SUMS) \
|
||||||
&& pip install --no-index --find-links=/build-wheels -r requirements.lock; \
|
&& pip install --no-index --find-links=/build-wheels -r requirements.lock; \
|
||||||
else python -c "import os; from urllib.parse import urlsplit; print('Docker build pip index host:', urlsplit(os.getenv('PIP_INDEX_URL', 'https://pypi.org/simple')).hostname, flush=True)" \
|
else python /build-tools/install.py --locked; fi
|
||||||
&& pip install -r requirements.lock; fi
|
|
||||||
COPY pigway_frame_flow ./pigway_frame_flow
|
COPY pigway_frame_flow ./pigway_frame_flow
|
||||||
COPY vendor ./vendor
|
COPY vendor ./vendor
|
||||||
ARG BUILD_COMMIT=development
|
ARG BUILD_COMMIT=development
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ services:
|
|||||||
args:
|
args:
|
||||||
DEBIAN_MIRROR: ${DEBIAN_MIRROR:-deb.debian.org}
|
DEBIAN_MIRROR: ${DEBIAN_MIRROR:-deb.debian.org}
|
||||||
DEBIAN_SECURITY_MIRROR: ${DEBIAN_SECURITY_MIRROR:-deb.debian.org}
|
DEBIAN_SECURITY_MIRROR: ${DEBIAN_SECURITY_MIRROR:-deb.debian.org}
|
||||||
|
DOCKER_PIP_INDEX_URL: ${DOCKER_PIP_INDEX_URL:-https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple}
|
||||||
|
DOCKER_PIP_FALLBACK_INDEX_URL: ${DOCKER_PIP_FALLBACK_INDEX_URL:-https://pypi.org/simple}
|
||||||
BUILD_COMMIT: ${BUILD_COMMIT:-development}
|
BUILD_COMMIT: ${BUILD_COMMIT:-development}
|
||||||
BUILD_TIME: ${BUILD_TIME:-unknown}
|
BUILD_TIME: ${BUILD_TIME:-unknown}
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|||||||
@@ -69,3 +69,20 @@ Publish a non-retained JSON message to the configured command topic:
|
|||||||
```
|
```
|
||||||
|
|
||||||
Replies appear on `<topic>/result` with operation_id or the standard error object. Poll the REST Operation API for completion. QoS duplicate messages with the same request_id cannot repeat the operation; retained commands are rejected. Automatic broker reconnection never re-sends frame writes. No broker is started or required for ordinary NAS operation.
|
Replies appear on `<topic>/result` with operation_id or the standard error object. Poll the REST Operation API for completion. QoS duplicate messages with the same request_id cannot repeat the operation; retained commands are rejected. Automatic broker reconnection never re-sends frame writes. No broker is started or required for ordinary NAS operation.
|
||||||
|
|
||||||
|
### Docker dependency downloads
|
||||||
|
|
||||||
|
Docker builds use `DOCKER_PIP_INDEX_URL` (default: Tsinghua PyPI mirror) and
|
||||||
|
`DOCKER_PIP_FALLBACK_INDEX_URL` (default: official PyPI). Configure these as
|
||||||
|
Gitea Variables for both Verify and production deployment. These public mirror
|
||||||
|
URLs must not contain credentials; build arguments are visible in build metadata.
|
||||||
|
CI tooling separately uses `CI_PIP_INDEX_URL` and `CI_PIP_FALLBACK_INDEX_URL`.
|
||||||
|
A failed mirror installation retries the same locked requirements against the
|
||||||
|
fallback, without extra indexes, version downgrades, or disabling build isolation.
|
||||||
|
Logs show index hosts and redact URL credentials/query parameters.
|
||||||
|
|
||||||
|
APT/font layers remain cached when Python dependencies change. The Python layer
|
||||||
|
changes only when the lock, installer, base image, or mirror arguments change;
|
||||||
|
application source changes reuse it. BuildKit pip cache survives layer misses.
|
||||||
|
An explicitly supplied, checksum-verified offline wheel directory still takes
|
||||||
|
precedence and never falls back to the network.
|
||||||
|
|||||||
@@ -70,12 +70,15 @@ def diagnostics():
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
locked = "--locked" in sys.argv[1:]
|
||||||
|
prefix = "DOCKER" if locked else "CI"
|
||||||
|
label = "Docker build" if locked else "CI Job"
|
||||||
diagnostics()
|
diagnostics()
|
||||||
indexes = list(
|
indexes = list(
|
||||||
dict.fromkeys(
|
dict.fromkeys(
|
||||||
[
|
[
|
||||||
os.getenv("CI_PIP_INDEX_URL") or PRIMARY,
|
os.getenv(f"{prefix}_PIP_INDEX_URL") or PRIMARY,
|
||||||
os.getenv("CI_PIP_FALLBACK_INDEX_URL") or FALLBACK,
|
os.getenv(f"{prefix}_PIP_FALLBACK_INDEX_URL") or FALLBACK,
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -84,7 +87,7 @@ def main():
|
|||||||
if parsed.scheme != "https" or not parsed.hostname:
|
if parsed.scheme != "https" or not parsed.hostname:
|
||||||
raise SystemExit("CI pip indexes must be valid HTTPS URLs")
|
raise SystemExit("CI pip indexes must be valid HTTPS URLs")
|
||||||
for index in indexes:
|
for index in indexes:
|
||||||
print("Effective CI Job pip index host:", host(index), flush=True)
|
print(f"Effective {label} pip index host:", host(index), flush=True)
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
[
|
[
|
||||||
sys.executable,
|
sys.executable,
|
||||||
@@ -99,8 +102,7 @@ def main():
|
|||||||
"30",
|
"30",
|
||||||
"--index-url",
|
"--index-url",
|
||||||
index,
|
index,
|
||||||
"-e",
|
*(["-r", "requirements.lock"] if locked else ["-e", ".[test]"]),
|
||||||
".[test]",
|
|
||||||
],
|
],
|
||||||
env=install_env(index),
|
env=install_env(index),
|
||||||
stdout=subprocess.PIPE,
|
stdout=subprocess.PIPE,
|
||||||
|
|||||||
@@ -364,6 +364,12 @@ class Deploy:
|
|||||||
command = ["build", "--pull", "-t", image]
|
command = ["build", "--pull", "-t", image]
|
||||||
now = datetime.now(UTC).isoformat()
|
now = datetime.now(UTC).isoformat()
|
||||||
for key, value in {
|
for key, value in {
|
||||||
|
"DOCKER_PIP_INDEX_URL": self.env.get("DOCKER_PIP_INDEX_URL")
|
||||||
|
or "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple",
|
||||||
|
"DOCKER_PIP_FALLBACK_INDEX_URL": self.env.get(
|
||||||
|
"DOCKER_PIP_FALLBACK_INDEX_URL"
|
||||||
|
)
|
||||||
|
or "https://pypi.org/simple",
|
||||||
"DEBIAN_MIRROR": self.env.get("DEBIAN_MIRROR") or "deb.debian.org",
|
"DEBIAN_MIRROR": self.env.get("DEBIAN_MIRROR") or "deb.debian.org",
|
||||||
"DEBIAN_SECURITY_MIRROR": self.env.get("DEBIAN_SECURITY_MIRROR")
|
"DEBIAN_SECURITY_MIRROR": self.env.get("DEBIAN_SECURITY_MIRROR")
|
||||||
or "deb.debian.org",
|
or "deb.debian.org",
|
||||||
|
|||||||
@@ -83,3 +83,27 @@ def test_sequential_fallback_keeps_editable_and_isolation(
|
|||||||
output = capsys.readouterr().out
|
output = capsys.readouterr().out
|
||||||
assert "fixture-secret" not in output and "fixture-token" not in output
|
assert "fixture-secret" not in output and "fixture-token" not in output
|
||||||
assert "mirror.example" in output and "pypi.org" in output
|
assert "mirror.example" in output and "pypi.org" in output
|
||||||
|
|
||||||
|
|
||||||
|
def test_docker_locked_install_has_independent_index_and_fallback(
|
||||||
|
installer, monkeypatch
|
||||||
|
):
|
||||||
|
monkeypatch.setattr(installer.sys, "argv", ["install.py", "--locked"])
|
||||||
|
monkeypatch.setenv("CI_PIP_INDEX_URL", "https://ci.example/simple")
|
||||||
|
monkeypatch.setenv("DOCKER_PIP_INDEX_URL", "https://docker.example/simple")
|
||||||
|
monkeypatch.setenv("DOCKER_PIP_FALLBACK_INDEX_URL", "https://pypi.org/simple")
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def run(command, **kwargs):
|
||||||
|
if "config" in command:
|
||||||
|
return SimpleNamespace(returncode=0, stdout="")
|
||||||
|
calls.append((command, kwargs["env"]))
|
||||||
|
return SimpleNamespace(returncode=0 if len(calls) == 2 else 1, stdout="")
|
||||||
|
|
||||||
|
monkeypatch.setattr(installer.subprocess, "run", run)
|
||||||
|
installer.main()
|
||||||
|
assert [env["PIP_INDEX_URL"] for _, env in calls] == [
|
||||||
|
"https://docker.example/simple",
|
||||||
|
"https://pypi.org/simple",
|
||||||
|
]
|
||||||
|
assert all(command[-2:] == ["-r", "requirements.lock"] for command, _ in calls)
|
||||||
|
|||||||
Reference in New Issue
Block a user