From d88c88f8762c3e4ab0710a7fef1b006c84824392 Mon Sep 17 00:00:00 2001 From: Pigway Date: Wed, 7 Oct 2026 17:37:33 +0800 Subject: [PATCH] Configure mainland build mirrors and persist BuildKit apt and pip caches --- .dockerignore | 4 +++ .env.example | 5 ++++ .github/workflows/verify.yml | 7 ++++-- Dockerfile | 19 ++++++++++----- README.md | 47 ++++++++++++++++++++++++++++++++++++ deploy/build-cn.env | 5 ++++ docker-compose.yml | 2 ++ scripts/build-verify.sh | 2 ++ 8 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 deploy/build-cn.env diff --git a/.dockerignore b/.dockerignore index 453765d..f920761 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,3 +10,7 @@ config *.egg-info build dist + +.mypy_cache +**/__pycache__ +**/*.pyc diff --git a/.env.example b/.env.example index f84668e..cc9c131 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,8 @@ BUILD_TIME=unknown # Login in the photo-source editor; only the session is saved in /config (mode 0600). SYNOLOGY_PHOTOS_URL= SYNOLOGY_PHOTOS_CERT_SHA256= + +# Build-only mirror hosts; official defaults work outside mainland China. +# For mainland NAS builds: source ./deploy/build-cn.env before build-verify.sh. +DEBIAN_MIRROR=deb.debian.org +DEBIAN_SECURITY_MIRROR=deb.debian.org diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 2ee0045..a91b597 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -3,6 +3,9 @@ on: [push, pull_request, workflow_dispatch] jobs: verify: runs-on: ubuntu-latest + env: + DEBIAN_MIRROR: ${{ vars.DEBIAN_MIRROR }} + DEBIAN_SECURITY_MIRROR: ${{ vars.DEBIAN_SECURITY_MIRROR }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -22,8 +25,8 @@ jobs: p=Path('.env') p.write_text(p.read_text().replace('FRAME_FLOW_TOKEN=\n', 'FRAME_FLOW_TOKEN='+secrets.token_urlsafe(32)+'\n')) PY - docker compose config --quiet - docker compose build --pull + . ./deploy/build-cn.env + ./scripts/build-verify.sh mkdir -p data/images data/generated data/cache config sudo chown -R 10333:10333 data config docker compose up -d diff --git a/Dockerfile b/Dockerfile index 0bbfaf3..a5b0f36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,19 @@ ARG PYTHON_BASE=python:3.14.5-slim-bookworm@sha256:ec58d916f9e24a6035cab2bdf07f6206c4cc092a16613c60597534711332d9d6 FROM ${PYTHON_BASE} ARG DEBIAN_MIRROR=deb.debian.org +ARG DEBIAN_SECURITY_MIRROR=deb.debian.org +ARG TARGETARCH ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 TZ=Asia/Shanghai \ DATA_DIR=/data CONFIG_DIR=/config WORKDIR /app # CJK fonts intentionally retained for readable Chinese rendering. -RUN --mount=type=bind,source=build-cache/debs,target=/build-debs,readonly sed -i "s|http://deb.debian.org|https://${DEBIAN_MIRROR}|g" /etc/apt/sources.list.d/debian.sources \ - && apt-get update \ +RUN --mount=type=cache,id=frameflow-bookworm-apt-${TARGETARCH},target=/var/cache/apt,sharing=locked \ + --mount=type=cache,id=frameflow-bookworm-lists-${TARGETARCH},target=/var/lib/apt/lists,sharing=locked \ + --mount=type=bind,source=build-cache/debs,target=/build-debs,readonly \ + rm -f /etc/apt/apt.conf.d/docker-clean \ + && printf 'Binary::apt::APT::Keep-Downloaded-Packages "true";\n' > /etc/apt/apt.conf.d/keep-cache \ + && sed -i "s|http://deb.debian.org/debian-security|https://${DEBIAN_SECURITY_MIRROR}/debian-security|g; s|http://deb.debian.org/debian|https://${DEBIAN_MIRROR}/debian|g" /etc/apt/sources.list.d/debian.sources \ + && apt-get -o Acquire::Retries=3 -o Acquire::https::Timeout=60 update \ && if [ -f /build-debs/fonts-noto-cjk.deb ]; then \ font_version=$(apt-cache policy fonts-noto-cjk | awk '$1=="Candidate:" {print $2}'); \ font_sha=$(apt-cache show "fonts-noto-cjk=$font_version" | awk '$1=="SHA256:" {print $2;exit}'); \ @@ -15,16 +22,16 @@ RUN --mount=type=bind,source=build-cache/debs,target=/build-debs,readonly sed -i apt-get install -y --no-install-recommends tzdata; \ dpkg -i /build-debs/fonts-noto-cjk.deb; \ else apt-get install -y --no-install-recommends fonts-noto-cjk tzdata; fi \ - && rm -rf /var/lib/apt/lists/* \ && groupadd --gid 10333 frameflow && useradd --uid 10333 --gid 10333 --no-create-home frameflow \ && mkdir -p /data/images /data/generated /data/cache /config \ && chown -R 10333:10333 /data /config COPY requirements.lock ./ -RUN --mount=type=bind,source=build-cache/wheels,target=/build-wheels,readonly \ +RUN --mount=type=cache,id=frameflow-pip-py314-${TARGETARCH},target=/root/.cache/pip,sharing=locked \ + --mount=type=bind,source=build-cache/wheels,target=/build-wheels,readonly \ if [ -f /build-wheels/SHA256SUMS ]; then \ (cd /build-wheels && sha256sum -c SHA256SUMS) \ - && pip install --no-cache-dir --no-index --find-links=/build-wheels -r requirements.lock; \ - else pip install --no-cache-dir -r requirements.lock; fi + && pip install --no-index --find-links=/build-wheels -r requirements.lock; \ + else pip install -r requirements.lock; fi COPY pigway_frame_flow ./pigway_frame_flow COPY vendor ./vendor ARG BUILD_COMMIT=development diff --git a/README.md b/README.md index b197f4a..a243678 100644 --- a/README.md +++ b/README.md @@ -237,3 +237,50 @@ Gallery supports read-only NAS folders and HTTPS WebDAV sources, random photo preview, and device-bound scheduled random photos. Source definitions are persisted in SQLite. See [setup and limits](docs/PHOTO_SOURCES.md). NAS 本地目录需要显式只读挂载；网盘先支持通用 WebDAV，也可使用同步到 NAS 的目录。 + +### Build networking and cache + +The Dockerfile retains the pinned Python base, locked Python dependencies, +`fonts-noto-cjk`, Debian repository signature checks, and offline artifact +SHA256 verification. Both production and CI use `scripts/build-verify.sh`, +which enables BuildKit and retains `docker compose build --pull`. + +APT mirror hosts are configurable through `DEBIAN_MIRROR` and +`DEBIAN_SECURITY_MIRROR` (hosts only, without scheme or path). Defaults are +`deb.debian.org`; HTTPS is used for both Debian and security repositories. +For mainland-China NAS builds, use the shared profile: + +```sh +. ./deploy/build-cn.env +./scripts/build-verify.sh +``` + +CI loads this same profile, using Tsinghua TUNA for both repositories unless +Gitea repository variables `DEBIAN_MIRROR` / `DEBIAN_SECURITY_MIRROR` override +it. Production may also set the two hosts directly in `.env` when calling the +build script without the profile. An alternate mainland mirror is +`mirrors.ustc.edu.cn`; availability and throughput depend on the Runner network. +No fallback disables signature verification or removes packages. + +Layer order stays: pinned base → APT/font dependencies → `requirements.lock` +→ pip dependencies → application/vendor source → build metadata. +A source-only change therefore reuses the APT and pip layers, provided the +same builder, base digest, mirror arguments, and optional offline artifacts +are used. Changing the lockfile rebuilds pip and later layers, not APT. +Changing the mirror or APT instructions rebuilds APT and later layers. + +Previously an APT layer miss redownloaded indexes/packages, and pip used +`--no-cache-dir`. Now locked BuildKit mounts retain APT indexes, downloaded +`.deb` files, and pip downloads/wheels outside the final image. Debian's +`docker-clean` hook is disabled so it cannot discard cached packages. +Re-executed APT steps still run `apt-get update` and validate repository +signatures; cached packages and pip artifacts can avoid repeat large downloads. +The initial build after this Dockerfile change will miss dependency layers; +subsequent builds benefit. No measured speedup is claimed before CI completes. + +Caches belong to the Docker builder, not the ephemeral CI workspace. Keep the +NAS Runner's builder storage across jobs; removing the builder, pruning caches, +or using a fresh Docker daemon loses the benefit. Cache mounts are disposable: +an empty cache still builds correctly. `--pull` remains enabled; a pinned base +digest check alone does not invalidate dependency layers. To deliberately +refresh installed APT versions, use `docker compose build --pull --no-cache`. diff --git a/deploy/build-cn.env b/deploy/build-cn.env new file mode 100644 index 0000000..9c81363 --- /dev/null +++ b/deploy/build-cn.env @@ -0,0 +1,5 @@ +# Shared mainland-China build profile; source this file before Compose builds. +# Nonempty environment / Gitea repository variables override these defaults. +DEBIAN_MIRROR=${DEBIAN_MIRROR:-mirrors.tuna.tsinghua.edu.cn} +DEBIAN_SECURITY_MIRROR=${DEBIAN_SECURITY_MIRROR:-mirrors.tuna.tsinghua.edu.cn} +export DEBIAN_MIRROR DEBIAN_SECURITY_MIRROR diff --git a/docker-compose.yml b/docker-compose.yml index df8d6df..57bc420 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,8 @@ services: build: context: . args: + DEBIAN_MIRROR: ${DEBIAN_MIRROR:-deb.debian.org} + DEBIAN_SECURITY_MIRROR: ${DEBIAN_SECURITY_MIRROR:-deb.debian.org} BUILD_COMMIT: ${BUILD_COMMIT:-development} BUILD_TIME: ${BUILD_TIME:-unknown} restart: unless-stopped diff --git a/scripts/build-verify.sh b/scripts/build-verify.sh index b2a0a17..0d68fc6 100755 --- a/scripts/build-verify.sh +++ b/scripts/build-verify.sh @@ -1,5 +1,7 @@ #!/bin/sh set -eu +export DOCKER_BUILDKIT=1 +export COMPOSE_DOCKER_CLI_BUILD=1 cd "$(dirname "$0")/.." DOCKER_BIN=${DOCKER_BIN:-docker} "$DOCKER_BIN" info >/dev/null