From c19b2b2af4cc3bb010d363a7c3d7116f992eff2d Mon Sep 17 00:00:00 2001 From: Brad Rodgers Date: Tue, 22 Sep 2026 15:45:57 -0400 Subject: [PATCH] ci: install docker + kustomize as static binaries, not distro packages CI has been failing silently since 2026-08-03. The run for 9f1cbca did execute (Gitea action_run #6, status=failure, 13m) but died before pushing an image, so no package version exists for that commit and ArgoCD has been serving the 2026-07-20 build ever since. Cause: the "Ensure docker + kustomize are available" step ran `apt-get install -y docker.io`. The runner's job image was node:18-bullseye and Debian 11 went EOL, so debian-security stopped serving those package versions. `apt-get update` still succeeds, so the failure surfaces only at install time as 404s on docker.io, containerd, systemd and libcap2. Fetch both tools as pinned static binaries instead. These have no distro archive dependency, so the same rot cannot recur when bookworm ages out. Verified in the real job environment: the static client installs and reaches the host daemon through the mounted socket. The runner job image has separately been moved to node:22-bookworm, which also resolves the immediate breakage, but the static-binary approach is kept as the durable fix. Also refresh the prerequisites header: items 3 and 4 still described the pre-ADR-0008 registry design (k3s now pulls over public HTTPS, and the private package's pull auth comes from External Secrets). --- .gitea/workflows/ci.yml | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 69b2f6b..7a20c09 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -8,10 +8,10 @@ # jobs can't see it, enable socket passthrough in act_runner config or switch this to kaniko. # 2. The RUNNER HOST's dockerd must treat 10.66.15.22:3000 as an insecure (HTTP) registry # ("insecure-registries": ["10.66.15.22:3000"] in /etc/docker/daemon.json). -# 3. K3S must trust the same HTTP registry — add a registries.yaml entry (k3s_registry_mirrors) -# for 10.66.15.22:3000, then restart k3s, or the pods can't pull the image. -# 4. The Gitea container package should be PUBLIC (simplest). If PRIVATE, add an imagePullSecret -# (see k8s/deployment.yaml) and grant the run token package:write. +# 3. K3S does NOT need registries.yaml — it pulls the same package over public HTTPS +# (git.7tl-homelab.com, valid cert). See ADR-0008. No node config, no cluster restart. +# 4. The Gitea package is PRIVATE; k3s pull auth comes from External Secrets -> +# k8s/externalsecret-registry.yaml -> `gitea-registry` imagePullSecret. See ADR-0008. # 5. secrets.GITEA_TOKEN (auto-injected per run) needs write access to this repo (tag-bump commit) # and the package registry. @@ -61,12 +61,32 @@ jobs: id: tag run: echo "sha=$(echo ${{ github.sha }} | cut -c1-12)" >> "$GITHUB_OUTPUT" + # Both tools are fetched as STATIC BINARIES, never from distro packages. + # History: this step used `apt-get install -y docker.io`, which broke CI on 2026-08-03 when + # Debian 11 (bullseye — the runner's job image is node:18-bullseye) went EOL and the + # debian-security pool stopped serving those package versions. `apt-get update` still + # succeeds, so the failure surfaces only at install time as a wall of 404s. Static binaries + # have no distro-archive dependency and pin an exact version, so this cannot rot the same way. - name: Ensure docker + kustomize are available + env: + DOCKER_CLI_VERSION: '28.3.2' + KUSTOMIZE_VERSION: 'v5.4.3' run: | - command -v docker >/dev/null 2>&1 || { apt-get update && apt-get install -y docker.io; } - command -v kustomize >/dev/null 2>&1 || \ - curl -sSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.4.3/kustomize_v5.4.3_linux_amd64.tar.gz" \ + set -euo pipefail + if ! command -v docker >/dev/null 2>&1; then + curl -sSL --retry 3 --retry-delay 5 \ + "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz" -o /tmp/docker.tgz + # We only need the client; the daemon is the runner host's, via the mounted socket. + tar xzf /tmp/docker.tgz -C /tmp --strip-components=1 docker/docker + install -m 0755 /tmp/docker /usr/local/bin/docker + fi + if ! command -v kustomize >/dev/null 2>&1; then + curl -sSL --retry 3 --retry-delay 5 \ + "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" \ | tar xz -C /usr/local/bin + fi + docker version --format 'docker client={{.Client.Version}} server={{.Server.Version}}' + kustomize version # CI_TOKEN: a Gitea Personal Access Token (scopes: write:package, write:repository) owned by # the repo owner, stored as a repo Actions secret. The auto GITEA_TOKEN lacks reliable package