diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index 16bea88a..410ab4a8 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -64,8 +64,18 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Buildx + # Connect to a persistent, out-of-cluster BuildKit deployment instead of + # spinning up a builder against a local Docker daemon. The arc-runner-set + # pods have no Docker daemon (no /var/run/docker.sock) and cannot run a + # privileged DinD sidecar under cluster policy, so we point Buildx at a + # remote buildkitd Service. The endpoint is provided as a repo/org secret + # (e.g. tcp://buildkit.buildkit.svc.cluster.local:1234), so it is not + # hardcoded here. See deploy/ci/buildkit-remote.yaml for the deployment. + - name: Set up Buildx (remote BuildKit) uses: docker/setup-buildx-action@v3 + with: + driver: remote + endpoint: ${{ secrets.BUILDKIT_ENDPOINT }} - name: Build and push channel image uses: docker/build-push-action@v6 diff --git a/deploy/ci/buildkit-remote.yaml b/deploy/ci/buildkit-remote.yaml new file mode 100644 index 00000000..654d9d27 --- /dev/null +++ b/deploy/ci/buildkit-remote.yaml @@ -0,0 +1,123 @@ +# Persistent, non-privileged BuildKit for the publish-images workflow. +# +# WHY this exists: the arc-runner-set runner pods have the docker/buildx CLIs +# but no Docker daemon and cannot run a privileged DinD sidecar under cluster +# policy. So image builds are delegated to this standalone buildkitd Service. +# The workflow (.github/workflows/publish-images.yml) points Buildx at it with +# driver: remote +# endpoint: ${{ secrets.BUILDKIT_ENDPOINT }} +# where BUILDKIT_ENDPOINT is set to: +# tcp://buildkit.buildkit.svc.cluster.local:1234 +# +# This runs rootless buildkitd, so it needs NO privileged: true. It only needs +# a few unconfined-ish knobs the rootless image documents. A persistent +# Deployment (not a per-job container) means the build cache in the PVC survives +# across runs, so the heavy frontend npm build is only paid in full once. +# +# Cameron: apply once with `kubectl apply -f deploy/ci/buildkit-remote.yaml`. +# This changes cluster infra only. It does NOT modify any application code or +# any branch of the repo. +--- +apiVersion: v1 +kind: Namespace +metadata: + name: buildkit +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: buildkit + namespace: buildkit + labels: + app: buildkit +spec: + replicas: 1 + selector: + matchLabels: + app: buildkit + template: + metadata: + labels: + app: buildkit + annotations: + # Required by rootless buildkitd to unshare user/network namespaces + # without full privilege. These are the values from the upstream + # rootless example; they are far weaker than privileged: true. + container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined + spec: + containers: + - name: buildkitd + image: moby/buildkit:v0.16.0-rootless + args: + - --addr + - unix:///run/user/1000/buildkit/buildkitd.sock + - --addr + - tcp://0.0.0.0:1234 + # TLS is terminated in-cluster; the Service is only reachable from + # runner pods on the internal network. If you want mTLS between the + # runner and buildkitd, mount certs here and add --tlscacert etc, + # then set the endpoint secret to the tcp:// address with certs. + - --oci-worker-no-process-sandbox + readinessProbe: + exec: + command: + - buildctl + - debug + - workers + initialDelaySeconds: 5 + periodSeconds: 30 + livenessProbe: + exec: + command: + - buildctl + - debug + - workers + initialDelaySeconds: 5 + periodSeconds: 30 + securityContext: + # Rootless: no privilege escalation, runs as uid 1000. The seccomp + # and AppArmor unconfined settings below are what rootless needs in + # place of privileged: true. + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: false + seccompProfile: + type: Unconfined + ports: + - containerPort: 1234 + name: buildkit + volumeMounts: + - name: buildkit-cache + mountPath: /home/user/.local/share/buildkit + volumes: + - name: buildkit-cache + persistentVolumeClaim: + claimName: buildkit-cache +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: buildkit-cache + namespace: buildkit +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + # Layer + build cache for the app image. Bump if the cache fills. + storage: 20Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: buildkit + namespace: buildkit +spec: + selector: + app: buildkit + ports: + - name: buildkit + port: 1234 + targetPort: 1234 + # ClusterIP only: reachable from runner pods, not exposed outside the cluster. + type: ClusterIP