e2c34d9553
arc-runner-set pods have no docker daemon and cannot run a privileged DinD sidecar under cluster policy, so Set up Buildx failed to connect to /var/run/docker.sock. Point Buildx at a persistent rootless buildkitd Service (driver: remote, endpoint from the BUILDKIT_ENDPOINT secret) and add its non-privileged Kubernetes manifest. Infra only; no application code changes.
124 lines
3.8 KiB
YAML
124 lines
3.8 KiB
YAML
# Persistent, non-privileged BuildKit for the publish-images workflow.
|
|
#
|
|
# WHY this exists: the arc-runner-set runner pods have the docker/buildx CLIs
|
|
# but no Docker daemon and cannot run a privileged DinD sidecar under cluster
|
|
# policy. So image builds are delegated to this standalone buildkitd Service.
|
|
# The workflow (.github/workflows/publish-images.yml) points Buildx at it with
|
|
# driver: remote
|
|
# endpoint: ${{ secrets.BUILDKIT_ENDPOINT }}
|
|
# where BUILDKIT_ENDPOINT is set to:
|
|
# tcp://buildkit.buildkit.svc.cluster.local:1234
|
|
#
|
|
# This runs rootless buildkitd, so it needs NO privileged: true. It only needs
|
|
# a few unconfined-ish knobs the rootless image documents. A persistent
|
|
# Deployment (not a per-job container) means the build cache in the PVC survives
|
|
# across runs, so the heavy frontend npm build is only paid in full once.
|
|
#
|
|
# Cameron: apply once with `kubectl apply -f deploy/ci/buildkit-remote.yaml`.
|
|
# This changes cluster infra only. It does NOT modify any application code or
|
|
# any branch of the repo.
|
|
---
|
|
apiVersion: v1
|
|
kind: Namespace
|
|
metadata:
|
|
name: buildkit
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: buildkit
|
|
namespace: buildkit
|
|
labels:
|
|
app: buildkit
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: buildkit
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: buildkit
|
|
annotations:
|
|
# Required by rootless buildkitd to unshare user/network namespaces
|
|
# without full privilege. These are the values from the upstream
|
|
# rootless example; they are far weaker than privileged: true.
|
|
container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined
|
|
spec:
|
|
containers:
|
|
- name: buildkitd
|
|
image: moby/buildkit:v0.16.0-rootless
|
|
args:
|
|
- --addr
|
|
- unix:///run/user/1000/buildkit/buildkitd.sock
|
|
- --addr
|
|
- tcp://0.0.0.0:1234
|
|
# TLS is terminated in-cluster; the Service is only reachable from
|
|
# runner pods on the internal network. If you want mTLS between the
|
|
# runner and buildkitd, mount certs here and add --tlscacert etc,
|
|
# then set the endpoint secret to the tcp:// address with certs.
|
|
- --oci-worker-no-process-sandbox
|
|
readinessProbe:
|
|
exec:
|
|
command:
|
|
- buildctl
|
|
- debug
|
|
- workers
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 30
|
|
livenessProbe:
|
|
exec:
|
|
command:
|
|
- buildctl
|
|
- debug
|
|
- workers
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 30
|
|
securityContext:
|
|
# Rootless: no privilege escalation, runs as uid 1000. The seccomp
|
|
# and AppArmor unconfined settings below are what rootless needs in
|
|
# place of privileged: true.
|
|
runAsUser: 1000
|
|
runAsGroup: 1000
|
|
allowPrivilegeEscalation: false
|
|
seccompProfile:
|
|
type: Unconfined
|
|
ports:
|
|
- containerPort: 1234
|
|
name: buildkit
|
|
volumeMounts:
|
|
- name: buildkit-cache
|
|
mountPath: /home/user/.local/share/buildkit
|
|
volumes:
|
|
- name: buildkit-cache
|
|
persistentVolumeClaim:
|
|
claimName: buildkit-cache
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: buildkit-cache
|
|
namespace: buildkit
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
resources:
|
|
requests:
|
|
# Layer + build cache for the app image. Bump if the cache fills.
|
|
storage: 20Gi
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: buildkit
|
|
namespace: buildkit
|
|
spec:
|
|
selector:
|
|
app: buildkit
|
|
ports:
|
|
- name: buildkit
|
|
port: 1234
|
|
targetPort: 1234
|
|
# ClusterIP only: reachable from runner pods, not exposed outside the cluster.
|
|
type: ClusterIP
|