build images via remote BuildKit instead of a local docker daemon

arc-runner-set pods have no docker daemon and cannot run a privileged DinD sidecar under cluster policy, so Set up Buildx failed to connect to /var/run/docker.sock. Point Buildx at a persistent rootless buildkitd Service (driver: remote, endpoint from the BUILDKIT_ENDPOINT secret) and add its non-privileged Kubernetes manifest. Infra only; no application code changes.

.github/workflows/publish-images.yml

@@ -64,8 +64,18 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set up Buildx
+      # Connect to a persistent, out-of-cluster BuildKit deployment instead of
+      # spinning up a builder against a local Docker daemon. The arc-runner-set
+      # pods have no Docker daemon (no /var/run/docker.sock) and cannot run a
+      # privileged DinD sidecar under cluster policy, so we point Buildx at a
+      # remote buildkitd Service. The endpoint is provided as a repo/org secret
+      # (e.g. tcp://buildkit.buildkit.svc.cluster.local:1234), so it is not
+      # hardcoded here. See deploy/ci/buildkit-remote.yaml for the deployment.
+      - name: Set up Buildx (remote BuildKit)
         uses: docker/setup-buildx-action@v3
+        with:
+          driver: remote
+          endpoint: ${{ secrets.BUILDKIT_ENDPOINT }}
 
       - name: Build and push channel image
         uses: docker/build-push-action@v6

deploy/ci/buildkit-remote.yaml

@@ -0,0 +1,123 @@
+# Persistent, non-privileged BuildKit for the publish-images workflow.
+#
+# WHY this exists: the arc-runner-set runner pods have the docker/buildx CLIs
+# but no Docker daemon and cannot run a privileged DinD sidecar under cluster
+# policy. So image builds are delegated to this standalone buildkitd Service.
+# The workflow (.github/workflows/publish-images.yml) points Buildx at it with
+#   driver: remote
+#   endpoint: ${{ secrets.BUILDKIT_ENDPOINT }}
+# where BUILDKIT_ENDPOINT is set to:
+#   tcp://buildkit.buildkit.svc.cluster.local:1234
+#
+# This runs rootless buildkitd, so it needs NO privileged: true. It only needs
+# a few unconfined-ish knobs the rootless image documents. A persistent
+# Deployment (not a per-job container) means the build cache in the PVC survives
+# across runs, so the heavy frontend npm build is only paid in full once.
+#
+# Cameron: apply once with `kubectl apply -f deploy/ci/buildkit-remote.yaml`.
+# This changes cluster infra only. It does NOT modify any application code or
+# any branch of the repo.
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: buildkit
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: buildkit
+  namespace: buildkit
+  labels:
+    app: buildkit
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: buildkit
+  template:
+    metadata:
+      labels:
+        app: buildkit
+      annotations:
+        # Required by rootless buildkitd to unshare user/network namespaces
+        # without full privilege. These are the values from the upstream
+        # rootless example; they are far weaker than privileged: true.
+        container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined
+    spec:
+      containers:
+        - name: buildkitd
+          image: moby/buildkit:v0.16.0-rootless
+          args:
+            - --addr
+            - unix:///run/user/1000/buildkit/buildkitd.sock
+            - --addr
+            - tcp://0.0.0.0:1234
+            # TLS is terminated in-cluster; the Service is only reachable from
+            # runner pods on the internal network. If you want mTLS between the
+            # runner and buildkitd, mount certs here and add --tlscacert etc,
+            # then set the endpoint secret to the tcp:// address with certs.
+            - --oci-worker-no-process-sandbox
+          readinessProbe:
+            exec:
+              command:
+                - buildctl
+                - debug
+                - workers
+            initialDelaySeconds: 5
+            periodSeconds: 30
+          livenessProbe:
+            exec:
+              command:
+                - buildctl
+                - debug
+                - workers
+            initialDelaySeconds: 5
+            periodSeconds: 30
+          securityContext:
+            # Rootless: no privilege escalation, runs as uid 1000. The seccomp
+            # and AppArmor unconfined settings below are what rootless needs in
+            # place of privileged: true.
+            runAsUser: 1000
+            runAsGroup: 1000
+            allowPrivilegeEscalation: false
+            seccompProfile:
+              type: Unconfined
+          ports:
+            - containerPort: 1234
+              name: buildkit
+          volumeMounts:
+            - name: buildkit-cache
+              mountPath: /home/user/.local/share/buildkit
+      volumes:
+        - name: buildkit-cache
+          persistentVolumeClaim:
+            claimName: buildkit-cache
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: buildkit-cache
+  namespace: buildkit
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      # Layer + build cache for the app image. Bump if the cache fills.
+      storage: 20Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: buildkit
+  namespace: buildkit
+spec:
+  selector:
+    app: buildkit
+  ports:
+    - name: buildkit
+      port: 1234
+      targetPort: 1234
+  # ClusterIP only: reachable from runner pods, not exposed outside the cluster.
+  type: ClusterIP