k8s-ds-amdgpu-dp-health.yaml

# TODO device plugin currently uses /dev/kfd to do health check
# which requires privileged container
# Might need to find better alternatives
#
# ***** this requires --allow-privileged=true set for both kube-apiserver and kubelet
#
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: amdgpu-device-plugin-daemonset
  namespace: kube-system
spec:
  template:
    metadata:
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ""
      labels:
        name: amdgpu-dp-ds
    spec:
      tolerations:
      - key: CriticalAddonsOnly
        operator: Exists
      containers:
      - image: rocm/k8s-device-plugin
        name: amdgpu-dp-cntr-health
        workingDir: /root
        command: ["./k8s-device-plugin"]
        args: ["-logtostderr=true", "-stderrthreshold=INFO", "-v=5", "-pulse=2"]
        securityContext:
          privileged: true #Needed for /dev
          capabilities:
            drop: ["ALL"]
        volumeMounts:
          - name: dp
            mountPath: /var/lib/kubelet/device-plugins
          - name: sys
            mountPath: /sys
          - name: dev
            mountPath: /dev
      volumes:
        - name: dp
          hostPath:
            path: /var/lib/kubelet/device-plugins
        - name: sys
          hostPath:
            path: /sys
        - name: dev # Needed for health check
          hostPath:
            path: /dev