Merge pull request #140 from Leaseweb/node_sa_deploy_manifest

feat: Use SA for node daemonset in deploy manifest and other improvem…

Author: hrak

deploy/k8s/controller-deployment.yaml

@@ -23,14 +23,39 @@ spec:
     spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: cloudstack-csi-controller
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: "app.kubernetes.io/name"
+                operator: In
+                values:
+                - cloudstack-csi-controller
+            topologyKey: "kubernetes.io/hostname"
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
       nodeSelector:
         kubernetes.io/os: linux
-        node-role.kubernetes.io/control-plane: ""
       tolerations:
-        - effect: NoExecute
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: CriticalAddonsOnly
           operator: Exists
-        - effect: NoSchedule
+        - effect: NoExecute
           operator: Exists
+          tolerationSeconds: 300
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        fsGroupChangePolicy: OnRootMismatch
 
       containers:
         - name: cloudstack-csi-controller
@@ -45,10 +70,6 @@ spec:
           env:
             - name: CSI_ENDPOINT
               value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
-          securityContext:
-            runAsNonRoot: true
-            runAsUser: 65532
-            runAsGroup: 65532
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
@@ -62,10 +83,21 @@ spec:
             httpGet:
               path: /healthz
               port: healthz
-            initialDelaySeconds: 30
-            timeoutSeconds: 10
-            periodSeconds: 180
-            failureThreshold: 3
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+          resources:
+            requests:
+              cpu: 10m
+              memory: 40Mi
+            limits:
+              memory: 256Mi
+          securityContext:
+            seccompProfile:
+              type: RuntimeDefault
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
 
         - name: external-provisioner
           image: registry.k8s.io/sig-storage/csi-provisioner:v5.0.1
@@ -89,6 +121,11 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
+          securityContext:
+            seccompProfile:
+              type: RuntimeDefault
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
 
         - name: external-attacher
           image: registry.k8s.io/sig-storage/csi-attacher:v4.6.1
@@ -109,6 +146,11 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
+          securityContext:
+            seccompProfile:
+              type: RuntimeDefault
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
 
         - name: external-resizer
           image: registry.k8s.io/sig-storage/csi-resizer:v1.11.1
@@ -129,6 +171,11 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
+          securityContext:
+            seccompProfile:
+              type: RuntimeDefault
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
 
         - name: liveness-probe
           image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0
@@ -141,6 +188,9 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
+          securityContext:
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
 
       volumes:
         - name: socket-dir

deploy/k8s/node-daemonset.yaml

@@ -9,25 +9,35 @@ spec:
       app.kubernetes.io/name: cloudstack-csi-node
   updateStrategy:
     type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: "10%"
   template:
     metadata:
       labels:
         app.kubernetes.io/name: cloudstack-csi-node
         app.kubernetes.io/part-of: cloudstack-csi-driver
     spec:
       priorityClassName: system-node-critical
+      serviceAccountName: cloudstack-csi-node
       nodeSelector:
         kubernetes.io/os: linux
+      terminationGracePeriodSeconds: 30
       tolerations:
-        - effect: NoExecute
-          operator: Exists
         - effect: NoSchedule
           operator: Exists
+        - effect: NoExecute
+          operator: Exists
+          tolerationSeconds: 300
+      securityContext:
+        runAsNonRoot: false
+        runAsUser: 0
+        runAsGroup: 0
+        fsGroup: 0
 
       containers:
         - name: cloudstack-csi-node
           image: cloudstack-csi-driver
-          imagePullPolicy: Always
+          imagePullPolicy: IfNotPresent
           args:
             - "node"
             - "--endpoint=$(CSI_ENDPOINT)"
@@ -42,11 +52,6 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
-          securityContext:
-            privileged: true
-            capabilities:
-              add: ["SYS_ADMIN"]
-            allowPrivilegeEscalation: true
           volumeMounts:
             - name: plugin-dir
               mountPath: /csi
@@ -59,6 +64,9 @@ spec:
               mountPath: /dev
             - name: cloud-init-dir
               mountPath: /run/cloud-init/
+            # Comment the above 2 lines and uncomment the next 2 lines for Ignition support
+            # - name: ignition-dir
+            #   mountPath: /run/metadata
             - name: cloudstack-conf
               mountPath: /etc/cloudstack-csi-driver
           ports:
@@ -70,21 +78,19 @@ spec:
               path: /healthz
               port: healthz
             initialDelaySeconds: 10
-            timeoutSeconds: 5
-            periodSeconds: 5
-            failureThreshold: 3
-
-        - name: liveness-probe
-          image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0
-          args:
-            - "--v=4"
-            - "--csi-address=$(ADDRESS)"
-          env:
-            - name: ADDRESS
-              value: /csi/csi.sock
-          volumeMounts:
-            - name: plugin-dir
-              mountPath: /csi
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+          resources:
+            limits:
+              cpu: "200m"
+              memory: 200Mi
+            requests:
+              cpu: "50m"
+              memory: 50Mi
+          securityContext:
+            # readOnlyRootFilesystem: true
+            privileged: true
 
         - name: node-driver-registrar
           image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.1
@@ -94,15 +100,6 @@ spec:
             - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
             - "--v=5"
             - "--health-port=9809"
-          livenessProbe:
-            httpGet:
-              path: /healthz
-              port: healthz
-            failureThreshold: 5
-            initialDelaySeconds: 60
-            periodSeconds: 10
-            successThreshold: 1
-            timeoutSeconds: 15
           env:
             - name: ADDRESS
               value: /csi/csi.sock
@@ -116,6 +113,48 @@ spec:
           ports:
             - containerPort: 9809
               name: healthz
+          livenessProbe:
+            exec:
+              command:
+                - /csi-node-driver-registrar
+                - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+                - --mode=kubelet-registration-probe
+            initialDelaySeconds: 30
+            periodSeconds: 90
+            timeoutSeconds: 15
+          resources:
+            limits:
+              cpu: "200m"
+              memory: 200Mi
+            requests:
+              cpu: "50m"
+              memory: 50Mi
+          securityContext:
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
+
+        - name: liveness-probe
+          image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - "--v=4"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+          resources:
+            limits:
+              cpu: "200m"
+              memory: 250Mi
+            requests:
+              cpu: "50m"
+              memory: 50Mi
+          securityContext:
+            readOnlyRootFilesystem: true
+            allowPrivilegeEscalation: false
 
       volumes:
         - name: plugin-dir
@@ -138,6 +177,11 @@ spec:
           hostPath:
             path: /run/cloud-init/
             type: Directory
+        # Comment the above 4 lines and uncomment the next 4 lines for Ignition support
+        # - name: ignition-dir
+        #   hostPath:
+        #     path: /run/metadata
+        #     type: Directory
         - name: cloudstack-conf
           secret:
             secretName: cloudstack-secret

deploy/k8s/rbac.yaml

@@ -49,3 +49,37 @@ roleRef:
   kind: ClusterRole
   name: cloudstack-csi-controller-role
   apiGroup: rbac.authorization.k8s.io
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: cloudstack-csi-node
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cloudstack-csi-node-role
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "patch"]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "volumeattachments" ]
+    verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "csinodes" ]
+    verbs: [ "get" ]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cloudstack-csi-node-binding
+subjects:
+  - kind: ServiceAccount
+    name: cloudstack-csi-node
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: cloudstack-csi-node-role
+  apiGroup: rbac.authorization.k8s.io