operator: drop rbac-proxy in favor of controller-runtime's authz/authn

rbac-proxy will be deprecated in 2025

Signed-off-by: Tuomas Katila <[email protected]>

Author: tkatila

cmd/operator/main.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/klog/v2/textlogger"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -97,12 +98,45 @@ func contains(arr []string, val string) bool {
 	return false
 }
 
+func createTLSCfgs(enableHTTP2 bool) []func(*tls.Config) {
+	tlsCfgFuncs := []func(*tls.Config){
+		func(cfg *tls.Config) {
+			cfg.MinVersion = tls.VersionTLS12
+			cfg.MaxVersion = tls.VersionTLS12
+			cfg.CipherSuites = []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+			}
+		},
+	}
+
+	// if the enable-http2 flag is false (the default), http/2 should be disabled
+	// due to its vulnerabilities. More specifically, disabling http/2 will
+	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
+	// Rapid Reset CVEs. For more information see:
+	// - https://github.com/advisories/GHSA-qppj-fm5r-hxr3
+	// - https://github.com/advisories/GHSA-4374-p667-p6c8
+	disableHTTP2 := func(cfg *tls.Config) {
+		setupLog.Info("disabling http/2")
+
+		cfg.NextProtos = []string{"http/1.1"}
+	}
+
+	if !enableHTTP2 {
+		tlsCfgFuncs = append(tlsCfgFuncs, disableHTTP2)
+	}
+
+	return tlsCfgFuncs
+}
+
 func main() {
 	var (
 		metricsAddr           string
 		probeAddr             string
 		devicePluginNamespace string
 		enableLeaderElection  bool
+		enableHTTP2           bool
+		secureMetrics         bool
 		pm                    *patcher.Manager
 	)
 
@@ -115,6 +149,9 @@ func main() {
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
+	flag.BoolVar(&secureMetrics, "metrics-secure", false, "If set the metrics endpoint is served securely")
+	flag.BoolVar(&enableHTTP2, "enable-http2", false,
+		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	flag.Var(&devices, "devices", "Device(s) to set up.")
 	flag.Parse()
 
@@ -134,27 +171,33 @@ func main() {
 		"sgx":  sgx.SetupReconciler,
 	}
 
-	tlsCfgFunc := func(cfg *tls.Config) {
-		cfg.MinVersion = tls.VersionTLS12
-		cfg.MaxVersion = tls.VersionTLS12
-		cfg.CipherSuites = []uint16{
-			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
-			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-		}
+	tlsCfgFuncs := createTLSCfgs(enableHTTP2)
+
+	webhookServer := webhook.NewServer(webhook.Options{
+		TLSOpts: tlsCfgFuncs,
+	})
+
+	// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
+	// More info:
+	// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/server
+	// - https://book.kubebuilder.io/reference/metrics.html
+	metricsServerOptions := metricsserver.Options{
+		BindAddress:   metricsAddr,
+		SecureServing: secureMetrics,
+		TLSOpts:       tlsCfgFuncs,
 	}
 
-	webhookOptions := webhook.Options{
-		Port: 9443,
-		TLSOpts: []func(*tls.Config){
-			tlsCfgFunc,
-		},
+	if secureMetrics {
+		// More info:
+		// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
+		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
-		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
+		Metrics:                metricsServerOptions,
 		Logger:                 ctrl.Log.WithName("intel-device-plugins-manager"),
-		WebhookServer:          webhook.NewServer(webhookOptions),
+		WebhookServer:          webhookServer,
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "d1c7b6d5.intel.com",

deployments/operator/default/kustomization.yaml

@@ -18,17 +18,19 @@ resources:
 - ../manager
 - ../webhook
 - ../certmanager
+# [METRICS] Expose the controller manager metrics service.
+- metrics_service.yaml
+
 
 patches:
-  # Protect the /metrics endpoint by putting it behind auth.
-  # If you want your controller-manager to expose the /metrics
-  # endpoint w/o any authn/z, please comment the following line.
-- path: manager_auth_proxy_patch.yaml
+# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
+# More info: https://book.kubebuilder.io/reference/metrics
+- path: manager_metrics_patch.yaml
   target:
-    name: controller-manager
-  # Enable webhook
+    kind: Deployment
 - path: manager_webhook_patch.yaml
   target:
+    kind: Deployment
     name: controller-manager
   # Enable certmanager integration
 - path: webhookcainjection_patch_mutate.yaml

deployments/operator/default/manager_auth_proxy_patch.yaml

@@ -0,0 +1,7 @@
+# This patch adds the args to allow exposing the metrics endpoint using HTTPS
+- op: add
+  path: /spec/template/spec/containers/0/args/0
+  value: "--metrics-bind-address=:8443"
+- op: add
+  path: /spec/template/spec/containers/0/args/0
+  value: "--metrics-secure"

deployments/operator/default/manager_metrics_patch.yaml

@@ -9,7 +9,7 @@ spec:
   ports:
   - name: https
     port: 8443
-    targetPort: https
+    protocol: TCP
+    targetPort: 8443
   selector:
     control-plane: controller-manager
-    manager: intel-deviceplugin-operator

deployments/operator/rbac/auth_proxy_service.yaml renamed to deployments/operator/default/metrics_service.yaml

@@ -30,6 +30,9 @@ spec:
       - image: docker.io/intel/intel-deviceplugin-operator:devel
         imagePullPolicy: IfNotPresent
         name: manager
+        args:
+          - "--health-probe-bind-address=:8081"
+          - "--leader-elect"
         livenessProbe:
           httpGet:
             path: /healthz

deployments/operator/manager/manager.yaml

@@ -0,0 +1,23 @@
+# This NetworkPolicy allows ingress traffic
+# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those
+# namespaces are able to gathering data from the metrics endpoint.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-metrics-traffic
+  namespace: system
+spec:
+  podSelector:
+    matchLabels:
+      control-plane: controller-manager
+  policyTypes:
+    - Ingress
+  ingress:
+    # This allows ingress traffic from any namespace with the label metrics: enabled
+    - from:
+      - namespaceSelector:
+          matchLabels:
+            metrics: enabled  # Only from namespaces with this label
+      ports:
+        - port: 8443
+          protocol: TCP

deployments/operator/network-policy/allow-metrics-traffic.yaml

@@ -0,0 +1,23 @@
+# This NetworkPolicy allows ingress traffic to your webhook server running
+# as part of the controller-manager from specific namespaces and pods. CR(s) which uses webhooks
+# will only work when applied in namespaces labeled with 'webhook: enabled'
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-webhook-traffic
+  namespace: system
+spec:
+  podSelector:
+    matchLabels:
+      control-plane: controller-manager
+  policyTypes:
+    - Ingress
+  ingress:
+    # This allows ingress traffic from any namespace with the label webhook: enabled
+    - from:
+      - namespaceSelector:
+          matchLabels:
+            webhook: enabled # Only from namespaces with this label
+      ports:
+        - port: 443
+          protocol: TCP

deployments/operator/network-policy/allow-webhook-traffic.yaml

@@ -0,0 +1,3 @@
+resources:
+- allow-webhook-traffic.yaml
+- allow-metrics-traffic.yaml

deployments/operator/network-policy/kustomization.yaml

@@ -4,10 +4,12 @@ resources:
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
 - gpu_manager_role.yaml
-# Comment the following 4 lines if you want to disable
-# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-# which protects your /metrics endpoint.
-- auth_proxy_service.yaml
-- auth_proxy_role.yaml
-- auth_proxy_role_binding.yaml
-- auth_proxy_client_clusterrole.yaml
+# The following RBAC configurations are used to protect
+# the metrics endpoint with authn/authz. These configurations
+# ensure that only authorized users and service accounts
+# can access the metrics endpoint. Comment the following
+# permissions if you want to disable this protection.
+# More info: https://book.kubebuilder.io/reference/metrics.html
+- metrics_auth_role.yaml
+- metrics_auth_role_binding.yaml
+- metrics_reader_role.yaml