Created InitContainer & resources for mtls patch

Bobbins228 · Bobbins228 · commit 05519da9ca57 · 2024-04-17T09:46:10.000+01:00
diff --git a/main.go b/main.go
@@ -47,6 +47,7 @@ import (
 	"sigs.k8s.io/yaml"
 
 	routev1 "github.com/openshift/api/route/v1"
+	routev1ClientSet "github.com/openshift/client-go/route/clientset/versioned"
 
 	"github.com/project-codeflare/codeflare-operator/pkg/config"
 	"github.com/project-codeflare/codeflare-operator/pkg/controllers"
@@ -56,12 +57,15 @@ import (
 )
 
 var (
-	scheme            = runtime.NewScheme()
-	setupLog          = ctrl.Log.WithName("setup")
-	OperatorVersion   = "UNKNOWN"
-	McadVersion       = "UNKNOWN"
-	InstaScaleVersion = "UNKNOWN"
-	BuildDate         = "UNKNOWN"
+	scheme                      = runtime.NewScheme()
+	setupLog                    = ctrl.Log.WithName("setup")
+	OperatorVersion             = "UNKNOWN"
+	McadVersion                 = "UNKNOWN"
+	InstaScaleVersion           = "UNKNOWN"
+	BuildDate                   = "UNKNOWN"
+	NameConsoleLink      string = "console"
+	NamespaceConsoleLink string = "openshift-console"
+	domain                      = ""
 )
 
 func init() {
@@ -147,8 +151,17 @@ func main() {
 	})
 	exitOnError(err, "unable to start manager")
 
+	if cfg.KubeRay.IngressDomain == "" {
+		routeClient, err := routev1ClientSet.NewForConfig(kubeConfig)
+		exitOnError(err, "unable to create Route Client Set")
+		domain, err = getOpenShiftDomainName(ctx, routeClient)
+		exitOnError(err, domain)
+	} else {
+		domain = cfg.KubeRay.IngressDomain
+	}
+
 	rayClusterDefaulter := &controllers.RayClusterDefaulter{}
-	exitOnError(rayClusterDefaulter.SetupWebhookWithManager(mgr), "error setting up webhook")
+	exitOnError(rayClusterDefaulter.SetupWebhookWithManager(mgr, domain), "error setting up webhook")
 
 	ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster"))
 	if ok {
@@ -248,3 +261,13 @@ func exitOnError(err error, msg string) {
 		os.Exit(1)
 	}
 }
+
+func getOpenShiftDomainName(ctx context.Context, routeClient routev1ClientSet.Interface) (string, error) {
+	route, err := routeClient.RouteV1().Routes(NamespaceConsoleLink).Get(ctx, NameConsoleLink, metav1.GetOptions{})
+	if err != nil {
+		return "error getting console route URL", err
+	}
+	domainIndex := strings.Index(route.Spec.Host, ".")
+	consoleLinkDomain := route.Spec.Host[domainIndex+1:]
+	return consoleLinkDomain, nil
+}
diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go
@@ -28,10 +28,14 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
-// log is for logging in this package.
-var rayclusterlog = logf.Log.WithName("raycluster-resource")
+var (
+	// log is for logging in this package.
+	rayclusterlog        = logf.Log.WithName("raycluster-resource")
+	baseDomain    string = ""
+)
 
-func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error {
+func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager, domain string) error {
+	baseDomain = domain
 	return ctrl.NewWebhookManagedBy(mgr).
 		For(&rayv1.RayCluster{}).
 		WithDefaulter(&RayClusterDefaulter{}).
@@ -49,17 +53,36 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e
 	raycluster := obj.(*rayv1.RayCluster)
 
 	rayclusterlog.Info("default", "name", raycluster.Name)
+	oauthExists := false
+	initHeadExists := false
+	workerHeadExists := false
 	// Check and add OAuth proxy if it does not exist.
-	alreadyExists := false
 	for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers {
 		if container.Name == "oauth-proxy" {
 			rayclusterlog.Info("OAuth sidecar already exists, no patch needed")
-			alreadyExists = true
+			oauthExists = true
+			break // exits the for loop
+		}
+	}
+
+	// Check for the create-cert Init Containers
+	for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.InitContainers {
+		if container.Name == "create-cert" {
+			rayclusterlog.Info("Head Init Containers already exist, no patch needed")
+			initHeadExists = true
+			break // exits the for loop
+		}
+	}
+	// Check fot the create-cert Init Container WorkerGroupSpec
+	for _, container := range raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers {
+		if container.Name == "create-cert" {
+			rayclusterlog.Info("Worker Init Containers already exist, no patch needed")
+			workerHeadExists = true
 			break // exits the for loop
 		}
 	}
 
-	if !alreadyExists {
+	if !oauthExists {
 		rayclusterlog.Info("Adding OAuth sidecar container")
 		// definition of the new container
 		newOAuthSidecar := corev1.Container{
@@ -119,5 +142,127 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e
 			raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy"
 		}
 	}
+	mtlsPatch(raycluster, initHeadExists, workerHeadExists)
 	return nil
 }
+
+func mtlsPatch(raycluster *rayv1.RayCluster, initHeadExists bool, workerHeadExists bool) {
+
+	rayclusterlog.Info("creating json patch for RayCluster initContainers")
+
+	// Volume Mounts for the Init Containers
+	key_volumes := []corev1.VolumeMount{
+		{
+			Name:      "ca-vol",
+			MountPath: "/home/ray/workspace/ca",
+			ReadOnly:  true,
+		},
+		{
+			Name:      "server-cert",
+			MountPath: "/home/ray/workspace/tls",
+			ReadOnly:  false,
+		},
+	}
+
+	// Service name for basic interactive
+	svcDomain := raycluster.Name + "-head-svc." + raycluster.Namespace + ".svc"
+	// Ca Secret generated by the SDK
+	secretName := `ca-secret-` + raycluster.Name
+
+	// Env variables for Worker & Head Containers
+	envList := []corev1.EnvVar{
+		{
+			Name: "MY_POD_IP",
+			ValueFrom: &corev1.EnvVarSource{
+				FieldRef: &corev1.ObjectFieldSelector{
+					FieldPath: "status.podIP",
+				},
+			},
+		},
+		{
+			Name:  "RAY_USE_TLS",
+			Value: "1",
+		},
+		{
+			Name:  "RAY_TLS_SERVER_CERT",
+			Value: "/home/ray/workspace/tls/server.crt",
+		},
+		{
+			Name:  "RAY_TLS_SERVER_KEY",
+			Value: "/home/ray/workspace/tls/server.key",
+		},
+		{
+			Name:  "RAY_TLS_CA_CERT",
+			Value: "/home/ray/workspace/tls/ca.crt",
+		},
+	}
+
+	// Volumes for the main container of Head and worker
+	caVolumes := []corev1.Volume{
+		{
+			Name: "ca-vol",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: secretName,
+				},
+			},
+		},
+		{
+			Name: "server-cert",
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{},
+			},
+		},
+	}
+
+	if !initHeadExists {
+		rayClientRoute := "rayclient-" + raycluster.Name + "-" + raycluster.Namespace + "." + baseDomain
+		initContainerHead := corev1.Container{
+			Name:  "create-cert",
+			Image: "quay.io/project-codeflare/ray:latest-py39-cu118",
+			Command: []string{
+				"sh",
+				"-c",
+				`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
+			},
+			VolumeMounts: key_volumes,
+		}
+
+		// Append the list of environment variables for the ray-head container
+		for index, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers {
+			if container.Name == "ray-head" {
+				raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[index].Env = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[index].Env, envList...)
+			}
+		}
+
+		// Append the create-cert Init Container
+		raycluster.Spec.HeadGroupSpec.Template.Spec.InitContainers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, initContainerHead)
+
+		// Append the CA volumes
+		raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, caVolumes...)
+	}
+
+	if !workerHeadExists {
+		initContainerWorker := corev1.Container{
+			Name:  "create-cert",
+			Image: "quay.io/project-codeflare/ray:latest-py39-cu118",
+			Command: []string{
+				"sh",
+				"-c",
+				`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
+			},
+			VolumeMounts: key_volumes,
+		}
+		// Append the CA volumes
+		raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes = append(raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVolumes...)
+		// Append the create-cert Init Container
+		raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = append(raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, initContainerWorker)
+
+		// Append the list of environment variables for the machine-learning container
+		for index, container := range raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers {
+			if container.Name == "machine-learning" {
+				raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[index].Env = append(raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[index].Env, envList...)
+			}
+		}
+	}
+}
