Skip to content

Commit eff323b

Browse files
KPostOfficeKPostOffice
committed
add hash to end of resource names to avoid name clash
Signed-off-by: Kevin <[email protected]>
1 parent f31a77d commit eff323b

File tree

4 files changed

+102
-25
lines changed

4 files changed

+102
-25
lines changed

‎pkg/controllers/raycluster_controller.go‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,10 @@ func isMTLSEnabled(cfg *config.KubeRayConfiguration) bool {
304304
}
305305

306306
func crbNameFromCluster(cluster *rayv1.RayCluster) string {
307-
return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz}
307+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
308+
return RCCUniqueName(cluster.Name + "-" + cluster.Namespace + "-auth")
309+
}
310+
return cluster.Name + "-" + cluster.Namespace + "-auth"
308311
}
309312

310313
func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.ClusterRoleBindingApplyConfiguration {
@@ -326,6 +329,9 @@ func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.Cluster
326329
}
327330

328331
func oauthServiceAccountNameFromCluster(cluster *rayv1.RayCluster) string {
332+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
333+
return RCCUniqueName(cluster.Name + "-oauth-proxy")
334+
}
329335
return cluster.Name + "-oauth-proxy"
330336
}
331337

@@ -363,10 +369,16 @@ func desiredClusterRoute(cluster *rayv1.RayCluster) *routev1ac.RouteApplyConfigu
363369
}
364370

365371
func oauthServiceNameFromCluster(cluster *rayv1.RayCluster) string {
372+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
373+
return RCCUniqueName(cluster.Name + "-oauth")
374+
}
366375
return cluster.Name + "-oauth"
367376
}
368377

369378
func oauthServiceTLSSecretName(cluster *rayv1.RayCluster) string {
379+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
380+
return RCCUniqueName(cluster.Name + "-proxy-tls-secret")
381+
}
370382
return cluster.Name + "-proxy-tls-secret"
371383
}
372384

@@ -389,6 +401,9 @@ func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfig
389401
}
390402

391403
func oauthSecretNameFromCluster(cluster *rayv1.RayCluster) string {
404+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
405+
return RCCUniqueName(cluster.Name + "-oauth-config")
406+
}
392407
return cluster.Name + "-oauth-config"
393408
}
394409

@@ -406,6 +421,9 @@ func desiredOAuthSecret(cluster *rayv1.RayCluster, cookieSalt string) *corev1ac.
406421
}
407422

408423
func caSecretNameFromCluster(cluster *rayv1.RayCluster) string {
424+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
425+
return RCCUniqueName(cluster.Name + "-ca-secret")
426+
}
409427
return "ca-secret-" + cluster.Name
410428
}
411429

@@ -462,8 +480,17 @@ func generateCACertificate() ([]byte, []byte, error) {
462480
return privateKeyPem, certPem, nil
463481
}
464482

483+
func workerNWPNameFromCluster(cluster *rayv1.RayCluster) string {
484+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
485+
return RCCUniqueName(cluster.Name + "-workers")
486+
}
487+
return cluster.Name + "-workers"
488+
}
489+
465490
func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.NetworkPolicyApplyConfiguration {
466-
return networkingv1ac.NetworkPolicy(cluster.Name+"-workers", cluster.Namespace).
491+
return networkingv1ac.NetworkPolicy(
492+
workerNWPNameFromCluster(cluster), cluster.Namespace,
493+
).
467494
WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}).
468495
WithSpec(networkingv1ac.NetworkPolicySpec().
469496
WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "worker"})).
@@ -477,14 +504,21 @@ func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.Netw
477504
WithOwnerReferences(ownerRefForRayCluster(cluster))
478505
}
479506

507+
func headNWPNameFromCluster(cluster *rayv1.RayCluster) string {
508+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
509+
return RCCUniqueName(cluster.Name + "-head")
510+
}
511+
return cluster.Name + "-head"
512+
}
513+
480514
func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConfiguration, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration {
481515
allSecuredPorts := []*networkingv1ac.NetworkPolicyPortApplyConfiguration{
482516
networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)),
483517
}
484518
if ptr.Deref(cfg.MTLSEnabled, true) {
485519
allSecuredPorts = append(allSecuredPorts, networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)))
486520
}
487-
return networkingv1ac.NetworkPolicy(cluster.Name+"-head", cluster.Namespace).
521+
return networkingv1ac.NetworkPolicy(headNWPNameFromCluster(cluster), cluster.Namespace).
488522
WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}).
489523
WithSpec(networkingv1ac.NetworkPolicySpec().
490524
WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})).
@@ -619,3 +653,7 @@ func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
619653

620654
return controller.Complete(r)
621655
}
656+
657+
func RCCUniqueName(s string) string {
658+
return s + "-" + seededHash(controllerName, s)
659+
}

‎pkg/controllers/raycluster_webhook.go‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ const (
3838
oauthProxyContainerName = "oauth-proxy"
3939
oauthProxyVolumeName = "proxy-tls-secret"
4040
initContainerName = "create-cert"
41+
newNameAnnotation = "ray.openshift.ai/hashed-names"
4142
)
4243

4344
// log is for logging in this package.
@@ -68,13 +69,21 @@ var _ webhook.CustomValidator = &rayClusterWebhook{}
6869
func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error {
6970
rayCluster := obj.(*rayv1.RayCluster)
7071

72+
// add annotation to use new names
73+
annotations := rayCluster.GetAnnotations()
74+
if annotations == nil {
75+
annotations = make(map[string]string)
76+
}
77+
annotations[newNameAnnotation] = "true"
78+
rayCluster.SetAnnotations(annotations)
79+
7180
if ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) {
7281
rayclusterlog.V(2).Info("Adding OAuth sidecar container")
7382
rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, oauthProxyContainer(rayCluster), withContainerName(oauthProxyContainerName))
7483

7584
rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), withVolumeName(oauthProxyVolumeName))
7685

77-
rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy"
86+
rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = oauthServiceAccountNameFromCluster(rayCluster)
7887
}
7988

8089
if ptr.Deref(w.Config.MTLSEnabled, true) {
@@ -218,7 +227,7 @@ func validateIngress(rayCluster *rayv1.RayCluster) field.ErrorList {
218227
func validateHeadGroupServiceAccountName(rayCluster *rayv1.RayCluster) field.ErrorList {
219228
var allErrors field.ErrorList
220229

221-
if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName != rayCluster.Name+"-oauth-proxy" {
230+
if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName != oauthServiceAccountNameFromCluster(rayCluster) {
222231
allErrors = append(allErrors, field.Invalid(
223232
field.NewPath("spec", "headGroupSpec", "template", "spec", "serviceAccountName"),
224233
rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName,
@@ -241,7 +250,7 @@ func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container {
241250
ValueFrom: &corev1.EnvVarSource{
242251
SecretKeyRef: &corev1.SecretKeySelector{
243252
LocalObjectReference: corev1.LocalObjectReference{
244-
Name: rayCluster.Name + "-oauth-config",
253+
Name: oauthSecretNameFromCluster(rayCluster),
245254
},
246255
Key: "cookie_secret",
247256
},
@@ -251,7 +260,7 @@ func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container {
251260
Args: []string{
252261
"--https-address=:8443",
253262
"--provider=openshift",
254-
"--openshift-service-account=" + rayCluster.Name + "-oauth-proxy",
263+
"--openshift-service-account=" + oauthServiceAccountNameFromCluster(rayCluster),
255264
"--upstream=http://localhost:8265",
256265
"--tls-cert=/etc/tls/private/tls.crt",
257266
"--tls-key=/etc/tls/private/tls.key",
@@ -273,7 +282,7 @@ func oauthProxyTLSSecretVolume(rayCluster *rayv1.RayCluster) corev1.Volume {
273282
Name: oauthProxyVolumeName,
274283
VolumeSource: corev1.VolumeSource{
275284
Secret: &corev1.SecretVolumeSource{
276-
SecretName: rayCluster.Name + "-proxy-tls-secret",
285+
SecretName: oauthServiceTLSSecretName(rayCluster),
277286
},
278287
},
279288
}
@@ -329,7 +338,7 @@ func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume {
329338
Name: "ca-vol",
330339
VolumeSource: corev1.VolumeSource{
331340
Secret: &corev1.SecretVolumeSource{
332-
SecretName: `ca-secret-` + rayCluster.Name,
341+
SecretName: caSecretNameFromCluster(rayCluster),
333342
},
334343
},
335344
},
@@ -343,9 +352,9 @@ func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume {
343352
}
344353

345354
func rayHeadInitContainer(rayCluster *rayv1.RayCluster, config *config.KubeRayConfiguration) corev1.Container {
346-
rayClientRoute := "rayclient-" + rayCluster.Name + "-" + rayCluster.Namespace + "." + config.IngressDomain
355+
rayClientRoute := rayClientNameFromCluster(rayCluster) + "-" + rayCluster.Namespace + "." + config.IngressDomain
347356
// Service name for basic interactive
348-
svcDomain := rayCluster.Name + "-head-svc." + rayCluster.Namespace + ".svc"
357+
svcDomain := serviceNameFromCluster(rayCluster) + "." + rayCluster.Namespace + ".svc"
349358

350359
initContainerHead := corev1.Container{
351360
Name: "create-cert",

‎pkg/controllers/raycluster_webhook_test.go‎

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ func TestRayClusterWebhookDefault(t *testing.T) {
124124

125125
t.Run("Expected required service account name for the head group", func(t *testing.T) {
126126
test.Expect(validRayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName).
127-
To(Equal(validRayCluster.Name+"-oauth-proxy"),
127+
To(Equal(oauthServiceAccountNameFromCluster(validRayCluster)),
128128
"Expected the service account name to be set correctly")
129129
})
130130

@@ -230,7 +230,16 @@ func TestRayClusterWebhookDefault(t *testing.T) {
230230

231231
func TestValidateCreate(t *testing.T) {
232232
test := support.NewTest(t)
233-
233+
emptyRayCluster := &rayv1.RayCluster{
234+
ObjectMeta: metav1.ObjectMeta{
235+
Name: rayClusterName,
236+
Namespace: namespace,
237+
Annotations: map[string]string{
238+
newNameAnnotation: "true",
239+
},
240+
},
241+
Spec: rayv1.RayClusterSpec{},
242+
}
234243
validRayCluster := &rayv1.RayCluster{
235244
ObjectMeta: metav1.ObjectMeta{
236245
Name: rayClusterName,
@@ -253,7 +262,7 @@ func TestValidateCreate(t *testing.T) {
253262
ValueFrom: &corev1.EnvVarSource{
254263
SecretKeyRef: &corev1.SecretKeySelector{
255264
LocalObjectReference: corev1.LocalObjectReference{
256-
Name: rayClusterName + "-oauth-config",
265+
Name: oauthSecretNameFromCluster(emptyRayCluster),
257266
},
258267
Key: "cookie_secret",
259268
},
@@ -263,7 +272,7 @@ func TestValidateCreate(t *testing.T) {
263272
Args: []string{
264273
"--https-address=:8443",
265274
"--provider=openshift",
266-
"--openshift-service-account=" + rayClusterName + "-oauth-proxy",
275+
"--openshift-service-account=" + oauthServiceAccountNameFromCluster(emptyRayCluster),
267276
"--upstream=http://localhost:8265",
268277
"--tls-cert=/etc/tls/private/tls.crt",
269278
"--tls-key=/etc/tls/private/tls.key",
@@ -284,12 +293,12 @@ func TestValidateCreate(t *testing.T) {
284293
Name: oauthProxyVolumeName,
285294
VolumeSource: corev1.VolumeSource{
286295
Secret: &corev1.SecretVolumeSource{
287-
SecretName: rayClusterName + "-proxy-tls-secret",
296+
SecretName: oauthServiceTLSSecretName(emptyRayCluster),
288297
},
289298
},
290299
},
291300
},
292-
ServiceAccountName: rayClusterName + "-oauth-proxy",
301+
ServiceAccountName: oauthServiceAccountNameFromCluster(emptyRayCluster),
293302
},
294303
},
295304
RayStartParams: map[string]string{},
@@ -351,7 +360,18 @@ func TestValidateCreate(t *testing.T) {
351360

352361
func TestValidateUpdate(t *testing.T) {
353362
test := support.NewTest(t)
354-
363+
emptyRayCluster := &rayv1.RayCluster{
364+
ObjectMeta: metav1.ObjectMeta{
365+
Name: rayClusterName,
366+
Namespace: namespace,
367+
Annotations: map[string]string{
368+
newNameAnnotation: "true",
369+
},
370+
},
371+
Spec: rayv1.RayClusterSpec{},
372+
}
373+
rayClientRoute := rayClientNameFromCluster(emptyRayCluster) + "-" + emptyRayCluster.Namespace + "." + rcWebhook.Config.IngressDomain
374+
svcDomain := serviceNameFromCluster(emptyRayCluster) + "." + emptyRayCluster.Namespace + ".svc"
355375
validRayCluster := &rayv1.RayCluster{
356376
ObjectMeta: metav1.ObjectMeta{
357377
Name: rayClusterName,
@@ -374,7 +394,7 @@ func TestValidateUpdate(t *testing.T) {
374394
ValueFrom: &corev1.EnvVarSource{
375395
SecretKeyRef: &corev1.SecretKeySelector{
376396
LocalObjectReference: corev1.LocalObjectReference{
377-
Name: rayClusterName + "-oauth-config",
397+
Name: oauthSecretNameFromCluster(emptyRayCluster),
378398
},
379399
Key: "cookie_secret",
380400
},
@@ -396,7 +416,7 @@ func TestValidateUpdate(t *testing.T) {
396416
Args: []string{
397417
"--https-address=:8443",
398418
"--provider=openshift",
399-
"--openshift-service-account=" + rayClusterName + "-oauth-proxy",
419+
"--openshift-service-account=" + oauthServiceAccountNameFromCluster(emptyRayCluster),
400420
"--upstream=http://localhost:8265",
401421
"--tls-cert=/etc/tls/private/tls.crt",
402422
"--tls-key=/etc/tls/private/tls.key",
@@ -419,7 +439,7 @@ func TestValidateUpdate(t *testing.T) {
419439
Command: []string{
420440
"sh",
421441
"-c",
422-
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = rayclient-` + rayClusterName + `-` + namespace + `.\nDNS.6 = ` + rayClusterName + `-head-svc.` + namespace + `.svc` + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
442+
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
423443
},
424444
VolumeMounts: []corev1.VolumeMount{
425445
{
@@ -440,15 +460,15 @@ func TestValidateUpdate(t *testing.T) {
440460
Name: oauthProxyVolumeName,
441461
VolumeSource: corev1.VolumeSource{
442462
Secret: &corev1.SecretVolumeSource{
443-
SecretName: rayClusterName + "-proxy-tls-secret",
463+
SecretName: oauthServiceTLSSecretName(emptyRayCluster),
444464
},
445465
},
446466
},
447467
{
448468
Name: "ca-vol",
449469
VolumeSource: corev1.VolumeSource{
450470
Secret: &corev1.SecretVolumeSource{
451-
SecretName: `ca-secret-` + rayClusterName,
471+
SecretName: caSecretNameFromCluster(emptyRayCluster),
452472
},
453473
},
454474
},
@@ -459,7 +479,7 @@ func TestValidateUpdate(t *testing.T) {
459479
},
460480
},
461481
},
462-
ServiceAccountName: rayClusterName + "-oauth-proxy",
482+
ServiceAccountName: oauthServiceAccountNameFromCluster(emptyRayCluster),
463483
},
464484
},
465485
RayStartParams: map[string]string{},
@@ -505,7 +525,7 @@ func TestValidateUpdate(t *testing.T) {
505525
Name: "ca-vol",
506526
VolumeSource: corev1.VolumeSource{
507527
Secret: &corev1.SecretVolumeSource{
508-
SecretName: `ca-secret-` + rayClusterName,
528+
SecretName: caSecretNameFromCluster(emptyRayCluster),
509529
},
510530
},
511531
},

‎pkg/controllers/support.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package controllers
22

33
import (
4+
"crypto/sha256"
5+
"fmt"
46
"os"
57

68
"github.com/go-logr/logr"
@@ -209,3 +211,11 @@ func ownerRefForRayCluster(cluster *rayv1.RayCluster) *v1.OwnerReferenceApplyCon
209211
WithUID(cluster.UID).
210212
WithController(true)
211213
}
214+
215+
var (
216+
hashLength = 8
217+
)
218+
219+
func seededHash(seed string, s string) string {
220+
return fmt.Sprintf("%x", sha256.Sum256([]byte(seed+s)))[:hashLength]
221+
}

0 commit comments

Comments
 (0)