Skip to content

Commit 2b020b4

Browse files
KPostOfficeKPostOffice
committed
add hash to end of resource names to avoid name clash
Signed-off-by: Kevin <[email protected]>
1 parent f31a77d commit 2b020b4

File tree

4 files changed

+93
-25
lines changed

4 files changed

+93
-25
lines changed

‎pkg/controllers/raycluster_controller.go‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,10 @@ func isMTLSEnabled(cfg *config.KubeRayConfiguration) bool {
304304
}
305305

306306
func crbNameFromCluster(cluster *rayv1.RayCluster) string {
307-
return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz}
307+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
308+
return RCCUniqueName(cluster.Name + "-" + cluster.Namespace + "-auth")
309+
}
310+
return cluster.Name + "-" + cluster.Namespace + "-auth"
308311
}
309312

310313
func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.ClusterRoleBindingApplyConfiguration {
@@ -326,6 +329,9 @@ func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.Cluster
326329
}
327330

328331
func oauthServiceAccountNameFromCluster(cluster *rayv1.RayCluster) string {
332+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
333+
return RCCUniqueName(cluster.Name + "-oauth-proxy")
334+
}
329335
return cluster.Name + "-oauth-proxy"
330336
}
331337

@@ -363,10 +369,16 @@ func desiredClusterRoute(cluster *rayv1.RayCluster) *routev1ac.RouteApplyConfigu
363369
}
364370

365371
func oauthServiceNameFromCluster(cluster *rayv1.RayCluster) string {
372+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
373+
return RCCUniqueName(cluster.Name + "-oauth")
374+
}
366375
return cluster.Name + "-oauth"
367376
}
368377

369378
func oauthServiceTLSSecretName(cluster *rayv1.RayCluster) string {
379+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
380+
return RCCUniqueName(cluster.Name + "-proxy-tls-secret")
381+
}
370382
return cluster.Name + "-proxy-tls-secret"
371383
}
372384

@@ -389,6 +401,9 @@ func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfig
389401
}
390402

391403
func oauthSecretNameFromCluster(cluster *rayv1.RayCluster) string {
404+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
405+
return RCCUniqueName(cluster.Name + "-oauth-config")
406+
}
392407
return cluster.Name + "-oauth-config"
393408
}
394409

@@ -406,6 +421,9 @@ func desiredOAuthSecret(cluster *rayv1.RayCluster, cookieSalt string) *corev1ac.
406421
}
407422

408423
func caSecretNameFromCluster(cluster *rayv1.RayCluster) string {
424+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
425+
return RCCUniqueName(cluster.Name + "-ca-secret")
426+
}
409427
return "ca-secret-" + cluster.Name
410428
}
411429

@@ -462,8 +480,17 @@ func generateCACertificate() ([]byte, []byte, error) {
462480
return privateKeyPem, certPem, nil
463481
}
464482

483+
func workerNWPNameFromCluster(cluster *rayv1.RayCluster) string {
484+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
485+
return RCCUniqueName(cluster.Name + "-workers")
486+
}
487+
return cluster.Name + "-workers"
488+
}
489+
465490
func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.NetworkPolicyApplyConfiguration {
466-
return networkingv1ac.NetworkPolicy(cluster.Name+"-workers", cluster.Namespace).
491+
return networkingv1ac.NetworkPolicy(
492+
workerNWPNameFromCluster(cluster), cluster.Namespace,
493+
).
467494
WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}).
468495
WithSpec(networkingv1ac.NetworkPolicySpec().
469496
WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "worker"})).
@@ -477,14 +504,21 @@ func desiredWorkersNetworkPolicy(cluster *rayv1.RayCluster) *networkingv1ac.Netw
477504
WithOwnerReferences(ownerRefForRayCluster(cluster))
478505
}
479506

507+
func headNWPNameFromCluster(cluster *rayv1.RayCluster) string {
508+
if val, ok := cluster.GetAnnotations()[newNameAnnotation]; ok && val == "true" {
509+
return RCCUniqueName(cluster.Name + "-head")
510+
}
511+
return cluster.Name + "-head"
512+
}
513+
480514
func desiredHeadNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConfiguration, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration {
481515
allSecuredPorts := []*networkingv1ac.NetworkPolicyPortApplyConfiguration{
482516
networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)),
483517
}
484518
if ptr.Deref(cfg.MTLSEnabled, true) {
485519
allSecuredPorts = append(allSecuredPorts, networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)))
486520
}
487-
return networkingv1ac.NetworkPolicy(cluster.Name+"-head", cluster.Namespace).
521+
return networkingv1ac.NetworkPolicy(headNWPNameFromCluster(cluster), cluster.Namespace).
488522
WithLabels(map[string]string{RayClusterNameLabel: cluster.Name}).
489523
WithSpec(networkingv1ac.NetworkPolicySpec().
490524
WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})).
@@ -619,3 +653,7 @@ func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
619653

620654
return controller.Complete(r)
621655
}
656+
657+
func RCCUniqueName(s string) string {
658+
return s + "-" + seededHash(controllerName, s)
659+
}

‎pkg/controllers/raycluster_webhook.go‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ const (
3838
oauthProxyContainerName = "oauth-proxy"
3939
oauthProxyVolumeName = "proxy-tls-secret"
4040
initContainerName = "create-cert"
41+
newNameAnnotation = "ray.openshift.ai/hashed-names"
4142
)
4243

4344
// log is for logging in this package.
@@ -68,13 +69,18 @@ var _ webhook.CustomValidator = &rayClusterWebhook{}
6869
func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error {
6970
rayCluster := obj.(*rayv1.RayCluster)
7071

72+
// add annotation to use new names
73+
annotations := rayCluster.GetAnnotations()
74+
annotations[newNameAnnotation] = "true"
75+
rayCluster.SetAnnotations(annotations)
76+
7177
if ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) {
7278
rayclusterlog.V(2).Info("Adding OAuth sidecar container")
7379
rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, oauthProxyContainer(rayCluster), withContainerName(oauthProxyContainerName))
7480

7581
rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), withVolumeName(oauthProxyVolumeName))
7682

77-
rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy"
83+
rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = oauthServiceAccountNameFromCluster(rayCluster)
7884
}
7985

8086
if ptr.Deref(w.Config.MTLSEnabled, true) {
@@ -218,7 +224,7 @@ func validateIngress(rayCluster *rayv1.RayCluster) field.ErrorList {
218224
func validateHeadGroupServiceAccountName(rayCluster *rayv1.RayCluster) field.ErrorList {
219225
var allErrors field.ErrorList
220226

221-
if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName != rayCluster.Name+"-oauth-proxy" {
227+
if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName != oauthServiceAccountNameFromCluster(rayCluster) {
222228
allErrors = append(allErrors, field.Invalid(
223229
field.NewPath("spec", "headGroupSpec", "template", "spec", "serviceAccountName"),
224230
rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName,
@@ -241,7 +247,7 @@ func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container {
241247
ValueFrom: &corev1.EnvVarSource{
242248
SecretKeyRef: &corev1.SecretKeySelector{
243249
LocalObjectReference: corev1.LocalObjectReference{
244-
Name: rayCluster.Name + "-oauth-config",
250+
Name: oauthSecretNameFromCluster(rayCluster),
245251
},
246252
Key: "cookie_secret",
247253
},
@@ -251,7 +257,7 @@ func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container {
251257
Args: []string{
252258
"--https-address=:8443",
253259
"--provider=openshift",
254-
"--openshift-service-account=" + rayCluster.Name + "-oauth-proxy",
260+
"--openshift-service-account=" + oauthServiceAccountNameFromCluster(rayCluster),
255261
"--upstream=http://localhost:8265",
256262
"--tls-cert=/etc/tls/private/tls.crt",
257263
"--tls-key=/etc/tls/private/tls.key",
@@ -273,7 +279,7 @@ func oauthProxyTLSSecretVolume(rayCluster *rayv1.RayCluster) corev1.Volume {
273279
Name: oauthProxyVolumeName,
274280
VolumeSource: corev1.VolumeSource{
275281
Secret: &corev1.SecretVolumeSource{
276-
SecretName: rayCluster.Name + "-proxy-tls-secret",
282+
SecretName: oauthServiceTLSSecretName(rayCluster),
277283
},
278284
},
279285
}
@@ -329,7 +335,7 @@ func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume {
329335
Name: "ca-vol",
330336
VolumeSource: corev1.VolumeSource{
331337
Secret: &corev1.SecretVolumeSource{
332-
SecretName: `ca-secret-` + rayCluster.Name,
338+
SecretName: caSecretNameFromCluster(rayCluster),
333339
},
334340
},
335341
},
@@ -343,9 +349,9 @@ func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume {
343349
}
344350

345351
func rayHeadInitContainer(rayCluster *rayv1.RayCluster, config *config.KubeRayConfiguration) corev1.Container {
346-
rayClientRoute := "rayclient-" + rayCluster.Name + "-" + rayCluster.Namespace + "." + config.IngressDomain
352+
rayClientRoute := rayClientNameFromCluster(rayCluster) + "-" + rayCluster.Namespace + "." + config.IngressDomain
347353
// Service name for basic interactive
348-
svcDomain := rayCluster.Name + "-head-svc." + rayCluster.Namespace + ".svc"
354+
svcDomain := serviceNameFromCluster(rayCluster) + "." + rayCluster.Namespace + ".svc"
349355

350356
initContainerHead := corev1.Container{
351357
Name: "create-cert",

‎pkg/controllers/raycluster_webhook_test.go‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ func TestRayClusterWebhookDefault(t *testing.T) {
124124

125125
t.Run("Expected required service account name for the head group", func(t *testing.T) {
126126
test.Expect(validRayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName).
127-
To(Equal(validRayCluster.Name+"-oauth-proxy"),
127+
To(Equal(oauthServiceAccountNameFromCluster(validRayCluster)),
128128
"Expected the service account name to be set correctly")
129129
})
130130

@@ -230,7 +230,13 @@ func TestRayClusterWebhookDefault(t *testing.T) {
230230

231231
func TestValidateCreate(t *testing.T) {
232232
test := support.NewTest(t)
233-
233+
emptyRayCluster := &rayv1.RayCluster{
234+
ObjectMeta: metav1.ObjectMeta{
235+
Name: rayClusterName,
236+
Namespace: namespace,
237+
},
238+
Spec: rayv1.RayClusterSpec{},
239+
}
234240
validRayCluster := &rayv1.RayCluster{
235241
ObjectMeta: metav1.ObjectMeta{
236242
Name: rayClusterName,
@@ -253,7 +259,7 @@ func TestValidateCreate(t *testing.T) {
253259
ValueFrom: &corev1.EnvVarSource{
254260
SecretKeyRef: &corev1.SecretKeySelector{
255261
LocalObjectReference: corev1.LocalObjectReference{
256-
Name: rayClusterName + "-oauth-config",
262+
Name: oauthSecretNameFromCluster(emptyRayCluster),
257263
},
258264
Key: "cookie_secret",
259265
},
@@ -263,7 +269,7 @@ func TestValidateCreate(t *testing.T) {
263269
Args: []string{
264270
"--https-address=:8443",
265271
"--provider=openshift",
266-
"--openshift-service-account=" + rayClusterName + "-oauth-proxy",
272+
"--openshift-service-account=" + oauthServiceAccountNameFromCluster(emptyRayCluster),
267273
"--upstream=http://localhost:8265",
268274
"--tls-cert=/etc/tls/private/tls.crt",
269275
"--tls-key=/etc/tls/private/tls.key",
@@ -284,12 +290,12 @@ func TestValidateCreate(t *testing.T) {
284290
Name: oauthProxyVolumeName,
285291
VolumeSource: corev1.VolumeSource{
286292
Secret: &corev1.SecretVolumeSource{
287-
SecretName: rayClusterName + "-proxy-tls-secret",
293+
SecretName: oauthServiceTLSSecretName(emptyRayCluster),
288294
},
289295
},
290296
},
291297
},
292-
ServiceAccountName: rayClusterName + "-oauth-proxy",
298+
ServiceAccountName: oauthServiceAccountNameFromCluster(emptyRayCluster),
293299
},
294300
},
295301
RayStartParams: map[string]string{},
@@ -351,7 +357,15 @@ func TestValidateCreate(t *testing.T) {
351357

352358
func TestValidateUpdate(t *testing.T) {
353359
test := support.NewTest(t)
354-
360+
emptyRayCluster := &rayv1.RayCluster{
361+
ObjectMeta: metav1.ObjectMeta{
362+
Name: rayClusterName,
363+
Namespace: namespace,
364+
},
365+
Spec: rayv1.RayClusterSpec{},
366+
}
367+
rayClientRoute := rayClientNameFromCluster(emptyRayCluster) + "-" + emptyRayCluster.Namespace + "." + rcWebhook.Config.IngressDomain
368+
svcDomain := serviceNameFromCluster(emptyRayCluster) + "." + emptyRayCluster.Namespace + ".svc"
355369
validRayCluster := &rayv1.RayCluster{
356370
ObjectMeta: metav1.ObjectMeta{
357371
Name: rayClusterName,
@@ -374,7 +388,7 @@ func TestValidateUpdate(t *testing.T) {
374388
ValueFrom: &corev1.EnvVarSource{
375389
SecretKeyRef: &corev1.SecretKeySelector{
376390
LocalObjectReference: corev1.LocalObjectReference{
377-
Name: rayClusterName + "-oauth-config",
391+
Name: oauthSecretNameFromCluster(emptyRayCluster),
378392
},
379393
Key: "cookie_secret",
380394
},
@@ -396,7 +410,7 @@ func TestValidateUpdate(t *testing.T) {
396410
Args: []string{
397411
"--https-address=:8443",
398412
"--provider=openshift",
399-
"--openshift-service-account=" + rayClusterName + "-oauth-proxy",
413+
"--openshift-service-account=" + oauthServiceAccountNameFromCluster(emptyRayCluster),
400414
"--upstream=http://localhost:8265",
401415
"--tls-cert=/etc/tls/private/tls.crt",
402416
"--tls-key=/etc/tls/private/tls.key",
@@ -419,7 +433,7 @@ func TestValidateUpdate(t *testing.T) {
419433
Command: []string{
420434
"sh",
421435
"-c",
422-
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = rayclient-` + rayClusterName + `-` + namespace + `.\nDNS.6 = ` + rayClusterName + `-head-svc.` + namespace + `.svc` + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
436+
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
423437
},
424438
VolumeMounts: []corev1.VolumeMount{
425439
{
@@ -440,15 +454,15 @@ func TestValidateUpdate(t *testing.T) {
440454
Name: oauthProxyVolumeName,
441455
VolumeSource: corev1.VolumeSource{
442456
Secret: &corev1.SecretVolumeSource{
443-
SecretName: rayClusterName + "-proxy-tls-secret",
457+
SecretName: oauthServiceTLSSecretName(emptyRayCluster),
444458
},
445459
},
446460
},
447461
{
448462
Name: "ca-vol",
449463
VolumeSource: corev1.VolumeSource{
450464
Secret: &corev1.SecretVolumeSource{
451-
SecretName: `ca-secret-` + rayClusterName,
465+
SecretName: caSecretNameFromCluster(emptyRayCluster),
452466
},
453467
},
454468
},
@@ -459,7 +473,7 @@ func TestValidateUpdate(t *testing.T) {
459473
},
460474
},
461475
},
462-
ServiceAccountName: rayClusterName + "-oauth-proxy",
476+
ServiceAccountName: oauthServiceAccountNameFromCluster(emptyRayCluster),
463477
},
464478
},
465479
RayStartParams: map[string]string{},
@@ -505,7 +519,7 @@ func TestValidateUpdate(t *testing.T) {
505519
Name: "ca-vol",
506520
VolumeSource: corev1.VolumeSource{
507521
Secret: &corev1.SecretVolumeSource{
508-
SecretName: `ca-secret-` + rayClusterName,
522+
SecretName: caSecretNameFromCluster(emptyRayCluster),
509523
},
510524
},
511525
},

‎pkg/controllers/support.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package controllers
22

33
import (
4+
"crypto/sha256"
5+
"fmt"
46
"os"
57

68
"github.com/go-logr/logr"
@@ -209,3 +211,11 @@ func ownerRefForRayCluster(cluster *rayv1.RayCluster) *v1.OwnerReferenceApplyCon
209211
WithUID(cluster.UID).
210212
WithController(true)
211213
}
214+
215+
var (
216+
hashLength = 8
217+
)
218+
219+
func seededHash(seed string, s string) string {
220+
return fmt.Sprintf("%x", sha256.Sum256([]byte(seed+s)))[:hashLength]
221+
}

0 commit comments

Comments
 (0)