@@ -40,104 +40,58 @@ func TestMNISTRayClusterSDK(t *testing.T) {
4040 test := With (t )
4141 test .T ().Parallel ()
4242
43- // Currently blocked by https://github.com/project-codeflare/codeflare-sdk/pull/251 , remove the skip once SDK with the PR is released
44- test .T ().Skip ("Requires https://github.com/project-codeflare/codeflare-sdk/pull/251" )
45-
4643 // Create a namespace
4744 namespace := test .NewTestNamespace ()
4845
4946 // Test configuration
50- config := & corev1.ConfigMap {
51- TypeMeta : metav1.TypeMeta {
52- APIVersion : corev1 .SchemeGroupVersion .String (),
53- Kind : "ConfigMap" ,
54- },
55- ObjectMeta : metav1.ObjectMeta {
56- Name : "mnist-raycluster-sdk" ,
57- Namespace : namespace .Name ,
47+ config := CreateConfigMap (test , namespace .Name , map [string ][]byte {
48+ // SDK script
49+ "mnist_raycluster_sdk.py" : ReadFile (test , "mnist_raycluster_sdk.py" ),
50+ // pip requirements
51+ "requirements.txt" : ReadFile (test , "mnist_pip_requirements.txt" ),
52+ // MNIST training script
53+ "mnist.py" : ReadFile (test , "mnist.py" ),
54+ })
55+
56+ // Create RBAC, retrieve token for user with limited rights
57+ policyRules := []rbacv1.PolicyRule {
58+ {
59+ Verbs : []string {"get" , "create" , "delete" , "list" , "patch" , "update" },
60+ APIGroups : []string {mcadv1beta1 .GroupName },
61+ Resources : []string {"appwrappers" },
5862 },
59- BinaryData : map [string ][]byte {
60- // SDK script
61- "mnist_raycluster_sdk.py" : ReadFile (test , "mnist_raycluster_sdk.py" ),
62- // pip requirements
63- "requirements.txt" : ReadFile (test , "mnist_pip_requirements.txt" ),
64- // MNIST training script
65- "mnist.py" : ReadFile (test , "mnist.py" ),
63+ {
64+ Verbs : []string {"get" , "list" },
65+ APIGroups : []string {rayv1 .GroupVersion .Group },
66+ Resources : []string {"rayclusters" , "rayclusters/status" },
6667 },
67- Immutable : Ptr (true ),
68- }
69- config , err := test .Client ().Core ().CoreV1 ().ConfigMaps (namespace .Name ).Create (test .Ctx (), config , metav1.CreateOptions {})
70- test .Expect (err ).NotTo (HaveOccurred ())
71- test .T ().Logf ("Created ConfigMap %s/%s successfully" , config .Namespace , config .Name )
72-
73- // SDK client RBAC
74- serviceAccount := & corev1.ServiceAccount {
75- TypeMeta : metav1.TypeMeta {
76- APIVersion : corev1 .SchemeGroupVersion .String (),
77- Kind : "ServiceAccount" ,
68+ {
69+ Verbs : []string {"get" , "list" },
70+ APIGroups : []string {"route.openshift.io" },
71+ Resources : []string {"routes" },
7872 },
79- ObjectMeta : metav1.ObjectMeta {
80- Name : "sdk-user" ,
81- Namespace : namespace .Name ,
73+ {
74+ Verbs : []string {"get" , "list" },
75+ APIGroups : []string {"networking.k8s.io" },
76+ Resources : []string {"ingresses" },
8277 },
8378 }
84- serviceAccount , err = test .Client ().Core ().CoreV1 ().ServiceAccounts (namespace .Name ).Create (test .Ctx (), serviceAccount , metav1.CreateOptions {})
85- test .Expect (err ).NotTo (HaveOccurred ())
8679
87- role := & rbacv1.Role {
88- TypeMeta : metav1.TypeMeta {
89- APIVersion : rbacv1 .SchemeGroupVersion .String (),
90- Kind : "Role" ,
91- },
92- ObjectMeta : metav1.ObjectMeta {
93- Name : "sdk" ,
94- Namespace : namespace .Name ,
95- },
96- Rules : []rbacv1.PolicyRule {
97- {
98- Verbs : []string {"get" , "create" , "delete" , "list" , "patch" , "update" },
99- APIGroups : []string {mcadv1beta1 .GroupName },
100- Resources : []string {"appwrappers" },
101- },
102- {
103- Verbs : []string {"get" , "list" },
104- APIGroups : []string {rayv1 .GroupVersion .Group },
105- Resources : []string {"rayclusters" , "rayclusters/status" },
106- },
107- {
108- Verbs : []string {"get" , "list" },
109- APIGroups : []string {"route.openshift.io" },
110- Resources : []string {"routes" },
111- },
80+ // Create cluster wide RBAC, required for SDK OpenShift check
81+ // TODO reevaluate once SDK change OpenShift detection logic
82+ clusterPolicyRules := []rbacv1.PolicyRule {
83+ {
84+ Verbs : []string {"get" , "list" },
85+ APIGroups : []string {"config.openshift.io" },
86+ Resources : []string {"ingresses" },
11287 },
11388 }
114- role , err = test .Client ().Core ().RbacV1 ().Roles (namespace .Name ).Create (test .Ctx (), role , metav1.CreateOptions {})
115- test .Expect (err ).NotTo (HaveOccurred ())
11689
117- roleBinding := & rbacv1.RoleBinding {
118- TypeMeta : metav1.TypeMeta {
119- APIVersion : rbacv1 .SchemeGroupVersion .String (),
120- Kind : "RoleBinding" ,
121- },
122- ObjectMeta : metav1.ObjectMeta {
123- Name : "sdk" ,
124- },
125- RoleRef : rbacv1.RoleRef {
126- APIGroup : rbacv1 .SchemeGroupVersion .Group ,
127- Kind : "Role" ,
128- Name : role .Name ,
129- },
130- Subjects : []rbacv1.Subject {
131- {
132- Kind : "ServiceAccount" ,
133- APIGroup : corev1 .SchemeGroupVersion .Group ,
134- Name : serviceAccount .Name ,
135- Namespace : serviceAccount .Namespace ,
136- },
137- },
138- }
139- _ , err = test .Client ().Core ().RbacV1 ().RoleBindings (namespace .Name ).Create (test .Ctx (), roleBinding , metav1.CreateOptions {})
140- test .Expect (err ).NotTo (HaveOccurred ())
90+ sa := CreateServiceAccount (test , namespace .Name )
91+ role := CreateRole (test , namespace .Name , policyRules )
92+ CreateRoleBinding (test , namespace .Name , sa , role )
93+ clusterRole := CreateClusterRole (test , clusterPolicyRules )
94+ CreateClusterRoleBinding (test , sa , clusterRole )
14195
14296 job := & batchv1.Job {
14397 TypeMeta : metav1.TypeMeta {
@@ -161,7 +115,8 @@ func TestMNISTRayClusterSDK(t *testing.T) {
161115 // See https://github.com/project-codeflare/codeflare-sdk/pull/146
162116 Image : "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e" ,
163117 Env : []corev1.EnvVar {
164- corev1.EnvVar {Name : "PYTHONUSERBASE" , Value : "/workdir" },
118+ {Name : "PYTHONUSERBASE" , Value : "/workdir" },
119+ {Name : "RAY_IMAGE" , Value : GetRayImage ()},
165120 },
166121 Command : []string {"/bin/sh" , "-c" , "pip install codeflare-sdk==" + GetCodeFlareSDKVersion () + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace .Name },
167122 VolumeMounts : []corev1.VolumeMount {
@@ -206,12 +161,31 @@ func TestMNISTRayClusterSDK(t *testing.T) {
206161 },
207162 },
208163 RestartPolicy : corev1 .RestartPolicyNever ,
209- ServiceAccountName : serviceAccount .Name ,
164+ ServiceAccountName : sa .Name ,
210165 },
211166 },
212167 },
213168 }
214- job , err = test .Client ().Core ().BatchV1 ().Jobs (namespace .Name ).Create (test .Ctx (), job , metav1.CreateOptions {})
169+ if GetClusterType (test ) == KindCluster {
170+ // Take first KinD node and redirect pod hostname requests there
171+ node := GetNodes (test )[0 ]
172+ hostname := GetClusterHostname (test )
173+ IP := GetNodeInternalIP (test , node )
174+
175+ test .T ().Logf ("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod" , hostname , IP )
176+ job .Spec .Template .Spec .HostAliases = []corev1.HostAlias {
177+ {
178+ IP : IP ,
179+ Hostnames : []string {hostname },
180+ },
181+ }
182+
183+ // Propagate hostname into Python code as env variable
184+ hostnameEnvVar := corev1.EnvVar {Name : "CLUSTER_HOSTNAME" , Value : hostname }
185+ job .Spec .Template .Spec .Containers [0 ].Env = append (job .Spec .Template .Spec .Containers [0 ].Env , hostnameEnvVar )
186+ }
187+
188+ job , err := test .Client ().Core ().BatchV1 ().Jobs (namespace .Name ).Create (test .Ctx (), job , metav1.CreateOptions {})
215189 test .Expect (err ).NotTo (HaveOccurred ())
216190 test .T ().Logf ("Created Job %s/%s successfully" , job .Namespace , job .Name )
217191
0 commit comments