@@ -40,104 +40,59 @@ func TestMNISTRayClusterSDK(t *testing.T) {
4040 test := With (t )
4141 test .T ().Parallel ()
4242
43- // Currently blocked by https://github.com/project-codeflare/codeflare-sdk/pull/251 , remove the skip once SDK with the PR is released
44- test .T ().Skip ("Requires https://github.com/project-codeflare/codeflare-sdk/pull/251" )
45-
4643 // Create a namespace
4744 namespace := test .NewTestNamespace ()
4845
4946 // Test configuration
50- config := & corev1.ConfigMap {
51- TypeMeta : metav1.TypeMeta {
52- APIVersion : corev1 .SchemeGroupVersion .String (),
53- Kind : "ConfigMap" ,
54- },
55- ObjectMeta : metav1.ObjectMeta {
56- Name : "mnist-raycluster-sdk" ,
57- Namespace : namespace .Name ,
47+ config := CreateConfigMap (test , namespace .Name , map [string ][]byte {
48+ // SDK script
49+ "mnist_raycluster_sdk.py" : ReadFile (test , "mnist_raycluster_sdk.py" ),
50+ // pip requirements
51+ "requirements.txt" : ReadFile (test , "mnist_pip_requirements.txt" ),
52+ // MNIST training script
53+ "mnist.py" : ReadFile (test , "mnist.py" ),
54+ })
55+
56+ // Create RBAC, retrieve token for user with limited rights
57+ policyRules := []rbacv1.PolicyRule {
58+ {
59+ Verbs : []string {"get" , "create" , "delete" , "list" , "patch" , "update" },
60+ APIGroups : []string {mcadv1beta1 .GroupName },
61+ Resources : []string {"appwrappers" },
5862 },
59- BinaryData : map [string ][]byte {
60- // SDK script
61- "mnist_raycluster_sdk.py" : ReadFile (test , "mnist_raycluster_sdk.py" ),
62- // pip requirements
63- "requirements.txt" : ReadFile (test , "mnist_pip_requirements.txt" ),
64- // MNIST training script
65- "mnist.py" : ReadFile (test , "mnist.py" ),
63+ {
64+ Verbs : []string {"get" , "list" },
65+ APIGroups : []string {rayv1 .GroupVersion .Group },
66+ Resources : []string {"rayclusters" , "rayclusters/status" },
6667 },
67- Immutable : Ptr (true ),
68- }
69- config , err := test .Client ().Core ().CoreV1 ().ConfigMaps (namespace .Name ).Create (test .Ctx (), config , metav1.CreateOptions {})
70- test .Expect (err ).NotTo (HaveOccurred ())
71- test .T ().Logf ("Created ConfigMap %s/%s successfully" , config .Namespace , config .Name )
72-
73- // SDK client RBAC
74- serviceAccount := & corev1.ServiceAccount {
75- TypeMeta : metav1.TypeMeta {
76- APIVersion : corev1 .SchemeGroupVersion .String (),
77- Kind : "ServiceAccount" ,
68+ {
69+ Verbs : []string {"get" , "list" },
70+ APIGroups : []string {"route.openshift.io" },
71+ Resources : []string {"routes" },
7872 },
79- ObjectMeta : metav1.ObjectMeta {
80- Name : "sdk-user" ,
81- Namespace : namespace .Name ,
73+ {
74+ Verbs : []string {"get" , "list" },
75+ APIGroups : []string {"networking.k8s.io" },
76+ Resources : []string {"ingresses" },
8277 },
8378 }
84- serviceAccount , err = test .Client ().Core ().CoreV1 ().ServiceAccounts (namespace .Name ).Create (test .Ctx (), serviceAccount , metav1.CreateOptions {})
85- test .Expect (err ).NotTo (HaveOccurred ())
8679
87- role := & rbacv1.Role {
88- TypeMeta : metav1.TypeMeta {
89- APIVersion : rbacv1 .SchemeGroupVersion .String (),
90- Kind : "Role" ,
91- },
92- ObjectMeta : metav1.ObjectMeta {
93- Name : "sdk" ,
94- Namespace : namespace .Name ,
95- },
96- Rules : []rbacv1.PolicyRule {
97- {
98- Verbs : []string {"get" , "create" , "delete" , "list" , "patch" , "update" },
99- APIGroups : []string {mcadv1beta1 .GroupName },
100- Resources : []string {"appwrappers" },
101- },
102- {
103- Verbs : []string {"get" , "list" },
104- APIGroups : []string {rayv1 .GroupVersion .Group },
105- Resources : []string {"rayclusters" , "rayclusters/status" },
106- },
107- {
108- Verbs : []string {"get" , "list" },
109- APIGroups : []string {"route.openshift.io" },
110- Resources : []string {"routes" },
111- },
80+ // Create cluster wide RBAC, required for SDK OpenShift check
81+ // TODO reevaluate once SDK change OpenShift detection logic
82+ clusterPolicyRules := []rbacv1.PolicyRule {
83+ {
84+ Verbs : []string {"get" , "list" },
85+ APIGroups : []string {"config.openshift.io" },
86+ Resources : []string {"ingresses" },
87+ ResourceNames : []string {"cluster" },
11288 },
11389 }
114- role , err = test .Client ().Core ().RbacV1 ().Roles (namespace .Name ).Create (test .Ctx (), role , metav1.CreateOptions {})
115- test .Expect (err ).NotTo (HaveOccurred ())
11690
117- roleBinding := & rbacv1.RoleBinding {
118- TypeMeta : metav1.TypeMeta {
119- APIVersion : rbacv1 .SchemeGroupVersion .String (),
120- Kind : "RoleBinding" ,
121- },
122- ObjectMeta : metav1.ObjectMeta {
123- Name : "sdk" ,
124- },
125- RoleRef : rbacv1.RoleRef {
126- APIGroup : rbacv1 .SchemeGroupVersion .Group ,
127- Kind : "Role" ,
128- Name : role .Name ,
129- },
130- Subjects : []rbacv1.Subject {
131- {
132- Kind : "ServiceAccount" ,
133- APIGroup : corev1 .SchemeGroupVersion .Group ,
134- Name : serviceAccount .Name ,
135- Namespace : serviceAccount .Namespace ,
136- },
137- },
138- }
139- _ , err = test .Client ().Core ().RbacV1 ().RoleBindings (namespace .Name ).Create (test .Ctx (), roleBinding , metav1.CreateOptions {})
140- test .Expect (err ).NotTo (HaveOccurred ())
91+ sa := CreateServiceAccount (test , namespace .Name )
92+ role := CreateRole (test , namespace .Name , policyRules )
93+ CreateRoleBinding (test , namespace .Name , sa , role )
94+ clusterRole := CreateClusterRole (test , clusterPolicyRules )
95+ CreateClusterRoleBinding (test , sa , clusterRole )
14196
14297 job := & batchv1.Job {
14398 TypeMeta : metav1.TypeMeta {
@@ -161,7 +116,8 @@ func TestMNISTRayClusterSDK(t *testing.T) {
161116 // See https://github.com/project-codeflare/codeflare-sdk/pull/146
162117 Image : "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e" ,
163118 Env : []corev1.EnvVar {
164- corev1.EnvVar {Name : "PYTHONUSERBASE" , Value : "/workdir" },
119+ {Name : "PYTHONUSERBASE" , Value : "/workdir" },
120+ {Name : "RAY_IMAGE" , Value : GetRayImage ()},
165121 },
166122 Command : []string {"/bin/sh" , "-c" , "pip install codeflare-sdk==" + GetCodeFlareSDKVersion () + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace .Name },
167123 VolumeMounts : []corev1.VolumeMount {
@@ -206,12 +162,31 @@ func TestMNISTRayClusterSDK(t *testing.T) {
206162 },
207163 },
208164 RestartPolicy : corev1 .RestartPolicyNever ,
209- ServiceAccountName : serviceAccount .Name ,
165+ ServiceAccountName : sa .Name ,
210166 },
211167 },
212168 },
213169 }
214- job , err = test .Client ().Core ().BatchV1 ().Jobs (namespace .Name ).Create (test .Ctx (), job , metav1.CreateOptions {})
170+ if GetClusterType (test ) == KindCluster {
171+ // Take first KinD node and redirect pod hostname requests there
172+ node := GetNodes (test )[0 ]
173+ hostname := GetClusterHostname (test )
174+ IP := GetNodeInternalIP (test , node )
175+
176+ test .T ().Logf ("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod" , hostname , IP )
177+ job .Spec .Template .Spec .HostAliases = []corev1.HostAlias {
178+ {
179+ IP : IP ,
180+ Hostnames : []string {hostname },
181+ },
182+ }
183+
184+ // Propagate hostname into Python code as env variable
185+ hostnameEnvVar := corev1.EnvVar {Name : "CLUSTER_HOSTNAME" , Value : hostname }
186+ job .Spec .Template .Spec .Containers [0 ].Env = append (job .Spec .Template .Spec .Containers [0 ].Env , hostnameEnvVar )
187+ }
188+
189+ job , err := test .Client ().Core ().BatchV1 ().Jobs (namespace .Name ).Create (test .Ctx (), job , metav1.CreateOptions {})
215190 test .Expect (err ).NotTo (HaveOccurred ())
216191 test .T ().Logf ("Created Job %s/%s successfully" , job .Namespace , job .Name )
217192
0 commit comments