Skip to content

Commit 26e5691

Browse files
sutaakarsutaakar
committed
Enable TestMNISTRayClusterSDK test
1 parent e95759a commit 26e5691

File tree

8 files changed

+79
-182
lines changed

8 files changed

+79
-182
lines changed

‎.github/actions/kind/action.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
name: "Set up KinD"
22
description: "Step to start and configure KinD cluster"
33

4+
inputs:
5+
kind-node-hostname:
6+
description: "Hostname of the main kind node"
7+
required: false
8+
default: kind
9+
410
runs:
511
using: "composite"
612
steps:
@@ -56,3 +62,13 @@ runs:
5662
curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/"${VERSION}"/deploy/static/provider/kind/deploy.yaml | sed "s/--publish-status-address=localhost/--report-node-internal-ip-address\\n - --status-update-interval=10/g" | kubectl apply -f -
5763
kubectl annotate ingressclass nginx "ingressclass.kubernetes.io/is-default-class=true"
5864
kubectl -n ingress-nginx wait --timeout=300s --for=condition=Available deployments --all
65+
66+
- name: Add ${{ inputs.kind-node-hostname }} host to machine hosts
67+
shell: bash
68+
run: echo "127.0.0.1 ${{ inputs.kind-node-hostname }}" | sudo tee -a /etc/hosts
69+
70+
- name: Set env variables for tests to properly leverage KinD cluster
71+
shell: bash
72+
run: |
73+
echo "CLUSTER_TYPE=KIND" >> $GITHUB_ENV
74+
echo "CLUSTER_HOSTNAME=kind" >> $GITHUB_ENV

‎.github/workflows/e2e_tests.yaml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ jobs:
7171
echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
7272
7373
set -euo pipefail
74-
go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
74+
go test -timeout 30m -run TestMNISTRayClusterSDK -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
7575
7676
- name: Print CodeFlare operator logs
7777
if: always() && steps.deploy.outcome == 'success'
@@ -85,6 +85,13 @@ jobs:
8585
echo "Printing KubeRay operator logs"
8686
kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
8787
88+
- name: Print Ingress
89+
if: always() && steps.deploy.outcome == 'success'
90+
run: |
91+
echo "Printing Ingress"
92+
kubectl describe ingress -A
93+
kubectl get ingress -A -o yaml
94+
8895
- name: Upload logs
8996
uses: actions/upload-artifact@v3
9097
if: always() && steps.deploy.outcome == 'success'

‎.github/workflows/olm_tests.yaml‎

Lines changed: 0 additions & 174 deletions
This file was deleted.

‎go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ require (
2121

2222
replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3
2323

24+
replace github.com/project-codeflare/codeflare-common => github.com/sutaakar/codeflare-common v0.0.0-20231110120152-dbaedcac175e
25+
2426
require (
2527
github.com/NYTimes/gziphandler v1.1.1 // indirect
2628
github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -391,8 +391,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
391391
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
392392
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
393393
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
394-
github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 h1:TRMLDP6IYt0CAd3+BkvY/r2lkpjI3sOsxf3tnQojZ9k=
395-
github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk=
396394
github.com/project-codeflare/instascale v0.3.0 h1:PSlwbqqUsFTkTQ5KUhMFRebfokySnEZwav97xZixLQs=
397395
github.com/project-codeflare/instascale v0.3.0/go.mod h1:IU1Wl+zqTpMpZ49BOcr6U+A6gF3AjcmFdKo9ZwP3TDI=
398396
github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 h1:dU2Ev0SijdNm30Y9mjdKJL1Fp6l07rnRBKhSbx1kX9g=
@@ -463,6 +461,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
463461
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
464462
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
465463
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
464+
github.com/sutaakar/codeflare-common v0.0.0-20231110120152-dbaedcac175e h1:xWNK1SCWHkmlpnelVjnmAYdoCTJ466B21Ssa1GGUbk4=
465+
github.com/sutaakar/codeflare-common v0.0.0-20231110120152-dbaedcac175e/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk=
466466
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
467467
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
468468
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=

‎test/e2e/kind.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ nodes:
3030
nodeRegistration:
3131
kubeletExtraArgs:
3232
node-labels: "ingress-ready=true"
33+
extraPortMappings:
34+
- containerPort: 80
35+
hostPort: 80
36+
listenAddress: "0.0.0.0"
37+
protocol: tcp
3338
EOF
3439

3540
echo "Deploying Ingress controller into KinD cluster"

‎test/e2e/mnist_raycluster_sdk.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import sys
2+
import os
23

34
from time import sleep
45

@@ -9,16 +10,36 @@
910

1011
namespace = sys.argv[1]
1112

13+
ingress_options = {}
14+
15+
host = os.getenv('CLUSTER_HOSTNAME')
16+
if host is not None:
17+
ingress_options = {
18+
"ingresses": [
19+
{
20+
"ingressName": "ray-dashboard",
21+
"port": 8265,
22+
"pathType": "Prefix",
23+
"path": "/",
24+
"host": host,
25+
},
26+
]
27+
}
28+
29+
1230
cluster = Cluster(ClusterConfiguration(
1331
name='mnist',
1432
namespace=namespace,
1533
num_workers=1,
34+
head_cpus='500m',
35+
head_memory=2,
1636
min_cpus='500m',
1737
max_cpus=1,
1838
min_memory=0.5,
19-
max_memory=1,
39+
max_memory=2,
2040
num_gpus=0,
2141
instascale=False,
42+
ingress_options = ingress_options
2243
))
2344

2445
cluster.up()

‎test/e2e/mnist_raycluster_sdk_test.go‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,6 @@ func TestMNISTRayClusterSDK(t *testing.T) {
4040
test := With(t)
4141
test.T().Parallel()
4242

43-
// Currently blocked by https://github.com/project-codeflare/codeflare-sdk/pull/251 , remove the skip once SDK with the PR is released
44-
test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/251")
45-
4643
// Create a namespace
4744
namespace := test.NewTestNamespace()
4845

@@ -109,6 +106,11 @@ func TestMNISTRayClusterSDK(t *testing.T) {
109106
APIGroups: []string{"route.openshift.io"},
110107
Resources: []string{"routes"},
111108
},
109+
{
110+
Verbs: []string{"get", "list"},
111+
APIGroups: []string{"networking.k8s.io"},
112+
Resources: []string{"ingresses"},
113+
},
112114
},
113115
}
114116
role, err = test.Client().Core().RbacV1().Roles(namespace.Name).Create(test.Ctx(), role, metav1.CreateOptions{})
@@ -163,7 +165,7 @@ func TestMNISTRayClusterSDK(t *testing.T) {
163165
Env: []corev1.EnvVar{
164166
corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"},
165167
},
166-
Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name},
168+
Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==0.11.0 && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name},
167169
VolumeMounts: []corev1.VolumeMount{
168170
{
169171
Name: "test",
@@ -211,6 +213,24 @@ func TestMNISTRayClusterSDK(t *testing.T) {
211213
},
212214
},
213215
}
216+
if GetClusterType(test) == KindCluster {
217+
// Take first KinD node and redirect pod hostname requests there
218+
node := GetNodes(test)[0]
219+
hostname := GetClusterHostname(test)
220+
IP := GetNodeInternalIP(test, node)
221+
222+
test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP)
223+
job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{
224+
{
225+
IP: IP,
226+
Hostnames: []string{hostname},
227+
},
228+
}
229+
230+
hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname}
231+
job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar)
232+
}
233+
214234
job, err = test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{})
215235
test.Expect(err).NotTo(HaveOccurred())
216236
test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name)

0 commit comments

Comments
 (0)