From a2e406714e64b0312b62fcbc6ea5afdade1c01d0 Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 29 Feb 2024 10:45:32 -0500 Subject: [PATCH 1/5] WIP on e2e testing --- internal/controller/appwrapper_controller.go | 10 +- .../controller/appwrapper_controller_test.go | 10 +- .../controller/appwrapper_fixtures_test.go | 2 +- internal/controller/workload_controller.go | 2 +- test/e2e/appwrapper_test.go | 11 +- test/e2e/e2e_test.go | 1 + test/e2e/fixtures_test.go | 135 ++++++++++++ test/e2e/util.go | 83 ------- test/e2e/util_test.go | 203 ++++++++++++++++++ 9 files changed, 359 insertions(+), 98 deletions(-) create mode 100644 test/e2e/fixtures_test.go delete mode 100644 test/e2e/util.go create mode 100644 test/e2e/util_test.go diff --git a/internal/controller/appwrapper_controller.go b/internal/controller/appwrapper_controller.go index bd7d14d..c224323 100644 --- a/internal/controller/appwrapper_controller.go +++ b/internal/controller/appwrapper_controller.go @@ -39,7 +39,7 @@ import ( ) const ( - appWrapperLabel = "workload.codeflare.dev/appwrapper" + AppWrapperLabel = "workload.codeflare.dev/appwrapper" appWrapperFinalizer = "workload.codeflare.dev/finalizer" ) @@ -275,7 +275,7 @@ func (r *AppWrapperReconciler) SetupWithManager(mgr ctrl.Manager) error { // podMapFunc maps pods to appwrappers func (r *AppWrapperReconciler) podMapFunc(ctx context.Context, obj client.Object) []reconcile.Request { pod := obj.(*v1.Pod) - if name, ok := pod.Labels[appWrapperLabel]; ok { + if name, ok := pod.Labels[AppWrapperLabel]; ok { if pod.Status.Phase == v1.PodSucceeded { return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: pod.Namespace, Name: name}}} } @@ -363,10 +363,10 @@ func (r *AppWrapperReconciler) workloadStatus(ctx context.Context, aw *workloadv pods := &v1.PodList{} if err := r.List(ctx, pods, client.InNamespace(aw.Namespace), - client.MatchingLabels{appWrapperLabel: aw.Name}); err != nil { + client.MatchingLabels{AppWrapperLabel: aw.Name}); err != nil { return nil, err } - summary := &podStatusSummary{expected: expectedPodCount(aw)} + summary := &podStatusSummary{expected: ExpectedPodCount(aw)} for _, pod := range pods.Items { switch pod.Status.Phase { @@ -392,7 +392,7 @@ func replicas(ps workloadv1beta2.AppWrapperPodSet) int32 { } } -func expectedPodCount(aw *workloadv1beta2.AppWrapper) int32 { +func ExpectedPodCount(aw *workloadv1beta2.AppWrapper) int32 { var expected int32 for _, c := range aw.Spec.Components { for _, s := range c.PodSets { diff --git a/internal/controller/appwrapper_controller_test.go b/internal/controller/appwrapper_controller_test.go index e2f3db4..ace4afa 100644 --- a/internal/controller/appwrapper_controller_test.go +++ b/internal/controller/appwrapper_controller_test.go @@ -78,10 +78,10 @@ var _ = Describe("AppWrapper Controller", func() { Expect((*AppWrapper)(aw).IsSuspended()).Should(BeFalse()) podStatus, err := awReconciler.workloadStatus(ctx, aw) Expect(err).NotTo(HaveOccurred()) - Expect(podStatus.pending).Should(Equal(expectedPodCount(aw))) + Expect(podStatus.pending).Should(Equal(ExpectedPodCount(aw))) By("Simulating all Pods Running") - Expect(setPodStatus(aw, v1.PodRunning, expectedPodCount(aw))).To(Succeed()) + Expect(setPodStatus(aw, v1.PodRunning, ExpectedPodCount(aw))).To(Succeed()) By("Reconciling: Running -> Running") _, err = awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName}) Expect(err).NotTo(HaveOccurred()) @@ -96,7 +96,7 @@ var _ = Describe("AppWrapper Controller", func() { Expect((*AppWrapper)(aw).PodsReady()).Should(BeTrue()) podStatus, err = awReconciler.workloadStatus(ctx, aw) Expect(err).NotTo(HaveOccurred()) - Expect(podStatus.running).Should(Equal(expectedPodCount(aw))) + Expect(podStatus.running).Should(Equal(ExpectedPodCount(aw))) _, finished := (*AppWrapper)(aw).Finished() Expect(finished).Should(BeFalse()) } @@ -152,11 +152,11 @@ var _ = Describe("AppWrapper Controller", func() { Expect((*AppWrapper)(aw).IsSuspended()).Should(BeFalse()) podStatus, err := awReconciler.workloadStatus(ctx, aw) Expect(err).NotTo(HaveOccurred()) - Expect(podStatus.running).Should(Equal(expectedPodCount(aw) - 1)) + Expect(podStatus.running).Should(Equal(ExpectedPodCount(aw) - 1)) Expect(podStatus.succeeded).Should(Equal(int32(1))) By("Simulating all Pods Completing") - Expect(setPodStatus(aw, v1.PodSucceeded, expectedPodCount(aw))).To(Succeed()) + Expect(setPodStatus(aw, v1.PodSucceeded, ExpectedPodCount(aw))).To(Succeed()) By("Reconciling: Running -> Succeeded") _, err = awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName}) Expect(err).NotTo(HaveOccurred()) diff --git a/internal/controller/appwrapper_fixtures_test.go b/internal/controller/appwrapper_fixtures_test.go index ec240d5..c6abd95 100644 --- a/internal/controller/appwrapper_fixtures_test.go +++ b/internal/controller/appwrapper_fixtures_test.go @@ -69,7 +69,7 @@ func setPodStatus(aw *workloadv1beta2.AppWrapper, phase v1.PodPhase, numToChange if numToChange <= 0 { return nil } - if awn, found := pod.Labels[appWrapperLabel]; found && awn == aw.Name { + if awn, found := pod.Labels[AppWrapperLabel]; found && awn == aw.Name { pod.Status.Phase = phase err = k8sClient.Status().Update(ctx, &pod) if err != nil { diff --git a/internal/controller/workload_controller.go b/internal/controller/workload_controller.go index 1d14b2e..473296f 100644 --- a/internal/controller/workload_controller.go +++ b/internal/controller/workload_controller.go @@ -120,7 +120,7 @@ func (aw *AppWrapper) RunWithPodSetsInfo(podSetsInfo []podset.PodSetInfo) error } } } - awLabels := map[string]string{appWrapperLabel: aw.Name} + awLabels := map[string]string{AppWrapperLabel: aw.Name} podSetsInfoIndex := 0 for componentIndex := range aw.Spec.Components { diff --git a/test/e2e/appwrapper_test.go b/test/e2e/appwrapper_test.go index 37260e6..9e265c7 100644 --- a/test/e2e/appwrapper_test.go +++ b/test/e2e/appwrapper_test.go @@ -18,7 +18,7 @@ package e2e import ( . "github.com/onsi/ginkgo/v2" - // . "github.com/onsi/gomega" + . "github.com/onsi/gomega" awv1b2 "github.com/project-codeflare/appwrapper/api/v1beta2" ) @@ -35,7 +35,12 @@ var _ = Describe("AppWrapper E2E Test", func() { cleanupTestObjects(ctx, appwrappers) }) - It("Dummy Test", func() { - By("Testing nothing of interest...") + Describe("Creation of Different GVKs", func() { + It("Pods", func() { + aw := createAppWrapper(ctx, pod(250), pod(250)) + appwrappers = append(appwrappers, aw) + Expect(waitAWPodsReady(ctx, aw)).Should(Succeed()) + }) + }) }) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 9f3cb4c..bf5add6 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -33,6 +33,7 @@ var _ = BeforeSuite(func() { log.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) ctx = extendContextWithClient(context.Background()) ensureNamespaceExists(ctx) + ensureTestQueuesExist(ctx) }) func TestE2E(t *testing.T) { diff --git a/test/e2e/fixtures_test.go b/test/e2e/fixtures_test.go new file mode 100644 index 0000000..b421423 --- /dev/null +++ b/test/e2e/fixtures_test.go @@ -0,0 +1,135 @@ +/* +Copyright 2024 IBM Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "math/rand" + "time" + + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" + + workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" +) + +const charset = "abcdefghijklmnopqrstuvwxyz0123456789" + +func randName(baseName string) string { + seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) + b := make([]byte, 6) + for i := range b { + b[i] = charset[seededRand.Intn(len(charset))] + } + return fmt.Sprintf("%s-%s", baseName, string(b)) +} + +const podYAML = ` +apiVersion: v1 +kind: Pod +metadata: + name: %v +spec: + restartPolicy: Never + containers: + - name: busybox + image: quay.io/project-codeflare/busybox:1.36 + command: ["sh", "-c", "sleep 10"] + resources: + requests: + cpu: %v` + +func pod(milliCPU int64) workloadv1beta2.AppWrapperComponent { + yamlString := fmt.Sprintf(podYAML, + randName("pod"), + resource.NewMilliQuantity(milliCPU, resource.DecimalSI)) + + jsonBytes, err := yaml.YAMLToJSON([]byte(yamlString)) + Expect(err).NotTo(HaveOccurred()) + replicas := int32(1) + return workloadv1beta2.AppWrapperComponent{ + PodSets: []workloadv1beta2.AppWrapperPodSet{{Replicas: &replicas, Path: "template"}}, + Template: runtime.RawExtension{Raw: jsonBytes}, + } +} + +const serviceYAML = ` +apiVersion: v1 +kind: Service +metadata: + name: %v +spec: + selector: + app: test + ports: + - protocol: TCP + port: 80 + targetPort: 8080` + +func service() workloadv1beta2.AppWrapperComponent { + yamlString := fmt.Sprintf(serviceYAML, randName("service")) + jsonBytes, err := yaml.YAMLToJSON([]byte(yamlString)) + Expect(err).NotTo(HaveOccurred()) + return workloadv1beta2.AppWrapperComponent{ + PodSets: []workloadv1beta2.AppWrapperPodSet{}, + Template: runtime.RawExtension{Raw: jsonBytes}, + } +} + +const deploymentYAML = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: %v + labels: + app: test +spec: + replicas: %v + selector: + matchLabels: + app: test + template: + metadata: + labels: + app: test + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: busybox + image: quay.io/project-codeflare/busybox:1.36 + command: ["sh", "-c", "sleep 10000"] + resources: + requests: + cpu: %v` + +func deployment(replicaCount int, milliCPU int64) workloadv1beta2.AppWrapperComponent { + yamlString := fmt.Sprintf(deploymentYAML, + randName("deployment"), + replicaCount, + resource.NewMilliQuantity(milliCPU, resource.DecimalSI)) + + jsonBytes, err := yaml.YAMLToJSON([]byte(yamlString)) + Expect(err).NotTo(HaveOccurred()) + replicas := int32(replicaCount) + return workloadv1beta2.AppWrapperComponent{ + PodSets: []workloadv1beta2.AppWrapperPodSet{{Replicas: &replicas, Path: "template.spec.template"}}, + Template: runtime.RawExtension{Raw: jsonBytes}, + } +} diff --git a/test/e2e/util.go b/test/e2e/util.go deleted file mode 100644 index 0a30586..0000000 --- a/test/e2e/util.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2024 IBM Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "context" - - // . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - awv1b2 "github.com/project-codeflare/appwrapper/api/v1beta2" -) - -const testNamespace = "test" - -type myKey struct { - key string -} - -func getClient(ctx context.Context) client.Client { - kubeClient := ctx.Value(myKey{key: "kubeclient"}) - return kubeClient.(client.Client) -} - -func extendContextWithClient(ctx context.Context) context.Context { - scheme := runtime.NewScheme() - _ = clientgoscheme.AddToScheme(scheme) - _ = awv1b2.AddToScheme(scheme) - kubeclient, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme}) - Expect(err).NotTo(HaveOccurred()) - return context.WithValue(ctx, myKey{key: "kubeclient"}, kubeclient) -} - -func ensureNamespaceExists(ctx context.Context) { - err := getClient(ctx).Create(ctx, &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: testNamespace, - }, - }) - Expect(client.IgnoreAlreadyExists(err)).NotTo(HaveOccurred()) -} - -func cleanupTestObjects(ctx context.Context, appwrappers []*awv1b2.AppWrapper) { - if appwrappers == nil { - return - } - - // TODO! -- pull code from e2e.util and adjust - - /* - for _, aw := range appwrappers { - awNamespace := aw.Namespace - awName := aw.Name - - err := deleteAppWrapper(ctx, aw.Name, aw.Namespace) - Expect(err).NotTo(HaveOccurred()) - err = waitAWPodsDeleted(ctx, awNamespace, awName) - Expect(err).NotTo(HaveOccurred()) - } - */ -} diff --git a/test/e2e/util_test.go b/test/e2e/util_test.go new file mode 100644 index 0000000..ae8ed71 --- /dev/null +++ b/test/e2e/util_test.go @@ -0,0 +1,203 @@ +/* +Copyright 2024 IBM Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "time" + + // . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kc "sigs.k8s.io/kueue/pkg/controller/constants" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" + "github.com/project-codeflare/appwrapper/internal/controller" +) + +const ( + testNamespace = "e2e-test" + testFlavorName = "e2e-test-flavor" + testQueueName = "e2e-test-queue" +) + +type myKey struct { + key string +} + +func getClient(ctx context.Context) client.Client { + kubeClient := ctx.Value(myKey{key: "kubeclient"}) + return kubeClient.(client.Client) +} + +func extendContextWithClient(ctx context.Context) context.Context { + scheme := runtime.NewScheme() + Expect(clientgoscheme.AddToScheme(scheme)).To(Succeed()) + Expect(workloadv1beta2.AddToScheme(scheme)).To(Succeed()) + Expect(kueue.AddToScheme(scheme)).To(Succeed()) + kubeclient, err := client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme}) + Expect(err).To(Succeed()) + return context.WithValue(ctx, myKey{key: "kubeclient"}, kubeclient) +} + +func ensureNamespaceExists(ctx context.Context) { + err := getClient(ctx).Create(ctx, &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + }, + }) + Expect(client.IgnoreAlreadyExists(err)).To(Succeed()) +} + +func ensureTestQueuesExist(ctx context.Context) { + rf := &kueue.ResourceFlavor{ObjectMeta: metav1.ObjectMeta{Name: testFlavorName}} + err := getClient(ctx).Create(ctx, rf) + Expect(client.IgnoreAlreadyExists(err)).To(Succeed()) + cq := &kueue.ClusterQueue{ + ObjectMeta: metav1.ObjectMeta{Name: testQueueName}, + Spec: kueue.ClusterQueueSpec{ + NamespaceSelector: &metav1.LabelSelector{}, + ResourceGroups: []kueue.ResourceGroup{{ + CoveredResources: []v1.ResourceName{v1.ResourceCPU}, + Flavors: []kueue.FlavorQuotas{{ + Name: testFlavorName, + Resources: []kueue.ResourceQuota{{Name: v1.ResourceCPU, NominalQuota: *resource.NewMilliQuantity(2000, resource.DecimalSI)}}, + }}, + }, + }, + }, + } + err = getClient(ctx).Create(ctx, cq) + Expect(client.IgnoreAlreadyExists(err)).To(Succeed()) + + lq := &kueue.LocalQueue{ + ObjectMeta: metav1.ObjectMeta{Name: testQueueName, Namespace: testNamespace}, + Spec: kueue.LocalQueueSpec{ClusterQueue: kueue.ClusterQueueReference(testQueueName)}, + } + err = getClient(ctx).Create(ctx, lq) + Expect(client.IgnoreAlreadyExists(err)).To(Succeed()) +} + +func cleanupTestObjects(ctx context.Context, appwrappers []*workloadv1beta2.AppWrapper) { + if appwrappers == nil { + return + } + for _, aw := range appwrappers { + awNamespace := aw.Namespace + awName := aw.Name + + err := deleteAppWrapper(ctx, aw.Name, aw.Namespace) + Expect(err).To(Succeed()) + err = waitAWPodsDeleted(ctx, awNamespace, awName) + Expect(err).To(Succeed()) + } +} + +func deleteAppWrapper(ctx context.Context, name string, namespace string) error { + foreground := metav1.DeletePropagationForeground + aw := &workloadv1beta2.AppWrapper{ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }} + return getClient(ctx).Delete(ctx, aw, &client.DeleteOptions{PropagationPolicy: &foreground}) +} + +func createAppWrapper(ctx context.Context, components ...workloadv1beta2.AppWrapperComponent) *workloadv1beta2.AppWrapper { + aw := toAppWrapper(components...) + Expect(getClient(ctx).Create(ctx, aw)).To(Succeed()) + return aw +} + +func toAppWrapper(components ...workloadv1beta2.AppWrapperComponent) *workloadv1beta2.AppWrapper { + return &workloadv1beta2.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: randName("aw"), + Namespace: testNamespace, + Annotations: map[string]string{kc.QueueLabel: testQueueName}, + }, + Spec: workloadv1beta2.AppWrapperSpec{Components: components}, + } +} + +func getAppWrapper(ctx context.Context, typeNamespacedName types.NamespacedName) *workloadv1beta2.AppWrapper { + aw := &workloadv1beta2.AppWrapper{} + err := getClient(ctx).Get(ctx, typeNamespacedName, aw) + Expect(err).NotTo(HaveOccurred()) + return aw +} + +func podsInPhase(awNamespace string, awName string, phase []v1.PodPhase, minimumPodCount int32) wait.ConditionWithContextFunc { + return func(ctx context.Context) (bool, error) { + podList := &v1.PodList{} + err := getClient(ctx).List(ctx, podList, &client.ListOptions{Namespace: awNamespace}) + if err != nil { + return false, err + } + + matchingPodCount := int32(0) + for _, pod := range podList.Items { + if awn, found := pod.Labels[controller.AppWrapperLabel]; found && awn == awName { + for _, p := range phase { + if pod.Status.Phase == p { + matchingPodCount++ + break + } + } + } + } + + return minimumPodCount <= matchingPodCount, nil + } +} + +func noPodsExist(awNamespace string, awName string) wait.ConditionWithContextFunc { + return func(ctx context.Context) (bool, error) { + podList := &v1.PodList{} + err := getClient(ctx).List(context.Background(), podList, &client.ListOptions{Namespace: awNamespace}) + if err != nil { + return false, err + } + + for _, podFromPodList := range podList.Items { + if awn, found := podFromPodList.Labels[controller.AppWrapperLabel]; found && awn == awName { + return false, nil + } + } + return true, nil + } +} + +func waitAWPodsDeleted(ctx context.Context, awNamespace string, awName string) error { + return wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 90*time.Second, true, noPodsExist(awNamespace, awName)) +} + +func waitAWPodsReady(ctx context.Context, aw *workloadv1beta2.AppWrapper) error { + numExpected := controller.ExpectedPodCount(aw) + phases := []v1.PodPhase{v1.PodRunning, v1.PodSucceeded} + return wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 90*time.Second, true, podsInPhase(aw.Namespace, aw.Name, phases, numExpected)) +} From bd69dac1214eb9e4b673a61566a138043715f6de Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 29 Feb 2024 12:37:26 -0500 Subject: [PATCH 2/5] update namespace/pod names --- hack/e2e-util.sh | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/hack/e2e-util.sh b/hack/e2e-util.sh index ba6d8d4..6720ced 100755 --- a/hack/e2e-util.sh +++ b/hack/e2e-util.sh @@ -202,23 +202,14 @@ function cleanup { # TODO: Need to update this for appwrapper system/controller - local mcad_dispatcher_pod=$(kubectl get pods -n mcad-system | grep mcad-controller | grep -v mcad-controller-runner | awk '{print $1}') - local mcad_runner_pod=$(kubectl get pods -n mcad-system | grep mcad-controller-runner | awk '{print $1}') - if [[ "$mcad_dispatcher_pod" != "" ]] + local appwrapper_controller_pod=$(kubectl get pods -n appwrapper-system | grep appwrapper-controller | awk '{print $1}') + if [[ "$appwrapper_controller_pod" != "" ]] then echo "====================================================================================" - echo "==========================>>>>> MCAD Controller Logs <<<<<==========================" + echo "==========================>>>>> AppWrapper Controller Logs <<<<<==========================" echo "====================================================================================" - echo "kubectl logs ${mcad_dispatcher_pod} -n mcad-system" - kubectl logs ${mcad_dispatcher_pod} -n mcad-system - fi - if [[ "$mcad_runner_pod" != "" ]] - then - echo "====================================================================================" - echo "==========================>>>>> MCAD Runner Logs <<<<<==============================" - echo "====================================================================================" - echo "kubectl logs ${mcad_runner_pod} -n mcad-system" - kubectl logs ${mcad_runner_pod} -n mcad-system + echo "kubectl logs ${appwrapper_controller_pod} -n appwrapper-system" + kubectl logs ${appwrapper_controller_pod} -n appwrapper-system fi fi From c43c719e0b8d280fd4878f8642216d195421f286 Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 29 Feb 2024 17:26:07 -0500 Subject: [PATCH 3/5] first set of basic e2e tests --- test/e2e/appwrapper_test.go | 50 ++++++++++++++++++++++++++++++++++++- test/e2e/fixtures_test.go | 41 ++++++++++++++++++++++++++++++ test/e2e/util_test.go | 26 +++++++++++++++++++ 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/test/e2e/appwrapper_test.go b/test/e2e/appwrapper_test.go index 9e265c7..6183bd3 100644 --- a/test/e2e/appwrapper_test.go +++ b/test/e2e/appwrapper_test.go @@ -35,12 +35,60 @@ var _ = Describe("AppWrapper E2E Test", func() { cleanupTestObjects(ctx, appwrappers) }) - Describe("Creation of Different GVKs", func() { + Describe("Creation of Fundamental GVKs", func() { It("Pods", func() { aw := createAppWrapper(ctx, pod(250), pod(250)) appwrappers = append(appwrappers, aw) Expect(waitAWPodsReady(ctx, aw)).Should(Succeed()) }) + It("Deployments", func() { + aw := createAppWrapper(ctx, deployment(2, 200)) + appwrappers = append(appwrappers, aw) + Expect(waitAWPodsReady(ctx, aw)).Should(Succeed()) + }) + It("StatefulSets", func() { + aw := createAppWrapper(ctx, statefulset(2, 200)) + appwrappers = append(appwrappers, aw) + Expect(waitAWPodsReady(ctx, aw)).Should(Succeed()) + }) + // TODO: Batch v1.Jobs + It("Mixed Basic Resources", func() { + aw := createAppWrapper(ctx, pod(100), deployment(2, 100), statefulset(2, 100), service()) + appwrappers = append(appwrappers, aw) + Expect(waitAWPodsReady(ctx, aw)).Should(Succeed()) + }) + }) + + Describe("Error Handling for Invalid Resources", func() { + // TODO: Replicate scenarios from the AdmissionController unit tests + + }) + + Describe("Queueing and Preemption", func() { + It("Basic Queuing", Label("slow"), func() { + By("Jobs should be admitted when there is available quota") + aw := createAppWrapper(ctx, deployment(2, 250)) + appwrappers = append(appwrappers, aw) + Expect(waitAWPodsReady(ctx, aw)).Should(Succeed()) + aw2 := createAppWrapper(ctx, deployment(2, 250)) + appwrappers = append(appwrappers, aw2) + Expect(waitAWPodsReady(ctx, aw2)).Should(Succeed()) + + By("Jobs should be queued when quota remains") + aw3 := createAppWrapper(ctx, deployment(2, 250)) + appwrappers = append(appwrappers, aw3) + Consistently(AppWrapperIsQueued(ctx, aw3), "20s").Should(BeTrue()) + + }) }) + + Describe("Detection of Completion Status", func() { + + }) + + Describe("Load Testing", Label("slow"), func() { + + }) + }) diff --git a/test/e2e/fixtures_test.go b/test/e2e/fixtures_test.go index b421423..e47af40 100644 --- a/test/e2e/fixtures_test.go +++ b/test/e2e/fixtures_test.go @@ -133,3 +133,44 @@ func deployment(replicaCount int, milliCPU int64) workloadv1beta2.AppWrapperComp Template: runtime.RawExtension{Raw: jsonBytes}, } } + +const statefulesetYAML = ` +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: %v + labels: + app: test +spec: + replicas: %v + selector: + matchLabels: + app: test + template: + metadata: + labels: + app: test + spec: + containers: + terminationGracePeriodSeconds: 0 + - name: busybox + image: quay.io/project-codeflare/busybox:1.36 + command: ["sh", "-c", "sleep 10000"] + resources: + requests: + cpu: %v` + +func statefulset(replicaCount int, milliCPU int64) workloadv1beta2.AppWrapperComponent { + yamlString := fmt.Sprintf(statefulesetYAML, + randName("statefulset"), + replicaCount, + resource.NewMilliQuantity(milliCPU, resource.DecimalSI)) + + jsonBytes, err := yaml.YAMLToJSON([]byte(yamlString)) + Expect(err).NotTo(HaveOccurred()) + replicas := int32(replicaCount) + return workloadv1beta2.AppWrapperComponent{ + PodSets: []workloadv1beta2.AppWrapperPodSet{{Replicas: &replicas, Path: "template.spec.template"}}, + Template: runtime.RawExtension{Raw: jsonBytes}, + } +} diff --git a/test/e2e/util_test.go b/test/e2e/util_test.go index ae8ed71..bbbdc41 100644 --- a/test/e2e/util_test.go +++ b/test/e2e/util_test.go @@ -21,6 +21,7 @@ import ( "time" // . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/resource" @@ -201,3 +202,28 @@ func waitAWPodsReady(ctx context.Context, aw *workloadv1beta2.AppWrapper) error phases := []v1.PodPhase{v1.PodRunning, v1.PodSucceeded} return wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 90*time.Second, true, podsInPhase(aw.Namespace, aw.Name, phases, numExpected)) } + +func AppWrapperIsQueued(ctx context.Context, aw *workloadv1beta2.AppWrapper) func(g gomega.Gomega) (bool, error) { + name := aw.Name + namespace := aw.Namespace + return func(g gomega.Gomega) (bool, error) { + aw := &workloadv1beta2.AppWrapper{} + err := getClient(ctx).Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, aw) + g.Expect(err).NotTo(gomega.HaveOccurred()) + if aw.Spec.Suspend != true { + return false, nil + } + podList := &v1.PodList{} + err = getClient(ctx).List(context.Background(), podList, &client.ListOptions{Namespace: namespace}) + if err != nil { + return false, err + } + + for _, podFromPodList := range podList.Items { + if awn, found := podFromPodList.Labels[controller.AppWrapperLabel]; found && awn == name { + return false, nil + } + } + return true, nil + } +} From 9fa9ad6ca399534d9711d64d9a03cbfe9689f077 Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 29 Feb 2024 17:55:41 -0500 Subject: [PATCH 4/5] refine first batch of tests --- test/e2e/appwrapper_test.go | 21 ++++++++++++++------- test/e2e/fixtures_test.go | 2 +- test/e2e/util_test.go | 23 ++++------------------- 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/test/e2e/appwrapper_test.go b/test/e2e/appwrapper_test.go index 6183bd3..13ae29c 100644 --- a/test/e2e/appwrapper_test.go +++ b/test/e2e/appwrapper_test.go @@ -17,17 +17,19 @@ limitations under the License. package e2e import ( + "time" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - awv1b2 "github.com/project-codeflare/appwrapper/api/v1beta2" + workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" ) var _ = Describe("AppWrapper E2E Test", func() { - var appwrappers []*awv1b2.AppWrapper + var appwrappers []*workloadv1beta2.AppWrapper BeforeEach(func() { - appwrappers = []*awv1b2.AppWrapper{} + appwrappers = []*workloadv1beta2.AppWrapper{} }) AfterEach(func() { @@ -67,18 +69,23 @@ var _ = Describe("AppWrapper E2E Test", func() { Describe("Queueing and Preemption", func() { It("Basic Queuing", Label("slow"), func() { By("Jobs should be admitted when there is available quota") - aw := createAppWrapper(ctx, deployment(2, 250)) + aw := createAppWrapper(ctx, deployment(2, 500)) appwrappers = append(appwrappers, aw) Expect(waitAWPodsReady(ctx, aw)).Should(Succeed()) - aw2 := createAppWrapper(ctx, deployment(2, 250)) + aw2 := createAppWrapper(ctx, deployment(2, 500)) appwrappers = append(appwrappers, aw2) Expect(waitAWPodsReady(ctx, aw2)).Should(Succeed()) - By("Jobs should be queued when quota remains") + By("Jobs should be queued when quota is exhausted") aw3 := createAppWrapper(ctx, deployment(2, 250)) appwrappers = append(appwrappers, aw3) - Consistently(AppWrapperIsQueued(ctx, aw3), "20s").Should(BeTrue()) + Eventually(AppWrapperPhase(ctx, aw3), 10*time.Second).Should(Equal(workloadv1beta2.AppWrapperSuspended)) + Consistently(AppWrapperPhase(ctx, aw3), 20*time.Second).Should(Equal(workloadv1beta2.AppWrapperSuspended)) + By("Queued job is admitted when quota becomes available") + Expect(deleteAppWrapper(ctx, aw.Name, aw.Namespace)).Should(Succeed()) + appwrappers = []*workloadv1beta2.AppWrapper{aw2, aw3} + Expect(waitAWPodsReady(ctx, aw3)).Should(Succeed()) }) }) diff --git a/test/e2e/fixtures_test.go b/test/e2e/fixtures_test.go index e47af40..cbe5dce 100644 --- a/test/e2e/fixtures_test.go +++ b/test/e2e/fixtures_test.go @@ -151,8 +151,8 @@ spec: labels: app: test spec: + terminationGracePeriodSeconds: 0 containers: - terminationGracePeriodSeconds: 0 - name: busybox image: quay.io/project-codeflare/busybox:1.36 command: ["sh", "-c", "sleep 10000"] diff --git a/test/e2e/util_test.go b/test/e2e/util_test.go index bbbdc41..19cad2a 100644 --- a/test/e2e/util_test.go +++ b/test/e2e/util_test.go @@ -21,7 +21,6 @@ import ( "time" // . "github.com/onsi/ginkgo/v2" - "github.com/onsi/gomega" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/resource" @@ -203,27 +202,13 @@ func waitAWPodsReady(ctx context.Context, aw *workloadv1beta2.AppWrapper) error return wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 90*time.Second, true, podsInPhase(aw.Namespace, aw.Name, phases, numExpected)) } -func AppWrapperIsQueued(ctx context.Context, aw *workloadv1beta2.AppWrapper) func(g gomega.Gomega) (bool, error) { +func AppWrapperPhase(ctx context.Context, aw *workloadv1beta2.AppWrapper) func(g Gomega) workloadv1beta2.AppWrapperPhase { name := aw.Name namespace := aw.Namespace - return func(g gomega.Gomega) (bool, error) { + return func(g Gomega) workloadv1beta2.AppWrapperPhase { aw := &workloadv1beta2.AppWrapper{} err := getClient(ctx).Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, aw) - g.Expect(err).NotTo(gomega.HaveOccurred()) - if aw.Spec.Suspend != true { - return false, nil - } - podList := &v1.PodList{} - err = getClient(ctx).List(context.Background(), podList, &client.ListOptions{Namespace: namespace}) - if err != nil { - return false, err - } - - for _, podFromPodList := range podList.Items { - if awn, found := podFromPodList.Labels[controller.AppWrapperLabel]; found && awn == name { - return false, nil - } - } - return true, nil + g.Expect(err).NotTo(HaveOccurred()) + return aw.Status.Phase } } From 724ba499b0cbee0a939fdad67c5ef193f571de2d Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 29 Feb 2024 18:15:49 -0500 Subject: [PATCH 5/5] ensure controller is running before initiating tests --- hack/e2e-util.sh | 12 ++++++++++-- hack/run-tests-on-cluster.sh | 4 +++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/hack/e2e-util.sh b/hack/e2e-util.sh index 6720ced..cc594ac 100755 --- a/hack/e2e-util.sh +++ b/hack/e2e-util.sh @@ -159,6 +159,16 @@ function configure_cluster { done } +function wait_for_appwrapper_controller { + # Sleep until the appwrapper controller is running + echo "Waiting for pods in the appwrapper-system namespace to become ready" + while [[ $(kubectl get pods -n appwrapper-system -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}' | tr ' ' '\n' | sort -u) != "True" ]] + do + echo -n "." && sleep 1; + done + echo "" +} + # clean up function cleanup { echo "==========================>>>>> Cleaning up... <<<<<==========================" @@ -200,8 +210,6 @@ function cleanup { echo "'all' Namespaces list..." kubectl get namespaces - # TODO: Need to update this for appwrapper system/controller - local appwrapper_controller_pod=$(kubectl get pods -n appwrapper-system | grep appwrapper-controller | awk '{print $1}') if [[ "$appwrapper_controller_pod" != "" ]] then diff --git a/hack/run-tests-on-cluster.sh b/hack/run-tests-on-cluster.sh index dcf6886..ccfcd3c 100755 --- a/hack/run-tests-on-cluster.sh +++ b/hack/run-tests-on-cluster.sh @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Run the e2e tests on an existing cluster with mcad already installed +# Run the e2e tests on an existing cluster with kueue and AppWrapper already installed export ROOT_DIR="$(dirname "$(dirname "$(readlink -fn "$0")")")" export GORACE=1 @@ -24,6 +24,8 @@ source ${ROOT_DIR}/hack/e2e-util.sh trap cleanup EXIT +wait_for_appwrapper_controller + run_kuttl_test_suite go run github.com/onsi/ginkgo/v2/ginkgo -v -fail-fast --procs 1 -timeout 130m ./test/e2e