initial batch of end-to-end tests

hack/e2e-util.sh

@@ -159,6 +159,16 @@ function configure_cluster {
   done
 }
 
+function wait_for_appwrapper_controller {
+    # Sleep until the appwrapper controller is running
+    echo "Waiting for pods in the appwrapper-system namespace to become ready"
+    while [[ $(kubectl get pods -n appwrapper-system -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}' | tr ' ' '\n' | sort -u) != "True" ]]
+    do
+        echo -n "." && sleep 1;
+    done
+    echo ""
+}
+
 # clean up
 function cleanup {
     echo "==========================>>>>> Cleaning up... <<<<<=========================="
@@ -200,25 +210,14 @@ function cleanup {
       echo "'all' Namespaces  list..."
       kubectl get namespaces
 
-      # TODO:  Need to update this for appwrapper system/controller
-
-      local mcad_dispatcher_pod=$(kubectl get pods -n mcad-system | grep mcad-controller | grep -v mcad-controller-runner | awk '{print $1}')
-      local mcad_runner_pod=$(kubectl get pods -n mcad-system | grep mcad-controller-runner | awk '{print $1}')
-      if [[ "$mcad_dispatcher_pod" != "" ]]
-      then
-        echo "===================================================================================="
-        echo "==========================>>>>> MCAD Controller Logs <<<<<=========================="
-        echo "===================================================================================="
-        echo "kubectl logs ${mcad_dispatcher_pod} -n mcad-system"
-        kubectl logs ${mcad_dispatcher_pod} -n mcad-system
-      fi
-      if [[ "$mcad_runner_pod" != "" ]]
+      local appwrapper_controller_pod=$(kubectl get pods -n appwrapper-system | grep appwrapper-controller | awk '{print $1}')
+      if [[ "$appwrapper_controller_pod" != "" ]]
       then
         echo "===================================================================================="
-        echo "==========================>>>>> MCAD Runner Logs <<<<<=============================="
+        echo "==========================>>>>> AppWrapper Controller Logs <<<<<=========================="
         echo "===================================================================================="
-        echo "kubectl logs ${mcad_runner_pod} -n mcad-system"
-        kubectl logs ${mcad_runner_pod} -n mcad-system
+        echo "kubectl logs ${appwrapper_controller_pod} -n appwrapper-system"
+        kubectl logs ${appwrapper_controller_pod} -n appwrapper-system
       fi
     fi
 

hack/run-tests-on-cluster.sh

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Run the e2e tests on an existing cluster with mcad already installed
+# Run the e2e tests on an existing cluster with kueue and AppWrapper already installed
 
 export ROOT_DIR="$(dirname "$(dirname "$(readlink -fn "$0")")")"
 export GORACE=1
@@ -24,6 +24,8 @@ source ${ROOT_DIR}/hack/e2e-util.sh
 
 trap cleanup EXIT
 
+wait_for_appwrapper_controller
+
 run_kuttl_test_suite
 go run github.com/onsi/ginkgo/v2/ginkgo -v -fail-fast --procs 1 -timeout 130m ./test/e2e
 

internal/controller/appwrapper_controller.go

@@ -39,7 +39,7 @@ import (
 )
 
 const (
-	appWrapperLabel     = "workload.codeflare.dev/appwrapper"
+	AppWrapperLabel     = "workload.codeflare.dev/appwrapper"
 	appWrapperFinalizer = "workload.codeflare.dev/finalizer"
 )
 
@@ -275,7 +275,7 @@ func (r *AppWrapperReconciler) SetupWithManager(mgr ctrl.Manager) error {
 // podMapFunc maps pods to appwrappers
 func (r *AppWrapperReconciler) podMapFunc(ctx context.Context, obj client.Object) []reconcile.Request {
 	pod := obj.(*v1.Pod)
-	if name, ok := pod.Labels[appWrapperLabel]; ok {
+	if name, ok := pod.Labels[AppWrapperLabel]; ok {
 		if pod.Status.Phase == v1.PodSucceeded {
 			return []reconcile.Request{{NamespacedName: types.NamespacedName{Namespace: pod.Namespace, Name: name}}}
 		}
@@ -363,10 +363,10 @@ func (r *AppWrapperReconciler) workloadStatus(ctx context.Context, aw *workloadv
 	pods := &v1.PodList{}
 	if err := r.List(ctx, pods,
 		client.InNamespace(aw.Namespace),
-		client.MatchingLabels{appWrapperLabel: aw.Name}); err != nil {
+		client.MatchingLabels{AppWrapperLabel: aw.Name}); err != nil {
 		return nil, err
 	}
-	summary := &podStatusSummary{expected: expectedPodCount(aw)}
+	summary := &podStatusSummary{expected: ExpectedPodCount(aw)}
 
 	for _, pod := range pods.Items {
 		switch pod.Status.Phase {
@@ -392,7 +392,7 @@ func replicas(ps workloadv1beta2.AppWrapperPodSet) int32 {
 	}
 }
 
-func expectedPodCount(aw *workloadv1beta2.AppWrapper) int32 {
+func ExpectedPodCount(aw *workloadv1beta2.AppWrapper) int32 {
 	var expected int32
 	for _, c := range aw.Spec.Components {
 		for _, s := range c.PodSets {

internal/controller/appwrapper_controller_test.go

@@ -78,10 +78,10 @@ var _ = Describe("AppWrapper Controller", func() {
 		Expect((*AppWrapper)(aw).IsSuspended()).Should(BeFalse())
 		podStatus, err := awReconciler.workloadStatus(ctx, aw)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(podStatus.pending).Should(Equal(expectedPodCount(aw)))
+		Expect(podStatus.pending).Should(Equal(ExpectedPodCount(aw)))
 
 		By("Simulating all Pods Running")
-		Expect(setPodStatus(aw, v1.PodRunning, expectedPodCount(aw))).To(Succeed())
+		Expect(setPodStatus(aw, v1.PodRunning, ExpectedPodCount(aw))).To(Succeed())
 		By("Reconciling: Running -> Running")
 		_, err = awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName})
 		Expect(err).NotTo(HaveOccurred())
@@ -96,7 +96,7 @@ var _ = Describe("AppWrapper Controller", func() {
 		Expect((*AppWrapper)(aw).PodsReady()).Should(BeTrue())
 		podStatus, err = awReconciler.workloadStatus(ctx, aw)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(podStatus.running).Should(Equal(expectedPodCount(aw)))
+		Expect(podStatus.running).Should(Equal(ExpectedPodCount(aw)))
 		_, finished := (*AppWrapper)(aw).Finished()
 		Expect(finished).Should(BeFalse())
 	}
@@ -152,11 +152,11 @@ var _ = Describe("AppWrapper Controller", func() {
 		Expect((*AppWrapper)(aw).IsSuspended()).Should(BeFalse())
 		podStatus, err := awReconciler.workloadStatus(ctx, aw)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(podStatus.running).Should(Equal(expectedPodCount(aw) - 1))
+		Expect(podStatus.running).Should(Equal(ExpectedPodCount(aw) - 1))
 		Expect(podStatus.succeeded).Should(Equal(int32(1)))
 
 		By("Simulating all Pods Completing")
-		Expect(setPodStatus(aw, v1.PodSucceeded, expectedPodCount(aw))).To(Succeed())
+		Expect(setPodStatus(aw, v1.PodSucceeded, ExpectedPodCount(aw))).To(Succeed())
 		By("Reconciling: Running -> Succeeded")
 		_, err = awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName})
 		Expect(err).NotTo(HaveOccurred())

internal/controller/appwrapper_fixtures_test.go

@@ -69,7 +69,7 @@ func setPodStatus(aw *workloadv1beta2.AppWrapper, phase v1.PodPhase, numToChange
 		if numToChange <= 0 {
 			return nil
 		}
-		if awn, found := pod.Labels[appWrapperLabel]; found && awn == aw.Name {
+		if awn, found := pod.Labels[AppWrapperLabel]; found && awn == aw.Name {
 			pod.Status.Phase = phase
 			err = k8sClient.Status().Update(ctx, &pod)
 			if err != nil {

internal/controller/workload_controller.go

@@ -120,7 +120,7 @@ func (aw *AppWrapper) RunWithPodSetsInfo(podSetsInfo []podset.PodSetInfo) error
 			}
 		}
 	}
-	awLabels := map[string]string{appWrapperLabel: aw.Name}
+	awLabels := map[string]string{AppWrapperLabel: aw.Name}
 
 	podSetsInfoIndex := 0
 	for componentIndex := range aw.Spec.Components {

test/e2e/appwrapper_test.go

@@ -17,25 +17,85 @@ limitations under the License.
 package e2e
 
 import (
+	"time"
+
 	. "github.com/onsi/ginkgo/v2"
-	// . "github.com/onsi/gomega"
+	. "github.com/onsi/gomega"
 
-	awv1b2 "github.com/project-codeflare/appwrapper/api/v1beta2"
+	workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2"
 )
 
 var _ = Describe("AppWrapper E2E Test", func() {
-	var appwrappers []*awv1b2.AppWrapper
+	var appwrappers []*workloadv1beta2.AppWrapper
 
 	BeforeEach(func() {
-		appwrappers = []*awv1b2.AppWrapper{}
+		appwrappers = []*workloadv1beta2.AppWrapper{}
 	})
 
 	AfterEach(func() {
 		By("Cleaning up test objects")
 		cleanupTestObjects(ctx, appwrappers)
 	})
 
-	It("Dummy Test", func() {
-		By("Testing nothing of interest...")
+	Describe("Creation of Fundamental GVKs", func() {
+		It("Pods", func() {
+			aw := createAppWrapper(ctx, pod(250), pod(250))
+			appwrappers = append(appwrappers, aw)
+			Expect(waitAWPodsReady(ctx, aw)).Should(Succeed())
+		})
+		It("Deployments", func() {
+			aw := createAppWrapper(ctx, deployment(2, 200))
+			appwrappers = append(appwrappers, aw)
+			Expect(waitAWPodsReady(ctx, aw)).Should(Succeed())
+		})
+		It("StatefulSets", func() {
+			aw := createAppWrapper(ctx, statefulset(2, 200))
+			appwrappers = append(appwrappers, aw)
+			Expect(waitAWPodsReady(ctx, aw)).Should(Succeed())
+		})
+		// TODO: Batch v1.Jobs
+		It("Mixed Basic Resources", func() {
+			aw := createAppWrapper(ctx, pod(100), deployment(2, 100), statefulset(2, 100), service())
+			appwrappers = append(appwrappers, aw)
+			Expect(waitAWPodsReady(ctx, aw)).Should(Succeed())
+		})
+	})
+
+	Describe("Error Handling for Invalid Resources", func() {
+		// TODO: Replicate scenarios from the AdmissionController unit tests
+
+	})
+
+	Describe("Queueing and Preemption", func() {
+		It("Basic Queuing", Label("slow"), func() {
+			By("Jobs should be admitted when there is available quota")
+			aw := createAppWrapper(ctx, deployment(2, 500))
+			appwrappers = append(appwrappers, aw)
+			Expect(waitAWPodsReady(ctx, aw)).Should(Succeed())
+			aw2 := createAppWrapper(ctx, deployment(2, 500))
+			appwrappers = append(appwrappers, aw2)
+			Expect(waitAWPodsReady(ctx, aw2)).Should(Succeed())
+
+			By("Jobs should be queued when quota is exhausted")
+			aw3 := createAppWrapper(ctx, deployment(2, 250))
+			appwrappers = append(appwrappers, aw3)
+			Eventually(AppWrapperPhase(ctx, aw3), 10*time.Second).Should(Equal(workloadv1beta2.AppWrapperSuspended))
+			Consistently(AppWrapperPhase(ctx, aw3), 20*time.Second).Should(Equal(workloadv1beta2.AppWrapperSuspended))
+
+			By("Queued job is admitted when quota becomes available")
+			Expect(deleteAppWrapper(ctx, aw.Name, aw.Namespace)).Should(Succeed())
+			appwrappers = []*workloadv1beta2.AppWrapper{aw2, aw3}
+			Expect(waitAWPodsReady(ctx, aw3)).Should(Succeed())
+		})
+
+	})
+
+	Describe("Detection of Completion Status", func() {
+
+	})
+
+	Describe("Load Testing", Label("slow"), func() {
+
 	})
+
 })

test/e2e/e2e_test.go

@@ -33,6 +33,7 @@ var _ = BeforeSuite(func() {
 	log.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
 	ctx = extendContextWithClient(context.Background())
 	ensureNamespaceExists(ctx)
+	ensureTestQueuesExist(ctx)
 })
 
 func TestE2E(t *testing.T) {