Skip to content

Commit 2be04f0

Browse files
dgrove-ossdgrove-oss
authored
unit tests for Controllers (#32)
1 parent 467ee65 commit 2be04f0

File tree

3 files changed

+252
-59
lines changed

3 files changed

+252
-59
lines changed

‎internal/controller/appwrapper_controller_test.go‎

Lines changed: 213 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -16,73 +16,230 @@ limitations under the License.
1616

1717
package controller
1818

19-
/*
20-
2119
import (
22-
"context"
23-
2420
. "github.com/onsi/ginkgo/v2"
2521
. "github.com/onsi/gomega"
26-
"k8s.io/apimachinery/pkg/api/errors"
22+
v1 "k8s.io/api/core/v1"
23+
"k8s.io/apimachinery/pkg/api/meta"
2724
"k8s.io/apimachinery/pkg/types"
25+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
2826
"sigs.k8s.io/controller-runtime/pkg/reconcile"
29-
30-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
28+
"sigs.k8s.io/kueue/pkg/podset"
29+
utilslices "sigs.k8s.io/kueue/pkg/util/slices"
3130

3231
workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2"
3332
)
3433

3534
var _ = Describe("AppWrapper Controller", func() {
36-
Context("When reconciling a resource", func() {
37-
const resourceName = "test-resource"
35+
var awReconciler *AppWrapperReconciler
36+
var awName types.NamespacedName
37+
markerPodSet := podset.PodSetInfo{
38+
Labels: map[string]string{"testkey1": "value1"},
39+
Annotations: map[string]string{"test2": "test2"},
40+
}
41+
var kueuePodSets []kueue.PodSet
42+
43+
advanceToRunning := func() {
44+
By("Reconciling: Empty -> Suspended")
45+
_, err := awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName})
46+
Expect(err).NotTo(HaveOccurred())
47+
48+
aw := getAppWrapper(awName)
49+
Expect(aw.Status.Phase).Should(Equal(workloadv1beta2.AppWrapperSuspended))
50+
Expect(controllerutil.ContainsFinalizer(aw, appWrapperFinalizer)).Should(BeTrue())
51+
52+
By("Updating aw.Spec by invoking RunWithPodSetsInfo")
53+
Expect((*AppWrapper)(aw).RunWithPodSetsInfo([]podset.PodSetInfo{markerPodSet, markerPodSet})).To(Succeed())
54+
Expect(aw.Spec.Suspend).To(BeFalse())
55+
Expect(k8sClient.Update(ctx, aw)).To(Succeed())
56+
57+
By("Reconciling: Suspended -> Resuming")
58+
_, err = awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName})
59+
Expect(err).NotTo(HaveOccurred())
60+
61+
aw = getAppWrapper(awName)
62+
Expect(aw.Status.Phase).Should(Equal(workloadv1beta2.AppWrapperResuming))
63+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed))).Should(BeTrue())
64+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.QuotaReserved))).Should(BeTrue())
65+
Expect((*AppWrapper)(aw).IsActive()).Should(BeTrue())
66+
Expect((*AppWrapper)(aw).IsSuspended()).Should(BeFalse())
67+
68+
By("Reconciling: Resuming -> Running")
69+
_, err = awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName})
70+
Expect(err).NotTo(HaveOccurred())
71+
72+
aw = getAppWrapper(awName)
73+
Expect(aw.Status.Phase).Should(Equal(workloadv1beta2.AppWrapperRunning))
74+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed))).Should(BeTrue())
75+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.QuotaReserved))).Should(BeTrue())
76+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.PodsReady))).Should(BeFalse())
77+
Expect((*AppWrapper)(aw).IsActive()).Should(BeTrue())
78+
Expect((*AppWrapper)(aw).IsSuspended()).Should(BeFalse())
79+
podStatus, err := awReconciler.workloadStatus(ctx, aw)
80+
Expect(err).NotTo(HaveOccurred())
81+
Expect(podStatus.pending).Should(Equal(expectedPodCount(aw)))
82+
83+
By("Simulating all Pods Running")
84+
Expect(setPodStatus(aw, v1.PodRunning, expectedPodCount(aw))).To(Succeed())
85+
By("Reconciling: Running -> Running")
86+
_, err = awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName})
87+
Expect(err).NotTo(HaveOccurred())
88+
89+
aw = getAppWrapper(awName)
90+
Expect(aw.Status.Phase).Should(Equal(workloadv1beta2.AppWrapperRunning))
91+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed))).Should(BeTrue())
92+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.QuotaReserved))).Should(BeTrue())
93+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.PodsReady))).Should(BeTrue())
94+
Expect((*AppWrapper)(aw).IsActive()).Should(BeTrue())
95+
Expect((*AppWrapper)(aw).IsSuspended()).Should(BeFalse())
96+
Expect((*AppWrapper)(aw).PodsReady()).Should(BeTrue())
97+
podStatus, err = awReconciler.workloadStatus(ctx, aw)
98+
Expect(err).NotTo(HaveOccurred())
99+
Expect(podStatus.running).Should(Equal(expectedPodCount(aw)))
100+
_, finished := (*AppWrapper)(aw).Finished()
101+
Expect(finished).Should(BeFalse())
102+
}
103+
104+
BeforeEach(func() {
105+
By("Create an AppWrapper containing two Pods")
106+
aw := toAppWrapper(pod(100), pod(100))
107+
aw.Spec.Suspend = true
108+
Expect(k8sClient.Create(ctx, aw)).To(Succeed())
109+
awName = types.NamespacedName{
110+
Name: aw.Name,
111+
Namespace: aw.Namespace,
112+
}
113+
awReconciler = &AppWrapperReconciler{
114+
Client: k8sClient,
115+
Scheme: k8sClient.Scheme(),
116+
}
117+
kueuePodSets = (*AppWrapper)(aw).PodSets()
118+
})
38119

39-
ctx := context.Background()
120+
AfterEach(func() {
121+
By("Cleanup the AppWrapper and ensure no Pods remain")
122+
aw := &workloadv1beta2.AppWrapper{}
123+
Expect(k8sClient.Get(ctx, awName, aw)).To(Succeed())
124+
Expect(k8sClient.Delete(ctx, aw)).To(Succeed())
40125

41-
typeNamespacedName := types.NamespacedName{
42-
Name: resourceName,
43-
Namespace: "default", // TODO(user):Modify as needed
44-
}
45-
appwrapper := &workloadv1beta2.AppWrapper{}
46-
47-
BeforeEach(func() {
48-
By("creating the custom resource for the Kind AppWrapper")
49-
err := k8sClient.Get(ctx, typeNamespacedName, appwrapper)
50-
if err != nil && errors.IsNotFound(err) {
51-
resource := &workloadv1beta2.AppWrapper{
52-
ObjectMeta: metav1.ObjectMeta{
53-
Name: resourceName,
54-
Namespace: "default",
55-
},
56-
// TODO(user): Specify other spec details if needed.
57-
}
58-
Expect(k8sClient.Create(ctx, resource)).To(Succeed())
59-
}
60-
})
61-
62-
AfterEach(func() {
63-
// TODO(user): Cleanup logic after each test, like removing the resource instance.
64-
resource := &workloadv1beta2.AppWrapper{}
65-
err := k8sClient.Get(ctx, typeNamespacedName, resource)
66-
Expect(err).NotTo(HaveOccurred())
67-
68-
By("Cleanup the specific resource instance AppWrapper")
69-
Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
70-
})
71-
It("should successfully reconcile the resource", func() {
72-
By("Reconciling the created resource")
73-
controllerReconciler := &AppWrapperReconciler{
74-
Client: k8sClient,
75-
Scheme: k8sClient.Scheme(),
76-
}
77-
78-
_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
79-
NamespacedName: typeNamespacedName,
80-
})
81-
Expect(err).NotTo(HaveOccurred())
82-
// TODO(user): Add more specific assertions depending on your controller's reconciliation logic.
83-
// Example: If you expect a certain status condition after reconciliation, verify it here.
84-
})
126+
By("Reconciling: Deletion processing")
127+
_, err := awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName})
128+
Expect(err).NotTo(HaveOccurred())
129+
130+
podStatus, err := awReconciler.workloadStatus(ctx, aw)
131+
Expect(err).NotTo(HaveOccurred())
132+
Expect(podStatus.failed + podStatus.succeeded + podStatus.running + podStatus.pending).Should(Equal(int32(0)))
85133
})
86-
})
87134

88-
*/
135+
It("Happy Path Lifecycle", func() {
136+
advanceToRunning()
137+
138+
By("Simulating one Pod Completing")
139+
aw := getAppWrapper(awName)
140+
Expect(setPodStatus(aw, v1.PodSucceeded, 1)).To(Succeed())
141+
By("Reconciling: Running -> Running")
142+
_, err := awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName})
143+
Expect(err).NotTo(HaveOccurred())
144+
145+
aw = getAppWrapper(awName)
146+
Expect(aw.Status.Phase).Should(Equal(workloadv1beta2.AppWrapperRunning))
147+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed))).Should(BeTrue())
148+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.QuotaReserved))).Should(BeTrue())
149+
Expect((*AppWrapper)(aw).IsActive()).Should(BeTrue())
150+
Expect((*AppWrapper)(aw).IsSuspended()).Should(BeFalse())
151+
podStatus, err := awReconciler.workloadStatus(ctx, aw)
152+
Expect(err).NotTo(HaveOccurred())
153+
Expect(podStatus.running).Should(Equal(expectedPodCount(aw) - 1))
154+
Expect(podStatus.succeeded).Should(Equal(int32(1)))
155+
156+
By("Simulating all Pods Completing")
157+
Expect(setPodStatus(aw, v1.PodSucceeded, expectedPodCount(aw))).To(Succeed())
158+
By("Reconciling: Running -> Succeeded")
159+
_, err = awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName})
160+
Expect(err).NotTo(HaveOccurred())
161+
162+
aw = getAppWrapper(awName)
163+
Expect(aw.Status.Phase).Should(Equal(workloadv1beta2.AppWrapperSucceeded))
164+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed))).Should(BeTrue())
165+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.QuotaReserved))).Should(BeFalse())
166+
Expect((*AppWrapper)(aw).IsActive()).Should(BeFalse())
167+
Expect((*AppWrapper)(aw).IsSuspended()).Should(BeFalse())
168+
_, finished := (*AppWrapper)(aw).Finished()
169+
Expect(finished).Should(BeTrue())
170+
})
171+
172+
It("Running Workloads can be Suspended", func() {
173+
advanceToRunning()
174+
175+
By("Updating aw.Spec by invoking RunWithPodSetsInfo")
176+
aw := getAppWrapper(awName)
177+
(*AppWrapper)(aw).Suspend()
178+
Expect((*AppWrapper)(aw).RestorePodSetsInfo(utilslices.Map(kueuePodSets, podset.FromPodSet))).To(BeTrue())
179+
Expect(k8sClient.Update(ctx, aw)).To(Succeed())
180+
181+
By("Reconciling: Running -> Suspending")
182+
_, err := awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName})
183+
Expect(err).NotTo(HaveOccurred())
184+
185+
aw = getAppWrapper(awName)
186+
Expect(aw.Status.Phase).Should(Equal(workloadv1beta2.AppWrapperSuspending))
187+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed))).Should(BeTrue())
188+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.QuotaReserved))).Should(BeTrue())
189+
Expect((*AppWrapper)(aw).IsActive()).Should(BeTrue())
190+
Expect((*AppWrapper)(aw).IsSuspended()).Should(BeTrue())
191+
192+
By("Reconciling: Suspending -> Suspended")
193+
_, err = awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName}) // initiate deletion
194+
Expect(err).NotTo(HaveOccurred())
195+
_, err = awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName}) // see deletion has completed
196+
Expect(err).NotTo(HaveOccurred())
197+
198+
aw = getAppWrapper(awName)
199+
Expect(aw.Status.Phase).Should(Equal(workloadv1beta2.AppWrapperSuspended))
200+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed))).Should(BeFalse())
201+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.QuotaReserved))).Should(BeFalse())
202+
Expect((*AppWrapper)(aw).IsActive()).Should(BeFalse())
203+
Expect((*AppWrapper)(aw).IsSuspended()).Should(BeTrue())
204+
podStatus, err := awReconciler.workloadStatus(ctx, aw)
205+
Expect(err).NotTo(HaveOccurred())
206+
Expect(podStatus.failed + podStatus.succeeded + podStatus.running + podStatus.pending).Should(Equal(int32(0)))
207+
})
208+
209+
It("A Pod Failure leads to a failed AppWrappers", func() {
210+
advanceToRunning()
211+
212+
By("Simulating one Pod Failing")
213+
aw := getAppWrapper(awName)
214+
Expect(setPodStatus(aw, v1.PodFailed, 1)).To(Succeed())
215+
216+
By("Reconciling: Running -> Failed")
217+
_, err := awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName}) // detect failure
218+
Expect(err).NotTo(HaveOccurred())
219+
220+
aw = getAppWrapper(awName)
221+
Expect(aw.Status.Phase).Should(Equal(workloadv1beta2.AppWrapperFailed))
222+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed))).Should(BeTrue())
223+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.QuotaReserved))).Should(BeTrue())
224+
Expect((*AppWrapper)(aw).IsActive()).Should(BeTrue())
225+
Expect((*AppWrapper)(aw).IsSuspended()).Should(BeFalse())
226+
_, finished := (*AppWrapper)(aw).Finished()
227+
Expect(finished).Should(BeFalse())
228+
229+
_, err = awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName}) // initiate deletion
230+
Expect(err).NotTo(HaveOccurred())
231+
_, err = awReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: awName}) // see deletion has completed
232+
Expect(err).NotTo(HaveOccurred())
233+
234+
aw = getAppWrapper(awName)
235+
Expect(aw.Status.Phase).Should(Equal(workloadv1beta2.AppWrapperFailed))
236+
237+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.ResourcesDeployed))).Should(BeFalse())
238+
Expect(meta.IsStatusConditionTrue(aw.Status.Conditions, string(workloadv1beta2.QuotaReserved))).Should(BeFalse())
239+
Expect((*AppWrapper)(aw).IsActive()).Should(BeFalse())
240+
Expect((*AppWrapper)(aw).IsSuspended()).Should(BeFalse())
241+
_, finished = (*AppWrapper)(aw).Finished()
242+
Expect(finished).Should(BeTrue())
243+
})
244+
245+
})

‎internal/controller/appwrapper_fixtures_test.go‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,12 @@ import (
2424
. "github.com/onsi/gomega"
2525

2626
workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2"
27+
v1 "k8s.io/api/core/v1"
2728
"k8s.io/apimachinery/pkg/api/resource"
2829
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2930
"k8s.io/apimachinery/pkg/runtime"
31+
"k8s.io/apimachinery/pkg/types"
32+
"sigs.k8s.io/controller-runtime/pkg/client"
3033
"sigs.k8s.io/yaml"
3134
)
3235

@@ -48,6 +51,36 @@ func toAppWrapper(components ...workloadv1beta2.AppWrapperComponent) *workloadv1
4851
}
4952
}
5053

54+
func getAppWrapper(typeNamespacedName types.NamespacedName) *workloadv1beta2.AppWrapper {
55+
aw := &workloadv1beta2.AppWrapper{}
56+
err := k8sClient.Get(ctx, typeNamespacedName, aw)
57+
Expect(err).NotTo(HaveOccurred())
58+
return aw
59+
}
60+
61+
// envTest doesn't have a Pod controller; so simulate it
62+
func setPodStatus(aw *workloadv1beta2.AppWrapper, phase v1.PodPhase, numToChange int32) error {
63+
podList := &v1.PodList{}
64+
err := k8sClient.List(ctx, podList, &client.ListOptions{Namespace: aw.Namespace})
65+
if err != nil {
66+
return err
67+
}
68+
for _, pod := range podList.Items {
69+
if numToChange <= 0 {
70+
return nil
71+
}
72+
if awn, found := pod.Labels[appWrapperLabel]; found && awn == aw.Name {
73+
pod.Status.Phase = phase
74+
err = k8sClient.Status().Update(ctx, &pod)
75+
if err != nil {
76+
return err
77+
}
78+
numToChange -= 1
79+
}
80+
}
81+
return nil
82+
}
83+
5184
const podYAML = `
5285
apiVersion: v1
5386
kind: Pod
@@ -58,7 +91,7 @@ spec:
5891
containers:
5992
- name: busybox
6093
image: quay.io/project-codeflare/busybox:1.36
61-
command: ["sh", "-c", "sleep 1000"]
94+
command: ["sh", "-c", "sleep 10"]
6295
resources:
6396
requests:
6497
cpu: %v`
@@ -88,7 +121,7 @@ spec:
88121
containers:
89122
- name: busybox
90123
image: quay.io/project-codeflare/busybox:1.36
91-
command: ["sh", "-c", "sleep 1000"]
124+
command: ["sh", "-c", "sleep 10"]
92125
resources:
93126
requests:
94127
cpu: %v`

‎internal/controller/suite_test.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import (
3535
rbacv1 "k8s.io/api/rbac/v1"
3636
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3737
apimachineryruntime "k8s.io/apimachinery/pkg/runtime"
38+
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
3839
"k8s.io/client-go/rest"
3940
ctrl "sigs.k8s.io/controller-runtime"
4041
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -66,7 +67,7 @@ func TestControllers(t *testing.T) {
6667
var _ = BeforeSuite(func() {
6768
logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
6869

69-
ctx, cancel = context.WithCancel(context.TODO())
70+
ctx, cancel = context.WithCancel(context.Background())
7071

7172
By("bootstrapping test environment")
7273
testEnv = &envtest.Environment{
@@ -100,6 +101,8 @@ var _ = BeforeSuite(func() {
100101
Expect(err).NotTo(HaveOccurred())
101102
err = rbacv1.AddToScheme(scheme)
102103
Expect(err).NotTo(HaveOccurred())
104+
err = clientgoscheme.AddToScheme(scheme)
105+
Expect(err).NotTo(HaveOccurred())
103106

104107
//+kubebuilder:scaffold:scheme
105108

0 commit comments

Comments
 (0)