port to AppWrapper controller using cert-controller

Author: dgrove-oss

.github/workflows/e2e_tests.yaml

@@ -55,6 +55,10 @@ jobs:
       - name: Setup and start KinD cluster
         uses: ./common/github-actions/kind
 
+      - name: Deploy Kueue
+        run: |
+          make kueue-e2e
+
       - name: Deploy CodeFlare stack
         id: deploy
         run: |

.github/workflows/olm_tests.yaml

@@ -54,6 +54,10 @@ jobs:
       - name: Setup and start KinD cluster
         uses: ./common/github-actions/kind
 
+      - name: Deploy Kueue
+        run: |
+          make kueue-e2e
+
       - name: Install OLM
         run: |
           kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml

Makefile

@@ -12,11 +12,14 @@ VERSION ?= v0.0.0-dev
 BUNDLE_VERSION ?= $(VERSION:v%=%)
 
 # APPWRAPPER_VERSION defines the default version of the AppWrapper controller
-APPWRAPPER_VERSION ?= v0.6.4
+APPWRAPPER_VERSION ?= v0.7.0
 APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper
 # Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version)
 APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION}
 
+# KUEUE_VERSION defines the default version of Kueue (used for testing)
+KUEUE_VERSION ?= v0.6.1
+
 # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing)
 KUBERAY_VERSION ?= v1.0.0
 
@@ -375,6 +378,10 @@ test-e2e: manifests fmt vet ## Run e2e tests.
 kind-e2e: ## Set up e2e KinD cluster
 	test/e2e/kind.sh
 
+.PHONY: kueue-e2e
+kueue-e2e: ## Deploy Kueue
+	KUEUE_VERSION=$(KUEUE_VERSION) test/e2e/kueue.sh
+
 .PHONY: setup-e2e
 setup-e2e: ## Set up e2e tests.
 	KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh

README.md

@@ -33,6 +33,8 @@ The e2e tests can be executed locally by running the following commands:
     ```bash
     # Create a KinD cluster
     make kind-e2e
+    # Deploy Kueue
+    make kueue-e2e
     # Install the CRDs
     make install
     ```

config/crd/appwrapper/kustomization.yaml

@@ -1,4 +1,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- github.com/project-codeflare/appwrapper/config/crd?ref=v0.6.4
+- github.com/project-codeflare/appwrapper/config/crd?ref=v0.7.0

config/crd/crd-appwrapper.yml

@@ -2,7 +2,6 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME
     controller-gen.kubebuilder.io/version: v0.14.0
   name: appwrappers.workload.codeflare.dev
 spec:
@@ -212,6 +211,10 @@ spec:
                 phase:
                   description: Phase of the AppWrapper object
                   type: string
+                resettingCount:
+                  description: Retries counts the number of times the AppWrapper has entered the Resetting Phase
+                  format: int32
+                  type: integer
               type: object
           type: object
       served: true

config/crd/kustomization.yaml

@@ -5,3 +5,15 @@ resources:
 - crd-appwrapper.yml
 
 #+kubebuilder:scaffold:crdkustomizeresource
+
+patches:
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
+# patches here are for enabling the conversion webhook for each CRD
+- path: patches/webhook_in_appwrappers.yaml
+#+kubebuilder:scaffold:crdkustomizewebhookpatch
+
+# [WEBHOOK] To enable webhook, uncomment the following section
+# the following config is for teaching kustomize how to do kustomization for CRDs.
+
+configurations:
+- kustomizeconfig.yaml

config/crd/kustomizeconfig.yaml

@@ -0,0 +1,19 @@
+# This file is for teaching kustomize how to substitute name and namespace reference in CRD
+nameReference:
+- kind: Service
+  version: v1
+  fieldSpecs:
+  - kind: CustomResourceDefinition
+    version: v1
+    group: apiextensions.k8s.io
+    path: spec/conversion/webhook/clientConfig/service/name
+
+namespace:
+- kind: CustomResourceDefinition
+  version: v1
+  group: apiextensions.k8s.io
+  path: spec/conversion/webhook/clientConfig/service/namespace
+  create: false
+
+varReference:
+- path: metadata/annotations

config/crd/patches/webhook_in_appwrappers.yaml

@@ -0,0 +1,16 @@
+# The following patch enables a conversion webhook for the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: appwrappers.workload.codeflare.dev
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          namespace: system
+          name: webhook-service
+          path: /convert
+      conversionReviewVersions:
+      - v1

config/default/kustomization.yaml

@@ -17,8 +17,17 @@ bases:
 - ../crd
 - ../rbac
 - ../manager
+- ../internalcert
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
+- ../webhook
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 # - ../prometheus
 
+patches:
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
+- path: manager_webhook_patch.yaml
+
 resources:
 - metrics_service.yaml