Adopt ConfigMap-based configuration of the operator

.github/workflows/CI-standalone.yaml

@@ -55,7 +55,7 @@ jobs:
       run: |
         make install -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }}
         make kind-push -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }}
-        make deploy-aw -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }}
+        make deploy -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }} ENV=standalone
 
     - name: Run E2E tests
       run: LABEL_FILTER="Standalone,Webhook" ./hack/run-tests-on-cluster.sh

.github/workflows/CI.yaml

@@ -58,7 +58,7 @@ jobs:
       run: |
         make install -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }}
         make kind-push -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }}
-        make deploy -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }}
+        make deploy -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }} ENV=default
 
     - name: Run E2E tests
       run: ./hack/run-tests-on-cluster.sh

Dockerfile

@@ -22,15 +22,13 @@ COPY pkg/ pkg/
 # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
 # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/unified/main.go
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager-aw cmd/standalone/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /
 COPY --from=builder /workspace/manager .
-COPY --from=builder /workspace/manager-aw .
 USER 65532:65532
 
 CMD ["/manager"]

Makefile

@@ -30,6 +30,10 @@ endif
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.29.0
 
+# The target deployment environment, that corresponds to the Kustomize directory
+# used to build the manifests.
+ENV ?= default
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -94,6 +98,18 @@ vet: ## Run go vet against code.
 test: manifests generate fmt vet envtest ## Run unit tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./api/... ./internal/... ./pkg/...) -v -ginkgo.v -coverprofile cover.out
 
+.PHONY: install
+install: manifests kustomize ## Install dev configuration into the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/dev | $(KUBECTL) apply -f -
+
+.PHONY: uninstall
+uninstall: manifests kustomize ## Uninstall dev configuration from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+	$(KUSTOMIZE) build config/dev | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
+
+.PHONY: run
+run: manifests generate fmt vet ## Run a controller from your host using the dev configurtation (webhooks are disabled).
+	NAMESPACE=dev go run ./cmd/main.go
+
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
 test-e2e:
@@ -116,22 +132,7 @@ build: manifests generate fmt vet ## Build manager binary.
 			-X 'main.BuildVersion=$(BUILD_VERSION)' \
 			-X 'main.BuildDate=$(BUILD_DATE)' \
 		" \
-		-o bin/manager cmd/unified/main.go
-	go build \
-		-ldflags " \
-			-X 'main.BuildVersion=$(BUILD_VERSION)' \
-			-X 'main.BuildDate=$(BUILD_DATE)' \
-		" \
-		-o bin/manager-aw cmd/standalone/main.go
-
-.PHONY: run
-run: manifests generate fmt vet ## Run a controller from your host (webhooks are disabled).
-	NAMESPACE=dev ENABLE_WEBHOOKS=false go run ./cmd/unified/main.go --metrics-bind-address=localhost:0 --health-probe-bind-address=localhost:0
-
-.PHONY: run-aw
-run-aw: manifests generate fmt vet ## Run a controller from your host (webhooks are disabled).
-	NAMESPACE=dev ENABLE_WEBHOOKS=false go run ./cmd/standalone/main.go --metrics-bind-address=localhost:0 --health-probe-bind-address=localhost:0
-
+		-o bin/manager cmd/main.go
 
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
@@ -173,7 +174,7 @@ build-installer: manifests generate kustomize ## Generate a consolidated YAML wi
 	fi
 	echo "---" >> dist/install.yaml  # Add a document separator before appending
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	$(KUSTOMIZE) build config/default >> dist/install.yaml
+	$(KUSTOMIZE) build config/$(ENV) >> dist/install.yaml
 	@$(call clean-manifests)
 
 ##@ Deployment
@@ -184,33 +185,15 @@ endif
 
 clean-manifests = (cd config/manager && $(KUSTOMIZE) edit set image controller=quay.io/ibm/appwrapper)
 
-.PHONY: install
-install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f -
-
-.PHONY: uninstall
-uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
-	$(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
-
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
-	@$(call clean-manifests)
-
-.PHONY: deploy-aw
-deploy-aw: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	$(KUSTOMIZE) build config/standalone | $(KUBECTL) apply -f -
+	$(KUSTOMIZE) build config/$(ENV) | $(KUBECTL) apply -f -
 	@$(call clean-manifests)
 
 .PHONY: undeploy
 undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
-	$(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
-
-.PHONY: undeploy-aw
-undeploy-aw: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
-	$(KUSTOMIZE) build config/standalone | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
+	$(KUSTOMIZE) build config/$(ENV) | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
 
 ##@ Dependencies
 

cmd/main.go

@@ -0,0 +1,205 @@
+/*
+Copyright 2024 IBM Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
+	// to ensure that exec-entrypoint and run can make use of them.
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/utils/ptr"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"sigs.k8s.io/yaml"
+
+	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
+
+	workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2"
+	"github.com/project-codeflare/appwrapper/pkg/config"
+	"github.com/project-codeflare/appwrapper/pkg/controller"
+	//+kubebuilder:scaffold:imports
+)
+
+var (
+	scheme       = runtime.NewScheme()
+	setupLog     = ctrl.Log.WithName("setup")
+	BuildVersion = "UNKNOWN"
+	BuildDate    = "UNKNOWN"
+)
+
+func init() {
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(kueue.AddToScheme(scheme))
+	utilruntime.Must(workloadv1beta2.AddToScheme(scheme))
+	//+kubebuilder:scaffold:scheme
+}
+
+func main() {
+	var configMapName string
+	flag.StringVar(&configMapName, "config", "appwrapper-operator-config",
+		"The name of the ConfigMap to load the operator configuration from. "+
+			"If it does not exist, the operator will create and initialise it.")
+
+	opts := zap.Options{
+		Development: true,
+	}
+	opts.BindFlags(flag.CommandLine)
+	flag.Parse()
+
+	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+	setupLog.Info("Build info", "version", BuildVersion, "date", BuildDate)
+
+	namespace, err := getNamespace()
+	exitOnError(err, "unable to get operator namespace")
+
+	cfg := &config.OperatorConfig{
+		AppWrapper:        config.NewAppWrapperConfig(),
+		CertManagement:    config.NewCertManagementConfig(namespace),
+		ControllerManager: config.NewControllerManagerConfig(),
+		WebhooksEnabled:   ptr.To(true),
+	}
+
+	k8sConfig, err := ctrl.GetConfig()
+	exitOnError(err, "unable to get client config")
+	k8sClient, err := client.New(k8sConfig, client.Options{Scheme: scheme})
+	exitOnError(err, "unable to create Kubernetes client")
+	ctx := ctrl.SetupSignalHandler()
+
+	cmName := types.NamespacedName{Namespace: namespace, Name: configMapName}
+	exitOnError(loadIntoOrCreate(ctx, k8sClient, cmName, cfg), "unable to initialise configuration")
+
+	setupLog.Info("Configuration", "config", cfg)
+	exitOnError(config.ValidateAppWrapperConfig(cfg.AppWrapper), "invalid appwrapper config")
+
+	tlsOpts := []func(*tls.Config){}
+	if !cfg.ControllerManager.EnableHTTP2 {
+		// Unless EnableHTTP2 was set to True, http/2 should be disabled
+		// due to its vulnerabilities. More specifically, disabling http/2 will
+		// prevent from being vulnerable to the HTTP/2 Stream Cancelation and
+		// Rapid Reset CVEs. For more information see:
+		// - https://github.com/advisories/GHSA-qppj-fm5r-hxr3
+		// - https://github.com/advisories/GHSA-4374-p667-p6c8
+		disableHTTP2 := func(c *tls.Config) {
+			setupLog.Info("disabling http/2")
+			c.NextProtos = []string{"http/1.1"}
+		}
+		tlsOpts = append(tlsOpts, disableHTTP2)
+	}
+
+	mgr, err := ctrl.NewManager(k8sConfig, ctrl.Options{
+		Scheme: scheme,
+		Metrics: metricsserver.Options{
+			BindAddress:   cfg.ControllerManager.Metrics.BindAddress,
+			SecureServing: cfg.ControllerManager.Metrics.SecureServing,
+			TLSOpts:       tlsOpts,
+		},
+		WebhookServer: webhook.NewServer(webhook.Options{
+			TLSOpts: tlsOpts,
+		}),
+		HealthProbeBindAddress: cfg.ControllerManager.Health.BindAddress,
+		LeaderElection:         cfg.ControllerManager.LeaderElection,
+		LeaderElectionID:       "f134c674.codeflare.dev",
+	})
+	exitOnError(err, "unable to start manager")
+
+	certsReady := make(chan struct{})
+
+	if *cfg.WebhooksEnabled {
+		exitOnError(controller.SetupCertManagement(mgr, cfg.CertManagement, certsReady), "Unable to set up cert rotation")
+	} else {
+		close(certsReady)
+	}
+
+	// Ascynchronous because controllers need to wait for certificate to be ready for webhooks to work
+	go controller.SetupControllers(ctx, mgr, cfg.AppWrapper, *cfg.WebhooksEnabled, certsReady, setupLog)
+
+	exitOnError(controller.SetupIndexers(ctx, mgr, cfg.AppWrapper), "unable to setup indexers")
+	exitOnError(controller.SetupProbeEndpoints(mgr, certsReady), "unable to setup probe endpoints")
+
+	setupLog.Info("starting manager")
+	exitOnError(mgr.Start(ctx), "problem running manager")
+}
+
+func getNamespace() (string, error) {
+	// This way assumes you've set the NAMESPACE environment variable either manually, when running
+	// the operator standalone, or using the downward API, when running the operator in-cluster.
+	if ns := os.Getenv("NAMESPACE"); ns != "" {
+		return ns, nil
+	}
+
+	// Fall back to the namespace associated with the service account token, if available
+	if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
+		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
+			return ns, nil
+		}
+	}
+
+	return "", fmt.Errorf("unable to determine current namespace")
+}
+
+func loadIntoOrCreate(ctx context.Context, k8sClient client.Client, cmName types.NamespacedName,
+	cfg *config.OperatorConfig) error {
+	configMap := &corev1.ConfigMap{}
+	err := k8sClient.Get(ctx, cmName, configMap)
+	if apierrors.IsNotFound(err) {
+		if content, err := yaml.Marshal(cfg); err == nil {
+			configMap := &corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{Name: cmName.Name, Namespace: cmName.Namespace},
+				Data:       map[string]string{"config.yaml": string(content)},
+			}
+			return k8sClient.Create(ctx, configMap)
+		} else {
+			return err
+		}
+	} else if err != nil {
+		return err
+	}
+
+	if len(configMap.Data) != 1 {
+		return fmt.Errorf("cannot resolve config from ConfigMap %s/%s", configMap.Namespace, configMap.Name)
+	}
+
+	for _, data := range configMap.Data {
+		return yaml.Unmarshal([]byte(data), cfg)
+	}
+
+	return nil
+}
+
+func exitOnError(err error, msg string) {
+	if err != nil {
+		setupLog.Error(err, msg)
+		os.Exit(1)
+	}
+}