diff --git a/.github/workflows/CI-standalone.yaml b/.github/workflows/CI-standalone.yaml index 7f12327..d2d3a99 100644 --- a/.github/workflows/CI-standalone.yaml +++ b/.github/workflows/CI-standalone.yaml @@ -55,7 +55,7 @@ jobs: run: | make install -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }} make kind-push -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }} - make deploy-aw -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }} + make deploy -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }} ENV=standalone - name: Run E2E tests run: LABEL_FILTER="Standalone,Webhook" ./hack/run-tests-on-cluster.sh diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 907481f..d63f02e 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -58,7 +58,7 @@ jobs: run: | make install -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }} make kind-push -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }} - make deploy -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }} + make deploy -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }} ENV=default - name: Run E2E tests run: ./hack/run-tests-on-cluster.sh diff --git a/Dockerfile b/Dockerfile index 6c9616b..20829ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,15 +22,13 @@ COPY pkg/ pkg/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/unified/main.go -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager-aw cmd/standalone/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=builder /workspace/manager . -COPY --from=builder /workspace/manager-aw . USER 65532:65532 CMD ["/manager"] diff --git a/Makefile b/Makefile index 8150161..da29954 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,10 @@ endif # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.29.0 +# The target deployment environment, that corresponds to the Kustomize directory +# used to build the manifests. +ENV ?= default + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -94,6 +98,18 @@ vet: ## Run go vet against code. test: manifests generate fmt vet envtest ## Run unit tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./api/... ./internal/... ./pkg/...) -v -ginkgo.v -coverprofile cover.out +.PHONY: install +install: manifests kustomize ## Install dev configuration into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/dev | $(KUBECTL) apply -f - + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall dev configuration from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/dev | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host using the dev configurtation (webhooks are disabled). + NAMESPACE=dev go run ./cmd/main.go + # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. test-e2e: @@ -116,22 +132,7 @@ build: manifests generate fmt vet ## Build manager binary. -X 'main.BuildVersion=$(BUILD_VERSION)' \ -X 'main.BuildDate=$(BUILD_DATE)' \ " \ - -o bin/manager cmd/unified/main.go - go build \ - -ldflags " \ - -X 'main.BuildVersion=$(BUILD_VERSION)' \ - -X 'main.BuildDate=$(BUILD_DATE)' \ - " \ - -o bin/manager-aw cmd/standalone/main.go - -.PHONY: run -run: manifests generate fmt vet ## Run a controller from your host (webhooks are disabled). - NAMESPACE=dev ENABLE_WEBHOOKS=false go run ./cmd/unified/main.go --metrics-bind-address=localhost:0 --health-probe-bind-address=localhost:0 - -.PHONY: run-aw -run-aw: manifests generate fmt vet ## Run a controller from your host (webhooks are disabled). - NAMESPACE=dev ENABLE_WEBHOOKS=false go run ./cmd/standalone/main.go --metrics-bind-address=localhost:0 --health-probe-bind-address=localhost:0 - + -o bin/manager cmd/main.go # If you wish to build the manager image targeting other platforms you can use the --platform flag. # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. @@ -173,7 +174,7 @@ build-installer: manifests generate kustomize ## Generate a consolidated YAML wi fi echo "---" >> dist/install.yaml # Add a document separator before appending cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default >> dist/install.yaml + $(KUSTOMIZE) build config/$(ENV) >> dist/install.yaml @$(call clean-manifests) ##@ Deployment @@ -184,33 +185,15 @@ endif clean-manifests = (cd config/manager && $(KUSTOMIZE) edit set image controller=quay.io/ibm/appwrapper) -.PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - - -.PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - - @$(call clean-manifests) - -.PHONY: deploy-aw -deploy-aw: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/standalone | $(KUBECTL) apply -f - + $(KUSTOMIZE) build config/$(ENV) | $(KUBECTL) apply -f - @$(call clean-manifests) .PHONY: undeploy undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: undeploy-aw -undeploy-aw: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/standalone | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + $(KUSTOMIZE) build config/$(ENV) | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - ##@ Dependencies diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..32a34da --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,205 @@ +/* +Copyright 2024 IBM Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "crypto/tls" + "flag" + "fmt" + "os" + "strings" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/utils/ptr" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/yaml" + + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + + workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" + "github.com/project-codeflare/appwrapper/pkg/config" + "github.com/project-codeflare/appwrapper/pkg/controller" + //+kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + BuildVersion = "UNKNOWN" + BuildDate = "UNKNOWN" +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(kueue.AddToScheme(scheme)) + utilruntime.Must(workloadv1beta2.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme +} + +func main() { + var configMapName string + flag.StringVar(&configMapName, "config", "appwrapper-operator-config", + "The name of the ConfigMap to load the operator configuration from. "+ + "If it does not exist, the operator will create and initialise it.") + + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + setupLog.Info("Build info", "version", BuildVersion, "date", BuildDate) + + namespace, err := getNamespace() + exitOnError(err, "unable to get operator namespace") + + cfg := &config.OperatorConfig{ + AppWrapper: config.NewAppWrapperConfig(), + CertManagement: config.NewCertManagementConfig(namespace), + ControllerManager: config.NewControllerManagerConfig(), + WebhooksEnabled: ptr.To(true), + } + + k8sConfig, err := ctrl.GetConfig() + exitOnError(err, "unable to get client config") + k8sClient, err := client.New(k8sConfig, client.Options{Scheme: scheme}) + exitOnError(err, "unable to create Kubernetes client") + ctx := ctrl.SetupSignalHandler() + + cmName := types.NamespacedName{Namespace: namespace, Name: configMapName} + exitOnError(loadIntoOrCreate(ctx, k8sClient, cmName, cfg), "unable to initialise configuration") + + setupLog.Info("Configuration", "config", cfg) + exitOnError(config.ValidateAppWrapperConfig(cfg.AppWrapper), "invalid appwrapper config") + + tlsOpts := []func(*tls.Config){} + if !cfg.ControllerManager.EnableHTTP2 { + // Unless EnableHTTP2 was set to True, http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancelation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + tlsOpts = append(tlsOpts, disableHTTP2) + } + + mgr, err := ctrl.NewManager(k8sConfig, ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{ + BindAddress: cfg.ControllerManager.Metrics.BindAddress, + SecureServing: cfg.ControllerManager.Metrics.SecureServing, + TLSOpts: tlsOpts, + }, + WebhookServer: webhook.NewServer(webhook.Options{ + TLSOpts: tlsOpts, + }), + HealthProbeBindAddress: cfg.ControllerManager.Health.BindAddress, + LeaderElection: cfg.ControllerManager.LeaderElection, + LeaderElectionID: "f134c674.codeflare.dev", + }) + exitOnError(err, "unable to start manager") + + certsReady := make(chan struct{}) + + if *cfg.WebhooksEnabled { + exitOnError(controller.SetupCertManagement(mgr, cfg.CertManagement, certsReady), "Unable to set up cert rotation") + } else { + close(certsReady) + } + + // Ascynchronous because controllers need to wait for certificate to be ready for webhooks to work + go controller.SetupControllers(ctx, mgr, cfg.AppWrapper, *cfg.WebhooksEnabled, certsReady, setupLog) + + exitOnError(controller.SetupIndexers(ctx, mgr, cfg.AppWrapper), "unable to setup indexers") + exitOnError(controller.SetupProbeEndpoints(mgr, certsReady), "unable to setup probe endpoints") + + setupLog.Info("starting manager") + exitOnError(mgr.Start(ctx), "problem running manager") +} + +func getNamespace() (string, error) { + // This way assumes you've set the NAMESPACE environment variable either manually, when running + // the operator standalone, or using the downward API, when running the operator in-cluster. + if ns := os.Getenv("NAMESPACE"); ns != "" { + return ns, nil + } + + // Fall back to the namespace associated with the service account token, if available + if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { + if ns := strings.TrimSpace(string(data)); len(ns) > 0 { + return ns, nil + } + } + + return "", fmt.Errorf("unable to determine current namespace") +} + +func loadIntoOrCreate(ctx context.Context, k8sClient client.Client, cmName types.NamespacedName, + cfg *config.OperatorConfig) error { + configMap := &corev1.ConfigMap{} + err := k8sClient.Get(ctx, cmName, configMap) + if apierrors.IsNotFound(err) { + if content, err := yaml.Marshal(cfg); err == nil { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: cmName.Name, Namespace: cmName.Namespace}, + Data: map[string]string{"config.yaml": string(content)}, + } + return k8sClient.Create(ctx, configMap) + } else { + return err + } + } else if err != nil { + return err + } + + if len(configMap.Data) != 1 { + return fmt.Errorf("cannot resolve config from ConfigMap %s/%s", configMap.Namespace, configMap.Name) + } + + for _, data := range configMap.Data { + return yaml.Unmarshal([]byte(data), cfg) + } + + return nil +} + +func exitOnError(err error, msg string) { + if err != nil { + setupLog.Error(err, msg) + os.Exit(1) + } +} diff --git a/cmd/standalone/main.go b/cmd/standalone/main.go deleted file mode 100644 index 7b8a9d1..0000000 --- a/cmd/standalone/main.go +++ /dev/null @@ -1,189 +0,0 @@ -/* -Copyright 2024 IBM Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "crypto/tls" - "flag" - "os" - "strings" - - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" - - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/webhook" - - workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" - "github.com/project-codeflare/appwrapper/pkg/config" - "github.com/project-codeflare/appwrapper/pkg/controller" - //+kubebuilder:scaffold:imports -) - -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") - BuildVersion = "UNKNOWN" - BuildDate = "UNKNOWN" -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(workloadv1beta2.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme -} - -func main() { - var metricsAddr string - var enableLeaderElection bool - var probeAddr string - var secureMetrics bool - var enableHTTP2 bool - - cfg := &config.OperatorConfig{ - AppWrapper: config.NewAppWrapperConfig(), - CertManagement: config.NewCertManagementConfig(namespaceOrDie()), - } - cfg.AppWrapper.StandaloneMode = true - cfg.AppWrapper.ManageJobsWithoutQueueName = false - - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") - flag.BoolVar(&enableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") - opts := zap.Options{ - Development: true, - } - opts.BindFlags(flag.CommandLine) - flag.Parse() - - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - setupLog.Info("Build info", "version", BuildVersion, "date", BuildDate) - setupLog.Info("Configuration", "config", cfg) - - if err := config.ValidateAppWrapperConfig(cfg.AppWrapper); err != nil { - setupLog.Error(err, "invalid appwrapper config") - os.Exit(1) - } - - // if the enable-http2 flag is false (the default), http/2 should be disabled - // due to its vulnerabilities. More specifically, disabling http/2 will - // prevent from being vulnerable to the HTTP/2 Stream Cancelation and - // Rapid Reset CVEs. For more information see: - // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 - // - https://github.com/advisories/GHSA-4374-p667-p6c8 - disableHTTP2 := func(c *tls.Config) { - setupLog.Info("disabling http/2") - c.NextProtos = []string{"http/1.1"} - } - - tlsOpts := []func(*tls.Config){} - if !enableHTTP2 { - tlsOpts = append(tlsOpts, disableHTTP2) - } - - webhookServer := webhook.NewServer(webhook.Options{ - TLSOpts: tlsOpts, - }) - - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{ - BindAddress: metricsAddr, - SecureServing: secureMetrics, - TLSOpts: tlsOpts, - }, - WebhookServer: webhookServer, - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "f134c674.codeflare.dev", - // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily - // when the Manager ends. This requires the binary to immediately end when the - // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly - // speeds up voluntary leader transitions as the new leader don't have to wait - // LeaseDuration time first. - // - // In the default scaffold provided, the program ends immediately after - // the manager stops, so would be fine to enable this option. However, - // if you are doing or is intended to do any operation such as perform cleanups - // after the manager stops then its usage might be unsafe. - // LeaderElectionReleaseOnCancel: true, - }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - ctx := ctrl.SetupSignalHandler() - certsReady := make(chan struct{}) - - if os.Getenv("ENABLE_WEBHOOKS") == "false" { - close(certsReady) - } else { - if err := controller.SetupCertManagement(mgr, cfg.CertManagement, certsReady); err != nil { - setupLog.Error(err, "Unable to set up cert rotation") - os.Exit(1) - } - } - - // Ascynchronous because controllers need to wait for certificate to be ready for webhooks to work - go controller.SetupControllers(ctx, mgr, cfg.AppWrapper, certsReady, setupLog) - - if err := controller.SetupIndexers(ctx, mgr, cfg.AppWrapper); err != nil { - setupLog.Error(err, "unable to setup indexers") - os.Exit(1) - } - - if err := controller.SetupProbeEndpoints(mgr, certsReady); err != nil { - setupLog.Error(err, "unable to setup probe endpoints") - os.Exit(1) - } - - setupLog.Info("starting manager") - if err := mgr.Start(ctx); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } -} - -func namespaceOrDie() string { - // This way assumes you've set the NAMESPACE environment variable either manually, when running - // the operator standalone, or using the downward API, when running the operator in-cluster. - if ns := os.Getenv("NAMESPACE"); ns != "" { - return ns - } - - // Fall back to the namespace associated with the service account token, if available - if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { - if ns := strings.TrimSpace(string(data)); len(ns) > 0 { - return ns - } - } - - panic("unable to determine current namespace") -} diff --git a/cmd/unified/main.go b/cmd/unified/main.go deleted file mode 100644 index 6f51531..0000000 --- a/cmd/unified/main.go +++ /dev/null @@ -1,191 +0,0 @@ -/* -Copyright 2024 IBM Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "crypto/tls" - "flag" - "os" - "strings" - - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" - - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/webhook" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - - workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2" - "github.com/project-codeflare/appwrapper/pkg/config" - "github.com/project-codeflare/appwrapper/pkg/controller" - //+kubebuilder:scaffold:imports -) - -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") - BuildVersion = "UNKNOWN" - BuildDate = "UNKNOWN" -) - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(kueue.AddToScheme(scheme)) - utilruntime.Must(workloadv1beta2.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme -} - -func main() { - var metricsAddr string - var enableLeaderElection bool - var probeAddr string - var secureMetrics bool - var enableHTTP2 bool - - cfg := &config.OperatorConfig{ - AppWrapper: config.NewAppWrapperConfig(), - CertManagement: config.NewCertManagementConfig(namespaceOrDie()), - } - - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") - flag.BoolVar(&enableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") - flag.BoolVar(&cfg.AppWrapper.ManageJobsWithoutQueueName, - "manage-no-queue", true, "Manage AppWrappers without queue names") - opts := zap.Options{ - Development: true, - } - opts.BindFlags(flag.CommandLine) - flag.Parse() - - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - setupLog.Info("Build info", "version", BuildVersion, "date", BuildDate) - setupLog.Info("Configuration", "config", cfg) - - if err := config.ValidateAppWrapperConfig(cfg.AppWrapper); err != nil { - setupLog.Error(err, "invalid appwrapper config") - os.Exit(1) - } - - // if the enable-http2 flag is false (the default), http/2 should be disabled - // due to its vulnerabilities. More specifically, disabling http/2 will - // prevent from being vulnerable to the HTTP/2 Stream Cancelation and - // Rapid Reset CVEs. For more information see: - // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 - // - https://github.com/advisories/GHSA-4374-p667-p6c8 - disableHTTP2 := func(c *tls.Config) { - setupLog.Info("disabling http/2") - c.NextProtos = []string{"http/1.1"} - } - - tlsOpts := []func(*tls.Config){} - if !enableHTTP2 { - tlsOpts = append(tlsOpts, disableHTTP2) - } - - webhookServer := webhook.NewServer(webhook.Options{ - TLSOpts: tlsOpts, - }) - - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{ - BindAddress: metricsAddr, - SecureServing: secureMetrics, - TLSOpts: tlsOpts, - }, - WebhookServer: webhookServer, - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "f134c674.codeflare.dev", - // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily - // when the Manager ends. This requires the binary to immediately end when the - // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly - // speeds up voluntary leader transitions as the new leader don't have to wait - // LeaseDuration time first. - // - // In the default scaffold provided, the program ends immediately after - // the manager stops, so would be fine to enable this option. However, - // if you are doing or is intended to do any operation such as perform cleanups - // after the manager stops then its usage might be unsafe. - // LeaderElectionReleaseOnCancel: true, - }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - ctx := ctrl.SetupSignalHandler() - certsReady := make(chan struct{}) - - if os.Getenv("ENABLE_WEBHOOKS") == "false" { - close(certsReady) - } else { - if err := controller.SetupCertManagement(mgr, cfg.CertManagement, certsReady); err != nil { - setupLog.Error(err, "Unable to set up cert rotation") - os.Exit(1) - } - } - - // Ascynchronous because controllers need to wait for certificate to be ready for webhooks to work - go controller.SetupControllers(ctx, mgr, cfg.AppWrapper, certsReady, setupLog) - - if err := controller.SetupIndexers(ctx, mgr, cfg.AppWrapper); err != nil { - setupLog.Error(err, "unable to setup indexers") - os.Exit(1) - } - - if err := controller.SetupProbeEndpoints(mgr, certsReady); err != nil { - setupLog.Error(err, "unable to setup probe endpoints") - os.Exit(1) - } - - setupLog.Info("starting manager") - if err := mgr.Start(ctx); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } -} - -func namespaceOrDie() string { - // This way assumes you've set the NAMESPACE environment variable either manually, when running - // the operator standalone, or using the downward API, when running the operator in-cluster. - if ns := os.Getenv("NAMESPACE"); ns != "" { - return ns - } - - // Fall back to the namespace associated with the service account token, if available - if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { - if ns := strings.TrimSpace(string(data)); len(ns) > 0 { - return ns - } - } - - panic("unable to determine current namespace") -} diff --git a/config/default/config.yaml b/config/default/config.yaml new file mode 100644 index 0000000..1731275 --- /dev/null +++ b/config/default/config.yaml @@ -0,0 +1,12 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: operator-config +data: + config.yaml: | + controllerManager: + health: + bindAddress: ":8081" + metrics: + bindAddress: "127.0.0.1:8080" + leaderElection: true diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index cbc56e7..4943108 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -15,6 +15,7 @@ namePrefix: appwrapper- # someName: someValue resources: +- config.yaml - ../crd - ../rbac - ../manager diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 70c3437..6f39130 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -32,8 +32,3 @@ spec: requests: cpu: 5m memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/config/dev/config.yaml b/config/dev/config.yaml new file mode 100644 index 0000000..c21bb86 --- /dev/null +++ b/config/dev/config.yaml @@ -0,0 +1,13 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: operator-config +data: + config.yaml: | + controllerManager: + health: + bindAddress: "localhost:0" + metrics: + bindAddress: "localhost:0" + leaderElection: false + webhooksEnabled: false diff --git a/config/dev/kustomization.yaml b/config/dev/kustomization.yaml new file mode 100644 index 0000000..e8403b9 --- /dev/null +++ b/config/dev/kustomization.yaml @@ -0,0 +1,14 @@ +# Adds namespace to all resources. +namespace: dev + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: appwrapper- + +resources: +- config.yaml +- namespace.yaml +- ../crd diff --git a/config/dev/namespace.yaml b/config/dev/namespace.yaml new file mode 100644 index 0000000..c172f26 --- /dev/null +++ b/config/dev/namespace.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: namespace + app.kubernetes.io/instance: system + app.kubernetes.io/component: manager + app.kubernetes.io/part-of: appwrapper + app.kubernetes.io/managed-by: kustomize + name: system diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d8afc7d..b30ad58 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -59,8 +59,6 @@ spec: containers: - command: - /manager - args: - - --leader-elect image: controller:latest name: manager securityContext: diff --git a/config/standalone/config.yaml b/config/standalone/config.yaml new file mode 100644 index 0000000..a4f0789 --- /dev/null +++ b/config/standalone/config.yaml @@ -0,0 +1,15 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: operator-config +data: + config.yaml: | + appwrapper: + standaloneMode: true + manageJobsWithoutQueueName: false + controllerManager: + health: + bindAddress: ":8081" + metrics: + bindAddress: "127.0.0.1:8080" + leaderElection: true diff --git a/config/standalone/kustomization.yaml b/config/standalone/kustomization.yaml index 16c2f89..4943108 100644 --- a/config/standalone/kustomization.yaml +++ b/config/standalone/kustomization.yaml @@ -15,6 +15,7 @@ namePrefix: appwrapper- # someName: someValue resources: +- config.yaml - ../crd - ../rbac - ../manager @@ -40,7 +41,7 @@ patches: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. # 'CERTMANAGER' needs to be enabled to use ca injection -# - path: webhookcainjection_patch.yaml +#- path: webhookcainjection_patch.yaml # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations diff --git a/config/standalone/manager_auth_proxy_patch.yaml b/config/standalone/manager_auth_proxy_patch.yaml index 8986687..6f39130 100644 --- a/config/standalone/manager_auth_proxy_patch.yaml +++ b/config/standalone/manager_auth_proxy_patch.yaml @@ -32,10 +32,3 @@ spec: requests: cpu: 5m memory: 64Mi - - name: manager - command: - - /manager-aw - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/pkg/config/config.go b/pkg/config/config.go index 2f24b90..894abc9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -22,8 +22,10 @@ import ( ) type OperatorConfig struct { - AppWrapper *AppWrapperConfig `json:"appWrapper,omitempty"` - CertManagement *CertManagementConfig `json:"certManagement,omitempty"` + AppWrapper *AppWrapperConfig `json:"appwrapper,omitempty"` + CertManagement *CertManagementConfig `json:"certManagement,omitempty"` + ControllerManager *ControllerManagerConfig `json:"controllerManager,omitempty"` + WebhooksEnabled *bool `json:"webhooksEnabled,omitempty"` } type AppWrapperConfig struct { @@ -52,6 +54,22 @@ type CertManagementConfig struct { WebhookSecretName string `json:"webhookSecretName,omitempty"` } +type ControllerManagerConfig struct { + Metrics MetricsConfiguration `json:"metrics,omitempty"` + Health HealthConfiguration `json:"health,omitempty"` + LeaderElection bool `json:"leaderElection,omitempty"` + EnableHTTP2 bool `json:"enableHTTP2,omitempty"` +} + +type MetricsConfiguration struct { + BindAddress string `json:"bindAddress,omitempty"` + SecureServing bool `json:"secureServing,omitempty"` +} + +type HealthConfiguration struct { + BindAddress string `json:"bindAddress,omitempty"` +} + // NewAppWrapperConfig constructs an AppWrapperConfig and fills in default values func NewAppWrapperConfig() *AppWrapperConfig { return &AppWrapperConfig{ @@ -102,3 +120,18 @@ func NewCertManagementConfig(namespace string) *CertManagementConfig { WebhookSecretName: "appwrapper-webhook-server-cert", } } + +// NewControllerRuntimeConfig constructs a ControllerRuntimeConfig and filles in default values +func NewControllerManagerConfig() *ControllerManagerConfig { + return &ControllerManagerConfig{ + Metrics: MetricsConfiguration{ + BindAddress: ":8080", + SecureServing: false, + }, + Health: HealthConfiguration{ + BindAddress: ":8081", + }, + LeaderElection: false, + EnableHTTP2: false, + } +} diff --git a/pkg/controller/setup.go b/pkg/controller/setup.go index 4a54b97..6ba0881 100644 --- a/pkg/controller/setup.go +++ b/pkg/controller/setup.go @@ -40,7 +40,7 @@ import ( // SetupControllers creates and configures all components of the AppWrapper controller func SetupControllers(ctx context.Context, mgr ctrl.Manager, awConfig *config.AppWrapperConfig, - certsReady chan struct{}, log logr.Logger) { + webhooksEnabled bool, certsReady chan struct{}, log logr.Logger) { log.Info("Waiting for certificates to be generated") <-certsReady @@ -74,7 +74,7 @@ func SetupControllers(ctx context.Context, mgr ctrl.Manager, awConfig *config.Ap os.Exit(1) } - if os.Getenv("ENABLE_WEBHOOKS") != "false" { + if webhooksEnabled { if err := (&webhook.AppWrapperWebhook{ Config: awConfig, }).SetupWebhookWithManager(mgr); err != nil {