replace cert manager with internal cert-controller (#66)

Author: dgrove-oss

.golangci.yml

@@ -25,6 +25,9 @@ issues:
     linters:
     - dupl
     - lll
+  - path: "pkg/*"
+    linters:
+    - lll
 linters:
   disable-all: true
   enable:

Makefile

@@ -122,11 +122,11 @@ build: manifests generate fmt vet ## Build manager binary.
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host (webhooks are disabled).
-	ENABLE_WEBHOOKS=false go run ./cmd/unified/main.go --metrics-bind-address=localhost:0 --health-probe-bind-address=localhost:0
+	NAMESPACE=dev ENABLE_WEBHOOKS=false go run ./cmd/unified/main.go --metrics-bind-address=localhost:0 --health-probe-bind-address=localhost:0
 
 .PHONY: run-aw
 run-aw: manifests generate fmt vet ## Run a controller from your host (webhooks are disabled).
-	ENABLE_WEBHOOKS=false go run ./cmd/standalone/main.go --metrics-bind-address=localhost:0 --health-probe-bind-address=localhost:0
+	NAMESPACE=dev ENABLE_WEBHOOKS=false go run ./cmd/standalone/main.go --metrics-bind-address=localhost:0 --health-probe-bind-address=localhost:0
 
 
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.

cmd/standalone/main.go

@@ -20,6 +20,7 @@ import (
 	"crypto/tls"
 	"flag"
 	"os"
+	"strings"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -29,7 +30,6 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -60,7 +60,7 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 
-	awConfig := config.NewConfig()
+	awConfig := config.NewConfig(namespaceOrDie())
 	awConfig.StandaloneMode = true
 	awConfig.ManageJobsWithoutQueueName = false
 
@@ -132,19 +132,27 @@ func main() {
 	}
 
 	ctx := ctrl.SetupSignalHandler()
-	err = controller.SetupWithManager(ctx, mgr, awConfig)
-	if err != nil {
-		setupLog.Error(err, "unable to start appwrapper controllers")
-		os.Exit(1)
+	certsReady := make(chan struct{})
+
+	if os.Getenv("ENABLE_WEBHOOKS") == "false" {
+		close(certsReady)
+	} else {
+		if err := controller.SetupCertManagement(mgr, &awConfig.CertManagement, certsReady); err != nil {
+			setupLog.Error(err, "Unable to set up cert rotation")
+			os.Exit(1)
+		}
 	}
 
-	//+kubebuilder:scaffold:builder
-	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
-		setupLog.Error(err, "unable to set up health check")
+	// Ascynchronous because controllers need to wait for certificate to be ready for webhooks to work
+	go controller.SetupControllers(ctx, mgr, awConfig, certsReady, setupLog)
+
+	if err := controller.SetupIndexers(ctx, mgr, awConfig); err != nil {
+		setupLog.Error(err, "unable to setup indexers")
 		os.Exit(1)
 	}
-	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
-		setupLog.Error(err, "unable to set up ready check")
+
+	if err := controller.SetupProbeEndpoints(mgr, certsReady); err != nil {
+		setupLog.Error(err, "unable to setup probe endpoints")
 		os.Exit(1)
 	}
 
@@ -154,3 +162,20 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func namespaceOrDie() string {
+	// This way assumes you've set the NAMESPACE environment variable either manually, when running
+	// the operator standalone, or using the downward API, when running the operator in-cluster.
+	if ns := os.Getenv("NAMESPACE"); ns != "" {
+		return ns
+	}
+
+	// Fall back to the namespace associated with the service account token, if available
+	if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
+		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
+			return ns
+		}
+	}
+
+	panic("unable to determine current namespace")
+}

cmd/unified/main.go

@@ -20,6 +20,7 @@ import (
 	"crypto/tls"
 	"flag"
 	"os"
+	"strings"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -29,7 +30,6 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -62,7 +62,7 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 
-	awConfig := config.NewConfig()
+	awConfig := config.NewConfig(namespaceOrDie())
 
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -133,19 +133,27 @@ func main() {
 	}
 
 	ctx := ctrl.SetupSignalHandler()
-	err = controller.SetupWithManager(ctx, mgr, awConfig)
-	if err != nil {
-		setupLog.Error(err, "unable to start appwrapper controllers")
-		os.Exit(1)
+	certsReady := make(chan struct{})
+
+	if os.Getenv("ENABLE_WEBHOOKS") == "false" {
+		close(certsReady)
+	} else {
+		if err := controller.SetupCertManagement(mgr, &awConfig.CertManagement, certsReady); err != nil {
+			setupLog.Error(err, "Unable to set up cert rotation")
+			os.Exit(1)
+		}
 	}
 
-	//+kubebuilder:scaffold:builder
-	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
-		setupLog.Error(err, "unable to set up health check")
+	// Ascynchronous because controllers need to wait for certificate to be ready for webhooks to work
+	go controller.SetupControllers(ctx, mgr, awConfig, certsReady, setupLog)
+
+	if err := controller.SetupIndexers(ctx, mgr, awConfig); err != nil {
+		setupLog.Error(err, "unable to setup indexers")
 		os.Exit(1)
 	}
-	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
-		setupLog.Error(err, "unable to set up ready check")
+
+	if err := controller.SetupProbeEndpoints(mgr, certsReady); err != nil {
+		setupLog.Error(err, "unable to setup probe endpoints")
 		os.Exit(1)
 	}
 
@@ -155,3 +163,20 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func namespaceOrDie() string {
+	// This way assumes you've set the NAMESPACE environment variable either manually, when running
+	// the operator standalone, or using the downward API, when running the operator in-cluster.
+	if ns := os.Getenv("NAMESPACE"); ns != "" {
+		return ns
+	}
+
+	// Fall back to the namespace associated with the service account token, if available
+	if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
+		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
+			return ns
+		}
+	}
+
+	panic("unable to determine current namespace")
+}

config/crd/kustomization.yaml

@@ -13,7 +13,7 @@ patches:
 
 # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix.
 # patches here are for enabling the CA injection for each CRD
-- path: patches/cainjection_in_appwrappers.yaml
+#- path: patches/cainjection_in_appwrappers.yaml
 #+kubebuilder:scaffold:crdkustomizecainjectionpatch
 
 # [WEBHOOK] To enable webhook, uncomment the following section