Omit informer has already started error

Author: aledbf

components/ws-manager-mk2/controllers/maintenance_controller.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
 	"sync"
 	"time"
 
@@ -17,7 +18,12 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
 var (
@@ -106,9 +112,44 @@ func (r *MaintenanceReconciler) setEnabledUntil(ctx context.Context, enabledUnti
 	log.FromContext(ctx).Info("maintenance mode state change", "enabledUntil", enabledUntil)
 }
 
-func (r *MaintenanceReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		Named("maintenance").
-		For(&corev1.ConfigMap{}).
-		Complete(r)
+func (r *MaintenanceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
+	c, err := controller.NewUnmanaged("maintenance-controller", mgr, controller.Options{Reconciler: r})
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		err = c.Start(ctx)
+		if err != nil {
+			log.FromContext(ctx).Error(err, "cannot start maintenance reconciler")
+			os.Exit(1)
+		}
+	}()
+
+	return c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), &handler.EnqueueRequestForObject{}, &filterConfigMap{})
+
+}
+
+type filterConfigMap struct {
+	predicate.Funcs
+}
+
+func (f filterConfigMap) Create(e event.CreateEvent) bool {
+	return f.filter(e.Object)
+}
+
+func (f filterConfigMap) Update(e event.UpdateEvent) bool {
+	return f.filter(e.ObjectNew)
+}
+
+func (f filterConfigMap) Generic(e event.GenericEvent) bool {
+	return f.filter(e.Object)
+}
+
+func (f filterConfigMap) filter(obj client.Object) bool {
+	if obj == nil {
+		return false
+	}
+
+	return obj.GetName() == configMapKey.Name && obj.GetNamespace() == configMapKey.Namespace
 }

components/ws-manager-mk2/controllers/workspace_controller.go

@@ -500,8 +500,8 @@ func (r *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		return []string{owner.Name}
 	}
 	err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Pod{}, wsOwnerKey, idx)
-	if err != nil {
-		return err
+	if err != nil && err.Error() == "informer has already started" {
+		return nil
 	}
 
 	return ctrl.NewControllerManagedBy(mgr).

components/ws-manager-mk2/main.go

@@ -112,18 +112,20 @@ func main() {
 		setupLog.Error(nil, "namespace cannot be empty")
 		os.Exit(1)
 	}
+
 	if cfg.Manager.SecretsNamespace == "" {
 		setupLog.Error(nil, "secretsNamespace cannot be empty")
 		os.Exit(1)
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:                 scheme,
-		MetricsBindAddress:     cfg.Prometheus.Addr,
-		Port:                   9443,
-		HealthProbeBindAddress: cfg.Health.Addr,
-		LeaderElection:         true,
-		LeaderElectionID:       "ws-manager-mk2-leader.gitpod.io",
+		Scheme:                        scheme,
+		MetricsBindAddress:            cfg.Prometheus.Addr,
+		Port:                          9443,
+		HealthProbeBindAddress:        cfg.Health.Addr,
+		LeaderElection:                true,
+		LeaderElectionID:              "ws-manager-mk2-leader.gitpod.io",
+		LeaderElectionReleaseOnCancel: true,
 		NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) {
 			opts.Namespaces = []string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}
 			return cache.New(config, opts)
@@ -134,53 +136,50 @@ func main() {
 		os.Exit(1)
 	}
 
-	go func() {
-		<-mgr.Elected()
-
-		maintenanceReconciler, err := controllers.NewMaintenanceReconciler(mgr.GetClient())
-		if err != nil {
-			setupLog.Error(err, "unable to create maintenance controller", "controller", "Maintenance")
-			os.Exit(1)
-		}
+	mgrCtx := ctrl.SetupSignalHandler()
 
-		workspaceReconciler, err := controllers.NewWorkspaceReconciler(
-			mgr.GetClient(), mgr.GetScheme(), mgr.GetEventRecorderFor("workspace"), &cfg.Manager, metrics.Registry, maintenanceReconciler)
-		if err != nil {
-			setupLog.Error(err, "unable to create controller", "controller", "Workspace")
-			os.Exit(1)
-		}
+	maintenanceReconciler, err := controllers.NewMaintenanceReconciler(mgr.GetClient())
+	if err != nil {
+		setupLog.Error(err, "unable to create maintenance controller", "controller", "Maintenance")
+		os.Exit(1)
+	}
 
-		activity := activity.NewWorkspaceActivity()
+	workspaceReconciler, err := controllers.NewWorkspaceReconciler(
+		mgr.GetClient(), mgr.GetScheme(), mgr.GetEventRecorderFor("workspace"), &cfg.Manager, metrics.Registry, maintenanceReconciler)
+	if err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "Workspace")
+		os.Exit(1)
+	}
 
-		timeoutReconciler, err := controllers.NewTimeoutReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("workspace"), cfg.Manager, activity, maintenanceReconciler)
-		if err != nil {
-			setupLog.Error(err, "unable to create timeout controller", "controller", "Timeout")
-			os.Exit(1)
-		}
+	activity := activity.NewWorkspaceActivity()
 
-		wsmanService, err := setupGRPCService(cfg, mgr.GetClient(), activity, maintenanceReconciler)
-		if err != nil {
-			setupLog.Error(err, "unable to start manager service")
-			os.Exit(1)
-		}
+	timeoutReconciler, err := controllers.NewTimeoutReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("workspace"), cfg.Manager, activity, maintenanceReconciler)
+	if err != nil {
+		setupLog.Error(err, "unable to create timeout controller", "controller", "Timeout")
+		os.Exit(1)
+	}
 
-		workspaceReconciler.OnReconcile = wsmanService.OnWorkspaceReconcile
+	wsmanService, err := setupGRPCService(cfg, mgr.GetClient(), activity, maintenanceReconciler)
+	if err != nil {
+		setupLog.Error(err, "unable to start manager service")
+		os.Exit(1)
+	}
 
-		if err = workspaceReconciler.SetupWithManager(mgr); err != nil {
-			setupLog.Error(err, "unable to setup workspace controller with manager", "controller", "Workspace")
-			os.Exit(1)
-		}
+	workspaceReconciler.OnReconcile = wsmanService.OnWorkspaceReconcile
+	if err = workspaceReconciler.SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to setup workspace controller with manager", "controller", "Workspace")
+		os.Exit(1)
+	}
 
-		if err = timeoutReconciler.SetupWithManager(mgr); err != nil {
-			setupLog.Error(err, "unable to setup timeout controller with manager", "controller", "Timeout")
-			os.Exit(1)
-		}
+	if err = timeoutReconciler.SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to setup timeout controller with manager", "controller", "Timeout")
+		os.Exit(1)
+	}
 
-		if err = maintenanceReconciler.SetupWithManager(mgr); err != nil {
-			setupLog.Error(err, "unable to setup maintenance controller with manager", "controller", "Maintenance")
-			os.Exit(1)
-		}
-	}()
+	if err = maintenanceReconciler.SetupWithManager(mgrCtx, mgr); err != nil {
+		setupLog.Error(err, "unable to setup maintenance controller with manager", "controller", "Maintenance")
+		os.Exit(1)
+	}
 
 	// if err = (&workspacev1.Workspace{}).SetupWebhookWithManager(mgr); err != nil {
 	// 	setupLog.Error(err, "unable to create webhook", "webhook", "Workspace")
@@ -199,7 +198,7 @@ func main() {
 	}
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	if err := mgr.Start(mgrCtx); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}

components/ws-manager-mk2/service/manager.go

@@ -16,25 +16,13 @@ import (
 	validation "github.com/go-ozzo/ozzo-validation"
 	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/xerrors"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/peer"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/timestamppb"
-
-	wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
-	"github.com/gitpod-io/gitpod/common-go/log"
-	"github.com/gitpod-io/gitpod/common-go/tracing"
-	"github.com/gitpod-io/gitpod/common-go/util"
-	"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
-	"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
-	wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
-	"github.com/gitpod-io/gitpod/ws-manager/api/config"
-	workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
-
-	csapi "github.com/gitpod-io/gitpod/content-service/api"
-	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -46,6 +34,17 @@ import (
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
+	"github.com/gitpod-io/gitpod/common-go/log"
+	"github.com/gitpod-io/gitpod/common-go/tracing"
+	"github.com/gitpod-io/gitpod/common-go/util"
+	csapi "github.com/gitpod-io/gitpod/content-service/api"
+	"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity"
+	"github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance"
+	wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api"
+	"github.com/gitpod-io/gitpod/ws-manager/api/config"
+	workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
 )
 
 const (