Bump sigs.k8s.io/controller-runtime to v0.14.4 (#5507)

* Bump sigs.k8s.io/controller-runtime to v0.14.4

* Update gofmt

vendor/sigs.k8s.io/kubefed/pkg/apis/core/common/constants.go

@@ -24,6 +24,8 @@ const (
 	ClusterReady ClusterConditionType = "Ready"
 	// ClusterOffline means the cluster is temporarily down or not reachable
 	ClusterOffline ClusterConditionType = "Offline"
+	// ClusterConfigMalformed means the cluster's configuration may be malformed.
+	ClusterConfigMalformed ClusterConditionType = "ConfigMalformed"
 )
 
 const (

vendor/sigs.k8s.io/kubefed/pkg/apis/core/v1beta1/kubefedcluster_types.go

@@ -70,6 +70,9 @@ type LocalSecretReference struct {
 type KubeFedClusterStatus struct {
 	// Conditions is an array of current cluster conditions.
 	Conditions []ClusterCondition `json:"conditions"`
+	// KubernetesVersion is the Kubernetes git version of the cluster.
+	// +optional
+	KubernetesVersion string `json:"kubernetesVersion,omitempty"`
 	// Zones are the names of availability zones in which the nodes of the cluster exist, e.g. 'us-east1-a'.
 	// +optional
 	Zones []string `json:"zones,omitempty"`
@@ -81,7 +84,8 @@ type KubeFedClusterStatus struct {
 // +kubebuilder:object:root=true
 // +kubebuilder:printcolumn:name=age,type=date,JSONPath=.metadata.creationTimestamp
 // +kubebuilder:printcolumn:name=ready,type=string,JSONPath=.status.conditions[?(@.type=='Ready')].status
-// +kubebuilder:resource:path=kubefedclusters
+// +kubebuilder:printcolumn:name=kubernetes-version,type=string,JSONPath=.status.kubernetesVersion
+// +kubebuilder:resource:path=kubefedclusters,shortName=kfc
 // +kubebuilder:subresource:status
 
 // KubeFedCluster configures KubeFed to be aware of a Kubernetes

vendor/sigs.k8s.io/kubefed/pkg/apis/core/v1beta1/kubefedconfig_types.go

@@ -48,6 +48,9 @@ type DurationConfig struct {
 	// Time to wait before giving up on an unhealthy cluster.
 	// +optional
 	UnavailableDelay *metav1.Duration `json:"unavailableDelay,omitempty"`
+	// Time to wait for all caches to sync before exit.
+	// +optional
+	CacheSyncTimeout *metav1.Duration `json:"cacheSyncTimeout,omitempty"`
 }
 type LeaderElectConfig struct {
 	// The duration that non-leader candidates will wait after observing a leadership

vendor/sigs.k8s.io/kubefed/pkg/apis/core/v1beta1/zz_generated.deepcopy.go

@@ -118,6 +118,11 @@ func (in *DurationConfig) DeepCopyInto(out *DurationConfig) {
 		*out = new(v1.Duration)
 		**out = **in
 	}
+	if in.CacheSyncTimeout != nil {
+		in, out := &in.CacheSyncTimeout, &out.CacheSyncTimeout
+		*out = new(v1.Duration)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DurationConfig.

vendor/sigs.k8s.io/kubefed/pkg/controller/util/cluster_util.go

@@ -124,26 +124,31 @@ func CustomizeTLSTransport(fedCluster *fedv1b1.KubeFedCluster, clientConfig *res
 		return errors.Errorf("Cluster %s transport error: %s", fedCluster.Name, err)
 	}
 
-	err = CustomizeCertificateValidation(fedCluster, transportConfig)
-	if err != nil {
-		return errors.Errorf("Cluster %s custom certificate validation error: %s", fedCluster.Name, err)
+	if transportConfig != nil {
+		err = CustomizeCertificateValidation(fedCluster, transportConfig)
+		if err != nil {
+			return errors.Errorf("Cluster %s custom certificate validation error: %s", fedCluster.Name, err)
+		}
+
+		// using the same defaults as http.DefaultTransport
+		clientConfig.Transport = &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+				DualStack: true,
+			}).DialContext,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+			TLSClientConfig:       transportConfig,
+		}
+		clientConfig.TLSClientConfig = restclient.TLSClientConfig{}
+	} else {
+		clientConfig.Insecure = true
 	}
 
-	// using the same defaults as http.DefaultTransport
-	clientConfig.Transport = &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
-			DualStack: true,
-		}).DialContext,
-		MaxIdleConns:          100,
-		IdleConnTimeout:       90 * time.Second,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig:       transportConfig,
-	}
-	clientConfig.TLSClientConfig = restclient.TLSClientConfig{}
 	return nil
 }
 

vendor/sigs.k8s.io/kubefed/pkg/controller/util/constants.go

@@ -20,9 +20,11 @@ import (
 	"time"
 )
 
-// Providing 0 duration to an informer indicates that resync should be delayed as long as possible
 const (
-	NoResyncPeriod time.Duration = 0 * time.Second
+	// Providing 0 duration to an informer indicates that resync should be delayed as long as possible
+	NoResyncPeriod = 0 * time.Second
+
+	SyncedPollPeriod = 10 * time.Second
 
 	NamespaceName = "namespaces"
 	NamespaceKind = "Namespace"
@@ -39,6 +41,12 @@ const (
 	StatusField   = "status"
 	MetadataField = "metadata"
 
+	// Service fields
+	HealthCheckNodePortField = "healthCheckNodePort"
+	ClusterIPField           = "clusterIP"
+	ClusterIPsField          = "clusterIPs"
+	PortsField               = "ports"
+
 	// ServiceAccount fields
 	SecretsField = "secrets"
 

vendor/sigs.k8s.io/kubefed/pkg/controller/util/controllerconfig.go

@@ -72,6 +72,7 @@ type ControllerConfig struct {
 	ClusterAvailableDelay         time.Duration
 	ClusterUnavailableDelay       time.Duration
 	MinimizeLatency               bool
+	CacheSyncTimeout              time.Duration
 	MaxConcurrentSyncReconciles   int64
 	MaxConcurrentStatusReconciles int64
 	SkipAdoptingResources         bool

vendor/sigs.k8s.io/kubefed/pkg/controller/util/federated_informer.go

@@ -534,33 +534,24 @@ func (fs *federatedStoreImpl) GetKeyFor(item interface{}) string {
 	return key
 }
 
-// Checks whether stores for all clusters form the lists (and only these) are there and
+// ClustersSynced checks whether stores for all clusters form the lists (and only these) are there and
 // are synced.
 func (fs *federatedStoreImpl) ClustersSynced(clusters []*fedv1b1.KubeFedCluster) bool {
-	// Get the list of informers to check under a lock and check it outside.
-	okSoFar, informersToCheck := func() (bool, []informer) {
-		fs.federatedInformer.Lock()
-		defer fs.federatedInformer.Unlock()
+	fs.federatedInformer.Lock()
+	defer fs.federatedInformer.Unlock()
 
-		if len(fs.federatedInformer.targetInformers) != len(clusters) {
-			return false, []informer{}
-		}
-		informersToCheck := make([]informer, 0, len(clusters))
-		for _, cluster := range clusters {
-			if targetInformer, found := fs.federatedInformer.targetInformers[cluster.Name]; found {
-				informersToCheck = append(informersToCheck, targetInformer)
-			} else {
-				return false, []informer{}
-			}
-		}
-		return true, informersToCheck
-	}()
-
-	if !okSoFar {
+	if len(fs.federatedInformer.targetInformers) != len(clusters) {
+		klog.V(4).Infof("The number of target informers mismatch with given clusters")
 		return false
 	}
-	for _, informerToCheck := range informersToCheck {
-		if !informerToCheck.controller.HasSynced() {
+	for _, cluster := range clusters {
+		if targetInformer, found := fs.federatedInformer.targetInformers[cluster.Name]; found {
+			if !targetInformer.controller.HasSynced() {
+				klog.V(4).Infof("Informer of cluster %q not synced", cluster.Name)
+				return false
+			}
+		} else {
+			klog.V(4).Infof("Informer of cluster %q not found", cluster.Name)
 			return false
 		}
 	}

vendor/sigs.k8s.io/kubefed/pkg/controller/util/worker.go

@@ -23,6 +23,8 @@ import (
 	"k8s.io/client-go/util/flowcontrol"
 	"k8s.io/client-go/util/workqueue"
 	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	"sigs.k8s.io/kubefed/pkg/metrics"
 )
 
 type ReconcileFunc func(qualifiedName QualifiedName) ReconciliationStatus
@@ -128,6 +130,8 @@ func (w *asyncWorker) EnqueueWithDelay(qualifiedName QualifiedName, delay time.D
 }
 
 func (w *asyncWorker) Run(stopChan <-chan struct{}) {
+	w.initMetrics()
+
 	StartBackoffGC(w.backoff, stopChan)
 	w.deliverer.StartWithHandler(func(item *DelayingDelivererItem) {
 		qualifiedName, ok := item.Value.(*QualifiedName)
@@ -183,16 +187,41 @@ func (w *asyncWorker) reconcileOnce() bool {
 		return true
 	}
 
+	metrics.ControllerRuntimeActiveWorkers.WithLabelValues(w.name).Add(1)
+	defer metrics.ControllerRuntimeActiveWorkers.WithLabelValues(w.name).Add(-1)
+	defer metrics.UpdateControllerRuntimeReconcileTimeFromStart(w.name, time.Now())
+
 	status := w.reconcile(qualifiedName)
 	switch status {
 	case StatusAllOK:
-		break
+		metrics.ControllerRuntimeReconcileTotal.WithLabelValues(w.name, labelSuccess).Inc()
 	case StatusError:
 		w.EnqueueForError(qualifiedName)
+		metrics.ControllerRuntimeReconcileErrors.WithLabelValues(w.name).Inc()
+		metrics.ControllerRuntimeReconcileTotal.WithLabelValues(w.name, labelError).Inc()
 	case StatusNeedsRecheck:
 		w.EnqueueForRetry(qualifiedName)
+		metrics.ControllerRuntimeReconcileTotal.WithLabelValues(w.name, labelNeedsRecheck).Inc()
 	case StatusNotSynced:
 		w.EnqueueForClusterSync(qualifiedName)
+		metrics.ControllerRuntimeReconcileTotal.WithLabelValues(w.name, labelNotSynced).Inc()
 	}
 	return true
 }
+
+const (
+	labelSuccess      = "success"
+	labelError        = "error"
+	labelNeedsRecheck = "needs_recheck"
+	labelNotSynced    = "not_synced"
+)
+
+func (w *asyncWorker) initMetrics() {
+	metrics.ControllerRuntimeActiveWorkers.WithLabelValues(w.name).Set(0)
+	metrics.ControllerRuntimeReconcileErrors.WithLabelValues(w.name).Add(0)
+	metrics.ControllerRuntimeReconcileTotal.WithLabelValues(w.name, labelSuccess).Add(0)
+	metrics.ControllerRuntimeReconcileTotal.WithLabelValues(w.name, labelError).Add(0)
+	metrics.ControllerRuntimeReconcileTotal.WithLabelValues(w.name, labelNeedsRecheck).Add(0)
+	metrics.ControllerRuntimeReconcileTotal.WithLabelValues(w.name, labelNotSynced).Add(0)
+	metrics.ControllerRuntimeWorkerCount.WithLabelValues(w.name).Set(float64(w.maxConcurrentReconciles))
+}

vendor/sigs.k8s.io/kubefed/pkg/kubefedctl/util/util.go

@@ -107,6 +107,12 @@ func ClusterServiceAccountName(joiningClusterName, hostClusterName string) strin
 	return fmt.Sprintf("%s-%s", joiningClusterName, hostClusterName)
 }
 
+// ClusterServiceAccountTokenSecretName returns the name of a service account token secret whose
+// credentials are used by the host cluster to access the client cluster.
+func ClusterServiceAccountTokenSecretName(joiningClusterName, hostClusterName string) string {
+	return fmt.Sprintf("%s-%s", joiningClusterName, hostClusterName)
+}
+
 // RoleName returns the name of a Role or ClusterRole and its
 // associated RoleBinding or ClusterRoleBinding that are used to allow
 // the service account to access necessary resources on the cluster.

vendor/sigs.k8s.io/kubefed/pkg/metrics/metrics.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
@@ -58,7 +59,7 @@ var (
 	reconcileFederatedResourcesDuration = prometheus.NewHistogram(
 		prometheus.HistogramOpts{
 			Name:    "reconcile_federated_resources_duration_seconds",
-			Help:    "Time taken to reconcile federated resources in the target clusters.",
+			Help:    "[Deprecated] Time taken to reconcile federated resources in the target clusters. Replaced by controller_runtime_reconcile_time_seconds.",
 			Buckets: []float64{0.01, 0.05, 0.1, 0.5, 1.0, 2.5, 5.0, 7.5, 10.0, 12.5, 15.0, 17.5, 20.0, 22.5, 25.0, 27.5, 30.0, 50.0, 75.0, 100.0, 1000.0},
 		},
 	)
@@ -90,7 +91,7 @@ var (
 	controllerRuntimeReconcileDuration = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Name:    "controller_runtime_reconcile_duration_seconds",
-			Help:    "Time taken by various parts of Kubefed controllers reconciliation loops.",
+			Help:    "[Deprecated] Time taken by various parts of Kubefed controllers reconciliation loops. Replaced by controller_runtime_reconcile_time_seconds.",
 			Buckets: []float64{0.01, 0.05, 0.1, 0.5, 1.0, 2.5, 5.0, 7.5, 10.0, 12.5, 15.0, 17.5, 20.0, 22.5, 25.0, 27.5, 30.0, 50.0, 75.0, 100.0, 1000.0},
 		}, []string{"controller"},
 	)
@@ -98,10 +99,37 @@ var (
 	controllerRuntimeReconcileDurationSummary = prometheus.NewSummaryVec(
 		prometheus.SummaryOpts{
 			Name:   "controller_runtime_reconcile_quantile_seconds",
-			Help:   "Quantiles of time taken by various parts of Kubefed controllers reconciliation loops.",
+			Help:   "[Deprecated] Quantiles of time taken by various parts of Kubefed controllers reconciliation loops. Replaced by controller_runtime_reconcile_time_seconds.",
 			MaxAge: time.Hour,
 		}, []string{"controller"},
 	)
+
+	ControllerRuntimeReconcileTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "controller_runtime_reconcile_total",
+		Help: "Total number of reconciliations per controller",
+	}, []string{"controller", "result"})
+
+	ControllerRuntimeReconcileErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "controller_runtime_reconcile_errors_total",
+		Help: "Total number of reconciliation errors per controller",
+	}, []string{"controller"})
+
+	ControllerRuntimeReconcileTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name: "controller_runtime_reconcile_time_seconds",
+		Help: "Length of time per reconciliation per controller",
+		Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
+			1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60},
+	}, []string{"controller"})
+
+	ControllerRuntimeWorkerCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "controller_runtime_max_concurrent_reconciles",
+		Help: "Maximum number of concurrent reconciles per controller",
+	}, []string{"controller"})
+
+	ControllerRuntimeActiveWorkers = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "controller_runtime_active_workers",
+		Help: "Number of currently used workers per controller",
+	}, []string{"controller"})
 )
 
 const (
@@ -117,6 +145,10 @@ const (
 // RegisterAll registers all metrics.
 func RegisterAll() {
 	metrics.Registry.MustRegister(
+		// expose process metrics like CPU, Memory, file descriptor usage etc.
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+		// expose Go runtime metrics like GC stats, memory stats etc.
+		collectors.NewGoCollector(),
 		kubefedClusterTotal,
 		joinedClusterTotal,
 		reconcileFederatedResourcesDuration,
@@ -127,6 +159,11 @@ func RegisterAll() {
 		dispatchOperationDuration,
 		controllerRuntimeReconcileDuration,
 		controllerRuntimeReconcileDurationSummary,
+		ControllerRuntimeReconcileTotal,
+		ControllerRuntimeReconcileErrors,
+		ControllerRuntimeReconcileTime,
+		ControllerRuntimeWorkerCount,
+		ControllerRuntimeActiveWorkers,
 	)
 }
 
@@ -203,10 +240,20 @@ func UpdateControllerReconcileDurationFromStart(controller string, start time.Ti
 
 // UpdateControllerReconcileDuration records the duration of the reconcile function of a controller
 func UpdateControllerReconcileDuration(controller string, duration time.Duration) {
-	if duration > LogReconcileLongDurationThreshold {
-		klog.V(4).Infof("Reconcile loop %s took %v to complete", controller, duration)
-	}
-
 	controllerRuntimeReconcileDurationSummary.WithLabelValues(controller).Observe(duration.Seconds())
 	controllerRuntimeReconcileDuration.WithLabelValues(controller).Observe(duration.Seconds())
 }
+
+// UpdateControllerRuntimeReconcileTimeFromStart records the duration of the reconcile loop of a controller
+func UpdateControllerRuntimeReconcileTimeFromStart(controller string, start time.Time) {
+	duration := time.Since(start)
+	UpdateControllerRuntimeReconcileTime(controller, duration)
+}
+
+// UpdateControllerRuntimeReconcileTime records the duration of the reconcile function of a controller
+func UpdateControllerRuntimeReconcileTime(controller string, duration time.Duration) {
+	if duration > LogReconcileLongDurationThreshold {
+		klog.V(4).Infof("Reconcile loop %s took %v to complete", controller, duration)
+	}
+	ControllerRuntimeReconcileTime.WithLabelValues(controller).Observe(duration.Seconds())
+}