feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

vendor/sigs.k8s.io/controller-runtime/pkg/cache/multi_namespace_cache.go

@@ -23,50 +23,42 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/client-go/rest"
 	toolscache "k8s.io/client-go/tools/cache"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/internal/objectutil"
-)
 
-// NewCacheFunc - Function for creating a new cache from the options and a rest config.
-type NewCacheFunc func(config *rest.Config, opts Options) (Cache, error)
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+)
 
 // a new global namespaced cache to handle cluster scoped resources.
 const globalCache = "_cluster-scope"
 
-// MultiNamespacedCacheBuilder - Builder function to create a new multi-namespaced cache.
-// This will scope the cache to a list of namespaces. Listing for all namespaces
-// will list for all the namespaces that this knows about. By default this will create
-// a global cache for cluster scoped resource. Note that this is not intended
-// to be used for excluding namespaces, this is better done via a Predicate. Also note that
-// you may face performance issues when using this with a high number of namespaces.
-func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
-	return func(config *rest.Config, opts Options) (Cache, error) {
-		opts, err := defaultOpts(config, opts)
-		if err != nil {
-			return nil, err
-		}
+func newMultiNamespaceCache(
+	newCache newCacheFunc,
+	scheme *runtime.Scheme,
+	restMapper apimeta.RESTMapper,
+	namespaces map[string]Config,
+	globalConfig *Config, // may be nil in which case no cache for cluster-scoped objects will be created
+) Cache {
+	// Create every namespace cache.
+	caches := map[string]Cache{}
+	for namespace, config := range namespaces {
+		caches[namespace] = newCache(config, namespace)
+	}
 
-		caches := map[string]Cache{}
+	// Create a cache for cluster scoped resources if requested
+	var clusterCache Cache
+	if globalConfig != nil {
+		clusterCache = newCache(*globalConfig, corev1.NamespaceAll)
+	}
 
-		// create a cache for cluster scoped resources
-		gCache, err := New(config, opts)
-		if err != nil {
-			return nil, fmt.Errorf("error creating global cache: %w", err)
-		}
-
-		for _, ns := range namespaces {
-			opts.Namespace = ns
-			c, err := New(config, opts)
-			if err != nil {
-				return nil, err
-			}
-			caches[ns] = c
-		}
-		return &multiNamespaceCache{namespaceToCache: caches, Scheme: opts.Scheme, RESTMapper: opts.Mapper, clusterCache: gCache}, nil
+	return &multiNamespaceCache{
+		namespaceToCache: caches,
+		Scheme:           scheme,
+		RESTMapper:       restMapper,
+		clusterCache:     clusterCache,
 	}
 }
 
@@ -75,90 +67,117 @@ func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
 // operator to a list of namespaces instead of watching every namespace
 // in the cluster.
 type multiNamespaceCache struct {
-	namespaceToCache map[string]Cache
 	Scheme           *runtime.Scheme
 	RESTMapper       apimeta.RESTMapper
+	namespaceToCache map[string]Cache
 	clusterCache     Cache
 }
 
 var _ Cache = &multiNamespaceCache{}
 
 // Methods for multiNamespaceCache to conform to the Informers interface.
-func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj client.Object) (Informer, error) {
-	informers := map[string]Informer{}
 
-	// If the object is clusterscoped, get the informer from clusterCache,
+func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error) {
+	// If the object is cluster scoped, get the informer from clusterCache,
 	// if not use the namespaced caches.
-	isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper)
+	isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper)
 	if err != nil {
 		return nil, err
 	}
 	if !isNamespaced {
-		clusterCacheInf, err := c.clusterCache.GetInformer(ctx, obj)
+		clusterCacheInformer, err := c.clusterCache.GetInformer(ctx, obj, opts...)
 		if err != nil {
 			return nil, err
 		}
-		informers[globalCache] = clusterCacheInf
 
-		return &multiNamespaceInformer{namespaceToInformer: informers}, nil
+		return &multiNamespaceInformer{
+			namespaceToInformer: map[string]Informer{
+				globalCache: clusterCacheInformer,
+			},
+		}, nil
 	}
 
+	namespaceToInformer := map[string]Informer{}
 	for ns, cache := range c.namespaceToCache {
-		informer, err := cache.GetInformer(ctx, obj)
+		informer, err := cache.GetInformer(ctx, obj, opts...)
 		if err != nil {
 			return nil, err
 		}
-		informers[ns] = informer
+		namespaceToInformer[ns] = informer
 	}
 
-	return &multiNamespaceInformer{namespaceToInformer: informers}, nil
+	return &multiNamespaceInformer{namespaceToInformer: namespaceToInformer}, nil
 }
 
-func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) {
-	informers := map[string]Informer{}
-
+func (c *multiNamespaceCache) RemoveInformer(ctx context.Context, obj client.Object) error {
 	// If the object is clusterscoped, get the informer from clusterCache,
 	// if not use the namespaced caches.
-	isNamespaced, err := objectutil.IsAPINamespacedWithGVK(gvk, c.Scheme, c.RESTMapper)
+	isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper)
+	if err != nil {
+		return err
+	}
+	if !isNamespaced {
+		return c.clusterCache.RemoveInformer(ctx, obj)
+	}
+
+	for _, cache := range c.namespaceToCache {
+		err := cache.RemoveInformer(ctx, obj)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error) {
+	// If the object is cluster scoped, get the informer from clusterCache,
+	// if not use the namespaced caches.
+	isNamespaced, err := apiutil.IsGVKNamespaced(gvk, c.RESTMapper)
 	if err != nil {
 		return nil, err
 	}
 	if !isNamespaced {
-		clusterCacheInf, err := c.clusterCache.GetInformerForKind(ctx, gvk)
+		clusterCacheInformer, err := c.clusterCache.GetInformerForKind(ctx, gvk, opts...)
 		if err != nil {
 			return nil, err
 		}
-		informers[globalCache] = clusterCacheInf
 
-		return &multiNamespaceInformer{namespaceToInformer: informers}, nil
+		return &multiNamespaceInformer{
+			namespaceToInformer: map[string]Informer{
+				globalCache: clusterCacheInformer,
+			},
+		}, nil
 	}
 
+	namespaceToInformer := map[string]Informer{}
 	for ns, cache := range c.namespaceToCache {
-		informer, err := cache.GetInformerForKind(ctx, gvk)
+		informer, err := cache.GetInformerForKind(ctx, gvk, opts...)
 		if err != nil {
 			return nil, err
 		}
-		informers[ns] = informer
+		namespaceToInformer[ns] = informer
 	}
 
-	return &multiNamespaceInformer{namespaceToInformer: informers}, nil
+	return &multiNamespaceInformer{namespaceToInformer: namespaceToInformer}, nil
 }
 
 func (c *multiNamespaceCache) Start(ctx context.Context) error {
 	// start global cache
-	go func() {
-		err := c.clusterCache.Start(ctx)
-		if err != nil {
-			log.Error(err, "cluster scoped cache failed to start")
-		}
-	}()
+	if c.clusterCache != nil {
+		go func() {
+			err := c.clusterCache.Start(ctx)
+			if err != nil {
+				log.Error(err, "cluster scoped cache failed to start")
+			}
+		}()
+	}
 
 	// start namespaced caches
 	for ns, cache := range c.namespaceToCache {
 		go func(ns string, cache Cache) {
-			err := cache.Start(ctx)
-			if err != nil {
-				log.Error(err, "multinamespace cache failed to start namespaced informer", "namespace", ns)
+			if err := cache.Start(ctx); err != nil {
+				log.Error(err, "multi-namespace cache failed to start namespaced informer", "namespace", ns)
 			}
 		}(ns, cache)
 	}
@@ -170,22 +189,22 @@ func (c *multiNamespaceCache) Start(ctx context.Context) error {
 func (c *multiNamespaceCache) WaitForCacheSync(ctx context.Context) bool {
 	synced := true
 	for _, cache := range c.namespaceToCache {
-		if s := cache.WaitForCacheSync(ctx); !s {
-			synced = s
+		if !cache.WaitForCacheSync(ctx) {
+			synced = false
 		}
 	}
 
 	// check if cluster scoped cache has synced
-	if !c.clusterCache.WaitForCacheSync(ctx) {
+	if c.clusterCache != nil && !c.clusterCache.WaitForCacheSync(ctx) {
 		synced = false
 	}
 	return synced
 }
 
 func (c *multiNamespaceCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error {
-	isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper)
+	isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper)
 	if err != nil {
-		return nil //nolint:nilerr
+		return err
 	}
 
 	if !isNamespaced {
@@ -201,7 +220,7 @@ func (c *multiNamespaceCache) IndexField(ctx context.Context, obj client.Object,
 }
 
 func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
-	isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper)
+	isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper)
 	if err != nil {
 		return err
 	}
@@ -213,9 +232,12 @@ func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj
 
 	cache, ok := c.namespaceToCache[key.Namespace]
 	if !ok {
+		if global, hasGlobal := c.namespaceToCache[metav1.NamespaceAll]; hasGlobal {
+			return global.Get(ctx, key, obj, opts...)
+		}
 		return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", key)
 	}
-	return cache.Get(ctx, key, obj)
+	return cache.Get(ctx, key, obj, opts...)
 }
 
 // List multi namespace cache will get all the objects in the namespaces that the cache is watching if asked for all namespaces.
@@ -223,7 +245,7 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList,
 	listOpts := client.ListOptions{}
 	listOpts.ApplyOptions(opts)
 
-	isNamespaced, err := objectutil.IsAPINamespaced(list, c.Scheme, c.RESTMapper)
+	isNamespaced, err := apiutil.IsObjectNamespaced(list, c.Scheme, c.RESTMapper)
 	if err != nil {
 		return err
 	}
@@ -236,7 +258,7 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList,
 	if listOpts.Namespace != corev1.NamespaceAll {
 		cache, ok := c.namespaceToCache[listOpts.Namespace]
 		if !ok {
-			return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", listOpts.Namespace)
+			return fmt.Errorf("unable to list: %v because of unknown namespace for the cache", listOpts.Namespace)
 		}
 		return cache.List(ctx, list, opts...)
 	}
@@ -269,12 +291,14 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList,
 			return fmt.Errorf("object: %T must be a list type", list)
 		}
 		allItems = append(allItems, items...)
+
 		// The last list call should have the most correct resource version.
 		resourceVersion = accessor.GetResourceVersion()
 		if limitSet {
 			// decrement Limit by the number of items
 			// fetched from the current namespace.
 			listOpts.Limit -= int64(len(items))
+
 			// if a Limit was set and the number of
 			// items read has reached this set limit,
 			// then stop reading.
@@ -293,42 +317,71 @@ type multiNamespaceInformer struct {
 	namespaceToInformer map[string]Informer
 }
 
+type handlerRegistration struct {
+	handles map[string]toolscache.ResourceEventHandlerRegistration
+}
+
+type syncer interface {
+	HasSynced() bool
+}
+
+// HasSynced asserts that the handler has been called for the full initial state of the informer.
+// This uses syncer to be compatible between client-go 1.27+ and older versions when the interface changed.
+func (h handlerRegistration) HasSynced() bool {
+	for _, reg := range h.handles {
+		if s, ok := reg.(syncer); ok {
+			if !s.HasSynced() {
+				return false
+			}
+		}
+	}
+	return true
+}
+
 var _ Informer = &multiNamespaceInformer{}
 
-// AddEventHandler adds the handler to each namespaced informer.
+// AddEventHandler adds the handler to each informer.
 func (i *multiNamespaceInformer) AddEventHandler(handler toolscache.ResourceEventHandler) (toolscache.ResourceEventHandlerRegistration, error) {
-	handles := make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer))
+	handles := handlerRegistration{
+		handles: make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer)),
+	}
+
 	for ns, informer := range i.namespaceToInformer {
 		registration, err := informer.AddEventHandler(handler)
 		if err != nil {
 			return nil, err
 		}
-		handles[ns] = registration
+		handles.handles[ns] = registration
 	}
+
 	return handles, nil
 }
 
 // AddEventHandlerWithResyncPeriod adds the handler with a resync period to each namespaced informer.
 func (i *multiNamespaceInformer) AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) (toolscache.ResourceEventHandlerRegistration, error) {
-	handles := make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer))
+	handles := handlerRegistration{
+		handles: make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer)),
+	}
+
 	for ns, informer := range i.namespaceToInformer {
 		registration, err := informer.AddEventHandlerWithResyncPeriod(handler, resyncPeriod)
 		if err != nil {
 			return nil, err
 		}
-		handles[ns] = registration
+		handles.handles[ns] = registration
 	}
+
 	return handles, nil
 }
 
-// RemoveEventHandler removes a formerly added event handler given by its registration handle.
+// RemoveEventHandler removes a previously added event handler given by its registration handle.
 func (i *multiNamespaceInformer) RemoveEventHandler(h toolscache.ResourceEventHandlerRegistration) error {
-	handles, ok := h.(map[string]toolscache.ResourceEventHandlerRegistration)
+	handles, ok := h.(handlerRegistration)
 	if !ok {
-		return fmt.Errorf("it is not the registration returned by multiNamespaceInformer")
+		return fmt.Errorf("registration is not a registration returned by multiNamespaceInformer")
 	}
 	for ns, informer := range i.namespaceToInformer {
-		registration, ok := handles[ns]
+		registration, ok := handles.handles[ns]
 		if !ok {
 			continue
 		}
@@ -339,7 +392,7 @@ func (i *multiNamespaceInformer) RemoveEventHandler(h toolscache.ResourceEventHa
 	return nil
 }
 
-// AddIndexers adds the indexer for each namespaced informer.
+// AddIndexers adds the indexers to each informer.
 func (i *multiNamespaceInformer) AddIndexers(indexers toolscache.Indexers) error {
 	for _, informer := range i.namespaceToInformer {
 		err := informer.AddIndexers(indexers)
@@ -350,11 +403,21 @@ func (i *multiNamespaceInformer) AddIndexers(indexers toolscache.Indexers) error
 	return nil
 }
 
-// HasSynced checks if each namespaced informer has synced.
+// HasSynced checks if each informer has synced.
 func (i *multiNamespaceInformer) HasSynced() bool {
 	for _, informer := range i.namespaceToInformer {
-		if ok := informer.HasSynced(); !ok {
-			return ok
+		if !informer.HasSynced() {
+			return false
+		}
+	}
+	return true
+}
+
+// IsStopped checks if each namespaced informer has stopped, returns false if any are still running.
+func (i *multiNamespaceInformer) IsStopped() bool {
+	for _, informer := range i.namespaceToInformer {
+		if stopped := informer.IsStopped(); !stopped {
+			return false
 		}
 	}
 	return true