feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

kube/pkg/quota/v1/evaluator/core/persistentvolumeclaims.go

@@ -17,9 +17,12 @@ limitations under the License.
 package core
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -29,7 +32,7 @@ import (
 
 	"kubesphere.io/kubesphere/kube/pkg/apis/core/v1/helper"
 	k8sfeatures "kubesphere.io/kubesphere/kube/pkg/features"
-	quota "kubesphere.io/kubesphere/kube/pkg/quota/v1"
+	"kubesphere.io/kubesphere/kube/pkg/quota/v1"
 	"kubesphere.io/kubesphere/kube/pkg/quota/v1/generic"
 )
 
@@ -51,29 +54,16 @@ var pvcResources = []corev1.ResourceName{
 // * bronze.storageclass.storage.k8s.io/requests.storage: 500Gi
 const storageClassSuffix string = ".storageclass.storage.k8s.io/"
 
-/* TODO: prune?
-// ResourceByStorageClass returns a quota resource name by storage class.
-func ResourceByStorageClass(storageClass string, resourceName corev1.ResourceName) corev1.ResourceName {
-	return corev1.ResourceName(string(storageClass + storageClassSuffix + string(resourceName)))
-}
-*/
-
-// V1ResourceByStorageClass returns a quota resource name by storage class.
-func V1ResourceByStorageClass(storageClass string, resourceName corev1.ResourceName) corev1.ResourceName {
-	return corev1.ResourceName(string(storageClass + storageClassSuffix + string(resourceName)))
-}
-
 // NewPersistentVolumeClaimEvaluator returns an evaluator that can evaluate persistent volume claims
-func NewPersistentVolumeClaimEvaluator(f quota.ListerForResourceFunc) quota.Evaluator {
-	listFuncByNamespace := generic.ListResourceUsingListerFunc(f, corev1.SchemeGroupVersion.WithResource("persistentvolumeclaims"))
-	pvcEvaluator := &pvcEvaluator{listFuncByNamespace: listFuncByNamespace}
+func NewPersistentVolumeClaimEvaluator(cache client.Reader) quota.Evaluator {
+	pvcEvaluator := &pvcEvaluator{cache: cache}
 	return pvcEvaluator
 }
 
 // pvcEvaluator knows how to evaluate quota usage for persistent volume claims
 type pvcEvaluator struct {
 	// listFuncByNamespace knows how to list pvc claims
-	listFuncByNamespace generic.ListFuncByNamespace
+	cache client.Reader
 }
 
 // Constraints verifies that all required resources are present on the item.
@@ -117,7 +107,7 @@ func (p *pvcEvaluator) UncoveredQuotaScopes(limitedScopes []corev1.ScopedResourc
 
 // MatchingResources takes the input specified list of resources and returns the set of resources it matches.
 func (p *pvcEvaluator) MatchingResources(items []corev1.ResourceName) []corev1.ResourceName {
-	result := []corev1.ResourceName{}
+	var result []corev1.ResourceName
 	for _, item := range items {
 		// match object count quota fields
 		if quota.Contains([]corev1.ResourceName{pvcObjectCountName}, item) {
@@ -170,9 +160,21 @@ func (p *pvcEvaluator) Usage(item runtime.Object) (corev1.ResourceList, error) {
 	return result, nil
 }
 
+func (p *pvcEvaluator) listPVC(namespace string) ([]runtime.Object, error) {
+	pvcList := &corev1.PersistentVolumeClaimList{}
+	if err := p.cache.List(context.Background(), pvcList, client.InNamespace(namespace)); err != nil {
+		return nil, err
+	}
+	pvcs := make([]runtime.Object, 0)
+	for _, pvc := range pvcList.Items {
+		pvcs = append(pvcs, &pvc)
+	}
+	return pvcs, nil
+}
+
 // UsageStats calculates aggregate usage for the object.
 func (p *pvcEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) {
-	return generic.CalculateUsageStats(options, p.listFuncByNamespace, generic.MatchesNoScopeFunc, p.Usage)
+	return generic.CalculateUsageStats(options, p.listPVC, generic.MatchesNoScopeFunc, p.Usage)
 }
 
 // ensure we implement required interface

kube/pkg/quota/v1/evaluator/core/pods.go

@@ -17,16 +17,18 @@ limitations under the License.
 package core
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"time"
 
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/utils/clock"
@@ -96,16 +98,14 @@ var validationSet = sets.New(
 )
 
 // NewPodEvaluator returns an evaluator that can evaluate pods
-func NewPodEvaluator(f quota.ListerForResourceFunc, clock clock.Clock) quota.Evaluator {
-	listFuncByNamespace := generic.ListResourceUsingListerFunc(f, corev1.SchemeGroupVersion.WithResource("pods"))
-	podEvaluator := &podEvaluator{listFuncByNamespace: listFuncByNamespace, clock: clock}
+func NewPodEvaluator(cache client.Reader, clock clock.Clock) quota.Evaluator {
+	podEvaluator := &podEvaluator{cache: cache, clock: clock}
 	return podEvaluator
 }
 
 // podEvaluator knows how to measure usage of pods.
 type podEvaluator struct {
-	// knows how to list pods
-	listFuncByNamespace generic.ListFuncByNamespace
+	cache client.Reader
 	// used to track time
 	clock clock.Clock
 }
@@ -212,7 +212,19 @@ func (p *podEvaluator) Usage(item runtime.Object) (corev1.ResourceList, error) {
 
 // UsageStats calculates aggregate usage for the object.
 func (p *podEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) {
-	return generic.CalculateUsageStats(options, p.listFuncByNamespace, podMatchesScopeFunc, p.Usage)
+	return generic.CalculateUsageStats(options, p.listPods, podMatchesScopeFunc, p.Usage)
+}
+
+func (p *podEvaluator) listPods(namespace string) ([]runtime.Object, error) {
+	podList := &corev1.PodList{}
+	if err := p.cache.List(context.Background(), podList, client.InNamespace(namespace)); err != nil {
+		return nil, err
+	}
+	pods := make([]runtime.Object, 0)
+	for _, pod := range podList.Items {
+		pods = append(pods, &pod)
+	}
+	return pods, nil
 }
 
 // verifies we implement the required interface.

kube/pkg/quota/v1/evaluator/core/registry.go

@@ -20,31 +20,32 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/utils/clock"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	quota "kubesphere.io/kubesphere/kube/pkg/quota/v1"
+	"kubesphere.io/kubesphere/kube/pkg/quota/v1"
 	"kubesphere.io/kubesphere/kube/pkg/quota/v1/generic"
 )
 
 // legacyObjectCountAliases are what we used to do simple object counting quota with mapped to alias
 var legacyObjectCountAliases = map[schema.GroupVersionResource]corev1.ResourceName{
-	corev1.SchemeGroupVersion.WithResource("configmaps"):             corev1.ResourceConfigMaps,
-	corev1.SchemeGroupVersion.WithResource("resourcequotas"):         corev1.ResourceQuotas,
-	corev1.SchemeGroupVersion.WithResource("replicationcontrollers"): corev1.ResourceReplicationControllers,
-	corev1.SchemeGroupVersion.WithResource("secrets"):                corev1.ResourceSecrets,
+	corev1.SchemeGroupVersion.WithResource(string(corev1.ResourceConfigMaps)):             corev1.ResourceConfigMaps,
+	corev1.SchemeGroupVersion.WithResource(string(corev1.ResourceQuotas)):                 corev1.ResourceQuotas,
+	corev1.SchemeGroupVersion.WithResource(string(corev1.ResourceReplicationControllers)): corev1.ResourceReplicationControllers,
+	corev1.SchemeGroupVersion.WithResource(string(corev1.ResourceSecrets)):                corev1.ResourceSecrets,
 }
 
 // NewEvaluators returns the list of static evaluators that manage more than counts
-func NewEvaluators(f quota.ListerForResourceFunc) []quota.Evaluator {
+func NewEvaluators(client client.Client) []quota.Evaluator {
 	// these evaluators have special logic
 	result := []quota.Evaluator{
-		NewPodEvaluator(f, clock.RealClock{}),
-		NewServiceEvaluator(f),
-		NewPersistentVolumeClaimEvaluator(f),
+		NewPodEvaluator(client, clock.RealClock{}),
+		NewServiceEvaluator(client),
+		NewPersistentVolumeClaimEvaluator(client),
 	}
 	// these evaluators require an alias for backwards compatibility
-	for gvr, alias := range legacyObjectCountAliases {
+	for gvk, alias := range legacyObjectCountAliases {
 		result = append(result,
-			generic.NewObjectCountEvaluator(gvr.GroupResource(), generic.ListResourceUsingListerFunc(f, gvr), alias))
+			generic.NewObjectCountEvaluator(gvk.GroupVersion().WithResource(string(alias)).GroupResource(), generic.ListResourceUsingCacheFunc(client, gvk), alias))
 	}
 	return result
 }

kube/pkg/quota/v1/evaluator/core/services.go

@@ -17,8 +17,11 @@ limitations under the License.
 package core
 
 import (
+	"context"
 	"fmt"
 
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -41,16 +44,15 @@ var serviceResources = []corev1.ResourceName{
 }
 
 // NewServiceEvaluator returns an evaluator that can evaluate services.
-func NewServiceEvaluator(f quota.ListerForResourceFunc) quota.Evaluator {
-	listFuncByNamespace := generic.ListResourceUsingListerFunc(f, corev1.SchemeGroupVersion.WithResource("services"))
-	serviceEvaluator := &serviceEvaluator{listFuncByNamespace: listFuncByNamespace}
+func NewServiceEvaluator(cache client.Reader) quota.Evaluator {
+	serviceEvaluator := &serviceEvaluator{cache: cache}
 	return serviceEvaluator
 }
 
 // serviceEvaluator knows how to measure usage for services.
 type serviceEvaluator struct {
 	// knows how to list items by namespace
-	listFuncByNamespace generic.ListFuncByNamespace
+	cache client.Reader
 }
 
 // Constraints verifies that all required resources are present on the item
@@ -131,9 +133,21 @@ func (p *serviceEvaluator) Usage(item runtime.Object) (corev1.ResourceList, erro
 	return result, nil
 }
 
+func (p *serviceEvaluator) listServices(namespace string) ([]runtime.Object, error) {
+	serviceList := &corev1.ServiceList{}
+	if err := p.cache.List(context.Background(), serviceList, client.InNamespace(namespace)); err != nil {
+		return nil, err
+	}
+	services := make([]runtime.Object, 0)
+	for _, svc := range serviceList.Items {
+		services = append(services, &svc)
+	}
+	return services, nil
+}
+
 // UsageStats calculates aggregate usage for the object.
 func (p *serviceEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) {
-	return generic.CalculateUsageStats(options, p.listFuncByNamespace, generic.MatchesNoScopeFunc, p.Usage)
+	return generic.CalculateUsageStats(options, p.listServices, generic.MatchesNoScopeFunc, p.Usage)
 }
 
 var _ quota.Evaluator = &serviceEvaluator{}

kube/pkg/quota/v1/generic/evaluator.go

@@ -17,109 +17,36 @@ limitations under the License.
 package generic
 
 import (
+	"context"
 	"fmt"
-	"sync/atomic"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/admission"
-	"k8s.io/client-go/informers"
-	"k8s.io/client-go/tools/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	quota "kubesphere.io/kubesphere/kube/pkg/quota/v1"
+	"kubesphere.io/kubesphere/kube/pkg/quota/v1"
 )
 
-// InformerForResourceFunc knows how to provision an informer
-type InformerForResourceFunc func(schema.GroupVersionResource) (informers.GenericInformer, error)
-
-// ListerFuncForResourceFunc knows how to provision a lister from an informer func.
-// The lister returns errors until the informer has synced.
-func ListerFuncForResourceFunc(f InformerForResourceFunc) quota.ListerForResourceFunc {
-	return func(gvr schema.GroupVersionResource) (cache.GenericLister, error) {
-		informer, err := f(gvr)
-		if err != nil {
-			return nil, err
-		}
-		return &protectedLister{
-			hasSynced:   cachedHasSynced(informer.Informer().HasSynced),
-			notReadyErr: fmt.Errorf("%v not yet synced", gvr),
-			delegate:    informer.Lister(),
-		}, nil
-	}
-}
-
-// cachedHasSynced returns a function that calls hasSynced() until it returns true once, then returns true
-func cachedHasSynced(hasSynced func() bool) func() bool {
-	cache := &atomic.Value{}
-	cache.Store(false)
-	return func() bool {
-		if cache.Load().(bool) {
-			// short-circuit if already synced
-			return true
-		}
-		if hasSynced() {
-			// remember we synced
-			cache.Store(true)
-			return true
-		}
-		return false
-	}
-}
-
-// protectedLister returns notReadyError if hasSynced returns false, otherwise delegates to delegate
-type protectedLister struct {
-	hasSynced   func() bool
-	notReadyErr error
-	delegate    cache.GenericLister
-}
-
-func (p *protectedLister) List(selector labels.Selector) (ret []runtime.Object, err error) {
-	if !p.hasSynced() {
-		return nil, p.notReadyErr
-	}
-	return p.delegate.List(selector)
-}
-func (p *protectedLister) Get(name string) (runtime.Object, error) {
-	if !p.hasSynced() {
-		return nil, p.notReadyErr
-	}
-	return p.delegate.Get(name)
-}
-func (p *protectedLister) ByNamespace(namespace string) cache.GenericNamespaceLister {
-	return &protectedNamespaceLister{p.hasSynced, p.notReadyErr, p.delegate.ByNamespace(namespace)}
-}
-
-// protectedNamespaceLister returns notReadyError if hasSynced returns false, otherwise delegates to delegate
-type protectedNamespaceLister struct {
-	hasSynced   func() bool
-	notReadyErr error
-	delegate    cache.GenericNamespaceLister
-}
-
-func (p *protectedNamespaceLister) List(selector labels.Selector) (ret []runtime.Object, err error) {
-	if !p.hasSynced() {
-		return nil, p.notReadyErr
-	}
-	return p.delegate.List(selector)
-}
-func (p *protectedNamespaceLister) Get(name string) (runtime.Object, error) {
-	if !p.hasSynced() {
-		return nil, p.notReadyErr
-	}
-	return p.delegate.Get(name)
-}
-
-// ListResourceUsingListerFunc returns a listing function based on the shared informer factory for the specified resource.
-func ListResourceUsingListerFunc(l quota.ListerForResourceFunc, resource schema.GroupVersionResource) ListFuncByNamespace {
+// ListResourceUsingCacheFunc returns a listing function based on the shared informer factory for the specified resource.
+func ListResourceUsingCacheFunc(cacheClient client.Client, gvr schema.GroupVersionResource) ListFuncByNamespace {
 	return func(namespace string) ([]runtime.Object, error) {
-		lister, err := l(resource)
+		gvk, err := cacheClient.RESTMapper().KindFor(gvr)
 		if err != nil {
 			return nil, err
 		}
-		return lister.ByNamespace(namespace).List(labels.Everything())
+		gvkObject, err := cacheClient.Scheme().New(gvk)
+		if err != nil {
+			return nil, err
+		}
+		objList := gvkObject.(client.ObjectList)
+		if err := cacheClient.List(context.Background(), objList); err != nil {
+			return nil, err
+		}
+		return meta.ExtractList(objList)
 	}
 }
 

kube/pkg/quota/v1/generic/registry.go

@@ -21,7 +21,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
 
-	quota "kubesphere.io/kubesphere/kube/pkg/quota/v1"
+	"kubesphere.io/kubesphere/kube/pkg/quota/v1"
 )
 
 // implements a basic registry

kube/pkg/quota/v1/install/registry.go

@@ -18,10 +18,11 @@ package install
 
 import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	quota "kubesphere.io/kubesphere/kube/pkg/quota/v1"
-	core "kubesphere.io/kubesphere/kube/pkg/quota/v1/evaluator/core"
-	generic "kubesphere.io/kubesphere/kube/pkg/quota/v1/generic"
+	"kubesphere.io/kubesphere/kube/pkg/quota/v1"
+	"kubesphere.io/kubesphere/kube/pkg/quota/v1/evaluator/core"
+	"kubesphere.io/kubesphere/kube/pkg/quota/v1/generic"
 )
 
 // NewQuotaConfigurationForAdmission returns a quota configuration for admission control.
@@ -31,8 +32,8 @@ func NewQuotaConfigurationForAdmission() quota.Configuration {
 }
 
 // NewQuotaConfigurationForControllers returns a quota configuration for controllers.
-func NewQuotaConfigurationForControllers(f quota.ListerForResourceFunc) quota.Configuration {
-	evaluators := core.NewEvaluators(f)
+func NewQuotaConfigurationForControllers(client client.Client) quota.Configuration {
+	evaluators := core.NewEvaluators(client)
 	return generic.NewConfiguration(evaluators, DefaultIgnoredResources())
 }
 

kube/pkg/quota/v1/interfaces.go

@@ -21,7 +21,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/admission"
-	"k8s.io/client-go/tools/cache"
 )
 
 // UsageStatsOptions is an options structs that describes how stats should be calculated
@@ -83,6 +82,3 @@ type Registry interface {
 	// List from registry
 	List() []Evaluator
 }
-
-// ListerForResourceFunc knows how to get a lister for a specific resource
-type ListerForResourceFunc func(schema.GroupVersionResource) (cache.GenericLister, error)