feat: kubesphere 4.0 (#6115)
* feat: kubesphere 4.0 Signed-off-by: ci-bot <ci-bot@kubesphere.io> * feat: kubesphere 4.0 Signed-off-by: ci-bot <ci-bot@kubesphere.io> --------- Signed-off-by: ci-bot <ci-bot@kubesphere.io> Co-authored-by: ks-ci-bot <ks-ci-bot@example.com> Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
committed by
GitHub
GitHub
parent
b5015ec7b9
commit
447a51f08b
@@ -1,127 +0,0 @@
|
||||
/*
|
||||
Copyright 2014 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package resourcequota
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
genericadmissioninitializer "k8s.io/apiserver/pkg/admission/initializer"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
"kubesphere.io/kubesphere/kube/pkg/quota/v1"
|
||||
"kubesphere.io/kubesphere/kube/pkg/quota/v1/generic"
|
||||
resourcequotaapi "kubesphere.io/kubesphere/kube/plugin/pkg/admission/resourcequota/apis/resourcequota"
|
||||
)
|
||||
|
||||
// QuotaAdmission implements an admission controller that can enforce quota constraints
|
||||
type QuotaAdmission struct {
|
||||
*admission.Handler
|
||||
config *resourcequotaapi.Configuration
|
||||
stopCh <-chan struct{}
|
||||
quotaConfiguration quota.Configuration
|
||||
numEvaluators int
|
||||
quotaAccessor *quotaAccessor
|
||||
evaluator Evaluator
|
||||
}
|
||||
|
||||
// WantsQuotaConfiguration defines a function which sets quota configuration for admission plugins that need it.
|
||||
type WantsQuotaConfiguration interface {
|
||||
SetQuotaConfiguration(quota.Configuration)
|
||||
admission.InitializationValidator
|
||||
}
|
||||
|
||||
var _ admission.ValidationInterface = &QuotaAdmission{}
|
||||
var _ = genericadmissioninitializer.WantsExternalKubeInformerFactory(&QuotaAdmission{})
|
||||
var _ = genericadmissioninitializer.WantsExternalKubeClientSet(&QuotaAdmission{})
|
||||
var _ = WantsQuotaConfiguration(&QuotaAdmission{})
|
||||
|
||||
type liveLookupEntry struct {
|
||||
expiry time.Time
|
||||
items []*corev1.ResourceQuota
|
||||
}
|
||||
|
||||
// NewResourceQuota configures an admission controller that can enforce quota constraints
|
||||
// using the provided registry. The registry must have the capability to handle group/kinds that
|
||||
// are persisted by the server this admission controller is intercepting
|
||||
func NewResourceQuota(config *resourcequotaapi.Configuration, numEvaluators int, stopCh <-chan struct{}) (*QuotaAdmission, error) {
|
||||
quotaAccessor, err := newQuotaAccessor()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &QuotaAdmission{
|
||||
Handler: admission.NewHandler(admission.Create, admission.Update),
|
||||
stopCh: stopCh,
|
||||
numEvaluators: numEvaluators,
|
||||
config: config,
|
||||
quotaAccessor: quotaAccessor,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SetExternalKubeClientSet registers the client into QuotaAdmission
|
||||
func (a *QuotaAdmission) SetExternalKubeClientSet(client kubernetes.Interface) {
|
||||
a.quotaAccessor.client = client
|
||||
}
|
||||
|
||||
// SetExternalKubeInformerFactory registers an informer factory into QuotaAdmission
|
||||
func (a *QuotaAdmission) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) {
|
||||
a.quotaAccessor.lister = f.Core().V1().ResourceQuotas().Lister()
|
||||
}
|
||||
|
||||
// SetQuotaConfiguration assigns and initializes configuration and evaluator for QuotaAdmission
|
||||
func (a *QuotaAdmission) SetQuotaConfiguration(c quota.Configuration) {
|
||||
a.quotaConfiguration = c
|
||||
a.evaluator = NewQuotaEvaluator(a.quotaAccessor, a.quotaConfiguration.IgnoredResources(), generic.NewRegistry(a.quotaConfiguration.Evaluators()), nil, a.config, a.numEvaluators, a.stopCh)
|
||||
}
|
||||
|
||||
// ValidateInitialization ensures an authorizer is set.
|
||||
func (a *QuotaAdmission) ValidateInitialization() error {
|
||||
if a.quotaAccessor == nil {
|
||||
return fmt.Errorf("missing quotaAccessor")
|
||||
}
|
||||
if a.quotaAccessor.client == nil {
|
||||
return fmt.Errorf("missing quotaAccessor.client")
|
||||
}
|
||||
if a.quotaAccessor.lister == nil {
|
||||
return fmt.Errorf("missing quotaAccessor.lister")
|
||||
}
|
||||
if a.quotaConfiguration == nil {
|
||||
return fmt.Errorf("missing quotaConfiguration")
|
||||
}
|
||||
if a.evaluator == nil {
|
||||
return fmt.Errorf("missing evaluator")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate makes admission decisions while enforcing quota
|
||||
func (a *QuotaAdmission) Validate(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces) (err error) {
|
||||
// ignore all operations that correspond to sub-resource actions
|
||||
if attr.GetSubresource() != "" {
|
||||
return nil
|
||||
}
|
||||
// ignore all operations that are not namespaced
|
||||
if attr.GetNamespace() == "" {
|
||||
return nil
|
||||
}
|
||||
return a.evaluator.Evaluate(attr)
|
||||
}
|
||||
@@ -1,21 +1,3 @@
|
||||
/*
|
||||
|
||||
Copyright 2021 The KubeSphere Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
|
||||
package resourcequota
|
||||
|
||||
import (
|
||||
|
||||
@@ -23,8 +23,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
@@ -35,6 +33,7 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
"kubesphere.io/kubesphere/kube/pkg/quota/v1"
|
||||
"kubesphere.io/kubesphere/kube/pkg/quota/v1/generic"
|
||||
|
||||
@@ -17,18 +17,7 @@ limitations under the License.
|
||||
package resourcequota
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apiserver/pkg/storage"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
corev1listers "k8s.io/client-go/listers/core/v1"
|
||||
)
|
||||
|
||||
// QuotaAccessor abstracts the get/set logic from the rest of the Evaluator. This could be a test stub, a straight passthrough,
|
||||
@@ -41,113 +30,3 @@ type QuotaAccessor interface {
|
||||
// GetQuotas gets all possible quotas for a given namespace
|
||||
GetQuotas(namespace string) ([]corev1.ResourceQuota, error)
|
||||
}
|
||||
|
||||
type quotaAccessor struct {
|
||||
client kubernetes.Interface
|
||||
|
||||
// lister can list/get quota objects from a shared informer's cache
|
||||
lister corev1listers.ResourceQuotaLister
|
||||
|
||||
// liveLookups holds the last few live lookups we've done to help ammortize cost on repeated lookup failures.
|
||||
// This lets us handle the case of latent caches, by looking up actual results for a namespace on cache miss/no results.
|
||||
// We track the lookup result here so that for repeated requests, we don't look it up very often.
|
||||
liveLookupCache *lru.Cache
|
||||
liveTTL time.Duration
|
||||
// updatedQuotas holds a cache of quotas that we've updated. This is used to pull the "really latest" during back to
|
||||
// back quota evaluations that touch the same quota doc. This only works because we can compare etcd resourceVersions
|
||||
// for the same resource as integers. Before this change: 22 updates with 12 conflicts. after this change: 15 updates with 0 conflicts
|
||||
updatedQuotas *lru.Cache
|
||||
}
|
||||
|
||||
// newQuotaAccessor creates an object that conforms to the QuotaAccessor interface to be used to retrieve quota objects.
|
||||
func newQuotaAccessor() (*quotaAccessor, error) {
|
||||
liveLookupCache, err := lru.New(100)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
updatedCache, err := lru.New(100)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// client and lister will be set when SetInternalKubeClientSet and SetInternalKubeInformerFactory are invoked
|
||||
return "aAccessor{
|
||||
liveLookupCache: liveLookupCache,
|
||||
liveTTL: time.Duration(30 * time.Second),
|
||||
updatedQuotas: updatedCache,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *quotaAccessor) UpdateQuotaStatus(newQuota *corev1.ResourceQuota) error {
|
||||
updatedQuota, err := e.client.CoreV1().ResourceQuotas(newQuota.Namespace).UpdateStatus(context.TODO(), newQuota, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
key := newQuota.Namespace + "/" + newQuota.Name
|
||||
e.updatedQuotas.Add(key, updatedQuota)
|
||||
return nil
|
||||
}
|
||||
|
||||
var storageVersioner = storage.APIObjectVersioner{}
|
||||
|
||||
// checkCache compares the passed quota against the value in the look-aside cache and returns the newer
|
||||
// if the cache is out of date, it deletes the stale entry. This only works because of etcd resourceVersions
|
||||
// being monotonically increasing integers
|
||||
func (e *quotaAccessor) checkCache(quota *corev1.ResourceQuota) *corev1.ResourceQuota {
|
||||
key := quota.Namespace + "/" + quota.Name
|
||||
uncastCachedQuota, ok := e.updatedQuotas.Get(key)
|
||||
if !ok {
|
||||
return quota
|
||||
}
|
||||
cachedQuota := uncastCachedQuota.(*corev1.ResourceQuota)
|
||||
|
||||
if storageVersioner.CompareResourceVersion(quota, cachedQuota) >= 0 {
|
||||
e.updatedQuotas.Remove(key)
|
||||
return quota
|
||||
}
|
||||
return cachedQuota
|
||||
}
|
||||
|
||||
func (e *quotaAccessor) GetQuotas(namespace string) ([]corev1.ResourceQuota, error) {
|
||||
// determine if there are any quotas in this namespace
|
||||
// if there are no quotas, we don't need to do anything
|
||||
items, err := e.lister.ResourceQuotas(namespace).List(labels.Everything())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error resolving quota: %v", err)
|
||||
}
|
||||
|
||||
// if there are no items held in our indexer, check our live-lookup LRU, if that misses, do the live lookup to prime it.
|
||||
if len(items) == 0 {
|
||||
lruItemObj, ok := e.liveLookupCache.Get(namespace)
|
||||
if !ok || lruItemObj.(liveLookupEntry).expiry.Before(time.Now()) {
|
||||
// TODO: If there are multiple operations at the same time and cache has just expired,
|
||||
// this may cause multiple List operations being issued at the same time.
|
||||
// If there is already in-flight List() for a given namespace, we should wait until
|
||||
// it is finished and cache is updated instead of doing the same, also to avoid
|
||||
// throttling - see #22422 for details.
|
||||
liveList, err := e.client.CoreV1().ResourceQuotas(namespace).List(context.TODO(), metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
newEntry := liveLookupEntry{expiry: time.Now().Add(e.liveTTL)}
|
||||
for i := range liveList.Items {
|
||||
newEntry.items = append(newEntry.items, &liveList.Items[i])
|
||||
}
|
||||
e.liveLookupCache.Add(namespace, newEntry)
|
||||
lruItemObj = newEntry
|
||||
}
|
||||
lruEntry := lruItemObj.(liveLookupEntry)
|
||||
items = append(items, lruEntry.items...)
|
||||
}
|
||||
|
||||
resourceQuotas := []corev1.ResourceQuota{}
|
||||
for i := range items {
|
||||
quota := items[i]
|
||||
quota = e.checkCache(quota)
|
||||
// always make a copy. We're going to muck around with this and we should never mutate the originals
|
||||
resourceQuotas = append(resourceQuotas, *quota)
|
||||
}
|
||||
|
||||
return resourceQuotas, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user