admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: kubesphere 4.0 (#6115)

Browse Source 
* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

337
kube/pkg/api/v1/pod/util.go
View File

@@ -1,337 +0,0 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package pod
import (
"fmt"
"time"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"kubesphere.io/kubesphere/kube/pkg/features"
)
// FindPort locates the container port for the given pod and portName. If the
// targetPort is a number, use that. If the targetPort is a string, look that
// string up in all named ports in all containers in the target pod. If no
// match is found, fail.
func FindPort(pod *v1.Pod, svcPort *v1.ServicePort) (int, error) {
portName := svcPort.TargetPort
switch portName.Type {
case intstr.String:
name := portName.StrVal
for _, container := range pod.Spec.Containers {
for _, port := range container.Ports {
if port.Name == name && port.Protocol == svcPort.Protocol {
return int(port.ContainerPort), nil
}
}
}
case intstr.Int:
return portName.IntValue(), nil
}
return 0, fmt.Errorf("no suitable port for manifest: %s", pod.UID)
}
// ContainerVisitor is called with each container spec, and returns true
// if visiting should continue.
type ContainerVisitor func(container *v1.Container) (shouldContinue bool)
// VisitContainers invokes the visitor function with a pointer to the container
// spec of every container in the given pod spec. If visitor returns false,
// visiting is short-circuited. VisitContainers returns true if visiting completes,
// false if visiting was short-circuited.
func VisitContainers(podSpec *v1.PodSpec, visitor ContainerVisitor) bool {
for i := range podSpec.InitContainers {
if !visitor(&podSpec.InitContainers[i]) {
return false
}
}
for i := range podSpec.Containers {
if !visitor(&podSpec.Containers[i]) {
return false
}
}
if utilfeature.DefaultFeatureGate.Enabled(features.EphemeralContainers) {
for i := range podSpec.EphemeralContainers {
if !visitor((*v1.Container)(&podSpec.EphemeralContainers[i].EphemeralContainerCommon)) {
return false
}
}
}
return true
}
// Visitor is called with each object name, and returns true if visiting should continue
type Visitor func(name string) (shouldContinue bool)
// VisitPodSecretNames invokes the visitor function with the name of every secret
// referenced by the pod spec. If visitor returns false, visiting is short-circuited.
// Transitive references (e.g. pod -> pvc -> pv -> secret) are not visited.
// Returns true if visiting completed, false if visiting was short-circuited.
func VisitPodSecretNames(pod *v1.Pod, visitor Visitor) bool {
for _, reference := range pod.Spec.ImagePullSecrets {
if !visitor(reference.Name) {
return false
}
}
VisitContainers(&pod.Spec, func(c *v1.Container) bool {
return visitContainerSecretNames(c, visitor)
})
var source *v1.VolumeSource
for i := range pod.Spec.Volumes {
source = &pod.Spec.Volumes[i].VolumeSource
switch {
case source.AzureFile != nil:
if len(source.AzureFile.SecretName) > 0 && !visitor(source.AzureFile.SecretName) {
return false
}
case source.CephFS != nil:
if source.CephFS.SecretRef != nil && !visitor(source.CephFS.SecretRef.Name) {
return false
}
case source.Cinder != nil:
if source.Cinder.SecretRef != nil && !visitor(source.Cinder.SecretRef.Name) {
return false
}
case source.FlexVolume != nil:
if source.FlexVolume.SecretRef != nil && !visitor(source.FlexVolume.SecretRef.Name) {
return false
}
case source.Projected != nil:
for j := range source.Projected.Sources {
if source.Projected.Sources[j].Secret != nil {
if !visitor(source.Projected.Sources[j].Secret.Name) {
return false
}
}
}
case source.RBD != nil:
if source.RBD.SecretRef != nil && !visitor(source.RBD.SecretRef.Name) {
return false
}
case source.Secret != nil:
if !visitor(source.Secret.SecretName) {
return false
}
case source.ScaleIO != nil:
if source.ScaleIO.SecretRef != nil && !visitor(source.ScaleIO.SecretRef.Name) {
return false
}
case source.ISCSI != nil:
if source.ISCSI.SecretRef != nil && !visitor(source.ISCSI.SecretRef.Name) {
return false
}
case source.StorageOS != nil:
if source.StorageOS.SecretRef != nil && !visitor(source.StorageOS.SecretRef.Name) {
return false
}
case source.CSI != nil:
if source.CSI.NodePublishSecretRef != nil && !visitor(source.CSI.NodePublishSecretRef.Name) {
return false
}
}
}
return true
}
func visitContainerSecretNames(container *v1.Container, visitor Visitor) bool {
for _, env := range container.EnvFrom {
if env.SecretRef != nil {
if !visitor(env.SecretRef.Name) {
return false
}
}
}
for _, envVar := range container.Env {
if envVar.ValueFrom != nil && envVar.ValueFrom.SecretKeyRef != nil {
if !visitor(envVar.ValueFrom.SecretKeyRef.Name) {
return false
}
}
}
return true
}
// VisitPodConfigmapNames invokes the visitor function with the name of every configmap
// referenced by the pod spec. If visitor returns false, visiting is short-circuited.
// Transitive references (e.g. pod -> pvc -> pv -> secret) are not visited.
// Returns true if visiting completed, false if visiting was short-circuited.
func VisitPodConfigmapNames(pod *v1.Pod, visitor Visitor) bool {
VisitContainers(&pod.Spec, func(c *v1.Container) bool {
return visitContainerConfigmapNames(c, visitor)
})
var source *v1.VolumeSource
for i := range pod.Spec.Volumes {
source = &pod.Spec.Volumes[i].VolumeSource
switch {
case source.Projected != nil:
for j := range source.Projected.Sources {
if source.Projected.Sources[j].ConfigMap != nil {
if !visitor(source.Projected.Sources[j].ConfigMap.Name) {
return false
}
}
}
case source.ConfigMap != nil:
if !visitor(source.ConfigMap.Name) {
return false
}
}
}
return true
}
func visitContainerConfigmapNames(container *v1.Container, visitor Visitor) bool {
for _, env := range container.EnvFrom {
if env.ConfigMapRef != nil {
if !visitor(env.ConfigMapRef.Name) {
return false
}
}
}
for _, envVar := range container.Env {
if envVar.ValueFrom != nil && envVar.ValueFrom.ConfigMapKeyRef != nil {
if !visitor(envVar.ValueFrom.ConfigMapKeyRef.Name) {
return false
}
}
}
return true
}
// GetContainerStatus extracts the status of container "name" from "statuses".
// It also returns if "name" exists.
func GetContainerStatus(statuses []v1.ContainerStatus, name string) (v1.ContainerStatus, bool) {
for i := range statuses {
if statuses[i].Name == name {
return statuses[i], true
}
}
return v1.ContainerStatus{}, false
}
// GetExistingContainerStatus extracts the status of container "name" from "statuses",
// It also returns if "name" exists.
func GetExistingContainerStatus(statuses []v1.ContainerStatus, name string) v1.ContainerStatus {
status, _ := GetContainerStatus(statuses, name)
return status
}
// IsPodAvailable returns true if a pod is available; false otherwise.
// Precondition for an available pod is that it must be ready. On top
// of that, there are two cases when a pod can be considered available:
// 1. minReadySeconds == 0, or
// 2. LastTransitionTime (is set) + minReadySeconds < current time
func IsPodAvailable(pod *v1.Pod, minReadySeconds int32, now metav1.Time) bool {
if !IsPodReady(pod) {
return false
}
c := GetPodReadyCondition(pod.Status)
minReadySecondsDuration := time.Duration(minReadySeconds) * time.Second
if minReadySeconds == 0 || !c.LastTransitionTime.IsZero() && c.LastTransitionTime.Add(minReadySecondsDuration).Before(now.Time) {
return true
}
return false
}
// IsPodReady returns true if a pod is ready; false otherwise.
func IsPodReady(pod *v1.Pod) bool {
return IsPodReadyConditionTrue(pod.Status)
}
// IsPodReadyConditionTrue returns true if a pod is ready; false otherwise.
func IsPodReadyConditionTrue(status v1.PodStatus) bool {
condition := GetPodReadyCondition(status)
return condition != nil && condition.Status == v1.ConditionTrue
}
// GetPodReadyCondition extracts the pod ready condition from the given status and returns that.
// Returns nil if the condition is not present.
func GetPodReadyCondition(status v1.PodStatus) *v1.PodCondition {
_, condition := GetPodCondition(&status, v1.PodReady)
return condition
}
// GetPodCondition extracts the provided condition from the given status and returns that.
// Returns nil and -1 if the condition is not present, and the index of the located condition.
func GetPodCondition(status *v1.PodStatus, conditionType v1.PodConditionType) (int, *v1.PodCondition) {
if status == nil {
return -1, nil
}
return GetPodConditionFromList(status.Conditions, conditionType)
}
// GetPodConditionFromList extracts the provided condition from the given list of condition and
// returns the index of the condition and the condition. Returns -1 and nil if the condition is not present.
func GetPodConditionFromList(conditions []v1.PodCondition, conditionType v1.PodConditionType) (int, *v1.PodCondition) {
if conditions == nil {
return -1, nil
}
for i := range conditions {
if conditions[i].Type == conditionType {
return i, &conditions[i]
}
}
return -1, nil
}
// UpdatePodCondition updates existing pod condition or creates a new one. Sets LastTransitionTime to now if the
// status has changed.
// Returns true if pod condition has changed or has been added.
func UpdatePodCondition(status *v1.PodStatus, condition *v1.PodCondition) bool {
condition.LastTransitionTime = metav1.Now()
// Try to find this pod condition.
conditionIndex, oldCondition := GetPodCondition(status, condition.Type)
if oldCondition == nil {
// We are adding new pod condition.
status.Conditions = append(status.Conditions, *condition)
return true
}
// We are updating an existing condition, so we need to check if it has changed.
if condition.Status == oldCondition.Status {
condition.LastTransitionTime = oldCondition.LastTransitionTime
}
isEqual := condition.Status == oldCondition.Status &&
condition.Reason == oldCondition.Reason &&
condition.Message == oldCondition.Message &&
condition.LastProbeTime.Equal(&oldCondition.LastProbeTime) &&
condition.LastTransitionTime.Equal(&oldCondition.LastTransitionTime)
status.Conditions[conditionIndex] = *condition
// Return true if one of the fields have changed.
return !isEqual
}
// GetPodPriority returns priority of the given pod.
func GetPodPriority(pod *v1.Pod) int32 {
if pod.Spec.Priority != nil {
return *pod.Spec.Priority
}
// When priority of a running pod is nil, it means it was created at a time
// that there was no global default priority class and the priority class
// name of the pod was empty. So, we resolve to the static default priority.
return 0
}

51
kube/pkg/apis/core/helper/helpers.go
View File

@@ -1,51 +0,0 @@
/*
Copyright 2021 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package helper
import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/conversion"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
)
// Semantic can do semantic deep equality checks for core objects.
// Example: apiequality.Semantic.DeepEqual(aPod, aPodWithNonNilButEmptyMaps) == true
var Semantic = conversion.EqualitiesOrDie(
func(a, b resource.Quantity) bool {
// Ignore formatting, only care that numeric value stayed the same.
// TODO: if we decide it's important, it should be safe to start comparing the format.
//
// Uninitialized quantities are equivalent to 0 quantities.
return a.Cmp(b) == 0
},
func(a, b metav1.MicroTime) bool {
return a.UTC() == b.UTC()
},
func(a, b metav1.Time) bool {
return a.UTC() == b.UTC()
},
func(a, b labels.Selector) bool {
return a.String() == b.String()
},
func(a, b fields.Selector) bool {
return a.String() == b.String()
},
)

403
kube/pkg/apis/core/v1/helper/helpers.go
View File

@@ -17,18 +17,13 @@ limitations under the License.
package helper
import (
"encoding/json"
"fmt"
"strings"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/apimachinery/pkg/util/validation"
"kubesphere.io/kubesphere/kube/pkg/apis/core/helper"
)
// IsExtendedResourceName returns true if:
@@ -62,404 +57,6 @@ func IsNativeResource(name v1.ResourceName) bool {
IsPrefixedNativeResource(name)
}
// IsHugePageResourceName returns true if the resource name has the huge page
// resource prefix.
func IsHugePageResourceName(name v1.ResourceName) bool {
return strings.HasPrefix(string(name), v1.ResourceHugePagesPrefix)
}
// HugePageResourceName returns a ResourceName with the canonical hugepage
// prefix prepended for the specified page size. The page size is converted
// to its canonical representation.
func HugePageResourceName(pageSize resource.Quantity) v1.ResourceName {
return v1.ResourceName(fmt.Sprintf("%s%s", v1.ResourceHugePagesPrefix, pageSize.String()))
}
// HugePageSizeFromResourceName returns the page size for the specified huge page
// resource name. If the specified input is not a valid huge page resource name
// an error is returned.
func HugePageSizeFromResourceName(name v1.ResourceName) (resource.Quantity, error) {
if !IsHugePageResourceName(name) {
return resource.Quantity{}, fmt.Errorf("resource name: %s is an invalid hugepage name", name)
}
pageSize := strings.TrimPrefix(string(name), v1.ResourceHugePagesPrefix)
return resource.ParseQuantity(pageSize)
}
// IsOvercommitAllowed returns true if the resource is in the default
// namespace and is not hugepages.
func IsOvercommitAllowed(name v1.ResourceName) bool {
return IsNativeResource(name) &&
!IsHugePageResourceName(name)
}
func IsAttachableVolumeResourceName(name v1.ResourceName) bool {
return strings.HasPrefix(string(name), v1.ResourceAttachableVolumesPrefix)
}
// Extended and Hugepages resources
func IsScalarResourceName(name v1.ResourceName) bool {
return IsExtendedResourceName(name) || IsHugePageResourceName(name) ||
IsPrefixedNativeResource(name) || IsAttachableVolumeResourceName(name)
}
// this function aims to check if the service's ClusterIP is set or not
// the objective is not to perform validation here
func IsServiceIPSet(service *v1.Service) bool {
return service.Spec.ClusterIP != v1.ClusterIPNone && service.Spec.ClusterIP != ""
}
// TODO: make method on LoadBalancerStatus?
func LoadBalancerStatusEqual(l, r *v1.LoadBalancerStatus) bool {
return ingressSliceEqual(l.Ingress, r.Ingress)
}
func ingressSliceEqual(lhs, rhs []v1.LoadBalancerIngress) bool {
if len(lhs) != len(rhs) {
return false
}
for i := range lhs {
if !ingressEqual(&lhs[i], &rhs[i]) {
return false
}
}
return true
}
func ingressEqual(lhs, rhs *v1.LoadBalancerIngress) bool {
if lhs.IP != rhs.IP {
return false
}
if lhs.Hostname != rhs.Hostname {
return false
}
return true
}
// GetAccessModesAsString returns a string representation of an array of access modes.
// modes, when present, are always in the same order: RWO,ROX,RWX.
func GetAccessModesAsString(modes []v1.PersistentVolumeAccessMode) string {
modes = removeDuplicateAccessModes(modes)
modesStr := []string{}
if containsAccessMode(modes, v1.ReadWriteOnce) {
modesStr = append(modesStr, "RWO")
}
if containsAccessMode(modes, v1.ReadOnlyMany) {
modesStr = append(modesStr, "ROX")
}
if containsAccessMode(modes, v1.ReadWriteMany) {
modesStr = append(modesStr, "RWX")
}
return strings.Join(modesStr, ",")
}
// GetAccessModesAsString returns an array of AccessModes from a string created by GetAccessModesAsString
func GetAccessModesFromString(modes string) []v1.PersistentVolumeAccessMode {
strmodes := strings.Split(modes, ",")
accessModes := []v1.PersistentVolumeAccessMode{}
for _, s := range strmodes {
s = strings.Trim(s, " ")
switch {
case s == "RWO":
accessModes = append(accessModes, v1.ReadWriteOnce)
case s == "ROX":
accessModes = append(accessModes, v1.ReadOnlyMany)
case s == "RWX":
accessModes = append(accessModes, v1.ReadWriteMany)
}
}
return accessModes
}
// removeDuplicateAccessModes returns an array of access modes without any duplicates
func removeDuplicateAccessModes(modes []v1.PersistentVolumeAccessMode) []v1.PersistentVolumeAccessMode {
accessModes := []v1.PersistentVolumeAccessMode{}
for _, m := range modes {
if !containsAccessMode(accessModes, m) {
accessModes = append(accessModes, m)
}
}
return accessModes
}
func containsAccessMode(modes []v1.PersistentVolumeAccessMode, mode v1.PersistentVolumeAccessMode) bool {
for _, m := range modes {
if m == mode {
return true
}
}
return false
}
// NodeSelectorRequirementsAsSelector converts the []NodeSelectorRequirement api type into a struct that implements
// labels.Selector.
func NodeSelectorRequirementsAsSelector(nsm []v1.NodeSelectorRequirement) (labels.Selector, error) {
if len(nsm) == 0 {
return labels.Nothing(), nil
}
selector := labels.NewSelector()
for _, expr := range nsm {
var op selection.Operator
switch expr.Operator {
case v1.NodeSelectorOpIn:
op = selection.In
case v1.NodeSelectorOpNotIn:
op = selection.NotIn
case v1.NodeSelectorOpExists:
op = selection.Exists
case v1.NodeSelectorOpDoesNotExist:
op = selection.DoesNotExist
case v1.NodeSelectorOpGt:
op = selection.GreaterThan
case v1.NodeSelectorOpLt:
op = selection.LessThan
default:
return nil, fmt.Errorf("%q is not a valid node selector operator", expr.Operator)
}
r, err := labels.NewRequirement(expr.Key, op, expr.Values)
if err != nil {
return nil, err
}
selector = selector.Add(*r)
}
return selector, nil
}
// NodeSelectorRequirementsAsFieldSelector converts the []NodeSelectorRequirement core type into a struct that implements
// fields.Selector.
func NodeSelectorRequirementsAsFieldSelector(nsm []v1.NodeSelectorRequirement) (fields.Selector, error) {
if len(nsm) == 0 {
return fields.Nothing(), nil
}
selectors := []fields.Selector{}
for _, expr := range nsm {
switch expr.Operator {
case v1.NodeSelectorOpIn:
if len(expr.Values) != 1 {
return nil, fmt.Errorf("unexpected number of value (%d) for node field selector operator %q",
len(expr.Values), expr.Operator)
}
selectors = append(selectors, fields.OneTermEqualSelector(expr.Key, expr.Values[0]))
case v1.NodeSelectorOpNotIn:
if len(expr.Values) != 1 {
return nil, fmt.Errorf("unexpected number of value (%d) for node field selector operator %q",
len(expr.Values), expr.Operator)
}
selectors = append(selectors, fields.OneTermNotEqualSelector(expr.Key, expr.Values[0]))
default:
return nil, fmt.Errorf("%q is not a valid node field selector operator", expr.Operator)
}
}
return fields.AndSelectors(selectors...), nil
}
// NodeSelectorRequirementKeysExistInNodeSelectorTerms checks if a NodeSelectorTerm with key is already specified in terms
func NodeSelectorRequirementKeysExistInNodeSelectorTerms(reqs []v1.NodeSelectorRequirement, terms []v1.NodeSelectorTerm) bool {
for _, req := range reqs {
for _, term := range terms {
for _, r := range term.MatchExpressions {
if r.Key == req.Key {
return true
}
}
}
}
return false
}
// MatchNodeSelectorTerms checks whether the node labels and fields match node selector terms in ORed;
// nil or empty term matches no objects.
func MatchNodeSelectorTerms(
nodeSelectorTerms []v1.NodeSelectorTerm,
nodeLabels labels.Set,
nodeFields fields.Set,
) bool {
for _, req := range nodeSelectorTerms {
// nil or empty term selects no objects
if len(req.MatchExpressions) == 0 && len(req.MatchFields) == 0 {
continue
}
if len(req.MatchExpressions) != 0 {
labelSelector, err := NodeSelectorRequirementsAsSelector(req.MatchExpressions)
if err != nil || !labelSelector.Matches(nodeLabels) {
continue
}
}
if len(req.MatchFields) != 0 {
fieldSelector, err := NodeSelectorRequirementsAsFieldSelector(req.MatchFields)
if err != nil || !fieldSelector.Matches(nodeFields) {
continue
}
}
return true
}
return false
}
// TopologySelectorRequirementsAsSelector converts the []TopologySelectorLabelRequirement api type into a struct
// that implements labels.Selector.
func TopologySelectorRequirementsAsSelector(tsm []v1.TopologySelectorLabelRequirement) (labels.Selector, error) {
if len(tsm) == 0 {
return labels.Nothing(), nil
}
selector := labels.NewSelector()
for _, expr := range tsm {
r, err := labels.NewRequirement(expr.Key, selection.In, expr.Values)
if err != nil {
return nil, err
}
selector = selector.Add(*r)
}
return selector, nil
}
// MatchTopologySelectorTerms checks whether given labels match topology selector terms in ORed;
// nil or empty term matches no objects; while empty term list matches all objects.
func MatchTopologySelectorTerms(topologySelectorTerms []v1.TopologySelectorTerm, lbls labels.Set) bool {
if len(topologySelectorTerms) == 0 {
// empty term list matches all objects
return true
}
for _, req := range topologySelectorTerms {
// nil or empty term selects no objects
if len(req.MatchLabelExpressions) == 0 {
continue
}
labelSelector, err := TopologySelectorRequirementsAsSelector(req.MatchLabelExpressions)
if err != nil || !labelSelector.Matches(lbls) {
continue
}
return true
}
return false
}
// AddOrUpdateTolerationInPodSpec tries to add a toleration to the toleration list in PodSpec.
// Returns true if something was updated, false otherwise.
func AddOrUpdateTolerationInPodSpec(spec *v1.PodSpec, toleration *v1.Toleration) bool {
podTolerations := spec.Tolerations
var newTolerations []v1.Toleration
updated := false
for i := range podTolerations {
if toleration.MatchToleration(&podTolerations[i]) {
if helper.Semantic.DeepEqual(toleration, podTolerations[i]) {
return false
}
newTolerations = append(newTolerations, *toleration)
updated = true
continue
}
newTolerations = append(newTolerations, podTolerations[i])
}
if !updated {
newTolerations = append(newTolerations, *toleration)
}
spec.Tolerations = newTolerations
return true
}
// AddOrUpdateTolerationInPod tries to add a toleration to the pod's toleration list.
// Returns true if something was updated, false otherwise.
func AddOrUpdateTolerationInPod(pod *v1.Pod, toleration *v1.Toleration) bool {
return AddOrUpdateTolerationInPodSpec(&pod.Spec, toleration)
}
// TolerationsTolerateTaint checks if taint is tolerated by any of the tolerations.
func TolerationsTolerateTaint(tolerations []v1.Toleration, taint *v1.Taint) bool {
for i := range tolerations {
if tolerations[i].ToleratesTaint(taint) {
return true
}
}
return false
}
type taintsFilterFunc func(*v1.Taint) bool
// TolerationsTolerateTaintsWithFilter checks if given tolerations tolerates
// all the taints that apply to the filter in given taint list.
func TolerationsTolerateTaintsWithFilter(tolerations []v1.Toleration, taints []v1.Taint, applyFilter taintsFilterFunc) bool {
if len(taints) == 0 {
return true
}
for i := range taints {
if applyFilter != nil && !applyFilter(&taints[i]) {
continue
}
if !TolerationsTolerateTaint(tolerations, &taints[i]) {
return false
}
}
return true
}
// Returns true and list of Tolerations matching all Taints if all are tolerated, or false otherwise.
func GetMatchingTolerations(taints []v1.Taint, tolerations []v1.Toleration) (bool, []v1.Toleration) {
if len(taints) == 0 {
return true, []v1.Toleration{}
}
if len(tolerations) == 0 && len(taints) > 0 {
return false, []v1.Toleration{}
}
result := []v1.Toleration{}
for i := range taints {
tolerated := false
for j := range tolerations {
if tolerations[j].ToleratesTaint(&taints[i]) {
result = append(result, tolerations[j])
tolerated = true
break
}
}
if !tolerated {
return false, []v1.Toleration{}
}
}
return true, result
}
func GetAvoidPodsFromNodeAnnotations(annotations map[string]string) (v1.AvoidPods, error) {
var avoidPods v1.AvoidPods
if len(annotations) > 0 && annotations[v1.PreferAvoidPodsAnnotationKey] != "" {
err := json.Unmarshal([]byte(annotations[v1.PreferAvoidPodsAnnotationKey]), &avoidPods)
if err != nil {
return avoidPods, err
}
}
return avoidPods, nil
}
// GetPersistentVolumeClass returns StorageClassName.
func GetPersistentVolumeClass(volume *v1.PersistentVolume) string {
// Use beta annotation first
if class, found := volume.Annotations[v1.BetaStorageClassAnnotation]; found {
return class
}
return volume.Spec.StorageClassName
}
// GetPersistentVolumeClaimClass returns StorageClassName. If no storage class was
// requested, it returns "".
func GetPersistentVolumeClaimClass(claim *v1.PersistentVolumeClaim) string {

21
kube/pkg/apis/core/v1/helper/qos/qos.go
View File

@@ -1,21 +1,3 @@
/*
Copyright 2021 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package qos
import (
@@ -26,9 +8,6 @@ import (
var supportedQoSComputeResources = sets.New(string(corev1.ResourceCPU), string(corev1.ResourceMemory))
// QOSList is a set of (resource name, QoS class) pairs.
type QOSList map[corev1.ResourceName]corev1.PodQOSClass
func isSupportedQoSComputeResource(name corev1.ResourceName) bool {
return supportedQoSComputeResources.Has(string(name))
}

653
kube/pkg/features/features.go
View File

@@ -21,660 +21,9 @@ import (
)
const (
// Every feature gate should add method here following this template:
//
// // owner: @username
// // alpha: v1.X
// MyFeature featuregate.Feature = "MyFeature"
// owner: @tallclair
// beta: v1.4
AppArmor featuregate.Feature = "AppArmor"
// owner: @mtaufen
// alpha: v1.4
// beta: v1.11
DynamicKubeletConfig featuregate.Feature = "DynamicKubeletConfig"
// owner: @pweil-
// alpha: v1.5
//
// Default userns=host for containers that are using other host namespaces, host mounts, the pod
// contains a privileged container, or specific non-namespaced capabilities (MKNOD, SYS_MODULE,
// SYS_TIME). This should only be enabled if user namespace remapping is enabled in the docker daemon.
ExperimentalHostUserNamespaceDefaultingGate featuregate.Feature = "ExperimentalHostUserNamespaceDefaulting"
// owner: @jiayingz
// beta: v1.10
//
// Enables support for Device Plugins
DevicePlugins featuregate.Feature = "DevicePlugins"
// owner: @dxist
// alpha: v1.16
//
// Enables support of HPA scaling to zero pods when an object or custom metric is configured.
HPAScaleToZero featuregate.Feature = "HPAScaleToZero"
// owner: @mikedanese
// alpha: v1.7
// beta: v1.12
//
// Gets a server certificate for the kubelet from the Certificate Signing
// Request API instead of generating one self signed and auto rotates the
// certificate as expiration approaches.
RotateKubeletServerCertificate featuregate.Feature = "RotateKubeletServerCertificate"
// owner: @jinxu
// beta: v1.10
//
// New local storage types to support local storage capacity isolation
LocalStorageCapacityIsolation featuregate.Feature = "LocalStorageCapacityIsolation"
// owner: @gnufied
// beta: v1.11
// Ability to Expand persistent volumes
ExpandPersistentVolumes featuregate.Feature = "ExpandPersistentVolumes"
// owner: @mlmhl
// beta: v1.15
// Ability to expand persistent volumes' file system without unmounting volumes.
ExpandInUsePersistentVolumes featuregate.Feature = "ExpandInUsePersistentVolumes"
// owner: @gnufied
// alpha: v1.14
// beta: v1.16
// Ability to expand CSI volumes
ExpandCSIVolumes featuregate.Feature = "ExpandCSIVolumes"
// owner: @verb
// alpha: v1.16
//
// Allows running an ephemeral container in pod namespaces to troubleshoot a running pod.
EphemeralContainers featuregate.Feature = "EphemeralContainers"
// owner: @sjenning
// alpha: v1.11
//
// Allows resource reservations at the QoS level preventing pods at lower QoS levels from
// bursting into resources requested at higher QoS levels (memory only for now)
QOSReserved featuregate.Feature = "QOSReserved"
// owner: @ConnorDoyle
// alpha: v1.8
// beta: v1.10
//
// Alternative container-level CPU affinity policies.
CPUManager featuregate.Feature = "CPUManager"
// owner: @szuecs
// alpha: v1.12
//
// Enable nodes to change CPUCFSQuotaPeriod
CPUCFSQuotaPeriod featuregate.Feature = "CustomCPUCFSQuotaPeriod"
// owner: @lmdaly
// alpha: v1.16
// beta: v1.18
//
// Enable resource managers to make NUMA aligned decisions
TopologyManager featuregate.Feature = "TopologyManager"
// owner: @sjenning
// beta: v1.11
//
// Enable pods to set sysctls on a pod
Sysctls featuregate.Feature = "Sysctls"
// owner @smarterclayton
// alpha: v1.16
// beta: v1.19
// ga: v1.21
//
// Enable legacy behavior to vary cluster functionality on the node-role.kubernetes.io labels. On by default (legacy), will be turned off in 1.18.
// Lock to false in v1.21 and remove in v1.22.
LegacyNodeRoleBehavior featuregate.Feature = "LegacyNodeRoleBehavior"
// owner @brendandburns
// alpha: v1.9
// beta: v1.19
// ga: v1.21
//
// Enable nodes to exclude themselves from service load balancers
ServiceNodeExclusion featuregate.Feature = "ServiceNodeExclusion"
// owner @smarterclayton
// alpha: v1.16
// beta: v1.19
// ga: v1.21
//
// Enable nodes to exclude themselves from network disruption checks
NodeDisruptionExclusion featuregate.Feature = "NodeDisruptionExclusion"
// owner: @saad-ali
// alpha: v1.12
// beta: v1.14
// GA: v1.18
// Enable all logic related to the CSIDriver API object in storage.k8s.io
CSIDriverRegistry featuregate.Feature = "CSIDriverRegistry"
// owner: @screeley44
// alpha: v1.9
// beta: v1.13
// ga: v1.18
//
// Enable Block volume support in containers.
BlockVolume featuregate.Feature = "BlockVolume"
// owner: @pospispa
// GA: v1.11
//
// Postpone deletion of a PV or a PVC when they are being used
StorageObjectInUseProtection featuregate.Feature = "StorageObjectInUseProtection"
// owner: @dims, @derekwaynecarr
// alpha: v1.10
// beta: v1.14
// GA: v1.20
//
// Implement support for limiting pids in pods
SupportPodPidsLimit featuregate.Feature = "SupportPodPidsLimit"
// owner: @mikedanese
// alpha: v1.13
//
// Migrate ServiceAccount volumes to use a projected volume consisting of a
// ServiceAccountTokenVolumeProjection. This feature adds new required flags
// to the API server.
BoundServiceAccountTokenVolume featuregate.Feature = "BoundServiceAccountTokenVolume"
// owner: @mtaufen
// alpha: v1.18
// beta: v1.20
//
// Enable OIDC discovery endpoints (issuer and JWKS URLs) for the service
// account issuer in the API server.
// Note these endpoints serve minimally-compliant discovery docs that are
// intended to be used for service account token verification.
ServiceAccountIssuerDiscovery featuregate.Feature = "ServiceAccountIssuerDiscovery"
// owner: @Random-Liu
// beta: v1.11
//
// Enable container log rotation for cri container runtime
CRIContainerLogRotation featuregate.Feature = "CRIContainerLogRotation"
// owner: @krmayankk
// beta: v1.14
//
// Enables control over the primary group ID of containers' init processes.
RunAsGroup featuregate.Feature = "RunAsGroup"
// owner: @saad-ali
// ga
//
// Allow mounting a subpath of a volume in a container
// Do not remove this feature gate even though it's GA
VolumeSubpath featuregate.Feature = "VolumeSubpath"
// owner: @ravig
// alpha: v1.11
//
// Include volume count on node to be considered for balanced resource allocation while scheduling.
// A node which has closer cpu,memory utilization and volume count is favoured by scheduler
// while making decisions.
BalanceAttachedNodeVolumes featuregate.Feature = "BalanceAttachedNodeVolumes"
// owner: @vladimirvivien
// alpha: v1.11
// beta: v1.14
// ga: v1.18
//
// Enables CSI to use raw block storage volumes
CSIBlockVolume featuregate.Feature = "CSIBlockVolume"
// owner: @pohly
// alpha: v1.14
// beta: v1.16
//
// Enables CSI Inline volumes support for pods
CSIInlineVolume featuregate.Feature = "CSIInlineVolume"
// owner: @pohly
// alpha: v1.19
//
// Enables tracking of available storage capacity that CSI drivers provide.
CSIStorageCapacity featuregate.Feature = "CSIStorageCapacity"
// owner: @alculquicondor
// beta: v1.20
//
// Enables the use of PodTopologySpread scheduling plugin to do default
// spreading and disables legacy SelectorSpread plugin.
DefaultPodTopologySpread featuregate.Feature = "DefaultPodTopologySpread"
// owner: @pohly
// alpha: v1.19
//
// Enables generic ephemeral inline volume support for pods
GenericEphemeralVolume featuregate.Feature = "GenericEphemeralVolume"
// owner: @chendave
// alpha: v1.21
//
// PreferNominatedNode tells scheduler whether the nominated node will be checked first before looping
// all the rest of nodes in the cluster.
// Enabling this feature also implies the preemptor pod might not be dispatched to the best candidate in
// some corner case, e.g. another node releases enough resources after the nominated node has been set
// and hence is the best candidate instead.
PreferNominatedNode featuregate.Feature = "PreferNominatedNode"
// owner: @tallclair
// alpha: v1.12
// beta: v1.14
// GA: v1.20
//
// Enables RuntimeClass, for selecting between multiple runtimes to run a pod.
RuntimeClass featuregate.Feature = "RuntimeClass"
// owner: @mtaufen
// alpha: v1.12
// beta: v1.14
// GA: v1.17
//
// Kubelet uses the new Lease API to report node heartbeats,
// (Kube) Node Lifecycle Controller uses these heartbeats as a node health signal.
NodeLease featuregate.Feature = "NodeLease"
// owner: @janosi
// alpha: v1.12
// beta: v1.18
// GA: v1.20
//
// Enables SCTP as new protocol for Service ports, NetworkPolicy, and ContainerPort in Pod/Containers definition
SCTPSupport featuregate.Feature = "SCTPSupport"
// owner: @xing-yang
// alpha: v1.12
// beta: v1.17
// GA: v1.20
//
// Enable volume snapshot data source support.
VolumeSnapshotDataSource featuregate.Feature = "VolumeSnapshotDataSource"
// owner: @jessfraz
// alpha: v1.12
//
// Enables control over ProcMountType for containers.
ProcMountType featuregate.Feature = "ProcMountType"
// owner: @janetkuo
// alpha: v1.12
//
// Allow TTL controller to clean up Pods and Jobs after they finish.
TTLAfterFinished featuregate.Feature = "TTLAfterFinished"
// owner: @dashpole
// alpha: v1.13
// beta: v1.15
//
// Enables the kubelet's pod resources grpc endpoint
KubeletPodResources featuregate.Feature = "KubeletPodResources"
// owner: @davidz627
// alpha: v1.14
// beta: v1.17
//
// Enables the in-tree storage to CSI Plugin migration feature.
CSIMigration featuregate.Feature = "CSIMigration"
// owner: @davidz627
// alpha: v1.14
// beta: v1.17
//
// Enables the GCE PD in-tree driver to GCE CSI Driver migration feature.
CSIMigrationGCE featuregate.Feature = "CSIMigrationGCE"
// owner: @davidz627
// alpha: v1.17
//
// Disables the GCE PD in-tree driver.
// Expects GCE PD CSI Driver to be installed and configured on all nodes.
CSIMigrationGCEComplete featuregate.Feature = "CSIMigrationGCEComplete"
// owner: @leakingtapan
// alpha: v1.14
// beta: v1.17
//
// Enables the AWS EBS in-tree driver to AWS EBS CSI Driver migration feature.
CSIMigrationAWS featuregate.Feature = "CSIMigrationAWS"
// owner: @leakingtapan
// alpha: v1.17
//
// Disables the AWS EBS in-tree driver.
// Expects AWS EBS CSI Driver to be installed and configured on all nodes.
CSIMigrationAWSComplete featuregate.Feature = "CSIMigrationAWSComplete"
// owner: @andyzhangx
// alpha: v1.15
// beta: v1.19
//
// Enables the Azure Disk in-tree driver to Azure Disk Driver migration feature.
CSIMigrationAzureDisk featuregate.Feature = "CSIMigrationAzureDisk"
// owner: @andyzhangx
// alpha: v1.17
//
// Disables the Azure Disk in-tree driver.
// Expects Azure Disk CSI Driver to be installed and configured on all nodes.
CSIMigrationAzureDiskComplete featuregate.Feature = "CSIMigrationAzureDiskComplete"
// owner: @andyzhangx
// alpha: v1.15
//
// Enables the Azure File in-tree driver to Azure File Driver migration feature.
CSIMigrationAzureFile featuregate.Feature = "CSIMigrationAzureFile"
// owner: @andyzhangx
// alpha: v1.17
//
// Disables the Azure File in-tree driver.
// Expects Azure File CSI Driver to be installed and configured on all nodes.
CSIMigrationAzureFileComplete featuregate.Feature = "CSIMigrationAzureFileComplete"
// owner: @divyenpatel
// beta: v1.19 (requires: vSphere vCenter/ESXi Version: 7.0u1, HW Version: VM version 15)
//
// Enables the vSphere in-tree driver to vSphere CSI Driver migration feature.
CSIMigrationvSphere featuregate.Feature = "CSIMigrationvSphere"
// owner: @divyenpatel
// beta: v1.19 (requires: vSphere vCenter/ESXi Version: 7.0u1, HW Version: VM version 15)
//
// Disables the vSphere in-tree driver.
// Expects vSphere CSI Driver to be installed and configured on all nodes.
CSIMigrationvSphereComplete featuregate.Feature = "CSIMigrationvSphereComplete"
// owner: @huffmanca
// alpha: v1.19
// beta: v1.20
//
// Determines if a CSI Driver supports applying fsGroup.
CSIVolumeFSGroupPolicy featuregate.Feature = "CSIVolumeFSGroupPolicy"
// owner: @gnufied
// alpha: v1.18
// beta: v1.20
// Allows user to configure volume permission change policy for fsGroups when mounting
// a volume in a Pod.
ConfigurableFSGroupPolicy featuregate.Feature = "ConfigurableFSGroupPolicy"
// owner: @RobertKrawitz, @derekwaynecarr
// beta: v1.15
// GA: v1.20
//
// Implement support for limiting pids in nodes
SupportNodePidsLimit featuregate.Feature = "SupportNodePidsLimit"
// owner: @wk8
// alpha: v1.14
// beta: v1.16
//
// Enables GMSA support for Windows workloads.
WindowsGMSA featuregate.Feature = "WindowsGMSA"
// owner: @bclau
// alpha: v1.16
// beta: v1.17
// GA: v1.18
//
// Enables support for running container entrypoints as different usernames than their default ones.
WindowsRunAsUserName featuregate.Feature = "WindowsRunAsUserName"
// owner: @adisky
// alpha: v1.14
// beta: v1.18
//
// Enables the OpenStack Cinder in-tree driver to OpenStack Cinder CSI Driver migration feature.
CSIMigrationOpenStack featuregate.Feature = "CSIMigrationOpenStack"
// owner: @adisky
// alpha: v1.17
//
// Disables the OpenStack Cinder in-tree driver.
// Expects the OpenStack Cinder CSI Driver to be installed and configured on all nodes.
CSIMigrationOpenStackComplete featuregate.Feature = "CSIMigrationOpenStackComplete"
// owner: @RobertKrawitz
// alpha: v1.15
//
// Allow use of filesystems for ephemeral storage monitoring.
// Only applies if LocalStorageCapacityIsolation is set.
LocalStorageCapacityIsolationFSQuotaMonitoring featuregate.Feature = "LocalStorageCapacityIsolationFSQuotaMonitoring"
// owner: @denkensk
// alpha: v1.15
// beta: v1.19
//
// Enables NonPreempting option for priorityClass and pod.
NonPreemptingPriority featuregate.Feature = "NonPreemptingPriority"
// owner: @egernst
// alpha: v1.16
// beta: v1.18
//
// Enables PodOverhead, for accounting pod overheads which are specific to a given RuntimeClass
PodOverhead featuregate.Feature = "PodOverhead"
// owner: @khenidak
// alpha: v1.15
//
// Enables ipv6 dual stack
IPv6DualStack featuregate.Feature = "IPv6DualStack"
// owner: @robscott @freehan
// alpha: v1.16
//
// Enable Endpoint Slices for more scalable Service endpoints.
EndpointSlice featuregate.Feature = "EndpointSlice"
// owner: @robscott @freehan
// alpha: v1.18
// beta: v1.19
//
// Enable Endpoint Slice consumption by kube-proxy for improved scalability.
EndpointSliceProxying featuregate.Feature = "EndpointSliceProxying"
// owner: @robscott @kumarvin123
// alpha: v1.19
//
// Enable Endpoint Slice consumption by kube-proxy in Windows for improved scalability.
WindowsEndpointSliceProxying featuregate.Feature = "WindowsEndpointSliceProxying"
// owner: @matthyx
// alpha: v1.16
// beta: v1.18
// GA: v1.20
//
// Enables the startupProbe in kubelet worker.
StartupProbe featuregate.Feature = "StartupProbe"
// owner: @deads2k
// beta: v1.17
//
// Enables the users to skip TLS verification of kubelets on pod logs requests
AllowInsecureBackendProxy featuregate.Feature = "AllowInsecureBackendProxy"
// owner: @mortent
// alpha: v1.3
// beta: v1.5
//
// Enable all logic related to the PodDisruptionBudget API object in policy
PodDisruptionBudget featuregate.Feature = "PodDisruptionBudget"
// owner: @alaypatel07, @soltysh
// alpha: v1.20
// beta: v1.21
//
// CronJobControllerV2 controls whether the controller manager starts old cronjob
// controller or new one which is implemented with informers and delaying queue
//
// This feature is deprecated, and will be removed in v1.22.
CronJobControllerV2 featuregate.Feature = "CronJobControllerV2"
// owner: @smarterclayton
// alpha: v1.21
//
// DaemonSets allow workloads to maintain availability during update per node
DaemonSetUpdateSurge featuregate.Feature = "DaemonSetUpdateSurge"
// owner: @m1093782566
// alpha: v1.17
//
// Enables topology aware service routing
ServiceTopology featuregate.Feature = "ServiceTopology"
// owner: @robscott
// alpha: v1.18
// beta: v1.19
// ga: v1.20
//
// Enables AppProtocol field for Services and Endpoints.
ServiceAppProtocol featuregate.Feature = "ServiceAppProtocol"
// owner: @wojtek-t
// alpha: v1.18
// beta: v1.19
// ga: v1.21
//
// Enables a feature to make secrets and configmaps data immutable.
ImmutableEphemeralVolumes featuregate.Feature = "ImmutableEphemeralVolumes"
// owner: @bart0sh
// alpha: v1.18
// beta: v1.19
//
// Enables usage of HugePages-<size> in a volume medium,
// e.g. emptyDir:
// medium: HugePages-1Gi
HugePageStorageMediumSize featuregate.Feature = "HugePageStorageMediumSize"
// owner: @derekwaynecarr
// alpha: v1.20
//
// Enables usage of hugepages-<size> in downward API.
DownwardAPIHugePages featuregate.Feature = "DownwardAPIHugePages"
// owner: @freehan
// GA: v1.18
//
// Enable ExternalTrafficPolicy for Service ExternalIPs.
// This is for bug fix #69811
ExternalPolicyForExternalIP featuregate.Feature = "ExternalPolicyForExternalIP"
// owner: @bswartz
// alpha: v1.18
//
// Enables usage of any object for volume data source in PVCs
AnyVolumeDataSource featuregate.Feature = "AnyVolumeDataSource"
// owner: @javidiaz
// alpha: v1.19
// beta: v1.20
//
// Allow setting the Fully Qualified Domain Name (FQDN) in the hostname of a Pod. If a Pod does not
// have FQDN, this feature has no effect.
SetHostnameAsFQDN featuregate.Feature = "SetHostnameAsFQDN"
// owner: @ksubrmnn
// alpha: v1.14
// beta: v1.20
//
// Allows kube-proxy to run in Overlay mode for Windows
WinOverlay featuregate.Feature = "WinOverlay"
// owner: @ksubrmnn
// alpha: v1.14
//
// Allows kube-proxy to create DSR loadbalancers for Windows
WinDSR featuregate.Feature = "WinDSR"
// owner: @RenaudWasTaken @dashpole
// alpha: v1.19
// beta: v1.20
//
// Disables Accelerator Metrics Collected by Kubelet
DisableAcceleratorUsageMetrics featuregate.Feature = "DisableAcceleratorUsageMetrics"
// owner: @arjunrn @mwielgus @josephburnett
// alpha: v1.20
//
// Add support for the HPA to scale based on metrics from individual containers
// in target pods
HPAContainerMetrics featuregate.Feature = "HPAContainerMetrics"
// owner: @zshihang
// alpha: v1.13
// beta: v1.20
//
// Allows kube-controller-manager to publish kube-root-ca.crt configmap to
// every namespace. This feature is a prerequisite of BoundServiceAccountTokenVolume.
RootCAConfigMap featuregate.Feature = "RootCAConfigMap"
// owner: @andrewsykim
// alpha: v1.20
//
// Enable Terminating condition in Endpoint Slices.
EndpointSliceTerminatingCondition featuregate.Feature = "EndpointSliceTerminatingCondition"
// owner: @robscott
// alpha: v1.20
//
// Enable NodeName field on Endpoint Slices.
EndpointSliceNodeName featuregate.Feature = "EndpointSliceNodeName"
// owner: @derekwaynecarr
// alpha: v1.20
//
// Enables kubelet support to size memory backed volumes
SizeMemoryBackedVolumes featuregate.Feature = "SizeMemoryBackedVolumes"
// owner: @andrewsykim @SergeyKanzhelev
// GA: v1.20
//
// Ensure kubelet respects exec probe timeouts. Feature gate exists in-case existing workloads
// may depend on old behavior where exec probe timeouts were ignored.
// Lock to default in v1.21 and remove in v1.22.
ExecProbeTimeout featuregate.Feature = "ExecProbeTimeout"
// owner: @andrewsykim
// alpha: v1.20
//
// Enable kubelet exec plugins for image pull credentials.
KubeletCredentialProviders featuregate.Feature = "KubeletCredentialProviders"
// owner: @zshihang
// alpha: v1.20
//
// Enable kubelet to pass pod's service account token to NodePublishVolume
// call of CSI driver which is mounting volumes for that pod.
CSIServiceAccountToken featuregate.Feature = "CSIServiceAccountToken"
// owner: @bobbypage
// alpha: v1.20
// Adds support for kubelet to detect node shutdown and gracefully terminate pods prior to the node being shutdown.
GracefulNodeShutdown featuregate.Feature = "GracefulNodeShutdown"
// owner: @andrewsykim @uablrek
// alpha: v1.20
//
// Allows control if NodePorts shall be created for services with "type: LoadBalancer" by defining the spec.AllocateLoadBalancerNodePorts field (bool)
ServiceLBNodePortControl featuregate.Feature = "ServiceLBNodePortControl"
// owner: @janosi
// alpha: v1.20
//
// Enables the usage of different protocols in the same Service with type=LoadBalancer
MixedProtocolLBService featuregate.Feature = "MixedProtocolLBService"
)

193
kube/pkg/openapi/apiservice.go Normal file
View File

@@ -0,0 +1,193 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package openapi
import (
"context"
"encoding/base64"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/httpstream"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/proxy"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
restclient "k8s.io/client-go/rest"
"k8s.io/klog/v2"
extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
)
type certKeyFunc func() ([]byte, []byte)
const (
aggregatedDiscoveryTimeout = 5 * time.Second
)
type ApiService interface {
Name() string
ResolveEndpoint() (*url.URL, error)
UpdateAPIService() error
ServeHTTP(w http.ResponseWriter, req *http.Request)
}
type defaultApiService struct {
apiService *extensionsv1alpha1.APIService
proxyTransport *http.Transport
restConfig *restclient.Config
proxyRoundTripper http.RoundTripper
proxyCurrentCertKeyContent certKeyFunc
}
func NewApiService(apiService *extensionsv1alpha1.APIService) ApiService {
return &defaultApiService{
apiService: apiService,
proxyCurrentCertKeyContent: func() (bytes []byte, bytes2 []byte) { return nil, nil },
}
}
func (d *defaultApiService) Name() string {
return d.apiService.Name
}
func (d *defaultApiService) ResolveEndpoint() (*url.URL, error) {
if d.apiService.Spec.Service != nil &&
d.apiService.Spec.Service.Name != "" &&
d.apiService.Spec.Service.Namespace != "" &&
*d.apiService.Spec.Service.Port != 0 {
return &url.URL{Scheme: "https", Host: fmt.Sprintf("%s.%s.svc:%d",
d.apiService.Spec.Service.Name, d.apiService.Spec.Service.Namespace, d.apiService.Spec.Service.Port)}, nil
}
if d.apiService.Spec.URL != nil && *d.apiService.Spec.URL != "" {
u, err := url.Parse(*d.apiService.Spec.URL)
if err != nil {
return nil, err
}
if d.apiService.Spec.InsecureSkipVerify {
u.Scheme = "http"
}
return u, nil
}
return nil, fmt.Errorf("cannot resolve an apiservice %s", d.Name())
}
func (d *defaultApiService) UpdateAPIService() error {
proxyClientCert, proxyClientKey := d.proxyCurrentCertKeyContent()
tlsConfig := restclient.TLSClientConfig{
Insecure: d.apiService.Spec.InsecureSkipVerify,
}
if !d.apiService.Spec.InsecureSkipVerify && len(d.apiService.Spec.CABundle) > 0 {
caData, err := base64.StdEncoding.DecodeString(string(d.apiService.Spec.CABundle))
if err != nil {
klog.Warning(err.Error())
return err
}
tlsConfig.ServerName = d.apiService.Spec.Service.Name + "." + d.apiService.Spec.Service.Namespace + ".svc"
tlsConfig.CertData = proxyClientCert
tlsConfig.KeyData = proxyClientKey
tlsConfig.CAData = caData
}
d.restConfig = &restclient.Config{
TLSClientConfig: tlsConfig,
}
if d.proxyTransport != nil && d.proxyTransport.DialContext != nil {
d.restConfig.Dial = d.proxyTransport.DialContext
}
proxyRoundTripper, err := restclient.TransportFor(d.restConfig)
if err != nil {
klog.Warning(err.Error())
return err
}
d.proxyRoundTripper = proxyRoundTripper
return nil
}
func (d *defaultApiService) ServeHTTP(w http.ResponseWriter, req *http.Request) {
//user, ok := genericapirequest.UserFrom(req.Context())
//if !ok {
// proxyError(w, req, "missing user", http.StatusInternalServerError)
// return
//}
// write a new location based on the existing request pointed at the target service
location, err := d.ResolveEndpoint()
if err != nil {
klog.Errorf("error resolving %s: %v", d.Name(), err)
proxyError(w, req, "service unavailable", http.StatusServiceUnavailable)
}
location.Path = req.URL.Path
location.RawQuery = req.URL.Query().Encode()
newReq, cancelFn := newRequestForProxy(location, req)
defer cancelFn()
if d.proxyRoundTripper == nil {
proxyError(w, req, "", http.StatusNotFound)
return
}
proxyRoundTripper := d.proxyRoundTripper
upgrade := httpstream.IsUpgradeRequest(req)
//proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), proxyRoundTripper)
//if upgrade {
//transport.SetAuthProxyHeaders(newReq, user.GetName(), user.GetGroups(), user.GetExtra())
//}
handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, upgrade, &responder{w: w})
handler.ServeHTTP(w, newReq)
}
type responder struct {
w http.ResponseWriter
}
func (r *responder) Object(statusCode int, obj runtime.Object) {
responsewriters.WriteRawJSON(statusCode, obj, r.w)
}
func (r *responder) Error(_ http.ResponseWriter, _ *http.Request, err error) {
http.Error(r.w, err.Error(), http.StatusServiceUnavailable)
}
func proxyError(w http.ResponseWriter, req *http.Request, error string, code int) {
http.Error(w, error, code)
}
// newRequestForProxy returns a shallow copy of the original request with a context that may include a timeout for discovery requests
func newRequestForProxy(location *url.URL, req *http.Request) (*http.Request, context.CancelFunc) {
newCtx := req.Context()
cancelFn := func() {}
if requestInfo, ok := genericapirequest.RequestInfoFrom(req.Context()); ok {
// trim leading and trailing slashes. Then "/apis/group/version" requests are for discovery, so if we have exactly three
// segments that we are going to proxy, we have a discovery request.
if !requestInfo.IsResourceRequest && len(strings.Split(strings.Trim(requestInfo.Path, "/"), "/")) == 3 {
// discovery requests are used by kubectl and others to determine which resources a server has. This is a cheap call that
// should be fast for every aggregated apiserver. Latency for aggregation is expected to be low (as for all extensions)
// so forcing a short timeout here helps responsiveness of all clients.
newCtx, cancelFn = context.WithTimeout(newCtx, aggregatedDiscoveryTimeout)
}
}
// WithContext creates a shallow clone of the request with the same context.
newReq := req.WithContext(newCtx)
newReq.Header = utilnet.CloneHeader(req.Header)
newReq.URL = location
newReq.Host = location.Host
return newReq, cancelFn
}

129
kube/pkg/openapi/downloader.go Normal file
View File

@@ -0,0 +1,129 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package openapi
import (
"fmt"
"net/http"
"time"
)
const (
specDownloadTimeout = time.Minute
)
type CacheableDownloader interface {
Name() string
GetV2() ([]byte, error)
GetV3() ([]byte, error)
UpdateDownloader(apiService ApiService) error
}
type Downloader struct{}
func NewDownloader() *Downloader {
return &Downloader{}
}
func (s *Downloader) Download(handler http.Handler, req *http.Request) (data []byte, err error) {
handler = http.TimeoutHandler(handler, specDownloadTimeout, "request timed out")
writer := newInMemoryResponseWriter()
handler.ServeHTTP(writer, req)
switch writer.respCode {
case http.StatusNotModified:
return nil, nil
case http.StatusNotFound:
return nil, ErrAPIServiceNotFound
case http.StatusOK:
return writer.data, nil
default:
return nil, fmt.Errorf("failed to retrieve openAPI spec, http error: %s", writer.String())
}
}
type inMemoryResponseWriter struct {
writeHeaderCalled bool
header http.Header
respCode int
data []byte
}
func newInMemoryResponseWriter() *inMemoryResponseWriter {
return &inMemoryResponseWriter{header: http.Header{}}
}
func (r *inMemoryResponseWriter) Header() http.Header {
return r.header
}
func (r *inMemoryResponseWriter) WriteHeader(code int) {
r.writeHeaderCalled = true
r.respCode = code
}
func (r *inMemoryResponseWriter) Write(in []byte) (int, error) {
if !r.writeHeaderCalled {
r.WriteHeader(http.StatusOK)
}
r.data = append(r.data, in...)
return len(in), nil
}
func (r *inMemoryResponseWriter) String() string {
s := fmt.Sprintf("ResponseCode: %d", r.respCode)
if r.data != nil {
s += fmt.Sprintf(", Body: %s", string(r.data))
}
if r.header != nil {
s += fmt.Sprintf(", Header: %s", r.header)
}
return s
}
type cacheableDownloader struct {
apiService ApiService
downloader *Downloader
}
func NewCacheableDownloader(apiService ApiService, downloader *Downloader) (CacheableDownloader, error) {
c := &cacheableDownloader{
apiService: apiService,
downloader: downloader,
}
if err := c.apiService.UpdateAPIService(); err != nil {
return nil, err
}
return c, nil
}
func (c *cacheableDownloader) Name() string {
return c.apiService.Name()
}
func (c *cacheableDownloader) Get(url string) ([]byte, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Add("Accept", "application/json")
return c.downloader.Download(c.apiService, req)
}
func (c *cacheableDownloader) GetV2() ([]byte, error) {
return c.Get("/openapi/v2")
}
func (c *cacheableDownloader) GetV3() ([]byte, error) {
return c.Get("/openapi/v3")
}
func (c *cacheableDownloader) UpdateDownloader(apiService ApiService) error {
c.apiService = apiService
return c.apiService.UpdateAPIService()
}

14
kube/pkg/openapi/error.go Normal file
View File

@@ -0,0 +1,14 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package openapi
import (
"errors"
)
var (
ErrAPIServiceNotFound = errors.New("resource not found")
)

208
kube/pkg/openapi/merge/default_pruning.go Normal file
View File

@@ -0,0 +1,208 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package merge
import "github.com/go-openapi/spec"
// PruneDefaults remove all the defaults recursively from all the
// schemas in the definitions, and does not modify the definitions in
// place.
func PruneDefaults(definitions spec.Definitions) spec.Definitions {
definitionsCloned := false
for k, v := range definitions {
if s := PruneDefaultsSchema(&v); s != &v {
if !definitionsCloned {
definitionsCloned = true
orig := definitions
definitions = make(spec.Definitions, len(orig))
for k2, v2 := range orig {
definitions[k2] = v2
}
}
definitions[k] = *s
}
}
return definitions
}
// PruneDefaultsSchema remove all the defaults recursively from the
// schema in place.
func PruneDefaultsSchema(schema *spec.Schema) *spec.Schema {
if schema == nil {
return nil
}
orig := schema
clone := func() {
if orig == schema {
schema = &spec.Schema{}
*schema = *orig
}
}
if schema.Default != nil {
clone()
schema.Default = nil
}
definitionsCloned := false
for k, v := range schema.Definitions {
if s := PruneDefaultsSchema(&v); s != &v {
if !definitionsCloned {
definitionsCloned = true
clone()
schema.Definitions = make(spec.Definitions, len(orig.Definitions))
for k2, v2 := range orig.Definitions {
schema.Definitions[k2] = v2
}
}
schema.Definitions[k] = *s
}
}
propertiesCloned := false
for k, v := range schema.Properties {
if s := PruneDefaultsSchema(&v); s != &v {
if !propertiesCloned {
propertiesCloned = true
clone()
schema.Properties = make(map[string]spec.Schema, len(orig.Properties))
for k2, v2 := range orig.Properties {
schema.Properties[k2] = v2
}
}
schema.Properties[k] = *s
}
}
patternPropertiesCloned := false
for k, v := range schema.PatternProperties {
if s := PruneDefaultsSchema(&v); s != &v {
if !patternPropertiesCloned {
patternPropertiesCloned = true
clone()
schema.PatternProperties = make(map[string]spec.Schema, len(orig.PatternProperties))
for k2, v2 := range orig.PatternProperties {
schema.PatternProperties[k2] = v2
}
}
schema.PatternProperties[k] = *s
}
}
dependenciesCloned := false
for k, v := range schema.Dependencies {
if s := PruneDefaultsSchema(v.Schema); s != v.Schema {
if !dependenciesCloned {
dependenciesCloned = true
clone()
schema.Dependencies = make(spec.Dependencies, len(orig.Dependencies))
for k2, v2 := range orig.Dependencies {
schema.Dependencies[k2] = v2
}
}
v.Schema = s
schema.Dependencies[k] = v
}
}
allOfCloned := false
for i := range schema.AllOf {
if s := PruneDefaultsSchema(&schema.AllOf[i]); s != &schema.AllOf[i] {
if !allOfCloned {
allOfCloned = true
clone()
schema.AllOf = make([]spec.Schema, len(orig.AllOf))
copy(schema.AllOf, orig.AllOf)
}
schema.AllOf[i] = *s
}
}
anyOfCloned := false
for i := range schema.AnyOf {
if s := PruneDefaultsSchema(&schema.AnyOf[i]); s != &schema.AnyOf[i] {
if !anyOfCloned {
anyOfCloned = true
clone()
schema.AnyOf = make([]spec.Schema, len(orig.AnyOf))
copy(schema.AnyOf, orig.AnyOf)
}
schema.AnyOf[i] = *s
}
}
oneOfCloned := false
for i := range schema.OneOf {
if s := PruneDefaultsSchema(&schema.OneOf[i]); s != &schema.OneOf[i] {
if !oneOfCloned {
oneOfCloned = true
clone()
schema.OneOf = make([]spec.Schema, len(orig.OneOf))
copy(schema.OneOf, orig.OneOf)
}
schema.OneOf[i] = *s
}
}
if schema.Not != nil {
if s := PruneDefaultsSchema(schema.Not); s != schema.Not {
clone()
schema.Not = s
}
}
if schema.AdditionalProperties != nil && schema.AdditionalProperties.Schema != nil {
if s := PruneDefaultsSchema(schema.AdditionalProperties.Schema); s != schema.AdditionalProperties.Schema {
clone()
schema.AdditionalProperties = &spec.SchemaOrBool{Schema: s, Allows: schema.AdditionalProperties.Allows}
}
}
if schema.AdditionalItems != nil && schema.AdditionalItems.Schema != nil {
if s := PruneDefaultsSchema(schema.AdditionalItems.Schema); s != schema.AdditionalItems.Schema {
clone()
schema.AdditionalItems = &spec.SchemaOrBool{Schema: s, Allows: schema.AdditionalItems.Allows}
}
}
if schema.Items != nil {
if schema.Items.Schema != nil {
if s := PruneDefaultsSchema(schema.Items.Schema); s != schema.Items.Schema {
clone()
schema.Items = &spec.SchemaOrArray{Schema: s}
}
} else {
itemsCloned := false
for i := range schema.Items.Schemas {
if s := PruneDefaultsSchema(&schema.Items.Schemas[i]); s != &schema.Items.Schemas[i] {
if !itemsCloned {
clone()
schema.Items = &spec.SchemaOrArray{
Schemas: make([]spec.Schema, len(orig.Items.Schemas)),
}
itemsCloned = true
copy(schema.Items.Schemas, orig.Items.Schemas)
}
schema.Items.Schemas[i] = *s
}
}
}
}
return schema
}

920
kube/pkg/openapi/merge/merge.go Normal file
View File

@@ -0,0 +1,920 @@
package merge
import (
"fmt"
"reflect"
"sort"
"strings"
"github.com/go-openapi/spec"
"k8s.io/kube-openapi/pkg/util"
)
const gvkKey = "x-kubesphere-group-version-kind"
// usedDefinitionForSpec returns a map with all used definitions in the provided spec as keys and true as values.
func usedDefinitionForSpec(root *spec.Swagger) map[string]bool {
usedDefinitions := map[string]bool{}
walkOnAllReferences(func(ref *spec.Ref) {
if refStr := ref.String(); refStr != "" && strings.HasPrefix(refStr, definitionPrefix) {
usedDefinitions[refStr[len(definitionPrefix):]] = true
}
}, root)
return usedDefinitions
}
// FilterSpecByPathsWithoutSideEffects removes unnecessary paths and definitions used by those paths.
// i.e. if a Path removed by this function, all definitions used by it and not used
// anywhere else will also be removed.
// It does not modify the input, but the output shares data structures with the input.
func FilterSpecByPathsWithoutSideEffects(sp *spec.Swagger, keepPathPrefixes []string) *spec.Swagger {
if sp.Paths == nil {
return sp
}
// Walk all references to find all used definitions. This function
// want to only deal with unused definitions resulted from filtering paths.
// Thus a definition will be removed only if it has been used before but
// it is unused because of a path prune.
initialUsedDefinitions := usedDefinitionForSpec(sp)
// First remove unwanted paths
prefixes := util.NewTrie(keepPathPrefixes)
ret := *sp
ret.Paths = &spec.Paths{
VendorExtensible: sp.Paths.VendorExtensible,
Paths: map[string]spec.PathItem{},
}
for path, pathItem := range sp.Paths.Paths {
if !prefixes.HasPrefix(path) {
continue
}
ret.Paths.Paths[path] = pathItem
}
// Walk all references to find all definition references.
usedDefinitions := usedDefinitionForSpec(&ret)
// Remove unused definitions
ret.Definitions = spec.Definitions{}
for k, v := range sp.Definitions {
if usedDefinitions[k] || !initialUsedDefinitions[k] {
ret.Definitions[k] = v
}
}
return &ret
}
// renameDefinitions renames definition references, without mutating the input.
// The output might share data structures with the input.
func renameDefinitions(s *spec.Swagger, renames map[string]string) *spec.Swagger {
refRenames := make(map[string]string, len(renames))
foundOne := false
for k, v := range renames {
refRenames[definitionPrefix+k] = definitionPrefix + v
if _, ok := s.Definitions[k]; ok {
foundOne = true
}
}
if !foundOne {
return s
}
ret := &spec.Swagger{}
*ret = *s
ret = ReplaceReferences(func(ref *spec.Ref) *spec.Ref {
refName := ref.String()
if newRef, found := refRenames[refName]; found {
ret := spec.MustCreateRef(newRef)
return &ret
}
return ref
}, ret)
renamedDefinitions := make(spec.Definitions, len(ret.Definitions))
for k, v := range ret.Definitions {
if newRef, found := renames[k]; found {
k = newRef
}
renamedDefinitions[k] = v
}
ret.Definitions = renamedDefinitions
return ret
}
// renameParameters renames parameter references, without mutating the input.
// The output might share data structures with the input.
func renameParameters(s *spec.Swagger, renames map[string]string) *spec.Swagger {
refRenames := make(map[string]string, len(renames))
foundOne := false
for k, v := range renames {
refRenames[parameterPrefix+k] = parameterPrefix + v
if _, ok := s.Parameters[k]; ok {
foundOne = true
}
}
if !foundOne {
return s
}
ret := &spec.Swagger{}
*ret = *s
ret = ReplaceReferences(func(ref *spec.Ref) *spec.Ref {
refName := ref.String()
if newRef, found := refRenames[refName]; found {
ret := spec.MustCreateRef(newRef)
return &ret
}
return ref
}, ret)
renamed := make(map[string]spec.Parameter, len(ret.Parameters))
for k, v := range ret.Parameters {
if newRef, found := renames[k]; found {
k = newRef
}
renamed[k] = v
}
ret.Parameters = renamed
return ret
}
// MergeSpecsIgnorePathConflictRenamingDefinitionsAndParameters is the same as
// MergeSpecs except it will ignore any path conflicts by keeping the paths of
// destination. It will rename definition and parameter conflicts.
func MergeSpecsIgnorePathConflictRenamingDefinitionsAndParameters(dest, source *spec.Swagger) error {
return mergeSpecs(dest, source, true, true, true)
}
// mergeSpecs merges source into dest while resolving conflicts.
// The source is not mutated.
func mergeSpecs(dest, source *spec.Swagger, renameModelConflicts, renameParameterConflicts, ignorePathConflicts bool) (err error) {
// Paths may be empty, due to [ACL constraints](http://goo.gl/8us55a#securityFiltering).
if source.Paths == nil {
// When a source spec does not have any path, that means none of the definitions
// are used thus we should not do anything
return nil
}
if dest.Paths == nil {
dest.Paths = &spec.Paths{}
}
if ignorePathConflicts {
keepPaths := []string{}
hasConflictingPath := false
for k := range source.Paths.Paths {
if _, found := dest.Paths.Paths[k]; !found {
keepPaths = append(keepPaths, k)
} else {
hasConflictingPath = true
}
}
if len(keepPaths) == 0 {
// There is nothing to merge. All paths are conflicting.
return nil
}
if hasConflictingPath {
source = FilterSpecByPathsWithoutSideEffects(source, keepPaths)
}
}
// Check for model conflicts and rename to make definitions conflict-free (modulo different GVKs)
usedNames := map[string]bool{}
for k := range dest.Definitions {
usedNames[k] = true
}
renames := map[string]string{}
DEFINITIONLOOP:
for k, v := range source.Definitions {
existing, found := dest.Definitions[k]
if !found || deepEqualDefinitionsModuloGVKs(&existing, &v) {
// skip for now, we copy them after the rename loop
continue
}
if !renameModelConflicts {
return fmt.Errorf("model name conflict in merging OpenAPI spec: %s", k)
}
// Reuse previously renamed model if one exists
var newName string
i := 1
for found {
i++
newName = fmt.Sprintf("%s_v%d", k, i)
existing, found = dest.Definitions[newName]
if found && deepEqualDefinitionsModuloGVKs(&existing, &v) {
renames[k] = newName
continue DEFINITIONLOOP
}
}
_, foundInSource := source.Definitions[newName]
for usedNames[newName] || foundInSource {
i++
newName = fmt.Sprintf("%s_v%d", k, i)
_, foundInSource = source.Definitions[newName]
}
renames[k] = newName
usedNames[newName] = true
}
source = renameDefinitions(source, renames)
// Check for parameter conflicts and rename to make parameters conflict-free
usedNames = map[string]bool{}
for k := range dest.Parameters {
usedNames[k] = true
}
renames = map[string]string{}
PARAMETERLOOP:
for k, p := range source.Parameters {
existing, found := dest.Parameters[k]
if !found || reflect.DeepEqual(&existing, &p) {
// skip for now, we copy them after the rename loop
continue
}
if !renameParameterConflicts {
return fmt.Errorf("parameter name conflict in merging OpenAPI spec: %s", k)
}
// Reuse previously renamed parameter if one exists
var newName string
i := 1
for found {
i++
newName = fmt.Sprintf("%s_v%d", k, i)
existing, found = dest.Parameters[newName]
if found && reflect.DeepEqual(&existing, &p) {
renames[k] = newName
continue PARAMETERLOOP
}
}
_, foundInSource := source.Parameters[newName]
for usedNames[newName] || foundInSource {
i++
newName = fmt.Sprintf("%s_v%d", k, i)
_, foundInSource = source.Parameters[newName]
}
renames[k] = newName
usedNames[newName] = true
}
source = renameParameters(source, renames)
// Now without conflict (modulo different GVKs), copy definitions to dest
for k, v := range source.Definitions {
if existing, found := dest.Definitions[k]; !found {
if dest.Definitions == nil {
dest.Definitions = make(spec.Definitions, len(source.Definitions))
}
dest.Definitions[k] = v
} else if merged, changed, err := mergedGVKs(&existing, &v); err != nil {
return err
} else if changed {
existing.Extensions[gvkKey] = merged
}
}
// Now without conflict, copy parameters to dest
for k, v := range source.Parameters {
if _, found := dest.Parameters[k]; !found {
if dest.Parameters == nil {
dest.Parameters = make(map[string]spec.Parameter, len(source.Parameters))
}
dest.Parameters[k] = v
}
}
// Check for path conflicts
for k, v := range source.Paths.Paths {
if _, found := dest.Paths.Paths[k]; found {
return fmt.Errorf("unable to merge: duplicated path %s", k)
}
// PathItem may be empty, due to [ACL constraints](http://goo.gl/8us55a#securityFiltering).
if dest.Paths.Paths == nil {
dest.Paths.Paths = map[string]spec.PathItem{}
}
dest.Paths.Paths[k] = v
}
return nil
}
// deepEqualDefinitionsModuloGVKs compares s1 and s2, but ignores the x-kubernetes-group-version-kind extension.
func deepEqualDefinitionsModuloGVKs(s1, s2 *spec.Schema) bool {
if s1 == nil {
return s2 == nil
} else if s2 == nil {
return false
}
if !reflect.DeepEqual(s1.Extensions, s2.Extensions) {
for k, v := range s1.Extensions {
if k == gvkKey {
continue
}
if !reflect.DeepEqual(v, s2.Extensions[k]) {
return false
}
}
len1 := len(s1.Extensions)
len2 := len(s2.Extensions)
if _, found := s1.Extensions[gvkKey]; found {
len1--
}
if _, found := s2.Extensions[gvkKey]; found {
len2--
}
if len1 != len2 {
return false
}
if s1.Extensions != nil {
shallowCopy := *s1
s1 = &shallowCopy
s1.Extensions = nil
}
if s2.Extensions != nil {
shallowCopy := *s2
s2 = &shallowCopy
s2.Extensions = nil
}
}
return reflect.DeepEqual(s1, s2)
}
// mergedGVKs merges the x-kubernetes-group-version-kind slices and returns the result, and whether
// s1's x-kubernetes-group-version-kind slice was changed at all.
func mergedGVKs(s1, s2 *spec.Schema) (interface{}, bool, error) {
gvk1, found1 := s1.Extensions[gvkKey]
gvk2, found2 := s2.Extensions[gvkKey]
if !found1 {
return gvk2, found2, nil
}
if !found2 {
return gvk1, false, nil
}
slice1, ok := gvk1.([]interface{})
if !ok {
return nil, false, fmt.Errorf("expected slice of GroupVersionKinds, got: %+v", slice1)
}
slice2, ok := gvk2.([]interface{})
if !ok {
return nil, false, fmt.Errorf("expected slice of GroupVersionKinds, got: %+v", slice2)
}
ret := make([]interface{}, len(slice1), len(slice1)+len(slice2))
keys := make([]string, 0, len(slice1)+len(slice2))
copy(ret, slice1)
seen := make(map[string]bool, len(slice1))
for _, x := range slice1 {
gvk, ok := x.(map[string]interface{})
if !ok {
return nil, false, fmt.Errorf(`expected {"group": <group>, "kind": <kind>, "version": <version>}, got: %#v`, x)
}
k := fmt.Sprintf("%s/%s.%s", gvk["group"], gvk["version"], gvk["kind"])
keys = append(keys, k)
seen[k] = true
}
changed := false
for _, x := range slice2 {
gvk, ok := x.(map[string]interface{})
if !ok {
return nil, false, fmt.Errorf(`expected {"group": <group>, "kind": <kind>, "version": <version>}, got: %#v`, x)
}
k := fmt.Sprintf("%s/%s.%s", gvk["group"], gvk["version"], gvk["kind"])
if seen[k] {
continue
}
ret = append(ret, x)
keys = append(keys, k)
changed = true
}
if changed {
sort.Sort(byKeys{ret, keys})
}
return ret, changed, nil
}
type byKeys struct {
values []interface{}
keys []string
}
func (b byKeys) Len() int {
return len(b.values)
}
func (b byKeys) Less(i, j int) bool {
return b.keys[i] < b.keys[j]
}
func (b byKeys) Swap(i, j int) {
b.values[i], b.values[j] = b.values[j], b.values[i]
b.keys[i], b.keys[j] = b.keys[j], b.keys[i]
}
func ReplaceReferences(walkRef func(ref *spec.Ref) *spec.Ref, sp *spec.Swagger) *spec.Swagger {
walker := &Walker{RefCallback: walkRef, SchemaCallback: SchemaCallBackNoop}
return walker.WalkRoot(sp)
}
type Walker struct {
// SchemaCallback will be called on each schema, taking the original schema,
// and before any other callbacks of the Walker.
// If the schema needs to be mutated, DO NOT mutate it in-place,
// always create a copy, mutate, and return it.
SchemaCallback func(schema *spec.Schema) *spec.Schema
// RefCallback will be called on each ref.
// If the ref needs to be mutated, DO NOT mutate it in-place,
// always create a copy, mutate, and return it.
RefCallback func(ref *spec.Ref) *spec.Ref
}
type SchemaCallbackFunc func(schema *spec.Schema) *spec.Schema
type RefCallbackFunc func(ref *spec.Ref) *spec.Ref
var SchemaCallBackNoop SchemaCallbackFunc = func(schema *spec.Schema) *spec.Schema {
return schema
}
var RefCallbackNoop RefCallbackFunc = func(ref *spec.Ref) *spec.Ref {
return ref
}
func (w *Walker) WalkRoot(swagger *spec.Swagger) *spec.Swagger {
if swagger == nil {
return nil
}
orig := swagger
cloned := false
clone := func() {
if !cloned {
cloned = true
swagger = &spec.Swagger{}
*swagger = *orig
}
}
parametersCloned := false
for k, v := range swagger.Parameters {
if p := w.walkParameter(&v); p != &v {
if !parametersCloned {
parametersCloned = true
clone()
swagger.Parameters = make(map[string]spec.Parameter, len(orig.Parameters))
for k2, v2 := range orig.Parameters {
swagger.Parameters[k2] = v2
}
}
swagger.Parameters[k] = *p
}
}
responsesCloned := false
for k, v := range swagger.Responses {
if r := w.walkResponse(&v); r != &v {
if !responsesCloned {
responsesCloned = true
clone()
swagger.Responses = make(map[string]spec.Response, len(orig.Responses))
for k2, v2 := range orig.Responses {
swagger.Responses[k2] = v2
}
}
swagger.Responses[k] = *r
}
}
definitionsCloned := false
for k, v := range swagger.Definitions {
if s := w.WalkSchema(&v); s != &v {
if !definitionsCloned {
definitionsCloned = true
clone()
swagger.Definitions = make(spec.Definitions, len(orig.Definitions))
for k2, v2 := range orig.Definitions {
swagger.Definitions[k2] = v2
}
}
swagger.Definitions[k] = *s
}
}
if swagger.Paths != nil {
if p := w.walkPaths(swagger.Paths); p != swagger.Paths {
clone()
swagger.Paths = p
}
}
return swagger
}
func (w *Walker) WalkSchema(schema *spec.Schema) *spec.Schema {
if schema == nil {
return nil
}
orig := schema
clone := func() {
if orig == schema {
schema = &spec.Schema{}
*schema = *orig
}
}
// Always run callback on the whole schema first
// so that SchemaCallback can take the original schema as input.
schema = w.SchemaCallback(schema)
if r := w.RefCallback(&schema.Ref); r != &schema.Ref {
clone()
schema.Ref = *r
}
definitionsCloned := false
for k, v := range schema.Definitions {
if s := w.WalkSchema(&v); s != &v {
if !definitionsCloned {
definitionsCloned = true
clone()
schema.Definitions = make(spec.Definitions, len(orig.Definitions))
for k2, v2 := range orig.Definitions {
schema.Definitions[k2] = v2
}
}
schema.Definitions[k] = *s
}
}
propertiesCloned := false
for k, v := range schema.Properties {
if s := w.WalkSchema(&v); s != &v {
if !propertiesCloned {
propertiesCloned = true
clone()
schema.Properties = make(map[string]spec.Schema, len(orig.Properties))
for k2, v2 := range orig.Properties {
schema.Properties[k2] = v2
}
}
schema.Properties[k] = *s
}
}
patternPropertiesCloned := false
for k, v := range schema.PatternProperties {
if s := w.WalkSchema(&v); s != &v {
if !patternPropertiesCloned {
patternPropertiesCloned = true
clone()
schema.PatternProperties = make(map[string]spec.Schema, len(orig.PatternProperties))
for k2, v2 := range orig.PatternProperties {
schema.PatternProperties[k2] = v2
}
}
schema.PatternProperties[k] = *s
}
}
allOfCloned := false
for i := range schema.AllOf {
if s := w.WalkSchema(&schema.AllOf[i]); s != &schema.AllOf[i] {
if !allOfCloned {
allOfCloned = true
clone()
schema.AllOf = make([]spec.Schema, len(orig.AllOf))
copy(schema.AllOf, orig.AllOf)
}
schema.AllOf[i] = *s
}
}
anyOfCloned := false
for i := range schema.AnyOf {
if s := w.WalkSchema(&schema.AnyOf[i]); s != &schema.AnyOf[i] {
if !anyOfCloned {
anyOfCloned = true
clone()
schema.AnyOf = make([]spec.Schema, len(orig.AnyOf))
copy(schema.AnyOf, orig.AnyOf)
}
schema.AnyOf[i] = *s
}
}
oneOfCloned := false
for i := range schema.OneOf {
if s := w.WalkSchema(&schema.OneOf[i]); s != &schema.OneOf[i] {
if !oneOfCloned {
oneOfCloned = true
clone()
schema.OneOf = make([]spec.Schema, len(orig.OneOf))
copy(schema.OneOf, orig.OneOf)
}
schema.OneOf[i] = *s
}
}
if schema.Not != nil {
if s := w.WalkSchema(schema.Not); s != schema.Not {
clone()
schema.Not = s
}
}
if schema.AdditionalProperties != nil && schema.AdditionalProperties.Schema != nil {
if s := w.WalkSchema(schema.AdditionalProperties.Schema); s != schema.AdditionalProperties.Schema {
clone()
schema.AdditionalProperties = &spec.SchemaOrBool{Schema: s, Allows: schema.AdditionalProperties.Allows}
}
}
if schema.AdditionalItems != nil && schema.AdditionalItems.Schema != nil {
if s := w.WalkSchema(schema.AdditionalItems.Schema); s != schema.AdditionalItems.Schema {
clone()
schema.AdditionalItems = &spec.SchemaOrBool{Schema: s, Allows: schema.AdditionalItems.Allows}
}
}
if schema.Items != nil {
if schema.Items.Schema != nil {
if s := w.WalkSchema(schema.Items.Schema); s != schema.Items.Schema {
clone()
schema.Items = &spec.SchemaOrArray{Schema: s}
}
} else {
itemsCloned := false
for i := range schema.Items.Schemas {
if s := w.WalkSchema(&schema.Items.Schemas[i]); s != &schema.Items.Schemas[i] {
if !itemsCloned {
clone()
schema.Items = &spec.SchemaOrArray{
Schemas: make([]spec.Schema, len(orig.Items.Schemas)),
}
itemsCloned = true
copy(schema.Items.Schemas, orig.Items.Schemas)
}
schema.Items.Schemas[i] = *s
}
}
}
}
return schema
}
func (w *Walker) walkParameter(param *spec.Parameter) *spec.Parameter {
if param == nil {
return nil
}
orig := param
cloned := false
clone := func() {
if !cloned {
cloned = true
param = &spec.Parameter{}
*param = *orig
}
}
if r := w.RefCallback(&param.Ref); r != &param.Ref {
clone()
param.Ref = *r
}
if s := w.WalkSchema(param.Schema); s != param.Schema {
clone()
param.Schema = s
}
if param.Items != nil {
if r := w.RefCallback(&param.Items.Ref); r != &param.Items.Ref {
param.Items.Ref = *r
}
}
return param
}
func (w *Walker) walkParameters(params []spec.Parameter) ([]spec.Parameter, bool) {
if params == nil {
return nil, false
}
orig := params
cloned := false
clone := func() {
if !cloned {
cloned = true
params = make([]spec.Parameter, len(params))
copy(params, orig)
}
}
for i := range params {
if s := w.walkParameter(&params[i]); s != &params[i] {
clone()
params[i] = *s
}
}
return params, cloned
}
func (w *Walker) walkResponse(resp *spec.Response) *spec.Response {
if resp == nil {
return nil
}
orig := resp
cloned := false
clone := func() {
if !cloned {
cloned = true
resp = &spec.Response{}
*resp = *orig
}
}
if r := w.RefCallback(&resp.Ref); r != &resp.Ref {
clone()
resp.Ref = *r
}
if s := w.WalkSchema(resp.Schema); s != resp.Schema {
clone()
resp.Schema = s
}
return resp
}
func (w *Walker) walkResponses(resps *spec.Responses) *spec.Responses {
if resps == nil {
return nil
}
orig := resps
cloned := false
clone := func() {
if !cloned {
cloned = true
resps = &spec.Responses{}
*resps = *orig
}
}
if r := w.walkResponse(resps.ResponsesProps.Default); r != resps.ResponsesProps.Default {
clone()
resps.Default = r
}
responsesCloned := false
for k, v := range resps.ResponsesProps.StatusCodeResponses {
if r := w.walkResponse(&v); r != &v {
if !responsesCloned {
responsesCloned = true
clone()
resps.ResponsesProps.StatusCodeResponses = make(map[int]spec.Response, len(orig.StatusCodeResponses))
for k2, v2 := range orig.StatusCodeResponses {
resps.ResponsesProps.StatusCodeResponses[k2] = v2
}
}
resps.ResponsesProps.StatusCodeResponses[k] = *r
}
}
return resps
}
func (w *Walker) walkOperation(op *spec.Operation) *spec.Operation {
if op == nil {
return nil
}
orig := op
cloned := false
clone := func() {
if !cloned {
cloned = true
op = &spec.Operation{}
*op = *orig
}
}
parametersCloned := false
for i := range op.Parameters {
if s := w.walkParameter(&op.Parameters[i]); s != &op.Parameters[i] {
if !parametersCloned {
parametersCloned = true
clone()
op.Parameters = make([]spec.Parameter, len(orig.Parameters))
copy(op.Parameters, orig.Parameters)
}
op.Parameters[i] = *s
}
}
if r := w.walkResponses(op.Responses); r != op.Responses {
clone()
op.Responses = r
}
return op
}
func (w *Walker) walkPathItem(pathItem *spec.PathItem) *spec.PathItem {
if pathItem == nil {
return nil
}
orig := pathItem
cloned := false
clone := func() {
if !cloned {
cloned = true
pathItem = &spec.PathItem{}
*pathItem = *orig
}
}
if p, changed := w.walkParameters(pathItem.Parameters); changed {
clone()
pathItem.Parameters = p
}
if op := w.walkOperation(pathItem.Get); op != pathItem.Get {
clone()
pathItem.Get = op
}
if op := w.walkOperation(pathItem.Head); op != pathItem.Head {
clone()
pathItem.Head = op
}
if op := w.walkOperation(pathItem.Delete); op != pathItem.Delete {
clone()
pathItem.Delete = op
}
if op := w.walkOperation(pathItem.Options); op != pathItem.Options {
clone()
pathItem.Options = op
}
if op := w.walkOperation(pathItem.Patch); op != pathItem.Patch {
clone()
pathItem.Patch = op
}
if op := w.walkOperation(pathItem.Post); op != pathItem.Post {
clone()
pathItem.Post = op
}
if op := w.walkOperation(pathItem.Put); op != pathItem.Put {
clone()
pathItem.Put = op
}
return pathItem
}
func (w *Walker) walkPaths(paths *spec.Paths) *spec.Paths {
if paths == nil {
return nil
}
orig := paths
cloned := false
clone := func() {
if !cloned {
cloned = true
paths = &spec.Paths{}
*paths = *orig
}
}
pathsCloned := false
for k, v := range paths.Paths {
if p := w.walkPathItem(&v); p != &v {
if !pathsCloned {
pathsCloned = true
clone()
paths.Paths = make(map[string]spec.PathItem, len(orig.Paths))
for k2, v2 := range orig.Paths {
paths.Paths[k2] = v2
}
}
paths.Paths[k] = *p
}
}
return paths
}

163
kube/pkg/openapi/merge/walker.go Normal file
View File

@@ -0,0 +1,163 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package merge
import (
"strings"
"github.com/go-openapi/spec"
)
const (
definitionPrefix = "#/definitions/"
parameterPrefix = "#/parameters/"
)
// Run a readonlyReferenceWalker method on all references of an OpenAPI spec
type readonlyReferenceWalker struct {
// walkRefCallback will be called on each reference. The input will never be nil.
walkRefCallback func(ref *spec.Ref)
// The spec to walk through.
root *spec.Swagger
}
// walkOnAllReferences recursively walks on all references, while following references into definitions.
// it calls walkRef on each found reference.
func walkOnAllReferences(walkRef func(ref *spec.Ref), root *spec.Swagger) {
alreadyVisited := map[string]bool{}
walker := &readonlyReferenceWalker{
root: root,
}
walker.walkRefCallback = func(ref *spec.Ref) {
walkRef(ref)
refStr := ref.String()
if refStr == "" || !strings.HasPrefix(refStr, definitionPrefix) {
return
}
defName := refStr[len(definitionPrefix):]
if _, found := root.Definitions[defName]; found && !alreadyVisited[refStr] {
alreadyVisited[refStr] = true
def := root.Definitions[defName]
walker.walkSchema(&def)
}
}
walker.Start()
}
func (s *readonlyReferenceWalker) walkSchema(schema *spec.Schema) {
if schema == nil {
return
}
s.walkRefCallback(&schema.Ref)
var v *spec.Schema
if len(schema.Definitions)+len(schema.Properties)+len(schema.PatternProperties) > 0 {
v = &spec.Schema{}
}
for k := range schema.Definitions {
*v = schema.Definitions[k]
s.walkSchema(v)
}
for k := range schema.Properties {
*v = schema.Properties[k]
s.walkSchema(v)
}
for k := range schema.PatternProperties {
*v = schema.PatternProperties[k]
s.walkSchema(v)
}
for i := range schema.AllOf {
s.walkSchema(&schema.AllOf[i])
}
for i := range schema.AnyOf {
s.walkSchema(&schema.AnyOf[i])
}
for i := range schema.OneOf {
s.walkSchema(&schema.OneOf[i])
}
if schema.Not != nil {
s.walkSchema(schema.Not)
}
if schema.AdditionalProperties != nil && schema.AdditionalProperties.Schema != nil {
s.walkSchema(schema.AdditionalProperties.Schema)
}
if schema.AdditionalItems != nil && schema.AdditionalItems.Schema != nil {
s.walkSchema(schema.AdditionalItems.Schema)
}
if schema.Items != nil {
if schema.Items.Schema != nil {
s.walkSchema(schema.Items.Schema)
}
for i := range schema.Items.Schemas {
s.walkSchema(&schema.Items.Schemas[i])
}
}
}
func (s *readonlyReferenceWalker) walkParams(params []spec.Parameter) {
if params == nil {
return
}
for _, param := range params {
s.walkRefCallback(&param.Ref)
s.walkSchema(param.Schema)
if param.Items != nil {
s.walkRefCallback(&param.Items.Ref)
}
}
}
func (s *readonlyReferenceWalker) walkResponse(resp *spec.Response) {
if resp == nil {
return
}
s.walkRefCallback(&resp.Ref)
s.walkSchema(resp.Schema)
}
func (s *readonlyReferenceWalker) walkOperation(op *spec.Operation) {
if op == nil {
return
}
s.walkParams(op.Parameters)
if op.Responses == nil {
return
}
s.walkResponse(op.Responses.Default)
for _, r := range op.Responses.StatusCodeResponses {
s.walkResponse(&r)
}
}
func (s *readonlyReferenceWalker) Start() {
if s.root.Paths == nil {
return
}
for _, pathItem := range s.root.Paths.Paths {
s.walkParams(pathItem.Parameters)
s.walkOperation(pathItem.Delete)
s.walkOperation(pathItem.Get)
s.walkOperation(pathItem.Head)
s.walkOperation(pathItem.Options)
s.walkOperation(pathItem.Patch)
s.walkOperation(pathItem.Post)
s.walkOperation(pathItem.Put)
}
}

102
kube/pkg/openapi/openapi.go Normal file
View File

@@ -0,0 +1,102 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package openapi
import (
"fmt"
"net/http"
"sync/atomic"
extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
)
type Cache[T any] struct {
value atomic.Pointer[T]
}
func (c *Cache[T]) Store(v T) {
c.value.Store(&v)
}
func (c *Cache[T]) Load() T {
return *c.value.Load()
}
type PathHandler interface {
Handle(path string, handler http.Handler)
}
type APIServiceManager interface {
AddUpdateApiService(apiService *extensionsv1alpha1.APIService) error
UpdateOpenApiSpec(apiServiceName string) error
RemoveApiService(apiServiceName string)
}
type OpenApiAggregatorServices struct {
apiService map[string]ApiService
downloaderMap map[string]CacheableDownloader
downloader *Downloader
}
func NewOpenApiAggregatorServices() *OpenApiAggregatorServices {
return &OpenApiAggregatorServices{
apiService: make(map[string]ApiService),
downloaderMap: make(map[string]CacheableDownloader),
downloader: NewDownloader(),
}
}
func (o *OpenApiAggregatorServices) AddUpdateApiService(apiService *extensionsv1alpha1.APIService) error {
openapiService := NewApiService(apiService)
o.apiService[apiService.Name] = openapiService
if d, ok := o.downloaderMap[apiService.Name]; ok {
if err := d.UpdateDownloader(openapiService); err != nil {
return err
}
} else {
cacheDownloader, err := NewCacheableDownloader(openapiService, o.downloader)
if err != nil {
return err
}
o.downloaderMap[apiService.Name] = cacheDownloader
}
return nil
}
func (o *OpenApiAggregatorServices) GetOpenApiSpecV2(apiServiceName string) ([]byte, error) {
if d, ok := o.downloaderMap[apiServiceName]; ok {
data, err := d.GetV2()
if err != nil {
return nil, fmt.Errorf("fetch ApiService %s openapi-v2 error: %s", apiServiceName, err)
}
return data, nil
}
return nil, fmt.Errorf("update OpenApiSpec failed beaseuse of apiService %s not found", apiServiceName)
}
func (o *OpenApiAggregatorServices) GetOpenApiSpecV3(apiServiceName string) ([]byte, error) {
if d, ok := o.downloaderMap[apiServiceName]; ok {
data, err := d.GetV3()
if err != nil {
return nil, fmt.Errorf("fetch ApiService %s openapi-v3 error: %s", apiServiceName, err)
}
return data, nil
}
return nil, fmt.Errorf("update OpenApiSpec failed beaseuse of apiService %s not found", apiServiceName)
}
func (o *OpenApiAggregatorServices) AddLocalApiService(name string) {
apiService := extensionsv1alpha1.APIService{}
apiService.Name = name
o.apiService[apiService.Name] = NewApiService(&apiService)
}
func (o *OpenApiAggregatorServices) RemoveApiService(apiServiceName string) {
delete(o.apiService, apiServiceName)
delete(o.downloaderMap, apiServiceName)
}

143
kube/pkg/openapi/v2/services.go Normal file
View File

@@ -0,0 +1,143 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package v2
import (
"fmt"
"net/http"
"github.com/NYTimes/gziphandler"
restfulspec "github.com/emicklei/go-restful-openapi/v2"
"github.com/go-openapi/spec"
"k8s.io/klog/v2"
extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
"kubesphere.io/kubesphere/kube/pkg/openapi"
"kubesphere.io/kubesphere/kube/pkg/openapi/merge"
)
var OpenApiPath = "/openapi/v2"
type OpenApiV2Services struct {
openApiSpecCache map[string]*openapi.Cache[*spec.Swagger]
openApiAggregatorService *openapi.OpenApiAggregatorServices
}
func NewOpenApiV2Services() *OpenApiV2Services {
return &OpenApiV2Services{
openApiSpecCache: make(map[string]*openapi.Cache[*spec.Swagger]),
openApiAggregatorService: openapi.NewOpenApiAggregatorServices(),
}
}
func (s *OpenApiV2Services) AddUpdateApiService(apiService *extensionsv1alpha1.APIService) error {
c := &openapi.Cache[*spec.Swagger]{}
c.Store(&spec.Swagger{})
s.openApiSpecCache[apiService.Name] = c
if err := s.openApiAggregatorService.AddUpdateApiService(apiService); err != nil {
return err
}
return s.UpdateOpenApiSpec(apiService.Name)
}
func (s *OpenApiV2Services) UpdateOpenApiSpec(apiServiceName string) error {
data, err := s.openApiAggregatorService.GetOpenApiSpecV2(apiServiceName)
if err != nil {
return err
}
openAPISpec := &spec.Swagger{}
if err := openAPISpec.UnmarshalJSON(data); err != nil {
return err
}
if cache, ok := s.openApiSpecCache[apiServiceName]; ok {
cache.Store(openAPISpec)
} else {
c := openapi.Cache[*spec.Swagger]{}
c.Store(openAPISpec)
s.openApiSpecCache[apiServiceName] = &c
}
return nil
}
func (s *OpenApiV2Services) RemoveApiService(apiServiceName string) {
s.openApiAggregatorService.RemoveApiService(apiServiceName)
delete(s.openApiSpecCache, apiServiceName)
}
func (s *OpenApiV2Services) MergeSpecCache() (*spec.Swagger, error) {
var merged *spec.Swagger
for i := range s.openApiSpecCache {
if cacheValue, ok := s.openApiSpecCache[i]; ok {
cacheSpec := cacheValue.Load()
if merged == nil {
merged = &spec.Swagger{}
*merged = *cacheSpec
merged.Paths = nil
merged.Definitions = nil
merged.Parameters = nil
}
if err := merge.MergeSpecsIgnorePathConflictRenamingDefinitionsAndParameters(merged, cacheSpec); err != nil {
return nil, fmt.Errorf("failed to build merge specs: %v", err)
}
}
}
return merged, nil
}
func (s *OpenApiV2Services) RegisterOpenAPIVersionedService(servePath string, handler openapi.PathHandler) {
handler.Handle(servePath, gziphandler.GzipHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
result, err := s.MergeSpecCache()
if err != nil {
klog.Errorf("Error in OpenAPI handler: %s", err)
// only return a 503 if we have no older cache data to serve
if result == nil {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
}
data, err := (*result).MarshalJSON()
if err != nil {
klog.Errorf("Error in OpenAPI handler: %s", err)
if data == nil {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
}
w.Write(data)
}),
))
}
func (s *OpenApiV2Services) AddLocalApiService(name string, val *spec.Swagger) {
s.openApiAggregatorService.AddLocalApiService(name)
c := &openapi.Cache[*spec.Swagger]{}
c.Store(val)
s.openApiSpecCache[name] = c
}
func BuildAndRegisterAggregator(
config *restfulspec.Config, pathHandler openapi.PathHandler) (*OpenApiV2Services, error) {
aggregatorOpenAPISpec := restfulspec.BuildSwagger(*config)
aggregatorOpenAPISpec.Definitions = merge.PruneDefaults(aggregatorOpenAPISpec.Definitions)
s := buildAndRegisterOpenApiV2ForLocalServices(OpenApiPath, aggregatorOpenAPISpec, pathHandler)
return s, nil
}
func buildAndRegisterOpenApiV2ForLocalServices(path string, aggregatorSpec *spec.Swagger, pathHandler openapi.PathHandler) *OpenApiV2Services {
s := NewOpenApiV2Services()
s.AddLocalApiService("kubeSphere_internal_local_delegation", aggregatorSpec)
if path == "" {
path = OpenApiPath
}
s.RegisterOpenAPIVersionedService(path, pathHandler)
return s
}

43
kube/pkg/openapi/v2/services_test.go Normal file
View File

@@ -0,0 +1,43 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package v2
import (
"testing"
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
)
func TestServiceAddUpdateApiService(t *testing.T) {
uri := "http://172.31.188.161:8080"
apiServer := extensionsv1alpha1.APIService{
TypeMeta: metav1.TypeMeta{},
ObjectMeta: metav1.ObjectMeta{
Name: "v1alpha1.local.kubesphere.io",
},
Spec: extensionsv1alpha1.APIServiceSpec{
Group: "local.kubesphere.io",
Version: "v1alpha1",
Endpoint: extensionsv1alpha1.Endpoint{
URL: &uri,
Service: nil,
CABundle: nil,
InsecureSkipVerify: false,
},
},
Status: extensionsv1alpha1.APIServiceStatus{},
}
openApiV2Services := NewOpenApiV2Services()
err := openApiV2Services.AddUpdateApiService(&apiServer)
assert.Equal(t, err, nil)
val, err := openApiV2Services.MergeSpecCache()
assert.Equal(t, err, nil)
t.Log(val)
}

154
kube/pkg/openapi/v3/services.go Normal file
View File

@@ -0,0 +1,154 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package v3
import (
"fmt"
"net/http"
"github.com/NYTimes/gziphandler"
restfulspec "github.com/emicklei/go-restful-openapi/v2"
openapibuilder "k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder"
"k8s.io/klog/v2"
"k8s.io/kube-openapi/pkg/openapiconv"
"k8s.io/kube-openapi/pkg/spec3"
spec2 "k8s.io/kube-openapi/pkg/validation/spec"
extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
"kubesphere.io/kubesphere/kube/pkg/openapi"
"kubesphere.io/kubesphere/kube/pkg/openapi/merge"
)
var OpenApiPath = "/openapi/v3"
type OpenApiV3Services struct {
openApiSpecCache map[string]*openapi.Cache[*spec3.OpenAPI]
openApiAggregatorService *openapi.OpenApiAggregatorServices
}
func NewOpenApiV3Services() *OpenApiV3Services {
return &OpenApiV3Services{
openApiSpecCache: make(map[string]*openapi.Cache[*spec3.OpenAPI]),
openApiAggregatorService: openapi.NewOpenApiAggregatorServices(),
}
}
func (s *OpenApiV3Services) AddUpdateApiService(apiService *extensionsv1alpha1.APIService) error {
c := &openapi.Cache[*spec3.OpenAPI]{}
c.Store(&spec3.OpenAPI{})
s.openApiSpecCache[apiService.Name] = c
if err := s.openApiAggregatorService.AddUpdateApiService(apiService); err != nil {
return err
}
return s.UpdateOpenApiSpec(apiService.Name)
}
func (s *OpenApiV3Services) UpdateOpenApiSpec(apiServiceName string) error {
data, err := s.openApiAggregatorService.GetOpenApiSpecV3(apiServiceName)
if err != nil {
return err
}
openAPISpec := &spec3.OpenAPI{}
if err := openAPISpec.UnmarshalJSON(data); err != nil {
return err
}
if cache, ok := s.openApiSpecCache[apiServiceName]; ok {
cache.Store(openAPISpec)
return nil
} else {
c := openapi.Cache[*spec3.OpenAPI]{}
c.Store(openAPISpec)
s.openApiSpecCache[apiServiceName] = &c
}
return nil
}
func (s *OpenApiV3Services) RemoveApiService(apiServiceName string) {
s.openApiAggregatorService.RemoveApiService(apiServiceName)
delete(s.openApiSpecCache, apiServiceName)
}
func (s *OpenApiV3Services) MergeSpecCache() (*spec3.OpenAPI, error) {
var merged *spec3.OpenAPI
var err error
for i := range s.openApiSpecCache {
if cacheValue, ok := s.openApiSpecCache[i]; ok {
cacheSpec := cacheValue.Load()
if merged == nil {
merged = &spec3.OpenAPI{}
*merged = *cacheSpec
merged.Paths = nil
}
if merged, err = openapibuilder.MergeSpecsV3(merged, cacheSpec); err != nil {
return nil, fmt.Errorf("failed to build merge specs: %v", err)
}
}
}
return merged, nil
}
func (s *OpenApiV3Services) RegisterOpenAPIVersionedService(servePath string, handler openapi.PathHandler) {
handler.Handle(servePath, gziphandler.GzipHandler(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
result, err := s.MergeSpecCache()
if err != nil {
klog.Errorf("Error in OpenAPI handler: %s", err)
// only return a 503 if we have no older cache data to serve
if result == nil {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
}
data, err := (*result).MarshalJSON()
if err != nil {
klog.Errorf("Error in OpenAPI handler: %s", err)
if data == nil {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
}
w.Write(data)
}),
))
}
func (s *OpenApiV3Services) AddLocalApiService(name string, val *spec3.OpenAPI) {
s.openApiAggregatorService.AddLocalApiService(name)
c := &openapi.Cache[*spec3.OpenAPI]{}
c.Store(val)
s.openApiSpecCache[name] = c
}
func BuildAndRegisterAggregator(
config *restfulspec.Config, pathHandler openapi.PathHandler) (*OpenApiV3Services, error) {
aggregatorOpenAPISpec := restfulspec.BuildSwagger(*config)
aggregatorOpenAPISpec.Definitions = merge.PruneDefaults(aggregatorOpenAPISpec.Definitions)
swaggerData, err := aggregatorOpenAPISpec.MarshalJSON()
if err != nil {
return nil, err
}
spec2Swagger := spec2.Swagger{}
if err = spec2Swagger.UnmarshalJSON(swaggerData); err != nil {
return nil, err
}
convertedOpenAPIV3 := openapiconv.ConvertV2ToV3(&spec2Swagger)
s := buildAndRegisterOpenApiV3ForLocalServices(OpenApiPath, convertedOpenAPIV3, pathHandler)
return s, nil
}
func buildAndRegisterOpenApiV3ForLocalServices(path string, aggregatorSpec *spec3.OpenAPI, pathHandler openapi.PathHandler) *OpenApiV3Services {
s := NewOpenApiV3Services()
s.AddLocalApiService("kubeSphere_internal_local_delegation", aggregatorSpec)
if path == "" {
path = OpenApiPath
}
s.RegisterOpenAPIVersionedService(path, pathHandler)
return s
}

6
kube/pkg/openapi/v3/services_test.go Normal file
View File

@@ -0,0 +1,6 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package v3

40
kube/pkg/quota/v1/evaluator/core/persistent_volume_claims.go → kube/pkg/quota/v1/evaluator/core/persistentvolumeclaims.go
View File

@@ -17,9 +17,12 @@ limitations under the License.
package core
import (
"context"
"fmt"
"strings"
"sigs.k8s.io/controller-runtime/pkg/client"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/runtime"
@@ -29,7 +32,7 @@ import (
"kubesphere.io/kubesphere/kube/pkg/apis/core/v1/helper"
k8sfeatures "kubesphere.io/kubesphere/kube/pkg/features"
quota "kubesphere.io/kubesphere/kube/pkg/quota/v1"
"kubesphere.io/kubesphere/kube/pkg/quota/v1"
"kubesphere.io/kubesphere/kube/pkg/quota/v1/generic"
)
@@ -51,29 +54,16 @@ var pvcResources = []corev1.ResourceName{
// * bronze.storageclass.storage.k8s.io/requests.storage: 500Gi
const storageClassSuffix string = ".storageclass.storage.k8s.io/"
/* TODO: prune?
// ResourceByStorageClass returns a quota resource name by storage class.
func ResourceByStorageClass(storageClass string, resourceName corev1.ResourceName) corev1.ResourceName {
return corev1.ResourceName(string(storageClass + storageClassSuffix + string(resourceName)))
}
*/
// V1ResourceByStorageClass returns a quota resource name by storage class.
func V1ResourceByStorageClass(storageClass string, resourceName corev1.ResourceName) corev1.ResourceName {
return corev1.ResourceName(string(storageClass + storageClassSuffix + string(resourceName)))
}
// NewPersistentVolumeClaimEvaluator returns an evaluator that can evaluate persistent volume claims
func NewPersistentVolumeClaimEvaluator(f quota.ListerForResourceFunc) quota.Evaluator {
listFuncByNamespace := generic.ListResourceUsingListerFunc(f, corev1.SchemeGroupVersion.WithResource("persistentvolumeclaims"))
pvcEvaluator := &pvcEvaluator{listFuncByNamespace: listFuncByNamespace}
func NewPersistentVolumeClaimEvaluator(cache client.Reader) quota.Evaluator {
pvcEvaluator := &pvcEvaluator{cache: cache}
return pvcEvaluator
}
// pvcEvaluator knows how to evaluate quota usage for persistent volume claims
type pvcEvaluator struct {
// listFuncByNamespace knows how to list pvc claims
listFuncByNamespace generic.ListFuncByNamespace
cache client.Reader
}
// Constraints verifies that all required resources are present on the item.
@@ -117,7 +107,7 @@ func (p *pvcEvaluator) UncoveredQuotaScopes(limitedScopes []corev1.ScopedResourc
// MatchingResources takes the input specified list of resources and returns the set of resources it matches.
func (p *pvcEvaluator) MatchingResources(items []corev1.ResourceName) []corev1.ResourceName {
result := []corev1.ResourceName{}
var result []corev1.ResourceName
for _, item := range items {
// match object count quota fields
if quota.Contains([]corev1.ResourceName{pvcObjectCountName}, item) {
@@ -170,9 +160,21 @@ func (p *pvcEvaluator) Usage(item runtime.Object) (corev1.ResourceList, error) {
return result, nil
}
func (p *pvcEvaluator) listPVC(namespace string) ([]runtime.Object, error) {
pvcList := &corev1.PersistentVolumeClaimList{}
if err := p.cache.List(context.Background(), pvcList, client.InNamespace(namespace)); err != nil {
return nil, err
}
pvcs := make([]runtime.Object, 0)
for _, pvc := range pvcList.Items {
pvcs = append(pvcs, &pvc)
}
return pvcs, nil
}
// UsageStats calculates aggregate usage for the object.
func (p *pvcEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) {
return generic.CalculateUsageStats(options, p.listFuncByNamespace, generic.MatchesNoScopeFunc, p.Usage)
return generic.CalculateUsageStats(options, p.listPVC, generic.MatchesNoScopeFunc, p.Usage)
}
// ensure we implement required interface

26
kube/pkg/quota/v1/evaluator/core/pods.go
View File

@@ -17,16 +17,18 @@ limitations under the License.
package core
import (
"context"
"fmt"
"strings"
"time"
"sigs.k8s.io/controller-runtime/pkg/client"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
"k8s.io/utils/clock"
@@ -96,16 +98,14 @@ var validationSet = sets.New(
)
// NewPodEvaluator returns an evaluator that can evaluate pods
func NewPodEvaluator(f quota.ListerForResourceFunc, clock clock.Clock) quota.Evaluator {
listFuncByNamespace := generic.ListResourceUsingListerFunc(f, corev1.SchemeGroupVersion.WithResource("pods"))
podEvaluator := &podEvaluator{listFuncByNamespace: listFuncByNamespace, clock: clock}
func NewPodEvaluator(cache client.Reader, clock clock.Clock) quota.Evaluator {
podEvaluator := &podEvaluator{cache: cache, clock: clock}
return podEvaluator
}
// podEvaluator knows how to measure usage of pods.
type podEvaluator struct {
// knows how to list pods
listFuncByNamespace generic.ListFuncByNamespace
cache client.Reader
// used to track time
clock clock.Clock
}
@@ -212,7 +212,19 @@ func (p *podEvaluator) Usage(item runtime.Object) (corev1.ResourceList, error) {
// UsageStats calculates aggregate usage for the object.
func (p *podEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) {
return generic.CalculateUsageStats(options, p.listFuncByNamespace, podMatchesScopeFunc, p.Usage)
return generic.CalculateUsageStats(options, p.listPods, podMatchesScopeFunc, p.Usage)
}
func (p *podEvaluator) listPods(namespace string) ([]runtime.Object, error) {
podList := &corev1.PodList{}
if err := p.cache.List(context.Background(), podList, client.InNamespace(namespace)); err != nil {
return nil, err
}
pods := make([]runtime.Object, 0)
for _, pod := range podList.Items {
pods = append(pods, &pod)
}
return pods, nil
}
// verifies we implement the required interface.

23
kube/pkg/quota/v1/evaluator/core/registry.go
View File

@@ -20,31 +20,32 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/utils/clock"
"sigs.k8s.io/controller-runtime/pkg/client"
quota "kubesphere.io/kubesphere/kube/pkg/quota/v1"
"kubesphere.io/kubesphere/kube/pkg/quota/v1"
"kubesphere.io/kubesphere/kube/pkg/quota/v1/generic"
)
// legacyObjectCountAliases are what we used to do simple object counting quota with mapped to alias
var legacyObjectCountAliases = map[schema.GroupVersionResource]corev1.ResourceName{
corev1.SchemeGroupVersion.WithResource("configmaps"): corev1.ResourceConfigMaps,
corev1.SchemeGroupVersion.WithResource("resourcequotas"): corev1.ResourceQuotas,
corev1.SchemeGroupVersion.WithResource("replicationcontrollers"): corev1.ResourceReplicationControllers,
corev1.SchemeGroupVersion.WithResource("secrets"): corev1.ResourceSecrets,
corev1.SchemeGroupVersion.WithResource(string(corev1.ResourceConfigMaps)): corev1.ResourceConfigMaps,
corev1.SchemeGroupVersion.WithResource(string(corev1.ResourceQuotas)): corev1.ResourceQuotas,
corev1.SchemeGroupVersion.WithResource(string(corev1.ResourceReplicationControllers)): corev1.ResourceReplicationControllers,
corev1.SchemeGroupVersion.WithResource(string(corev1.ResourceSecrets)): corev1.ResourceSecrets,
}
// NewEvaluators returns the list of static evaluators that manage more than counts
func NewEvaluators(f quota.ListerForResourceFunc) []quota.Evaluator {
func NewEvaluators(client client.Client) []quota.Evaluator {
// these evaluators have special logic
result := []quota.Evaluator{
NewPodEvaluator(f, clock.RealClock{}),
NewServiceEvaluator(f),
NewPersistentVolumeClaimEvaluator(f),
NewPodEvaluator(client, clock.RealClock{}),
NewServiceEvaluator(client),
NewPersistentVolumeClaimEvaluator(client),
}
// these evaluators require an alias for backwards compatibility
for gvr, alias := range legacyObjectCountAliases {
for gvk, alias := range legacyObjectCountAliases {
result = append(result,
generic.NewObjectCountEvaluator(gvr.GroupResource(), generic.ListResourceUsingListerFunc(f, gvr), alias))
generic.NewObjectCountEvaluator(gvk.GroupVersion().WithResource(string(alias)).GroupResource(), generic.ListResourceUsingCacheFunc(client, gvk), alias))
}
return result
}

24
kube/pkg/quota/v1/evaluator/core/services.go
View File

@@ -17,8 +17,11 @@ limitations under the License.
package core
import (
"context"
"fmt"
"sigs.k8s.io/controller-runtime/pkg/client"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/runtime"
@@ -41,16 +44,15 @@ var serviceResources = []corev1.ResourceName{
}
// NewServiceEvaluator returns an evaluator that can evaluate services.
func NewServiceEvaluator(f quota.ListerForResourceFunc) quota.Evaluator {
listFuncByNamespace := generic.ListResourceUsingListerFunc(f, corev1.SchemeGroupVersion.WithResource("services"))
serviceEvaluator := &serviceEvaluator{listFuncByNamespace: listFuncByNamespace}
func NewServiceEvaluator(cache client.Reader) quota.Evaluator {
serviceEvaluator := &serviceEvaluator{cache: cache}
return serviceEvaluator
}
// serviceEvaluator knows how to measure usage for services.
type serviceEvaluator struct {
// knows how to list items by namespace
listFuncByNamespace generic.ListFuncByNamespace
cache client.Reader
}
// Constraints verifies that all required resources are present on the item
@@ -131,9 +133,21 @@ func (p *serviceEvaluator) Usage(item runtime.Object) (corev1.ResourceList, erro
return result, nil
}
func (p *serviceEvaluator) listServices(namespace string) ([]runtime.Object, error) {
serviceList := &corev1.ServiceList{}
if err := p.cache.List(context.Background(), serviceList, client.InNamespace(namespace)); err != nil {
return nil, err
}
services := make([]runtime.Object, 0)
for _, svc := range serviceList.Items {
services = append(services, &svc)
}
return services, nil
}
// UsageStats calculates aggregate usage for the object.
func (p *serviceEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) {
return generic.CalculateUsageStats(options, p.listFuncByNamespace, generic.MatchesNoScopeFunc, p.Usage)
return generic.CalculateUsageStats(options, p.listServices, generic.MatchesNoScopeFunc, p.Usage)
}
var _ quota.Evaluator = &serviceEvaluator{}

105
kube/pkg/quota/v1/generic/evaluator.go
View File

@@ -17,109 +17,36 @@ limitations under the License.
package generic
import (
"context"
"fmt"
"sync/atomic"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/client-go/informers"
"k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
quota "kubesphere.io/kubesphere/kube/pkg/quota/v1"
"kubesphere.io/kubesphere/kube/pkg/quota/v1"
)
// InformerForResourceFunc knows how to provision an informer
type InformerForResourceFunc func(schema.GroupVersionResource) (informers.GenericInformer, error)
// ListerFuncForResourceFunc knows how to provision a lister from an informer func.
// The lister returns errors until the informer has synced.
func ListerFuncForResourceFunc(f InformerForResourceFunc) quota.ListerForResourceFunc {
return func(gvr schema.GroupVersionResource) (cache.GenericLister, error) {
informer, err := f(gvr)
if err != nil {
return nil, err
}
return &protectedLister{
hasSynced: cachedHasSynced(informer.Informer().HasSynced),
notReadyErr: fmt.Errorf("%v not yet synced", gvr),
delegate: informer.Lister(),
}, nil
}
}
// cachedHasSynced returns a function that calls hasSynced() until it returns true once, then returns true
func cachedHasSynced(hasSynced func() bool) func() bool {
cache := &atomic.Value{}
cache.Store(false)
return func() bool {
if cache.Load().(bool) {
// short-circuit if already synced
return true
}
if hasSynced() {
// remember we synced
cache.Store(true)
return true
}
return false
}
}
// protectedLister returns notReadyError if hasSynced returns false, otherwise delegates to delegate
type protectedLister struct {
hasSynced func() bool
notReadyErr error
delegate cache.GenericLister
}
func (p *protectedLister) List(selector labels.Selector) (ret []runtime.Object, err error) {
if !p.hasSynced() {
return nil, p.notReadyErr
}
return p.delegate.List(selector)
}
func (p *protectedLister) Get(name string) (runtime.Object, error) {
if !p.hasSynced() {
return nil, p.notReadyErr
}
return p.delegate.Get(name)
}
func (p *protectedLister) ByNamespace(namespace string) cache.GenericNamespaceLister {
return &protectedNamespaceLister{p.hasSynced, p.notReadyErr, p.delegate.ByNamespace(namespace)}
}
// protectedNamespaceLister returns notReadyError if hasSynced returns false, otherwise delegates to delegate
type protectedNamespaceLister struct {
hasSynced func() bool
notReadyErr error
delegate cache.GenericNamespaceLister
}
func (p *protectedNamespaceLister) List(selector labels.Selector) (ret []runtime.Object, err error) {
if !p.hasSynced() {
return nil, p.notReadyErr
}
return p.delegate.List(selector)
}
func (p *protectedNamespaceLister) Get(name string) (runtime.Object, error) {
if !p.hasSynced() {
return nil, p.notReadyErr
}
return p.delegate.Get(name)
}
// ListResourceUsingListerFunc returns a listing function based on the shared informer factory for the specified resource.
func ListResourceUsingListerFunc(l quota.ListerForResourceFunc, resource schema.GroupVersionResource) ListFuncByNamespace {
// ListResourceUsingCacheFunc returns a listing function based on the shared informer factory for the specified resource.
func ListResourceUsingCacheFunc(cacheClient client.Client, gvr schema.GroupVersionResource) ListFuncByNamespace {
return func(namespace string) ([]runtime.Object, error) {
lister, err := l(resource)
gvk, err := cacheClient.RESTMapper().KindFor(gvr)
if err != nil {
return nil, err
}
return lister.ByNamespace(namespace).List(labels.Everything())
gvkObject, err := cacheClient.Scheme().New(gvk)
if err != nil {
return nil, err
}
objList := gvkObject.(client.ObjectList)
if err := cacheClient.List(context.Background(), objList); err != nil {
return nil, err
}
return meta.ExtractList(objList)
}
}

2
kube/pkg/quota/v1/generic/registry.go
View File

@@ -21,7 +21,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
quota "kubesphere.io/kubesphere/kube/pkg/quota/v1"
"kubesphere.io/kubesphere/kube/pkg/quota/v1"
)
// implements a basic registry

11
kube/pkg/quota/v1/install/registry.go
View File

@@ -18,10 +18,11 @@ package install
import (
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client"
quota "kubesphere.io/kubesphere/kube/pkg/quota/v1"
core "kubesphere.io/kubesphere/kube/pkg/quota/v1/evaluator/core"
generic "kubesphere.io/kubesphere/kube/pkg/quota/v1/generic"
"kubesphere.io/kubesphere/kube/pkg/quota/v1"
"kubesphere.io/kubesphere/kube/pkg/quota/v1/evaluator/core"
"kubesphere.io/kubesphere/kube/pkg/quota/v1/generic"
)
// NewQuotaConfigurationForAdmission returns a quota configuration for admission control.
@@ -31,8 +32,8 @@ func NewQuotaConfigurationForAdmission() quota.Configuration {
}
// NewQuotaConfigurationForControllers returns a quota configuration for controllers.
func NewQuotaConfigurationForControllers(f quota.ListerForResourceFunc) quota.Configuration {
evaluators := core.NewEvaluators(f)
func NewQuotaConfigurationForControllers(client client.Client) quota.Configuration {
evaluators := core.NewEvaluators(client)
return generic.NewConfiguration(evaluators, DefaultIgnoredResources())
}

4
kube/pkg/quota/v1/interfaces.go
View File

@@ -21,7 +21,6 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/client-go/tools/cache"
)
// UsageStatsOptions is an options structs that describes how stats should be calculated
@@ -83,6 +82,3 @@ type Registry interface {
// List from registry
List() []Evaluator
}
// ListerForResourceFunc knows how to get a lister for a specific resource
type ListerForResourceFunc func(schema.GroupVersionResource) (cache.GenericLister, error)

127
kube/plugin/pkg/admission/resourcequota/admission.go
View File

@@ -1,127 +0,0 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package resourcequota
import (
"context"
"fmt"
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apiserver/pkg/admission"
genericadmissioninitializer "k8s.io/apiserver/pkg/admission/initializer"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"kubesphere.io/kubesphere/kube/pkg/quota/v1"
"kubesphere.io/kubesphere/kube/pkg/quota/v1/generic"
resourcequotaapi "kubesphere.io/kubesphere/kube/plugin/pkg/admission/resourcequota/apis/resourcequota"
)
// QuotaAdmission implements an admission controller that can enforce quota constraints
type QuotaAdmission struct {
*admission.Handler
config *resourcequotaapi.Configuration
stopCh <-chan struct{}
quotaConfiguration quota.Configuration
numEvaluators int
quotaAccessor *quotaAccessor
evaluator Evaluator
}
// WantsQuotaConfiguration defines a function which sets quota configuration for admission plugins that need it.
type WantsQuotaConfiguration interface {
SetQuotaConfiguration(quota.Configuration)
admission.InitializationValidator
}
var _ admission.ValidationInterface = &QuotaAdmission{}
var _ = genericadmissioninitializer.WantsExternalKubeInformerFactory(&QuotaAdmission{})
var _ = genericadmissioninitializer.WantsExternalKubeClientSet(&QuotaAdmission{})
var _ = WantsQuotaConfiguration(&QuotaAdmission{})
type liveLookupEntry struct {
expiry time.Time
items []*corev1.ResourceQuota
}
// NewResourceQuota configures an admission controller that can enforce quota constraints
// using the provided registry. The registry must have the capability to handle group/kinds that
// are persisted by the server this admission controller is intercepting
func NewResourceQuota(config *resourcequotaapi.Configuration, numEvaluators int, stopCh <-chan struct{}) (*QuotaAdmission, error) {
quotaAccessor, err := newQuotaAccessor()
if err != nil {
return nil, err
}
return &QuotaAdmission{
Handler: admission.NewHandler(admission.Create, admission.Update),
stopCh: stopCh,
numEvaluators: numEvaluators,
config: config,
quotaAccessor: quotaAccessor,
}, nil
}
// SetExternalKubeClientSet registers the client into QuotaAdmission
func (a *QuotaAdmission) SetExternalKubeClientSet(client kubernetes.Interface) {
a.quotaAccessor.client = client
}
// SetExternalKubeInformerFactory registers an informer factory into QuotaAdmission
func (a *QuotaAdmission) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) {
a.quotaAccessor.lister = f.Core().V1().ResourceQuotas().Lister()
}
// SetQuotaConfiguration assigns and initializes configuration and evaluator for QuotaAdmission
func (a *QuotaAdmission) SetQuotaConfiguration(c quota.Configuration) {
a.quotaConfiguration = c
a.evaluator = NewQuotaEvaluator(a.quotaAccessor, a.quotaConfiguration.IgnoredResources(), generic.NewRegistry(a.quotaConfiguration.Evaluators()), nil, a.config, a.numEvaluators, a.stopCh)
}
// ValidateInitialization ensures an authorizer is set.
func (a *QuotaAdmission) ValidateInitialization() error {
if a.quotaAccessor == nil {
return fmt.Errorf("missing quotaAccessor")
}
if a.quotaAccessor.client == nil {
return fmt.Errorf("missing quotaAccessor.client")
}
if a.quotaAccessor.lister == nil {
return fmt.Errorf("missing quotaAccessor.lister")
}
if a.quotaConfiguration == nil {
return fmt.Errorf("missing quotaConfiguration")
}
if a.evaluator == nil {
return fmt.Errorf("missing evaluator")
}
return nil
}
// Validate makes admission decisions while enforcing quota
func (a *QuotaAdmission) Validate(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces) (err error) {
// ignore all operations that correspond to sub-resource actions
if attr.GetSubresource() != "" {
return nil
}
// ignore all operations that are not namespaced
if attr.GetNamespace() == "" {
return nil
}
return a.evaluator.Evaluate(attr)
}

18
kube/plugin/pkg/admission/resourcequota/apis/resourcequota/types.go
View File

@@ -1,21 +1,3 @@
/*
Copyright 2021 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package resourcequota
import (

3
kube/plugin/pkg/admission/resourcequota/controller.go
View File

@@ -23,8 +23,6 @@ import (
"sync"
"time"
"k8s.io/klog/v2"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
@@ -35,6 +33,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/admission"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/kube/pkg/quota/v1"
"kubesphere.io/kubesphere/kube/pkg/quota/v1/generic"

121
kube/plugin/pkg/admission/resourcequota/resource_access.go
View File

@@ -17,18 +17,7 @@ limitations under the License.
package resourcequota
import (
"context"
"fmt"
"time"
lru "github.com/hashicorp/golang-lru"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apiserver/pkg/storage"
"k8s.io/client-go/kubernetes"
corev1listers "k8s.io/client-go/listers/core/v1"
)
// QuotaAccessor abstracts the get/set logic from the rest of the Evaluator. This could be a test stub, a straight passthrough,
@@ -41,113 +30,3 @@ type QuotaAccessor interface {
// GetQuotas gets all possible quotas for a given namespace
GetQuotas(namespace string) ([]corev1.ResourceQuota, error)
}
type quotaAccessor struct {
client kubernetes.Interface
// lister can list/get quota objects from a shared informer's cache
lister corev1listers.ResourceQuotaLister
// liveLookups holds the last few live lookups we've done to help ammortize cost on repeated lookup failures.
// This lets us handle the case of latent caches, by looking up actual results for a namespace on cache miss/no results.
// We track the lookup result here so that for repeated requests, we don't look it up very often.
liveLookupCache *lru.Cache
liveTTL time.Duration
// updatedQuotas holds a cache of quotas that we've updated. This is used to pull the "really latest" during back to
// back quota evaluations that touch the same quota doc. This only works because we can compare etcd resourceVersions
// for the same resource as integers. Before this change: 22 updates with 12 conflicts. after this change: 15 updates with 0 conflicts
updatedQuotas *lru.Cache
}
// newQuotaAccessor creates an object that conforms to the QuotaAccessor interface to be used to retrieve quota objects.
func newQuotaAccessor() (*quotaAccessor, error) {
liveLookupCache, err := lru.New(100)
if err != nil {
return nil, err
}
updatedCache, err := lru.New(100)
if err != nil {
return nil, err
}
// client and lister will be set when SetInternalKubeClientSet and SetInternalKubeInformerFactory are invoked
return &quotaAccessor{
liveLookupCache: liveLookupCache,
liveTTL: time.Duration(30 * time.Second),
updatedQuotas: updatedCache,
}, nil
}
func (e *quotaAccessor) UpdateQuotaStatus(newQuota *corev1.ResourceQuota) error {
updatedQuota, err := e.client.CoreV1().ResourceQuotas(newQuota.Namespace).UpdateStatus(context.TODO(), newQuota, metav1.UpdateOptions{})
if err != nil {
return err
}
key := newQuota.Namespace + "/" + newQuota.Name
e.updatedQuotas.Add(key, updatedQuota)
return nil
}
var storageVersioner = storage.APIObjectVersioner{}
// checkCache compares the passed quota against the value in the look-aside cache and returns the newer
// if the cache is out of date, it deletes the stale entry. This only works because of etcd resourceVersions
// being monotonically increasing integers
func (e *quotaAccessor) checkCache(quota *corev1.ResourceQuota) *corev1.ResourceQuota {
key := quota.Namespace + "/" + quota.Name
uncastCachedQuota, ok := e.updatedQuotas.Get(key)
if !ok {
return quota
}
cachedQuota := uncastCachedQuota.(*corev1.ResourceQuota)
if storageVersioner.CompareResourceVersion(quota, cachedQuota) >= 0 {
e.updatedQuotas.Remove(key)
return quota
}
return cachedQuota
}
func (e *quotaAccessor) GetQuotas(namespace string) ([]corev1.ResourceQuota, error) {
// determine if there are any quotas in this namespace
// if there are no quotas, we don't need to do anything
items, err := e.lister.ResourceQuotas(namespace).List(labels.Everything())
if err != nil {
return nil, fmt.Errorf("error resolving quota: %v", err)
}
// if there are no items held in our indexer, check our live-lookup LRU, if that misses, do the live lookup to prime it.
if len(items) == 0 {
lruItemObj, ok := e.liveLookupCache.Get(namespace)
if !ok || lruItemObj.(liveLookupEntry).expiry.Before(time.Now()) {
// TODO: If there are multiple operations at the same time and cache has just expired,
// this may cause multiple List operations being issued at the same time.
// If there is already in-flight List() for a given namespace, we should wait until
// it is finished and cache is updated instead of doing the same, also to avoid
// throttling - see #22422 for details.
liveList, err := e.client.CoreV1().ResourceQuotas(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return nil, err
}
newEntry := liveLookupEntry{expiry: time.Now().Add(e.liveTTL)}
for i := range liveList.Items {
newEntry.items = append(newEntry.items, &liveList.Items[i])
}
e.liveLookupCache.Add(namespace, newEntry)
lruItemObj = newEntry
}
lruEntry := lruItemObj.(liveLookupEntry)
items = append(items, lruEntry.items...)
}
resourceQuotas := []corev1.ResourceQuota{}
for i := range items {
quota := items[i]
quota = e.checkCache(quota)
// always make a copy. We're going to muck around with this and we should never mutate the originals
resourceQuotas = append(resourceQuotas, *quota)
}
return resourceQuotas, nil
}