feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions

View File

@@ -1,31 +1,18 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package node
import (
"fmt"
"sort"
"context"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/labels"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/informers"
resourceheper "k8s.io/kubectl/pkg/util/resource"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"kubesphere.io/kubesphere/pkg/api"
"kubesphere.io/kubesphere/pkg/apiserver/query"
@@ -34,123 +21,54 @@ import (
// Those annotations were added to node only for display purposes
const (
nodeCPURequests = "node.kubesphere.io/cpu-requests"
nodeMemoryRequests = "node.kubesphere.io/memory-requests"
nodeCPULimits = "node.kubesphere.io/cpu-limits"
nodeMemoryLimits = "node.kubesphere.io/memory-limits"
nodeCPURequestsFraction = "node.kubesphere.io/cpu-requests-fraction"
nodeCPULimitsFraction = "node.kubesphere.io/cpu-limits-fraction"
nodeMemoryRequestsFraction = "node.kubesphere.io/memory-requests-fraction"
nodeMemoryLimitsFraction = "node.kubesphere.io/memory-limits-fraction"
nodeConfigOK v1.NodeConditionType = "ConfigOK"
nodeKubeletReady v1.NodeConditionType = "KubeletReady"
statusRunning = "running"
statusWarning = "warning"
statusUnschedulable = "unschedulable"
nodeCPURequests = "node.kubesphere.io/cpu-requests"
nodeMemoryRequests = "node.kubesphere.io/memory-requests"
nodeCPULimits = "node.kubesphere.io/cpu-limits"
nodeMemoryLimits = "node.kubesphere.io/memory-limits"
nodeCPURequestsFraction = "node.kubesphere.io/cpu-requests-fraction"
nodeCPULimitsFraction = "node.kubesphere.io/cpu-limits-fraction"
nodeMemoryRequestsFraction = "node.kubesphere.io/memory-requests-fraction"
nodeMemoryLimitsFraction = "node.kubesphere.io/memory-limits-fraction"
nodeConfigOK corev1.NodeConditionType = "ConfigOK"
nodeKubeletReady corev1.NodeConditionType = "KubeletReady"
statusRunning = "running"
statusWarning = "warning"
statusUnschedulable = "unschedulable"
)
type nodesGetter struct {
informers informers.SharedInformerFactory
cache runtimeclient.Reader
}
func New(informers informers.SharedInformerFactory) v1alpha3.Interface {
return &nodesGetter{
informers: informers,
}
func New(cache runtimeclient.Reader) v1alpha3.Interface {
return &nodesGetter{cache: cache}
}
func (c *nodesGetter) Get(_, name string) (runtime.Object, error) {
node, err := c.informers.Core().V1().Nodes().Lister().Get(name)
if err != nil {
return nil, err
}
// ignore the error, skip annotating process if error happened
pods, _ := c.informers.Core().V1().Pods().Lister().Pods("").List(labels.Everything())
// Never mutate original objects!
// Caches are shared across controllers,
// this means that if you mutate your "copy" (actually a reference or shallow copy) of an object,
// you'll mess up other controllers (not just your own).
// Also, if the mutated field is a map,
// a "concurrent map (read &) write" panic might occur,
// causing the ks-apiserver to crash.
// Refer:
// https://github.com/kubesphere/kubesphere/issues/4357
// https://github.com/kubesphere/kubesphere/issues/3469
// https://github.com/kubesphere/kubesphere/pull/4599
// https://github.com/kubernetes/community/blob/master/contributors/devel/sig-api-machinery/controllers.md
node = node.DeepCopy()
c.annotateNode(node, pods)
return node, nil
node := &corev1.Node{}
return node, c.cache.Get(context.Background(), types.NamespacedName{Name: name}, node)
}
func (c *nodesGetter) List(_ string, q *query.Query) (*api.ListResult, error) {
nodes, err := c.informers.Core().V1().Nodes().Lister().List(q.Selector())
if err != nil {
nodes := &corev1.NodeList{}
if err := c.cache.List(context.Background(), nodes,
client.MatchingLabelsSelector{Selector: q.Selector()}); err != nil {
return nil, err
}
var filtered []*v1.Node
for _, object := range nodes {
selected := true
for field, value := range q.Filters {
if !c.filter(object, query.Filter{Field: field, Value: value}) {
selected = false
break
}
}
if selected {
filtered = append(filtered, object)
}
var result []runtime.Object
for _, item := range nodes.Items {
result = append(result, item.DeepCopy())
}
// sort by sortBy field
sort.Slice(filtered, func(i, j int) bool {
if !q.Ascending {
return c.compare(filtered[i], filtered[j], q.SortBy)
}
return !c.compare(filtered[i], filtered[j], q.SortBy)
})
total := len(filtered)
if q.Pagination == nil {
q.Pagination = query.NoPagination
}
start, end := q.Pagination.GetValidPagination(total)
selectedNodes := filtered[start:end]
// ignore the error, skip annotating process if error happened
pods, _ := c.informers.Core().V1().Pods().Lister().Pods("").List(labels.Everything())
var nonTerminatedPodsList []*v1.Pod
for _, pod := range pods {
if pod.Status.Phase != v1.PodSucceeded && pod.Status.Phase != v1.PodFailed {
nonTerminatedPodsList = append(nonTerminatedPodsList, pod)
}
}
var result = make([]interface{}, 0)
for _, node := range selectedNodes {
node = node.DeepCopy()
c.annotateNode(node, nonTerminatedPodsList)
result = append(result, node)
}
return &api.ListResult{
TotalItems: total,
Items: result,
}, nil
return v1alpha3.DefaultList(result, q, c.compare, c.filter), nil
}
func (c *nodesGetter) compare(left runtime.Object, right runtime.Object, field query.Field) bool {
leftNode, ok := left.(*v1.Node)
leftNode, ok := left.(*corev1.Node)
if !ok {
return false
}
rightNode, ok := right.(*v1.Node)
rightNode, ok := right.(*corev1.Node)
if !ok {
return false
}
@@ -159,7 +77,7 @@ func (c *nodesGetter) compare(left runtime.Object, right runtime.Object, field q
}
func (c *nodesGetter) filter(object runtime.Object, filter query.Filter) bool {
node, ok := object.(*v1.Node)
node, ok := object.(*corev1.Node)
if !ok {
return false
}
@@ -167,80 +85,10 @@ func (c *nodesGetter) filter(object runtime.Object, filter query.Filter) bool {
case query.FieldStatus:
return getNodeStatus(node) == string(filter.Value)
}
return v1alpha3.DefaultObjectMetaFilter(node.ObjectMeta, filter)
}
// annotateNode adds cpu/memory requests usage data to node's annotations
// this operation mutates the *v1.Node passed in
// so DO A DEEPCOPY before calling
func (c *nodesGetter) annotateNode(node *v1.Node, pods []*v1.Pod) {
if node.Annotations == nil {
node.Annotations = make(map[string]string)
}
if len(pods) == 0 {
return
}
var nodePods []*v1.Pod
for _, pod := range pods {
if pod.Spec.NodeName == node.Name {
nodePods = append(nodePods, pod)
}
}
reqs, limits := c.getPodsTotalRequestAndLimits(nodePods)
cpuReqs, cpuLimits, memoryReqs, memoryLimits := reqs[v1.ResourceCPU], limits[v1.ResourceCPU], reqs[v1.ResourceMemory], limits[v1.ResourceMemory]
node.Annotations[nodeCPURequests] = cpuReqs.String()
node.Annotations[nodeCPULimits] = cpuLimits.String()
node.Annotations[nodeMemoryRequests] = memoryReqs.String()
node.Annotations[nodeMemoryLimits] = memoryLimits.String()
fractionCpuReqs, fractionCpuLimits := float64(0), float64(0)
allocatable := node.Status.Allocatable
if allocatable.Cpu().MilliValue() != 0 {
fractionCpuReqs = float64(cpuReqs.MilliValue()) / float64(allocatable.Cpu().MilliValue()) * 100
fractionCpuLimits = float64(cpuLimits.MilliValue()) / float64(allocatable.Cpu().MilliValue()) * 100
}
fractionMemoryReqs, fractionMemoryLimits := float64(0), float64(0)
if allocatable.Memory().Value() != 0 {
fractionMemoryReqs = float64(memoryReqs.Value()) / float64(allocatable.Memory().Value()) * 100
fractionMemoryLimits = float64(memoryLimits.Value()) / float64(allocatable.Memory().Value()) * 100
}
node.Annotations[nodeCPURequestsFraction] = fmt.Sprintf("%d%%", int(fractionCpuReqs))
node.Annotations[nodeCPULimitsFraction] = fmt.Sprintf("%d%%", int(fractionCpuLimits))
node.Annotations[nodeMemoryRequestsFraction] = fmt.Sprintf("%d%%", int(fractionMemoryReqs))
node.Annotations[nodeMemoryLimitsFraction] = fmt.Sprintf("%d%%", int(fractionMemoryLimits))
}
func (c *nodesGetter) getPodsTotalRequestAndLimits(pods []*v1.Pod) (reqs map[v1.ResourceName]resource.Quantity, limits map[v1.ResourceName]resource.Quantity) {
reqs, limits = map[v1.ResourceName]resource.Quantity{}, map[v1.ResourceName]resource.Quantity{}
for _, pod := range pods {
podReqs, podLimits := resourceheper.PodRequestsAndLimits(pod)
for podReqName, podReqValue := range podReqs {
if value, ok := reqs[podReqName]; !ok {
reqs[podReqName] = podReqValue.DeepCopy()
} else {
value.Add(podReqValue)
reqs[podReqName] = value
}
}
for podLimitName, podLimitValue := range podLimits {
if value, ok := limits[podLimitName]; !ok {
limits[podLimitName] = podLimitValue.DeepCopy()
} else {
value.Add(podLimitValue)
limits[podLimitName] = value
}
}
}
return
}
func getNodeStatus(node *v1.Node) string {
func getNodeStatus(node *corev1.Node) string {
if node.Spec.Unschedulable {
return statusUnschedulable
}
@@ -253,17 +101,17 @@ func getNodeStatus(node *v1.Node) string {
return statusRunning
}
var expectedConditions = map[v1.NodeConditionType]v1.ConditionStatus{
v1.NodeMemoryPressure: v1.ConditionFalse,
v1.NodeDiskPressure: v1.ConditionFalse,
v1.NodePIDPressure: v1.ConditionFalse,
v1.NodeNetworkUnavailable: v1.ConditionFalse,
nodeConfigOK: v1.ConditionTrue,
nodeKubeletReady: v1.ConditionTrue,
v1.NodeReady: v1.ConditionTrue,
var expectedConditions = map[corev1.NodeConditionType]corev1.ConditionStatus{
corev1.NodeMemoryPressure: corev1.ConditionFalse,
corev1.NodeDiskPressure: corev1.ConditionFalse,
corev1.NodePIDPressure: corev1.ConditionFalse,
corev1.NodeNetworkUnavailable: corev1.ConditionFalse,
nodeConfigOK: corev1.ConditionTrue,
nodeKubeletReady: corev1.ConditionTrue,
corev1.NodeReady: corev1.ConditionTrue,
}
func isUnhealthyStatus(condition v1.NodeCondition) bool {
func isUnhealthyStatus(condition corev1.NodeCondition) bool {
expectedStatus := expectedConditions[condition.Type]
if expectedStatus != "" && condition.Status != expectedStatus {
return true

View File

@@ -1,283 +0,0 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package node
import (
"strconv"
"testing"
"github.com/google/go-cmp/cmp"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
"kubesphere.io/kubesphere/pkg/api"
"kubesphere.io/kubesphere/pkg/apiserver/query"
"kubesphere.io/kubesphere/pkg/models/resources/v1alpha3"
)
// mergeResourceLists will merge resoure lists. When two lists have the same resourece, the value from
// the last list will be present in the result
func mergeResourceLists(resourceLists ...corev1.ResourceList) corev1.ResourceList {
result := corev1.ResourceList{}
for _, rl := range resourceLists {
for resource, quantity := range rl {
result[resource] = quantity
}
}
return result
}
func getResourceList(cpu, memory string) corev1.ResourceList {
res := corev1.ResourceList{}
if cpu != "" {
res[corev1.ResourceCPU] = resource.MustParse(cpu)
}
if memory != "" {
res[corev1.ResourceMemory] = resource.MustParse(memory)
}
return res
}
var nodeAllocatable = mergeResourceLists(getResourceList("4", "12Gi"))
var node = &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
},
Status: corev1.NodeStatus{
Allocatable: nodeAllocatable,
},
}
var pods = []*corev1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
Namespace: "foo",
Name: "pod-with-resources",
},
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
},
Spec: corev1.PodSpec{
NodeName: node.Name,
Containers: []corev1.Container{
{
Name: "cpu-mem",
Image: "image:latest",
Resources: corev1.ResourceRequirements{
Requests: getResourceList("1", "1Gi"),
Limits: getResourceList("2", "2Gi"),
},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
{
ObjectMeta: metav1.ObjectMeta{
Namespace: "foo2",
Name: "pod-with-resources",
},
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
},
Spec: corev1.PodSpec{
NodeName: node.Name,
Containers: []corev1.Container{
{
Name: "cpu-mem",
Image: "image:latest",
Resources: corev1.ResourceRequirements{
Requests: getResourceList("1", "1Gi"),
Limits: getResourceList("2", "2Gi"),
},
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
},
},
}
var expectedAnnotations = map[string]string{
nodeCPURequests: "2",
nodeCPULimits: "4",
nodeCPURequestsFraction: "50%",
nodeCPULimitsFraction: "100%",
nodeMemoryRequests: "2Gi",
nodeMemoryLimits: "4Gi",
nodeMemoryRequestsFraction: "16%",
nodeMemoryLimitsFraction: "33%",
}
func TestNodesGetterGet(t *testing.T) {
fake := fake.NewSimpleClientset(node, pods[0], pods[1])
informer := informers.NewSharedInformerFactory(fake, 0)
informer.Core().V1().Nodes().Informer().GetIndexer().Add(node)
for _, pod := range pods {
informer.Core().V1().Pods().Informer().GetIndexer().Add(pod)
}
nodeGetter := New(informer)
got, err := nodeGetter.Get("", node.Name)
if err != nil {
t.Fatal(err)
}
nodeGot := got.(*corev1.Node)
if diff := cmp.Diff(nodeGot.Annotations, expectedAnnotations); len(diff) != 0 {
t.Errorf("%T, diff(-got, +expected), %v", expectedAnnotations, nodeGot.Annotations)
}
}
func TestListNodes(t *testing.T) {
tests := []struct {
query *query.Query
expected *api.ListResult
}{
{
&query.Query{
Pagination: &query.Pagination{
Limit: 1,
Offset: 0,
},
SortBy: query.FieldName,
Ascending: false,
Filters: map[query.Field]query.Value{query.FieldName: query.Value(node2.Name)},
},
&api.ListResult{
Items: []interface{}{
node2Expected,
},
TotalItems: 1,
},
},
{
&query.Query{
Pagination: &query.Pagination{
Limit: 1,
Offset: 0,
},
SortBy: query.FieldName,
Ascending: false,
Filters: map[query.Field]query.Value{query.FieldStatus: query.Value(statusUnschedulable)},
},
&api.ListResult{
Items: []interface{}{
node1Expected,
},
TotalItems: 1,
},
},
{
&query.Query{
Pagination: &query.Pagination{
Limit: 1,
Offset: 0,
},
SortBy: query.FieldName,
Ascending: false,
Filters: map[query.Field]query.Value{query.FieldStatus: query.Value(statusRunning)},
},
&api.ListResult{
Items: []interface{}{
node2Expected,
},
TotalItems: 1,
},
},
}
getter := prepare()
for index, test := range tests {
t.Run(strconv.Itoa(index), func(t *testing.T) {
got, err := getter.List("", test.query)
if err != nil {
t.Error(err)
}
if diff := cmp.Diff(got, test.expected); diff != "" {
t.Errorf("%T differ (-got, +want): %s", test.expected, diff)
}
})
}
}
var (
node1 = &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node1",
},
Spec: corev1.NodeSpec{
Unschedulable: true,
},
}
node1Expected = &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node1",
Annotations: map[string]string{},
},
Spec: corev1.NodeSpec{
Unschedulable: true,
},
}
node2 = &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node2",
},
Spec: corev1.NodeSpec{
Unschedulable: false,
},
}
node2Expected = &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node2",
Annotations: map[string]string{},
},
Spec: corev1.NodeSpec{
Unschedulable: false,
},
}
nodes = []*corev1.Node{node1, node2}
)
func prepare() v1alpha3.Interface {
fake := fake.NewSimpleClientset(node1, node2)
informer := informers.NewSharedInformerFactory(fake, 0)
for _, node := range nodes {
informer.Core().V1().Nodes().Informer().GetIndexer().Add(node)
}
return New(informer)
}