admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3698 from zhu733756/fix-bug-for-pod-metrics-on-edge-node

Browse Source 
fix bug where the edge node container groups tab could not see pod metrics
This commit is contained in:
KubeSphere CI Bot
2021-04-11 13:32:01 +08:00
committed by GitHub
parent 0f3e48ba30 f6dd42f494
commit f75e049187
12 changed files with 431 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
pkg/simple/client/monitoring/metricsserver/metricsserver.go
View File

@@ -110,13 +110,22 @@ func (m metricsServer) parseEdgePods(opts *monitoring.QueryOptions) map[string]b
edgePods := make(map[string]bool)
var filters []string
r, _ := regexp.Compile(`\s*\|\s*|\$`)
filters := r.Split(opts.ResourceFilter, -1)
if opts.ResourceFilter != "" {
filters = r.Split(opts.ResourceFilter, -1)
}
if opts.NamespacedResourcesFilter != "" {
filters = r.Split(opts.NamespacedResourcesFilter, -1)
}
for _, p := range filters {
if p != "" {
edgePods[p] = true
if p == "" || p == ".*" {
continue
}
edgePods[p] = true
}
return edgePods
@@ -140,8 +149,10 @@ func (m metricsServer) getNodeMetricsFromMetricsAPI() (*metricsapi.NodeMetricsLi
}
// pods metrics of edge nodes
func (m metricsServer) getPodMetricsFromMetricsAPI(edgePods map[string]bool, podName string, ns string) ([]metricsapi.PodMetrics, error) {
func (m metricsServer) getPodMetricsFromMetricsAPI(edgePods map[string]bool, opts *monitoring.QueryOptions) ([]metricsapi.PodMetrics, error) {
mc := m.metricsClient.MetricsV1beta1()
podName := opts.PodName
ns := opts.NamespaceName
// single pod request
if ns != "" && podName != "" {
@@ -160,23 +171,10 @@ func (m metricsServer) getPodMetricsFromMetricsAPI(edgePods map[string]bool, pod
return []metricsapi.PodMetrics{*metrics}, nil
}
if len(edgePods) == 0 {
return nil, nil
}
var isNamespacedEdgePod bool
for p, _ := range edgePods {
if ok := strings.Contains(p, "/"); ok {
isNamespacedEdgePod = true
}
break
}
combinedPodMetrics := []metricsapi.PodMetrics{}
// handle cases with when edgePodName contains namespaceName
if isNamespacedEdgePod {
if opts.NamespacedResourcesFilter != "" {
for p, _ := range edgePods {
splitedPodName := strings.Split(p, "/")
ns, p = strings.ReplaceAll(splitedPodName[0], " ", ""), strings.ReplaceAll(splitedPodName[1], " ", "")
@@ -390,6 +388,15 @@ func (m metricsServer) GetNodeLevelNamedMetrics(metrics []string, ts time.Time,
metricValues[enm].Metadata["role"] = "edge"
}
for _, addr := range status[m.Name].Addresses {
if addr.Type == v1.NodeInternalIP {
for _, enm := range edgeNodeMetrics {
metricValues[enm].Metadata["host_ip"] = addr.Address
}
break
}
}
for k, v := range metricsMap {
switch k {
case metricsNodeCPUUsage:
@@ -445,7 +452,7 @@ func (m metricsServer) GetPodLevelNamedMetrics(metrics []string, ts time.Time, o
return res
}
podMetricsFromMetricsAPI, err := m.getPodMetricsFromMetricsAPI(edgePods, opts.PodName, opts.NamespaceName)
podMetricsFromMetricsAPI, err := m.getPodMetricsFromMetricsAPI(edgePods, opts)
if err != nil {
klog.Errorf("Get pod metrics of edge nodes error %v\n", err)
return m.parseErrorResp(metrics, err)
@@ -668,7 +675,7 @@ func (m metricsServer) GetPodLevelNamedMetricsOverTime(metrics []string, start,
return res
}
podMetricsFromMetricsAPI, err := m.getPodMetricsFromMetricsAPI(edgePods, opts.PodName, opts.NamespaceName)
podMetricsFromMetricsAPI, err := m.getPodMetricsFromMetricsAPI(edgePods, opts)
if err != nil {
klog.Errorf("Get pod metrics of edge nodes error %v\n", err)
return m.parseErrorResp(metrics, err)

111
pkg/simple/client/monitoring/metricsserver/metricsserver_test.go
View File

@@ -2,7 +2,6 @@ package metricsserver
import (
"fmt"
"strings"
"testing"
"time"
@@ -75,14 +74,14 @@ var node2 = &v1.Node{
var pod1 = &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
Namespace: "kubeedge",
Namespace: "ns1",
},
}
var pod2 = &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod2",
Namespace: "kubeedge",
Namespace: "ns2",
},
}
@@ -135,7 +134,7 @@ var (
},
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
Namespace: "kubeedge",
Namespace: "ns1",
},
Timestamp: metav1.Time{Time: metricsTime},
Window: metav1.Duration{Duration: time.Minute},
@@ -159,7 +158,7 @@ var (
},
ObjectMeta: metav1.ObjectMeta{
Name: "pod2",
Namespace: "kubeedge",
Namespace: "ns2",
},
Timestamp: metav1.Time{Time: metricsTime},
Window: metav1.Duration{Duration: time.Minute},
@@ -215,6 +214,7 @@ func TestGetNamedMetrics(t *testing.T) {
podMetricsTests := []struct {
metrics []string
filter string
namespacedFilter string
expected string
podName string
namespaceName string
@@ -222,13 +222,15 @@ func TestGetNamedMetrics(t *testing.T) {
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "pod1$",
namespacedFilter: "",
expected: "metrics-vector-4.json",
podName: "",
namespaceName: "",
},
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "default/pod2$",
filter: "pod1|pod2$",
namespacedFilter: "",
expected: "metrics-vector-5.json",
podName: "",
namespaceName: "",
@@ -236,9 +238,26 @@ func TestGetNamedMetrics(t *testing.T) {
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "",
namespacedFilter: "ns2/pod2$",
expected: "metrics-vector-6.json",
podName: "",
namespaceName: "",
},
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "",
namespacedFilter: "ns1/pod1|ns2/pod2$",
expected: "metrics-vector-7.json",
podName: "",
namespaceName: "",
},
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "",
namespacedFilter: "",
expected: "metrics-vector-8.json",
podName: "pod1",
namespaceName: "kubeedge",
namespaceName: "ns1",
},
}
@@ -292,18 +311,38 @@ func TestGetNamedMetrics(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if tt.podName == "pod1" || strings.Contains(tt.filter, "pod1") {
fakeMetricsclient.ClearActions()
if tt.podName == "pod1" || tt.filter == "pod1$" || tt.namespacedFilter == "ns1/pod1$" {
fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
return true, &podMetric1, nil
})
} else {
} else if tt.podName == "pod2" || tt.filter == "pod2$" || tt.namespacedFilter == "ns2/pod2$" {
fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
return true, &podMetric2, nil
})
} else {
fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
ns := action.GetNamespace()
if ns == "ns1" {
return true, &podMetric1, nil
} else if ns == "ns2" {
return true, &podMetric2, nil
} else {
return true, &metricsV1beta1.PodMetricsList{}, nil
}
})
}
client := NewMetricsServer(fakeK8sClient, true, fakeMetricsclient)
result := client.GetNamedMetrics(tt.metrics, time.Now(), monitoring.PodOption{ResourceFilter: tt.filter, PodName: tt.podName, NamespaceName: tt.namespaceName})
result := client.GetNamedMetrics(
tt.metrics,
time.Now(),
monitoring.PodOption{
ResourceFilter: tt.filter,
NamespacedResourcesFilter: tt.namespacedFilter,
PodName: tt.podName,
NamespaceName: tt.namespaceName,
})
if diff := cmp.Diff(result, expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", expected, diff)
}
@@ -337,6 +376,7 @@ func TestGetNamedMetricsOverTime(t *testing.T) {
podMetricsTests := []struct {
metrics []string
filter string
namespacedFilter string
expected string
podName string
namespaceName string
@@ -344,13 +384,15 @@ func TestGetNamedMetricsOverTime(t *testing.T) {
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "pod1$",
namespacedFilter: "",
expected: "metrics-matrix-4.json",
podName: "",
namespaceName: "",
},
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "default/pod2$",
filter: "pod1|pod2$",
namespacedFilter: "",
expected: "metrics-matrix-5.json",
podName: "",
namespaceName: "",
@@ -358,9 +400,26 @@ func TestGetNamedMetricsOverTime(t *testing.T) {
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "",
namespacedFilter: "ns2/pod2$",
expected: "metrics-matrix-6.json",
podName: "",
namespaceName: "",
},
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "",
namespacedFilter: "ns1/pod1|ns2/pod2$",
expected: "metrics-matrix-7.json",
podName: "",
namespaceName: "",
},
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "",
namespacedFilter: "",
expected: "metrics-matrix-8.json",
podName: "pod1",
namespaceName: "kubeedge",
namespaceName: "ns1",
},
}
@@ -406,6 +465,7 @@ func TestGetNamedMetricsOverTime(t *testing.T) {
}
for i, tt := range podMetricsTests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
expected := make([]monitoring.Metric, 0)
err := jsonFromFile(tt.expected, &expected)
@@ -413,18 +473,39 @@ func TestGetNamedMetricsOverTime(t *testing.T) {
t.Fatal(err)
}
if tt.podName == "pod1" || strings.Contains(tt.filter, "pod1") {
fakeMetricsclient.ClearActions()
if tt.podName == "pod1" || tt.filter == "pod1$" || tt.namespacedFilter == "ns1/pod1$" {
fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
return true, &podMetric1, nil
})
} else {
} else if tt.podName == "pod2" || tt.filter == "pod2$" || tt.namespacedFilter == "ns2/pod2$" {
fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
return true, &podMetric2, nil
})
} else {
fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
ns := action.GetNamespace()
if ns == "ns1" {
return true, &podMetric1, nil
} else if ns == "ns2" {
return true, &podMetric2, nil
} else {
return true, &metricsV1beta1.PodMetricsList{}, nil
}
})
}
client := NewMetricsServer(fakeK8sClient, true, fakeMetricsclient)
result := client.GetNamedMetricsOverTime(tt.metrics, time.Now().Add(-time.Minute*3), time.Now(), time.Minute, monitoring.PodOption{ResourceFilter: tt.filter, PodName: tt.podName, NamespaceName: tt.namespaceName})
result := client.GetNamedMetricsOverTime(
tt.metrics, time.Now().Add(-time.Minute*3),
time.Now(),
time.Minute,
monitoring.PodOption{
ResourceFilter: tt.filter,
NamespacedResourcesFilter: tt.namespacedFilter,
PodName: tt.podName,
NamespaceName: tt.namespaceName,
})
if diff := cmp.Diff(result, expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", expected, diff)
}

4
pkg/simple/client/monitoring/metricsserver/testdata/metrics-matrix-4.json vendored
View File

@@ -6,7 +6,7 @@
"result": [
{
"metric": {
"namespace": "kubeedge",
"namespace": "ns1",
"pod":"pod1"
},
"values":[
@@ -26,7 +26,7 @@
"result": [
{
"metric": {
"namespace": "kubeedge",
"namespace": "ns1",
"pod":"pod1"
},
"values": [

28
pkg/simple/client/monitoring/metricsserver/testdata/metrics-matrix-5.json vendored
View File

@@ -6,7 +6,19 @@
"result": [
{
"metric": {
"namespace": "kubeedge",
"namespace": "ns1",
"pod":"pod1"
},
"values":[
[
1616675696,
"0.001"
]
]
},
{
"metric": {
"namespace": "ns2",
"pod":"pod2"
},
"values": [
@@ -26,7 +38,19 @@
"result": [
{
"metric": {
"namespace": "kubeedge",
"namespace": "ns1",
"pod":"pod1"
},
"values": [
[
1616675696,
"1"
]
]
},
{
"metric": {
"namespace": "ns2",
"pod":"pod2"
},
"values": [

12
pkg/simple/client/monitoring/metricsserver/testdata/metrics-matrix-6.json vendored
View File

@@ -6,13 +6,13 @@
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod1"
"namespace": "ns2",
"pod":"pod2"
},
"values": [
[
1616675696,
"0.001"
"0.002"
]
]
}
@@ -26,13 +26,13 @@
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod1"
"namespace": "ns2",
"pod":"pod2"
},
"values": [
[
1616675696,
"1"
"2"
]
]
}

66
pkg/simple/client/monitoring/metricsserver/testdata/metrics-matrix-7.json vendored Normal file
View File

@@ -0,0 +1,66 @@
[
{
"metric_name": "pod_cpu_usage",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"namespace": "ns1",
"pod":"pod1"
},
"values":[
[
1616675696,
"0.001"
]
]
},
{
"metric": {
"namespace": "ns2",
"pod":"pod2"
},
"values": [
[
1616675696,
"0.002"
]
]
}
]
}
},
{
"metric_name": "pod_memory_usage_wo_cache",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"namespace": "ns1",
"pod":"pod1"
},
"values": [
[
1616675696,
"1"
]
]
},
{
"metric": {
"namespace": "ns2",
"pod":"pod2"
},
"values": [
[
1616675696,
"2"
]
]
}
]
}
}
]

42
pkg/simple/client/monitoring/metricsserver/testdata/metrics-matrix-8.json vendored Normal file
View File

@@ -0,0 +1,42 @@
[
{
"metric_name": "pod_cpu_usage",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"namespace": "ns1",
"pod":"pod1"
},
"values":[
[
1616675696,
"0.001"
]
]
}
]
}
},
{
"metric_name": "pod_memory_usage_wo_cache",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"namespace": "ns1",
"pod":"pod1"
},
"values": [
[
1616675696,
"1"
]
]
}
]
}
}
]

4
pkg/simple/client/monitoring/metricsserver/testdata/metrics-vector-4.json vendored
View File

@@ -6,7 +6,7 @@
"result": [
{
"metric": {
"namespace": "kubeedge",
"namespace": "ns1",
"pod":"pod1"
},
"value": [
@@ -24,7 +24,7 @@
"result": [
{
"metric": {
"namespace": "kubeedge",
"namespace": "ns1",
"pod":"pod1"
},
"value": [

24
pkg/simple/client/monitoring/metricsserver/testdata/metrics-vector-5.json vendored
View File

@@ -6,7 +6,17 @@
"result": [
{
"metric": {
"namespace": "kubeedge",
"namespace": "ns1",
"pod":"pod1"
},
"value": [
1616675696,
"0.001"
]
},
{
"metric": {
"namespace": "ns2",
"pod":"pod2"
},
"value": [
@@ -24,7 +34,17 @@
"result": [
{
"metric": {
"namespace": "kubeedge",
"namespace": "ns1",
"pod":"pod1"
},
"value": [
1616675696,
"1"
]
},
{
"metric": {
"namespace": "ns2",
"pod":"pod2"
},
"value": [

12
pkg/simple/client/monitoring/metricsserver/testdata/metrics-vector-6.json vendored
View File

@@ -6,12 +6,12 @@
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod1"
"namespace": "ns2",
"pod":"pod2"
},
"value": [
1616675696,
"0.001"
"0.002"
]
}
]
@@ -24,12 +24,12 @@
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod1"
"namespace": "ns2",
"pod":"pod2"
},
"value": [
1616675696,
"1"
"2"
]
}
]

58
pkg/simple/client/monitoring/metricsserver/testdata/metrics-vector-7.json vendored Normal file
View File

@@ -0,0 +1,58 @@
[
{
"metric_name": "pod_cpu_usage",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"namespace": "ns1",
"pod":"pod1"
},
"value": [
1616675696,
"0.001"
]
},
{
"metric": {
"namespace": "ns2",
"pod":"pod2"
},
"value": [
1616675696,
"0.002"
]
}
]
}
},
{
"metric_name": "pod_memory_usage_wo_cache",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"namespace": "ns1",
"pod":"pod1"
},
"value": [
1616675696,
"1"
]
},
{
"metric": {
"namespace": "ns2",
"pod":"pod2"
},
"value": [
1616675696,
"2"
]
}
]
}
}
]

38
pkg/simple/client/monitoring/metricsserver/testdata/metrics-vector-8.json vendored Normal file
View File

@@ -0,0 +1,38 @@
[
{
"metric_name": "pod_cpu_usage",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"namespace": "ns1",
"pod":"pod1"
},
"value": [
1616675696,
"0.001"
]
}
]
}
},
{
"metric_name": "pod_memory_usage_wo_cache",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"namespace": "ns1",
"pod":"pod1"
},
"value": [
1616675696,
"1"
]
}
]
}
}
]