refactor monitoring modules

Signed-off-by: huanggze <loganhuang@yunify.com>

pkg/apiserver/monitoring/monitoring.go

@@ -33,13 +33,13 @@ func MonitorPod(request *restful.Request, response *restful.Response) {
 		var res *metrics.FormatedMetric
 		if !nullRule {
 			metricsStr := prometheus.SendMonitoringRequest(prometheus.PrometheusEndpoint, queryType, params)
-			res = metrics.ReformatJson(metricsStr, metricName, map[string]string{"pod_name": ""})
+			res = metrics.ReformatJson(metricsStr, metricName, map[string]string{metrics.MetricLevelPodName: ""})
 		}
 		response.WriteAsJson(res)
 
 	} else {
 		// multiple
-		rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelPod)
+		rawMetrics := metrics.GetPodLevelMetrics(requestParams)
 		// sorting
 		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
 		// paging
@@ -52,7 +52,7 @@ func MonitorContainer(request *restful.Request, response *restful.Response) {
 	requestParams := prometheus.ParseMonitoringRequestParams(request)
 	metricName := requestParams.MetricsName
 	if requestParams.MetricsFilter != "" {
-		rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelContainer)
+		rawMetrics := metrics.GetContainerLevelMetrics(requestParams)
 		// sorting
 		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
 		// paging
@@ -70,7 +70,7 @@ func MonitorContainer(request *restful.Request, response *restful.Response) {
 func MonitorWorkload(request *restful.Request, response *restful.Response) {
 	requestParams := prometheus.ParseMonitoringRequestParams(request)
 
-	rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkload)
+	rawMetrics := metrics.GetWorkloadLevelMetrics(requestParams)
 
 	// sorting
 	sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
@@ -89,7 +89,7 @@ func MonitorAllWorkspaces(request *restful.Request, response *restful.Response)
 	tp := requestParams.Tp
 	if tp == "statistics" {
 		// merge multiple metric: all-devops, all-roles, all-projects...this api is designed for admin
-		res := metrics.MonitorAllWorkspacesStatistics()
+		res := metrics.GetAllWorkspacesStatistics()
 		response.WriteAsJson(res)
 
 	} else if tp == "rank" {
@@ -114,7 +114,7 @@ func MonitorOneWorkspace(request *restful.Request, response *restful.Response) {
 	tp := requestParams.Tp
 	if tp == "rank" {
 		// multiple
-		rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkspace)
+		rawMetrics := metrics.GetWorkspaceLevelMetrics(requestParams)
 
 		// sorting
 		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
@@ -130,7 +130,7 @@ func MonitorOneWorkspace(request *restful.Request, response *restful.Response) {
 		res := metrics.MonitorOneWorkspaceStatistics(wsName)
 		response.WriteAsJson(res)
 	} else {
-		res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkspace)
+		res := metrics.GetWorkspaceLevelMetrics(requestParams)
 		response.WriteAsJson(res)
 	}
 }
@@ -138,7 +138,7 @@ func MonitorOneWorkspace(request *restful.Request, response *restful.Response) {
 func MonitorNamespace(request *restful.Request, response *restful.Response) {
 	requestParams := prometheus.ParseMonitoringRequestParams(request)
 	// multiple
-	rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelNamespace)
+	rawMetrics := metrics.GetNamespaceLevelMetrics(requestParams)
 
 	// sorting
 	sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
@@ -155,12 +155,12 @@ func MonitorCluster(request *restful.Request, response *restful.Response) {
 		// single
 		queryType, params := metrics.AssembleClusterMetricRequestInfo(requestParams, metricName)
 		metricsStr := prometheus.SendMonitoringRequest(prometheus.PrometheusEndpoint, queryType, params)
-		res := metrics.ReformatJson(metricsStr, metricName, map[string]string{"cluster": "local"})
+		res := metrics.ReformatJson(metricsStr, metricName, map[string]string{metrics.MetricLevelCluster: "local"})
 
 		response.WriteAsJson(res)
 	} else {
 		// multiple
-		res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelCluster)
+		res := metrics.GetClusterLevelMetrics(requestParams)
 		response.WriteAsJson(res)
 	}
 }
@@ -173,7 +173,7 @@ func MonitorNode(request *restful.Request, response *restful.Response) {
 		// single
 		queryType, params := metrics.AssembleNodeMetricRequestInfo(requestParams, metricName)
 		metricsStr := prometheus.SendMonitoringRequest(prometheus.PrometheusEndpoint, queryType, params)
-		res := metrics.ReformatJson(metricsStr, metricName, map[string]string{"node": ""})
+		res := metrics.ReformatJson(metricsStr, metricName, map[string]string{metrics.MetricLevelNode: ""})
 		// The raw node-exporter result doesn't include ip address information
 		// Thereby, append node ip address to .data.result[].metric
 
@@ -183,7 +183,7 @@ func MonitorNode(request *restful.Request, response *restful.Response) {
 		response.WriteAsJson(res)
 	} else {
 		// multiple
-		rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelNode)
+		rawMetrics := metrics.GetNodeLevelMetrics(requestParams)
 		nodeAddress := metrics.GetNodeAddressInfo()
 
 		for i := 0; i < len(rawMetrics.Results); i++ {
@@ -206,7 +206,7 @@ func MonitorComponent(request *restful.Request, response *restful.Response) {
 		requestParams.MetricsFilter = requestParams.ComponentName + "_.*"
 	}
 
-	rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelComponent)
+	rawMetrics := metrics.GetComponentLevelMetrics(requestParams)
 
 	response.WriteAsJson(rawMetrics)
 }

pkg/models/metrics/metrics.go

@@ -294,7 +294,7 @@ func AddNodeAddressMetric(nodeMetric *FormatedMetric, nodeAddress *map[string][]
 		metricDesc := nodeMetric.Data.Result[i][ResultItemMetric]
 		metricDescMap, ensure := metricDesc.(map[string]interface{})
 		if ensure {
-			if nodeId, exist := metricDescMap["resource_name"]; exist {
+			if nodeId, exist := metricDescMap[ResultItemMetricResourceName]; exist {
 				addr, exist := (*nodeAddress)[nodeId.(string)]
 				if exist {
 					metricDescMap["address"] = addr
@@ -307,7 +307,7 @@ func AddNodeAddressMetric(nodeMetric *FormatedMetric, nodeAddress *map[string][]
 func MonitorContainer(monitoringRequest *client.MonitoringRequestParams, metricName string) *FormatedMetric {
 	queryType, params := AssembleContainerMetricRequestInfo(monitoringRequest, metricName)
 	metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
-	res := ReformatJson(metricsStr, metricName, map[string]string{"container_name": ""})
+	res := ReformatJson(metricsStr, metricName, map[string]string{MetricLevelContainerName: ""})
 	return res
 }
 
@@ -481,8 +481,7 @@ func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, w
 
 			queryType, params := AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName)
 			metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
-			ch <- ReformatJson(metricsStr, metricName, map[string]string{"resource_name": ws}) // It's adding "resource_name" field
+			ch <- ReformatJson(metricsStr, metricName, map[string]string{ResultItemMetricResourceName: ws})
-
 			wg.Done()
 		}(metricName)
 	}
@@ -508,7 +507,7 @@ func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, w
 	wsAllch <- &metricsArray
 }
 
-func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resourceType string) *FormatedLevelMetric {
+func GetClusterLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
 	metricsFilter := monitoringRequest.MetricsFilter
 	if metricsFilter == "" {
 		metricsFilter = ".*"
@@ -517,9 +516,6 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 	var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
 	var wg sync.WaitGroup
 
-	switch resourceType {
-	case MetricLevelCluster:
-		{
 	for _, metricName := range ClusterMetricsNames {
 		matched, err := regexp.MatchString(metricsFilter, metricName)
 		if err == nil && matched {
@@ -527,14 +523,38 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 			go func(metricName string) {
 				queryType, params := AssembleClusterMetricRequestInfo(monitoringRequest, metricName)
 				metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
-						ch <- ReformatJson(metricsStr, metricName, map[string]string{"cluster": "local"})
+				ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelCluster: "local"})
 				wg.Done()
 			}(metricName)
 		}
 	}
+
+	wg.Wait()
+	close(ch)
+
+	var metricsArray []FormatedMetric
+
+	for oneMetric := range ch {
+		if oneMetric != nil {
+			metricsArray = append(metricsArray, *oneMetric)
 		}
-	case MetricLevelNode:
+	}
-		{
+
+	return &FormatedLevelMetric{
+		MetricsLevel: MetricLevelCluster,
+		Results:      metricsArray,
+	}
+}
+
+func GetNodeLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
+	metricsFilter := monitoringRequest.MetricsFilter
+	if metricsFilter == "" {
+		metricsFilter = ".*"
+	}
+
+	var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
+	var wg sync.WaitGroup
+
 	for _, metricName := range NodeMetricsNames {
 		matched, err := regexp.MatchString(metricsFilter, metricName)
 		if err == nil && matched {
@@ -542,14 +562,38 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 			go func(metricName string) {
 				queryType, params := AssembleNodeMetricRequestInfo(monitoringRequest, metricName)
 				metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
-						ch <- ReformatJson(metricsStr, metricName, map[string]string{"node": ""})
+				ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelNode: ""})
 				wg.Done()
 			}(metricName)
 		}
 	}
+
+	wg.Wait()
+	close(ch)
+
+	var metricsArray []FormatedMetric
+
+	for oneMetric := range ch {
+		if oneMetric != nil {
+			metricsArray = append(metricsArray, *oneMetric)
 		}
-	case MetricLevelWorkspace:
+	}
-		{
+
+	return &FormatedLevelMetric{
+		MetricsLevel: MetricLevelNode,
+		Results:      metricsArray,
+	}
+}
+
+func GetWorkspaceLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
+	metricsFilter := monitoringRequest.MetricsFilter
+	if metricsFilter == "" {
+		metricsFilter = ".*"
+	}
+
+	var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
+	var wg sync.WaitGroup
+
 	// a specific workspace's metrics
 	if monitoringRequest.WsName != "" {
 		namespaceArray, err := workspaces.WorkspaceNamespaces(monitoringRequest.WsName)
@@ -581,7 +625,7 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 
 							queryType, params := AssembleNamespaceMetricRequestInfoByNamesapce(monitoringRequest, namespace, metricName)
 							metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
-									chForOneMetric <- ReformatJson(metricsStr, metricName, map[string]string{"resource_name": namespace})
+							chForOneMetric <- ReformatJson(metricsStr, metricName, map[string]string{ResultItemMetricResourceName: namespace})
 							wgForOneMetric.Done()
 						}(metricName, ns)
 					}
@@ -590,13 +634,13 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 					close(chForOneMetric)
 
 					// ranking is for vector type result only
-							aggregatedResult := FormatedMetric{MetricName: metricName, Status: "success", Data: FormatedMetricData{Result: []map[string]interface{}{}, ResultType: ResultTypeVector}}
+					aggregatedResult := FormatedMetric{MetricName: metricName, Status: MetricStatusSuccess, Data: FormatedMetricData{Result: []map[string]interface{}{}, ResultType: ResultTypeVector}}
 
 					for oneMetric := range chForOneMetric {
 
 						if oneMetric != nil {
 
-									// wrapper layer 1: append .data.result[0]
+							// append .data.result[0]
 							if len(oneMetric.Data.Result) > 0 {
 								aggregatedResult.Data.Result = append(aggregatedResult.Data.Result, oneMetric.Data.Result[0])
 							}
@@ -625,7 +669,7 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 					go func(metricName string, workspace string) {
 						queryType, params := AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName)
 						metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
-								ch <- ReformatJson(metricsStr, metricName, map[string]string{"resource_name": workspace})
+						ch <- ReformatJson(metricsStr, metricName, map[string]string{ResultItemMetricResourceName: workspace})
 						wg.Done()
 					}(metricName, workspace)
 				}
@@ -633,7 +677,6 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 		}
 	} else {
 		// sum all workspaces
-
 		for _, metricName := range WorkspaceMetricsNames {
 			matched, err := regexp.MatchString(metricsFilter, metricName)
 			if err == nil && matched {
@@ -643,16 +686,40 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 				go func(metricName string) {
 					queryType, params := AssembleAllWorkspaceMetricRequestInfo(monitoringRequest, nil, metricName)
 					metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
-							ch <- ReformatJson(metricsStr, metricName, map[string]string{"workspace": "workspaces"})
+					ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelWorkspace: "workspaces"})
 
 					wg.Done()
 				}(metricName)
 			}
 		}
 	}
+
+	wg.Wait()
+	close(ch)
+
+	var metricsArray []FormatedMetric
+
+	for oneMetric := range ch {
+		if oneMetric != nil {
+			metricsArray = append(metricsArray, *oneMetric)
 		}
-	case MetricLevelNamespace:
+	}
-		{
+
+	return &FormatedLevelMetric{
+		MetricsLevel: MetricLevelWorkspace,
+		Results:      metricsArray,
+	}
+}
+
+func GetNamespaceLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
+	metricsFilter := monitoringRequest.MetricsFilter
+	if metricsFilter == "" {
+		metricsFilter = ".*"
+	}
+
+	var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
+	var wg sync.WaitGroup
+
 	for _, metricName := range NamespaceMetricsNames {
 		matched, err := regexp.MatchString(metricsFilter, metricName)
 		if err == nil && matched {
@@ -662,16 +729,40 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 				queryType, params := AssembleNamespaceMetricRequestInfo(monitoringRequest, metricName)
 				metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
 
-						rawResult := ReformatJson(metricsStr, metricName, map[string]string{"namespace": ""})
+				rawResult := ReformatJson(metricsStr, metricName, map[string]string{MetricLevelNamespace: ""})
 				ch <- rawResult
 
 				wg.Done()
 			}(metricName)
 		}
 	}
+
+	wg.Wait()
+	close(ch)
+
+	var metricsArray []FormatedMetric
+
+	for oneMetric := range ch {
+		if oneMetric != nil {
+			metricsArray = append(metricsArray, *oneMetric)
 		}
-	case MetricLevelWorkload:
+	}
-		{
+
+	return &FormatedLevelMetric{
+		MetricsLevel: MetricLevelNamespace,
+		Results:      metricsArray,
+	}
+}
+
+func GetWorkloadLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
+	metricsFilter := monitoringRequest.MetricsFilter
+	if metricsFilter == "" {
+		metricsFilter = ".*"
+	}
+
+	var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
+	var wg sync.WaitGroup
+
 	if monitoringRequest.WorkloadName == "" {
 		for _, metricName := range WorkloadMetricsNames {
 			matched, err := regexp.MatchString(metricsFilter, metricName)
@@ -680,7 +771,7 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 				go func(metricName string) {
 					queryType, params := AssembleAllWorkloadMetricRequestInfo(monitoringRequest, metricName)
 					metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
-							reformattedResult := ReformatJson(metricsStr, metricName, map[string]string{"workload": ""})
+					reformattedResult := ReformatJson(metricsStr, metricName, map[string]string{MetricLevelWorkload: ""})
 					// no need to append a null result
 					ch <- reformattedResult
 					wg.Done()
@@ -697,7 +788,7 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 					queryType, params, nullRule := AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest, metricName)
 					if !nullRule {
 						metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
-								fmtMetrics := ReformatJson(metricsStr, metricName, map[string]string{"pod_name": ""})
+						fmtMetrics := ReformatJson(metricsStr, metricName, map[string]string{MetricLevelPodName: ""})
 						unifyMetricHistoryTimeRange(fmtMetrics)
 						ch <- fmtMetrics
 					}
@@ -706,9 +797,33 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 			}
 		}
 	}
+
+	wg.Wait()
+	close(ch)
+
+	var metricsArray []FormatedMetric
+
+	for oneMetric := range ch {
+		if oneMetric != nil {
+			metricsArray = append(metricsArray, *oneMetric)
 		}
-	case MetricLevelPod:
+	}
-		{
+
+	return &FormatedLevelMetric{
+		MetricsLevel: MetricLevelWorkload,
+		Results:      metricsArray,
+	}
+}
+
+func GetPodLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
+	metricsFilter := monitoringRequest.MetricsFilter
+	if metricsFilter == "" {
+		metricsFilter = ".*"
+	}
+
+	var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
+	var wg sync.WaitGroup
+
 	for _, metricName := range PodMetricsNames {
 		matched, err := regexp.MatchString(metricsFilter, metricName)
 		if err == nil && matched {
@@ -717,7 +832,7 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 				queryType, params, nullRule := AssemblePodMetricRequestInfo(monitoringRequest, metricName)
 				if !nullRule {
 					metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
-							ch <- ReformatJson(metricsStr, metricName, map[string]string{"pod_name": ""})
+					ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelPodName: ""})
 				} else {
 					ch <- nil
 				}
@@ -725,9 +840,33 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 			}(metricName)
 		}
 	}
+
+	wg.Wait()
+	close(ch)
+
+	var metricsArray []FormatedMetric
+
+	for oneMetric := range ch {
+		if oneMetric != nil {
+			metricsArray = append(metricsArray, *oneMetric)
 		}
-	case MetricLevelContainer:
+	}
-		{
+
+	return &FormatedLevelMetric{
+		MetricsLevel: MetricLevelPod,
+		Results:      metricsArray,
+	}
+}
+
+func GetContainerLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
+	metricsFilter := monitoringRequest.MetricsFilter
+	if metricsFilter == "" {
+		metricsFilter = ".*"
+	}
+
+	var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
+	var wg sync.WaitGroup
+
 	for _, metricName := range ContainerMetricsNames {
 		matched, err := regexp.MatchString(metricsFilter, metricName)
 		if err == nil && matched {
@@ -735,14 +874,38 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 			go func(metricName string) {
 				queryType, params := AssembleContainerMetricRequestInfo(monitoringRequest, metricName)
 				metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
-						ch <- ReformatJson(metricsStr, metricName, map[string]string{"container_name": ""})
+				ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelContainerName: ""})
 				wg.Done()
 			}(metricName)
 		}
 	}
+
+	wg.Wait()
+	close(ch)
+
+	var metricsArray []FormatedMetric
+
+	for oneMetric := range ch {
+		if oneMetric != nil {
+			metricsArray = append(metricsArray, *oneMetric)
 		}
-	case MetricLevelComponent:
+	}
-		{
+
+	return &FormatedLevelMetric{
+		MetricsLevel: MetricLevelContainer,
+		Results:      metricsArray,
+	}
+}
+
+func GetComponentLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
+	metricsFilter := monitoringRequest.MetricsFilter
+	if metricsFilter == "" {
+		metricsFilter = ".*"
+	}
+
+	var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
+	var wg sync.WaitGroup
+
 	for _, metricName := range ComponentMetricsNames {
 		matched, err := regexp.MatchString(metricsFilter, metricName)
 		if err == nil && matched {
@@ -750,9 +913,9 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 			go func(metricName string) {
 				queryType, params := AssembleComponentRequestInfo(monitoringRequest, metricName)
 				metricsStr := client.SendMonitoringRequest(client.SecondaryPrometheusEndpoint, queryType, params)
-						formattedJson := ReformatJson(metricsStr, metricName, map[string]string{"resource_name": monitoringRequest.ComponentName})
+				formattedJson := ReformatJson(metricsStr, metricName, map[string]string{ResultItemMetricResourceName: monitoringRequest.ComponentName})
 
-						if metricName == "etcd_server_list" {
+				if metricName == EtcdServerList {
 
 					nodeMap := make(map[string]string, 0)
 
@@ -787,8 +950,6 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 			}(metricName)
 		}
 	}
-		}
-	}
 
 	wg.Wait()
 	close(ch)
@@ -801,14 +962,13 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 		}
 	}
 
-	// wrapper layer 2:
 	return &FormatedLevelMetric{
-		MetricsLevel: resourceType,
+		MetricsLevel: MetricLevelComponent,
 		Results:      metricsArray,
 	}
 }
 
-func MonitorAllWorkspacesStatistics() *FormatedLevelMetric {
+func GetAllWorkspacesStatistics() *FormatedLevelMetric {
 
 	wg := sync.WaitGroup{}
 	var metricsArray []FormatedMetric

pkg/models/metrics/metricsruleconst.go

@@ -21,6 +21,7 @@ const (
 	MetricStatusSuccess          = "success"
 	ResultItemMetric             = "metric"
 	ResultItemMetricResource     = "resource"
+	ResultItemMetricResourceName = "resource_name"
 	ResultItemMetricNodeIp       = "node_ip"
 	ResultItemMetricNodeName     = "node_name"
 	ResultItemValue              = "value"
@@ -90,6 +91,12 @@ const (
 	WorkspaceJoinedKey = "label_kubesphere_io_workspace"
 )
 
+// The metrics need to include extra info out of prometheus
+// eg. add node name info to the etcd_server_list metric
+const (
+	EtcdServerList = "etcd_server_list"
+)
+
 type MetricMap map[string]string
 
 var ClusterMetricsNames = []string{

pkg/models/metrics/namespaces.go

@@ -41,14 +41,14 @@ func GetNamespacesWithMetrics(namespaces []*v1.Namespace) []*v1.Namespace {
 		MetricsFilter:   "namespace_cpu_usage|namespace_memory_usage_wo_cache|namespace_pod_count",
 	}
 
-	rawMetrics := MonitorAllMetrics(&params, MetricLevelNamespace)
+	rawMetrics := GetNamespaceLevelMetrics(&params)
 
 	for _, result := range rawMetrics.Results {
 		for _, data := range result.Data.Result {
-			metricDescMap, ok := data["metric"].(map[string]interface{})
+			metricDescMap, ok := data[ResultItemMetric].(map[string]interface{})
 			if ok {
-				if ns, exist := metricDescMap["resource_name"]; exist {
+				if ns, exist := metricDescMap[ResultItemMetricResourceName]; exist {
-					timeAndValue, ok := data["value"].([]interface{})
+					timeAndValue, ok := data[ResultItemValue].([]interface{})
 					if ok && len(timeAndValue) == 2 {
 						for i := 0; i < len(namespaces); i++ {
 							if namespaces[i].Name == ns {

pkg/models/metrics/util.go

@@ -89,8 +89,8 @@ func Sort(sortMetricName string, sortType string, rawMetrics *FormatedLevelMetri
 						v1, _ := strconv.ParseFloat(value1[len(value1)-1].(string), 64)
 						v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64)
 						if v1 == v2 {
-							resourceName1 := (*p)[ResultItemMetric].(map[string]interface{})["resource_name"]
+							resourceName1 := (*p)[ResultItemMetric].(map[string]interface{})[ResultItemMetricResourceName]
-							resourceName2 := (*q)[ResultItemMetric].(map[string]interface{})["resource_name"]
+							resourceName2 := (*q)[ResultItemMetric].(map[string]interface{})[ResultItemMetricResourceName]
 							return resourceName1.(string) < resourceName2.(string)
 						}
 
@@ -105,8 +105,8 @@ func Sort(sortMetricName string, sortType string, rawMetrics *FormatedLevelMetri
 						v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64)
 
 						if v1 == v2 {
-							resourceName1 := (*p)[ResultItemMetric].(map[string]interface{})["resource_name"]
+							resourceName1 := (*p)[ResultItemMetric].(map[string]interface{})[ResultItemMetricResourceName]
-							resourceName2 := (*q)[ResultItemMetric].(map[string]interface{})["resource_name"]
+							resourceName2 := (*q)[ResultItemMetric].(map[string]interface{})[ResultItemMetricResourceName]
 							return resourceName1.(string) > resourceName2.(string)
 						}
 
@@ -116,8 +116,8 @@ func Sort(sortMetricName string, sortType string, rawMetrics *FormatedLevelMetri
 
 				for _, r := range metricItem.Data.Result {
 					// record the ordering of resource_name to indexMap
-					// example: {"metric":{"resource_name": "Deployment:xxx"},"value":[1541142931.731,"3"]}
+					// example: {"metric":{ResultItemMetricResourceName: "Deployment:xxx"},"value":[1541142931.731,"3"]}
-					resourceName, exist := r[ResultItemMetric].(map[string]interface{})["resource_name"]
+					resourceName, exist := r[ResultItemMetric].(map[string]interface{})[ResultItemMetricResourceName]
 					if exist {
 						if _, exist := indexMap[resourceName.(string)]; !exist {
 							indexMap[resourceName.(string)] = i
@@ -129,7 +129,7 @@ func Sort(sortMetricName string, sortType string, rawMetrics *FormatedLevelMetri
 
 			// iterator all metric to find max metricItems length
 			for _, r := range metricItem.Data.Result {
-				k, ok := r[ResultItemMetric].(map[string]interface{})["resource_name"]
+				k, ok := r[ResultItemMetric].(map[string]interface{})[ResultItemMetricResourceName]
 				if ok {
 					currentResourceMap[k.(string)] = 1
 				}
@@ -158,7 +158,7 @@ func Sort(sortMetricName string, sortType string, rawMetrics *FormatedLevelMetri
 			sortedMetric := make([]map[string]interface{}, len(indexMap))
 			for j := 0; j < len(re.Data.Result); j++ {
 				r := re.Data.Result[j]
-				k, exist := r[ResultItemMetric].(map[string]interface{})["resource_name"]
+				k, exist := r[ResultItemMetric].(map[string]interface{})[ResultItemMetricResourceName]
 				if exist {
 					index, exist := indexMap[k.(string)]
 					if exist {
@@ -290,9 +290,9 @@ func ReformatJson(metric string, metricsName string, needAddParams map[string]st
 				for n := range needAddParams {
 					if v, ok := metricMap[n]; ok {
 						delete(metricMap, n)
-						metricMap["resource_name"] = v
+						metricMap[ResultItemMetricResourceName] = v
 					} else {
-						metricMap["resource_name"] = needAddParams[n]
+						metricMap[ResultItemMetricResourceName] = needAddParams[n]
 					}
 				}
 			}