kubesphere/pkg/models/metrics/metrics.go

/*
Copyright 2018 The KubeSphere Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
	"encoding/json"
	"fmt"
	"net/url"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"k8s.io/api/core/v1"
	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"kubesphere.io/kubesphere/pkg/client"
	"kubesphere.io/kubesphere/pkg/models"
	"kubesphere.io/kubesphere/pkg/models/workspaces"
)

var nodeStatusDelLables = []string{"endpoint", "instance", "job", "namespace", "pod", "service"}

const (
	ChannelMaxCapacityWorkspaceMetric = 400
	ChannelMaxCapacity                = 100
)

type PagedFormatedLevelMetric struct {
	CurrentPage int                 `json:"page"`
	TotalPage   int                 `json:"total_page"`
	Message     string              `json:"msg"`
	Metric      FormatedLevelMetric `json:"metrics"`
}

type FormatedLevelMetric struct {
	MetricsLevel string           `json:"metrics_level"`
	Results      []FormatedMetric `json:"results"`
}

type FormatedMetric struct {
	MetricName string             `json:"metric_name, omitempty"`
	Status     string             `json:"status"`
	Data       FormatedMetricData `json:"data, omitempty"`
}

type FormatedMetricData struct {
	Result     []map[string]interface{} `json:"result"`
	ResultType string                   `json:"resultType"`
}

type MetricResultValues []MetricResultValue

type MetricResultValue struct {
	timestamp float64
	value     string
}

type MetricItem struct {
	MetricLabel map[string]string `json:"metric"`
	Value       []interface{}     `json:"value"`
}

type CommonMetricsResult struct {
	Status string            `json:"status"`
	Data   CommonMetricsData `json:"data"`
}

type CommonMetricsData struct {
	Result     []CommonResultItem `json:"result"`
	ResultType string             `json:"resultType"`
}

type CommonResultItem struct {
	KubePodMetric KubePodMetric `json:"metric"`
	Value         interface{}   `json:"value"`
}

type KubePodMetric struct {
	CreatedByKind string `json:"created_by_kind"`
	CreatedByName string `json:"created_by_name"`
	Namespace     string `json:"namespace"`
	Pod           string `json:"pod"`
}

type ComponentStatus struct {
	Name            string               `json:"metric_name,omitempty"`
	Namespace       string               `json:"namespace,omitempty"`
	Labels          map[string]string    `json:"labels,omitempty"`
	ComponentStatus []OneComponentStatus `json:"component"`
}

type OneComponentStatus struct {
	// Valid value: "Healthy"
	Type string `json:"type"`
	// Valid values for "Healthy": "True", "False", or "Unknown".
	Status string `json:"status"`
	// Message about the condition for a component.
	Message string `json:"message,omitempty"`
	// Condition error code for a component.
	Error string `json:"error,omitempty"`
}

func getPodNameRegexInWorkload(res string) string {

	data := []byte(res)
	var dat CommonMetricsResult
	jsonErr := json.Unmarshal(data, &dat)
	if jsonErr != nil {
		glog.Errorln("json parse failed", jsonErr)
	}
	var podNames []string
	for _, item := range dat.Data.Result {
		podName := item.KubePodMetric.Pod
		podNames = append(podNames, podName)
	}
	podNamesFilter := "^(" + strings.Join(podNames, "|") + ")$"
	return podNamesFilter
}

func AssembleWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
	rule := MakeWorkloadRule(monitoringRequest.WorkloadKind, monitoringRequest.WorkloadName, monitoringRequest.NsName)
	paramValues := monitoringRequest.Params
	params := makeRequestParamString(rule, paramValues)

	res := client.SendMonitoringRequest(client.DefaultQueryType, params)

	podNamesFilter := getPodNameRegexInWorkload(res)

	queryType := monitoringRequest.QueryType
	rule = MakePodPromQL(metricName, monitoringRequest.NsName, "", "", podNamesFilter)
	params = makeRequestParamString(rule, paramValues)

	return queryType, params
}

func AssemblePodMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) {
	queryType := monitoringRequest.QueryType

	paramValues := monitoringRequest.Params

	rule := MakePodPromQL(metricName, monitoringRequest.NsName, monitoringRequest.NodeId, monitoringRequest.PodName, monitoringRequest.PodsFilter)
	params := makeRequestParamString(rule, paramValues)
	return queryType, params, rule == ""
}

func GetMetric(queryType, params, metricName string) *FormatedMetric {
	res := client.SendMonitoringRequest(queryType, params)
	formatedMetric := ReformatJson(res, metricName)
	return formatedMetric
}

func GetNodeAddressInfo() *map[string][]v1.NodeAddress {
	nodeList, _ := client.NewK8sClient().CoreV1().Nodes().List(metaV1.ListOptions{})
	var nodeAddress = make(map[string][]v1.NodeAddress)
	for _, node := range nodeList.Items {
		nodeAddress[node.Name] = node.Status.Addresses
	}
	return &nodeAddress
}

func AddNodeAddressMetric(nodeMetric *FormatedMetric, nodeAddress *map[string][]v1.NodeAddress) {

	for i := 0; i < len(nodeMetric.Data.Result); i++ {
		metricDesc := nodeMetric.Data.Result[i][ResultItemMetric]
		metricDescMap := metricDesc.(map[string]interface{})
		if nodeId, exist := metricDescMap["node"]; exist {
			addr, exist := (*nodeAddress)[nodeId.(string)]
			if exist {
				metricDescMap["address"] = addr
			}
		}
	}
}

func MonitorContainer(monitoringRequest *client.MonitoringRequestParams) *FormatedMetric {
	queryType := monitoringRequest.QueryType

	paramValues := monitoringRequest.Params
	rule := MakeContainerPromQL(monitoringRequest.NsName, monitoringRequest.PodName, monitoringRequest.ContainerName, monitoringRequest.MetricsName, monitoringRequest.ContainersFilter)
	params := makeRequestParamString(rule, paramValues)

	res := GetMetric(queryType, params, monitoringRequest.MetricsName)
	return res
}

func AssembleNamespaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
	queryType := monitoringRequest.QueryType

	paramValues := monitoringRequest.Params
	rule := MakeNamespacePromQL(monitoringRequest.NsName, monitoringRequest.NsFilter, metricName)
	params := makeRequestParamString(rule, paramValues)

	return queryType, params
}

func AssembleWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) {
	nsFilter := "^(" + strings.Join(namespaceList, "|") + ")$"

	queryType := monitoringRequest.QueryType

	rule := MakeWorkspacePromQL(metricName, nsFilter)
	paramValues := monitoringRequest.Params
	params := makeRequestParamString(rule, paramValues)
	return queryType, params
}

func makeRequestParamString(rule string, paramValues url.Values) string {

	var values = make(url.Values)
	for key, v := range paramValues {
		values.Set(key, v[0])
	}

	values.Set("query", rule)

	params := values.Encode()

	return params
}

func filterNamespace(nsFilter string, namespaceList []string) []string {
	var newNSlist []string
	if nsFilter == "" {
		nsFilter = ".*"
	}
	for _, ns := range namespaceList {
		bol, _ := regexp.MatchString(nsFilter, ns)
		if bol {
			newNSlist = append(newNSlist, ns)
		}
	}
	return newNSlist
}

func MonitorAllWorkspaces(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
	metricsFilter := monitoringRequest.MetricsFilter
	if strings.Trim(metricsFilter, " ") == "" {
		metricsFilter = ".*"
	}
	var filterMetricsName []string
	for _, metricName := range WorkspaceMetricsNames {
		bol, err := regexp.MatchString(metricsFilter, metricName)
		if err == nil && bol {
			filterMetricsName = append(filterMetricsName, metricName)
		}
	}

	var wgAll sync.WaitGroup
	var wsAllch = make(chan *[]FormatedMetric, ChannelMaxCapacityWorkspaceMetric)

	workspaceNamespaceMap, _, _ := workspaces.GetAllOrgAndProjList()
	for ws, _ := range workspaceNamespaceMap {
		bol, err := regexp.MatchString(monitoringRequest.WsFilter, ws)
		if err == nil && bol {
			// a workspace
			wgAll.Add(1)
			go collectWorkspaceMetric(monitoringRequest, ws, filterMetricsName, &wgAll, wsAllch)
		}
	}

	wgAll.Wait()
	close(wsAllch)

	fmtMetricMap := make(map[string]FormatedMetric)
	for oneWsMetric := range wsAllch {
		if oneWsMetric != nil {
			// aggregate workspace metric
			for _, metric := range *oneWsMetric {
				fm, exist := fmtMetricMap[metric.MetricName]
				if exist {
					if metric.Status == "error" {
						fm.Status = metric.Status
					}
					fm.Data.Result = append(fm.Data.Result, metric.Data.Result...)
					fmtMetricMap[metric.MetricName] = fm
				} else {
					fmtMetricMap[metric.MetricName] = metric
				}
			}
		}
	}

	var metricArray = make([]FormatedMetric, 0)
	for _, metric := range fmtMetricMap {
		metricArray = append(metricArray, metric)
	}

	return &FormatedLevelMetric{
		MetricsLevel: MetricLevelClusterWorkspace,
		Results:      metricArray,
	}
}

func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, ws string, filterMetricsName []string, wgAll *sync.WaitGroup, wsAllch chan *[]FormatedMetric) {
	defer wgAll.Done()
	var wg sync.WaitGroup
	var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
	namespaceArray, err := workspaces.WorkspaceNamespaces(ws)
	if err != nil {
		glog.Errorln(err.Error())
	}
	// add by namespace
	for _, metricName := range filterMetricsName {
		wg.Add(1)
		go func(metricName string) {
			defer func() {
				if err := recover(); err != nil {
					glog.Errorln(err)
				}
			}()

			queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName)
			ch <- GetMetric(queryType, params, metricName)

			wg.Done()
		}(metricName)
	}

	wg.Wait()
	close(ch)

	var metricsArray []FormatedMetric
	for oneMetric := range ch {
		if oneMetric != nil {
			// add "workspace" filed to oneMetric `metric` field
			for i := 0; i < len(oneMetric.Data.Result); i++ {
				tmap := oneMetric.Data.Result[i]["metric"].(map[string]interface{})
				tmap[MetricLevelWorkspace] = ws
				oneMetric.Data.Result[i]["metric"] = tmap
			}
			metricsArray = append(metricsArray, *oneMetric)
		}
	}

	wsAllch <- &metricsArray
}

func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resourceType string) *FormatedLevelMetric {
	metricsFilter := monitoringRequest.MetricsFilter
	if metricsFilter == "" {
		metricsFilter = ".*"
	}

	var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
	var wg sync.WaitGroup

	switch resourceType {
	case MetricLevelCluster:
		{
			for _, metricName := range ClusterMetricsNames {
				bol, err := regexp.MatchString(metricsFilter, metricName)
				if err == nil && bol {
					wg.Add(1)
					go func(metricName string) {
						queryType, params := AssembleClusterMetricRequestInfo(monitoringRequest, metricName)
						clusterMetrics := GetMetric(queryType, params, metricName)

						// for this special case, get namespace history which in a workspace by determining namespace label
						if metricName == MetricNameWorkspaceAllProjectCount {
							clusterMetrics = MonitorWorkspaceNamespaceHistory(clusterMetrics)
						}

						ch <- clusterMetrics

						wg.Done()
					}(metricName)
				}
			}
		}
	case MetricLevelNode:
		{
			for _, metricName := range NodeMetricsNames {
				bol, err := regexp.MatchString(metricsFilter, metricName)
				if err == nil && bol {
					wg.Add(1)
					go func(metricName string) {
						queryType, params := AssembleNodeMetricRequestInfo(monitoringRequest, metricName)
						ch <- GetMetric(queryType, params, metricName)
						wg.Done()
					}(metricName)
				}
			}
		}
	case MetricLevelWorkspace:
		{
			namespaceArray, err := workspaces.WorkspaceNamespaces(monitoringRequest.WsName)
			if err != nil {
				glog.Errorln(err.Error())
			}
			namespaceArray = filterNamespace(monitoringRequest.NsFilter, namespaceArray)

			if monitoringRequest.Tp == "" {
				for _, metricName := range WorkspaceMetricsNames {
					bol, err := regexp.MatchString(metricsFilter, metricName)
					if err == nil && bol {
						wg.Add(1)
						go func(metricName string) {
							queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName)
							ch <- GetMetric(queryType, params, metricName)
							wg.Done()
						}(metricName)
					}
				}
			} else {
				for _, metricName := range NamespaceMetricsNames {
					bol, err := regexp.MatchString(metricsFilter, metricName)
					ns := "^(" + strings.Join(namespaceArray, "|") + ")$"
					monitoringRequest.NsFilter = ns
					if err == nil && bol {
						wg.Add(1)
						go func(metricName string) {
							queryType, params := AssembleNamespaceMetricRequestInfo(monitoringRequest, metricName)
							ch <- GetMetric(queryType, params, metricName)
							wg.Done()
						}(metricName)
					}
				}
			}
		}
	case MetricLevelNamespace:
		{
			for _, metricName := range NamespaceMetricsNames {
				bol, err := regexp.MatchString(metricsFilter, metricName)
				if err == nil && bol {
					wg.Add(1)
					go func(metricName string) {
						queryType, params := AssembleNamespaceMetricRequestInfo(monitoringRequest, metricName)
						ch <- GetMetric(queryType, params, metricName)
						wg.Done()
					}(metricName)
				}
			}
		}
	case MetricLevelWorkload:
		{
			for _, metricName := range WorkloadMetricsNames {
				bol, err := regexp.MatchString(metricsFilter, metricName)
				if err == nil && bol {
					wg.Add(1)
					go func(metricName string) {
						metricName = strings.TrimLeft(metricName, "workload_")
						queryType, params := AssembleWorkloadMetricRequestInfo(monitoringRequest, metricName)
						ch <- GetMetric(queryType, params, metricName)
						wg.Done()
					}(metricName)
				}
			}
		}
	case MetricLevelPod:
		{
			for _, metricName := range PodMetricsNames {
				bol, err := regexp.MatchString(metricsFilter, metricName)
				if err == nil && bol {
					wg.Add(1)
					go func(metricName string) {
						queryType, params, nullRule := AssemblePodMetricRequestInfo(monitoringRequest, metricName)
						if !nullRule {
							ch <- GetMetric(queryType, params, metricName)
						} else {
							ch <- nil
						}
						wg.Done()
					}(metricName)
				}
			}
		}
	}

	wg.Wait()
	close(ch)

	var metricsArray []FormatedMetric

	for oneMetric := range ch {
		if oneMetric != nil {
			metricsArray = append(metricsArray, *oneMetric)
		}
	}

	return &FormatedLevelMetric{
		MetricsLevel: resourceType,
		Results:      metricsArray,
	}
}

func MonitorWorkspaceNamespaceHistory(metric *FormatedMetric) *FormatedMetric {
	resultType := metric.Data.ResultType
	//metric.Status
	metricName := metric.MetricName

	for i := 0; i < len(metric.Data.Result); i++ {
		metricItem := metric.Data.Result[i]

		if resultType == ResultTypeVector {
			timeAndValue, sure := metricItem[ResultItemValue].([]interface{})
			if !sure {
				return metric
			}
			metric := getNamespaceHistoryMetric(timeAndValue[0].(float64), metricName)

			workspaceNamespaceCount := calcWorkspaceNamespace(metric)

			timeAndValue[1] = fmt.Sprintf("%d", workspaceNamespaceCount)

		} else if resultType == ResultTypeMatrix {

			values, sure := metricItem[ResultItemValues].([]interface{})
			if !sure {
				return metric
			}

			for _, valueItem := range values {
				timeAndValue, sure := valueItem.([]interface{})
				if !sure {
					return metric
				}

				metric := getNamespaceHistoryMetric(timeAndValue[0].(float64), metricName)

				workspaceNamespaceCount := calcWorkspaceNamespace(metric)

				timeAndValue[1] = fmt.Sprintf("%d", workspaceNamespaceCount)
			}
		}
	}

	return metric
}

func getNamespaceHistoryMetric(timestamp float64, metricName string) *FormatedMetric {
	var timeRelatedParams = make(url.Values)
	timeRelatedParams.Set("time", fmt.Sprintf("%f", timestamp))
	timeRelatedParams.Set("query", NamespaceLabelRule)
	params := timeRelatedParams.Encode()
	metric := GetMetric(client.DefaultQueryType, params, metricName)
	return metric
}

// calculate all namespace which belong to workspaces
func calcWorkspaceNamespace(metric *FormatedMetric) int {
	if metric.Status == "error" {
		glog.Errorf("failed when retrive namespace history, the metric is %v", metric.Data.Result)
		return 0
	}

	var workspaceNamespaceCount = 0

	for _, result := range metric.Data.Result {
		tmpMap := result[ResultItemMetric].(map[string]interface{})
		wsName, exist := tmpMap[WorkspaceJoinedKey]

		if exist && wsName != "" {
			workspaceNamespaceCount += 1
		}
	}

	return workspaceNamespaceCount
}

func MonitorAllWorkspacesStatistics() *FormatedLevelMetric {

	wg := sync.WaitGroup{}
	var metricsArray []FormatedMetric
	timestamp := time.Now().Unix()

	var orgResultItem *FormatedMetric
	var devopsResultItem *FormatedMetric
	var clusterProjResultItem *FormatedMetric
	var workspaceProjResultItem *FormatedMetric
	var accountResultItem *FormatedMetric

	wsMap, projs, errProj := workspaces.GetAllOrgAndProjList()

	wg.Add(5)

	go func() {
		orgNums, errOrg := workspaces.GetAllOrgNums()
		if errOrg != nil {
			glog.Errorln(errOrg.Error())
		}
		orgResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllOrganizationCount, WorkspaceResourceKindOrganization, orgNums, errOrg)
		wg.Done()
	}()

	go func() {
		devOpsProjectNums, errDevops := workspaces.GetAllDevOpsProjectsNums()
		if errDevops != nil {
			glog.Errorln(errDevops.Error())
		}
		devopsResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllDevopsCount, WorkspaceResourceKindDevops, devOpsProjectNums, errDevops)
		wg.Done()
	}()

	go func() {
		var projNums = 0
		for _, v := range wsMap {
			projNums += len(v)
		}
		if errProj != nil {
			glog.Errorln(errProj.Error())
		}
		workspaceProjResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllProjectCount, WorkspaceResourceKindNamespace, projNums, errProj)
		wg.Done()
	}()

	go func() {
		projNums := len(projs)
		if errProj != nil {
			glog.Errorln(errProj.Error())
		}
		clusterProjResultItem = getSpecificMetricItem(timestamp, MetricNameClusterAllProjectCount, WorkspaceResourceKindNamespace, projNums, errProj)
		wg.Done()
	}()

	go func() {
		actNums, errAct := workspaces.GetAllAccountNums()
		if errAct != nil {
			glog.Errorln(errAct.Error())
		}
		accountResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllAccountCount, WorkspaceResourceKindAccount, actNums, errAct)
		wg.Done()
	}()

	wg.Wait()

	metricsArray = append(metricsArray, *orgResultItem, *devopsResultItem, *workspaceProjResultItem, *accountResultItem, *clusterProjResultItem)

	return &FormatedLevelMetric{
		MetricsLevel: MetricLevelWorkspace,
		Results:      metricsArray,
	}
}

func MonitorOneWorkspaceStatistics(wsName string) *FormatedLevelMetric {

	var nsMetrics *FormatedMetric
	var devopsMetrics *FormatedMetric
	var memberMetrics *FormatedMetric
	var roleMetrics *FormatedMetric

	wg := sync.WaitGroup{}
	wg.Add(4)

	var fMetricsArray []FormatedMetric
	timestamp := int64(time.Now().Unix())

	go func() {
		// add namespaces(project) metric
		namespaces, errNs := workspaces.WorkspaceNamespaces(wsName)
		namespaces, noneExistentNs := getExistingNamespace(namespaces)
		if len(noneExistentNs) != 0 {
			nsStr := strings.Join(noneExistentNs, "|")
			errStr := "the namespaces " + nsStr + " do not exist"
			if errNs == nil {
				errNs = errors.New(errStr)
			} else {
				errNs = errors.New(errNs.Error() + "\t" + errStr)
			}
		}
		if errNs != nil {
			glog.Errorln(errNs.Error())
		}
		nsMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceNamespaceCount, WorkspaceResourceKindNamespace, len(namespaces), errNs)
		wg.Done()
	}()

	go func() {
		devOpsProjects, errDevOps := workspaces.GetDevOpsProjects(wsName)
		if errDevOps != nil {
			glog.Errorln(errDevOps.Error())
		}
		// add devops metric
		devopsMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceDevopsCount, WorkspaceResourceKindDevops, len(devOpsProjects), errDevOps)
		wg.Done()
	}()

	go func() {
		members, errMemb := workspaces.GetOrgMembers(wsName)
		if errMemb != nil {
			glog.Errorln(errMemb.Error())
		}
		// add member metric
		memberMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceMemberCount, WorkspaceResourceKindMember, len(members), errMemb)
		wg.Done()
	}()

	go func() {
		roles, errRole := workspaces.GetOrgRoles(wsName)
		if errRole != nil {
			glog.Errorln(errRole.Error())
		}
		// add role metric
		roleMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceRoleCount, WorkspaceResourceKindRole, len(roles), errRole)
		wg.Done()
	}()

	wg.Wait()

	fMetricsArray = append(fMetricsArray, *nsMetrics, *devopsMetrics, *memberMetrics, *roleMetrics)

	return &FormatedLevelMetric{
		MetricsLevel: MetricLevelWorkspace,
		Results:      fMetricsArray,
	}
}

func getSpecificMetricItem(timestamp int64, metricName string, resource string, count int, err error, resourceType ...string) *FormatedMetric {
	var nsMetrics FormatedMetric
	nsMetrics.MetricName = metricName
	nsMetrics.Data.ResultType = ResultTypeVector
	resultItem := make(map[string]interface{})
	tmp := make(map[string]string)

	if len(resourceType) > 0 {
		tmp[resourceType[0]] = resource
	} else {
		tmp[ResultItemMetricResource] = resource
	}

	if err == nil {
		nsMetrics.Status = MetricStatusSuccess
	} else {
		nsMetrics.Status = MetricStatusError
		resultItem["errormsg"] = err.Error()
	}

	resultItem[ResultItemMetric] = tmp
	resultItem[ResultItemValue] = []interface{}{timestamp, count}
	nsMetrics.Data.Result = make([]map[string]interface{}, 1)
	nsMetrics.Data.Result[0] = resultItem
	return &nsMetrics
}

// k8s component(controller, scheduler, etcd) status
func MonitorComponentStatus(monitoringRequest *client.MonitoringRequestParams) *[]interface{} {
	componentList, err := client.NewK8sClient().CoreV1().ComponentStatuses().List(metaV1.ListOptions{})
	if err != nil {
		glog.Errorln(err.Error())
	}

	var componentStatusList []*ComponentStatus
	for _, item := range componentList.Items {
		var status []OneComponentStatus
		for _, cond := range item.Conditions {
			status = append(status, OneComponentStatus{
				Type:    string(cond.Type),
				Status:  string(cond.Status),
				Message: cond.Message,
				Error:   cond.Error,
			})
		}

		componentStatusList = append(componentStatusList, &ComponentStatus{
			Name:            item.Name,
			Namespace:       item.Namespace,
			Labels:          item.Labels,
			ComponentStatus: status,
		})
	}

	// node status
	queryType := monitoringRequest.QueryType
	paramValues := monitoringRequest.Params
	paramValues.Set("query", NodeStatusRule)
	params := paramValues.Encode()
	res := client.SendMonitoringRequest(queryType, params)

	nodeStatusMetric := ReformatJson(res, "node_status", nodeStatusDelLables...)
	nodeStatusMetric = ReformatNodeStatusField(nodeStatusMetric)

	var normalNodes []string
	var abnormalNodes []string
	for _, result := range nodeStatusMetric.Data.Result {
		tmap := result[ResultItemMetric].(map[string]interface{})
		if tmap[MetricStatus].(string) == "false" {
			abnormalNodes = append(abnormalNodes, tmap[MetricLevelNode].(string))
		} else {
			normalNodes = append(normalNodes, tmap[MetricLevelNode].(string))
		}
	}

	Components, err := models.GetAllComponentsStatus()

	if err != nil {
		glog.Error(err.Error())
	}

	var namspaceComponentHealthyMap = make(map[string]int)
	var namspaceComponentTotalMap = make(map[string]int)

	for _, ns := range models.SYSTEM_NAMESPACES {
		nsStatus, exist := Components[ns]
		if exist {
			for _, nsStatusItem := range nsStatus.(map[string]interface{}) {
				component := nsStatusItem.(models.Component)
				namspaceComponentTotalMap[ns] += 1
				if component.HealthyBackends == component.TotalBackends {
					namspaceComponentHealthyMap[ns] += 1
				}
			}
		}
	}

	timestamp := int64(time.Now().Unix())

	onlineMetricItems := makeMetricItems(timestamp, namspaceComponentHealthyMap, MetricLevelNamespace)
	metricItems := makeMetricItems(timestamp, namspaceComponentTotalMap, MetricLevelNamespace)

	var assembleList []interface{}
	assembleList = append(assembleList, nodeStatusMetric)

	for _, statusItem := range componentStatusList {
		assembleList = append(assembleList, statusItem)
	}

	assembleList = append(assembleList, FormatedMetric{
		Data: FormatedMetricData{
			Result:     *onlineMetricItems,
			ResultType: ResultTypeVector,
		},
		MetricName: MetricNameComponentOnLine,
		Status:     MetricStatusSuccess,
	})

	assembleList = append(assembleList, FormatedMetric{
		Data: FormatedMetricData{
			Result:     *metricItems,
			ResultType: ResultTypeVector,
		},
		MetricName: MetricNameComponentLine,
		Status:     MetricStatusSuccess,
	})

	return &assembleList
}

func makeMetricItems(timestamp int64, statusMap map[string]int, resourceType string) *[]map[string]interface{} {
	var metricItems []map[string]interface{}

	for ns, count := range statusMap {
		metricItems = append(metricItems, map[string]interface{}{
			ResultItemMetric: map[string]string{resourceType: ns},
			ResultItemValue:  []interface{}{timestamp, fmt.Sprintf("%d", count)},
		})
	}
	return &metricItems
}

// monitor k8s event, there are two status: Normal, Warning
func MonitorEvents(nsFilter string) *[]v1.Event {
	namespaceMap, err := getAllNamespace()
	if err != nil {
		glog.Errorln(err.Error())
	}
	var nsList = make([]string, 0)
	for ns, _ := range namespaceMap {
		nsList = append(nsList, ns)
	}

	filterNS := filterNamespace(nsFilter, nsList)
	var eventsList = make([]v1.Event, 0)

	for _, ns := range filterNS {
		events, err := client.NewK8sClient().CoreV1().Events(ns).List(metaV1.ListOptions{})
		if err != nil {
			glog.Errorln(err.Error())
		} else {
			for _, item := range events.Items {
				eventsList = append(eventsList, item)
			}
		}
	}
	return &eventsList
}

func AssembleClusterMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
	queryType := monitoringRequest.QueryType
	paramValues := monitoringRequest.Params
	rule := MakeClusterRule(metricName)
	params := makeRequestParamString(rule, paramValues)

	return queryType, params
}

func AssembleNodeMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
	queryType := monitoringRequest.QueryType
	paramValues := monitoringRequest.Params
	rule := MakeNodeRule(monitoringRequest.NodeId, monitoringRequest.NodesFilter, metricName)
	params := makeRequestParamString(rule, paramValues)

	return queryType, params
}

func getExistingNamespace(namespaces []string) ([]string, []string) {
	namespaceMap, err := getAllNamespace()
	var existedNS []string
	var noneExistedNS []string
	if err != nil {
		return namespaces, nil
	}
	for _, ns := range namespaces {
		if _, exist := namespaceMap[ns]; exist {
			existedNS = append(existedNS, ns)
		} else {
			noneExistedNS = append(noneExistedNS, ns)
		}
	}
	return existedNS, noneExistedNS
}

func getAllNamespace() (map[string]int, error) {
	k8sClient := client.NewK8sClient()
	nsList, err := k8sClient.CoreV1().Namespaces().List(metaV1.ListOptions{})
	if err != nil {
		glog.Errorln(err.Error())
		return nil, err
	}
	namespaceMap := make(map[string]int)
	for _, item := range nsList.Items {
		namespaceMap[item.Name] = 0
	}
	return namespaceMap, nil
}

func MonitorWorkloadCount(namespace string) *FormatedMetric {
	quotaMetric, err := models.GetNamespaceQuota(namespace)
	fMetric := convertQuota2MetricStruct(quotaMetric)

	// whether the namespace in request parameters exists?
	namespaceMap, err := getAllNamespace()

	_, exist := namespaceMap[namespace]
	if err != nil {
		exist = true
	}

	if !exist || err != nil {
		fMetric.Status = MetricStatusError
		fMetric.Data.ResultType = ""
		errInfo := make(map[string]interface{})
		if err != nil {
			errInfo["errormsg"] = err.Error()
		} else {
			errInfo["errormsg"] = "namespace " + namespace + " does not exist"
		}
		fMetric.Data.Result = []map[string]interface{}{errInfo}
	}

	return fMetric
}

func convertQuota2MetricStruct(quotaMetric *models.ResourceQuota) *FormatedMetric {
	var fMetric FormatedMetric
	fMetric.MetricName = MetricNameWorkloadCount
	fMetric.Status = MetricStatusSuccess
	fMetric.Data.ResultType = ResultTypeVector
	timestamp := int64(time.Now().Unix())
	var resultItems []map[string]interface{}

	hardMap := make(map[string]string)
	for resourceName, v := range quotaMetric.Data.Hard {
		hardMap[resourceName.String()] = v.String()
	}

	for resourceName, v := range quotaMetric.Data.Used {
		resultItem := make(map[string]interface{})
		tmp := make(map[string]string)
		tmp[ResultItemMetricResource] = resourceName.String()
		resultItem[ResultItemMetric] = tmp
		resultItem[ResultItemValue] = []interface{}{timestamp, hardMap[resourceName.String()], v.String()}
		resultItems = append(resultItems, resultItem)
	}

	fMetric.Data.Result = resultItems
	return &fMetric
}