diff --git a/pkg/apis/v1alpha/monitoring/monitor_handler.go b/pkg/apis/v1alpha/monitoring/monitor_handler.go index 6471b2f3a..90f9e2baa 100755 --- a/pkg/apis/v1alpha/monitoring/monitor_handler.go +++ b/pkg/apis/v1alpha/monitoring/monitor_handler.go @@ -51,9 +51,21 @@ func (u Monitor) monitorPod(request *restful.Request, response *restful.Response func (u Monitor) monitorContainer(request *restful.Request, response *restful.Response) { requestParams := client.ParseMonitoringRequestParams(request) - res := metrics.MonitorContainer(requestParams) + metricName := requestParams.MetricsName + if requestParams.MetricsFilter != "" { + rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelContainer) + // sorting + sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelContainerName) + // paging + pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + + response.WriteAsJson(pagedMetrics) + + } else { + res := metrics.MonitorContainer(requestParams, metricName) + response.WriteAsJson(res) + } - response.WriteAsJson(res) } func (u Monitor) monitorWorkload(request *restful.Request, response *restful.Response) { @@ -323,13 +335,36 @@ func Register(ws *restful.WebService, subPath string) { Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) + ws.Route(ws.GET(subPath+"/nodes/{node_id}/pods/{pod_name}/containers").To(u.monitorContainer). + Filter(route.RouteLogging). + Doc("monitor specific pod level metrics by nodeid"). + Param(ws.PathParameter("node_id", "specific node").DataType("string").Required(true)). + Param(ws.PathParameter("pod_name", "specific pod").DataType("string").Required(true)). + Param(ws.QueryParameter("containers_filter", "container re2 expression filter").DataType("string").Required(false).DefaultValue("")). + Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...").DataType("string").Required(false)). + Param(ws.QueryParameter("metrics_name", "metrics name cpu memory...").DataType("string").Required(true).DefaultValue("pod_memory_utilisation_wo_cache")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). + Param(ws.QueryParameter("type", "rank, statistic").DataType("string").Required(false).DefaultValue("rank")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + ws.Route(ws.GET(subPath+"/namespaces/{ns_name}/pods/{pod_name}/containers").To(u.monitorContainer). Filter(route.RouteLogging). Doc("monitor containers level metrics"). Param(ws.PathParameter("ns_name", "specific namespace").DataType("string").Required(true).DefaultValue("monitoring")). Param(ws.PathParameter("pod_name", "specific pod").DataType("string").Required(true).DefaultValue("")). Param(ws.QueryParameter("containers_filter", "container re2 expression filter").DataType("string").Required(false).DefaultValue("")). + Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...").DataType("string").Required(false)). Param(ws.QueryParameter("metrics_name", "metrics name cpu memory...").DataType("string").Required(true).DefaultValue("container_memory_utilisation_wo_cache")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). + Param(ws.QueryParameter("type", "rank, statistic").DataType("string").Required(false).DefaultValue("rank")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) diff --git a/pkg/models/metrics/metrics.go b/pkg/models/metrics/metrics.go index 46375167e..32b92a4ca 100644 --- a/pkg/models/metrics/metrics.go +++ b/pkg/models/metrics/metrics.go @@ -120,51 +120,6 @@ type OneComponentStatus struct { Error string `json:"error,omitempty"` } -func renameWorkload(formatedMetric *FormatedMetric, relationMap map[string]string) { - if formatedMetric.Status == MetricStatusSuccess { - for i := 0; i < len(formatedMetric.Data.Result); i++ { - metricDesc := formatedMetric.Data.Result[i][ResultItemMetric] - metricDescMap, ensure := metricDesc.(map[string]interface{}) - if ensure { - if wl, exist := metricDescMap[MetricLevelWorkload]; exist { - if deployName, exist := relationMap[wl.(string)]; exist { - metricDescMap[MetricLevelWorkload] = deployName - } - } - } - } - } -} - -func getReplicaAndDeployRelation(nsName string) map[string]string { - rule := strings.Replace(WorkloadReplicaSetOwnerRule, "$1", nsName, -1) - - params := makeRequestParamString(rule, make(url.Values)) - - res := client.SendMonitoringRequest(client.DefaultQueryType, params) - formatedMetric := ReformatJson(res, "") - var relationMap = make(map[string]string) - if formatedMetric.Status == MetricStatusSuccess { - for i := 0; i < len(formatedMetric.Data.Result); i++ { - metricDesc := formatedMetric.Data.Result[i][ResultItemMetric] - metricDescMap, ensure := metricDesc.(map[string]interface{}) - if ensure { - if ownerKind, exist := metricDescMap["owner_kind"]; exist && ownerKind == ReplicaSet { - if ownerName, exist := metricDescMap["owner_name"]; exist { - replicaName, sure := ownerName.(string) - if sure { - deployName := replicaName[:strings.LastIndex(replicaName, "-")] - relationMap[replicaName] = deployName - } - } - } - } - - } - } - return relationMap -} - func getPodNameRegexInWorkload(res string) string { data := []byte(res) @@ -237,7 +192,7 @@ func unifyMetricHistoryTimeRange(fmtMetrics *FormatedMetric) { } } -func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { +func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) { nsName := monitoringRequest.NsName wkName := monitoringRequest.WorkloadName @@ -254,7 +209,7 @@ func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *client.Monitor rule = MakePodPromQL(metricName, nsName, "", "", podNamesFilter) params = makeRequestParamString(rule, paramValues) - return queryType, params + return queryType, params, rule == "" } func AssembleAllWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { @@ -311,15 +266,20 @@ func AddNodeAddressMetric(nodeMetric *FormatedMetric, nodeAddress *map[string][] } } -func MonitorContainer(monitoringRequest *client.MonitoringRequestParams) *FormatedMetric { +func MonitorContainer(monitoringRequest *client.MonitoringRequestParams, metricName string) *FormatedMetric { + queryType, params := AssembleContainerMetricRequestInfo(monitoringRequest, metricName) + res := GetMetric(queryType, params, metricName) + return res +} + +func AssembleContainerMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { queryType := monitoringRequest.QueryType paramValues := monitoringRequest.Params - rule := MakeContainerPromQL(monitoringRequest.NsName, monitoringRequest.PodName, monitoringRequest.ContainerName, monitoringRequest.MetricsName, monitoringRequest.ContainersFilter) + rule := MakeContainerPromQL(monitoringRequest.NsName, monitoringRequest.NodeId, monitoringRequest.PodName, monitoringRequest.ContainerName, metricName, monitoringRequest.ContainersFilter) params := makeRequestParamString(rule, paramValues) - res := GetMetric(queryType, params, monitoringRequest.MetricsName) - return res + return queryType, params } func AssembleNamespaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { @@ -646,20 +606,14 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour case MetricLevelWorkload: { if monitoringRequest.Tp == "rank" { - // get relationship between replicaset and deployment - relationMap := getReplicaAndDeployRelation(monitoringRequest.NsName) + for _, metricName := range WorkloadMetricsNames { bol, err := regexp.MatchString(metricsFilter, metricName) if err == nil && bol { wg.Add(1) go func(metricName string) { queryType, params := AssembleAllWorkloadMetricRequestInfo(monitoringRequest, metricName) - fmtMetrics := GetMetric(queryType, params, metricName) - - // rename replica workload name - renameWorkload(fmtMetrics, relationMap) - - ch <- fmtMetrics + ch <- GetMetric(queryType, params, metricName) wg.Done() }(metricName) } @@ -671,10 +625,12 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour wg.Add(1) go func(metricName string) { metricName = strings.TrimLeft(metricName, "workload_") - queryType, params := AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest, metricName) - fmtMetrics := GetMetric(queryType, params, metricName) - unifyMetricHistoryTimeRange(fmtMetrics) - ch <- fmtMetrics + queryType, params, nullRule := AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest, metricName) + if !nullRule { + fmtMetrics := GetMetric(queryType, params, metricName) + unifyMetricHistoryTimeRange(fmtMetrics) + ch <- fmtMetrics + } wg.Done() }(metricName) } @@ -699,6 +655,20 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour } } } + case MetricLevelContainer: + { + for _, metricName := range ContainerMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params := AssembleContainerMetricRequestInfo(monitoringRequest, metricName) + ch <- GetMetric(queryType, params, metricName) + wg.Done() + }(metricName) + } + } + } } wg.Wait() diff --git a/pkg/models/metrics/metricsrule.go b/pkg/models/metrics/metricsrule.go index e575696b0..b8cf9fdf5 100755 --- a/pkg/models/metrics/metricsrule.go +++ b/pkg/models/metrics/metricsrule.go @@ -84,27 +84,31 @@ func MakeSpecificWorkspacePromQL(metricsName, nsFilter string) string { return promql } -func MakeContainerPromQL(nsName, podName, containerName, metricName, containerFilter string) string { - var promql = "" - if containerName == "" { - // all containers maybe use filter - metricName += "_all" +func MakeContainerPromQL(nsName, nodeId, podName, containerName, metricName, containerFilter string) string { + var promql string + + if nsName != "" { + // get container metrics from namespace-pod promql = RulePromQLTmplMap[metricName] promql = strings.Replace(promql, "$1", nsName, -1) - promql = strings.Replace(promql, "$2", podName, -1) + } else { + // get container metrics from node-pod + promql = RulePromQLTmplMap[metricName+"_node"] + promql = strings.Replace(promql, "$1", nodeId, -1) + } + + promql = strings.Replace(promql, "$2", podName, -1) + + if containerName == "" { if containerFilter == "" { containerFilter = ".*" } - promql = strings.Replace(promql, "$3", containerFilter, -1) - return promql + } else { + promql = strings.Replace(promql, "$3", containerName, -1) } - promql = RulePromQLTmplMap[metricName] - promql = strings.Replace(promql, "$1", nsName, -1) - promql = strings.Replace(promql, "$2", podName, -1) - promql = strings.Replace(promql, "$3", containerName, -1) return promql } diff --git a/pkg/models/metrics/metricsruleconst.go b/pkg/models/metrics/metricsruleconst.go index 823323864..35ac3b8ff 100644 --- a/pkg/models/metrics/metricsruleconst.go +++ b/pkg/models/metrics/metricsruleconst.go @@ -63,6 +63,7 @@ const ( MetricLevelPod = "pod" MetricLevelPodName = "pod_name" MetricLevelContainer = "container" + MetricLevelContainerName = "container_name" MetricLevelWorkload = "workload" ) @@ -279,6 +280,21 @@ var WorkloadMetricsNames = []string{ "workload_pod_memory_usage_wo_cache", "workload_pod_net_bytes_transmitted", "workload_pod_net_bytes_received", + + "workload_deployment_replica", + "workload_deployment_replica_available", + "workload_statefulset_replica", + "workload_statefulset_replica_available", + "workload_daemonset_replica", + "workload_daemonset_replica_available", +} + +var ContainerMetricsNames = []string{ + "container_cpu_usage", + "container_memory_usage", + "container_memory_usage_wo_cache", + //"container_net_bytes_transmitted", + //"container_net_bytes_received", } var RulePromQLTmplMap = MetricMap{ @@ -298,10 +314,10 @@ var RulePromQLTmplMap = MetricMap{ "cluster_disk_write_iops": "sum(node:data_volume_iops_writes:sum)", "cluster_disk_read_throughput": "sum(node:data_volume_throughput_bytes_read:sum)", "cluster_disk_write_throughput": "sum(node:data_volume_throughput_bytes_written:sum)", - "cluster_disk_size_usage": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, - "cluster_disk_size_utilisation": `(sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))) / sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, - "cluster_disk_size_capacity": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, - "cluster_disk_size_available": `sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, + "cluster_disk_size_usage": `sum(sum by (node) ((node_filesystem_size{device="overlay", mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{device="overlay", mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, + "cluster_disk_size_utilisation": `(sum(sum by (node) ((node_filesystem_size{device="overlay", mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{device="overlay", mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))) / sum(sum by (node) ((node_filesystem_size{device="overlay", mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, + "cluster_disk_size_capacity": `sum(sum by (node) ((node_filesystem_size{device="overlay", mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, + "cluster_disk_size_available": `sum(sum by (node) ((node_filesystem_avail{device="overlay", mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, "cluster_disk_inode_total": `sum(node:disk_inodes_total:)`, "cluster_disk_inode_usage": `sum(node:disk_inodes_total:) - sum(node:disk_inodes_free:)`, @@ -384,10 +400,10 @@ var RulePromQLTmplMap = MetricMap{ // node:data_volume_throughput_bytes_written:sum{node=~"i-5xcldxos|i-6soe9zl1"} "node_disk_write_throughput": "node:data_volume_throughput_bytes_written:sum", - "node_disk_size_capacity": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_disk_size_available": `sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_disk_size_usage": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) -sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_disk_size_utilisation": `sum by (node) (((node_filesystem_size{mountpoint="/", job="node-exporter"} - node_filesystem_avail{mountpoint="/", job="node-exporter"}) / node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + "node_disk_size_capacity": `sum by (node) ((node_filesystem_size{device="overlay", mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + "node_disk_size_available": `sum by (node) ((node_filesystem_avail{device="overlay", mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + "node_disk_size_usage": `sum by (node) ((node_filesystem_size{device="overlay", mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) -sum by (node) ((node_filesystem_avail{device="overlay", mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + "node_disk_size_utilisation": `sum by (node) (((node_filesystem_size{device="overlay", mountpoint="/", job="node-exporter"} - node_filesystem_avail{device="overlay", mountpoint="/", job="node-exporter"}) / node_filesystem_size{device="overlay", mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, "node_disk_inode_total": `node:disk_inodes_total:$1`, "node_disk_inode_usage": `node:disk_inodes_total:$1 - node:disk_inodes_free:$1`, @@ -485,28 +501,38 @@ var RulePromQLTmplMap = MetricMap{ "pod_net_bytes_transmitted_all": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`, "pod_net_bytes_received_all": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`, - "pod_cpu_usage_node": `sum by (node, pod_name) (irate(container_cpu_usage_seconds_total{job="kubelet",pod_name!="", pod_name=~"$2", image!=""}[5m]) * on (namespace, pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node=~"$3"}, "pod_name", "", "pod", "_name"))`, - "pod_memory_usage_node": `sum by (node, pod_name) (container_memory_usage_bytes{job="kubelet",pod_name!="", pod_name=~"$2", image!=""} * on (namespace, pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node=~"$3"}, "pod_name", "", "pod", "_name"))`, - "pod_memory_usage_wo_cache_node": `sum by (node, pod_name) ((container_memory_usage_bytes{job="kubelet",pod_name!="", pod_name=~"$2", image!=""} - container_memory_cache{job="kubelet",pod_name!="", pod_name=~"$2", image!=""}) * on (namespace, pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node=~"$3"}, "pod_name", "", "pod", "_name"))`, - "pod_net_bytes_transmitted_node": `sum by (node, pod_name) (irate(container_network_transmit_bytes_total{pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]) * on (pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node=~"$3"}, "pod_name", "", "pod", "_name"))`, - "pod_net_bytes_received_node": `sum by (node, pod_name) (irate(container_network_receive_bytes_total{pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]) * on (pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node=~"$3"}, "pod_name", "", "pod", "_name"))`, + "pod_cpu_usage_node": `sum by (node, pod_name) (irate(container_cpu_usage_seconds_total{job="kubelet",pod_name!="", pod_name=~"$2", image!=""}[5m]) * on (namespace, pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node="$3"}, "pod_name", "", "pod", "_name"))`, + "pod_memory_usage_node": `sum by (node, pod_name) (container_memory_usage_bytes{job="kubelet",pod_name!="", pod_name=~"$2", image!=""} * on (namespace, pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node="$3"}, "pod_name", "", "pod", "_name"))`, + "pod_memory_usage_wo_cache_node": `sum by (node, pod_name) ((container_memory_usage_bytes{job="kubelet",pod_name!="", pod_name=~"$2", image!=""} - container_memory_cache{job="kubelet",pod_name!="", pod_name=~"$2", image!=""}) * on (namespace, pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node="$3"}, "pod_name", "", "pod", "_name"))`, + "pod_net_bytes_transmitted_node": `sum by (node, pod_name) (irate(container_network_transmit_bytes_total{pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]) * on (pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node="$3"}, "pod_name", "", "pod", "_name"))`, + "pod_net_bytes_received_node": `sum by (node, pod_name) (irate(container_network_receive_bytes_total{pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]) * on (pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node="$3"}, "pod_name", "", "pod", "_name"))`, // workload - "workload_pod_cpu_usage": `sum(label_replace(sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$2", pod_name!="", pod_name=~"$3", image!=""}[5m])) by (namespace, pod_name) and on (pod_name) label_join(kube_pod_owner{namespace="$2", owner_name!=""}, "pod_name", "", "pod", "_name"), "workload", "$1", "pod_name", "(.+)-(.+)")) by (namespace, workload)`, - "workload_pod_memory_usage": `sum(label_replace(sum(container_memory_usage_bytes{job="kubelet", namespace="$2", pod_name!="", pod_name=~"$3", image!=""}) by (namespace, pod_name) and on (pod_name) label_join(kube_pod_owner{namespace="$2", owner_name!=""}, "pod_name", "", "pod", "_name"), "workload", "$1", "pod_name", "(.+)-(.+)")) by (namespace, workload)`, - "workload_pod_memory_usage_wo_cache": `sum(label_replace(sum(container_memory_usage_bytes{job="kubelet", namespace="$2", pod_name!="", pod_name=~"$3", image!=""} - container_memory_cache{job="kubelet", namespace="$2", pod_name!="", pod_name=~"$3", image!=""}) by (namespace, pod_name) and on (pod_name) label_join(kube_pod_owner{namespace="$2", owner_name!=""}, "pod_name", "", "pod", "_name"), "workload", "$1", "pod_name", "(.+)-(.+)")) by (namespace, workload)`, - "workload_pod_net_bytes_transmitted": `sum(label_replace(sum(irate(container_network_transmit_bytes_total{namespace="$2", pod_name!="", pod_name=~"$3", interface="eth0", job="kubelet"}[5m])) by (namespace, pod_name) and on (pod_name) label_join(kube_pod_owner{namespace="$2", owner_name!=""}, "pod_name", "", "pod", "_name"), "workload", "$1", "pod_name", "(.+)-(.+)")) by (namespace, workload)`, - "workload_pod_net_bytes_received": `sum(label_replace(sum(irate(container_network_receive_bytes_total{namespace="$2", pod_name!="", pod_name=~"$3", interface="eth0", job="kubelet"}[5m])) by (namespace, pod_name) and on (pod_name) label_join(kube_pod_owner{namespace="$2", owner_name!=""}, "pod_name", "", "pod", "_name"), "workload", "$1", "pod_name", "(.+)-(.+)")) by (namespace, workload)`, + "workload_pod_cpu_usage": `label_join(sum(label_replace(label_replace(label_replace(label_join(label_join(label_replace(sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$2", pod_name!="", pod_name=~"$3", image!=""}[5m])) by (namespace, pod_name) * on (pod_name) group_left(owner_kind) label_join(label_replace(kube_pod_owner{namespace="$2", pod=~".*"}, "owner_kind", "POD", "owner_kind", ""), "pod_name", "", "pod", "_name") , "postfix", "-POD", "owner_kind", "POD"), "pod_name", "", "pod_name", "postfix"), "dist", "-", "owner_kind", "pod_name"), "pod_name", "$1", "dist", "ReplicaSet-(.+)-(.+)"), "workload", "$1", "pod_name", "(.+)-(.+)"), "owner_kind", "Deployment", "owner_kind", "ReplicaSet.*")) by (namespace, workload, owner_kind), "workload", ":", "owner_kind", "workload")`, + "workload_pod_memory_usage": `label_join(sum(label_replace(label_replace(label_replace(label_join(label_join(label_replace(sum(container_memory_usage_bytes{job="kubelet", namespace="$2", pod_name!="", pod_name=~"$3", image!=""}) by (namespace, pod_name) * on (pod_name) group_left(owner_kind) label_join(label_replace(kube_pod_owner{namespace="$2", pod=~".*"}, "owner_kind", "POD", "owner_kind", ""), "pod_name", "", "pod", "_name") , "postfix", "-POD", "owner_kind", "POD"), "pod_name", "", "pod_name", "postfix"), "dist", "-", "owner_kind", "pod_name"), "pod_name", "$1", "dist", "ReplicaSet-(.+)-(.+)"), "workload", "$1", "pod_name", "(.+)-(.+)"), "owner_kind", "Deployment", "owner_kind", "ReplicaSet.*")) by (namespace, workload, owner_kind), "workload", ":", "owner_kind", "workload")`, + "workload_pod_memory_usage_wo_cache": `label_join(sum(label_replace(label_replace(label_replace(label_join(label_join(label_replace(sum(container_memory_usage_bytes{job="kubelet", namespace="$2", pod_name!="", pod_name=~"$3", image!=""} - container_memory_cache{job="kubelet", namespace="$2", pod_name!="", pod_name=~"$3", image!=""}) by (namespace, pod_name) * on (pod_name) group_left(owner_kind) label_join(label_replace(kube_pod_owner{namespace="$2", pod=~".*"}, "owner_kind", "POD", "owner_kind", ""), "pod_name", "", "pod", "_name") , "postfix", "-POD", "owner_kind", "POD"), "pod_name", "", "pod_name", "postfix"), "dist", "-", "owner_kind", "pod_name"), "pod_name", "$1", "dist", "ReplicaSet-(.+)-(.+)"), "workload", "$1", "pod_name", "(.+)-(.+)"), "owner_kind", "Deployment", "owner_kind", "ReplicaSet.*")) by (namespace, workload, owner_kind), "workload", ":", "owner_kind", "workload")`, + "workload_pod_net_bytes_transmitted": `label_join(sum(label_replace(label_replace(label_replace(label_join(label_join(label_replace(sum(irate(container_network_transmit_bytes_total{namespace="$2", pod_name!="", pod_name=~"$3", interface="eth0", job="kubelet"}[5m])) by (namespace, pod_name) * on (pod_name) group_left(owner_kind) label_join(label_replace(kube_pod_owner{namespace="$2", pod=~".*"}, "owner_kind", "POD", "owner_kind", ""), "pod_name", "", "pod", "_name") , "postfix", "-POD", "owner_kind", "POD"), "pod_name", "", "pod_name", "postfix"), "dist", "-", "owner_kind", "pod_name"), "pod_name", "$1", "dist", "ReplicaSet-(.+)-(.+)"), "workload", "$1", "pod_name", "(.+)-(.+)"), "owner_kind", "Deployment", "owner_kind", "ReplicaSet.*")) by (namespace, workload, owner_kind), "workload", ":", "owner_kind", "workload")`, + "workload_pod_net_bytes_received": `label_join(sum(label_replace(label_replace(label_replace(label_join(label_join(label_replace(sum(irate(container_network_receive_bytes_total{namespace="$2", pod_name!="", pod_name=~"$3", interface="eth0", job="kubelet"}[5m])) by (namespace, pod_name) * on (pod_name) group_left(owner_kind) label_join(label_replace(kube_pod_owner{namespace="$2", pod=~".*"}, "owner_kind", "POD", "owner_kind", ""), "pod_name", "", "pod", "_name") , "postfix", "-POD", "owner_kind", "POD"), "pod_name", "", "pod_name", "postfix"), "dist", "-", "owner_kind", "pod_name"), "pod_name", "$1", "dist", "ReplicaSet-(.+)-(.+)"), "workload", "$1", "pod_name", "(.+)-(.+)"), "owner_kind", "Deployment", "owner_kind", "ReplicaSet.*")) by (namespace, workload, owner_kind), "workload", ":", "owner_kind", "workload")`, + + "workload_deployment_replica": `label_join(sum (label_join(label_replace(kube_deployment_spec_replicas{namespace="$2", deployment=~"$3"}, "owner_kind", "Deployment", "", ""), "workload", "", "deployment")) by (namespace, owner_kind, workload), "workload", ":", "owner_kind", "workload")`, + "workload_deployment_replica_available": `label_join(sum (label_join(label_replace(kube_deployment_status_replicas_available{namespace="$2", deployment=~"$3"}, "owner_kind", "Deployment", "", ""), "workload", "", "deployment")) by (namespace, owner_kind, workload), "workload", ":", "owner_kind", "workload")`, + "workload_statefulset_replica": `label_join(sum (label_join(label_replace(kube_statefulset_replicas{namespace="$2", statefulset=~"$3"}, "owner_kind", "StatefulSet", "", ""), "workload", "", "statefulset")) by (namespace, owner_kind, workload), "workload", ":", "owner_kind", "workload")`, + "workload_statefulset_replica_available": `label_join(sum (label_join(label_replace(kube_statefulset_status_replicas_ready{namespace="$2", statefulset=~"$3"}, "owner_kind", "StatefulSet", "", ""), "workload", "", "statefulset")) by (namespace, owner_kind, workload), "workload", ":", "owner_kind", "workload")`, + "workload_daemonset_replica": `label_join(sum (label_join(label_replace(kube_daemonset_status_desired_number_scheduled{namespace="$2", daemonset=~"$3"}, "owner_kind", "DaemonSet", "", ""), "workload", "", "daemonset")) by (namespace, owner_kind, workload), "workload", ":", "owner_kind", "workload")`, + "workload_daemonset_replica_available": `label_join(sum (label_join(label_replace(kube_daemonset_status_current_number_scheduled{namespace="$2", daemonset=~"$3"}, "owner_kind", "DaemonSet", "", ""), "workload", "", "daemonset")) by (namespace, owner_kind, workload), "workload", ":", "owner_kind", "workload")`, // container - "container_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name="$3"}[5m])) by (namespace, pod_name, container_name)`, - "container_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}[5m])) by (namespace, pod_name, container_name)`, + "container_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name!="POD", container_name=~"$3"}[5m])) by (namespace, pod_name, container_name)`, + "container_memory_usage": `sum(container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name!="POD", container_name=~"$3"}) by (namespace, pod_name, container_name)`, + "container_memory_usage_wo_cache": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name!="POD", container_name=~"$3"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name!="POD", container_name=~"$3"}`, + "container_net_bytes_transmitted": `sum(irate(container_network_transmit_bytes_total{job="kubelet", namespace="$1", pod_name="$2", container_name="POD", interface="eth0"}[5m])) by (namespace, pod_name, container_name)`, + "container_net_bytes_received": `sum(irate(container_network_receive_bytes_total{job="kubelet", namespace="$1", pod_name="$2", container_name="POD", interface="eth0"}[5m])) by (namespace, pod_name, container_name)`, - "container_memory_usage_wo_cache": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name="$3"}`, - "container_memory_usage_wo_cache_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`, - - "container_memory_usage": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"}`, - "container_memory_usage_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`, + "container_cpu_usage_node": `sum by (node, pod_name, container_name) (irate(container_cpu_usage_seconds_total{job="kubelet", pod_name="$2", container_name!="POD", container_name!="", container_name=~"$3", image!=""}[5m]) * on (pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node="$1"}, "pod_name", "", "pod", "_name"))`, + "container_memory_usage_node": `sum by (node, pod_name, container_name) (container_memory_usage_bytes{job="kubelet", pod_name="$2", container_name!="POD", container_name!="", container_name=~"$3", image!=""} * on (pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node="$1"}, "pod_name", "", "pod", "_name"))`, + "container_memory_usage_wo_cache_node": `sum by (node, pod_name, container_name) ((container_memory_usage_bytes{job="kubelet", pod_name="$2", container_name!="POD", container_name!="", container_name=~"$3", image!=""} - container_memory_cache{job="kubelet", pod_name="$2", container_name!="POD", container_name!="", container_name=~"$3", image!=""}) * on (pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node="$1"}, "pod_name", "", "pod", "_name"))`, + "container_net_bytes_transmitted_node": `sum by (node, pod_name, container_name) (irate(container_network_transmit_bytes_total{job="kubelet", interface="eth0", pod_name="$2", container_name="POD", container_name!="", image!=""}[5m]) * on (pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node="$1"}, "pod_name", "", "pod", "_name"))`, + "container_net_bytes_received_node": `sum by (node, pod_name, container_name) (irate(container_network_receive_bytes_total{job="kubelet", interface="eth0", pod_name="$2", container_name="POD", container_name!="", image!=""}[5m]) * on (pod_name) group_left(node) label_join(node_namespace_pod:kube_pod_info:{node="$1"}, "pod_name", "", "pod", "_name"))`, // workspace "workspace_cpu_usage": `sum(namespace:container_cpu_usage_seconds_total:sum_rate{namespace!="", namespace$1})`, diff --git a/pkg/models/metrics/util.go b/pkg/models/metrics/util.go index 41e56f6cc..5e7d85517 100644 --- a/pkg/models/metrics/util.go +++ b/pkg/models/metrics/util.go @@ -113,9 +113,13 @@ func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelM // for some reasons, 'metric' may not contain `resourceType` field // example: {"metric":{},"value":[1541142931.731,"3"]} k, exist := r[ResultItemMetric].(map[string]interface{})[resourceType] + key := k.(string) if exist { - indexMap[k.(string)] = i - i = i + 1 + if _, exist := indexMap[key]; !exist { + indexMap[key] = i + i = i + 1 + } + } } }