From 6a623c4c660fba0347092bf55455ca4f5cfd963f Mon Sep 17 00:00:00 2001 From: huanggze Date: Wed, 5 Feb 2020 20:52:19 +0800 Subject: [PATCH] update container metrics Signed-off-by: huanggze --- pkg/apis/monitoring/v1alpha2/register.go | 3 ++ pkg/models/metrics/metrics.go | 8 ++--- pkg/models/metrics/metrics_rules.go | 43 ++++++++++++++++++++++-- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/pkg/apis/monitoring/v1alpha2/register.go b/pkg/apis/monitoring/v1alpha2/register.go index ebc8a1fb1..d765d8fe3 100644 --- a/pkg/apis/monitoring/v1alpha2/register.go +++ b/pkg/apis/monitoring/v1alpha2/register.go @@ -405,5 +405,8 @@ func addWebService(c *restful.Container) error { Produces(restful.MIME_JSON) c.Add(ws) + + metrics.CompatibleMetrics() + return nil } diff --git a/pkg/models/metrics/metrics.go b/pkg/models/metrics/metrics.go index 2fe5cbb62..574df5283 100644 --- a/pkg/models/metrics/metrics.go +++ b/pkg/models/metrics/metrics.go @@ -313,7 +313,7 @@ func GetPodMetrics(params RequestParams) *Response { // add label resouce_name for _, item := range response.Data.Result { - item.Metric["resource_name"] = item.Metric["pod_name"] + item.Metric["resource_name"] = item.Metric[labelNamePod] } ch <- APIResponse{ @@ -364,7 +364,7 @@ func GetContainerMetrics(params RequestParams) *Response { // add label resouce_name for _, item := range response.Data.Result { - item.Metric["resource_name"] = item.Metric["container_name"] + item.Metric["resource_name"] = item.Metric[labelNameContainer] } ch <- APIResponse{ @@ -616,9 +616,9 @@ func makePromqlForContainer(metricName string, params RequestParams) string { var containerSelector string if params.ContainerName != "" { - containerSelector = fmt.Sprintf(`pod_name="%s", namespace="%s", container_name="%s"`, params.PodName, params.NamespaceName, params.ContainerName) + containerSelector = fmt.Sprintf(`%s="%s", namespace="%s", %s="%s"`, labelNamePod, params.PodName, params.NamespaceName, labelNameContainer, params.ContainerName) } else { - containerSelector = fmt.Sprintf(`pod_name="%s", namespace="%s", container_name=~"%s"`, params.PodName, params.NamespaceName, params.ResourcesFilter) + containerSelector = fmt.Sprintf(`%s="%s", namespace="%s", %s=~"%s"`, labelNamePod, params.PodName, params.NamespaceName, labelNameContainer, params.ResourcesFilter) } return strings.Replace(exp, "$1", containerSelector, -1) diff --git a/pkg/models/metrics/metrics_rules.go b/pkg/models/metrics/metrics_rules.go index 8879a5af6..828d75c9a 100644 --- a/pkg/models/metrics/metrics_rules.go +++ b/pkg/models/metrics/metrics_rules.go @@ -13,6 +13,16 @@ limitations under the License. package metrics +import ( + "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/simple/client" +) + +var ( + labelNamePod = "pod_name" + labelNameContainer = "container_name" +) + const ( // TODO: expose the following metrics in prometheus format MetricClusterWorkspaceCount = "cluster_workspace_count" @@ -435,14 +445,14 @@ var metricsPromqlMap = map[string]string{ // pod "pod_cpu_usage": `round(label_join(sum by (namespace, pod_name) (irate(container_cpu_usage_seconds_total{job="kubelet", pod_name!="", image!=""}[5m])), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}, 0.001)`, "pod_memory_usage": `label_join(sum by (namespace, pod_name) (container_memory_usage_bytes{job="kubelet", pod_name!="", image!=""}), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}`, - "pod_memory_usage_wo_cache": `label_join(sum by (namespace, pod_name) (container_memory_usage_bytes{job="kubelet", pod_name!="", image!=""} - container_memory_cache{job="kubelet", pod_name!="", image!=""}), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}`, + "pod_memory_usage_wo_cache": `label_join(sum by (namespace, pod_name) (container_memory_working_set_bytes{job="kubelet", pod_name!="", image!=""}), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}`, "pod_net_bytes_transmitted": `label_join(sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{pod_name!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet"}[5m])), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}`, "pod_net_bytes_received": `label_join(sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{pod_name!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet"}[5m])), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}`, // container "container_cpu_usage": `round(sum by (namespace, pod_name, container_name) (irate(container_cpu_usage_seconds_total{job="kubelet", container_name!="POD", container_name!="", image!="", $1}[5m])), 0.001)`, "container_memory_usage": `sum by (namespace, pod_name, container_name) (container_memory_usage_bytes{job="kubelet", container_name!="POD", container_name!="", image!="", $1})`, - "container_memory_usage_wo_cache": `sum by (namespace, pod_name, container_name) (container_memory_usage_bytes{job="kubelet", container_name!="POD", container_name!="", image!="", $1} - container_memory_cache{job="kubelet", container_name!="POD", container_name!="", image!="", $1})`, + "container_memory_usage_wo_cache": `sum by (namespace, pod_name, container_name) (container_memory_working_set_bytes{job="kubelet", container_name!="POD", container_name!="", image!="", $1})`, // pvc "pvc_inodes_available": `max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_inodes_free) * on (namespace, persistentvolumeclaim) group_left (storageclass) kube_persistentvolumeclaim_info{$1}`, @@ -506,3 +516,32 @@ var metricsPromqlMap = map[string]string{ "prometheus_up_sum": `prometheus:up:sum`, "prometheus_tsdb_head_samples_appended_rate": `prometheus:prometheus_tsdb_head_samples_appended:sum_rate`, } + +// As of Kubernetes v1.16, any Prometheus queries that match `pod_name` and +// `container_name` labels must be updated to use `pod` and `container` instead. +func CompatibleMetrics() { + version, err := client.ClientSets().K8s().Discovery().ServerVersion() + if err != nil { + klog.Errorf("fail to fetch k8s version: %v", err) + return + } + + if version.Minor >= "16" { + labelNamePod = "pod" + labelNameContainer = "container" + + m := metricsPromqlMap + m["workspace_net_bytes_transmitted"] = `sum by (label_kubesphere_io_workspace) (sum by (namespace) (irate(container_network_transmit_bytes_total{namespace!="", pod!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet"}[5m])) * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})` + m["workspace_net_bytes_received"] = `sum by (label_kubesphere_io_workspace) (sum by (namespace) (irate(container_network_receive_bytes_total{namespace!="", pod!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet"}[5m])) * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})` + m["namespace_net_bytes_transmitted"] = `sum by (namespace) (irate(container_network_transmit_bytes_total{namespace!="", pod!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet"}[5m]) * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})` + m["namespace_net_bytes_received"] = `sum by (namespace) (irate(container_network_receive_bytes_total{namespace!="", pod!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet"}[5m]) * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})` + m["pod_cpu_usage"] = `round(sum by (namespace, pod) (irate(container_cpu_usage_seconds_total{job="kubelet", pod!="", image!=""}[5m])) * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}, 0.001)` + m["pod_memory_usage"] = `sum by (namespace, pod) (container_memory_usage_bytes{job="kubelet", pod!="", image!=""}) * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}` + m["pod_memory_usage_wo_cache"] = `sum by (namespace, pod) (container_memory_working_set_bytes{job="kubelet", pod!="", image!=""}) * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}` + m["pod_net_bytes_transmitted"] = `sum by (namespace, pod) (irate(container_network_transmit_bytes_total{pod!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet"}[5m])) * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}` + m["pod_net_bytes_received"] = `sum by (namespace, pod) (irate(container_network_receive_bytes_total{pod!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet"}[5m])) * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}` + m["container_cpu_usage"] = `round(sum by (namespace, pod, container) (irate(container_cpu_usage_seconds_total{job="kubelet", container!="POD", container!="", image!="", $1}[5m])), 0.001)` + m["container_memory_usage"] = `sum by (namespace, pod, container) (container_memory_usage_bytes{job="kubelet", container!="POD", container!="", image!="", $1})` + m["container_memory_usage_wo_cache"] = `sum by (namespace, pod, container) (container_memory_working_set_bytes{job="kubelet", container!="POD", container!="", image!="", $1})` + } +}