From a38bb3784db3ce066a8c906893c31e4f8c8f9f9f Mon Sep 17 00:00:00 2001 From: jeff Date: Tue, 6 Nov 2018 14:48:15 +0800 Subject: [PATCH 1/4] component status --- .../v1alpha/components/components_handler.go | 23 +- pkg/models/components.go | 197 ++++++++---------- 2 files changed, 100 insertions(+), 120 deletions(-) diff --git a/pkg/apis/v1alpha/components/components_handler.go b/pkg/apis/v1alpha/components/components_handler.go index b4d16f548..342de69e7 100644 --- a/pkg/apis/v1alpha/components/components_handler.go +++ b/pkg/apis/v1alpha/components/components_handler.go @@ -30,22 +30,33 @@ func Register(ws *restful.WebService, subPath string) { ws.Route(ws.GET(subPath).To(handleGetComponents).Filter(route.RouteLogging)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) + ws.Route(ws.GET(subPath+"/{namespace}/{componentName}").To(handleGetComponentStatus). + Filter(route.RouteLogging)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) } -//get all components +// get a specific component status +func handleGetComponentStatus(request *restful.Request, response *restful.Response) { + namespace := request.PathParameter("namespace") + componentName := request.PathParameter("componentName") + if component, err := models.GetComponentStatus(namespace, componentName); err != nil { + response.WriteHeaderAndEntity(http.StatusInternalServerError, constants.MessageResponse{Message: err.Error()}) + } else { + response.WriteAsJson(component) + } +} + +// get all components func handleGetComponents(request *restful.Request, response *restful.Response) { - result, err := models.GetComponents() + result, err := models.GetAllComponentsStatus() if err != nil { - response.WriteHeaderAndEntity(http.StatusInternalServerError, constants.MessageResponse{Message: err.Error()}) - } else { - response.WriteAsJson(result) - } } diff --git a/pkg/models/components.go b/pkg/models/components.go index d58fab4f2..2d3cb0cbc 100644 --- a/pkg/models/components.go +++ b/pkg/models/components.go @@ -19,150 +19,119 @@ package models import ( "time" + "k8s.io/apimachinery/pkg/labels" + "github.com/golang/glog" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "kubesphere.io/kubesphere/pkg/client" - "kubesphere.io/kubesphere/pkg/constants" ) -type ComponentsCount struct { - KubernetesCount int `json:"kubernetesCount"` - OpenpitrixCount int `json:"openpitrixCount"` - KubesphereCount int `json:"kubesphereCount"` +// Namespaces need to watch +var SYSTEM_NAMESPACES = [...]string{"kubesphere-system", "openpitrix-system", "kube-system"} + +type Component struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + SelfLink string `json:"selfLink"` + Label interface{} `json:"label"` + StartedAt time.Time `json:"startedAt"` + TotalBackends int `json:"totalBackends"` + HealthyBackends int `json:"healthyBackends"` } -type Components struct { - Name string `json:"name"` - Namespace string `json:"namespace"` - SelfLink string `json:"selfLink"` - Label interface{} `json:"label"` - HealthStatus string `json:"healthStatus"` - CreateTime time.Time `json:"createTime"` -} - -/*** -* get all components from k8s and kubesphere system -* - */ -func GetComponents() (map[string]interface{}, error) { - - result := make(map[string]interface{}) - componentsList := make([]Components, 0) +func GetComponentStatus(namespace string, componentName string) (interface{}, error) { k8sClient := client.NewK8sClient() - var count ComponentsCount - var components Components - label := "" - namespaces := []string{constants.KubeSystemNamespace, constants.OpenPitrixNamespace, constants.KubeSphereNamespace} - for _, ns := range namespaces { + if service, err := k8sClient.CoreV1().Services(namespace).Get(componentName, meta_v1.GetOptions{}); err != nil { + glog.Error(err) + return nil, err + } else { + set := labels.Set(service.Spec.Selector) - if ns == constants.KubeSystemNamespace { - label = "kubernetes.io/cluster-service=true" - } else if ns == constants.OpenPitrixNamespace { - label = "app=openpitrix" - } else { - label = "app=kubesphere" + component := Component{ + Name: service.Name, + Namespace: service.Namespace, + SelfLink: service.SelfLink, + Label: service.Spec.Selector, + StartedAt: service.CreationTimestamp.Time, + HealthyBackends: 0, + TotalBackends: 0, } - option := meta_v1.ListOptions{ - - LabelSelector: label, - } - servicelists, err := k8sClient.CoreV1().Services(ns).List(option) - - if err != nil { + if pods, err := k8sClient.CoreV1().Pods(namespace).List(meta_v1.ListOptions{LabelSelector: set.AsSelector().String()}); err != nil { glog.Error(err) - - return result, err + return nil, err + } else { + for _, v := range pods.Items { + for _, c := range v.Status.ContainerStatuses { + component.TotalBackends++ + if c.Ready { + component.HealthyBackends++ + } + } + } } - if len(servicelists.Items) > 0 { + return component, nil + } - for _, service := range servicelists.Items { +} - switch ns { +func GetAllComponentsStatus() (map[string]interface{}, error) { - case constants.KubeSystemNamespace: - count.KubernetesCount++ - case constants.OpenPitrixNamespace: - count.OpenpitrixCount++ - default: - count.KubesphereCount++ - } + status := make(map[string]interface{}) + var err error - components.Name = service.Name - components.Namespace = service.Namespace - components.CreateTime = service.CreationTimestamp.Time - components.Label = service.Spec.Selector - components.SelfLink = service.SelfLink - label := service.Spec.Selector - combination := "" - for key, val := range label { + k8sClient := client.NewK8sClient() - labelstr := key + "=" + val + for _, ns := range SYSTEM_NAMESPACES { - if combination == "" { + nsStatus := make(map[string]interface{}) - combination = labelstr + services, err := k8sClient.CoreV1().Services(ns).List(meta_v1.ListOptions{}) + if err != nil { + glog.Error(err) + continue + } - } else { + for _, service := range services.Items { - combination = combination + "," + labelstr - - } - - } - option := meta_v1.ListOptions{ - LabelSelector: combination, - } - podsList, err := k8sClient.CoreV1().Pods(ns).List(option) - - if err != nil { - - glog.Error(err) - return result, err - } - - if len(podsList.Items) > 0 { - var health bool - for _, pod := range podsList.Items { - - for _, status := range pod.Status.ContainerStatuses { - - if status.Ready == false { - health = status.Ready - break - } else { - health = status.Ready - } - - } - - if health == false { - components.HealthStatus = "unhealth" - break - } - - } - - if health == true { - components.HealthStatus = "health" - } - - } else { - components.HealthStatus = "unhealth" - } - - componentsList = append(componentsList, components) + set := labels.Set(service.Spec.Selector) + if len(set) == 0 { + continue } + component := Component{ + Name: service.Name, + Namespace: service.Namespace, + SelfLink: service.SelfLink, + Label: service.Spec.Selector, + StartedAt: service.CreationTimestamp.Time, + HealthyBackends: 0, + TotalBackends: 0, + } + + if pods, err := k8sClient.CoreV1().Pods(ns).List(meta_v1.ListOptions{LabelSelector: set.AsSelector().String()}); err != nil { + glog.Error(err) + continue + } else { + for _, v := range pods.Items { + for _, c := range v.Status.ContainerStatuses { + component.TotalBackends++ + if c.Ready { + component.HealthyBackends++ + } + } + } + } + + nsStatus[service.Name] = component } + status[ns] = nsStatus } - result["count"] = count - result["item"] = componentsList - return result, nil + return status, err } From 3a38a83dd264a96b9110bc33bd56a5ffc6b7bbab Mon Sep 17 00:00:00 2001 From: jeff Date: Thu, 8 Nov 2018 11:29:42 +0800 Subject: [PATCH 2/4] component status --- pkg/models/components.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/models/components.go b/pkg/models/components.go index 2d3cb0cbc..12142f9ab 100644 --- a/pkg/models/components.go +++ b/pkg/models/components.go @@ -64,10 +64,12 @@ func GetComponentStatus(namespace string, componentName string) (interface{}, er return nil, err } else { for _, v := range pods.Items { + component.TotalBackends++ + component.HealthyBackends++ for _, c := range v.Status.ContainerStatuses { - component.TotalBackends++ - if c.Ready { - component.HealthyBackends++ + if !c.Ready { + component.HealthyBackends-- + break } } } @@ -118,10 +120,12 @@ func GetAllComponentsStatus() (map[string]interface{}, error) { continue } else { for _, v := range pods.Items { + component.TotalBackends++ + component.HealthyBackends++ for _, c := range v.Status.ContainerStatuses { - component.TotalBackends++ - if c.Ready { - component.HealthyBackends++ + if !c.Ready { + component.HealthyBackends-- + break } } } From f9057a070531e0cfcc120c38b2d15e3838b01838 Mon Sep 17 00:00:00 2001 From: Carman Zhang Date: Fri, 9 Nov 2018 17:57:36 +0800 Subject: [PATCH 3/4] reformat monitoring apis --- .../v1alpha/monitoring/monitor_handler.go | 257 ++++- pkg/client/prometheusclient.go | 127 ++- pkg/models/metrics/metrics.go | 993 ++++++++++++++++++ pkg/models/metrics/metricscollector.go | 539 ---------- pkg/models/metrics/metricsconst.go | 236 ----- pkg/models/metrics/metricsrule.go | 173 ++- pkg/models/metrics/metricsruleconst.go | 560 ++++++++++ pkg/models/metrics/metricsstruct.go | 69 -- pkg/models/metrics/util.go | 268 +++++ pkg/models/workspaces/workspaces.go | 40 + 10 files changed, 2232 insertions(+), 1030 deletions(-) create mode 100644 pkg/models/metrics/metrics.go delete mode 100755 pkg/models/metrics/metricscollector.go delete mode 100755 pkg/models/metrics/metricsconst.go create mode 100644 pkg/models/metrics/metricsruleconst.go delete mode 100755 pkg/models/metrics/metricsstruct.go create mode 100644 pkg/models/metrics/util.go diff --git a/pkg/apis/v1alpha/monitoring/monitor_handler.go b/pkg/apis/v1alpha/monitoring/monitor_handler.go index 59725ee39..36b580722 100755 --- a/pkg/apis/v1alpha/monitoring/monitor_handler.go +++ b/pkg/apis/v1alpha/monitoring/monitor_handler.go @@ -13,8 +13,6 @@ limitations under the License. package monitoring import ( - "strings" - "github.com/emicklei/go-restful" "github.com/emicklei/go-restful-openapi" @@ -23,94 +21,194 @@ import ( "kubesphere.io/kubesphere/pkg/models/metrics" ) -func (u MonitorResource) monitorPod(request *restful.Request, response *restful.Response) { - podName := strings.Trim(request.PathParameter("pod_name"), " ") +func (u Monitor) monitorPod(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + podName := requestParams.PodName + metricName := requestParams.MetricsName if podName != "" { // single pod single metric - metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") - res := metrics.MonitorPodSingleMetric(request, metricsName) + queryType, params, nullRule := metrics.AssemblePodMetricRequestInfo(requestParams, metricName) + var res *metrics.FormatedMetric + if !nullRule { + res = metrics.GetMetric(queryType, params, metricName) + } response.WriteAsJson(res) + } else { - // multiple pod multiple metric - res := metrics.MonitorAllMetrics(request) - response.WriteAsJson(res) + // multiple + rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelPod) + // sorting + sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelPodName) + // paging + pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + + response.WriteAsJson(pagedMetrics) } } -func (u MonitorResource) monitorContainer(request *restful.Request, response *restful.Response) { - metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") - promql := metrics.MakeContainerPromQL(request) - res := client.SendPrometheusRequest(request, promql) - cleanedJson := metrics.ReformatJson(res, metricsName) - response.WriteAsJson(cleanedJson) +func (u Monitor) monitorContainer(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + res := metrics.MonitorContainer(requestParams) + + response.WriteAsJson(res) } -func (u MonitorResource) monitorWorkload(request *restful.Request, response *restful.Response) { - wlKind := request.PathParameter("workload_kind") - if strings.Trim(wlKind, " ") == "" { +func (u Monitor) monitorWorkload(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + wlKind := requestParams.WorkloadKind + if wlKind == "" { // count all workloads figure - //metricName := "workload_count" - res := metrics.MonitorWorkloadCount(request) + res := metrics.MonitorWorkloadCount(requestParams.NsName) response.WriteAsJson(res) } else { - res := metrics.MonitorAllMetrics(request) + res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkload) response.WriteAsJson(res) } } // merge multiple metric: all-devops, all-roles, all-projects...this api is designed for admin -func (u MonitorResource) monitorWorkspaceUserInfo(request *restful.Request, response *restful.Response) { - res := metrics.MonitorWorkspaceUserInfo(request) +func (u Monitor) monitorAllWorkspacesStatistics(request *restful.Request, response *restful.Response) { + res := metrics.MonitorAllWorkspacesStatistics() response.WriteAsJson(res) } // merge multiple metric: devops, roles, projects... -func (u MonitorResource) monitorWorkspaceResourceLevelMetrics(request *restful.Request, response *restful.Response) { - res := metrics.MonitorWorkspaceResourceLevelMetrics(request) +func (u Monitor) monitorOneWorkspaceStatistics(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + wsName := requestParams.WsName + res := metrics.MonitorOneWorkspaceStatistics(wsName) response.WriteAsJson(res) } -func (u MonitorResource) monitorWorkspacePodLevelMetrics(request *restful.Request, response *restful.Response) { - res := metrics.MonitorAllMetrics(request) - response.WriteAsJson(res) +func (u Monitor) monitorAllWorkspaces(request *restful.Request, response *restful.Response) { + + requestParams := client.ParseMonitoringRequestParams(request) + + rawMetrics := metrics.MonitorAllWorkspaces(requestParams) + // sorting + sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelWorkspace) + // paging + pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + + response.WriteAsJson(pagedMetrics) } -func (u MonitorResource) monitorNamespace(request *restful.Request, response *restful.Response) { - nsName := strings.Trim(request.PathParameter("ns_name"), " ") +func (u Monitor) monitorOneWorkspace(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + + tp := requestParams.Tp + if tp != "" { + // multiple + rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkspace) + // sorting + sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelNamespace) + // paging + pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + + response.WriteAsJson(pagedMetrics) + + } else { + res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkspace) + response.WriteAsJson(res) + } +} + +func (u Monitor) monitorNamespace(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + metricName := requestParams.MetricsName + nsName := requestParams.NsName if nsName != "" { // single - metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") - res := metrics.MonitorNamespaceSingleMetric(request, metricsName) + queryType, params := metrics.AssembleNamespaceMetricRequestInfo(requestParams, metricName) + res := metrics.GetMetric(queryType, params, metricName) response.WriteAsJson(res) } else { // multiple - res := metrics.MonitorAllMetrics(request) - response.WriteAsJson(res) + rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelNamespace) + // sorting + sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelNamespace) + // paging + pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + + response.WriteAsJson(pagedMetrics) } } -func (u MonitorResource) monitorNodeorCluster(request *restful.Request, response *restful.Response) { - metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") - //var res *metrics.FormatedMetric - if metricsName != "" { +func (u Monitor) monitorCluster(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + + metricName := requestParams.MetricsName + if metricName != "" { // single - res := metrics.MonitorNodeorClusterSingleMetric(request, metricsName) + queryType, params := metrics.AssembleClusterMetricRequestInfo(requestParams, metricName) + res := metrics.GetMetric(queryType, params, metricName) + + if metricName == metrics.MetricNameWorkspaceAllProjectCount { + res = metrics.MonitorWorkspaceNamespaceHistory(res) + } + response.WriteAsJson(res) } else { // multiple - res := metrics.MonitorAllMetrics(request) + res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelCluster) response.WriteAsJson(res) } } -type MonitorResource struct { +func (u Monitor) monitorNode(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + + metricName := requestParams.MetricsName + if metricName != "" { + // single + queryType, params := metrics.AssembleNodeMetricRequestInfo(requestParams, metricName) + res := metrics.GetMetric(queryType, params, metricName) + nodeAddress := metrics.GetNodeAddressInfo() + metrics.AddNodeAddressMetric(res, nodeAddress) + response.WriteAsJson(res) + } else { + // multiple + rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelNode) + nodeAddress := metrics.GetNodeAddressInfo() + + for i := 0; i < len(rawMetrics.Results); i++ { + metrics.AddNodeAddressMetric(&rawMetrics.Results[i], nodeAddress) + } + + // sorting + sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelNode) + // paging + pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + + response.WriteAsJson(pagedMetrics) + } +} + +// k8s component(controller, scheduler, etcd) status +func (u Monitor) monitorComponentStatus(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + + status := metrics.MonitorComponentStatus(requestParams) + response.WriteAsJson(status) +} + +func (u Monitor) monitorEvents(request *restful.Request, response *restful.Response) { + // k8s component healthy status + requestParams := client.ParseMonitoringRequestParams(request) + + nsFilter := requestParams.NsFilter + events := metrics.MonitorEvents(nsFilter) + response.WriteAsJson(events) +} + +type Monitor struct { } func Register(ws *restful.WebService, subPath string) { tags := []string{"monitoring apis"} - u := MonitorResource{} + u := Monitor{} - ws.Route(ws.GET(subPath+"/clusters").To(u.monitorNodeorCluster). + ws.Route(ws.GET(subPath+"/clusters").To(u.monitorCluster). Filter(route.RouteLogging). Doc("monitor cluster level metrics"). Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("cluster_cpu_utilisation")). @@ -118,16 +216,20 @@ func Register(ws *restful.WebService, subPath string) { Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) - ws.Route(ws.GET(subPath+"/nodes").To(u.monitorNodeorCluster). + ws.Route(ws.GET(subPath+"/nodes").To(u.monitorNode). Filter(route.RouteLogging). Doc("monitor nodes level metrics"). Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("node_cpu_utilisation")). Param(ws.QueryParameter("nodes_filter", "node re2 expression filter").DataType("string").Required(false).DefaultValue("")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) - ws.Route(ws.GET(subPath+"/nodes/{node_id}").To(u.monitorNodeorCluster). + ws.Route(ws.GET(subPath+"/nodes/{node_id}").To(u.monitorNode). Filter(route.RouteLogging). Doc("monitor specific node level metrics"). Param(ws.PathParameter("node_id", "specific node").DataType("string").Required(true).DefaultValue("")). @@ -141,6 +243,10 @@ func Register(ws *restful.WebService, subPath string) { Doc("monitor namespaces level metrics"). Param(ws.QueryParameter("namespaces_filter", "namespaces re2 expression filter").DataType("string").Required(false).DefaultValue("")). Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("namespace_memory_utilisation")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) @@ -158,8 +264,12 @@ func Register(ws *restful.WebService, subPath string) { Filter(route.RouteLogging). Doc("monitor pods level metrics"). Param(ws.PathParameter("ns_name", "specific namespace").DataType("string").Required(true).DefaultValue("monitoring")). - Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("")). Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("pod_memory_utilisation_wo_cache")). + Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) @@ -178,8 +288,12 @@ func Register(ws *restful.WebService, subPath string) { Filter(route.RouteLogging). Doc("monitor pods level metrics by nodeid"). Param(ws.PathParameter("node_id", "specific node").DataType("string").Required(true).DefaultValue("i-k89a62il")). - Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("openpitrix.*")). Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("pod_memory_utilisation_wo_cache")). + Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("openpitrix.*")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) @@ -236,27 +350,64 @@ func Register(ws *restful.WebService, subPath string) { Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) - ws.Route(ws.GET(subPath+"/workspaces/{workspace_name}/pods").To(u.monitorWorkspacePodLevelMetrics). + // merge multiple metric: devops, roles, projects... + ws.Route(ws.GET(subPath+"/cluster_workspaces/{workspace_name}/_statistics").To(u.monitorOneWorkspaceStatistics). Filter(route.RouteLogging). Doc("monitor specific workspace level metrics"). Param(ws.PathParameter("workspace_name", "workspace name").DataType("string").Required(true)). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + // list all namespace in this workspace by selected metrics + ws.Route(ws.GET(subPath+"/cluster_workspaces/{workspace_name}").To(u.monitorOneWorkspace). + Filter(route.RouteLogging). + Doc("monitor workspaces level metrics"). + Param(ws.PathParameter("workspace_name", "workspace name").DataType("string").Required(true)). Param(ws.QueryParameter("namespaces_filter", "namespaces filter").DataType("string").Required(false).DefaultValue("k.*")). - Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...").DataType("string").Required(false).DefaultValue("tenant_memory_utilisation_wo_cache")). + Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("namespace_memory_utilisation_wo_cache")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). + Param(ws.QueryParameter("type", "page number").DataType("string").Required(false).DefaultValue("1")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) - ws.Route(ws.GET(subPath+"/workspaces/{workspace_name}").To(u.monitorWorkspaceResourceLevelMetrics). + // metrics from system users or projects, merge multiple metric: all-devops, all-roles, all-projects... + ws.Route(ws.GET(subPath+"/cluster_workspaces/_statistics").To(u.monitorAllWorkspacesStatistics). Filter(route.RouteLogging). Doc("monitor specific workspace level metrics"). - Param(ws.PathParameter("workspace_name", "workspace name").DataType("string").Required(true)). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) - ws.Route(ws.GET(subPath+"/workspaces").To(u.monitorWorkspaceUserInfo). + // metrics from prometheus + ws.Route(ws.GET(subPath+"/cluster_workspaces").To(u.monitorAllWorkspaces). Filter(route.RouteLogging). - Doc("monitor specific workspace level metrics"). + Doc("monitor workspaces level metrics"). + Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("workspace_memory_utilisation")). + Param(ws.QueryParameter("workspaces_filter", "workspaces re2 expression filter").DataType("string").Required(false).DefaultValue(".*")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/events").To(u.monitorEvents). + Filter(route.RouteLogging). + Doc("monitor k8s events"). + Param(ws.QueryParameter("namespaces_filter", "namespaces filter").DataType("string").Required(false).DefaultValue(".*")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/components").To(u.monitorComponentStatus). + Filter(route.RouteLogging). + Doc("monitor k8s components status"). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) diff --git a/pkg/client/prometheusclient.go b/pkg/client/prometheusclient.go index 95992936f..a7c776a48 100644 --- a/pkg/client/prometheusclient.go +++ b/pkg/client/prometheusclient.go @@ -16,13 +16,12 @@ import ( "io/ioutil" "net/http" "net/url" - "strconv" + "strings" "time" "github.com/emicklei/go-restful" "github.com/golang/glog" - "github.com/pkg/errors" ) const ( @@ -33,13 +32,38 @@ const ( PrometheusEndpointUrl = DefaultScheme + "://" + DefaultPrometheusService + ":" + DefaultPrometheusPort + PrometheusApiPath DefaultQueryStep = "10m" DefaultQueryTimeout = "30s" + RangeQueryType = "query_range?" + DefaultQueryType = "query?" ) +type MonitoringRequestParams struct { + Params url.Values + QueryType string + SortMetricName string + SortType string + PageNum string + LimitNum string + Tp string + MetricsFilter string + NodesFilter string + WsFilter string + NsFilter string + PodsFilter string + ContainersFilter string + MetricsName string + WorkloadName string + NodeId string + WsName string + NsName string + PodName string + ContainerName string + WorkloadKind string +} + var client = &http.Client{} -func SendRequest(postfix string, params string) string { - epurl := PrometheusEndpointUrl + postfix + params - //glog.Info("monitoring epurl:>", epurl) +func SendMonitoringRequest(queryType string, params string) string { + epurl := PrometheusEndpointUrl + queryType + params response, err := client.Get(epurl) if err != nil { glog.Error(err) @@ -56,34 +80,57 @@ func SendRequest(postfix string, params string) string { return "" } -func SendPrometheusRequest(request *restful.Request, recordingRule string) string { - paramsMap, bol, err := ParseRequestHeader(request) - if err != nil { - glog.Error(err) - return "" - } +func ParseMonitoringRequestParams(request *restful.Request) *MonitoringRequestParams { + instantTime := strings.Trim(request.QueryParameter("time"), " ") + start := strings.Trim(request.QueryParameter("start"), " ") + end := strings.Trim(request.QueryParameter("end"), " ") + step := strings.Trim(request.QueryParameter("step"), " ") + timeout := strings.Trim(request.QueryParameter("timeout"), " ") - var res = "" - var postfix = "" - if bol { - // range query - postfix = "query_range?" - } else { - // query - postfix = "query?" - } - paramsMap.Set("query", recordingRule) - params := paramsMap.Encode() - res = SendRequest(postfix, params) - return res -} + sortMetricName := strings.Trim(request.QueryParameter("sort_metric"), " ") + sortType := strings.Trim(request.QueryParameter("sort_type"), " ") + pageNum := strings.Trim(request.QueryParameter("page"), " ") + limitNum := strings.Trim(request.QueryParameter("limit"), " ") + tp := strings.Trim(request.QueryParameter("type"), " ") -func ParseRequestHeader(request *restful.Request) (url.Values, bool, error) { - instantTime := request.QueryParameter("time") - start := request.QueryParameter("start") - end := request.QueryParameter("end") - step := request.QueryParameter("step") - timeout := request.QueryParameter("timeout") + metricsFilter := strings.Trim(request.QueryParameter("metrics_filter"), " ") + nodesFilter := strings.Trim(request.QueryParameter("nodes_filter"), " ") + wsFilter := strings.Trim(request.QueryParameter("workspaces_filter"), " ") + nsFilter := strings.Trim(request.QueryParameter("namespaces_filter"), " ") + podsFilter := strings.Trim(request.QueryParameter("pods_filter"), " ") + containersFilter := strings.Trim(request.QueryParameter("containers_filter"), " ") + + metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") + workloadName := strings.Trim(request.QueryParameter("workload_name"), " ") + + nodeId := strings.Trim(request.PathParameter("node_id"), " ") + wsName := strings.Trim(request.PathParameter("workspace_name"), " ") + nsName := strings.Trim(request.PathParameter("ns_name"), " ") + podName := strings.Trim(request.PathParameter("pod_name"), " ") + containerName := strings.Trim(request.PathParameter("container_name"), " ") + workloadKind := strings.Trim(request.PathParameter("workload_kind"), " ") + + var requestParams = MonitoringRequestParams{ + SortMetricName: sortMetricName, + SortType: sortType, + PageNum: pageNum, + LimitNum: limitNum, + Tp: tp, + MetricsFilter: metricsFilter, + NodesFilter: nodesFilter, + WsFilter: wsFilter, + NsFilter: nsFilter, + PodsFilter: podsFilter, + ContainersFilter: containersFilter, + MetricsName: metricsName, + WorkloadName: workloadName, + NodeId: nodeId, + WsName: wsName, + NsName: nsName, + PodName: podName, + ContainerName: containerName, + WorkloadKind: workloadKind, + } if timeout == "" { timeout = DefaultQueryTimeout @@ -93,25 +140,35 @@ func ParseRequestHeader(request *restful.Request) (url.Values, bool, error) { } // Whether query or query_range request u := url.Values{} + if start != "" && end != "" { u.Set("start", convertTimeGranularity(start)) u.Set("end", convertTimeGranularity(end)) u.Set("step", step) u.Set("timeout", timeout) - return u, true, nil + requestParams.QueryType = RangeQueryType + requestParams.Params = u + return &requestParams } if instantTime != "" { u.Set("time", instantTime) u.Set("timeout", timeout) - return u, false, nil + requestParams.QueryType = DefaultQueryType + requestParams.Params = u + return &requestParams } else { //u.Set("time", strconv.FormatInt(int64(time.Now().Unix()), 10)) u.Set("timeout", timeout) - return u, false, nil + requestParams.QueryType = DefaultQueryType + requestParams.Params = u + return &requestParams } glog.Errorln("Parse request %s failed", u) - return u, false, errors.Errorf("Parse request time range %s failed", u) + requestParams.QueryType = DefaultQueryType + requestParams.Params = u + + return &requestParams } func convertTimeGranularity(ts string) string { diff --git a/pkg/models/metrics/metrics.go b/pkg/models/metrics/metrics.go new file mode 100644 index 000000000..0127dd5ab --- /dev/null +++ b/pkg/models/metrics/metrics.go @@ -0,0 +1,993 @@ +/* +Copyright 2018 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "encoding/json" + "fmt" + "net/url" + "regexp" + "strings" + "sync" + "time" + + "github.com/golang/glog" + "github.com/pkg/errors" + "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "kubesphere.io/kubesphere/pkg/client" + "kubesphere.io/kubesphere/pkg/models" + "kubesphere.io/kubesphere/pkg/models/workspaces" +) + +var nodeStatusDelLables = []string{"endpoint", "instance", "job", "namespace", "pod", "service"} + +const ( + ChannelMaxCapacityWorkspaceMetric = 400 + ChannelMaxCapacity = 100 +) + +type PagedFormatedLevelMetric struct { + CurrentPage int `json:"page"` + TotalPage int `json:"total_page"` + Message string `json:"msg"` + Metric FormatedLevelMetric `json:"metrics"` +} + +type FormatedLevelMetric struct { + MetricsLevel string `json:"metrics_level"` + Results []FormatedMetric `json:"results"` +} + +type FormatedMetric struct { + MetricName string `json:"metric_name, omitempty"` + Status string `json:"status"` + Data FormatedMetricData `json:"data, omitempty"` +} + +type FormatedMetricData struct { + Result []map[string]interface{} `json:"result"` + ResultType string `json:"resultType"` +} + +type MetricResultValues []MetricResultValue + +type MetricResultValue struct { + timestamp float64 + value string +} + +type MetricItem struct { + MetricLabel map[string]string `json:"metric"` + Value []interface{} `json:"value"` +} + +type CommonMetricsResult struct { + Status string `json:"status"` + Data CommonMetricsData `json:"data"` +} + +type CommonMetricsData struct { + Result []CommonResultItem `json:"result"` + ResultType string `json:"resultType"` +} + +type CommonResultItem struct { + KubePodMetric KubePodMetric `json:"metric"` + Value interface{} `json:"value"` +} + +type KubePodMetric struct { + CreatedByKind string `json:"created_by_kind"` + CreatedByName string `json:"created_by_name"` + Namespace string `json:"namespace"` + Pod string `json:"pod"` +} + +type ComponentStatus struct { + Name string `json:"metric_name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ComponentStatus []OneComponentStatus `json:"component"` +} + +type OneComponentStatus struct { + // Valid value: "Healthy" + Type string `json:"type"` + // Valid values for "Healthy": "True", "False", or "Unknown". + Status string `json:"status"` + // Message about the condition for a component. + Message string `json:"message,omitempty"` + // Condition error code for a component. + Error string `json:"error,omitempty"` +} + +func getPodNameRegexInWorkload(res string) string { + + data := []byte(res) + var dat CommonMetricsResult + jsonErr := json.Unmarshal(data, &dat) + if jsonErr != nil { + glog.Errorln("json parse failed", jsonErr) + } + var podNames []string + for _, item := range dat.Data.Result { + podName := item.KubePodMetric.Pod + podNames = append(podNames, podName) + } + podNamesFilter := "^(" + strings.Join(podNames, "|") + ")$" + return podNamesFilter +} + +func AssembleWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { + rule := MakeWorkloadRule(monitoringRequest.WorkloadKind, monitoringRequest.WorkloadName, monitoringRequest.NsName) + paramValues := monitoringRequest.Params + params := makeRequestParamString(rule, paramValues) + + res := client.SendMonitoringRequest(client.DefaultQueryType, params) + + podNamesFilter := getPodNameRegexInWorkload(res) + + queryType := monitoringRequest.QueryType + rule = MakePodPromQL(metricName, monitoringRequest.NsName, "", "", podNamesFilter) + params = makeRequestParamString(rule, paramValues) + + return queryType, params +} + +func AssemblePodMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) { + queryType := monitoringRequest.QueryType + + paramValues := monitoringRequest.Params + + rule := MakePodPromQL(metricName, monitoringRequest.NsName, monitoringRequest.NodeId, monitoringRequest.PodName, monitoringRequest.PodsFilter) + params := makeRequestParamString(rule, paramValues) + return queryType, params, rule == "" +} + +func GetMetric(queryType, params, metricName string) *FormatedMetric { + res := client.SendMonitoringRequest(queryType, params) + formatedMetric := ReformatJson(res, metricName) + return formatedMetric +} + +func GetNodeAddressInfo() *map[string][]v1.NodeAddress { + nodeList, _ := client.NewK8sClient().CoreV1().Nodes().List(metaV1.ListOptions{}) + var nodeAddress = make(map[string][]v1.NodeAddress) + for _, node := range nodeList.Items { + nodeAddress[node.Name] = node.Status.Addresses + } + return &nodeAddress +} + +func AddNodeAddressMetric(nodeMetric *FormatedMetric, nodeAddress *map[string][]v1.NodeAddress) { + + for i := 0; i < len(nodeMetric.Data.Result); i++ { + metricDesc := nodeMetric.Data.Result[i][ResultItemMetric] + metricDescMap := metricDesc.(map[string]interface{}) + if nodeId, exist := metricDescMap["node"]; exist { + addr, exist := (*nodeAddress)[nodeId.(string)] + if exist { + metricDescMap["address"] = addr + } + } + } +} + +func MonitorContainer(monitoringRequest *client.MonitoringRequestParams) *FormatedMetric { + queryType := monitoringRequest.QueryType + + paramValues := monitoringRequest.Params + rule := MakeContainerPromQL(monitoringRequest.NsName, monitoringRequest.PodName, monitoringRequest.ContainerName, monitoringRequest.MetricsName, monitoringRequest.ContainersFilter) + params := makeRequestParamString(rule, paramValues) + + res := GetMetric(queryType, params, monitoringRequest.MetricsName) + return res +} + +func AssembleNamespaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { + queryType := monitoringRequest.QueryType + + paramValues := monitoringRequest.Params + rule := MakeNamespacePromQL(monitoringRequest.NsName, monitoringRequest.NsFilter, metricName) + params := makeRequestParamString(rule, paramValues) + + return queryType, params +} + +func AssembleWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) { + nsFilter := "^(" + strings.Join(namespaceList, "|") + ")$" + + queryType := monitoringRequest.QueryType + + rule := MakeWorkspacePromQL(metricName, nsFilter) + paramValues := monitoringRequest.Params + params := makeRequestParamString(rule, paramValues) + return queryType, params +} + +func makeRequestParamString(rule string, paramValues url.Values) string { + + var values = make(url.Values) + for key, v := range paramValues { + values.Set(key, v[0]) + } + + values.Set("query", rule) + + params := values.Encode() + + return params +} + +func filterNamespace(nsFilter string, namespaceList []string) []string { + var newNSlist []string + if nsFilter == "" { + nsFilter = ".*" + } + for _, ns := range namespaceList { + bol, _ := regexp.MatchString(nsFilter, ns) + if bol { + newNSlist = append(newNSlist, ns) + } + } + return newNSlist +} + +func MonitorAllWorkspaces(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric { + metricsFilter := monitoringRequest.MetricsFilter + if strings.Trim(metricsFilter, " ") == "" { + metricsFilter = ".*" + } + var filterMetricsName []string + for _, metricName := range WorkspaceMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + filterMetricsName = append(filterMetricsName, metricName) + } + } + + var wgAll sync.WaitGroup + var wsAllch = make(chan *[]FormatedMetric, ChannelMaxCapacityWorkspaceMetric) + + workspaceNamespaceMap, _, _ := workspaces.GetAllOrgAndProjList() + for ws, _ := range workspaceNamespaceMap { + bol, err := regexp.MatchString(monitoringRequest.WsFilter, ws) + if err == nil && bol { + // a workspace + wgAll.Add(1) + go collectWorkspaceMetric(monitoringRequest, ws, filterMetricsName, &wgAll, wsAllch) + } + } + + wgAll.Wait() + close(wsAllch) + + fmtMetricMap := make(map[string]FormatedMetric) + for oneWsMetric := range wsAllch { + if oneWsMetric != nil { + // aggregate workspace metric + for _, metric := range *oneWsMetric { + fm, exist := fmtMetricMap[metric.MetricName] + if exist { + if metric.Status == "error" { + fm.Status = metric.Status + } + fm.Data.Result = append(fm.Data.Result, metric.Data.Result...) + fmtMetricMap[metric.MetricName] = fm + } else { + fmtMetricMap[metric.MetricName] = metric + } + } + } + } + + var metricArray = make([]FormatedMetric, 0) + for _, metric := range fmtMetricMap { + metricArray = append(metricArray, metric) + } + + return &FormatedLevelMetric{ + MetricsLevel: MetricLevelClusterWorkspace, + Results: metricArray, + } +} + +func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, ws string, filterMetricsName []string, wgAll *sync.WaitGroup, wsAllch chan *[]FormatedMetric) { + defer wgAll.Done() + var wg sync.WaitGroup + var ch = make(chan *FormatedMetric, ChannelMaxCapacity) + namespaceArray, err := workspaces.WorkspaceNamespaces(ws) + if err != nil { + glog.Errorln(err.Error()) + } + // add by namespace + for _, metricName := range filterMetricsName { + wg.Add(1) + go func(metricName string) { + defer func() { + if err := recover(); err != nil { + glog.Errorln(err) + } + }() + + queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName) + ch <- GetMetric(queryType, params, metricName) + + wg.Done() + }(metricName) + } + + wg.Wait() + close(ch) + + var metricsArray []FormatedMetric + for oneMetric := range ch { + if oneMetric != nil { + // add "workspace" filed to oneMetric `metric` field + for i := 0; i < len(oneMetric.Data.Result); i++ { + tmap := oneMetric.Data.Result[i]["metric"].(map[string]interface{}) + tmap[MetricLevelWorkspace] = ws + oneMetric.Data.Result[i]["metric"] = tmap + } + metricsArray = append(metricsArray, *oneMetric) + } + } + + wsAllch <- &metricsArray +} + +func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resourceType string) *FormatedLevelMetric { + metricsFilter := monitoringRequest.MetricsFilter + if metricsFilter == "" { + metricsFilter = ".*" + } + + var ch = make(chan *FormatedMetric, ChannelMaxCapacity) + var wg sync.WaitGroup + + switch resourceType { + case MetricLevelCluster: + { + for _, metricName := range ClusterMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params := AssembleClusterMetricRequestInfo(monitoringRequest, metricName) + clusterMetrics := GetMetric(queryType, params, metricName) + + // for this special case, get namespace history which in a workspace by determining namespace label + if metricName == MetricNameWorkspaceAllProjectCount { + clusterMetrics = MonitorWorkspaceNamespaceHistory(clusterMetrics) + } + + ch <- clusterMetrics + + wg.Done() + }(metricName) + } + } + } + case MetricLevelNode: + { + for _, metricName := range NodeMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params := AssembleNodeMetricRequestInfo(monitoringRequest, metricName) + ch <- GetMetric(queryType, params, metricName) + wg.Done() + }(metricName) + } + } + } + case MetricLevelWorkspace: + { + namespaceArray, err := workspaces.WorkspaceNamespaces(monitoringRequest.WsName) + if err != nil { + glog.Errorln(err.Error()) + } + namespaceArray = filterNamespace(monitoringRequest.NsFilter, namespaceArray) + + if monitoringRequest.Tp == "" { + for _, metricName := range WorkspaceMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName) + ch <- GetMetric(queryType, params, metricName) + wg.Done() + }(metricName) + } + } + } else { + for _, metricName := range NamespaceMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + ns := "^(" + strings.Join(namespaceArray, "|") + ")$" + monitoringRequest.NsFilter = ns + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params := AssembleNamespaceMetricRequestInfo(monitoringRequest, metricName) + ch <- GetMetric(queryType, params, metricName) + wg.Done() + }(metricName) + } + } + } + } + case MetricLevelNamespace: + { + for _, metricName := range NamespaceMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params := AssembleNamespaceMetricRequestInfo(monitoringRequest, metricName) + ch <- GetMetric(queryType, params, metricName) + wg.Done() + }(metricName) + } + } + } + case MetricLevelWorkload: + { + for _, metricName := range WorkloadMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + metricName = strings.TrimLeft(metricName, "workload_") + queryType, params := AssembleWorkloadMetricRequestInfo(monitoringRequest, metricName) + ch <- GetMetric(queryType, params, metricName) + wg.Done() + }(metricName) + } + } + } + case MetricLevelPod: + { + for _, metricName := range PodMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params, nullRule := AssemblePodMetricRequestInfo(monitoringRequest, metricName) + if !nullRule { + ch <- GetMetric(queryType, params, metricName) + } else { + ch <- nil + } + wg.Done() + }(metricName) + } + } + } + } + + wg.Wait() + close(ch) + + var metricsArray []FormatedMetric + + for oneMetric := range ch { + if oneMetric != nil { + metricsArray = append(metricsArray, *oneMetric) + } + } + + return &FormatedLevelMetric{ + MetricsLevel: resourceType, + Results: metricsArray, + } +} + +func MonitorWorkspaceNamespaceHistory(metric *FormatedMetric) *FormatedMetric { + resultType := metric.Data.ResultType + //metric.Status + metricName := metric.MetricName + + for i := 0; i < len(metric.Data.Result); i++ { + metricItem := metric.Data.Result[i] + + if resultType == ResultTypeVector { + timeAndValue, sure := metricItem[ResultItemValue].([]interface{}) + if !sure { + return metric + } + metric := getNamespaceHistoryMetric(timeAndValue[0].(float64), metricName) + + workspaceNamespaceCount := calcWorkspaceNamespace(metric) + + timeAndValue[1] = fmt.Sprintf("%d", workspaceNamespaceCount) + + } else if resultType == ResultTypeMatrix { + + values, sure := metricItem[ResultItemValues].([]interface{}) + if !sure { + return metric + } + + for _, valueItem := range values { + timeAndValue, sure := valueItem.([]interface{}) + if !sure { + return metric + } + + metric := getNamespaceHistoryMetric(timeAndValue[0].(float64), metricName) + + workspaceNamespaceCount := calcWorkspaceNamespace(metric) + + timeAndValue[1] = fmt.Sprintf("%d", workspaceNamespaceCount) + } + } + } + + return metric +} + +func getNamespaceHistoryMetric(timestamp float64, metricName string) *FormatedMetric { + var timeRelatedParams = make(url.Values) + timeRelatedParams.Set("time", fmt.Sprintf("%f", timestamp)) + timeRelatedParams.Set("query", NamespaceLabelRule) + params := timeRelatedParams.Encode() + metric := GetMetric(client.DefaultQueryType, params, metricName) + return metric +} + +// calculate all namespace which belong to workspaces +func calcWorkspaceNamespace(metric *FormatedMetric) int { + if metric.Status == "error" { + glog.Errorf("failed when retrive namespace history, the metric is %v", metric.Data.Result) + return 0 + } + + var workspaceNamespaceCount = 0 + + for _, result := range metric.Data.Result { + tmpMap := result[ResultItemMetric].(map[string]interface{}) + wsName, exist := tmpMap[WorkspaceJoinedKey] + + if exist && wsName != "" { + workspaceNamespaceCount += 1 + } + } + + return workspaceNamespaceCount +} + +func MonitorAllWorkspacesStatistics() *FormatedLevelMetric { + + wg := sync.WaitGroup{} + var metricsArray []FormatedMetric + timestamp := time.Now().Unix() + + var orgResultItem *FormatedMetric + var devopsResultItem *FormatedMetric + var clusterProjResultItem *FormatedMetric + var workspaceProjResultItem *FormatedMetric + var accountResultItem *FormatedMetric + + wsMap, projs, errProj := workspaces.GetAllOrgAndProjList() + + wg.Add(5) + + go func() { + orgNums, errOrg := workspaces.GetAllOrgNums() + if errOrg != nil { + glog.Errorln(errOrg.Error()) + } + orgResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllOrganizationCount, WorkspaceResourceKindOrganization, orgNums, errOrg) + wg.Done() + }() + + go func() { + devOpsProjectNums, errDevops := workspaces.GetAllDevOpsProjectsNums() + if errDevops != nil { + glog.Errorln(errDevops.Error()) + } + devopsResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllDevopsCount, WorkspaceResourceKindDevops, devOpsProjectNums, errDevops) + wg.Done() + }() + + go func() { + var projNums = 0 + for _, v := range wsMap { + projNums += len(v) + } + if errProj != nil { + glog.Errorln(errProj.Error()) + } + workspaceProjResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllProjectCount, WorkspaceResourceKindNamespace, projNums, errProj) + wg.Done() + }() + + go func() { + projNums := len(projs) + if errProj != nil { + glog.Errorln(errProj.Error()) + } + clusterProjResultItem = getSpecificMetricItem(timestamp, MetricNameClusterAllProjectCount, WorkspaceResourceKindNamespace, projNums, errProj) + wg.Done() + }() + + go func() { + actNums, errAct := workspaces.GetAllAccountNums() + if errAct != nil { + glog.Errorln(errAct.Error()) + } + accountResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllAccountCount, WorkspaceResourceKindAccount, actNums, errAct) + wg.Done() + }() + + wg.Wait() + + metricsArray = append(metricsArray, *orgResultItem, *devopsResultItem, *workspaceProjResultItem, *accountResultItem, *clusterProjResultItem) + + return &FormatedLevelMetric{ + MetricsLevel: MetricLevelWorkspace, + Results: metricsArray, + } +} + +func MonitorOneWorkspaceStatistics(wsName string) *FormatedLevelMetric { + + var nsMetrics *FormatedMetric + var devopsMetrics *FormatedMetric + var memberMetrics *FormatedMetric + var roleMetrics *FormatedMetric + + wg := sync.WaitGroup{} + wg.Add(4) + + var fMetricsArray []FormatedMetric + timestamp := int64(time.Now().Unix()) + + go func() { + // add namespaces(project) metric + namespaces, errNs := workspaces.WorkspaceNamespaces(wsName) + namespaces, noneExistentNs := getExistingNamespace(namespaces) + if len(noneExistentNs) != 0 { + nsStr := strings.Join(noneExistentNs, "|") + errStr := "the namespaces " + nsStr + " do not exist" + if errNs == nil { + errNs = errors.New(errStr) + } else { + errNs = errors.New(errNs.Error() + "\t" + errStr) + } + } + if errNs != nil { + glog.Errorln(errNs.Error()) + } + nsMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceNamespaceCount, WorkspaceResourceKindNamespace, len(namespaces), errNs) + wg.Done() + }() + + go func() { + devOpsProjects, errDevOps := workspaces.GetDevOpsProjects(wsName) + if errDevOps != nil { + glog.Errorln(errDevOps.Error()) + } + // add devops metric + devopsMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceDevopsCount, WorkspaceResourceKindDevops, len(devOpsProjects), errDevOps) + wg.Done() + }() + + go func() { + members, errMemb := workspaces.GetOrgMembers(wsName) + if errMemb != nil { + glog.Errorln(errMemb.Error()) + } + // add member metric + memberMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceMemberCount, WorkspaceResourceKindMember, len(members), errMemb) + wg.Done() + }() + + go func() { + roles, errRole := workspaces.GetOrgRoles(wsName) + if errRole != nil { + glog.Errorln(errRole.Error()) + } + // add role metric + roleMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceRoleCount, WorkspaceResourceKindRole, len(roles), errRole) + wg.Done() + }() + + wg.Wait() + + fMetricsArray = append(fMetricsArray, *nsMetrics, *devopsMetrics, *memberMetrics, *roleMetrics) + + return &FormatedLevelMetric{ + MetricsLevel: MetricLevelWorkspace, + Results: fMetricsArray, + } +} + +func getSpecificMetricItem(timestamp int64, metricName string, resource string, count int, err error, resourceType ...string) *FormatedMetric { + var nsMetrics FormatedMetric + nsMetrics.MetricName = metricName + nsMetrics.Data.ResultType = ResultTypeVector + resultItem := make(map[string]interface{}) + tmp := make(map[string]string) + + if len(resourceType) > 0 { + tmp[resourceType[0]] = resource + } else { + tmp[ResultItemMetricResource] = resource + } + + if err == nil { + nsMetrics.Status = MetricStatusSuccess + } else { + nsMetrics.Status = MetricStatusError + resultItem["errormsg"] = err.Error() + } + + resultItem[ResultItemMetric] = tmp + resultItem[ResultItemValue] = []interface{}{timestamp, count} + nsMetrics.Data.Result = make([]map[string]interface{}, 1) + nsMetrics.Data.Result[0] = resultItem + return &nsMetrics +} + +// k8s component(controller, scheduler, etcd) status +func MonitorComponentStatus(monitoringRequest *client.MonitoringRequestParams) *[]interface{} { + componentList, err := client.NewK8sClient().CoreV1().ComponentStatuses().List(metaV1.ListOptions{}) + if err != nil { + glog.Errorln(err.Error()) + } + + var componentStatusList []*ComponentStatus + for _, item := range componentList.Items { + var status []OneComponentStatus + for _, cond := range item.Conditions { + status = append(status, OneComponentStatus{ + Type: string(cond.Type), + Status: string(cond.Status), + Message: cond.Message, + Error: cond.Error, + }) + } + + componentStatusList = append(componentStatusList, &ComponentStatus{ + Name: item.Name, + Namespace: item.Namespace, + Labels: item.Labels, + ComponentStatus: status, + }) + } + + // node status + queryType := monitoringRequest.QueryType + paramValues := monitoringRequest.Params + paramValues.Set("query", NodeStatusRule) + params := paramValues.Encode() + res := client.SendMonitoringRequest(queryType, params) + + nodeStatusMetric := ReformatJson(res, "node_status", nodeStatusDelLables...) + nodeStatusMetric = ReformatNodeStatusField(nodeStatusMetric) + + var normalNodes []string + var abnormalNodes []string + for _, result := range nodeStatusMetric.Data.Result { + tmap := result[ResultItemMetric].(map[string]interface{}) + if tmap[MetricStatus].(string) == "false" { + abnormalNodes = append(abnormalNodes, tmap[MetricLevelNode].(string)) + } else { + normalNodes = append(normalNodes, tmap[MetricLevelNode].(string)) + } + } + + Components, err := models.GetAllComponentsStatus() + + if err != nil { + glog.Error(err.Error()) + } + + var namspaceComponentHealthyMap = make(map[string]int) + var namspaceComponentTotalMap = make(map[string]int) + + for _, ns := range models.SYSTEM_NAMESPACES { + nsStatus, exist := Components[ns] + if exist { + for _, nsStatusItem := range nsStatus.(map[string]interface{}) { + component := nsStatusItem.(models.Component) + namspaceComponentTotalMap[ns] += 1 + if component.HealthyBackends == component.TotalBackends { + namspaceComponentHealthyMap[ns] += 1 + } + } + } + } + + timestamp := int64(time.Now().Unix()) + + onlineMetricItems := makeMetricItems(timestamp, namspaceComponentHealthyMap, MetricLevelNamespace) + metricItems := makeMetricItems(timestamp, namspaceComponentTotalMap, MetricLevelNamespace) + + var assembleList []interface{} + assembleList = append(assembleList, nodeStatusMetric) + + for _, statusItem := range componentStatusList { + assembleList = append(assembleList, statusItem) + } + + assembleList = append(assembleList, FormatedMetric{ + Data: FormatedMetricData{ + Result: *onlineMetricItems, + ResultType: ResultTypeVector, + }, + MetricName: MetricNameComponentOnLine, + Status: MetricStatusSuccess, + }) + + assembleList = append(assembleList, FormatedMetric{ + Data: FormatedMetricData{ + Result: *metricItems, + ResultType: ResultTypeVector, + }, + MetricName: MetricNameComponentLine, + Status: MetricStatusSuccess, + }) + + return &assembleList +} + +func makeMetricItems(timestamp int64, statusMap map[string]int, resourceType string) *[]map[string]interface{} { + var metricItems []map[string]interface{} + + for ns, count := range statusMap { + metricItems = append(metricItems, map[string]interface{}{ + ResultItemMetric: map[string]string{resourceType: ns}, + ResultItemValue: []interface{}{timestamp, fmt.Sprintf("%d", count)}, + }) + } + return &metricItems +} + +// monitor k8s event, there are two status: Normal, Warning +func MonitorEvents(nsFilter string) *[]v1.Event { + namespaceMap, err := getAllNamespace() + if err != nil { + glog.Errorln(err.Error()) + } + var nsList = make([]string, 0) + for ns, _ := range namespaceMap { + nsList = append(nsList, ns) + } + + filterNS := filterNamespace(nsFilter, nsList) + var eventsList = make([]v1.Event, 0) + + for _, ns := range filterNS { + events, err := client.NewK8sClient().CoreV1().Events(ns).List(metaV1.ListOptions{}) + if err != nil { + glog.Errorln(err.Error()) + } else { + for _, item := range events.Items { + eventsList = append(eventsList, item) + } + } + } + return &eventsList +} + +func AssembleClusterMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { + queryType := monitoringRequest.QueryType + paramValues := monitoringRequest.Params + rule := MakeClusterRule(metricName) + params := makeRequestParamString(rule, paramValues) + + return queryType, params +} + +func AssembleNodeMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { + queryType := monitoringRequest.QueryType + paramValues := monitoringRequest.Params + rule := MakeNodeRule(monitoringRequest.NodeId, monitoringRequest.NodesFilter, metricName) + params := makeRequestParamString(rule, paramValues) + + return queryType, params +} + +func getExistingNamespace(namespaces []string) ([]string, []string) { + namespaceMap, err := getAllNamespace() + var existedNS []string + var noneExistedNS []string + if err != nil { + return namespaces, nil + } + for _, ns := range namespaces { + if _, exist := namespaceMap[ns]; exist { + existedNS = append(existedNS, ns) + } else { + noneExistedNS = append(noneExistedNS, ns) + } + } + return existedNS, noneExistedNS +} + +func getAllNamespace() (map[string]int, error) { + k8sClient := client.NewK8sClient() + nsList, err := k8sClient.CoreV1().Namespaces().List(metaV1.ListOptions{}) + if err != nil { + glog.Errorln(err.Error()) + return nil, err + } + namespaceMap := make(map[string]int) + for _, item := range nsList.Items { + namespaceMap[item.Name] = 0 + } + return namespaceMap, nil +} + +func MonitorWorkloadCount(namespace string) *FormatedMetric { + quotaMetric, err := models.GetNamespaceQuota(namespace) + fMetric := convertQuota2MetricStruct(quotaMetric) + + // whether the namespace in request parameters exists? + namespaceMap, err := getAllNamespace() + + _, exist := namespaceMap[namespace] + if err != nil { + exist = true + } + + if !exist || err != nil { + fMetric.Status = MetricStatusError + fMetric.Data.ResultType = "" + errInfo := make(map[string]interface{}) + if err != nil { + errInfo["errormsg"] = err.Error() + } else { + errInfo["errormsg"] = "namespace " + namespace + " does not exist" + } + fMetric.Data.Result = []map[string]interface{}{errInfo} + } + + return fMetric +} + +func convertQuota2MetricStruct(quotaMetric *models.ResourceQuota) *FormatedMetric { + var fMetric FormatedMetric + fMetric.MetricName = MetricNameWorkloadCount + fMetric.Status = MetricStatusSuccess + fMetric.Data.ResultType = ResultTypeVector + timestamp := int64(time.Now().Unix()) + var resultItems []map[string]interface{} + + hardMap := make(map[string]string) + for resourceName, v := range quotaMetric.Data.Hard { + hardMap[resourceName.String()] = v.String() + } + + for resourceName, v := range quotaMetric.Data.Used { + resultItem := make(map[string]interface{}) + tmp := make(map[string]string) + tmp[ResultItemMetricResource] = resourceName.String() + resultItem[ResultItemMetric] = tmp + resultItem[ResultItemValue] = []interface{}{timestamp, hardMap[resourceName.String()], v.String()} + resultItems = append(resultItems, resultItem) + } + + fMetric.Data.Result = resultItems + return &fMetric +} diff --git a/pkg/models/metrics/metricscollector.go b/pkg/models/metrics/metricscollector.go deleted file mode 100755 index d534135dc..000000000 --- a/pkg/models/metrics/metricscollector.go +++ /dev/null @@ -1,539 +0,0 @@ -/* -Copyright 2018 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -import ( - "encoding/json" - "regexp" - "strings" - - "github.com/emicklei/go-restful" - "github.com/golang/glog" - - "time" - - "github.com/pkg/errors" - "k8s.io/api/core/v1" - metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "sync" - - "k8s.io/apimachinery/pkg/labels" - v12 "k8s.io/client-go/listers/core/v1" - - "kubesphere.io/kubesphere/pkg/client" - "kubesphere.io/kubesphere/pkg/models" - "kubesphere.io/kubesphere/pkg/models/controllers" - "kubesphere.io/kubesphere/pkg/models/workspaces" -) - -func getPodNameRegexInWorkload(request *restful.Request) string { - promql := MakeWorkloadRule(request) - res := client.SendPrometheusRequest(request, promql) - data := []byte(res) - var dat CommonMetricsResult - jsonErr := json.Unmarshal(data, &dat) - if jsonErr != nil { - glog.Errorln("json parse failed", jsonErr) - } - var podNames []string - for _, x := range dat.Data.Result { - podName := x.KubePodMetric.Pod - podNames = append(podNames, podName) - } - podNamesFilter := "^(" + strings.Join(podNames, "|") + ")$" - return podNamesFilter -} - -func MonitorWorkloadSingleMetric(request *restful.Request, metricsName string) *FormatedMetric { - nsName := strings.Trim(request.PathParameter("ns_name"), " ") - podNamesFilter := getPodNameRegexInWorkload(request) - newPromql := MakePodPromQL(request, []string{metricsName, nsName, "", "", podNamesFilter}) - podMetrics := client.SendPrometheusRequest(request, newPromql) - cleanedJson := ReformatJson(podMetrics, metricsName) - return cleanedJson -} - -func MonitorPodSingleMetric(request *restful.Request, metricsName string) *FormatedMetric { - nsName := strings.Trim(request.PathParameter("ns_name"), " ") - nodeID := strings.Trim(request.PathParameter("node_id"), " ") - podName := strings.Trim(request.PathParameter("pod_name"), " ") - podFilter := strings.Trim(request.QueryParameter("pods_filter"), " ") - params := []string{metricsName, nsName, nodeID, podName, podFilter} - promql := MakePodPromQL(request, params) - if promql != "" { - res := client.SendPrometheusRequest(request, promql) - cleanedJson := ReformatJson(res, metricsName) - return cleanedJson - } - return nil -} - -func MonitorNamespaceSingleMetric(request *restful.Request, metricsName string) *FormatedMetric { - recordingRule := MakeNamespacePromQL(request, metricsName) - res := client.SendPrometheusRequest(request, recordingRule) - cleanedJson := ReformatJson(res, metricsName) - return cleanedJson -} - -// maybe this function is time consuming -func ReformatJson(metric string, metricsName string) *FormatedMetric { - var formatMetric FormatedMetric - err := json.Unmarshal([]byte(metric), &formatMetric) - if err != nil { - glog.Errorln("Unmarshal metric json failed", err) - } - if formatMetric.MetricName == "" { - formatMetric.MetricName = metricsName - } - // retrive metrics success - if formatMetric.Status == MetricStatusSuccess { - result := formatMetric.Data.Result - for _, res := range result { - metric, ok := res[ResultItemMetric] - me := metric.(map[string]interface{}) - if ok { - delete(me, "__name__") - } - } - } - return &formatMetric -} - -func collectNodeorClusterMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) { - metric := MonitorNodeorClusterSingleMetric(request, metricsName) - ch <- metric -} - -func collectNamespaceMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) { - metric := MonitorNamespaceSingleMetric(request, metricsName) - ch <- metric -} - -func collectWorkloadMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) { - metricsName = strings.TrimLeft(metricsName, "workload_") - metric := MonitorWorkloadSingleMetric(request, metricsName) - ch <- metric -} - -func collectWorkspaceMetrics(request *restful.Request, metricsName string, namespaceList []string, ch chan<- *FormatedMetric) { - mertic := monitorWorkspaceSingleMertic(request, metricsName, namespaceList) - ch <- mertic -} - -func collectPodMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) { - metric := MonitorPodSingleMetric(request, metricsName) - ch <- metric -} - -func monitorWorkspaceSingleMertic(request *restful.Request, metricsName string, namespaceList []string) *FormatedMetric { - namespaceRe2 := "^(" + strings.Join(namespaceList, "|") + ")$" - newpromql := MakeWorkspacePromQL(metricsName, namespaceRe2) - podMetrics := client.SendPrometheusRequest(request, newpromql) - cleanedJson := ReformatJson(podMetrics, metricsName) - return cleanedJson -} - -func filterNamespace(request *restful.Request, namespaceList []string) []string { - var newNSlist []string - nsFilter := strings.Trim(request.QueryParameter("namespaces_filter"), " ") - if nsFilter == "" { - nsFilter = ".*" - } - for _, ns := range namespaceList { - bol, _ := regexp.MatchString(nsFilter, ns) - if bol { - newNSlist = append(newNSlist, ns) - } - } - return newNSlist -} - -func MonitorAllMetrics(request *restful.Request) FormatedLevelMetric { - metricsName := strings.Trim(request.QueryParameter("metrics_filter"), " ") - if metricsName == "" { - metricsName = ".*" - } - path := request.SelectedRoutePath() - sourceType := path[strings.LastIndex(path, "/")+1 : len(path)-1] - if strings.Contains(path, MetricLevelWorkload) { - sourceType = MetricLevelWorkload - } else if strings.Contains(path, MetricLevelWorkspace) { - sourceType = MetricLevelWorkspace - } - var ch = make(chan *FormatedMetric, 10) - for _, metricName := range MetricsNames { - bol, err := regexp.MatchString(metricsName, metricName) - if !bol { - continue - } - if err != nil { - glog.Errorln("regex match failed", err) - continue - } - if strings.HasPrefix(metricName, sourceType) { - if sourceType == MetricLevelCluster || sourceType == MetricLevelNode { - go collectNodeorClusterMetrics(request, metricName, ch) - } else if sourceType == MetricLevelNamespace { - go collectNamespaceMetrics(request, metricName, ch) - } else if sourceType == MetricLevelPod { - go collectPodMetrics(request, metricName, ch) - } else if sourceType == MetricLevelWorkload { - go collectWorkloadMetrics(request, metricName, ch) - } else if sourceType == MetricLevelWorkspace { - name := request.PathParameter("workspace_name") - namespaces, err := workspaces.WorkspaceNamespaces(name) - - if err != nil { - glog.Errorln(err) - } - namespaces = filterNamespace(request, namespaces) - go collectWorkspaceMetrics(request, metricName, namespaces, ch) - } - } - } - var metricsArray []FormatedMetric - var tempJson *FormatedMetric - for _, k := range MetricsNames { - bol, err := regexp.MatchString(metricsName, k) - if !bol { - continue - } - if err != nil { - glog.Errorln("regex match failed") - continue - } - if strings.HasPrefix(k, sourceType) { - tempJson = <-ch - if tempJson != nil { - metricsArray = append(metricsArray, *tempJson) - } - } - } - return FormatedLevelMetric{ - MetricsLevel: sourceType, - Results: metricsArray, - } -} - -func MonitorWorkspaceUserInfo(req *restful.Request) FormatedLevelMetric { - - var metricsArray []FormatedMetric - timestamp := time.Now().Unix() - - wg := sync.WaitGroup{} - var orgResultItem FormatedMetric - var dvpResultItem FormatedMetric - var projResultItem FormatedMetric - var actResultItem FormatedMetric - - wg.Add(4) - - go func() { - orgNums, errOrg := workspaces.GetAllOrgNums() - orgResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllOrganizationCount, WorkspaceResourceKindOrganization, orgNums, errOrg) - wg.Done() - }() - go func() { - devOpsProjectNums, errDevops := workspaces.GetAllDevOpsProjectsNums() - dvpResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllDevopsCount, WorkspaceResourceKindDevops, devOpsProjectNums, errDevops) - wg.Done() - }() - go func() { - projNums, errProj := workspaces.GetAllProjectNums() - projResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllProjectCount, WorkspaceResourceKindNamespace, projNums, errProj) - wg.Done() - }() - go func() { - actNums, errAct := workspaces.GetAllAccountNums() - actResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllAccountCount, WorkspaceResourceKindAccount, actNums, errAct) - wg.Done() - }() - - wg.Wait() - metricsArray = append(metricsArray, orgResultItem, dvpResultItem, projResultItem, actResultItem) - - return FormatedLevelMetric{ - MetricsLevel: MetricLevelWorkspace, - Results: metricsArray, - } -} - -//func getWorkspaceMetricItem(timestamp int64, namespaceNums int64, resourceName string, err error) FormatedMetric { -// var fMetric FormatedMetric -// fMetric.Data.ResultType = ResultTypeVector -// fMetric.MetricName = MetricNameWorkspaceInfoCount -// fMetric.Status = MetricStatusSuccess -// if err != nil { -// fMetric.Status = MetricStatusError -// } -// resultItem := make(map[string]interface{}) -// tmp := make(map[string]string) -// tmp[ResultItemMetricResource] = resourceName -// resultItem[ResultItemMetric] = tmp -// resultItem[ResultItemValue] = []interface{}{timestamp, strconv.FormatInt(namespaceNums, 10)} -// return fMetric -//} - -func MonitorWorkspaceResourceLevelMetrics(request *restful.Request) FormatedLevelMetric { - wsName := request.PathParameter("workspace_name") - namspaces, errNs := workspaces.WorkspaceNamespaces(wsName) - - devOpsProjects, errDevOps := workspaces.GetDevOpsProjects(wsName) - members, errMemb := workspaces.GetOrgMembers(wsName) - roles, errRole := workspaces.GetOrgRoles(wsName) - - var fMetricsArray []FormatedMetric - timestamp := int64(time.Now().Unix()) - namespaces, noneExistentNs := getExistingNamespace(namspaces) - if len(noneExistentNs) != 0 { - nsStr := strings.Join(noneExistentNs, "|") - errStr := "the namespaces " + nsStr + " do not exist" - if errNs == nil { - errNs = errors.New(errStr) - } else { - errNs = errors.New(errNs.Error() + "\t" + errStr) - } - } - - // add namespaces(project) metric - nsMetrics := getSpecificMetricItem(timestamp, MetricNameWorkspaceNamespaceCount, WorkspaceResourceKindNamespace, len(namespaces), errNs) - // add devops metric - devopsMetrics := getSpecificMetricItem(timestamp, MetricNameWorkspaceDevopsCount, WorkspaceResourceKindDevops, len(devOpsProjects), errDevOps) - // add member metric - memberMetrics := getSpecificMetricItem(timestamp, MetricNameWorkspaceMemberCount, WorkspaceResourceKindMember, len(members), errMemb) - // add role metric - roleMetrics := getSpecificMetricItem(timestamp, MetricNameWorkspaceRoleCount, WorkspaceResourceKindRole, len(roles), errRole) - // add workloads count metric - wlMetrics := getWorkspaceWorkloadCountMetrics(namespaces) - // add pods count metric - podsCountMetrics := getWorkspacePodsCountMetrics(request, namespaces) - fMetricsArray = append(fMetricsArray, nsMetrics, devopsMetrics, memberMetrics, roleMetrics, wlMetrics, *podsCountMetrics) - - return FormatedLevelMetric{ - MetricsLevel: MetricLevelWorkspace, - Results: fMetricsArray, - } -} - -func getWorkspacePodsCountMetrics(request *restful.Request, namespaces []string) *FormatedMetric { - metricName := MetricNameNamespacePodCount - var recordingRule = RulePromQLTmplMap[metricName] - nsFilter := "^(" + strings.Join(namespaces, "|") + ")$" - recordingRule = strings.Replace(recordingRule, "$1", nsFilter, -1) - res := client.SendPrometheusRequest(request, recordingRule) - cleanedJson := ReformatJson(res, metricName) - return cleanedJson -} - -func getWorkspaceWorkloadCountMetrics(namespaces []string) FormatedMetric { - var wlQuotaMetrics models.ResourceQuota - wlQuotaMetrics.NameSpace = strings.Join(namespaces, "|") - wlQuotaMetrics.Data.Used = make(v1.ResourceList, 1) - wlQuotaMetrics.Data.Hard = make(v1.ResourceList, 1) - for _, ns := range namespaces { - quotaMetric, err := models.GetNamespaceQuota(ns) - if err != nil { - glog.Errorln(err) - continue - } - // sum all resources used along namespaces - quotaUsed := quotaMetric.Data.Used - for resourceName, quantity := range quotaUsed { - if _, ok := wlQuotaMetrics.Data.Used[resourceName]; ok { - tmpQuantity := wlQuotaMetrics.Data.Used[v1.ResourceName(resourceName)] - tmpQuantity.Add(quantity) - wlQuotaMetrics.Data.Used[v1.ResourceName(resourceName)] = tmpQuantity - } else { - wlQuotaMetrics.Data.Used[v1.ResourceName(resourceName)] = quantity.DeepCopy() - } - } - - // sum all resources hard along namespaces - quotaHard := quotaMetric.Data.Hard - for resourceName, quantity := range quotaHard { - if _, ok := wlQuotaMetrics.Data.Hard[resourceName]; ok { - tmpQuantity := wlQuotaMetrics.Data.Hard[v1.ResourceName(resourceName)] - tmpQuantity.Add(quantity) - wlQuotaMetrics.Data.Hard[v1.ResourceName(resourceName)] = tmpQuantity - } else { - wlQuotaMetrics.Data.Hard[v1.ResourceName(resourceName)] = quantity.DeepCopy() - } - } - } - wlMetrics := convertQuota2MetricStruct(&wlQuotaMetrics) - return wlMetrics -} - -func getSpecificMetricItem(timestamp int64, metricName string, kind string, count int, err error) FormatedMetric { - var nsMetrics FormatedMetric - nsMetrics.MetricName = metricName - nsMetrics.Data.ResultType = ResultTypeVector - resultItem := make(map[string]interface{}) - tmp := make(map[string]string) - tmp[ResultItemMetricResource] = kind - if err == nil { - nsMetrics.Status = MetricStatusSuccess - } else { - nsMetrics.Status = MetricStatusError - resultItem["errorinfo"] = err.Error() - } - - resultItem[ResultItemMetric] = tmp - resultItem[ResultItemValue] = []interface{}{timestamp, count} - nsMetrics.Data.Result = make([]map[string]interface{}, 1) - nsMetrics.Data.Result[0] = resultItem - return nsMetrics -} - -func MonitorNodeorClusterSingleMetric(request *restful.Request, metricsName string) *FormatedMetric { - // support cluster node statistic, include healthy nodes and unhealthy nodes - var res string - var fMetric FormatedMetric - timestamp := int64(time.Now().Unix()) - - if metricsName == "cluster_node_online" { - onlineNodes, _ := getNodeHealthyConditionMetric() - fMetric = getSpecificMetricItem(timestamp, MetricNameClusterHealthyNodeCount, "node_count", len(onlineNodes), nil) - } else if metricsName == "cluster_node_offline" { - _, offlineNodes := getNodeHealthyConditionMetric() - fMetric = getSpecificMetricItem(timestamp, MetricNameClusterUnhealthyNodeCount, "node_count", len(offlineNodes), nil) - } else if metricsName == "cluster_node_total" { - onlineNodes, offlineNodes := getNodeHealthyConditionMetric() - fMetric = getSpecificMetricItem(timestamp, MetricNameClusterNodeCount, "node_count", len(onlineNodes)+len(offlineNodes), nil) - } else { - recordingRule := MakeNodeorClusterRule(request, metricsName) - res = client.SendPrometheusRequest(request, recordingRule) - fMetric = *ReformatJson(res, metricsName) - } - return &fMetric -} - -func getNodeHealthyConditionMetric() ([]string, []string) { - nodeList, err := client.NewK8sClient().CoreV1().Nodes().List(metaV1.ListOptions{}) - if err != nil { - glog.Errorln(err) - return nil, nil - } - var onlineNodes []string - var offlineNodes []string - for _, node := range nodeList.Items { - nodeName := node.Labels["kubernetes.io/hostname"] - nodeRole := node.Labels["role"] - bol := true - for _, cond := range node.Status.Conditions { - if cond.Type == "Ready" && cond.Status == "Unknown" { - bol = false - break - } - } - if nodeRole != "log" { - if bol { - // reachable node - onlineNodes = append(onlineNodes, nodeName) - } else { - // unreachable node - offlineNodes = append(offlineNodes, nodeName) - } - } - } - return onlineNodes, offlineNodes -} - -func getExistingNamespace(namespaces []string) ([]string, []string) { - namespaceMap, err := getAllNamespace() - var existedNs []string - var noneExistedNs []string - if err != nil { - return namespaces, nil - } - for _, ns := range namespaces { - if _, ok := namespaceMap[ns]; ok { - existedNs = append(existedNs, ns) - } else { - noneExistedNs = append(noneExistedNs, ns) - } - } - return existedNs, noneExistedNs -} - -func getAllNamespace() (map[string]int, error) { - lister := controllers.ResourceControllers.Controllers[controllers.Namespaces].Lister().(v12.NamespaceLister) - nsList, err := lister.List(labels.Everything()) - if err != nil { - glog.Errorln(err) - return nil, err - } - namespaceMap := make(map[string]int) - for _, item := range nsList { - namespaceMap[item.Name] = 0 - } - return namespaceMap, nil -} - -func MonitorWorkloadCount(request *restful.Request) FormatedMetric { - namespace := strings.Trim(request.PathParameter("ns_name"), " ") - - quotaMetric, err := models.GetNamespaceQuota(namespace) - fMetric := convertQuota2MetricStruct(quotaMetric) - - // whether the namespace in request parameters exists? - namespaceMap, e := getAllNamespace() - _, ok := namespaceMap[namespace] - if e != nil { - ok = true - } - if !ok || err != nil { - fMetric.Status = MetricStatusError - fMetric.Data.ResultType = "" - errInfo := make(map[string]interface{}) - if err != nil { - errInfo["errormsg"] = err.Error() - } else { - errInfo["errormsg"] = "namespace " + namespace + " does not exist" - } - fMetric.Data.Result = []map[string]interface{}{errInfo} - } - - return fMetric -} - -func convertQuota2MetricStruct(quotaMetric *models.ResourceQuota) FormatedMetric { - var fMetric FormatedMetric - fMetric.MetricName = MetricNameWorkloadCount - fMetric.Status = MetricStatusSuccess - fMetric.Data.ResultType = ResultTypeVector - timestamp := int64(time.Now().Unix()) - var resultItems []map[string]interface{} - - hardMap := make(map[string]string) - for resourceName, v := range quotaMetric.Data.Hard { - hardMap[resourceName.String()] = v.String() - } - - for resourceName, v := range quotaMetric.Data.Used { - resultItem := make(map[string]interface{}) - tmp := make(map[string]string) - tmp[ResultItemMetricResource] = resourceName.String() - resultItem[ResultItemMetric] = tmp - resultItem[ResultItemValue] = []interface{}{timestamp, hardMap[resourceName.String()], v.String()} - resultItems = append(resultItems, resultItem) - } - - fMetric.Data.Result = resultItems - return fMetric -} diff --git a/pkg/models/metrics/metricsconst.go b/pkg/models/metrics/metricsconst.go deleted file mode 100755 index a43cd52b5..000000000 --- a/pkg/models/metrics/metricsconst.go +++ /dev/null @@ -1,236 +0,0 @@ -/* -Copyright 2018 The KubeSphere Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -const ( - ResultTypeVector = "vector" - ResultTypeMatrix = "matrix" - MetricStatusError = "error" - MetricStatusSuccess = "success" - ResultItemMetric = "metric" - ResultItemMetricResource = "resource" - ResultItemValue = "value" -) - -const ( - MetricNameWorkloadCount = "workload_count" - MetricNameNamespacePodCount = "namespace_pod_count" - - MetricNameWorkspaceAllOrganizationCount = "workspace_all_organization_count" - MetricNameWorkspaceAllAccountCount = "workspace_all_account_count" - MetricNameWorkspaceAllProjectCount = "workspace_all_project_count" - MetricNameWorkspaceAllDevopsCount = "workspace_all_devops_project_count" - - MetricNameWorkspaceNamespaceCount = "workspace_namespace_count" - MetricNameWorkspaceDevopsCount = "workspace_devops_project_count" - MetricNameWorkspaceMemberCount = "workspace_member_count" - MetricNameWorkspaceRoleCount = "workspace_role_count" - - MetricNameClusterHealthyNodeCount = "cluster_node_online" - MetricNameClusterUnhealthyNodeCount = "cluster_node_offline" - MetricNameClusterNodeCount = "cluster_node_total" -) - -const ( - WorkspaceResourceKindOrganization = "organization" - WorkspaceResourceKindAccount = "account" - WorkspaceResourceKindNamespace = "namespace" - WorkspaceResourceKindDevops = "devops" - WorkspaceResourceKindMember = "member" - WorkspaceResourceKindRole = "role" -) - -const ( - MetricLevelCluster = "cluster" - MetricLevelNode = "node" - MetricLevelWorkspace = "workspace" - MetricLevelNamespace = "namespace" - MetricLevelPod = "pod" - MetricLevelContainer = "container" - MetricLevelWorkload = "workload" -) - -type MetricMap map[string]string - -var MetricsNames = []string{ - "cluster_cpu_utilisation", - "cluster_cpu_usage", - "cluster_cpu_total", - "cluster_memory_utilisation", - "cluster_pod_count", - "cluster_memory_bytes_available", - "cluster_memory_bytes_total", - "cluster_memory_bytes_usage", - "cluster_net_utilisation", - "cluster_net_bytes_transmitted", - "cluster_net_bytes_received", - "cluster_disk_read_iops", - "cluster_disk_write_iops", - "cluster_disk_read_throughput", - "cluster_disk_write_throughput", - "cluster_disk_size_usage", - "cluster_disk_size_utilisation", - "cluster_disk_size_capacity", - "cluster_disk_size_available", - "cluster_node_online", - "cluster_node_offline", - "cluster_node_total", - - "node_cpu_utilisation", - "node_cpu_total", - "node_cpu_usage", - "node_memory_utilisation", - "node_memory_bytes_usage", - "node_memory_bytes_available", - "node_memory_bytes_total", - "node_net_utilisation", - "node_net_bytes_transmitted", - "node_net_bytes_received", - "node_disk_read_iops", - "node_disk_write_iops", - "node_disk_read_throughput", - "node_disk_write_throughput", - "node_disk_size_capacity", - "node_disk_size_available", - "node_disk_size_usage", - "node_disk_size_utilisation", - "node_pod_count", - "node_pod_quota", - - "namespace_cpu_usage", - "namespace_memory_usage", - "namespace_memory_usage_wo_cache", - "namespace_net_bytes_transmitted", - "namespace_net_bytes_received", - "namespace_pod_count", - - "pod_cpu_usage", - "pod_memory_usage", - "pod_memory_usage_wo_cache", - "pod_net_bytes_transmitted", - "pod_net_bytes_received", - - "workload_pod_cpu_usage", - "workload_pod_memory_usage", - "workload_pod_memory_usage_wo_cache", - "workload_pod_net_bytes_transmitted", - "workload_pod_net_bytes_received", - //"container_cpu_usage", - //"container_memory_usage_wo_cache", - //"container_memory_usage", - - "workspace_cpu_usage", - "workspace_memory_usage", - "workspace_memory_usage_wo_cache", - "workspace_net_bytes_transmitted", - "workspace_net_bytes_received", - "workspace_pod_count", -} - -var RulePromQLTmplMap = MetricMap{ - //cluster - "cluster_cpu_utilisation": ":node_cpu_utilisation:avg1m", - "cluster_cpu_usage": `sum (irate(container_cpu_usage_seconds_total{job="kubelet", image!=""}[5m]))`, - "cluster_cpu_total": "sum(node:node_num_cpu:sum)", - "cluster_memory_utilisation": ":node_memory_utilisation:", - "cluster_pod_count": `count(kube_pod_info unless on(pod) kube_pod_completion_time unless on(node) kube_node_labels{label_role="log"})`, - "cluster_memory_bytes_available": "sum(node:node_memory_bytes_available:sum)", - "cluster_memory_bytes_total": "sum(node:node_memory_bytes_total:sum)", - "cluster_memory_bytes_usage": "sum(node:node_memory_bytes_total:sum) - sum(node:node_memory_bytes_available:sum)", - "cluster_net_utilisation": ":node_net_utilisation:sum_irate", - "cluster_net_bytes_transmitted": "sum(node:node_net_bytes_transmitted:sum_irate)", - "cluster_net_bytes_received": "sum(node:node_net_bytes_received:sum_irate)", - "cluster_disk_read_iops": "sum(node:data_volume_iops_reads:sum)", - "cluster_disk_write_iops": "sum(node:data_volume_iops_writes:sum)", - "cluster_disk_read_throughput": "sum(node:data_volume_throughput_bytes_read:sum)", - "cluster_disk_write_throughput": "sum(node:data_volume_throughput_bytes_written:sum)", - "cluster_disk_size_usage": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, - "cluster_disk_size_utilisation": `(sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))) / sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, - "cluster_disk_size_capacity": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, - "cluster_disk_size_available": `sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, - - //node - "node_cpu_utilisation": "node:node_cpu_utilisation:avg1m", - "node_cpu_total": "node:node_num_cpu:sum", - "node_memory_utilisation": "node:node_memory_utilisation:", - "node_memory_bytes_available": "node:node_memory_bytes_available:sum", - "node_memory_bytes_total": "node:node_memory_bytes_total:sum", - // Node network utilisation (bytes received + bytes transmitted per second) - "node_net_utilisation": "node:node_net_utilisation:sum_irate", - // Node network bytes transmitted per second - "node_net_bytes_transmitted": "node:node_net_bytes_transmitted:sum_irate", - // Node network bytes received per second - "node_net_bytes_received": "node:node_net_bytes_received:sum_irate", - - // node:data_volume_iops_reads:sum{node=~"i-5xcldxos|i-6soe9zl1"} - "node_disk_read_iops": "node:data_volume_iops_reads:sum", - // node:data_volume_iops_writes:sum{node=~"i-5xcldxos|i-6soe9zl1"} - "node_disk_write_iops": "node:data_volume_iops_writes:sum", - // node:data_volume_throughput_bytes_read:sum{node=~"i-5xcldxos|i-6soe9zl1"} - "node_disk_read_throughput": "node:data_volume_throughput_bytes_read:sum", - // node:data_volume_throughput_bytes_written:sum{node=~"i-5xcldxos|i-6soe9zl1"} - "node_disk_write_throughput": "node:data_volume_throughput_bytes_written:sum", - - "node_disk_size_capacity": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_disk_size_available": `sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_disk_size_usage": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) -sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_disk_size_utilisation": `sum by (node) (((node_filesystem_size{mountpoint="/", job="node-exporter"} - node_filesystem_avail{mountpoint="/", job="node-exporter"}) / node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_pod_count": `count(kube_pod_info$1 unless on(pod) kube_pod_completion_time) by (node)`, - // without log node: unless on(node) kube_node_labels{label_role="log"} - "node_pod_quota": `sum(kube_node_status_capacity_pods$1) by (node)`, - "node_cpu_usage": `sum by (node) (label_join(irate(container_cpu_usage_seconds_total{job="kubelet", image!=""}[5m]), "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_memory_bytes_usage": "node:node_memory_bytes_total:sum$1 - node:node_memory_bytes_available:sum$1", - - //namespace - "namespace_cpu_usage": `namespace:container_cpu_usage_seconds_total:sum_rate{namespace=~"$1"}`, - "namespace_memory_usage": `namespace:container_memory_usage_bytes:sum{namespace=~"$1"}`, - "namespace_memory_usage_wo_cache": `namespace:container_memory_usage_bytes_wo_cache:sum{namespace=~"$1"}`, - "namespace_net_bytes_transmitted": `sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m]))`, - "namespace_net_bytes_received": `sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m]))`, - "namespace_pod_count": `count(kube_pod_info{namespace=~"$1"} unless on(pod) kube_pod_completion_time) by (namespace)`, - - // pod - "pod_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name="$2", image!=""}[5m])) by (namespace, pod_name)`, - "pod_memory_usage": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""}) by (namespace, pod_name)`, - "pod_memory_usage_wo_cache": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name="$2",image!=""}) by (namespace, pod_name)`, - "pod_net_bytes_transmitted": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[5m]))`, - "pod_net_bytes_received": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[5m]))`, - - "pod_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}[5m])) by (namespace, pod_name)`, - "pod_memory_usage_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`, - "pod_memory_usage_wo_cache_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`, - "pod_net_bytes_transmitted_all": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`, - "pod_net_bytes_received_all": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`, - - "pod_cpu_usage_node": `sum by (node, pod) (label_join(irate(container_cpu_usage_seconds_total{job="kubelet",pod_name=~"$2", image!=""}[5m]), "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, - "pod_memory_usage_node": `sum by (node, pod) (label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, - "pod_memory_usage_wo_cache_node": `sum by (node, pod) ((label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") - label_join(container_memory_cache{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name")) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, - - // container - "container_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name="$3"}[5m])) by (namespace, pod_name, container_name)`, - "container_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}[5m])) by (namespace, pod_name, container_name)`, - - "container_memory_usage_wo_cache": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name="$3"}`, - "container_memory_usage_wo_cache_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`, - "container_memory_usage": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"}`, - "container_memory_usage_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`, - - // enterprise - "workspace_cpu_usage": `sum(namespace:container_cpu_usage_seconds_total:sum_rate{namespace =~"$1"})`, - "workspace_memory_usage": `sum(namespace:container_memory_usage_bytes:sum{namespace =~"$1"})`, - "workspace_memory_usage_wo_cache": `sum(namespace:container_memory_usage_bytes_wo_cache:sum{namespace =~"$1"})`, - "workspace_net_bytes_transmitted": `sum(sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m])))`, - "workspace_net_bytes_received": `sum(sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m])))`, - "workspace_pod_count": `sum(count(kube_pod_info{namespace=~"$1"} unless on(pod) kube_pod_completion_time) by (namespace))`, -} diff --git a/pkg/models/metrics/metricsrule.go b/pkg/models/metrics/metricsrule.go index ee3ee3b70..b2321b6b1 100755 --- a/pkg/models/metrics/metricsrule.go +++ b/pkg/models/metrics/metricsrule.go @@ -15,142 +15,119 @@ package metrics import ( "strings" - - "github.com/emicklei/go-restful" ) -func MakeWorkloadRule(request *restful.Request) string { - // kube_pod_info{created_by_kind="DaemonSet",created_by_name="fluent-bit",endpoint="https-main", - // host_ip="192.168.0.14",instance="10.244.114.187:8443",job="kube-state-metrics", - // namespace="kube-system",node="i-k89a62il",pod="fluent-bit-l5vxr", - // pod_ip="10.244.114.175",service="kube-state-metrics"} - rule := `kube_pod_info{created_by_kind="$1",created_by_name=$2,namespace="$3"}` - kind := strings.Trim(request.PathParameter("workload_kind"), " ") - name := strings.Trim(request.QueryParameter("workload_name"), " ") - namespace := strings.Trim(request.PathParameter("ns_name"), " ") +func MakeWorkloadRule(wkKind, wkName, namespace string) string { + var rule = PodInfoRule if namespace == "" { namespace = ".*" } + // alertnatives values: Deployment StatefulSet ReplicaSet DaemonSet + wkKind = strings.ToLower(wkKind) - // kind alertnatives values: Deployment StatefulSet ReplicaSet DaemonSet - kind = strings.ToLower(kind) - - switch kind { + switch wkKind { case "deployment": - kind = "ReplicaSet" - if name != "" { - name = "~\"" + name + ".*\"" + wkKind = ReplicaSet + if wkName != "" { + wkName = "~\"" + wkName + ".*\"" } else { - name = "~\".*\"" + wkName = "~\".*\"" } - rule = strings.Replace(rule, "$1", kind, -1) - rule = strings.Replace(rule, "$2", name, -1) + rule = strings.Replace(rule, "$1", wkKind, -1) + rule = strings.Replace(rule, "$2", wkName, -1) rule = strings.Replace(rule, "$3", namespace, -1) return rule case "replicaset": - kind = "ReplicaSet" + wkKind = ReplicaSet case "statefulset": - kind = "StatefulSet" + wkKind = StatefulSet case "daemonset": - kind = "DaemonSet" + wkKind = DaemonSet } - if name == "" { - name = "~\".*\"" + if wkName == "" { + wkName = "~\".*\"" } else { - name = "\"" + name + "\"" + wkName = "\"" + wkName + "\"" } - rule = strings.Replace(rule, "$1", kind, -1) - rule = strings.Replace(rule, "$2", name, -1) + rule = strings.Replace(rule, "$1", wkKind, -1) + rule = strings.Replace(rule, "$2", wkName, -1) rule = strings.Replace(rule, "$3", namespace, -1) return rule } -func MakeWorkspacePromQL(metricsName string, namespaceRe2 string) string { +func MakeWorkspacePromQL(metricsName string, nsFilter string) string { promql := RulePromQLTmplMap[metricsName] - promql = strings.Replace(promql, "$1", namespaceRe2, -1) + promql = strings.Replace(promql, "$1", nsFilter, -1) return promql } -func MakeContainerPromQL(request *restful.Request) string { - nsName := strings.Trim(request.PathParameter("ns_name"), " ") - poName := strings.Trim(request.PathParameter("pod_name"), " ") - containerName := strings.Trim(request.PathParameter("container_name"), " ") - // metricType container_cpu_utilisation container_memory_utilisation container_memory_utilisation_wo_cache - metricType := strings.Trim(request.QueryParameter("metrics_name"), " ") +func MakeContainerPromQL(nsName, podName, containerName, metricName, containerFilter string) string { var promql = "" if containerName == "" { // all containers maybe use filter - metricType += "_all" - promql = RulePromQLTmplMap[metricType] + metricName += "_all" + promql = RulePromQLTmplMap[metricName] promql = strings.Replace(promql, "$1", nsName, -1) - promql = strings.Replace(promql, "$2", poName, -1) - containerFilter := strings.Trim(request.QueryParameter("containers_filter"), " ") + promql = strings.Replace(promql, "$2", podName, -1) + if containerFilter == "" { containerFilter = ".*" } + promql = strings.Replace(promql, "$3", containerFilter, -1) return promql } - promql = RulePromQLTmplMap[metricType] + promql = RulePromQLTmplMap[metricName] promql = strings.Replace(promql, "$1", nsName, -1) - promql = strings.Replace(promql, "$2", poName, -1) + promql = strings.Replace(promql, "$2", podName, -1) promql = strings.Replace(promql, "$3", containerName, -1) return promql } -func MakePodPromQL(request *restful.Request, params []string) string { - metricType := params[0] - nsName := params[1] - nodeID := params[2] - podName := params[3] - podFilter := params[4] +func MakePodPromQL(metricName, nsName, nodeID, podName, podFilter string) string { + + if podFilter == "" { + podFilter = ".*" + } + var promql = "" if nsName != "" { // get pod metrics by namespace if podName != "" { // specific pod - promql = RulePromQLTmplMap[metricType] + promql = RulePromQLTmplMap[metricName] promql = strings.Replace(promql, "$1", nsName, -1) promql = strings.Replace(promql, "$2", podName, -1) } else { // all pods - metricType += "_all" - promql = RulePromQLTmplMap[metricType] - if podFilter == "" { - podFilter = ".*" - } + metricName += "_all" + promql = RulePromQLTmplMap[metricName] + promql = strings.Replace(promql, "$1", nsName, -1) promql = strings.Replace(promql, "$2", podFilter, -1) } } else if nodeID != "" { // get pod metrics by nodeid - metricType += "_node" - promql = RulePromQLTmplMap[metricType] + metricName += "_node" + promql = RulePromQLTmplMap[metricName] promql = strings.Replace(promql, "$3", nodeID, -1) if podName != "" { // specific pod promql = strings.Replace(promql, "$2", podName, -1) } else { - // choose pod use re2 expression - podFilter := strings.Trim(request.QueryParameter("pods_filter"), " ") - if podFilter == "" { - podFilter = ".*" - } promql = strings.Replace(promql, "$2", podFilter, -1) } } return promql } -func MakeNamespacePromQL(request *restful.Request, metricsName string) string { - nsName := strings.Trim(request.PathParameter("ns_name"), " ") - metricType := metricsName - var recordingRule = RulePromQLTmplMap[metricType] - nsFilter := strings.Trim(request.QueryParameter("namespaces_filter"), " ") +func MakeNamespacePromQL(nsName string, nsFilter string, metricsName string) string { + var recordingRule = RulePromQLTmplMap[metricsName] + if nsName != "" { nsFilter = nsName } else { @@ -162,37 +139,37 @@ func MakeNamespacePromQL(request *restful.Request, metricsName string) string { return recordingRule } -func MakeNodeorClusterRule(request *restful.Request, metricsName string) string { - nodeID := request.PathParameter("node_id") +// cluster rule +func MakeClusterRule(metricsName string) string { var rule = RulePromQLTmplMap[metricsName] - - if strings.Contains(request.SelectedRoutePath(), "monitoring/cluster") { - // cluster - return rule - } else { - // node - nodesFilter := strings.Trim(request.QueryParameter("nodes_filter"), " ") - if nodesFilter == "" { - nodesFilter = ".*" - } - if strings.Contains(metricsName, "disk_size") || strings.Contains(metricsName, "pod") || strings.Contains(metricsName, "usage") { - // disk size promql - if nodeID != "" { - nodesFilter = "{" + "node" + "=" + "\"" + nodeID + "\"" + "}" - } else { - nodesFilter = "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}" - } - rule = strings.Replace(rule, "$1", nodesFilter, -1) - } else { - // cpu, memory, network, disk_iops rules - if nodeID != "" { - // specific node - rule = rule + "{" + "node" + "=" + "\"" + nodeID + "\"" + "}" - } else { - // all nodes or specific nodes filted with re2 syntax - rule = rule + "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}" - } - } - } + return rule +} + +// node rule +func MakeNodeRule(nodeID string, nodesFilter string, metricsName string) string { + var rule = RulePromQLTmplMap[metricsName] + + if nodesFilter == "" { + nodesFilter = ".*" + } + if strings.Contains(metricsName, "disk_size") || strings.Contains(metricsName, "pod") || strings.Contains(metricsName, "usage") || strings.Contains(metricsName, "inode") { + // disk size promql + if nodeID != "" { + nodesFilter = "{" + "node" + "=" + "\"" + nodeID + "\"" + "}" + } else { + nodesFilter = "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}" + } + rule = strings.Replace(rule, "$1", nodesFilter, -1) + } else { + // cpu, memory, network, disk_iops rules + if nodeID != "" { + // specific node + rule = rule + "{" + "node" + "=" + "\"" + nodeID + "\"" + "}" + } else { + // all nodes or specific nodes filted with re2 syntax + rule = rule + "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}" + } + } + return rule } diff --git a/pkg/models/metrics/metricsruleconst.go b/pkg/models/metrics/metricsruleconst.go new file mode 100644 index 000000000..05896333d --- /dev/null +++ b/pkg/models/metrics/metricsruleconst.go @@ -0,0 +1,560 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +const ( + ResultTypeVector = "vector" + ResultTypeMatrix = "matrix" + MetricStatus = "status" + MetricStatusError = "error" + MetricStatusSuccess = "success" + ResultItemMetric = "metric" + ResultItemMetricResource = "resource" + ResultItemValue = "value" + ResultItemValues = "values" + ResultSortTypeDesc = "desc" + ResultSortTypeAsce = "asce" +) + +const ( + MetricNameWorkloadCount = "workload_count" + MetricNameNamespacePodCount = "namespace_pod_count" + + MetricNameWorkspaceAllOrganizationCount = "workspace_all_organization_count" + MetricNameWorkspaceAllAccountCount = "workspace_all_account_count" + MetricNameWorkspaceAllProjectCount = "workspace_all_project_count" + MetricNameWorkspaceAllDevopsCount = "workspace_all_devops_project_count" + MetricNameClusterAllProjectCount = "cluster_namespace_count" + + MetricNameWorkspaceNamespaceCount = "workspace_namespace_count" + MetricNameWorkspaceDevopsCount = "workspace_devops_project_count" + MetricNameWorkspaceMemberCount = "workspace_member_count" + MetricNameWorkspaceRoleCount = "workspace_role_count" + MetricNameComponentOnLine = "component_online_count" + MetricNameComponentLine = "component_count" +) + +const ( + WorkspaceResourceKindOrganization = "organization" + WorkspaceResourceKindAccount = "account" + WorkspaceResourceKindNamespace = "namespace" + WorkspaceResourceKindDevops = "devops" + WorkspaceResourceKindMember = "member" + WorkspaceResourceKindRole = "role" +) + +const ( + MetricLevelCluster = "cluster" + MetricLevelClusterWorkspace = "cluster_workspace" + MetricLevelNode = "node" + MetricLevelWorkspace = "workspace" + MetricLevelNamespace = "namespace" + MetricLevelPod = "pod" + MetricLevelPodName = "pod_name" + MetricLevelContainer = "container" + MetricLevelWorkload = "workload" +) + +const ( + ReplicaSet = "ReplicaSet" + StatefulSet = "StatefulSet" + DaemonSet = "DaemonSet" +) + +const ( + NodeStatusRule = `kube_node_status_condition{condition="Ready"} > 0` + PodInfoRule = `kube_pod_info{created_by_kind="$1",created_by_name=$2,namespace="$3"}` + NamespaceLabelRule = `kube_namespace_labels` +) + +const ( + WorkspaceJoinedKey = "label_kubesphere_io_workspace" +) + +type MetricMap map[string]string + +var ClusterMetricsNames = []string{ + "cluster_cpu_utilisation", + "cluster_cpu_usage", + "cluster_cpu_total", + "cluster_memory_utilisation", + "cluster_memory_bytes_available", + "cluster_memory_bytes_total", + "cluster_memory_bytes_usage", + "cluster_net_utilisation", + "cluster_net_bytes_transmitted", + "cluster_net_bytes_received", + "cluster_disk_read_iops", + "cluster_disk_write_iops", + "cluster_disk_read_throughput", + "cluster_disk_write_throughput", + "cluster_disk_size_usage", + "cluster_disk_size_utilisation", + "cluster_disk_size_capacity", + "cluster_disk_size_available", + "cluster_disk_inode_total", + "cluster_disk_inode_usage", + "cluster_disk_inode_utilisation", + + "cluster_node_online", + "cluster_node_offline", + "cluster_node_total", + + "cluster_pod_count", + "cluster_pod_quota", + "cluster_pod_utilisation", + "cluster_pod_running_count", + "cluster_pod_succeeded_count", + "cluster_pod_abnormal_count", + "cluster_ingresses_extensions_count", + "cluster_cronjob_count", + "cluster_pvc_count", + "cluster_daemonset_count", + "cluster_deployment_count", + "cluster_endpoint_count", + "cluster_hpa_count", + "cluster_job_count", + "cluster_statefulset_count", + "cluster_replicaset_count", + "cluster_service_count", + "cluster_secret_count", + + "cluster_namespace_count", + "workspace_all_project_count", +} +var NodeMetricsNames = []string{ + "node_cpu_utilisation", + "node_cpu_total", + "node_cpu_usage", + "node_memory_utilisation", + "node_memory_bytes_usage", + "node_memory_bytes_available", + "node_memory_bytes_total", + "node_net_utilisation", + "node_net_bytes_transmitted", + "node_net_bytes_received", + "node_disk_read_iops", + "node_disk_write_iops", + "node_disk_read_throughput", + "node_disk_write_throughput", + "node_disk_size_capacity", + "node_disk_size_available", + "node_disk_size_usage", + "node_disk_size_utilisation", + + "node_disk_inode_total", + "node_disk_inode_usage", + "node_disk_inode_utilisation", + + "node_pod_count", + "node_pod_quota", + "node_pod_utilisation", + "node_pod_running_count", + "node_pod_succeeded_count", + "node_pod_abnormal_count", +} +var WorkspaceMetricsNames = []string{ + "workspace_cpu_usage", + "workspace_memory_usage", + "workspace_memory_usage_wo_cache", + "workspace_net_bytes_transmitted", + "workspace_net_bytes_received", + "workspace_pod_count", + "workspace_pod_running_count", + "workspace_pod_succeeded_count", + "workspace_pod_abnormal_count", + "workspace_ingresses_extensions_count", + + "workspace_cronjob_count", + "workspace_pvc_count", + "workspace_daemonset_count", + "workspace_deployment_count", + "workspace_endpoint_count", + "workspace_hpa_count", + "workspace_job_count", + "workspace_statefulset_count", + "workspace_replicaset_count", + "workspace_service_count", + "workspace_secret_count", +} +var NamespaceMetricsNames = []string{ + "namespace_cpu_usage", + "namespace_memory_usage", + "namespace_memory_usage_wo_cache", + "namespace_net_bytes_transmitted", + "namespace_net_bytes_received", + "namespace_pod_count", + "namespace_pod_running_count", + "namespace_pod_succeeded_count", + "namespace_pod_abnormal_count", + + "namespace_configmap_count_used", + "namespace_jobs_batch_count_used", + "namespace_roles_count_used", + "namespace_memory_limit_used", + "namespace_pvc_used", + "namespace_memory_request_used", + "namespace_pvc_count_used", + "namespace_cronjobs_batch_count_used", + "namespace_ingresses_extensions_count_used", + "namespace_cpu_limit_used", + "namespace_storage_request_used", + "namespace_deployment_count_used", + "namespace_pod_count_used", + "namespace_statefulset_count_used", + "namespace_daemonset_count_used", + "namespace_secret_count_used", + "namespace_service_count_used", + "namespace_cpu_request_used", + "namespace_service_loadbalancer_used", + + "namespace_configmap_count_hard", + "namespace_jobs_batch_count_hard", + "namespace_roles_count_hard", + "namespace_memory_limit_hard", + "namespace_pvc_hard", + "namespace_memory_request_hard", + "namespace_pvc_count_hard", + "namespace_cronjobs_batch_count_hard", + "namespace_ingresses_extensions_count_hard", + "namespace_cpu_limit_hard", + "namespace_storage_request_hard", + "namespace_deployment_count_hard", + "namespace_pod_count_hard", + "namespace_statefulset_count_hard", + "namespace_daemonset_count_hard", + "namespace_secret_count_hard", + "namespace_service_count_hard", + "namespace_cpu_request_hard", + "namespace_service_loadbalancer_hard", + + "namespace_cronjob_count", + "namespace_pvc_count", + "namespace_daemonset_count", + "namespace_deployment_count", + "namespace_endpoint_count", + "namespace_hpa_count", + "namespace_job_count", + "namespace_statefulset_count", + "namespace_replicaset_count", + "namespace_service_count", + "namespace_secret_count", +} + +var PodMetricsNames = []string{ + "pod_cpu_usage", + "pod_memory_usage", + "pod_memory_usage_wo_cache", + "pod_net_bytes_transmitted", + "pod_net_bytes_received", +} + +var WorkloadMetricsNames = []string{ + "workload_pod_cpu_usage", + "workload_pod_memory_usage", + "workload_pod_memory_usage_wo_cache", + "workload_pod_net_bytes_transmitted", + "workload_pod_net_bytes_received", +} + +var RulePromQLTmplMap = MetricMap{ + //cluster + "cluster_cpu_utilisation": ":node_cpu_utilisation:avg1m", + "cluster_cpu_usage": `:node_cpu_utilisation:avg1m * sum(node:node_num_cpu:sum)`, + "cluster_cpu_total": "sum(node:node_num_cpu:sum)", + "cluster_memory_utilisation": ":node_memory_utilisation:", + "cluster_memory_bytes_available": "sum(node:node_memory_bytes_available:sum)", + "cluster_memory_bytes_total": "sum(node:node_memory_bytes_total:sum)", + "cluster_memory_bytes_usage": "sum(node:node_memory_bytes_total:sum) - sum(node:node_memory_bytes_available:sum)", + "cluster_net_utilisation": ":node_net_utilisation:sum_irate", + "cluster_net_bytes_transmitted": "sum(node:node_net_bytes_transmitted:sum_irate)", + "cluster_net_bytes_received": "sum(node:node_net_bytes_received:sum_irate)", + "cluster_disk_read_iops": "sum(node:data_volume_iops_reads:sum)", + "cluster_disk_write_iops": "sum(node:data_volume_iops_writes:sum)", + "cluster_disk_read_throughput": "sum(node:data_volume_throughput_bytes_read:sum)", + "cluster_disk_write_throughput": "sum(node:data_volume_throughput_bytes_written:sum)", + "cluster_disk_size_usage": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, + "cluster_disk_size_utilisation": `(sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))) / sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, + "cluster_disk_size_capacity": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, + "cluster_disk_size_available": `sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, + + "cluster_disk_inode_total": `sum(node:disk_inodes_total:)`, + "cluster_disk_inode_usage": `sum(node:disk_inodes_total:) - sum(node:disk_inodes_free:)`, + "cluster_disk_inode_utilisation": `1 - sum(node:disk_inodes_free:) / sum(node:disk_inodes_total:)`, + + "cluster_namespace_count": `count(kube_namespace_annotations)`, + "workspace_all_project_count": `count(kube_namespace_annotations)`, + + // cluster_pod_count = cluster_pod_running_count + cluster_pod_succeeded_count + cluster_pod_abnormal_count + "cluster_pod_count": `sum(kube_pod_status_phase)`, + "cluster_pod_utilisation": `sum(kube_pod_status_phase) / sum(kube_node_status_capacity_pods)`, + "cluster_pod_running_count": `sum(kube_pod_status_phase{phase="Running"})`, + "cluster_pod_succeeded_count": `sum(kube_pod_status_phase{phase="Succeeded"})`, + "cluster_pod_abnormal_count": `sum(kube_pod_status_phase{phase=~"Failed|Pending|Unknown"})`, + + "cluster_node_online": `sum(kube_node_status_condition{condition="Ready",status="true"})`, + "cluster_node_offline": `sum(kube_node_status_condition{condition="Ready",status=~"unknown|false"})`, + "cluster_node_total": `sum(kube_node_status_condition{condition="Ready"})`, + "cluster_pod_quota": `sum(kube_node_status_capacity_pods)`, + "cluster_ingresses_extensions_count": `sum(kube_resourcequota{type="used", resource="count/ingresses.extensions"}) by (resource, type)`, + + "cluster_configmap_count_used": `sum(kube_resourcequota{type="used", resource="count/configmaps"}) by (resource, type)`, + "cluster_jobs_batch_count_used": `sum(kube_resourcequota{type="used", resource="count/jobs.batch"}) by (resource, type)`, + "cluster_roles_count_used": `sum(kube_resourcequota{type="used", resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`, + "cluster_memory_limit_used": `sum(kube_resourcequota{type="used", resource="limits.memory"}) by (resource, type)`, + "cluster_pvc_used": `sum(kube_resourcequota{type="used", resource="persistentvolumeclaims"}) by (resource, type)`, + "cluster_memory_request_used": `sum(kube_resourcequota{type="used", resource="requests.memory"}) by (resource, type)`, + "cluster_pvc_count_used": `sum(kube_resourcequota{type="used", resource="count/persistentvolumeclaims"}) by (resource, type)`, + "cluster_cronjobs_batch_count_used": `sum(kube_resourcequota{type="used", resource="count/cronjobs.batch"}) by (resource, type)`, + "cluster_ingresses_extensions_count_used": `sum(kube_resourcequota{type="used", resource="count/ingresses.extensions"}) by (resource, type)`, + "cluster_cpu_limit_used": `sum(kube_resourcequota{type="used", resource="limits.cpu"}) by (resource, type)`, + "cluster_storage_request_used": `sum(kube_resourcequota{type="used", resource="requests.storage"}) by (resource, type)`, + "cluster_deployment_count_used": `sum(kube_resourcequota{type="used", resource="count/deployments.apps"}) by (resource, type)`, + "cluster_pod_count_used": `sum(kube_resourcequota{type="used", resource="count/pods"}) by (resource, type)`, + "cluster_statefulset_count_used": `sum(kube_resourcequota{type="used", resource="count/statefulsets.apps"}) by (resource, type)`, + "cluster_daemonset_count_used": `sum(kube_resourcequota{type="used", resource="count/daemonsets.apps"}) by (resource, type)`, + "cluster_secret_count_used": `sum(kube_resourcequota{type="used", resource="count/secrets"}) by (resource, type)`, + "cluster_service_count_used": `sum(kube_resourcequota{type="used", resource="count/services"}) by (resource, type)`, + "cluster_cpu_request_used": `sum(kube_resourcequota{type="used", resource="requests.cpu"}) by (resource, type)`, + "cluster_service_loadbalancer_used": `sum(kube_resourcequota{type="used", resource="services.loadbalancers"}) by (resource, type)`, + + "cluster_configmap_count_hard": `sum(kube_resourcequota{type="hard", resource="count/configmaps"}) by (resource, type)`, + "cluster_jobs_batch_count_hard": `sum(kube_resourcequota{type="hard", resource="count/jobs.batch"}) by (resource, type)`, + "cluster_roles_count_hard": `sum(kube_resourcequota{type="hard", resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`, + "cluster_memory_limit_hard": `sum(kube_resourcequota{type="hard", resource="limits.memory"}) by (resource, type)`, + "cluster_pvc_hard": `sum(kube_resourcequota{type="hard", resource="persistentvolumeclaims"}) by (resource, type)`, + "cluster_memory_request_hard": `sum(kube_resourcequota{type="hard", resource="requests.memory"}) by (resource, type)`, + "cluster_pvc_count_hard": `sum(kube_resourcequota{type="hard", resource="count/persistentvolumeclaims"}) by (resource, type)`, + "cluster_cronjobs_batch_count_hard": `sum(kube_resourcequota{type="hard", resource="count/cronjobs.batch"}) by (resource, type)`, + "cluster_ingresses_extensions_count_hard": `sum(kube_resourcequota{type="hard", resource="count/ingresses.extensions"}) by (resource, type)`, + "cluster_cpu_limit_hard": `sum(kube_resourcequota{type="hard", resource="limits.cpu"}) by (resource, type)`, + "cluster_storage_request_hard": `sum(kube_resourcequota{type="hard", resource="requests.storage"}) by (resource, type)`, + "cluster_deployment_count_hard": `sum(kube_resourcequota{type="hard", resource="count/deployments.apps"}) by (resource, type)`, + "cluster_pod_count_hard": `sum(kube_resourcequota{type="hard", resource="count/pods"}) by (resource, type)`, + "cluster_statefulset_count_hard": `sum(kube_resourcequota{type="hard", resource="count/statefulsets.apps"}) by (resource, type)`, + "cluster_daemonset_count_hard": `sum(kube_resourcequota{type="hard", resource="count/daemonsets.apps"}) by (resource, type)`, + "cluster_secret_count_hard": `sum(kube_resourcequota{type="hard", resource="count/secrets"}) by (resource, type)`, + "cluster_service_count_hard": `sum(kube_resourcequota{type="hard", resource="count/services"}) by (resource, type)`, + "cluster_cpu_request_hard": `sum(kube_resourcequota{type="hard", resource="requests.cpu"}) by (resource, type)`, + "cluster_service_loadbalancer_hard": `sum(kube_resourcequota{type="hard", resource="services.loadbalancers"}) by (resource, type)`, + + "cluster_cronjob_count": `sum(kube_cronjob_labels)`, + "cluster_pvc_count": `sum(kube_persistentvolumeclaim_info)`, + "cluster_daemonset_count": `sum(kube_daemonset_labels)`, + "cluster_deployment_count": `sum(kube_deployment_labels)`, + "cluster_endpoint_count": `sum(kube_endpoint_labels)`, + "cluster_hpa_count": `sum(kube_hpa_labels)`, + "cluster_job_count": `sum(kube_job_labels)`, + "cluster_statefulset_count": `sum(kube_statefulset_labels)`, + "cluster_replicaset_count": `count(kube_replicaset_created)`, + "cluster_service_count": `sum(kube_service_info)`, + "cluster_secret_count": `sum(kube_secret_info)`, + "cluster_pv_count": `sum(kube_persistentvolume_labels)`, + + //node + "node_cpu_utilisation": "node:node_cpu_utilisation:avg1m", + "node_cpu_total": "node:node_num_cpu:sum", + "node_memory_utilisation": "node:node_memory_utilisation:", + "node_memory_bytes_available": "node:node_memory_bytes_available:sum", + "node_memory_bytes_total": "node:node_memory_bytes_total:sum", + // Node network utilisation (bytes received + bytes transmitted per second) + "node_net_utilisation": "node:node_net_utilisation:sum_irate", + // Node network bytes transmitted per second + "node_net_bytes_transmitted": "node:node_net_bytes_transmitted:sum_irate", + // Node network bytes received per second + "node_net_bytes_received": "node:node_net_bytes_received:sum_irate", + + // node:data_volume_iops_reads:sum{node=~"i-5xcldxos|i-6soe9zl1"} + "node_disk_read_iops": "node:data_volume_iops_reads:sum", + // node:data_volume_iops_writes:sum{node=~"i-5xcldxos|i-6soe9zl1"} + "node_disk_write_iops": "node:data_volume_iops_writes:sum", + // node:data_volume_throughput_bytes_read:sum{node=~"i-5xcldxos|i-6soe9zl1"} + "node_disk_read_throughput": "node:data_volume_throughput_bytes_read:sum", + // node:data_volume_throughput_bytes_written:sum{node=~"i-5xcldxos|i-6soe9zl1"} + "node_disk_write_throughput": "node:data_volume_throughput_bytes_written:sum", + + "node_disk_size_capacity": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + "node_disk_size_available": `sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + "node_disk_size_usage": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) -sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + "node_disk_size_utilisation": `sum by (node) (((node_filesystem_size{mountpoint="/", job="node-exporter"} - node_filesystem_avail{mountpoint="/", job="node-exporter"}) / node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + + "node_disk_inode_total": `node:disk_inodes_total:$1`, + "node_disk_inode_usage": `node:disk_inodes_total:$1 - node:disk_inodes_free:$1`, + "node_disk_inode_utilisation": `(1 - (node:disk_inodes_free:$1 / node:disk_inodes_total:$1))`, + + "node_pod_count": `count(kube_pod_info$1) by (node)`, + "node_pod_quota": `sum(kube_node_status_capacity_pods$1) by (node)`, + "node_pod_utilisation": `sum(kube_pod_info$1) by (node) / sum(kube_node_status_capacity_pods$1) by (node)`, + "node_pod_running_count": `count(kube_pod_info$1 unless on (pod) (kube_pod_status_phase{phase=~"Failed|Pending|Unknown|Succeeded"} > 0)) by (node)`, + "node_pod_succeeded_count": `count(kube_pod_info$1 unless on (pod) (kube_pod_status_phase{phase=~"Failed|Pending|Unknown|Running"} > 0)) by (node)`, + "node_pod_abnormal_count": `count(kube_pod_info$1 unless on (pod) (kube_pod_status_phase{phase=~"Succeeded|Running"} > 0)) by (node)`, + + // without log node: unless on(node) kube_node_labels{label_role="log"} + "node_cpu_usage": `node:node_cpu_utilisation:avg1m$1 * node:node_num_cpu:sum$1`, + "node_memory_bytes_usage": "node:node_memory_bytes_total:sum$1 - node:node_memory_bytes_available:sum$1", + + //namespace + "namespace_cpu_usage": `namespace:container_cpu_usage_seconds_total:sum_rate{namespace=~"$1"}`, + "namespace_memory_usage": `namespace:container_memory_usage_bytes:sum{namespace=~"$1"}`, + "namespace_memory_usage_wo_cache": `namespace:container_memory_usage_bytes_wo_cache:sum{namespace=~"$1"}`, + "namespace_net_bytes_transmitted": `sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m]))`, + "namespace_net_bytes_received": `sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m]))`, + "namespace_pod_count": `sum(kube_pod_status_phase{namespace=~"$1"}) by (namespace)`, + "namespace_pod_running_count": `sum(kube_pod_status_phase{phase="Running", namespace=~"$1"}) by (namespace)`, + "namespace_pod_succeeded_count": `sum(kube_pod_status_phase{phase="Succeeded", namespace=~"$1"}) by (namespace)`, + "namespace_pod_abnormal_count": `sum(kube_pod_status_phase{phase=~"Failed|Pending|Unknown", namespace=~"$1"}) by (namespace)`, + + "namespace_configmap_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/configmaps"}) by (namespace, resource, type)`, + "namespace_jobs_batch_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/jobs.batch"}) by (namespace, resource, type)`, + "namespace_roles_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/roles.rbac.authorization.k8s.io"}) by (namespace, resource, type)`, + "namespace_memory_limit_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="limits.memory"}) by (namespace, resource, type)`, + "namespace_pvc_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="persistentvolumeclaims"}) by (namespace, resource, type)`, + "namespace_memory_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.memory"}) by (namespace, resource, type)`, + "namespace_pvc_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/persistentvolumeclaims"}) by (namespace, resource, type)`, + "namespace_cronjobs_batch_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/cronjobs.batch"}) by (namespace, resource, type)`, + "namespace_ingresses_extensions_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/ingresses.extensions"}) by (namespace, resource, type)`, + "namespace_cpu_limit_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="limits.cpu"}) by (namespace, resource, type)`, + "namespace_storage_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.storage"}) by (namespace, resource, type)`, + "namespace_deployment_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/deployments.apps"}) by (namespace, resource, type)`, + "namespace_pod_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/pods"}) by (namespace, resource, type)`, + "namespace_statefulset_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/statefulsets.apps"}) by (namespace, resource, type)`, + "namespace_daemonset_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/daemonsets.apps"}) by (namespace, resource, type)`, + "namespace_secret_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/secrets"}) by (namespace, resource, type)`, + "namespace_service_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/services"}) by (namespace, resource, type)`, + "namespace_cpu_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.cpu"}) by (namespace, resource, type)`, + "namespace_service_loadbalancer_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="services.loadbalancers"}) by (namespace, resource, type)`, + + "namespace_configmap_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/configmaps"}) by (namespace, resource, type)`, + "namespace_jobs_batch_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/jobs.batch"}) by (namespace, resource, type)`, + "namespace_roles_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/roles.rbac.authorization.k8s.io"}) by (namespace, resource, type)`, + "namespace_memory_limit_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="limits.memory"}) by (namespace, resource, type)`, + "namespace_pvc_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="persistentvolumeclaims"}) by (namespace, resource, type)`, + "namespace_memory_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.memory"}) by (namespace, resource, type)`, + "namespace_pvc_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/persistentvolumeclaims"}) by (namespace, resource, type)`, + "namespace_cronjobs_batch_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/cronjobs.batch"}) by (namespace, resource, type)`, + "namespace_ingresses_extensions_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/ingresses.extensions"}) by (namespace, resource, type)`, + "namespace_cpu_limit_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="limits.cpu"}) by (namespace, resource, type)`, + "namespace_storage_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.storage"}) by (namespace, resource, type)`, + "namespace_deployment_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/deployments.apps"}) by (namespace, resource, type)`, + "namespace_pod_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/pods"}) by (namespace, resource, type)`, + "namespace_statefulset_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/statefulsets.apps"}) by (namespace, resource, type)`, + "namespace_daemonset_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/daemonsets.apps"}) by (namespace, resource, type)`, + "namespace_secret_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/secrets"}) by (namespace, resource, type)`, + "namespace_service_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/services"}) by (namespace, resource, type)`, + "namespace_cpu_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.cpu"}) by (namespace, resource, type)`, + "namespace_service_loadbalancer_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="services.loadbalancers"}) by (namespace, resource, type)`, + + "namespace_cronjob_count": `sum(kube_cronjob_labels{namespace=~"$1"}) by (namespace)`, + "namespace_pvc_count": `sum(kube_persistentvolumeclaim_info{namespace=~"$1"}) by (namespace)`, + "namespace_daemonset_count": `sum(kube_daemonset_labels{namespace=~"$1"}) by (namespace)`, + "namespace_deployment_count": `sum(kube_deployment_labels{namespace=~"$1"}) by (namespace)`, + "namespace_endpoint_count": `sum(kube_endpoint_labels{namespace=~"$1"}) by (namespace)`, + "namespace_hpa_count": `sum(kube_hpa_labels{namespace=~"$1"}) by (namespace)`, + "namespace_job_count": `sum(kube_job_labels{namespace=~"$1"}) by (namespace)`, + "namespace_statefulset_count": `sum(kube_statefulset_labels{namespace=~"$1"}) by (namespace)`, + "namespace_replicaset_count": `count(kube_replicaset_created{namespace=~"$1"}) by (namespace)`, + "namespace_service_count": `sum(kube_service_info{namespace=~"$1"}) by (namespace)`, + "namespace_secret_count": `sum(kube_secret_info{namespace=~"$1"}) by (namespace)`, + + // pod + "pod_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name="$2", image!=""}[5m])) by (namespace, pod_name)`, + "pod_memory_usage": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""}) by (namespace, pod_name)`, + "pod_memory_usage_wo_cache": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name="$2",image!=""}) by (namespace, pod_name)`, + "pod_net_bytes_transmitted": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[5m]))`, + "pod_net_bytes_received": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[5m]))`, + + "pod_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}[5m])) by (namespace, pod_name)`, + "pod_memory_usage_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`, + "pod_memory_usage_wo_cache_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`, + "pod_net_bytes_transmitted_all": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`, + "pod_net_bytes_received_all": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`, + + "pod_cpu_usage_node": `sum by (node, pod) (label_join(irate(container_cpu_usage_seconds_total{job="kubelet",pod_name=~"$2", image!=""}[5m]), "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, + "pod_memory_usage_node": `sum by (node, pod) (label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, + "pod_memory_usage_wo_cache_node": `sum by (node, pod) ((label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") - label_join(container_memory_cache{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name")) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, + + // container + "container_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name="$3"}[5m])) by (namespace, pod_name, container_name)`, + "container_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}[5m])) by (namespace, pod_name, container_name)`, + + "container_memory_usage_wo_cache": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name="$3"}`, + "container_memory_usage_wo_cache_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`, + + "container_memory_usage": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"}`, + "container_memory_usage_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`, + + // workspace + "workspace_cpu_usage": `sum(namespace:container_cpu_usage_seconds_total:sum_rate{namespace =~"$1"})`, + "workspace_memory_usage": `sum(namespace:container_memory_usage_bytes:sum{namespace =~"$1"})`, + "workspace_memory_usage_wo_cache": `sum(namespace:container_memory_usage_bytes_wo_cache:sum{namespace =~"$1"})`, + "workspace_net_bytes_transmitted": `sum(sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m])))`, + "workspace_net_bytes_received": `sum(sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m])))`, + "workspace_pod_count": `sum(kube_pod_status_phase{namespace=~"$1"})`, + "workspace_pod_running_count": `sum(kube_pod_status_phase{phase="Running", namespace=~"$1"})`, + "workspace_pod_succeeded_count": `sum(kube_pod_status_phase{phase="Succeeded", namespace=~"$1"})`, + "workspace_pod_abnormal_count": `sum(kube_pod_status_phase{phase=~"Failed|Pending|Unknown", namespace=~"$1"})`, + + "workspace_configmap_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/configmaps"}) by (resource, type)`, + "workspace_jobs_batch_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/jobs.batch"}) by (resource, type)`, + "workspace_roles_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`, + "workspace_memory_limit_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="limits.memory"}) by (resource, type)`, + "workspace_pvc_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="persistentvolumeclaims"}) by (resource, type)`, + "workspace_memory_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.memory"}) by (resource, type)`, + "workspace_pvc_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/persistentvolumeclaims"}) by (resource, type)`, + "workspace_cronjobs_batch_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/cronjobs.batch"}) by (resource, type)`, + "workspace_ingresses_extensions_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/ingresses.extensions"}) by (resource, type)`, + "workspace_cpu_limit_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="limits.cpu"}) by (resource, type)`, + "workspace_storage_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.storage"}) by (resource, type)`, + "workspace_deployment_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/deployments.apps"}) by (resource, type)`, + "workspace_pod_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/pods"}) by (resource, type)`, + "workspace_statefulset_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/statefulsets.apps"}) by (resource, type)`, + "workspace_daemonset_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/daemonsets.apps"}) by (resource, type)`, + "workspace_secret_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/secrets"}) by (resource, type)`, + "workspace_service_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/services"}) by (resource, type)`, + "workspace_cpu_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.cpu"}) by (resource, type)`, + "workspace_service_loadbalancer_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="services.loadbalancers"}) by (resource, type)`, + + "workspace_configmap_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/configmaps"}) by (resource, type)`, + "workspace_jobs_batch_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/jobs.batch"}) by (resource, type)`, + "workspace_roles_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`, + "workspace_memory_limit_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="limits.memory"}) by (resource, type)`, + "workspace_pvc_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="persistentvolumeclaims"}) by (resource, type)`, + "workspace_memory_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.memory"}) by (resource, type)`, + "workspace_pvc_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/persistentvolumeclaims"}) by (resource, type)`, + "workspace_cronjobs_batch_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/cronjobs.batch"}) by (resource, type)`, + "workspace_ingresses_extensions_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/ingresses.extensions"}) by (resource, type)`, + "workspace_cpu_limit_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="limits.cpu"}) by (resource, type)`, + "workspace_storage_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.storage"}) by (resource, type)`, + "workspace_deployment_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/deployments.apps"}) by (resource, type)`, + "workspace_pod_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/pods"}) by (resource, type)`, + "workspace_statefulset_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/statefulsets.apps"}) by (resource, type)`, + "workspace_daemonset_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/daemonsets.apps"}) by (resource, type)`, + "workspace_secret_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/secrets"}) by (resource, type)`, + "workspace_service_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/services"}) by (resource, type)`, + "workspace_cpu_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.cpu"}) by (resource, type)`, + "workspace_service_loadbalancer_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="services.loadbalancers"}) by (resource, type)`, + "workspace_ingresses_extensions_count": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/ingresses.extensions"}) by (resource, type)`, + + "workspace_cronjob_count": `sum(kube_cronjob_labels{namespace=~"$1"})`, + "workspace_pvc_count": `sum(kube_persistentvolumeclaim_info{namespace=~"$1"})`, + "workspace_daemonset_count": `sum(kube_daemonset_labels{namespace=~"$1"})`, + "workspace_deployment_count": `sum(kube_deployment_labels{namespace=~"$1"})`, + "workspace_endpoint_count": `sum(kube_endpoint_labels{namespace=~"$1"})`, + "workspace_hpa_count": `sum(kube_hpa_labels{namespace=~"$1"})`, + "workspace_job_count": `sum(kube_job_labels{namespace=~"$1"})`, + "workspace_statefulset_count": `sum(kube_statefulset_labels{namespace=~"$1"})`, + "workspace_replicaset_count": `count(kube_replicaset_created{namespace=~"$1"})`, + "workspace_service_count": `sum(kube_service_info{namespace=~"$1"})`, + "workspace_secret_count": `sum(kube_secret_info{namespace=~"$1"})`, +} diff --git a/pkg/models/metrics/metricsstruct.go b/pkg/models/metrics/metricsstruct.go deleted file mode 100755 index 4c77783bc..000000000 --- a/pkg/models/metrics/metricsstruct.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2018 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -type FormatedLevelMetric struct { - MetricsLevel string `json:"metrics_level"` - Results []FormatedMetric `json:"results"` -} - -type FormatedMetric struct { - MetricName string `json:"metric_name, omitempty"` - Status string `json:"status"` - Data FormatedMetricData `json:"data, omitempty"` -} - -type FormatedMetricData struct { - Result []map[string]interface{} `json:"result"` - ResultType string `json:"resultType"` -} - -type CommonMetricsResult struct { - Status string `json:"status"` - Data CommonMetricsData `json:"data"` -} - -type CommonMetricsData struct { - Result []CommonResultItem `json:"result"` - ResultType string `json:"resultType"` -} - -type CommonResultItem struct { - KubePodMetric KubePodMetric `json:"metric"` - Value interface{} `json:"value"` -} - -/** -"__name__": "kube_pod_info", -"created_by_kind": "\\u003cnone\\u003e", -"created_by_name": "\\u003cnone\\u003e", -"endpoint": "https-main", -"host_ip": "192.168.0.13", -"instance": "10.244.114.187:8443", -"job": "kube-state-metrics", -"namespace": "kube-system", -"node": "i-39p7faw6", -"pod": "cloud-controller-manager-i-39p7faw6", -"pod_ip": "192.168.0.13", -"service": "kube-state-metrics" -*/ -type KubePodMetric struct { - CreatedByKind string `json:"created_by_kind"` - CreatedByName string `json:"created_by_name"` - Namespace string `json:"namespace"` - Pod string `json:"pod"` -} diff --git a/pkg/models/metrics/util.go b/pkg/models/metrics/util.go new file mode 100644 index 000000000..78f62e46e --- /dev/null +++ b/pkg/models/metrics/util.go @@ -0,0 +1,268 @@ +/* +Copyright 2018 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "encoding/json" + "math" + "sort" + "strconv" + "unicode" + + "github.com/golang/glog" +) + +const ( + DefaultPageLimit = 5 + DefaultPage = 1 +) + +type FormatedMetricDataWrapper struct { + fmtMetricData FormatedMetricData + by func(p, q *map[string]interface{}) bool +} + +func (wrapper FormatedMetricDataWrapper) Len() int { + return len(wrapper.fmtMetricData.Result) +} + +func (wrapper FormatedMetricDataWrapper) Less(i, j int) bool { + return wrapper.by(&wrapper.fmtMetricData.Result[i], &wrapper.fmtMetricData.Result[j]) +} + +func (wrapper FormatedMetricDataWrapper) Swap(i, j int) { + wrapper.fmtMetricData.Result[i], wrapper.fmtMetricData.Result[j] = wrapper.fmtMetricData.Result[j], wrapper.fmtMetricData.Result[i] +} + +// sorted metric by ascending or descending order +func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelMetric, resourceType string) (*FormatedLevelMetric, int) { + var maxLength = 0 + for _, metricItem := range fmtLevelMetric.Results { + if metricItem.Data.ResultType == ResultTypeVector && metricItem.Status == MetricStatusSuccess { + if maxLength < len(metricItem.Data.Result) { + maxLength = len(metricItem.Data.Result) + } + } + } + + if sortMetricName == "" { + return fmtLevelMetric, maxLength + } + + // default sort type is descending order + if sortType == "" { + sortType = ResultSortTypeDesc + } + + var currentResourceMap = make(map[string]int) + + // indexMap store sorted index for each node/namespace/pod + var indexMap = make(map[string]int) + for _, metricItem := range fmtLevelMetric.Results { + if metricItem.Data.ResultType == ResultTypeVector && metricItem.Status == MetricStatusSuccess { + if metricItem.MetricName == sortMetricName { + if sortType == ResultSortTypeAsce { + // desc + sort.Sort(FormatedMetricDataWrapper{metricItem.Data, func(p, q *map[string]interface{}) bool { + value1 := (*p)[ResultItemValue].([]interface{}) + value2 := (*q)[ResultItemValue].([]interface{}) + v1, _ := strconv.ParseFloat(value1[len(value1)-1].(string), 64) + v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64) + return v1 < v2 + }}) + } else { + // desc + sort.Sort(FormatedMetricDataWrapper{metricItem.Data, func(p, q *map[string]interface{}) bool { + value1 := (*p)[ResultItemValue].([]interface{}) + value2 := (*q)[ResultItemValue].([]interface{}) + v1, _ := strconv.ParseFloat(value1[len(value1)-1].(string), 64) + v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64) + return v1 > v2 + }}) + } + + i := 0 + for _, r := range metricItem.Data.Result { + // for some reasons, 'metric' may not contain `resourceType` field + // example: {"metric":{},"value":[1541142931.731,"3"]} + k, exist := r[ResultItemMetric].(map[string]interface{})[resourceType] + if exist { + indexMap[k.(string)] = i + i = i + 1 + } + } + } + + // iterator all metric to find max metricItems length + for _, r := range metricItem.Data.Result { + k, ok := r[ResultItemMetric].(map[string]interface{})[resourceType] + if ok { + currentResourceMap[k.(string)] = 1 + } + } + + } + } + + // sort other metric + for i := 0; i < len(fmtLevelMetric.Results); i++ { + re := fmtLevelMetric.Results[i] + if re.MetricName != sortMetricName && re.Data.ResultType == ResultTypeVector && re.Status == MetricStatusSuccess { + sortedMetric := make([]map[string]interface{}, len(indexMap)) + noneSortedMetric := make([]map[string]interface{}, 0) + for j := 0; j < len(re.Data.Result); j++ { + r := re.Data.Result[j] + k, ok := r[ResultItemMetric].(map[string]interface{})[resourceType] + if ok { + index, exist := indexMap[k.(string)] + if exist { + sortedMetric[index] = r + } else { + noneSortedMetric = append(noneSortedMetric, r) + } + } + } + + sortedMetric = append(sortedMetric, noneSortedMetric...) + fmtLevelMetric.Results[i].Data.Result = sortedMetric + } + } + + return fmtLevelMetric, maxLength +} + +func Page(pageNum string, limitNum string, fmtLevelMetric *FormatedLevelMetric, maxLength int) interface{} { + // matrix type can not be sorted + for _, metricItem := range fmtLevelMetric.Results { + if metricItem.Data.ResultType != ResultTypeVector { + return fmtLevelMetric + } + } + + var page = DefaultPage + + if pageNum != "" { + p, err := strconv.Atoi(pageNum) + if err != nil { + glog.Errorln(err) + } else { + if p > 0 { + page = p + } + } + } else { + // the default mode is none paging + return fmtLevelMetric + } + + var limit = DefaultPageLimit + + if limitNum != "" { + l, err := strconv.Atoi(limitNum) + if err != nil { + glog.Errorln(err) + } else { + if l > 0 { + limit = l + } + } + } + + // the i page: [(page-1) * limit, (page) * limit - 1] + start := (page - 1) * limit + end := (page)*limit - 1 + + for i := 0; i < len(fmtLevelMetric.Results); i++ { + // only pageing when result type is `vector` and result status is `success` + if fmtLevelMetric.Results[i].Data.ResultType != ResultTypeVector || fmtLevelMetric.Results[i].Status != MetricStatusSuccess { + continue + } + resultLen := len(fmtLevelMetric.Results[i].Data.Result) + if start >= resultLen { + fmtLevelMetric.Results[i].Data.Result = nil + continue + } + if end >= resultLen { + end = resultLen - 1 + } + slice := fmtLevelMetric.Results[i].Data.Result[start : end+1] + fmtLevelMetric.Results[i].Data.Result = slice + } + + allPage := int(math.Ceil(float64(maxLength) / float64(limit))) + + return &PagedFormatedLevelMetric{ + Message: "paged", + TotalPage: allPage, + CurrentPage: page, + Metric: *fmtLevelMetric, + } +} + +// maybe this function is time consuming +func ReformatJson(metric string, metricsName string, needDelParams ...string) *FormatedMetric { + var formatMetric FormatedMetric + err := json.Unmarshal([]byte(metric), &formatMetric) + if err != nil { + glog.Errorln("Unmarshal metric json failed", err) + } + if formatMetric.MetricName == "" { + if metricsName != "" { + formatMetric.MetricName = metricsName + } + } + // retrive metrics success + if formatMetric.Status == MetricStatusSuccess { + result := formatMetric.Data.Result + for _, res := range result { + metric, exist := res[ResultItemMetric] + metricMap := metric.(map[string]interface{}) + if exist { + delete(metricMap, "__name__") + } + if len(needDelParams) > 0 { + for _, p := range needDelParams { + delete(metricMap, p) + } + } + } + } + + return &formatMetric +} + +func ReformatNodeStatusField(nodeMetric *FormatedMetric) *FormatedMetric { + metricCount := len(nodeMetric.Data.Result) + for i := 0; i < metricCount; i++ { + metric, exist := nodeMetric.Data.Result[i][ResultItemMetric] + if exist { + status, exist := metric.(map[string]interface{})[MetricStatus] + if exist { + status = UpperFirstLetter(status.(string)) + metric.(map[string]interface{})[MetricStatus] = status + } + } + } + return nodeMetric +} + +func UpperFirstLetter(str string) string { + for i, ch := range str { + return string(unicode.ToUpper(ch)) + str[i+1:] + } + return "" +} diff --git a/pkg/models/workspaces/workspaces.go b/pkg/models/workspaces/workspaces.go index a840c3fc8..8667a89cc 100644 --- a/pkg/models/workspaces/workspaces.go +++ b/pkg/models/workspaces/workspaces.go @@ -24,6 +24,8 @@ import ( clientV1 "k8s.io/client-go/listers/core/v1" "k8s.io/kubernetes/pkg/util/slice" + "github.com/golang/glog" + "kubesphere.io/kubesphere/pkg/client" "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/models/controllers" @@ -31,6 +33,10 @@ import ( ksErr "kubesphere.io/kubesphere/pkg/util/errors" ) +const ( + WorkspaceKey = "kubesphere.io/workspace" +) + var WorkSpaceRoles = []string{"admin", "operator", "viewer"} func UnBindDevopsProject(workspace string, devops string) error { @@ -1307,3 +1313,37 @@ func GetAllAccountNums() (int, error) { } return 0, errors.New("not found") } + +// get cluster organizations name which contains at least one namespace, +func GetAllOrgAndProjList() (map[string][]string, map[string]string, error) { + nsList, err := client.NewK8sClient().CoreV1().Namespaces().List(meta_v1.ListOptions{}) + if err != nil { + glog.Errorln(err) + return nil, nil, err + } + + var workspaceNamespaceMap = make(map[string][]string) + var namespaceWorkspaceMap = make(map[string]string) + + for _, item := range nsList.Items { + ws, exist := item.Labels[WorkspaceKey] + ns := item.Name + if exist { + if nsArray, exist := workspaceNamespaceMap[ws]; exist { + nsArray = append(nsArray, ns) + workspaceNamespaceMap[ws] = nsArray + } else { + var nsArray []string + nsArray = append(nsArray, ns) + workspaceNamespaceMap[ws] = nsArray + } + + namespaceWorkspaceMap[ns] = ws + } else { + // this namespace do not belong to any workspaces + namespaceWorkspaceMap[ns] = "" + } + } + + return workspaceNamespaceMap, namespaceWorkspaceMap, nil +} From beb7efdac04047a6a0b4d7056a2abc9558c46651 Mon Sep 17 00:00:00 2001 From: Carman Zhang Date: Wed, 14 Nov 2018 16:04:24 +0800 Subject: [PATCH 4/4] fixed deployment-pods metrics --- .../v1alpha/monitoring/monitor_handler.go | 69 +++----- pkg/models/metrics/metrics.go | 166 +++++++++++++----- pkg/models/metrics/metricsrule.go | 2 +- pkg/models/metrics/util.go | 4 +- 4 files changed, 155 insertions(+), 86 deletions(-) diff --git a/pkg/apis/v1alpha/monitoring/monitor_handler.go b/pkg/apis/v1alpha/monitoring/monitor_handler.go index 36b580722..262b3d116 100755 --- a/pkg/apis/v1alpha/monitoring/monitor_handler.go +++ b/pkg/apis/v1alpha/monitoring/monitor_handler.go @@ -1,9 +1,12 @@ /* Copyright 2018 The KubeSphere Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -66,38 +69,31 @@ func (u Monitor) monitorWorkload(request *restful.Request, response *restful.Res } } -// merge multiple metric: all-devops, all-roles, all-projects...this api is designed for admin -func (u Monitor) monitorAllWorkspacesStatistics(request *restful.Request, response *restful.Response) { - res := metrics.MonitorAllWorkspacesStatistics() - response.WriteAsJson(res) -} - -// merge multiple metric: devops, roles, projects... -func (u Monitor) monitorOneWorkspaceStatistics(request *restful.Request, response *restful.Response) { - requestParams := client.ParseMonitoringRequestParams(request) - wsName := requestParams.WsName - res := metrics.MonitorOneWorkspaceStatistics(wsName) - response.WriteAsJson(res) -} - func (u Monitor) monitorAllWorkspaces(request *restful.Request, response *restful.Response) { requestParams := client.ParseMonitoringRequestParams(request) - rawMetrics := metrics.MonitorAllWorkspaces(requestParams) - // sorting - sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelWorkspace) - // paging - pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + if requestParams.Tp == "_statistics" { + // merge multiple metric: all-devops, all-roles, all-projects...this api is designed for admin + res := metrics.MonitorAllWorkspacesStatistics() - response.WriteAsJson(pagedMetrics) + response.WriteAsJson(res) + } else { + rawMetrics := metrics.MonitorAllWorkspaces(requestParams) + // sorting + sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelWorkspace) + // paging + pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + + response.WriteAsJson(pagedMetrics) + } } func (u Monitor) monitorOneWorkspace(request *restful.Request, response *restful.Response) { requestParams := client.ParseMonitoringRequestParams(request) tp := requestParams.Tp - if tp != "" { + if tp == "rank" { // multiple rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkspace) // sorting @@ -107,6 +103,12 @@ func (u Monitor) monitorOneWorkspace(request *restful.Request, response *restful response.WriteAsJson(pagedMetrics) + } else if tp == "_statistics" { + wsName := requestParams.WsName + + // merge multiple metric: devops, roles, projects... + res := metrics.MonitorOneWorkspaceStatistics(wsName) + response.WriteAsJson(res) } else { res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkspace) response.WriteAsJson(res) @@ -350,17 +352,8 @@ func Register(ws *restful.WebService, subPath string) { Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) - // merge multiple metric: devops, roles, projects... - ws.Route(ws.GET(subPath+"/cluster_workspaces/{workspace_name}/_statistics").To(u.monitorOneWorkspaceStatistics). - Filter(route.RouteLogging). - Doc("monitor specific workspace level metrics"). - Param(ws.PathParameter("workspace_name", "workspace name").DataType("string").Required(true)). - Metadata(restfulspec.KeyOpenAPITags, tags)). - Consumes(restful.MIME_JSON, restful.MIME_XML). - Produces(restful.MIME_JSON) - // list all namespace in this workspace by selected metrics - ws.Route(ws.GET(subPath+"/cluster_workspaces/{workspace_name}").To(u.monitorOneWorkspace). + ws.Route(ws.GET(subPath+"/workspaces/{workspace_name}").To(u.monitorOneWorkspace). Filter(route.RouteLogging). Doc("monitor workspaces level metrics"). Param(ws.PathParameter("workspace_name", "workspace name").DataType("string").Required(true)). @@ -370,21 +363,12 @@ func Register(ws *restful.WebService, subPath string) { Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). - Param(ws.QueryParameter("type", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("type", "rank, statistic").DataType("string").Required(false).DefaultValue("rank")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) - // metrics from system users or projects, merge multiple metric: all-devops, all-roles, all-projects... - ws.Route(ws.GET(subPath+"/cluster_workspaces/_statistics").To(u.monitorAllWorkspacesStatistics). - Filter(route.RouteLogging). - Doc("monitor specific workspace level metrics"). - Metadata(restfulspec.KeyOpenAPITags, tags)). - Consumes(restful.MIME_JSON, restful.MIME_XML). - Produces(restful.MIME_JSON) - - // metrics from prometheus - ws.Route(ws.GET(subPath+"/cluster_workspaces").To(u.monitorAllWorkspaces). + ws.Route(ws.GET(subPath+"/workspaces").To(u.monitorAllWorkspaces). Filter(route.RouteLogging). Doc("monitor workspaces level metrics"). Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("workspace_memory_utilisation")). @@ -393,6 +377,7 @@ func Register(ws *restful.WebService, subPath string) { Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). + Param(ws.QueryParameter("type", "rank, statistic").DataType("string").Required(false).DefaultValue("rank")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) diff --git a/pkg/models/metrics/metrics.go b/pkg/models/metrics/metrics.go index 0127dd5ab..5287ab71c 100644 --- a/pkg/models/metrics/metrics.go +++ b/pkg/models/metrics/metrics.go @@ -30,6 +30,9 @@ import ( "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "runtime/debug" + "sort" + "kubesphere.io/kubesphere/pkg/client" "kubesphere.io/kubesphere/pkg/models" "kubesphere.io/kubesphere/pkg/models/workspaces" @@ -38,7 +41,7 @@ import ( var nodeStatusDelLables = []string{"endpoint", "instance", "job", "namespace", "pod", "service"} const ( - ChannelMaxCapacityWorkspaceMetric = 400 + ChannelMaxCapacityWorkspaceMetric = 800 ChannelMaxCapacity = 100 ) @@ -126,6 +129,7 @@ func getPodNameRegexInWorkload(res string) string { glog.Errorln("json parse failed", jsonErr) } var podNames []string + for _, item := range dat.Data.Result { podName := item.KubePodMetric.Pod podNames = append(podNames, podName) @@ -134,8 +138,66 @@ func getPodNameRegexInWorkload(res string) string { return podNamesFilter } +func unifyMetricHistoryTimeRange(fmtMetrics *FormatedMetric) { + + var timestampMap = make(map[float64]bool) + + if fmtMetrics.Data.ResultType == ResultTypeMatrix { + for i, _ := range fmtMetrics.Data.Result { + values, exist := fmtMetrics.Data.Result[i][ResultItemValues] + if exist { + valueArray, sure := values.([]interface{}) + if sure { + for j, _ := range valueArray { + timeAndValue := valueArray[j].([]interface{}) + timestampMap[timeAndValue[0].(float64)] = true + } + } + } + } + } + + timestampArray := make([]float64, len(timestampMap)) + i := 0 + for timestamp, _ := range timestampMap { + timestampArray[i] = timestamp + i++ + } + sort.Float64s(timestampArray) + + if fmtMetrics.Data.ResultType == ResultTypeMatrix { + for i := 0; i < len(fmtMetrics.Data.Result); i++ { + + values, exist := fmtMetrics.Data.Result[i][ResultItemValues] + if exist { + valueArray, sure := values.([]interface{}) + if sure { + + formatValueArray := make([][]interface{}, len(timestampArray)) + j := 0 + + for k, _ := range timestampArray { + valueItem, sure := valueArray[j].([]interface{}) + if sure && valueItem[0].(float64) == timestampArray[k] { + formatValueArray[k] = []interface{}{int64(timestampArray[k]), valueItem[1]} + j++ + } else { + formatValueArray[k] = []interface{}{int64(timestampArray[k]), "-1"} + } + } + fmtMetrics.Data.Result[i][ResultItemValues] = formatValueArray + } + } + } + } +} + func AssembleWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { - rule := MakeWorkloadRule(monitoringRequest.WorkloadKind, monitoringRequest.WorkloadName, monitoringRequest.NsName) + + nsName := monitoringRequest.NsName + wkName := monitoringRequest.WorkloadName + + rule := MakeWorkloadRule(monitoringRequest.WorkloadKind, wkName, nsName) paramValues := monitoringRequest.Params params := makeRequestParamString(rule, paramValues) @@ -144,7 +206,7 @@ func AssembleWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringReque podNamesFilter := getPodNameRegexInWorkload(res) queryType := monitoringRequest.QueryType - rule = MakePodPromQL(metricName, monitoringRequest.NsName, "", "", podNamesFilter) + rule = MakePodPromQL(metricName, nsName, "", "", podNamesFilter) params = makeRequestParamString(rule, paramValues) return queryType, params @@ -167,7 +229,10 @@ func GetMetric(queryType, params, metricName string) *FormatedMetric { } func GetNodeAddressInfo() *map[string][]v1.NodeAddress { - nodeList, _ := client.NewK8sClient().CoreV1().Nodes().List(metaV1.ListOptions{}) + nodeList, err := client.NewK8sClient().CoreV1().Nodes().List(metaV1.ListOptions{}) + if err != nil { + glog.Errorln(err.Error()) + } var nodeAddress = make(map[string][]v1.NodeAddress) for _, node := range nodeList.Items { nodeAddress[node.Name] = node.Status.Addresses @@ -179,11 +244,13 @@ func AddNodeAddressMetric(nodeMetric *FormatedMetric, nodeAddress *map[string][] for i := 0; i < len(nodeMetric.Data.Result); i++ { metricDesc := nodeMetric.Data.Result[i][ResultItemMetric] - metricDescMap := metricDesc.(map[string]interface{}) - if nodeId, exist := metricDescMap["node"]; exist { - addr, exist := (*nodeAddress)[nodeId.(string)] - if exist { - metricDescMap["address"] = addr + metricDescMap, ensure := metricDesc.(map[string]interface{}) + if ensure { + if nodeId, exist := metricDescMap["node"]; exist { + addr, exist := (*nodeAddress)[nodeId.(string)] + if exist { + metricDescMap["address"] = addr + } } } } @@ -223,6 +290,13 @@ func AssembleWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequ func makeRequestParamString(rule string, paramValues url.Values) string { + defer func() { + if err := recover(); err != nil { + glog.Errorln(err) + debug.PrintStack() + } + }() + var values = make(url.Values) for key, v := range paramValues { values.Set(key, v[0]) @@ -265,7 +339,12 @@ func MonitorAllWorkspaces(monitoringRequest *client.MonitoringRequestParams) *Fo var wgAll sync.WaitGroup var wsAllch = make(chan *[]FormatedMetric, ChannelMaxCapacityWorkspaceMetric) - workspaceNamespaceMap, _, _ := workspaces.GetAllOrgAndProjList() + workspaceNamespaceMap, _, err := workspaces.GetAllOrgAndProjList() + + if err != nil { + glog.Errorln(err.Error()) + } + for ws, _ := range workspaceNamespaceMap { bol, err := regexp.MatchString(monitoringRequest.WsFilter, ws) if err == nil && bol { @@ -320,11 +399,6 @@ func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, w for _, metricName := range filterMetricsName { wg.Add(1) go func(metricName string) { - defer func() { - if err := recover(); err != nil { - glog.Errorln(err) - } - }() queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName) ch <- GetMetric(queryType, params, metricName) @@ -341,9 +415,11 @@ func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, w if oneMetric != nil { // add "workspace" filed to oneMetric `metric` field for i := 0; i < len(oneMetric.Data.Result); i++ { - tmap := oneMetric.Data.Result[i]["metric"].(map[string]interface{}) - tmap[MetricLevelWorkspace] = ws - oneMetric.Data.Result[i]["metric"] = tmap + tmap, sure := oneMetric.Data.Result[i][ResultItemMetric].(map[string]interface{}) + if sure { + tmap[MetricLevelWorkspace] = ws + oneMetric.Data.Result[i][ResultItemMetric] = tmap + } } metricsArray = append(metricsArray, *oneMetric) } @@ -406,19 +482,7 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour } namespaceArray = filterNamespace(monitoringRequest.NsFilter, namespaceArray) - if monitoringRequest.Tp == "" { - for _, metricName := range WorkspaceMetricsNames { - bol, err := regexp.MatchString(metricsFilter, metricName) - if err == nil && bol { - wg.Add(1) - go func(metricName string) { - queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName) - ch <- GetMetric(queryType, params, metricName) - wg.Done() - }(metricName) - } - } - } else { + if monitoringRequest.Tp == "rank" { for _, metricName := range NamespaceMetricsNames { bol, err := regexp.MatchString(metricsFilter, metricName) ns := "^(" + strings.Join(namespaceArray, "|") + ")$" @@ -432,6 +496,19 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour }(metricName) } } + + } else { + for _, metricName := range WorkspaceMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName) + ch <- GetMetric(queryType, params, metricName) + wg.Done() + }(metricName) + } + } } } case MetricLevelNamespace: @@ -457,7 +534,9 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour go func(metricName string) { metricName = strings.TrimLeft(metricName, "workload_") queryType, params := AssembleWorkloadMetricRequestInfo(monitoringRequest, metricName) - ch <- GetMetric(queryType, params, metricName) + fmtMetrics := GetMetric(queryType, params, metricName) + unifyMetricHistoryTimeRange(fmtMetrics) + ch <- fmtMetrics wg.Done() }(metricName) } @@ -563,11 +642,13 @@ func calcWorkspaceNamespace(metric *FormatedMetric) int { var workspaceNamespaceCount = 0 for _, result := range metric.Data.Result { - tmpMap := result[ResultItemMetric].(map[string]interface{}) - wsName, exist := tmpMap[WorkspaceJoinedKey] + tmpMap, sure := result[ResultItemMetric].(map[string]interface{}) + if sure { + wsName, exist := tmpMap[WorkspaceJoinedKey] - if exist && wsName != "" { - workspaceNamespaceCount += 1 + if exist && wsName != "" { + workspaceNamespaceCount += 1 + } } } @@ -788,11 +869,14 @@ func MonitorComponentStatus(monitoringRequest *client.MonitoringRequestParams) * var normalNodes []string var abnormalNodes []string for _, result := range nodeStatusMetric.Data.Result { - tmap := result[ResultItemMetric].(map[string]interface{}) - if tmap[MetricStatus].(string) == "false" { - abnormalNodes = append(abnormalNodes, tmap[MetricLevelNode].(string)) - } else { - normalNodes = append(normalNodes, tmap[MetricLevelNode].(string)) + tmap, sure := result[ResultItemMetric].(map[string]interface{}) + + if sure { + if tmap[MetricStatus].(string) == "false" { + abnormalNodes = append(abnormalNodes, tmap[MetricLevelNode].(string)) + } else { + normalNodes = append(normalNodes, tmap[MetricLevelNode].(string)) + } } } diff --git a/pkg/models/metrics/metricsrule.go b/pkg/models/metrics/metricsrule.go index b2321b6b1..f0ff0811e 100755 --- a/pkg/models/metrics/metricsrule.go +++ b/pkg/models/metrics/metricsrule.go @@ -29,7 +29,7 @@ func MakeWorkloadRule(wkKind, wkName, namespace string) string { case "deployment": wkKind = ReplicaSet if wkName != "" { - wkName = "~\"" + wkName + ".*\"" + wkName = "~\"^" + wkName + `-(\\w)+$"` } else { wkName = "~\".*\"" } diff --git a/pkg/models/metrics/util.go b/pkg/models/metrics/util.go index 78f62e46e..36b081461 100644 --- a/pkg/models/metrics/util.go +++ b/pkg/models/metrics/util.go @@ -230,8 +230,8 @@ func ReformatJson(metric string, metricsName string, needDelParams ...string) *F result := formatMetric.Data.Result for _, res := range result { metric, exist := res[ResultItemMetric] - metricMap := metric.(map[string]interface{}) - if exist { + metricMap, sure := metric.(map[string]interface{}) + if exist && sure { delete(metricMap, "__name__") } if len(needDelParams) > 0 {