diff --git a/pkg/apis/v1alpha/monitoring/monitor_handler.go b/pkg/apis/v1alpha/monitoring/monitor_handler.go index 59725ee39..36b580722 100755 --- a/pkg/apis/v1alpha/monitoring/monitor_handler.go +++ b/pkg/apis/v1alpha/monitoring/monitor_handler.go @@ -13,8 +13,6 @@ limitations under the License. package monitoring import ( - "strings" - "github.com/emicklei/go-restful" "github.com/emicklei/go-restful-openapi" @@ -23,94 +21,194 @@ import ( "kubesphere.io/kubesphere/pkg/models/metrics" ) -func (u MonitorResource) monitorPod(request *restful.Request, response *restful.Response) { - podName := strings.Trim(request.PathParameter("pod_name"), " ") +func (u Monitor) monitorPod(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + podName := requestParams.PodName + metricName := requestParams.MetricsName if podName != "" { // single pod single metric - metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") - res := metrics.MonitorPodSingleMetric(request, metricsName) + queryType, params, nullRule := metrics.AssemblePodMetricRequestInfo(requestParams, metricName) + var res *metrics.FormatedMetric + if !nullRule { + res = metrics.GetMetric(queryType, params, metricName) + } response.WriteAsJson(res) + } else { - // multiple pod multiple metric - res := metrics.MonitorAllMetrics(request) - response.WriteAsJson(res) + // multiple + rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelPod) + // sorting + sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelPodName) + // paging + pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + + response.WriteAsJson(pagedMetrics) } } -func (u MonitorResource) monitorContainer(request *restful.Request, response *restful.Response) { - metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") - promql := metrics.MakeContainerPromQL(request) - res := client.SendPrometheusRequest(request, promql) - cleanedJson := metrics.ReformatJson(res, metricsName) - response.WriteAsJson(cleanedJson) +func (u Monitor) monitorContainer(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + res := metrics.MonitorContainer(requestParams) + + response.WriteAsJson(res) } -func (u MonitorResource) monitorWorkload(request *restful.Request, response *restful.Response) { - wlKind := request.PathParameter("workload_kind") - if strings.Trim(wlKind, " ") == "" { +func (u Monitor) monitorWorkload(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + wlKind := requestParams.WorkloadKind + if wlKind == "" { // count all workloads figure - //metricName := "workload_count" - res := metrics.MonitorWorkloadCount(request) + res := metrics.MonitorWorkloadCount(requestParams.NsName) response.WriteAsJson(res) } else { - res := metrics.MonitorAllMetrics(request) + res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkload) response.WriteAsJson(res) } } // merge multiple metric: all-devops, all-roles, all-projects...this api is designed for admin -func (u MonitorResource) monitorWorkspaceUserInfo(request *restful.Request, response *restful.Response) { - res := metrics.MonitorWorkspaceUserInfo(request) +func (u Monitor) monitorAllWorkspacesStatistics(request *restful.Request, response *restful.Response) { + res := metrics.MonitorAllWorkspacesStatistics() response.WriteAsJson(res) } // merge multiple metric: devops, roles, projects... -func (u MonitorResource) monitorWorkspaceResourceLevelMetrics(request *restful.Request, response *restful.Response) { - res := metrics.MonitorWorkspaceResourceLevelMetrics(request) +func (u Monitor) monitorOneWorkspaceStatistics(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + wsName := requestParams.WsName + res := metrics.MonitorOneWorkspaceStatistics(wsName) response.WriteAsJson(res) } -func (u MonitorResource) monitorWorkspacePodLevelMetrics(request *restful.Request, response *restful.Response) { - res := metrics.MonitorAllMetrics(request) - response.WriteAsJson(res) +func (u Monitor) monitorAllWorkspaces(request *restful.Request, response *restful.Response) { + + requestParams := client.ParseMonitoringRequestParams(request) + + rawMetrics := metrics.MonitorAllWorkspaces(requestParams) + // sorting + sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelWorkspace) + // paging + pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + + response.WriteAsJson(pagedMetrics) } -func (u MonitorResource) monitorNamespace(request *restful.Request, response *restful.Response) { - nsName := strings.Trim(request.PathParameter("ns_name"), " ") +func (u Monitor) monitorOneWorkspace(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + + tp := requestParams.Tp + if tp != "" { + // multiple + rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkspace) + // sorting + sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelNamespace) + // paging + pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + + response.WriteAsJson(pagedMetrics) + + } else { + res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkspace) + response.WriteAsJson(res) + } +} + +func (u Monitor) monitorNamespace(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + metricName := requestParams.MetricsName + nsName := requestParams.NsName if nsName != "" { // single - metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") - res := metrics.MonitorNamespaceSingleMetric(request, metricsName) + queryType, params := metrics.AssembleNamespaceMetricRequestInfo(requestParams, metricName) + res := metrics.GetMetric(queryType, params, metricName) response.WriteAsJson(res) } else { // multiple - res := metrics.MonitorAllMetrics(request) - response.WriteAsJson(res) + rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelNamespace) + // sorting + sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelNamespace) + // paging + pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + + response.WriteAsJson(pagedMetrics) } } -func (u MonitorResource) monitorNodeorCluster(request *restful.Request, response *restful.Response) { - metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") - //var res *metrics.FormatedMetric - if metricsName != "" { +func (u Monitor) monitorCluster(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + + metricName := requestParams.MetricsName + if metricName != "" { // single - res := metrics.MonitorNodeorClusterSingleMetric(request, metricsName) + queryType, params := metrics.AssembleClusterMetricRequestInfo(requestParams, metricName) + res := metrics.GetMetric(queryType, params, metricName) + + if metricName == metrics.MetricNameWorkspaceAllProjectCount { + res = metrics.MonitorWorkspaceNamespaceHistory(res) + } + response.WriteAsJson(res) } else { // multiple - res := metrics.MonitorAllMetrics(request) + res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelCluster) response.WriteAsJson(res) } } -type MonitorResource struct { +func (u Monitor) monitorNode(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + + metricName := requestParams.MetricsName + if metricName != "" { + // single + queryType, params := metrics.AssembleNodeMetricRequestInfo(requestParams, metricName) + res := metrics.GetMetric(queryType, params, metricName) + nodeAddress := metrics.GetNodeAddressInfo() + metrics.AddNodeAddressMetric(res, nodeAddress) + response.WriteAsJson(res) + } else { + // multiple + rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelNode) + nodeAddress := metrics.GetNodeAddressInfo() + + for i := 0; i < len(rawMetrics.Results); i++ { + metrics.AddNodeAddressMetric(&rawMetrics.Results[i], nodeAddress) + } + + // sorting + sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelNode) + // paging + pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount) + + response.WriteAsJson(pagedMetrics) + } +} + +// k8s component(controller, scheduler, etcd) status +func (u Monitor) monitorComponentStatus(request *restful.Request, response *restful.Response) { + requestParams := client.ParseMonitoringRequestParams(request) + + status := metrics.MonitorComponentStatus(requestParams) + response.WriteAsJson(status) +} + +func (u Monitor) monitorEvents(request *restful.Request, response *restful.Response) { + // k8s component healthy status + requestParams := client.ParseMonitoringRequestParams(request) + + nsFilter := requestParams.NsFilter + events := metrics.MonitorEvents(nsFilter) + response.WriteAsJson(events) +} + +type Monitor struct { } func Register(ws *restful.WebService, subPath string) { tags := []string{"monitoring apis"} - u := MonitorResource{} + u := Monitor{} - ws.Route(ws.GET(subPath+"/clusters").To(u.monitorNodeorCluster). + ws.Route(ws.GET(subPath+"/clusters").To(u.monitorCluster). Filter(route.RouteLogging). Doc("monitor cluster level metrics"). Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("cluster_cpu_utilisation")). @@ -118,16 +216,20 @@ func Register(ws *restful.WebService, subPath string) { Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) - ws.Route(ws.GET(subPath+"/nodes").To(u.monitorNodeorCluster). + ws.Route(ws.GET(subPath+"/nodes").To(u.monitorNode). Filter(route.RouteLogging). Doc("monitor nodes level metrics"). Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("node_cpu_utilisation")). Param(ws.QueryParameter("nodes_filter", "node re2 expression filter").DataType("string").Required(false).DefaultValue("")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) - ws.Route(ws.GET(subPath+"/nodes/{node_id}").To(u.monitorNodeorCluster). + ws.Route(ws.GET(subPath+"/nodes/{node_id}").To(u.monitorNode). Filter(route.RouteLogging). Doc("monitor specific node level metrics"). Param(ws.PathParameter("node_id", "specific node").DataType("string").Required(true).DefaultValue("")). @@ -141,6 +243,10 @@ func Register(ws *restful.WebService, subPath string) { Doc("monitor namespaces level metrics"). Param(ws.QueryParameter("namespaces_filter", "namespaces re2 expression filter").DataType("string").Required(false).DefaultValue("")). Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("namespace_memory_utilisation")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) @@ -158,8 +264,12 @@ func Register(ws *restful.WebService, subPath string) { Filter(route.RouteLogging). Doc("monitor pods level metrics"). Param(ws.PathParameter("ns_name", "specific namespace").DataType("string").Required(true).DefaultValue("monitoring")). - Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("")). Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("pod_memory_utilisation_wo_cache")). + Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) @@ -178,8 +288,12 @@ func Register(ws *restful.WebService, subPath string) { Filter(route.RouteLogging). Doc("monitor pods level metrics by nodeid"). Param(ws.PathParameter("node_id", "specific node").DataType("string").Required(true).DefaultValue("i-k89a62il")). - Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("openpitrix.*")). Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("pod_memory_utilisation_wo_cache")). + Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("openpitrix.*")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) @@ -236,27 +350,64 @@ func Register(ws *restful.WebService, subPath string) { Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) - ws.Route(ws.GET(subPath+"/workspaces/{workspace_name}/pods").To(u.monitorWorkspacePodLevelMetrics). + // merge multiple metric: devops, roles, projects... + ws.Route(ws.GET(subPath+"/cluster_workspaces/{workspace_name}/_statistics").To(u.monitorOneWorkspaceStatistics). Filter(route.RouteLogging). Doc("monitor specific workspace level metrics"). Param(ws.PathParameter("workspace_name", "workspace name").DataType("string").Required(true)). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + // list all namespace in this workspace by selected metrics + ws.Route(ws.GET(subPath+"/cluster_workspaces/{workspace_name}").To(u.monitorOneWorkspace). + Filter(route.RouteLogging). + Doc("monitor workspaces level metrics"). + Param(ws.PathParameter("workspace_name", "workspace name").DataType("string").Required(true)). Param(ws.QueryParameter("namespaces_filter", "namespaces filter").DataType("string").Required(false).DefaultValue("k.*")). - Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...").DataType("string").Required(false).DefaultValue("tenant_memory_utilisation_wo_cache")). + Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("namespace_memory_utilisation_wo_cache")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). + Param(ws.QueryParameter("type", "page number").DataType("string").Required(false).DefaultValue("1")). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) - ws.Route(ws.GET(subPath+"/workspaces/{workspace_name}").To(u.monitorWorkspaceResourceLevelMetrics). + // metrics from system users or projects, merge multiple metric: all-devops, all-roles, all-projects... + ws.Route(ws.GET(subPath+"/cluster_workspaces/_statistics").To(u.monitorAllWorkspacesStatistics). Filter(route.RouteLogging). Doc("monitor specific workspace level metrics"). - Param(ws.PathParameter("workspace_name", "workspace name").DataType("string").Required(true)). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) - ws.Route(ws.GET(subPath+"/workspaces").To(u.monitorWorkspaceUserInfo). + // metrics from prometheus + ws.Route(ws.GET(subPath+"/cluster_workspaces").To(u.monitorAllWorkspaces). Filter(route.RouteLogging). - Doc("monitor specific workspace level metrics"). + Doc("monitor workspaces level metrics"). + Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("workspace_memory_utilisation")). + Param(ws.QueryParameter("workspaces_filter", "workspaces re2 expression filter").DataType("string").Required(false).DefaultValue(".*")). + Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")). + Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/events").To(u.monitorEvents). + Filter(route.RouteLogging). + Doc("monitor k8s events"). + Param(ws.QueryParameter("namespaces_filter", "namespaces filter").DataType("string").Required(false).DefaultValue(".*")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/components").To(u.monitorComponentStatus). + Filter(route.RouteLogging). + Doc("monitor k8s components status"). Metadata(restfulspec.KeyOpenAPITags, tags)). Consumes(restful.MIME_JSON, restful.MIME_XML). Produces(restful.MIME_JSON) diff --git a/pkg/client/prometheusclient.go b/pkg/client/prometheusclient.go index 95992936f..a7c776a48 100644 --- a/pkg/client/prometheusclient.go +++ b/pkg/client/prometheusclient.go @@ -16,13 +16,12 @@ import ( "io/ioutil" "net/http" "net/url" - "strconv" + "strings" "time" "github.com/emicklei/go-restful" "github.com/golang/glog" - "github.com/pkg/errors" ) const ( @@ -33,13 +32,38 @@ const ( PrometheusEndpointUrl = DefaultScheme + "://" + DefaultPrometheusService + ":" + DefaultPrometheusPort + PrometheusApiPath DefaultQueryStep = "10m" DefaultQueryTimeout = "30s" + RangeQueryType = "query_range?" + DefaultQueryType = "query?" ) +type MonitoringRequestParams struct { + Params url.Values + QueryType string + SortMetricName string + SortType string + PageNum string + LimitNum string + Tp string + MetricsFilter string + NodesFilter string + WsFilter string + NsFilter string + PodsFilter string + ContainersFilter string + MetricsName string + WorkloadName string + NodeId string + WsName string + NsName string + PodName string + ContainerName string + WorkloadKind string +} + var client = &http.Client{} -func SendRequest(postfix string, params string) string { - epurl := PrometheusEndpointUrl + postfix + params - //glog.Info("monitoring epurl:>", epurl) +func SendMonitoringRequest(queryType string, params string) string { + epurl := PrometheusEndpointUrl + queryType + params response, err := client.Get(epurl) if err != nil { glog.Error(err) @@ -56,34 +80,57 @@ func SendRequest(postfix string, params string) string { return "" } -func SendPrometheusRequest(request *restful.Request, recordingRule string) string { - paramsMap, bol, err := ParseRequestHeader(request) - if err != nil { - glog.Error(err) - return "" - } +func ParseMonitoringRequestParams(request *restful.Request) *MonitoringRequestParams { + instantTime := strings.Trim(request.QueryParameter("time"), " ") + start := strings.Trim(request.QueryParameter("start"), " ") + end := strings.Trim(request.QueryParameter("end"), " ") + step := strings.Trim(request.QueryParameter("step"), " ") + timeout := strings.Trim(request.QueryParameter("timeout"), " ") - var res = "" - var postfix = "" - if bol { - // range query - postfix = "query_range?" - } else { - // query - postfix = "query?" - } - paramsMap.Set("query", recordingRule) - params := paramsMap.Encode() - res = SendRequest(postfix, params) - return res -} + sortMetricName := strings.Trim(request.QueryParameter("sort_metric"), " ") + sortType := strings.Trim(request.QueryParameter("sort_type"), " ") + pageNum := strings.Trim(request.QueryParameter("page"), " ") + limitNum := strings.Trim(request.QueryParameter("limit"), " ") + tp := strings.Trim(request.QueryParameter("type"), " ") -func ParseRequestHeader(request *restful.Request) (url.Values, bool, error) { - instantTime := request.QueryParameter("time") - start := request.QueryParameter("start") - end := request.QueryParameter("end") - step := request.QueryParameter("step") - timeout := request.QueryParameter("timeout") + metricsFilter := strings.Trim(request.QueryParameter("metrics_filter"), " ") + nodesFilter := strings.Trim(request.QueryParameter("nodes_filter"), " ") + wsFilter := strings.Trim(request.QueryParameter("workspaces_filter"), " ") + nsFilter := strings.Trim(request.QueryParameter("namespaces_filter"), " ") + podsFilter := strings.Trim(request.QueryParameter("pods_filter"), " ") + containersFilter := strings.Trim(request.QueryParameter("containers_filter"), " ") + + metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") + workloadName := strings.Trim(request.QueryParameter("workload_name"), " ") + + nodeId := strings.Trim(request.PathParameter("node_id"), " ") + wsName := strings.Trim(request.PathParameter("workspace_name"), " ") + nsName := strings.Trim(request.PathParameter("ns_name"), " ") + podName := strings.Trim(request.PathParameter("pod_name"), " ") + containerName := strings.Trim(request.PathParameter("container_name"), " ") + workloadKind := strings.Trim(request.PathParameter("workload_kind"), " ") + + var requestParams = MonitoringRequestParams{ + SortMetricName: sortMetricName, + SortType: sortType, + PageNum: pageNum, + LimitNum: limitNum, + Tp: tp, + MetricsFilter: metricsFilter, + NodesFilter: nodesFilter, + WsFilter: wsFilter, + NsFilter: nsFilter, + PodsFilter: podsFilter, + ContainersFilter: containersFilter, + MetricsName: metricsName, + WorkloadName: workloadName, + NodeId: nodeId, + WsName: wsName, + NsName: nsName, + PodName: podName, + ContainerName: containerName, + WorkloadKind: workloadKind, + } if timeout == "" { timeout = DefaultQueryTimeout @@ -93,25 +140,35 @@ func ParseRequestHeader(request *restful.Request) (url.Values, bool, error) { } // Whether query or query_range request u := url.Values{} + if start != "" && end != "" { u.Set("start", convertTimeGranularity(start)) u.Set("end", convertTimeGranularity(end)) u.Set("step", step) u.Set("timeout", timeout) - return u, true, nil + requestParams.QueryType = RangeQueryType + requestParams.Params = u + return &requestParams } if instantTime != "" { u.Set("time", instantTime) u.Set("timeout", timeout) - return u, false, nil + requestParams.QueryType = DefaultQueryType + requestParams.Params = u + return &requestParams } else { //u.Set("time", strconv.FormatInt(int64(time.Now().Unix()), 10)) u.Set("timeout", timeout) - return u, false, nil + requestParams.QueryType = DefaultQueryType + requestParams.Params = u + return &requestParams } glog.Errorln("Parse request %s failed", u) - return u, false, errors.Errorf("Parse request time range %s failed", u) + requestParams.QueryType = DefaultQueryType + requestParams.Params = u + + return &requestParams } func convertTimeGranularity(ts string) string { diff --git a/pkg/models/metrics/metrics.go b/pkg/models/metrics/metrics.go new file mode 100644 index 000000000..0127dd5ab --- /dev/null +++ b/pkg/models/metrics/metrics.go @@ -0,0 +1,993 @@ +/* +Copyright 2018 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "encoding/json" + "fmt" + "net/url" + "regexp" + "strings" + "sync" + "time" + + "github.com/golang/glog" + "github.com/pkg/errors" + "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "kubesphere.io/kubesphere/pkg/client" + "kubesphere.io/kubesphere/pkg/models" + "kubesphere.io/kubesphere/pkg/models/workspaces" +) + +var nodeStatusDelLables = []string{"endpoint", "instance", "job", "namespace", "pod", "service"} + +const ( + ChannelMaxCapacityWorkspaceMetric = 400 + ChannelMaxCapacity = 100 +) + +type PagedFormatedLevelMetric struct { + CurrentPage int `json:"page"` + TotalPage int `json:"total_page"` + Message string `json:"msg"` + Metric FormatedLevelMetric `json:"metrics"` +} + +type FormatedLevelMetric struct { + MetricsLevel string `json:"metrics_level"` + Results []FormatedMetric `json:"results"` +} + +type FormatedMetric struct { + MetricName string `json:"metric_name, omitempty"` + Status string `json:"status"` + Data FormatedMetricData `json:"data, omitempty"` +} + +type FormatedMetricData struct { + Result []map[string]interface{} `json:"result"` + ResultType string `json:"resultType"` +} + +type MetricResultValues []MetricResultValue + +type MetricResultValue struct { + timestamp float64 + value string +} + +type MetricItem struct { + MetricLabel map[string]string `json:"metric"` + Value []interface{} `json:"value"` +} + +type CommonMetricsResult struct { + Status string `json:"status"` + Data CommonMetricsData `json:"data"` +} + +type CommonMetricsData struct { + Result []CommonResultItem `json:"result"` + ResultType string `json:"resultType"` +} + +type CommonResultItem struct { + KubePodMetric KubePodMetric `json:"metric"` + Value interface{} `json:"value"` +} + +type KubePodMetric struct { + CreatedByKind string `json:"created_by_kind"` + CreatedByName string `json:"created_by_name"` + Namespace string `json:"namespace"` + Pod string `json:"pod"` +} + +type ComponentStatus struct { + Name string `json:"metric_name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ComponentStatus []OneComponentStatus `json:"component"` +} + +type OneComponentStatus struct { + // Valid value: "Healthy" + Type string `json:"type"` + // Valid values for "Healthy": "True", "False", or "Unknown". + Status string `json:"status"` + // Message about the condition for a component. + Message string `json:"message,omitempty"` + // Condition error code for a component. + Error string `json:"error,omitempty"` +} + +func getPodNameRegexInWorkload(res string) string { + + data := []byte(res) + var dat CommonMetricsResult + jsonErr := json.Unmarshal(data, &dat) + if jsonErr != nil { + glog.Errorln("json parse failed", jsonErr) + } + var podNames []string + for _, item := range dat.Data.Result { + podName := item.KubePodMetric.Pod + podNames = append(podNames, podName) + } + podNamesFilter := "^(" + strings.Join(podNames, "|") + ")$" + return podNamesFilter +} + +func AssembleWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { + rule := MakeWorkloadRule(monitoringRequest.WorkloadKind, monitoringRequest.WorkloadName, monitoringRequest.NsName) + paramValues := monitoringRequest.Params + params := makeRequestParamString(rule, paramValues) + + res := client.SendMonitoringRequest(client.DefaultQueryType, params) + + podNamesFilter := getPodNameRegexInWorkload(res) + + queryType := monitoringRequest.QueryType + rule = MakePodPromQL(metricName, monitoringRequest.NsName, "", "", podNamesFilter) + params = makeRequestParamString(rule, paramValues) + + return queryType, params +} + +func AssemblePodMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) { + queryType := monitoringRequest.QueryType + + paramValues := monitoringRequest.Params + + rule := MakePodPromQL(metricName, monitoringRequest.NsName, monitoringRequest.NodeId, monitoringRequest.PodName, monitoringRequest.PodsFilter) + params := makeRequestParamString(rule, paramValues) + return queryType, params, rule == "" +} + +func GetMetric(queryType, params, metricName string) *FormatedMetric { + res := client.SendMonitoringRequest(queryType, params) + formatedMetric := ReformatJson(res, metricName) + return formatedMetric +} + +func GetNodeAddressInfo() *map[string][]v1.NodeAddress { + nodeList, _ := client.NewK8sClient().CoreV1().Nodes().List(metaV1.ListOptions{}) + var nodeAddress = make(map[string][]v1.NodeAddress) + for _, node := range nodeList.Items { + nodeAddress[node.Name] = node.Status.Addresses + } + return &nodeAddress +} + +func AddNodeAddressMetric(nodeMetric *FormatedMetric, nodeAddress *map[string][]v1.NodeAddress) { + + for i := 0; i < len(nodeMetric.Data.Result); i++ { + metricDesc := nodeMetric.Data.Result[i][ResultItemMetric] + metricDescMap := metricDesc.(map[string]interface{}) + if nodeId, exist := metricDescMap["node"]; exist { + addr, exist := (*nodeAddress)[nodeId.(string)] + if exist { + metricDescMap["address"] = addr + } + } + } +} + +func MonitorContainer(monitoringRequest *client.MonitoringRequestParams) *FormatedMetric { + queryType := monitoringRequest.QueryType + + paramValues := monitoringRequest.Params + rule := MakeContainerPromQL(monitoringRequest.NsName, monitoringRequest.PodName, monitoringRequest.ContainerName, monitoringRequest.MetricsName, monitoringRequest.ContainersFilter) + params := makeRequestParamString(rule, paramValues) + + res := GetMetric(queryType, params, monitoringRequest.MetricsName) + return res +} + +func AssembleNamespaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { + queryType := monitoringRequest.QueryType + + paramValues := monitoringRequest.Params + rule := MakeNamespacePromQL(monitoringRequest.NsName, monitoringRequest.NsFilter, metricName) + params := makeRequestParamString(rule, paramValues) + + return queryType, params +} + +func AssembleWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) { + nsFilter := "^(" + strings.Join(namespaceList, "|") + ")$" + + queryType := monitoringRequest.QueryType + + rule := MakeWorkspacePromQL(metricName, nsFilter) + paramValues := monitoringRequest.Params + params := makeRequestParamString(rule, paramValues) + return queryType, params +} + +func makeRequestParamString(rule string, paramValues url.Values) string { + + var values = make(url.Values) + for key, v := range paramValues { + values.Set(key, v[0]) + } + + values.Set("query", rule) + + params := values.Encode() + + return params +} + +func filterNamespace(nsFilter string, namespaceList []string) []string { + var newNSlist []string + if nsFilter == "" { + nsFilter = ".*" + } + for _, ns := range namespaceList { + bol, _ := regexp.MatchString(nsFilter, ns) + if bol { + newNSlist = append(newNSlist, ns) + } + } + return newNSlist +} + +func MonitorAllWorkspaces(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric { + metricsFilter := monitoringRequest.MetricsFilter + if strings.Trim(metricsFilter, " ") == "" { + metricsFilter = ".*" + } + var filterMetricsName []string + for _, metricName := range WorkspaceMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + filterMetricsName = append(filterMetricsName, metricName) + } + } + + var wgAll sync.WaitGroup + var wsAllch = make(chan *[]FormatedMetric, ChannelMaxCapacityWorkspaceMetric) + + workspaceNamespaceMap, _, _ := workspaces.GetAllOrgAndProjList() + for ws, _ := range workspaceNamespaceMap { + bol, err := regexp.MatchString(monitoringRequest.WsFilter, ws) + if err == nil && bol { + // a workspace + wgAll.Add(1) + go collectWorkspaceMetric(monitoringRequest, ws, filterMetricsName, &wgAll, wsAllch) + } + } + + wgAll.Wait() + close(wsAllch) + + fmtMetricMap := make(map[string]FormatedMetric) + for oneWsMetric := range wsAllch { + if oneWsMetric != nil { + // aggregate workspace metric + for _, metric := range *oneWsMetric { + fm, exist := fmtMetricMap[metric.MetricName] + if exist { + if metric.Status == "error" { + fm.Status = metric.Status + } + fm.Data.Result = append(fm.Data.Result, metric.Data.Result...) + fmtMetricMap[metric.MetricName] = fm + } else { + fmtMetricMap[metric.MetricName] = metric + } + } + } + } + + var metricArray = make([]FormatedMetric, 0) + for _, metric := range fmtMetricMap { + metricArray = append(metricArray, metric) + } + + return &FormatedLevelMetric{ + MetricsLevel: MetricLevelClusterWorkspace, + Results: metricArray, + } +} + +func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, ws string, filterMetricsName []string, wgAll *sync.WaitGroup, wsAllch chan *[]FormatedMetric) { + defer wgAll.Done() + var wg sync.WaitGroup + var ch = make(chan *FormatedMetric, ChannelMaxCapacity) + namespaceArray, err := workspaces.WorkspaceNamespaces(ws) + if err != nil { + glog.Errorln(err.Error()) + } + // add by namespace + for _, metricName := range filterMetricsName { + wg.Add(1) + go func(metricName string) { + defer func() { + if err := recover(); err != nil { + glog.Errorln(err) + } + }() + + queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName) + ch <- GetMetric(queryType, params, metricName) + + wg.Done() + }(metricName) + } + + wg.Wait() + close(ch) + + var metricsArray []FormatedMetric + for oneMetric := range ch { + if oneMetric != nil { + // add "workspace" filed to oneMetric `metric` field + for i := 0; i < len(oneMetric.Data.Result); i++ { + tmap := oneMetric.Data.Result[i]["metric"].(map[string]interface{}) + tmap[MetricLevelWorkspace] = ws + oneMetric.Data.Result[i]["metric"] = tmap + } + metricsArray = append(metricsArray, *oneMetric) + } + } + + wsAllch <- &metricsArray +} + +func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resourceType string) *FormatedLevelMetric { + metricsFilter := monitoringRequest.MetricsFilter + if metricsFilter == "" { + metricsFilter = ".*" + } + + var ch = make(chan *FormatedMetric, ChannelMaxCapacity) + var wg sync.WaitGroup + + switch resourceType { + case MetricLevelCluster: + { + for _, metricName := range ClusterMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params := AssembleClusterMetricRequestInfo(monitoringRequest, metricName) + clusterMetrics := GetMetric(queryType, params, metricName) + + // for this special case, get namespace history which in a workspace by determining namespace label + if metricName == MetricNameWorkspaceAllProjectCount { + clusterMetrics = MonitorWorkspaceNamespaceHistory(clusterMetrics) + } + + ch <- clusterMetrics + + wg.Done() + }(metricName) + } + } + } + case MetricLevelNode: + { + for _, metricName := range NodeMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params := AssembleNodeMetricRequestInfo(monitoringRequest, metricName) + ch <- GetMetric(queryType, params, metricName) + wg.Done() + }(metricName) + } + } + } + case MetricLevelWorkspace: + { + namespaceArray, err := workspaces.WorkspaceNamespaces(monitoringRequest.WsName) + if err != nil { + glog.Errorln(err.Error()) + } + namespaceArray = filterNamespace(monitoringRequest.NsFilter, namespaceArray) + + if monitoringRequest.Tp == "" { + for _, metricName := range WorkspaceMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName) + ch <- GetMetric(queryType, params, metricName) + wg.Done() + }(metricName) + } + } + } else { + for _, metricName := range NamespaceMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + ns := "^(" + strings.Join(namespaceArray, "|") + ")$" + monitoringRequest.NsFilter = ns + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params := AssembleNamespaceMetricRequestInfo(monitoringRequest, metricName) + ch <- GetMetric(queryType, params, metricName) + wg.Done() + }(metricName) + } + } + } + } + case MetricLevelNamespace: + { + for _, metricName := range NamespaceMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params := AssembleNamespaceMetricRequestInfo(monitoringRequest, metricName) + ch <- GetMetric(queryType, params, metricName) + wg.Done() + }(metricName) + } + } + } + case MetricLevelWorkload: + { + for _, metricName := range WorkloadMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + metricName = strings.TrimLeft(metricName, "workload_") + queryType, params := AssembleWorkloadMetricRequestInfo(monitoringRequest, metricName) + ch <- GetMetric(queryType, params, metricName) + wg.Done() + }(metricName) + } + } + } + case MetricLevelPod: + { + for _, metricName := range PodMetricsNames { + bol, err := regexp.MatchString(metricsFilter, metricName) + if err == nil && bol { + wg.Add(1) + go func(metricName string) { + queryType, params, nullRule := AssemblePodMetricRequestInfo(monitoringRequest, metricName) + if !nullRule { + ch <- GetMetric(queryType, params, metricName) + } else { + ch <- nil + } + wg.Done() + }(metricName) + } + } + } + } + + wg.Wait() + close(ch) + + var metricsArray []FormatedMetric + + for oneMetric := range ch { + if oneMetric != nil { + metricsArray = append(metricsArray, *oneMetric) + } + } + + return &FormatedLevelMetric{ + MetricsLevel: resourceType, + Results: metricsArray, + } +} + +func MonitorWorkspaceNamespaceHistory(metric *FormatedMetric) *FormatedMetric { + resultType := metric.Data.ResultType + //metric.Status + metricName := metric.MetricName + + for i := 0; i < len(metric.Data.Result); i++ { + metricItem := metric.Data.Result[i] + + if resultType == ResultTypeVector { + timeAndValue, sure := metricItem[ResultItemValue].([]interface{}) + if !sure { + return metric + } + metric := getNamespaceHistoryMetric(timeAndValue[0].(float64), metricName) + + workspaceNamespaceCount := calcWorkspaceNamespace(metric) + + timeAndValue[1] = fmt.Sprintf("%d", workspaceNamespaceCount) + + } else if resultType == ResultTypeMatrix { + + values, sure := metricItem[ResultItemValues].([]interface{}) + if !sure { + return metric + } + + for _, valueItem := range values { + timeAndValue, sure := valueItem.([]interface{}) + if !sure { + return metric + } + + metric := getNamespaceHistoryMetric(timeAndValue[0].(float64), metricName) + + workspaceNamespaceCount := calcWorkspaceNamespace(metric) + + timeAndValue[1] = fmt.Sprintf("%d", workspaceNamespaceCount) + } + } + } + + return metric +} + +func getNamespaceHistoryMetric(timestamp float64, metricName string) *FormatedMetric { + var timeRelatedParams = make(url.Values) + timeRelatedParams.Set("time", fmt.Sprintf("%f", timestamp)) + timeRelatedParams.Set("query", NamespaceLabelRule) + params := timeRelatedParams.Encode() + metric := GetMetric(client.DefaultQueryType, params, metricName) + return metric +} + +// calculate all namespace which belong to workspaces +func calcWorkspaceNamespace(metric *FormatedMetric) int { + if metric.Status == "error" { + glog.Errorf("failed when retrive namespace history, the metric is %v", metric.Data.Result) + return 0 + } + + var workspaceNamespaceCount = 0 + + for _, result := range metric.Data.Result { + tmpMap := result[ResultItemMetric].(map[string]interface{}) + wsName, exist := tmpMap[WorkspaceJoinedKey] + + if exist && wsName != "" { + workspaceNamespaceCount += 1 + } + } + + return workspaceNamespaceCount +} + +func MonitorAllWorkspacesStatistics() *FormatedLevelMetric { + + wg := sync.WaitGroup{} + var metricsArray []FormatedMetric + timestamp := time.Now().Unix() + + var orgResultItem *FormatedMetric + var devopsResultItem *FormatedMetric + var clusterProjResultItem *FormatedMetric + var workspaceProjResultItem *FormatedMetric + var accountResultItem *FormatedMetric + + wsMap, projs, errProj := workspaces.GetAllOrgAndProjList() + + wg.Add(5) + + go func() { + orgNums, errOrg := workspaces.GetAllOrgNums() + if errOrg != nil { + glog.Errorln(errOrg.Error()) + } + orgResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllOrganizationCount, WorkspaceResourceKindOrganization, orgNums, errOrg) + wg.Done() + }() + + go func() { + devOpsProjectNums, errDevops := workspaces.GetAllDevOpsProjectsNums() + if errDevops != nil { + glog.Errorln(errDevops.Error()) + } + devopsResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllDevopsCount, WorkspaceResourceKindDevops, devOpsProjectNums, errDevops) + wg.Done() + }() + + go func() { + var projNums = 0 + for _, v := range wsMap { + projNums += len(v) + } + if errProj != nil { + glog.Errorln(errProj.Error()) + } + workspaceProjResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllProjectCount, WorkspaceResourceKindNamespace, projNums, errProj) + wg.Done() + }() + + go func() { + projNums := len(projs) + if errProj != nil { + glog.Errorln(errProj.Error()) + } + clusterProjResultItem = getSpecificMetricItem(timestamp, MetricNameClusterAllProjectCount, WorkspaceResourceKindNamespace, projNums, errProj) + wg.Done() + }() + + go func() { + actNums, errAct := workspaces.GetAllAccountNums() + if errAct != nil { + glog.Errorln(errAct.Error()) + } + accountResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllAccountCount, WorkspaceResourceKindAccount, actNums, errAct) + wg.Done() + }() + + wg.Wait() + + metricsArray = append(metricsArray, *orgResultItem, *devopsResultItem, *workspaceProjResultItem, *accountResultItem, *clusterProjResultItem) + + return &FormatedLevelMetric{ + MetricsLevel: MetricLevelWorkspace, + Results: metricsArray, + } +} + +func MonitorOneWorkspaceStatistics(wsName string) *FormatedLevelMetric { + + var nsMetrics *FormatedMetric + var devopsMetrics *FormatedMetric + var memberMetrics *FormatedMetric + var roleMetrics *FormatedMetric + + wg := sync.WaitGroup{} + wg.Add(4) + + var fMetricsArray []FormatedMetric + timestamp := int64(time.Now().Unix()) + + go func() { + // add namespaces(project) metric + namespaces, errNs := workspaces.WorkspaceNamespaces(wsName) + namespaces, noneExistentNs := getExistingNamespace(namespaces) + if len(noneExistentNs) != 0 { + nsStr := strings.Join(noneExistentNs, "|") + errStr := "the namespaces " + nsStr + " do not exist" + if errNs == nil { + errNs = errors.New(errStr) + } else { + errNs = errors.New(errNs.Error() + "\t" + errStr) + } + } + if errNs != nil { + glog.Errorln(errNs.Error()) + } + nsMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceNamespaceCount, WorkspaceResourceKindNamespace, len(namespaces), errNs) + wg.Done() + }() + + go func() { + devOpsProjects, errDevOps := workspaces.GetDevOpsProjects(wsName) + if errDevOps != nil { + glog.Errorln(errDevOps.Error()) + } + // add devops metric + devopsMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceDevopsCount, WorkspaceResourceKindDevops, len(devOpsProjects), errDevOps) + wg.Done() + }() + + go func() { + members, errMemb := workspaces.GetOrgMembers(wsName) + if errMemb != nil { + glog.Errorln(errMemb.Error()) + } + // add member metric + memberMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceMemberCount, WorkspaceResourceKindMember, len(members), errMemb) + wg.Done() + }() + + go func() { + roles, errRole := workspaces.GetOrgRoles(wsName) + if errRole != nil { + glog.Errorln(errRole.Error()) + } + // add role metric + roleMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceRoleCount, WorkspaceResourceKindRole, len(roles), errRole) + wg.Done() + }() + + wg.Wait() + + fMetricsArray = append(fMetricsArray, *nsMetrics, *devopsMetrics, *memberMetrics, *roleMetrics) + + return &FormatedLevelMetric{ + MetricsLevel: MetricLevelWorkspace, + Results: fMetricsArray, + } +} + +func getSpecificMetricItem(timestamp int64, metricName string, resource string, count int, err error, resourceType ...string) *FormatedMetric { + var nsMetrics FormatedMetric + nsMetrics.MetricName = metricName + nsMetrics.Data.ResultType = ResultTypeVector + resultItem := make(map[string]interface{}) + tmp := make(map[string]string) + + if len(resourceType) > 0 { + tmp[resourceType[0]] = resource + } else { + tmp[ResultItemMetricResource] = resource + } + + if err == nil { + nsMetrics.Status = MetricStatusSuccess + } else { + nsMetrics.Status = MetricStatusError + resultItem["errormsg"] = err.Error() + } + + resultItem[ResultItemMetric] = tmp + resultItem[ResultItemValue] = []interface{}{timestamp, count} + nsMetrics.Data.Result = make([]map[string]interface{}, 1) + nsMetrics.Data.Result[0] = resultItem + return &nsMetrics +} + +// k8s component(controller, scheduler, etcd) status +func MonitorComponentStatus(monitoringRequest *client.MonitoringRequestParams) *[]interface{} { + componentList, err := client.NewK8sClient().CoreV1().ComponentStatuses().List(metaV1.ListOptions{}) + if err != nil { + glog.Errorln(err.Error()) + } + + var componentStatusList []*ComponentStatus + for _, item := range componentList.Items { + var status []OneComponentStatus + for _, cond := range item.Conditions { + status = append(status, OneComponentStatus{ + Type: string(cond.Type), + Status: string(cond.Status), + Message: cond.Message, + Error: cond.Error, + }) + } + + componentStatusList = append(componentStatusList, &ComponentStatus{ + Name: item.Name, + Namespace: item.Namespace, + Labels: item.Labels, + ComponentStatus: status, + }) + } + + // node status + queryType := monitoringRequest.QueryType + paramValues := monitoringRequest.Params + paramValues.Set("query", NodeStatusRule) + params := paramValues.Encode() + res := client.SendMonitoringRequest(queryType, params) + + nodeStatusMetric := ReformatJson(res, "node_status", nodeStatusDelLables...) + nodeStatusMetric = ReformatNodeStatusField(nodeStatusMetric) + + var normalNodes []string + var abnormalNodes []string + for _, result := range nodeStatusMetric.Data.Result { + tmap := result[ResultItemMetric].(map[string]interface{}) + if tmap[MetricStatus].(string) == "false" { + abnormalNodes = append(abnormalNodes, tmap[MetricLevelNode].(string)) + } else { + normalNodes = append(normalNodes, tmap[MetricLevelNode].(string)) + } + } + + Components, err := models.GetAllComponentsStatus() + + if err != nil { + glog.Error(err.Error()) + } + + var namspaceComponentHealthyMap = make(map[string]int) + var namspaceComponentTotalMap = make(map[string]int) + + for _, ns := range models.SYSTEM_NAMESPACES { + nsStatus, exist := Components[ns] + if exist { + for _, nsStatusItem := range nsStatus.(map[string]interface{}) { + component := nsStatusItem.(models.Component) + namspaceComponentTotalMap[ns] += 1 + if component.HealthyBackends == component.TotalBackends { + namspaceComponentHealthyMap[ns] += 1 + } + } + } + } + + timestamp := int64(time.Now().Unix()) + + onlineMetricItems := makeMetricItems(timestamp, namspaceComponentHealthyMap, MetricLevelNamespace) + metricItems := makeMetricItems(timestamp, namspaceComponentTotalMap, MetricLevelNamespace) + + var assembleList []interface{} + assembleList = append(assembleList, nodeStatusMetric) + + for _, statusItem := range componentStatusList { + assembleList = append(assembleList, statusItem) + } + + assembleList = append(assembleList, FormatedMetric{ + Data: FormatedMetricData{ + Result: *onlineMetricItems, + ResultType: ResultTypeVector, + }, + MetricName: MetricNameComponentOnLine, + Status: MetricStatusSuccess, + }) + + assembleList = append(assembleList, FormatedMetric{ + Data: FormatedMetricData{ + Result: *metricItems, + ResultType: ResultTypeVector, + }, + MetricName: MetricNameComponentLine, + Status: MetricStatusSuccess, + }) + + return &assembleList +} + +func makeMetricItems(timestamp int64, statusMap map[string]int, resourceType string) *[]map[string]interface{} { + var metricItems []map[string]interface{} + + for ns, count := range statusMap { + metricItems = append(metricItems, map[string]interface{}{ + ResultItemMetric: map[string]string{resourceType: ns}, + ResultItemValue: []interface{}{timestamp, fmt.Sprintf("%d", count)}, + }) + } + return &metricItems +} + +// monitor k8s event, there are two status: Normal, Warning +func MonitorEvents(nsFilter string) *[]v1.Event { + namespaceMap, err := getAllNamespace() + if err != nil { + glog.Errorln(err.Error()) + } + var nsList = make([]string, 0) + for ns, _ := range namespaceMap { + nsList = append(nsList, ns) + } + + filterNS := filterNamespace(nsFilter, nsList) + var eventsList = make([]v1.Event, 0) + + for _, ns := range filterNS { + events, err := client.NewK8sClient().CoreV1().Events(ns).List(metaV1.ListOptions{}) + if err != nil { + glog.Errorln(err.Error()) + } else { + for _, item := range events.Items { + eventsList = append(eventsList, item) + } + } + } + return &eventsList +} + +func AssembleClusterMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { + queryType := monitoringRequest.QueryType + paramValues := monitoringRequest.Params + rule := MakeClusterRule(metricName) + params := makeRequestParamString(rule, paramValues) + + return queryType, params +} + +func AssembleNodeMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { + queryType := monitoringRequest.QueryType + paramValues := monitoringRequest.Params + rule := MakeNodeRule(monitoringRequest.NodeId, monitoringRequest.NodesFilter, metricName) + params := makeRequestParamString(rule, paramValues) + + return queryType, params +} + +func getExistingNamespace(namespaces []string) ([]string, []string) { + namespaceMap, err := getAllNamespace() + var existedNS []string + var noneExistedNS []string + if err != nil { + return namespaces, nil + } + for _, ns := range namespaces { + if _, exist := namespaceMap[ns]; exist { + existedNS = append(existedNS, ns) + } else { + noneExistedNS = append(noneExistedNS, ns) + } + } + return existedNS, noneExistedNS +} + +func getAllNamespace() (map[string]int, error) { + k8sClient := client.NewK8sClient() + nsList, err := k8sClient.CoreV1().Namespaces().List(metaV1.ListOptions{}) + if err != nil { + glog.Errorln(err.Error()) + return nil, err + } + namespaceMap := make(map[string]int) + for _, item := range nsList.Items { + namespaceMap[item.Name] = 0 + } + return namespaceMap, nil +} + +func MonitorWorkloadCount(namespace string) *FormatedMetric { + quotaMetric, err := models.GetNamespaceQuota(namespace) + fMetric := convertQuota2MetricStruct(quotaMetric) + + // whether the namespace in request parameters exists? + namespaceMap, err := getAllNamespace() + + _, exist := namespaceMap[namespace] + if err != nil { + exist = true + } + + if !exist || err != nil { + fMetric.Status = MetricStatusError + fMetric.Data.ResultType = "" + errInfo := make(map[string]interface{}) + if err != nil { + errInfo["errormsg"] = err.Error() + } else { + errInfo["errormsg"] = "namespace " + namespace + " does not exist" + } + fMetric.Data.Result = []map[string]interface{}{errInfo} + } + + return fMetric +} + +func convertQuota2MetricStruct(quotaMetric *models.ResourceQuota) *FormatedMetric { + var fMetric FormatedMetric + fMetric.MetricName = MetricNameWorkloadCount + fMetric.Status = MetricStatusSuccess + fMetric.Data.ResultType = ResultTypeVector + timestamp := int64(time.Now().Unix()) + var resultItems []map[string]interface{} + + hardMap := make(map[string]string) + for resourceName, v := range quotaMetric.Data.Hard { + hardMap[resourceName.String()] = v.String() + } + + for resourceName, v := range quotaMetric.Data.Used { + resultItem := make(map[string]interface{}) + tmp := make(map[string]string) + tmp[ResultItemMetricResource] = resourceName.String() + resultItem[ResultItemMetric] = tmp + resultItem[ResultItemValue] = []interface{}{timestamp, hardMap[resourceName.String()], v.String()} + resultItems = append(resultItems, resultItem) + } + + fMetric.Data.Result = resultItems + return &fMetric +} diff --git a/pkg/models/metrics/metricscollector.go b/pkg/models/metrics/metricscollector.go deleted file mode 100755 index d534135dc..000000000 --- a/pkg/models/metrics/metricscollector.go +++ /dev/null @@ -1,539 +0,0 @@ -/* -Copyright 2018 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -import ( - "encoding/json" - "regexp" - "strings" - - "github.com/emicklei/go-restful" - "github.com/golang/glog" - - "time" - - "github.com/pkg/errors" - "k8s.io/api/core/v1" - metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "sync" - - "k8s.io/apimachinery/pkg/labels" - v12 "k8s.io/client-go/listers/core/v1" - - "kubesphere.io/kubesphere/pkg/client" - "kubesphere.io/kubesphere/pkg/models" - "kubesphere.io/kubesphere/pkg/models/controllers" - "kubesphere.io/kubesphere/pkg/models/workspaces" -) - -func getPodNameRegexInWorkload(request *restful.Request) string { - promql := MakeWorkloadRule(request) - res := client.SendPrometheusRequest(request, promql) - data := []byte(res) - var dat CommonMetricsResult - jsonErr := json.Unmarshal(data, &dat) - if jsonErr != nil { - glog.Errorln("json parse failed", jsonErr) - } - var podNames []string - for _, x := range dat.Data.Result { - podName := x.KubePodMetric.Pod - podNames = append(podNames, podName) - } - podNamesFilter := "^(" + strings.Join(podNames, "|") + ")$" - return podNamesFilter -} - -func MonitorWorkloadSingleMetric(request *restful.Request, metricsName string) *FormatedMetric { - nsName := strings.Trim(request.PathParameter("ns_name"), " ") - podNamesFilter := getPodNameRegexInWorkload(request) - newPromql := MakePodPromQL(request, []string{metricsName, nsName, "", "", podNamesFilter}) - podMetrics := client.SendPrometheusRequest(request, newPromql) - cleanedJson := ReformatJson(podMetrics, metricsName) - return cleanedJson -} - -func MonitorPodSingleMetric(request *restful.Request, metricsName string) *FormatedMetric { - nsName := strings.Trim(request.PathParameter("ns_name"), " ") - nodeID := strings.Trim(request.PathParameter("node_id"), " ") - podName := strings.Trim(request.PathParameter("pod_name"), " ") - podFilter := strings.Trim(request.QueryParameter("pods_filter"), " ") - params := []string{metricsName, nsName, nodeID, podName, podFilter} - promql := MakePodPromQL(request, params) - if promql != "" { - res := client.SendPrometheusRequest(request, promql) - cleanedJson := ReformatJson(res, metricsName) - return cleanedJson - } - return nil -} - -func MonitorNamespaceSingleMetric(request *restful.Request, metricsName string) *FormatedMetric { - recordingRule := MakeNamespacePromQL(request, metricsName) - res := client.SendPrometheusRequest(request, recordingRule) - cleanedJson := ReformatJson(res, metricsName) - return cleanedJson -} - -// maybe this function is time consuming -func ReformatJson(metric string, metricsName string) *FormatedMetric { - var formatMetric FormatedMetric - err := json.Unmarshal([]byte(metric), &formatMetric) - if err != nil { - glog.Errorln("Unmarshal metric json failed", err) - } - if formatMetric.MetricName == "" { - formatMetric.MetricName = metricsName - } - // retrive metrics success - if formatMetric.Status == MetricStatusSuccess { - result := formatMetric.Data.Result - for _, res := range result { - metric, ok := res[ResultItemMetric] - me := metric.(map[string]interface{}) - if ok { - delete(me, "__name__") - } - } - } - return &formatMetric -} - -func collectNodeorClusterMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) { - metric := MonitorNodeorClusterSingleMetric(request, metricsName) - ch <- metric -} - -func collectNamespaceMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) { - metric := MonitorNamespaceSingleMetric(request, metricsName) - ch <- metric -} - -func collectWorkloadMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) { - metricsName = strings.TrimLeft(metricsName, "workload_") - metric := MonitorWorkloadSingleMetric(request, metricsName) - ch <- metric -} - -func collectWorkspaceMetrics(request *restful.Request, metricsName string, namespaceList []string, ch chan<- *FormatedMetric) { - mertic := monitorWorkspaceSingleMertic(request, metricsName, namespaceList) - ch <- mertic -} - -func collectPodMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) { - metric := MonitorPodSingleMetric(request, metricsName) - ch <- metric -} - -func monitorWorkspaceSingleMertic(request *restful.Request, metricsName string, namespaceList []string) *FormatedMetric { - namespaceRe2 := "^(" + strings.Join(namespaceList, "|") + ")$" - newpromql := MakeWorkspacePromQL(metricsName, namespaceRe2) - podMetrics := client.SendPrometheusRequest(request, newpromql) - cleanedJson := ReformatJson(podMetrics, metricsName) - return cleanedJson -} - -func filterNamespace(request *restful.Request, namespaceList []string) []string { - var newNSlist []string - nsFilter := strings.Trim(request.QueryParameter("namespaces_filter"), " ") - if nsFilter == "" { - nsFilter = ".*" - } - for _, ns := range namespaceList { - bol, _ := regexp.MatchString(nsFilter, ns) - if bol { - newNSlist = append(newNSlist, ns) - } - } - return newNSlist -} - -func MonitorAllMetrics(request *restful.Request) FormatedLevelMetric { - metricsName := strings.Trim(request.QueryParameter("metrics_filter"), " ") - if metricsName == "" { - metricsName = ".*" - } - path := request.SelectedRoutePath() - sourceType := path[strings.LastIndex(path, "/")+1 : len(path)-1] - if strings.Contains(path, MetricLevelWorkload) { - sourceType = MetricLevelWorkload - } else if strings.Contains(path, MetricLevelWorkspace) { - sourceType = MetricLevelWorkspace - } - var ch = make(chan *FormatedMetric, 10) - for _, metricName := range MetricsNames { - bol, err := regexp.MatchString(metricsName, metricName) - if !bol { - continue - } - if err != nil { - glog.Errorln("regex match failed", err) - continue - } - if strings.HasPrefix(metricName, sourceType) { - if sourceType == MetricLevelCluster || sourceType == MetricLevelNode { - go collectNodeorClusterMetrics(request, metricName, ch) - } else if sourceType == MetricLevelNamespace { - go collectNamespaceMetrics(request, metricName, ch) - } else if sourceType == MetricLevelPod { - go collectPodMetrics(request, metricName, ch) - } else if sourceType == MetricLevelWorkload { - go collectWorkloadMetrics(request, metricName, ch) - } else if sourceType == MetricLevelWorkspace { - name := request.PathParameter("workspace_name") - namespaces, err := workspaces.WorkspaceNamespaces(name) - - if err != nil { - glog.Errorln(err) - } - namespaces = filterNamespace(request, namespaces) - go collectWorkspaceMetrics(request, metricName, namespaces, ch) - } - } - } - var metricsArray []FormatedMetric - var tempJson *FormatedMetric - for _, k := range MetricsNames { - bol, err := regexp.MatchString(metricsName, k) - if !bol { - continue - } - if err != nil { - glog.Errorln("regex match failed") - continue - } - if strings.HasPrefix(k, sourceType) { - tempJson = <-ch - if tempJson != nil { - metricsArray = append(metricsArray, *tempJson) - } - } - } - return FormatedLevelMetric{ - MetricsLevel: sourceType, - Results: metricsArray, - } -} - -func MonitorWorkspaceUserInfo(req *restful.Request) FormatedLevelMetric { - - var metricsArray []FormatedMetric - timestamp := time.Now().Unix() - - wg := sync.WaitGroup{} - var orgResultItem FormatedMetric - var dvpResultItem FormatedMetric - var projResultItem FormatedMetric - var actResultItem FormatedMetric - - wg.Add(4) - - go func() { - orgNums, errOrg := workspaces.GetAllOrgNums() - orgResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllOrganizationCount, WorkspaceResourceKindOrganization, orgNums, errOrg) - wg.Done() - }() - go func() { - devOpsProjectNums, errDevops := workspaces.GetAllDevOpsProjectsNums() - dvpResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllDevopsCount, WorkspaceResourceKindDevops, devOpsProjectNums, errDevops) - wg.Done() - }() - go func() { - projNums, errProj := workspaces.GetAllProjectNums() - projResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllProjectCount, WorkspaceResourceKindNamespace, projNums, errProj) - wg.Done() - }() - go func() { - actNums, errAct := workspaces.GetAllAccountNums() - actResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllAccountCount, WorkspaceResourceKindAccount, actNums, errAct) - wg.Done() - }() - - wg.Wait() - metricsArray = append(metricsArray, orgResultItem, dvpResultItem, projResultItem, actResultItem) - - return FormatedLevelMetric{ - MetricsLevel: MetricLevelWorkspace, - Results: metricsArray, - } -} - -//func getWorkspaceMetricItem(timestamp int64, namespaceNums int64, resourceName string, err error) FormatedMetric { -// var fMetric FormatedMetric -// fMetric.Data.ResultType = ResultTypeVector -// fMetric.MetricName = MetricNameWorkspaceInfoCount -// fMetric.Status = MetricStatusSuccess -// if err != nil { -// fMetric.Status = MetricStatusError -// } -// resultItem := make(map[string]interface{}) -// tmp := make(map[string]string) -// tmp[ResultItemMetricResource] = resourceName -// resultItem[ResultItemMetric] = tmp -// resultItem[ResultItemValue] = []interface{}{timestamp, strconv.FormatInt(namespaceNums, 10)} -// return fMetric -//} - -func MonitorWorkspaceResourceLevelMetrics(request *restful.Request) FormatedLevelMetric { - wsName := request.PathParameter("workspace_name") - namspaces, errNs := workspaces.WorkspaceNamespaces(wsName) - - devOpsProjects, errDevOps := workspaces.GetDevOpsProjects(wsName) - members, errMemb := workspaces.GetOrgMembers(wsName) - roles, errRole := workspaces.GetOrgRoles(wsName) - - var fMetricsArray []FormatedMetric - timestamp := int64(time.Now().Unix()) - namespaces, noneExistentNs := getExistingNamespace(namspaces) - if len(noneExistentNs) != 0 { - nsStr := strings.Join(noneExistentNs, "|") - errStr := "the namespaces " + nsStr + " do not exist" - if errNs == nil { - errNs = errors.New(errStr) - } else { - errNs = errors.New(errNs.Error() + "\t" + errStr) - } - } - - // add namespaces(project) metric - nsMetrics := getSpecificMetricItem(timestamp, MetricNameWorkspaceNamespaceCount, WorkspaceResourceKindNamespace, len(namespaces), errNs) - // add devops metric - devopsMetrics := getSpecificMetricItem(timestamp, MetricNameWorkspaceDevopsCount, WorkspaceResourceKindDevops, len(devOpsProjects), errDevOps) - // add member metric - memberMetrics := getSpecificMetricItem(timestamp, MetricNameWorkspaceMemberCount, WorkspaceResourceKindMember, len(members), errMemb) - // add role metric - roleMetrics := getSpecificMetricItem(timestamp, MetricNameWorkspaceRoleCount, WorkspaceResourceKindRole, len(roles), errRole) - // add workloads count metric - wlMetrics := getWorkspaceWorkloadCountMetrics(namespaces) - // add pods count metric - podsCountMetrics := getWorkspacePodsCountMetrics(request, namespaces) - fMetricsArray = append(fMetricsArray, nsMetrics, devopsMetrics, memberMetrics, roleMetrics, wlMetrics, *podsCountMetrics) - - return FormatedLevelMetric{ - MetricsLevel: MetricLevelWorkspace, - Results: fMetricsArray, - } -} - -func getWorkspacePodsCountMetrics(request *restful.Request, namespaces []string) *FormatedMetric { - metricName := MetricNameNamespacePodCount - var recordingRule = RulePromQLTmplMap[metricName] - nsFilter := "^(" + strings.Join(namespaces, "|") + ")$" - recordingRule = strings.Replace(recordingRule, "$1", nsFilter, -1) - res := client.SendPrometheusRequest(request, recordingRule) - cleanedJson := ReformatJson(res, metricName) - return cleanedJson -} - -func getWorkspaceWorkloadCountMetrics(namespaces []string) FormatedMetric { - var wlQuotaMetrics models.ResourceQuota - wlQuotaMetrics.NameSpace = strings.Join(namespaces, "|") - wlQuotaMetrics.Data.Used = make(v1.ResourceList, 1) - wlQuotaMetrics.Data.Hard = make(v1.ResourceList, 1) - for _, ns := range namespaces { - quotaMetric, err := models.GetNamespaceQuota(ns) - if err != nil { - glog.Errorln(err) - continue - } - // sum all resources used along namespaces - quotaUsed := quotaMetric.Data.Used - for resourceName, quantity := range quotaUsed { - if _, ok := wlQuotaMetrics.Data.Used[resourceName]; ok { - tmpQuantity := wlQuotaMetrics.Data.Used[v1.ResourceName(resourceName)] - tmpQuantity.Add(quantity) - wlQuotaMetrics.Data.Used[v1.ResourceName(resourceName)] = tmpQuantity - } else { - wlQuotaMetrics.Data.Used[v1.ResourceName(resourceName)] = quantity.DeepCopy() - } - } - - // sum all resources hard along namespaces - quotaHard := quotaMetric.Data.Hard - for resourceName, quantity := range quotaHard { - if _, ok := wlQuotaMetrics.Data.Hard[resourceName]; ok { - tmpQuantity := wlQuotaMetrics.Data.Hard[v1.ResourceName(resourceName)] - tmpQuantity.Add(quantity) - wlQuotaMetrics.Data.Hard[v1.ResourceName(resourceName)] = tmpQuantity - } else { - wlQuotaMetrics.Data.Hard[v1.ResourceName(resourceName)] = quantity.DeepCopy() - } - } - } - wlMetrics := convertQuota2MetricStruct(&wlQuotaMetrics) - return wlMetrics -} - -func getSpecificMetricItem(timestamp int64, metricName string, kind string, count int, err error) FormatedMetric { - var nsMetrics FormatedMetric - nsMetrics.MetricName = metricName - nsMetrics.Data.ResultType = ResultTypeVector - resultItem := make(map[string]interface{}) - tmp := make(map[string]string) - tmp[ResultItemMetricResource] = kind - if err == nil { - nsMetrics.Status = MetricStatusSuccess - } else { - nsMetrics.Status = MetricStatusError - resultItem["errorinfo"] = err.Error() - } - - resultItem[ResultItemMetric] = tmp - resultItem[ResultItemValue] = []interface{}{timestamp, count} - nsMetrics.Data.Result = make([]map[string]interface{}, 1) - nsMetrics.Data.Result[0] = resultItem - return nsMetrics -} - -func MonitorNodeorClusterSingleMetric(request *restful.Request, metricsName string) *FormatedMetric { - // support cluster node statistic, include healthy nodes and unhealthy nodes - var res string - var fMetric FormatedMetric - timestamp := int64(time.Now().Unix()) - - if metricsName == "cluster_node_online" { - onlineNodes, _ := getNodeHealthyConditionMetric() - fMetric = getSpecificMetricItem(timestamp, MetricNameClusterHealthyNodeCount, "node_count", len(onlineNodes), nil) - } else if metricsName == "cluster_node_offline" { - _, offlineNodes := getNodeHealthyConditionMetric() - fMetric = getSpecificMetricItem(timestamp, MetricNameClusterUnhealthyNodeCount, "node_count", len(offlineNodes), nil) - } else if metricsName == "cluster_node_total" { - onlineNodes, offlineNodes := getNodeHealthyConditionMetric() - fMetric = getSpecificMetricItem(timestamp, MetricNameClusterNodeCount, "node_count", len(onlineNodes)+len(offlineNodes), nil) - } else { - recordingRule := MakeNodeorClusterRule(request, metricsName) - res = client.SendPrometheusRequest(request, recordingRule) - fMetric = *ReformatJson(res, metricsName) - } - return &fMetric -} - -func getNodeHealthyConditionMetric() ([]string, []string) { - nodeList, err := client.NewK8sClient().CoreV1().Nodes().List(metaV1.ListOptions{}) - if err != nil { - glog.Errorln(err) - return nil, nil - } - var onlineNodes []string - var offlineNodes []string - for _, node := range nodeList.Items { - nodeName := node.Labels["kubernetes.io/hostname"] - nodeRole := node.Labels["role"] - bol := true - for _, cond := range node.Status.Conditions { - if cond.Type == "Ready" && cond.Status == "Unknown" { - bol = false - break - } - } - if nodeRole != "log" { - if bol { - // reachable node - onlineNodes = append(onlineNodes, nodeName) - } else { - // unreachable node - offlineNodes = append(offlineNodes, nodeName) - } - } - } - return onlineNodes, offlineNodes -} - -func getExistingNamespace(namespaces []string) ([]string, []string) { - namespaceMap, err := getAllNamespace() - var existedNs []string - var noneExistedNs []string - if err != nil { - return namespaces, nil - } - for _, ns := range namespaces { - if _, ok := namespaceMap[ns]; ok { - existedNs = append(existedNs, ns) - } else { - noneExistedNs = append(noneExistedNs, ns) - } - } - return existedNs, noneExistedNs -} - -func getAllNamespace() (map[string]int, error) { - lister := controllers.ResourceControllers.Controllers[controllers.Namespaces].Lister().(v12.NamespaceLister) - nsList, err := lister.List(labels.Everything()) - if err != nil { - glog.Errorln(err) - return nil, err - } - namespaceMap := make(map[string]int) - for _, item := range nsList { - namespaceMap[item.Name] = 0 - } - return namespaceMap, nil -} - -func MonitorWorkloadCount(request *restful.Request) FormatedMetric { - namespace := strings.Trim(request.PathParameter("ns_name"), " ") - - quotaMetric, err := models.GetNamespaceQuota(namespace) - fMetric := convertQuota2MetricStruct(quotaMetric) - - // whether the namespace in request parameters exists? - namespaceMap, e := getAllNamespace() - _, ok := namespaceMap[namespace] - if e != nil { - ok = true - } - if !ok || err != nil { - fMetric.Status = MetricStatusError - fMetric.Data.ResultType = "" - errInfo := make(map[string]interface{}) - if err != nil { - errInfo["errormsg"] = err.Error() - } else { - errInfo["errormsg"] = "namespace " + namespace + " does not exist" - } - fMetric.Data.Result = []map[string]interface{}{errInfo} - } - - return fMetric -} - -func convertQuota2MetricStruct(quotaMetric *models.ResourceQuota) FormatedMetric { - var fMetric FormatedMetric - fMetric.MetricName = MetricNameWorkloadCount - fMetric.Status = MetricStatusSuccess - fMetric.Data.ResultType = ResultTypeVector - timestamp := int64(time.Now().Unix()) - var resultItems []map[string]interface{} - - hardMap := make(map[string]string) - for resourceName, v := range quotaMetric.Data.Hard { - hardMap[resourceName.String()] = v.String() - } - - for resourceName, v := range quotaMetric.Data.Used { - resultItem := make(map[string]interface{}) - tmp := make(map[string]string) - tmp[ResultItemMetricResource] = resourceName.String() - resultItem[ResultItemMetric] = tmp - resultItem[ResultItemValue] = []interface{}{timestamp, hardMap[resourceName.String()], v.String()} - resultItems = append(resultItems, resultItem) - } - - fMetric.Data.Result = resultItems - return fMetric -} diff --git a/pkg/models/metrics/metricsconst.go b/pkg/models/metrics/metricsconst.go deleted file mode 100755 index a43cd52b5..000000000 --- a/pkg/models/metrics/metricsconst.go +++ /dev/null @@ -1,236 +0,0 @@ -/* -Copyright 2018 The KubeSphere Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -const ( - ResultTypeVector = "vector" - ResultTypeMatrix = "matrix" - MetricStatusError = "error" - MetricStatusSuccess = "success" - ResultItemMetric = "metric" - ResultItemMetricResource = "resource" - ResultItemValue = "value" -) - -const ( - MetricNameWorkloadCount = "workload_count" - MetricNameNamespacePodCount = "namespace_pod_count" - - MetricNameWorkspaceAllOrganizationCount = "workspace_all_organization_count" - MetricNameWorkspaceAllAccountCount = "workspace_all_account_count" - MetricNameWorkspaceAllProjectCount = "workspace_all_project_count" - MetricNameWorkspaceAllDevopsCount = "workspace_all_devops_project_count" - - MetricNameWorkspaceNamespaceCount = "workspace_namespace_count" - MetricNameWorkspaceDevopsCount = "workspace_devops_project_count" - MetricNameWorkspaceMemberCount = "workspace_member_count" - MetricNameWorkspaceRoleCount = "workspace_role_count" - - MetricNameClusterHealthyNodeCount = "cluster_node_online" - MetricNameClusterUnhealthyNodeCount = "cluster_node_offline" - MetricNameClusterNodeCount = "cluster_node_total" -) - -const ( - WorkspaceResourceKindOrganization = "organization" - WorkspaceResourceKindAccount = "account" - WorkspaceResourceKindNamespace = "namespace" - WorkspaceResourceKindDevops = "devops" - WorkspaceResourceKindMember = "member" - WorkspaceResourceKindRole = "role" -) - -const ( - MetricLevelCluster = "cluster" - MetricLevelNode = "node" - MetricLevelWorkspace = "workspace" - MetricLevelNamespace = "namespace" - MetricLevelPod = "pod" - MetricLevelContainer = "container" - MetricLevelWorkload = "workload" -) - -type MetricMap map[string]string - -var MetricsNames = []string{ - "cluster_cpu_utilisation", - "cluster_cpu_usage", - "cluster_cpu_total", - "cluster_memory_utilisation", - "cluster_pod_count", - "cluster_memory_bytes_available", - "cluster_memory_bytes_total", - "cluster_memory_bytes_usage", - "cluster_net_utilisation", - "cluster_net_bytes_transmitted", - "cluster_net_bytes_received", - "cluster_disk_read_iops", - "cluster_disk_write_iops", - "cluster_disk_read_throughput", - "cluster_disk_write_throughput", - "cluster_disk_size_usage", - "cluster_disk_size_utilisation", - "cluster_disk_size_capacity", - "cluster_disk_size_available", - "cluster_node_online", - "cluster_node_offline", - "cluster_node_total", - - "node_cpu_utilisation", - "node_cpu_total", - "node_cpu_usage", - "node_memory_utilisation", - "node_memory_bytes_usage", - "node_memory_bytes_available", - "node_memory_bytes_total", - "node_net_utilisation", - "node_net_bytes_transmitted", - "node_net_bytes_received", - "node_disk_read_iops", - "node_disk_write_iops", - "node_disk_read_throughput", - "node_disk_write_throughput", - "node_disk_size_capacity", - "node_disk_size_available", - "node_disk_size_usage", - "node_disk_size_utilisation", - "node_pod_count", - "node_pod_quota", - - "namespace_cpu_usage", - "namespace_memory_usage", - "namespace_memory_usage_wo_cache", - "namespace_net_bytes_transmitted", - "namespace_net_bytes_received", - "namespace_pod_count", - - "pod_cpu_usage", - "pod_memory_usage", - "pod_memory_usage_wo_cache", - "pod_net_bytes_transmitted", - "pod_net_bytes_received", - - "workload_pod_cpu_usage", - "workload_pod_memory_usage", - "workload_pod_memory_usage_wo_cache", - "workload_pod_net_bytes_transmitted", - "workload_pod_net_bytes_received", - //"container_cpu_usage", - //"container_memory_usage_wo_cache", - //"container_memory_usage", - - "workspace_cpu_usage", - "workspace_memory_usage", - "workspace_memory_usage_wo_cache", - "workspace_net_bytes_transmitted", - "workspace_net_bytes_received", - "workspace_pod_count", -} - -var RulePromQLTmplMap = MetricMap{ - //cluster - "cluster_cpu_utilisation": ":node_cpu_utilisation:avg1m", - "cluster_cpu_usage": `sum (irate(container_cpu_usage_seconds_total{job="kubelet", image!=""}[5m]))`, - "cluster_cpu_total": "sum(node:node_num_cpu:sum)", - "cluster_memory_utilisation": ":node_memory_utilisation:", - "cluster_pod_count": `count(kube_pod_info unless on(pod) kube_pod_completion_time unless on(node) kube_node_labels{label_role="log"})`, - "cluster_memory_bytes_available": "sum(node:node_memory_bytes_available:sum)", - "cluster_memory_bytes_total": "sum(node:node_memory_bytes_total:sum)", - "cluster_memory_bytes_usage": "sum(node:node_memory_bytes_total:sum) - sum(node:node_memory_bytes_available:sum)", - "cluster_net_utilisation": ":node_net_utilisation:sum_irate", - "cluster_net_bytes_transmitted": "sum(node:node_net_bytes_transmitted:sum_irate)", - "cluster_net_bytes_received": "sum(node:node_net_bytes_received:sum_irate)", - "cluster_disk_read_iops": "sum(node:data_volume_iops_reads:sum)", - "cluster_disk_write_iops": "sum(node:data_volume_iops_writes:sum)", - "cluster_disk_read_throughput": "sum(node:data_volume_throughput_bytes_read:sum)", - "cluster_disk_write_throughput": "sum(node:data_volume_throughput_bytes_written:sum)", - "cluster_disk_size_usage": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, - "cluster_disk_size_utilisation": `(sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))) / sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, - "cluster_disk_size_capacity": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, - "cluster_disk_size_available": `sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, - - //node - "node_cpu_utilisation": "node:node_cpu_utilisation:avg1m", - "node_cpu_total": "node:node_num_cpu:sum", - "node_memory_utilisation": "node:node_memory_utilisation:", - "node_memory_bytes_available": "node:node_memory_bytes_available:sum", - "node_memory_bytes_total": "node:node_memory_bytes_total:sum", - // Node network utilisation (bytes received + bytes transmitted per second) - "node_net_utilisation": "node:node_net_utilisation:sum_irate", - // Node network bytes transmitted per second - "node_net_bytes_transmitted": "node:node_net_bytes_transmitted:sum_irate", - // Node network bytes received per second - "node_net_bytes_received": "node:node_net_bytes_received:sum_irate", - - // node:data_volume_iops_reads:sum{node=~"i-5xcldxos|i-6soe9zl1"} - "node_disk_read_iops": "node:data_volume_iops_reads:sum", - // node:data_volume_iops_writes:sum{node=~"i-5xcldxos|i-6soe9zl1"} - "node_disk_write_iops": "node:data_volume_iops_writes:sum", - // node:data_volume_throughput_bytes_read:sum{node=~"i-5xcldxos|i-6soe9zl1"} - "node_disk_read_throughput": "node:data_volume_throughput_bytes_read:sum", - // node:data_volume_throughput_bytes_written:sum{node=~"i-5xcldxos|i-6soe9zl1"} - "node_disk_write_throughput": "node:data_volume_throughput_bytes_written:sum", - - "node_disk_size_capacity": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_disk_size_available": `sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_disk_size_usage": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) -sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_disk_size_utilisation": `sum by (node) (((node_filesystem_size{mountpoint="/", job="node-exporter"} - node_filesystem_avail{mountpoint="/", job="node-exporter"}) / node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_pod_count": `count(kube_pod_info$1 unless on(pod) kube_pod_completion_time) by (node)`, - // without log node: unless on(node) kube_node_labels{label_role="log"} - "node_pod_quota": `sum(kube_node_status_capacity_pods$1) by (node)`, - "node_cpu_usage": `sum by (node) (label_join(irate(container_cpu_usage_seconds_total{job="kubelet", image!=""}[5m]), "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, - "node_memory_bytes_usage": "node:node_memory_bytes_total:sum$1 - node:node_memory_bytes_available:sum$1", - - //namespace - "namespace_cpu_usage": `namespace:container_cpu_usage_seconds_total:sum_rate{namespace=~"$1"}`, - "namespace_memory_usage": `namespace:container_memory_usage_bytes:sum{namespace=~"$1"}`, - "namespace_memory_usage_wo_cache": `namespace:container_memory_usage_bytes_wo_cache:sum{namespace=~"$1"}`, - "namespace_net_bytes_transmitted": `sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m]))`, - "namespace_net_bytes_received": `sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m]))`, - "namespace_pod_count": `count(kube_pod_info{namespace=~"$1"} unless on(pod) kube_pod_completion_time) by (namespace)`, - - // pod - "pod_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name="$2", image!=""}[5m])) by (namespace, pod_name)`, - "pod_memory_usage": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""}) by (namespace, pod_name)`, - "pod_memory_usage_wo_cache": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name="$2",image!=""}) by (namespace, pod_name)`, - "pod_net_bytes_transmitted": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[5m]))`, - "pod_net_bytes_received": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[5m]))`, - - "pod_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}[5m])) by (namespace, pod_name)`, - "pod_memory_usage_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`, - "pod_memory_usage_wo_cache_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`, - "pod_net_bytes_transmitted_all": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`, - "pod_net_bytes_received_all": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`, - - "pod_cpu_usage_node": `sum by (node, pod) (label_join(irate(container_cpu_usage_seconds_total{job="kubelet",pod_name=~"$2", image!=""}[5m]), "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, - "pod_memory_usage_node": `sum by (node, pod) (label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, - "pod_memory_usage_wo_cache_node": `sum by (node, pod) ((label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") - label_join(container_memory_cache{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name")) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, - - // container - "container_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name="$3"}[5m])) by (namespace, pod_name, container_name)`, - "container_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}[5m])) by (namespace, pod_name, container_name)`, - - "container_memory_usage_wo_cache": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name="$3"}`, - "container_memory_usage_wo_cache_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`, - "container_memory_usage": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"}`, - "container_memory_usage_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`, - - // enterprise - "workspace_cpu_usage": `sum(namespace:container_cpu_usage_seconds_total:sum_rate{namespace =~"$1"})`, - "workspace_memory_usage": `sum(namespace:container_memory_usage_bytes:sum{namespace =~"$1"})`, - "workspace_memory_usage_wo_cache": `sum(namespace:container_memory_usage_bytes_wo_cache:sum{namespace =~"$1"})`, - "workspace_net_bytes_transmitted": `sum(sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m])))`, - "workspace_net_bytes_received": `sum(sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m])))`, - "workspace_pod_count": `sum(count(kube_pod_info{namespace=~"$1"} unless on(pod) kube_pod_completion_time) by (namespace))`, -} diff --git a/pkg/models/metrics/metricsrule.go b/pkg/models/metrics/metricsrule.go index ee3ee3b70..b2321b6b1 100755 --- a/pkg/models/metrics/metricsrule.go +++ b/pkg/models/metrics/metricsrule.go @@ -15,142 +15,119 @@ package metrics import ( "strings" - - "github.com/emicklei/go-restful" ) -func MakeWorkloadRule(request *restful.Request) string { - // kube_pod_info{created_by_kind="DaemonSet",created_by_name="fluent-bit",endpoint="https-main", - // host_ip="192.168.0.14",instance="10.244.114.187:8443",job="kube-state-metrics", - // namespace="kube-system",node="i-k89a62il",pod="fluent-bit-l5vxr", - // pod_ip="10.244.114.175",service="kube-state-metrics"} - rule := `kube_pod_info{created_by_kind="$1",created_by_name=$2,namespace="$3"}` - kind := strings.Trim(request.PathParameter("workload_kind"), " ") - name := strings.Trim(request.QueryParameter("workload_name"), " ") - namespace := strings.Trim(request.PathParameter("ns_name"), " ") +func MakeWorkloadRule(wkKind, wkName, namespace string) string { + var rule = PodInfoRule if namespace == "" { namespace = ".*" } + // alertnatives values: Deployment StatefulSet ReplicaSet DaemonSet + wkKind = strings.ToLower(wkKind) - // kind alertnatives values: Deployment StatefulSet ReplicaSet DaemonSet - kind = strings.ToLower(kind) - - switch kind { + switch wkKind { case "deployment": - kind = "ReplicaSet" - if name != "" { - name = "~\"" + name + ".*\"" + wkKind = ReplicaSet + if wkName != "" { + wkName = "~\"" + wkName + ".*\"" } else { - name = "~\".*\"" + wkName = "~\".*\"" } - rule = strings.Replace(rule, "$1", kind, -1) - rule = strings.Replace(rule, "$2", name, -1) + rule = strings.Replace(rule, "$1", wkKind, -1) + rule = strings.Replace(rule, "$2", wkName, -1) rule = strings.Replace(rule, "$3", namespace, -1) return rule case "replicaset": - kind = "ReplicaSet" + wkKind = ReplicaSet case "statefulset": - kind = "StatefulSet" + wkKind = StatefulSet case "daemonset": - kind = "DaemonSet" + wkKind = DaemonSet } - if name == "" { - name = "~\".*\"" + if wkName == "" { + wkName = "~\".*\"" } else { - name = "\"" + name + "\"" + wkName = "\"" + wkName + "\"" } - rule = strings.Replace(rule, "$1", kind, -1) - rule = strings.Replace(rule, "$2", name, -1) + rule = strings.Replace(rule, "$1", wkKind, -1) + rule = strings.Replace(rule, "$2", wkName, -1) rule = strings.Replace(rule, "$3", namespace, -1) return rule } -func MakeWorkspacePromQL(metricsName string, namespaceRe2 string) string { +func MakeWorkspacePromQL(metricsName string, nsFilter string) string { promql := RulePromQLTmplMap[metricsName] - promql = strings.Replace(promql, "$1", namespaceRe2, -1) + promql = strings.Replace(promql, "$1", nsFilter, -1) return promql } -func MakeContainerPromQL(request *restful.Request) string { - nsName := strings.Trim(request.PathParameter("ns_name"), " ") - poName := strings.Trim(request.PathParameter("pod_name"), " ") - containerName := strings.Trim(request.PathParameter("container_name"), " ") - // metricType container_cpu_utilisation container_memory_utilisation container_memory_utilisation_wo_cache - metricType := strings.Trim(request.QueryParameter("metrics_name"), " ") +func MakeContainerPromQL(nsName, podName, containerName, metricName, containerFilter string) string { var promql = "" if containerName == "" { // all containers maybe use filter - metricType += "_all" - promql = RulePromQLTmplMap[metricType] + metricName += "_all" + promql = RulePromQLTmplMap[metricName] promql = strings.Replace(promql, "$1", nsName, -1) - promql = strings.Replace(promql, "$2", poName, -1) - containerFilter := strings.Trim(request.QueryParameter("containers_filter"), " ") + promql = strings.Replace(promql, "$2", podName, -1) + if containerFilter == "" { containerFilter = ".*" } + promql = strings.Replace(promql, "$3", containerFilter, -1) return promql } - promql = RulePromQLTmplMap[metricType] + promql = RulePromQLTmplMap[metricName] promql = strings.Replace(promql, "$1", nsName, -1) - promql = strings.Replace(promql, "$2", poName, -1) + promql = strings.Replace(promql, "$2", podName, -1) promql = strings.Replace(promql, "$3", containerName, -1) return promql } -func MakePodPromQL(request *restful.Request, params []string) string { - metricType := params[0] - nsName := params[1] - nodeID := params[2] - podName := params[3] - podFilter := params[4] +func MakePodPromQL(metricName, nsName, nodeID, podName, podFilter string) string { + + if podFilter == "" { + podFilter = ".*" + } + var promql = "" if nsName != "" { // get pod metrics by namespace if podName != "" { // specific pod - promql = RulePromQLTmplMap[metricType] + promql = RulePromQLTmplMap[metricName] promql = strings.Replace(promql, "$1", nsName, -1) promql = strings.Replace(promql, "$2", podName, -1) } else { // all pods - metricType += "_all" - promql = RulePromQLTmplMap[metricType] - if podFilter == "" { - podFilter = ".*" - } + metricName += "_all" + promql = RulePromQLTmplMap[metricName] + promql = strings.Replace(promql, "$1", nsName, -1) promql = strings.Replace(promql, "$2", podFilter, -1) } } else if nodeID != "" { // get pod metrics by nodeid - metricType += "_node" - promql = RulePromQLTmplMap[metricType] + metricName += "_node" + promql = RulePromQLTmplMap[metricName] promql = strings.Replace(promql, "$3", nodeID, -1) if podName != "" { // specific pod promql = strings.Replace(promql, "$2", podName, -1) } else { - // choose pod use re2 expression - podFilter := strings.Trim(request.QueryParameter("pods_filter"), " ") - if podFilter == "" { - podFilter = ".*" - } promql = strings.Replace(promql, "$2", podFilter, -1) } } return promql } -func MakeNamespacePromQL(request *restful.Request, metricsName string) string { - nsName := strings.Trim(request.PathParameter("ns_name"), " ") - metricType := metricsName - var recordingRule = RulePromQLTmplMap[metricType] - nsFilter := strings.Trim(request.QueryParameter("namespaces_filter"), " ") +func MakeNamespacePromQL(nsName string, nsFilter string, metricsName string) string { + var recordingRule = RulePromQLTmplMap[metricsName] + if nsName != "" { nsFilter = nsName } else { @@ -162,37 +139,37 @@ func MakeNamespacePromQL(request *restful.Request, metricsName string) string { return recordingRule } -func MakeNodeorClusterRule(request *restful.Request, metricsName string) string { - nodeID := request.PathParameter("node_id") +// cluster rule +func MakeClusterRule(metricsName string) string { var rule = RulePromQLTmplMap[metricsName] - - if strings.Contains(request.SelectedRoutePath(), "monitoring/cluster") { - // cluster - return rule - } else { - // node - nodesFilter := strings.Trim(request.QueryParameter("nodes_filter"), " ") - if nodesFilter == "" { - nodesFilter = ".*" - } - if strings.Contains(metricsName, "disk_size") || strings.Contains(metricsName, "pod") || strings.Contains(metricsName, "usage") { - // disk size promql - if nodeID != "" { - nodesFilter = "{" + "node" + "=" + "\"" + nodeID + "\"" + "}" - } else { - nodesFilter = "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}" - } - rule = strings.Replace(rule, "$1", nodesFilter, -1) - } else { - // cpu, memory, network, disk_iops rules - if nodeID != "" { - // specific node - rule = rule + "{" + "node" + "=" + "\"" + nodeID + "\"" + "}" - } else { - // all nodes or specific nodes filted with re2 syntax - rule = rule + "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}" - } - } - } + return rule +} + +// node rule +func MakeNodeRule(nodeID string, nodesFilter string, metricsName string) string { + var rule = RulePromQLTmplMap[metricsName] + + if nodesFilter == "" { + nodesFilter = ".*" + } + if strings.Contains(metricsName, "disk_size") || strings.Contains(metricsName, "pod") || strings.Contains(metricsName, "usage") || strings.Contains(metricsName, "inode") { + // disk size promql + if nodeID != "" { + nodesFilter = "{" + "node" + "=" + "\"" + nodeID + "\"" + "}" + } else { + nodesFilter = "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}" + } + rule = strings.Replace(rule, "$1", nodesFilter, -1) + } else { + // cpu, memory, network, disk_iops rules + if nodeID != "" { + // specific node + rule = rule + "{" + "node" + "=" + "\"" + nodeID + "\"" + "}" + } else { + // all nodes or specific nodes filted with re2 syntax + rule = rule + "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}" + } + } + return rule } diff --git a/pkg/models/metrics/metricsruleconst.go b/pkg/models/metrics/metricsruleconst.go new file mode 100644 index 000000000..05896333d --- /dev/null +++ b/pkg/models/metrics/metricsruleconst.go @@ -0,0 +1,560 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +const ( + ResultTypeVector = "vector" + ResultTypeMatrix = "matrix" + MetricStatus = "status" + MetricStatusError = "error" + MetricStatusSuccess = "success" + ResultItemMetric = "metric" + ResultItemMetricResource = "resource" + ResultItemValue = "value" + ResultItemValues = "values" + ResultSortTypeDesc = "desc" + ResultSortTypeAsce = "asce" +) + +const ( + MetricNameWorkloadCount = "workload_count" + MetricNameNamespacePodCount = "namespace_pod_count" + + MetricNameWorkspaceAllOrganizationCount = "workspace_all_organization_count" + MetricNameWorkspaceAllAccountCount = "workspace_all_account_count" + MetricNameWorkspaceAllProjectCount = "workspace_all_project_count" + MetricNameWorkspaceAllDevopsCount = "workspace_all_devops_project_count" + MetricNameClusterAllProjectCount = "cluster_namespace_count" + + MetricNameWorkspaceNamespaceCount = "workspace_namespace_count" + MetricNameWorkspaceDevopsCount = "workspace_devops_project_count" + MetricNameWorkspaceMemberCount = "workspace_member_count" + MetricNameWorkspaceRoleCount = "workspace_role_count" + MetricNameComponentOnLine = "component_online_count" + MetricNameComponentLine = "component_count" +) + +const ( + WorkspaceResourceKindOrganization = "organization" + WorkspaceResourceKindAccount = "account" + WorkspaceResourceKindNamespace = "namespace" + WorkspaceResourceKindDevops = "devops" + WorkspaceResourceKindMember = "member" + WorkspaceResourceKindRole = "role" +) + +const ( + MetricLevelCluster = "cluster" + MetricLevelClusterWorkspace = "cluster_workspace" + MetricLevelNode = "node" + MetricLevelWorkspace = "workspace" + MetricLevelNamespace = "namespace" + MetricLevelPod = "pod" + MetricLevelPodName = "pod_name" + MetricLevelContainer = "container" + MetricLevelWorkload = "workload" +) + +const ( + ReplicaSet = "ReplicaSet" + StatefulSet = "StatefulSet" + DaemonSet = "DaemonSet" +) + +const ( + NodeStatusRule = `kube_node_status_condition{condition="Ready"} > 0` + PodInfoRule = `kube_pod_info{created_by_kind="$1",created_by_name=$2,namespace="$3"}` + NamespaceLabelRule = `kube_namespace_labels` +) + +const ( + WorkspaceJoinedKey = "label_kubesphere_io_workspace" +) + +type MetricMap map[string]string + +var ClusterMetricsNames = []string{ + "cluster_cpu_utilisation", + "cluster_cpu_usage", + "cluster_cpu_total", + "cluster_memory_utilisation", + "cluster_memory_bytes_available", + "cluster_memory_bytes_total", + "cluster_memory_bytes_usage", + "cluster_net_utilisation", + "cluster_net_bytes_transmitted", + "cluster_net_bytes_received", + "cluster_disk_read_iops", + "cluster_disk_write_iops", + "cluster_disk_read_throughput", + "cluster_disk_write_throughput", + "cluster_disk_size_usage", + "cluster_disk_size_utilisation", + "cluster_disk_size_capacity", + "cluster_disk_size_available", + "cluster_disk_inode_total", + "cluster_disk_inode_usage", + "cluster_disk_inode_utilisation", + + "cluster_node_online", + "cluster_node_offline", + "cluster_node_total", + + "cluster_pod_count", + "cluster_pod_quota", + "cluster_pod_utilisation", + "cluster_pod_running_count", + "cluster_pod_succeeded_count", + "cluster_pod_abnormal_count", + "cluster_ingresses_extensions_count", + "cluster_cronjob_count", + "cluster_pvc_count", + "cluster_daemonset_count", + "cluster_deployment_count", + "cluster_endpoint_count", + "cluster_hpa_count", + "cluster_job_count", + "cluster_statefulset_count", + "cluster_replicaset_count", + "cluster_service_count", + "cluster_secret_count", + + "cluster_namespace_count", + "workspace_all_project_count", +} +var NodeMetricsNames = []string{ + "node_cpu_utilisation", + "node_cpu_total", + "node_cpu_usage", + "node_memory_utilisation", + "node_memory_bytes_usage", + "node_memory_bytes_available", + "node_memory_bytes_total", + "node_net_utilisation", + "node_net_bytes_transmitted", + "node_net_bytes_received", + "node_disk_read_iops", + "node_disk_write_iops", + "node_disk_read_throughput", + "node_disk_write_throughput", + "node_disk_size_capacity", + "node_disk_size_available", + "node_disk_size_usage", + "node_disk_size_utilisation", + + "node_disk_inode_total", + "node_disk_inode_usage", + "node_disk_inode_utilisation", + + "node_pod_count", + "node_pod_quota", + "node_pod_utilisation", + "node_pod_running_count", + "node_pod_succeeded_count", + "node_pod_abnormal_count", +} +var WorkspaceMetricsNames = []string{ + "workspace_cpu_usage", + "workspace_memory_usage", + "workspace_memory_usage_wo_cache", + "workspace_net_bytes_transmitted", + "workspace_net_bytes_received", + "workspace_pod_count", + "workspace_pod_running_count", + "workspace_pod_succeeded_count", + "workspace_pod_abnormal_count", + "workspace_ingresses_extensions_count", + + "workspace_cronjob_count", + "workspace_pvc_count", + "workspace_daemonset_count", + "workspace_deployment_count", + "workspace_endpoint_count", + "workspace_hpa_count", + "workspace_job_count", + "workspace_statefulset_count", + "workspace_replicaset_count", + "workspace_service_count", + "workspace_secret_count", +} +var NamespaceMetricsNames = []string{ + "namespace_cpu_usage", + "namespace_memory_usage", + "namespace_memory_usage_wo_cache", + "namespace_net_bytes_transmitted", + "namespace_net_bytes_received", + "namespace_pod_count", + "namespace_pod_running_count", + "namespace_pod_succeeded_count", + "namespace_pod_abnormal_count", + + "namespace_configmap_count_used", + "namespace_jobs_batch_count_used", + "namespace_roles_count_used", + "namespace_memory_limit_used", + "namespace_pvc_used", + "namespace_memory_request_used", + "namespace_pvc_count_used", + "namespace_cronjobs_batch_count_used", + "namespace_ingresses_extensions_count_used", + "namespace_cpu_limit_used", + "namespace_storage_request_used", + "namespace_deployment_count_used", + "namespace_pod_count_used", + "namespace_statefulset_count_used", + "namespace_daemonset_count_used", + "namespace_secret_count_used", + "namespace_service_count_used", + "namespace_cpu_request_used", + "namespace_service_loadbalancer_used", + + "namespace_configmap_count_hard", + "namespace_jobs_batch_count_hard", + "namespace_roles_count_hard", + "namespace_memory_limit_hard", + "namespace_pvc_hard", + "namespace_memory_request_hard", + "namespace_pvc_count_hard", + "namespace_cronjobs_batch_count_hard", + "namespace_ingresses_extensions_count_hard", + "namespace_cpu_limit_hard", + "namespace_storage_request_hard", + "namespace_deployment_count_hard", + "namespace_pod_count_hard", + "namespace_statefulset_count_hard", + "namespace_daemonset_count_hard", + "namespace_secret_count_hard", + "namespace_service_count_hard", + "namespace_cpu_request_hard", + "namespace_service_loadbalancer_hard", + + "namespace_cronjob_count", + "namespace_pvc_count", + "namespace_daemonset_count", + "namespace_deployment_count", + "namespace_endpoint_count", + "namespace_hpa_count", + "namespace_job_count", + "namespace_statefulset_count", + "namespace_replicaset_count", + "namespace_service_count", + "namespace_secret_count", +} + +var PodMetricsNames = []string{ + "pod_cpu_usage", + "pod_memory_usage", + "pod_memory_usage_wo_cache", + "pod_net_bytes_transmitted", + "pod_net_bytes_received", +} + +var WorkloadMetricsNames = []string{ + "workload_pod_cpu_usage", + "workload_pod_memory_usage", + "workload_pod_memory_usage_wo_cache", + "workload_pod_net_bytes_transmitted", + "workload_pod_net_bytes_received", +} + +var RulePromQLTmplMap = MetricMap{ + //cluster + "cluster_cpu_utilisation": ":node_cpu_utilisation:avg1m", + "cluster_cpu_usage": `:node_cpu_utilisation:avg1m * sum(node:node_num_cpu:sum)`, + "cluster_cpu_total": "sum(node:node_num_cpu:sum)", + "cluster_memory_utilisation": ":node_memory_utilisation:", + "cluster_memory_bytes_available": "sum(node:node_memory_bytes_available:sum)", + "cluster_memory_bytes_total": "sum(node:node_memory_bytes_total:sum)", + "cluster_memory_bytes_usage": "sum(node:node_memory_bytes_total:sum) - sum(node:node_memory_bytes_available:sum)", + "cluster_net_utilisation": ":node_net_utilisation:sum_irate", + "cluster_net_bytes_transmitted": "sum(node:node_net_bytes_transmitted:sum_irate)", + "cluster_net_bytes_received": "sum(node:node_net_bytes_received:sum_irate)", + "cluster_disk_read_iops": "sum(node:data_volume_iops_reads:sum)", + "cluster_disk_write_iops": "sum(node:data_volume_iops_writes:sum)", + "cluster_disk_read_throughput": "sum(node:data_volume_throughput_bytes_read:sum)", + "cluster_disk_write_throughput": "sum(node:data_volume_throughput_bytes_written:sum)", + "cluster_disk_size_usage": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, + "cluster_disk_size_utilisation": `(sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))) / sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, + "cluster_disk_size_capacity": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, + "cluster_disk_size_available": `sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`, + + "cluster_disk_inode_total": `sum(node:disk_inodes_total:)`, + "cluster_disk_inode_usage": `sum(node:disk_inodes_total:) - sum(node:disk_inodes_free:)`, + "cluster_disk_inode_utilisation": `1 - sum(node:disk_inodes_free:) / sum(node:disk_inodes_total:)`, + + "cluster_namespace_count": `count(kube_namespace_annotations)`, + "workspace_all_project_count": `count(kube_namespace_annotations)`, + + // cluster_pod_count = cluster_pod_running_count + cluster_pod_succeeded_count + cluster_pod_abnormal_count + "cluster_pod_count": `sum(kube_pod_status_phase)`, + "cluster_pod_utilisation": `sum(kube_pod_status_phase) / sum(kube_node_status_capacity_pods)`, + "cluster_pod_running_count": `sum(kube_pod_status_phase{phase="Running"})`, + "cluster_pod_succeeded_count": `sum(kube_pod_status_phase{phase="Succeeded"})`, + "cluster_pod_abnormal_count": `sum(kube_pod_status_phase{phase=~"Failed|Pending|Unknown"})`, + + "cluster_node_online": `sum(kube_node_status_condition{condition="Ready",status="true"})`, + "cluster_node_offline": `sum(kube_node_status_condition{condition="Ready",status=~"unknown|false"})`, + "cluster_node_total": `sum(kube_node_status_condition{condition="Ready"})`, + "cluster_pod_quota": `sum(kube_node_status_capacity_pods)`, + "cluster_ingresses_extensions_count": `sum(kube_resourcequota{type="used", resource="count/ingresses.extensions"}) by (resource, type)`, + + "cluster_configmap_count_used": `sum(kube_resourcequota{type="used", resource="count/configmaps"}) by (resource, type)`, + "cluster_jobs_batch_count_used": `sum(kube_resourcequota{type="used", resource="count/jobs.batch"}) by (resource, type)`, + "cluster_roles_count_used": `sum(kube_resourcequota{type="used", resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`, + "cluster_memory_limit_used": `sum(kube_resourcequota{type="used", resource="limits.memory"}) by (resource, type)`, + "cluster_pvc_used": `sum(kube_resourcequota{type="used", resource="persistentvolumeclaims"}) by (resource, type)`, + "cluster_memory_request_used": `sum(kube_resourcequota{type="used", resource="requests.memory"}) by (resource, type)`, + "cluster_pvc_count_used": `sum(kube_resourcequota{type="used", resource="count/persistentvolumeclaims"}) by (resource, type)`, + "cluster_cronjobs_batch_count_used": `sum(kube_resourcequota{type="used", resource="count/cronjobs.batch"}) by (resource, type)`, + "cluster_ingresses_extensions_count_used": `sum(kube_resourcequota{type="used", resource="count/ingresses.extensions"}) by (resource, type)`, + "cluster_cpu_limit_used": `sum(kube_resourcequota{type="used", resource="limits.cpu"}) by (resource, type)`, + "cluster_storage_request_used": `sum(kube_resourcequota{type="used", resource="requests.storage"}) by (resource, type)`, + "cluster_deployment_count_used": `sum(kube_resourcequota{type="used", resource="count/deployments.apps"}) by (resource, type)`, + "cluster_pod_count_used": `sum(kube_resourcequota{type="used", resource="count/pods"}) by (resource, type)`, + "cluster_statefulset_count_used": `sum(kube_resourcequota{type="used", resource="count/statefulsets.apps"}) by (resource, type)`, + "cluster_daemonset_count_used": `sum(kube_resourcequota{type="used", resource="count/daemonsets.apps"}) by (resource, type)`, + "cluster_secret_count_used": `sum(kube_resourcequota{type="used", resource="count/secrets"}) by (resource, type)`, + "cluster_service_count_used": `sum(kube_resourcequota{type="used", resource="count/services"}) by (resource, type)`, + "cluster_cpu_request_used": `sum(kube_resourcequota{type="used", resource="requests.cpu"}) by (resource, type)`, + "cluster_service_loadbalancer_used": `sum(kube_resourcequota{type="used", resource="services.loadbalancers"}) by (resource, type)`, + + "cluster_configmap_count_hard": `sum(kube_resourcequota{type="hard", resource="count/configmaps"}) by (resource, type)`, + "cluster_jobs_batch_count_hard": `sum(kube_resourcequota{type="hard", resource="count/jobs.batch"}) by (resource, type)`, + "cluster_roles_count_hard": `sum(kube_resourcequota{type="hard", resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`, + "cluster_memory_limit_hard": `sum(kube_resourcequota{type="hard", resource="limits.memory"}) by (resource, type)`, + "cluster_pvc_hard": `sum(kube_resourcequota{type="hard", resource="persistentvolumeclaims"}) by (resource, type)`, + "cluster_memory_request_hard": `sum(kube_resourcequota{type="hard", resource="requests.memory"}) by (resource, type)`, + "cluster_pvc_count_hard": `sum(kube_resourcequota{type="hard", resource="count/persistentvolumeclaims"}) by (resource, type)`, + "cluster_cronjobs_batch_count_hard": `sum(kube_resourcequota{type="hard", resource="count/cronjobs.batch"}) by (resource, type)`, + "cluster_ingresses_extensions_count_hard": `sum(kube_resourcequota{type="hard", resource="count/ingresses.extensions"}) by (resource, type)`, + "cluster_cpu_limit_hard": `sum(kube_resourcequota{type="hard", resource="limits.cpu"}) by (resource, type)`, + "cluster_storage_request_hard": `sum(kube_resourcequota{type="hard", resource="requests.storage"}) by (resource, type)`, + "cluster_deployment_count_hard": `sum(kube_resourcequota{type="hard", resource="count/deployments.apps"}) by (resource, type)`, + "cluster_pod_count_hard": `sum(kube_resourcequota{type="hard", resource="count/pods"}) by (resource, type)`, + "cluster_statefulset_count_hard": `sum(kube_resourcequota{type="hard", resource="count/statefulsets.apps"}) by (resource, type)`, + "cluster_daemonset_count_hard": `sum(kube_resourcequota{type="hard", resource="count/daemonsets.apps"}) by (resource, type)`, + "cluster_secret_count_hard": `sum(kube_resourcequota{type="hard", resource="count/secrets"}) by (resource, type)`, + "cluster_service_count_hard": `sum(kube_resourcequota{type="hard", resource="count/services"}) by (resource, type)`, + "cluster_cpu_request_hard": `sum(kube_resourcequota{type="hard", resource="requests.cpu"}) by (resource, type)`, + "cluster_service_loadbalancer_hard": `sum(kube_resourcequota{type="hard", resource="services.loadbalancers"}) by (resource, type)`, + + "cluster_cronjob_count": `sum(kube_cronjob_labels)`, + "cluster_pvc_count": `sum(kube_persistentvolumeclaim_info)`, + "cluster_daemonset_count": `sum(kube_daemonset_labels)`, + "cluster_deployment_count": `sum(kube_deployment_labels)`, + "cluster_endpoint_count": `sum(kube_endpoint_labels)`, + "cluster_hpa_count": `sum(kube_hpa_labels)`, + "cluster_job_count": `sum(kube_job_labels)`, + "cluster_statefulset_count": `sum(kube_statefulset_labels)`, + "cluster_replicaset_count": `count(kube_replicaset_created)`, + "cluster_service_count": `sum(kube_service_info)`, + "cluster_secret_count": `sum(kube_secret_info)`, + "cluster_pv_count": `sum(kube_persistentvolume_labels)`, + + //node + "node_cpu_utilisation": "node:node_cpu_utilisation:avg1m", + "node_cpu_total": "node:node_num_cpu:sum", + "node_memory_utilisation": "node:node_memory_utilisation:", + "node_memory_bytes_available": "node:node_memory_bytes_available:sum", + "node_memory_bytes_total": "node:node_memory_bytes_total:sum", + // Node network utilisation (bytes received + bytes transmitted per second) + "node_net_utilisation": "node:node_net_utilisation:sum_irate", + // Node network bytes transmitted per second + "node_net_bytes_transmitted": "node:node_net_bytes_transmitted:sum_irate", + // Node network bytes received per second + "node_net_bytes_received": "node:node_net_bytes_received:sum_irate", + + // node:data_volume_iops_reads:sum{node=~"i-5xcldxos|i-6soe9zl1"} + "node_disk_read_iops": "node:data_volume_iops_reads:sum", + // node:data_volume_iops_writes:sum{node=~"i-5xcldxos|i-6soe9zl1"} + "node_disk_write_iops": "node:data_volume_iops_writes:sum", + // node:data_volume_throughput_bytes_read:sum{node=~"i-5xcldxos|i-6soe9zl1"} + "node_disk_read_throughput": "node:data_volume_throughput_bytes_read:sum", + // node:data_volume_throughput_bytes_written:sum{node=~"i-5xcldxos|i-6soe9zl1"} + "node_disk_write_throughput": "node:data_volume_throughput_bytes_written:sum", + + "node_disk_size_capacity": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + "node_disk_size_available": `sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + "node_disk_size_usage": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) -sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + "node_disk_size_utilisation": `sum by (node) (((node_filesystem_size{mountpoint="/", job="node-exporter"} - node_filesystem_avail{mountpoint="/", job="node-exporter"}) / node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + + "node_disk_inode_total": `node:disk_inodes_total:$1`, + "node_disk_inode_usage": `node:disk_inodes_total:$1 - node:disk_inodes_free:$1`, + "node_disk_inode_utilisation": `(1 - (node:disk_inodes_free:$1 / node:disk_inodes_total:$1))`, + + "node_pod_count": `count(kube_pod_info$1) by (node)`, + "node_pod_quota": `sum(kube_node_status_capacity_pods$1) by (node)`, + "node_pod_utilisation": `sum(kube_pod_info$1) by (node) / sum(kube_node_status_capacity_pods$1) by (node)`, + "node_pod_running_count": `count(kube_pod_info$1 unless on (pod) (kube_pod_status_phase{phase=~"Failed|Pending|Unknown|Succeeded"} > 0)) by (node)`, + "node_pod_succeeded_count": `count(kube_pod_info$1 unless on (pod) (kube_pod_status_phase{phase=~"Failed|Pending|Unknown|Running"} > 0)) by (node)`, + "node_pod_abnormal_count": `count(kube_pod_info$1 unless on (pod) (kube_pod_status_phase{phase=~"Succeeded|Running"} > 0)) by (node)`, + + // without log node: unless on(node) kube_node_labels{label_role="log"} + "node_cpu_usage": `node:node_cpu_utilisation:avg1m$1 * node:node_num_cpu:sum$1`, + "node_memory_bytes_usage": "node:node_memory_bytes_total:sum$1 - node:node_memory_bytes_available:sum$1", + + //namespace + "namespace_cpu_usage": `namespace:container_cpu_usage_seconds_total:sum_rate{namespace=~"$1"}`, + "namespace_memory_usage": `namespace:container_memory_usage_bytes:sum{namespace=~"$1"}`, + "namespace_memory_usage_wo_cache": `namespace:container_memory_usage_bytes_wo_cache:sum{namespace=~"$1"}`, + "namespace_net_bytes_transmitted": `sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m]))`, + "namespace_net_bytes_received": `sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m]))`, + "namespace_pod_count": `sum(kube_pod_status_phase{namespace=~"$1"}) by (namespace)`, + "namespace_pod_running_count": `sum(kube_pod_status_phase{phase="Running", namespace=~"$1"}) by (namespace)`, + "namespace_pod_succeeded_count": `sum(kube_pod_status_phase{phase="Succeeded", namespace=~"$1"}) by (namespace)`, + "namespace_pod_abnormal_count": `sum(kube_pod_status_phase{phase=~"Failed|Pending|Unknown", namespace=~"$1"}) by (namespace)`, + + "namespace_configmap_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/configmaps"}) by (namespace, resource, type)`, + "namespace_jobs_batch_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/jobs.batch"}) by (namespace, resource, type)`, + "namespace_roles_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/roles.rbac.authorization.k8s.io"}) by (namespace, resource, type)`, + "namespace_memory_limit_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="limits.memory"}) by (namespace, resource, type)`, + "namespace_pvc_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="persistentvolumeclaims"}) by (namespace, resource, type)`, + "namespace_memory_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.memory"}) by (namespace, resource, type)`, + "namespace_pvc_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/persistentvolumeclaims"}) by (namespace, resource, type)`, + "namespace_cronjobs_batch_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/cronjobs.batch"}) by (namespace, resource, type)`, + "namespace_ingresses_extensions_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/ingresses.extensions"}) by (namespace, resource, type)`, + "namespace_cpu_limit_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="limits.cpu"}) by (namespace, resource, type)`, + "namespace_storage_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.storage"}) by (namespace, resource, type)`, + "namespace_deployment_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/deployments.apps"}) by (namespace, resource, type)`, + "namespace_pod_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/pods"}) by (namespace, resource, type)`, + "namespace_statefulset_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/statefulsets.apps"}) by (namespace, resource, type)`, + "namespace_daemonset_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/daemonsets.apps"}) by (namespace, resource, type)`, + "namespace_secret_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/secrets"}) by (namespace, resource, type)`, + "namespace_service_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/services"}) by (namespace, resource, type)`, + "namespace_cpu_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.cpu"}) by (namespace, resource, type)`, + "namespace_service_loadbalancer_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="services.loadbalancers"}) by (namespace, resource, type)`, + + "namespace_configmap_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/configmaps"}) by (namespace, resource, type)`, + "namespace_jobs_batch_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/jobs.batch"}) by (namespace, resource, type)`, + "namespace_roles_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/roles.rbac.authorization.k8s.io"}) by (namespace, resource, type)`, + "namespace_memory_limit_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="limits.memory"}) by (namespace, resource, type)`, + "namespace_pvc_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="persistentvolumeclaims"}) by (namespace, resource, type)`, + "namespace_memory_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.memory"}) by (namespace, resource, type)`, + "namespace_pvc_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/persistentvolumeclaims"}) by (namespace, resource, type)`, + "namespace_cronjobs_batch_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/cronjobs.batch"}) by (namespace, resource, type)`, + "namespace_ingresses_extensions_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/ingresses.extensions"}) by (namespace, resource, type)`, + "namespace_cpu_limit_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="limits.cpu"}) by (namespace, resource, type)`, + "namespace_storage_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.storage"}) by (namespace, resource, type)`, + "namespace_deployment_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/deployments.apps"}) by (namespace, resource, type)`, + "namespace_pod_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/pods"}) by (namespace, resource, type)`, + "namespace_statefulset_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/statefulsets.apps"}) by (namespace, resource, type)`, + "namespace_daemonset_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/daemonsets.apps"}) by (namespace, resource, type)`, + "namespace_secret_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/secrets"}) by (namespace, resource, type)`, + "namespace_service_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/services"}) by (namespace, resource, type)`, + "namespace_cpu_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.cpu"}) by (namespace, resource, type)`, + "namespace_service_loadbalancer_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="services.loadbalancers"}) by (namespace, resource, type)`, + + "namespace_cronjob_count": `sum(kube_cronjob_labels{namespace=~"$1"}) by (namespace)`, + "namespace_pvc_count": `sum(kube_persistentvolumeclaim_info{namespace=~"$1"}) by (namespace)`, + "namespace_daemonset_count": `sum(kube_daemonset_labels{namespace=~"$1"}) by (namespace)`, + "namespace_deployment_count": `sum(kube_deployment_labels{namespace=~"$1"}) by (namespace)`, + "namespace_endpoint_count": `sum(kube_endpoint_labels{namespace=~"$1"}) by (namespace)`, + "namespace_hpa_count": `sum(kube_hpa_labels{namespace=~"$1"}) by (namespace)`, + "namespace_job_count": `sum(kube_job_labels{namespace=~"$1"}) by (namespace)`, + "namespace_statefulset_count": `sum(kube_statefulset_labels{namespace=~"$1"}) by (namespace)`, + "namespace_replicaset_count": `count(kube_replicaset_created{namespace=~"$1"}) by (namespace)`, + "namespace_service_count": `sum(kube_service_info{namespace=~"$1"}) by (namespace)`, + "namespace_secret_count": `sum(kube_secret_info{namespace=~"$1"}) by (namespace)`, + + // pod + "pod_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name="$2", image!=""}[5m])) by (namespace, pod_name)`, + "pod_memory_usage": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""}) by (namespace, pod_name)`, + "pod_memory_usage_wo_cache": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name="$2",image!=""}) by (namespace, pod_name)`, + "pod_net_bytes_transmitted": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[5m]))`, + "pod_net_bytes_received": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[5m]))`, + + "pod_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}[5m])) by (namespace, pod_name)`, + "pod_memory_usage_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`, + "pod_memory_usage_wo_cache_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`, + "pod_net_bytes_transmitted_all": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`, + "pod_net_bytes_received_all": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`, + + "pod_cpu_usage_node": `sum by (node, pod) (label_join(irate(container_cpu_usage_seconds_total{job="kubelet",pod_name=~"$2", image!=""}[5m]), "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, + "pod_memory_usage_node": `sum by (node, pod) (label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, + "pod_memory_usage_wo_cache_node": `sum by (node, pod) ((label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") - label_join(container_memory_cache{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name")) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, + + // container + "container_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name="$3"}[5m])) by (namespace, pod_name, container_name)`, + "container_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}[5m])) by (namespace, pod_name, container_name)`, + + "container_memory_usage_wo_cache": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name="$3"}`, + "container_memory_usage_wo_cache_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`, + + "container_memory_usage": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"}`, + "container_memory_usage_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`, + + // workspace + "workspace_cpu_usage": `sum(namespace:container_cpu_usage_seconds_total:sum_rate{namespace =~"$1"})`, + "workspace_memory_usage": `sum(namespace:container_memory_usage_bytes:sum{namespace =~"$1"})`, + "workspace_memory_usage_wo_cache": `sum(namespace:container_memory_usage_bytes_wo_cache:sum{namespace =~"$1"})`, + "workspace_net_bytes_transmitted": `sum(sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m])))`, + "workspace_net_bytes_received": `sum(sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m])))`, + "workspace_pod_count": `sum(kube_pod_status_phase{namespace=~"$1"})`, + "workspace_pod_running_count": `sum(kube_pod_status_phase{phase="Running", namespace=~"$1"})`, + "workspace_pod_succeeded_count": `sum(kube_pod_status_phase{phase="Succeeded", namespace=~"$1"})`, + "workspace_pod_abnormal_count": `sum(kube_pod_status_phase{phase=~"Failed|Pending|Unknown", namespace=~"$1"})`, + + "workspace_configmap_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/configmaps"}) by (resource, type)`, + "workspace_jobs_batch_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/jobs.batch"}) by (resource, type)`, + "workspace_roles_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`, + "workspace_memory_limit_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="limits.memory"}) by (resource, type)`, + "workspace_pvc_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="persistentvolumeclaims"}) by (resource, type)`, + "workspace_memory_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.memory"}) by (resource, type)`, + "workspace_pvc_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/persistentvolumeclaims"}) by (resource, type)`, + "workspace_cronjobs_batch_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/cronjobs.batch"}) by (resource, type)`, + "workspace_ingresses_extensions_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/ingresses.extensions"}) by (resource, type)`, + "workspace_cpu_limit_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="limits.cpu"}) by (resource, type)`, + "workspace_storage_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.storage"}) by (resource, type)`, + "workspace_deployment_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/deployments.apps"}) by (resource, type)`, + "workspace_pod_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/pods"}) by (resource, type)`, + "workspace_statefulset_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/statefulsets.apps"}) by (resource, type)`, + "workspace_daemonset_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/daemonsets.apps"}) by (resource, type)`, + "workspace_secret_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/secrets"}) by (resource, type)`, + "workspace_service_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/services"}) by (resource, type)`, + "workspace_cpu_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.cpu"}) by (resource, type)`, + "workspace_service_loadbalancer_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="services.loadbalancers"}) by (resource, type)`, + + "workspace_configmap_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/configmaps"}) by (resource, type)`, + "workspace_jobs_batch_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/jobs.batch"}) by (resource, type)`, + "workspace_roles_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`, + "workspace_memory_limit_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="limits.memory"}) by (resource, type)`, + "workspace_pvc_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="persistentvolumeclaims"}) by (resource, type)`, + "workspace_memory_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.memory"}) by (resource, type)`, + "workspace_pvc_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/persistentvolumeclaims"}) by (resource, type)`, + "workspace_cronjobs_batch_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/cronjobs.batch"}) by (resource, type)`, + "workspace_ingresses_extensions_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/ingresses.extensions"}) by (resource, type)`, + "workspace_cpu_limit_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="limits.cpu"}) by (resource, type)`, + "workspace_storage_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.storage"}) by (resource, type)`, + "workspace_deployment_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/deployments.apps"}) by (resource, type)`, + "workspace_pod_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/pods"}) by (resource, type)`, + "workspace_statefulset_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/statefulsets.apps"}) by (resource, type)`, + "workspace_daemonset_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/daemonsets.apps"}) by (resource, type)`, + "workspace_secret_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/secrets"}) by (resource, type)`, + "workspace_service_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/services"}) by (resource, type)`, + "workspace_cpu_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.cpu"}) by (resource, type)`, + "workspace_service_loadbalancer_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="services.loadbalancers"}) by (resource, type)`, + "workspace_ingresses_extensions_count": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/ingresses.extensions"}) by (resource, type)`, + + "workspace_cronjob_count": `sum(kube_cronjob_labels{namespace=~"$1"})`, + "workspace_pvc_count": `sum(kube_persistentvolumeclaim_info{namespace=~"$1"})`, + "workspace_daemonset_count": `sum(kube_daemonset_labels{namespace=~"$1"})`, + "workspace_deployment_count": `sum(kube_deployment_labels{namespace=~"$1"})`, + "workspace_endpoint_count": `sum(kube_endpoint_labels{namespace=~"$1"})`, + "workspace_hpa_count": `sum(kube_hpa_labels{namespace=~"$1"})`, + "workspace_job_count": `sum(kube_job_labels{namespace=~"$1"})`, + "workspace_statefulset_count": `sum(kube_statefulset_labels{namespace=~"$1"})`, + "workspace_replicaset_count": `count(kube_replicaset_created{namespace=~"$1"})`, + "workspace_service_count": `sum(kube_service_info{namespace=~"$1"})`, + "workspace_secret_count": `sum(kube_secret_info{namespace=~"$1"})`, +} diff --git a/pkg/models/metrics/metricsstruct.go b/pkg/models/metrics/metricsstruct.go deleted file mode 100755 index 4c77783bc..000000000 --- a/pkg/models/metrics/metricsstruct.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2018 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -type FormatedLevelMetric struct { - MetricsLevel string `json:"metrics_level"` - Results []FormatedMetric `json:"results"` -} - -type FormatedMetric struct { - MetricName string `json:"metric_name, omitempty"` - Status string `json:"status"` - Data FormatedMetricData `json:"data, omitempty"` -} - -type FormatedMetricData struct { - Result []map[string]interface{} `json:"result"` - ResultType string `json:"resultType"` -} - -type CommonMetricsResult struct { - Status string `json:"status"` - Data CommonMetricsData `json:"data"` -} - -type CommonMetricsData struct { - Result []CommonResultItem `json:"result"` - ResultType string `json:"resultType"` -} - -type CommonResultItem struct { - KubePodMetric KubePodMetric `json:"metric"` - Value interface{} `json:"value"` -} - -/** -"__name__": "kube_pod_info", -"created_by_kind": "\\u003cnone\\u003e", -"created_by_name": "\\u003cnone\\u003e", -"endpoint": "https-main", -"host_ip": "192.168.0.13", -"instance": "10.244.114.187:8443", -"job": "kube-state-metrics", -"namespace": "kube-system", -"node": "i-39p7faw6", -"pod": "cloud-controller-manager-i-39p7faw6", -"pod_ip": "192.168.0.13", -"service": "kube-state-metrics" -*/ -type KubePodMetric struct { - CreatedByKind string `json:"created_by_kind"` - CreatedByName string `json:"created_by_name"` - Namespace string `json:"namespace"` - Pod string `json:"pod"` -} diff --git a/pkg/models/metrics/util.go b/pkg/models/metrics/util.go new file mode 100644 index 000000000..78f62e46e --- /dev/null +++ b/pkg/models/metrics/util.go @@ -0,0 +1,268 @@ +/* +Copyright 2018 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "encoding/json" + "math" + "sort" + "strconv" + "unicode" + + "github.com/golang/glog" +) + +const ( + DefaultPageLimit = 5 + DefaultPage = 1 +) + +type FormatedMetricDataWrapper struct { + fmtMetricData FormatedMetricData + by func(p, q *map[string]interface{}) bool +} + +func (wrapper FormatedMetricDataWrapper) Len() int { + return len(wrapper.fmtMetricData.Result) +} + +func (wrapper FormatedMetricDataWrapper) Less(i, j int) bool { + return wrapper.by(&wrapper.fmtMetricData.Result[i], &wrapper.fmtMetricData.Result[j]) +} + +func (wrapper FormatedMetricDataWrapper) Swap(i, j int) { + wrapper.fmtMetricData.Result[i], wrapper.fmtMetricData.Result[j] = wrapper.fmtMetricData.Result[j], wrapper.fmtMetricData.Result[i] +} + +// sorted metric by ascending or descending order +func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelMetric, resourceType string) (*FormatedLevelMetric, int) { + var maxLength = 0 + for _, metricItem := range fmtLevelMetric.Results { + if metricItem.Data.ResultType == ResultTypeVector && metricItem.Status == MetricStatusSuccess { + if maxLength < len(metricItem.Data.Result) { + maxLength = len(metricItem.Data.Result) + } + } + } + + if sortMetricName == "" { + return fmtLevelMetric, maxLength + } + + // default sort type is descending order + if sortType == "" { + sortType = ResultSortTypeDesc + } + + var currentResourceMap = make(map[string]int) + + // indexMap store sorted index for each node/namespace/pod + var indexMap = make(map[string]int) + for _, metricItem := range fmtLevelMetric.Results { + if metricItem.Data.ResultType == ResultTypeVector && metricItem.Status == MetricStatusSuccess { + if metricItem.MetricName == sortMetricName { + if sortType == ResultSortTypeAsce { + // desc + sort.Sort(FormatedMetricDataWrapper{metricItem.Data, func(p, q *map[string]interface{}) bool { + value1 := (*p)[ResultItemValue].([]interface{}) + value2 := (*q)[ResultItemValue].([]interface{}) + v1, _ := strconv.ParseFloat(value1[len(value1)-1].(string), 64) + v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64) + return v1 < v2 + }}) + } else { + // desc + sort.Sort(FormatedMetricDataWrapper{metricItem.Data, func(p, q *map[string]interface{}) bool { + value1 := (*p)[ResultItemValue].([]interface{}) + value2 := (*q)[ResultItemValue].([]interface{}) + v1, _ := strconv.ParseFloat(value1[len(value1)-1].(string), 64) + v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64) + return v1 > v2 + }}) + } + + i := 0 + for _, r := range metricItem.Data.Result { + // for some reasons, 'metric' may not contain `resourceType` field + // example: {"metric":{},"value":[1541142931.731,"3"]} + k, exist := r[ResultItemMetric].(map[string]interface{})[resourceType] + if exist { + indexMap[k.(string)] = i + i = i + 1 + } + } + } + + // iterator all metric to find max metricItems length + for _, r := range metricItem.Data.Result { + k, ok := r[ResultItemMetric].(map[string]interface{})[resourceType] + if ok { + currentResourceMap[k.(string)] = 1 + } + } + + } + } + + // sort other metric + for i := 0; i < len(fmtLevelMetric.Results); i++ { + re := fmtLevelMetric.Results[i] + if re.MetricName != sortMetricName && re.Data.ResultType == ResultTypeVector && re.Status == MetricStatusSuccess { + sortedMetric := make([]map[string]interface{}, len(indexMap)) + noneSortedMetric := make([]map[string]interface{}, 0) + for j := 0; j < len(re.Data.Result); j++ { + r := re.Data.Result[j] + k, ok := r[ResultItemMetric].(map[string]interface{})[resourceType] + if ok { + index, exist := indexMap[k.(string)] + if exist { + sortedMetric[index] = r + } else { + noneSortedMetric = append(noneSortedMetric, r) + } + } + } + + sortedMetric = append(sortedMetric, noneSortedMetric...) + fmtLevelMetric.Results[i].Data.Result = sortedMetric + } + } + + return fmtLevelMetric, maxLength +} + +func Page(pageNum string, limitNum string, fmtLevelMetric *FormatedLevelMetric, maxLength int) interface{} { + // matrix type can not be sorted + for _, metricItem := range fmtLevelMetric.Results { + if metricItem.Data.ResultType != ResultTypeVector { + return fmtLevelMetric + } + } + + var page = DefaultPage + + if pageNum != "" { + p, err := strconv.Atoi(pageNum) + if err != nil { + glog.Errorln(err) + } else { + if p > 0 { + page = p + } + } + } else { + // the default mode is none paging + return fmtLevelMetric + } + + var limit = DefaultPageLimit + + if limitNum != "" { + l, err := strconv.Atoi(limitNum) + if err != nil { + glog.Errorln(err) + } else { + if l > 0 { + limit = l + } + } + } + + // the i page: [(page-1) * limit, (page) * limit - 1] + start := (page - 1) * limit + end := (page)*limit - 1 + + for i := 0; i < len(fmtLevelMetric.Results); i++ { + // only pageing when result type is `vector` and result status is `success` + if fmtLevelMetric.Results[i].Data.ResultType != ResultTypeVector || fmtLevelMetric.Results[i].Status != MetricStatusSuccess { + continue + } + resultLen := len(fmtLevelMetric.Results[i].Data.Result) + if start >= resultLen { + fmtLevelMetric.Results[i].Data.Result = nil + continue + } + if end >= resultLen { + end = resultLen - 1 + } + slice := fmtLevelMetric.Results[i].Data.Result[start : end+1] + fmtLevelMetric.Results[i].Data.Result = slice + } + + allPage := int(math.Ceil(float64(maxLength) / float64(limit))) + + return &PagedFormatedLevelMetric{ + Message: "paged", + TotalPage: allPage, + CurrentPage: page, + Metric: *fmtLevelMetric, + } +} + +// maybe this function is time consuming +func ReformatJson(metric string, metricsName string, needDelParams ...string) *FormatedMetric { + var formatMetric FormatedMetric + err := json.Unmarshal([]byte(metric), &formatMetric) + if err != nil { + glog.Errorln("Unmarshal metric json failed", err) + } + if formatMetric.MetricName == "" { + if metricsName != "" { + formatMetric.MetricName = metricsName + } + } + // retrive metrics success + if formatMetric.Status == MetricStatusSuccess { + result := formatMetric.Data.Result + for _, res := range result { + metric, exist := res[ResultItemMetric] + metricMap := metric.(map[string]interface{}) + if exist { + delete(metricMap, "__name__") + } + if len(needDelParams) > 0 { + for _, p := range needDelParams { + delete(metricMap, p) + } + } + } + } + + return &formatMetric +} + +func ReformatNodeStatusField(nodeMetric *FormatedMetric) *FormatedMetric { + metricCount := len(nodeMetric.Data.Result) + for i := 0; i < metricCount; i++ { + metric, exist := nodeMetric.Data.Result[i][ResultItemMetric] + if exist { + status, exist := metric.(map[string]interface{})[MetricStatus] + if exist { + status = UpperFirstLetter(status.(string)) + metric.(map[string]interface{})[MetricStatus] = status + } + } + } + return nodeMetric +} + +func UpperFirstLetter(str string) string { + for i, ch := range str { + return string(unicode.ToUpper(ch)) + str[i+1:] + } + return "" +} diff --git a/pkg/models/workspaces/workspaces.go b/pkg/models/workspaces/workspaces.go index a840c3fc8..8667a89cc 100644 --- a/pkg/models/workspaces/workspaces.go +++ b/pkg/models/workspaces/workspaces.go @@ -24,6 +24,8 @@ import ( clientV1 "k8s.io/client-go/listers/core/v1" "k8s.io/kubernetes/pkg/util/slice" + "github.com/golang/glog" + "kubesphere.io/kubesphere/pkg/client" "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/models/controllers" @@ -31,6 +33,10 @@ import ( ksErr "kubesphere.io/kubesphere/pkg/util/errors" ) +const ( + WorkspaceKey = "kubesphere.io/workspace" +) + var WorkSpaceRoles = []string{"admin", "operator", "viewer"} func UnBindDevopsProject(workspace string, devops string) error { @@ -1307,3 +1313,37 @@ func GetAllAccountNums() (int, error) { } return 0, errors.New("not found") } + +// get cluster organizations name which contains at least one namespace, +func GetAllOrgAndProjList() (map[string][]string, map[string]string, error) { + nsList, err := client.NewK8sClient().CoreV1().Namespaces().List(meta_v1.ListOptions{}) + if err != nil { + glog.Errorln(err) + return nil, nil, err + } + + var workspaceNamespaceMap = make(map[string][]string) + var namespaceWorkspaceMap = make(map[string]string) + + for _, item := range nsList.Items { + ws, exist := item.Labels[WorkspaceKey] + ns := item.Name + if exist { + if nsArray, exist := workspaceNamespaceMap[ws]; exist { + nsArray = append(nsArray, ns) + workspaceNamespaceMap[ws] = nsArray + } else { + var nsArray []string + nsArray = append(nsArray, ns) + workspaceNamespaceMap[ws] = nsArray + } + + namespaceWorkspaceMap[ns] = ws + } else { + // this namespace do not belong to any workspaces + namespaceWorkspaceMap[ns] = "" + } + } + + return workspaceNamespaceMap, namespaceWorkspaceMap, nil +}