Merge pull request #201 from carmanzhang/monitor
Refactor monitor module
This commit is contained in:
zryfish
@@ -13,8 +13,6 @@ limitations under the License.
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/emicklei/go-restful"
|
||||
"github.com/emicklei/go-restful-openapi"
|
||||
|
||||
@@ -23,94 +21,194 @@ import (
|
||||
"kubesphere.io/kubesphere/pkg/models/metrics"
|
||||
)
|
||||
|
||||
func (u MonitorResource) monitorPod(request *restful.Request, response *restful.Response) {
|
||||
podName := strings.Trim(request.PathParameter("pod_name"), " ")
|
||||
func (u Monitor) monitorPod(request *restful.Request, response *restful.Response) {
|
||||
requestParams := client.ParseMonitoringRequestParams(request)
|
||||
podName := requestParams.PodName
|
||||
metricName := requestParams.MetricsName
|
||||
if podName != "" {
|
||||
// single pod single metric
|
||||
metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ")
|
||||
res := metrics.MonitorPodSingleMetric(request, metricsName)
|
||||
queryType, params, nullRule := metrics.AssemblePodMetricRequestInfo(requestParams, metricName)
|
||||
var res *metrics.FormatedMetric
|
||||
if !nullRule {
|
||||
res = metrics.GetMetric(queryType, params, metricName)
|
||||
}
|
||||
response.WriteAsJson(res)
|
||||
|
||||
} else {
|
||||
// multiple pod multiple metric
|
||||
res := metrics.MonitorAllMetrics(request)
|
||||
response.WriteAsJson(res)
|
||||
// multiple
|
||||
rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelPod)
|
||||
// sorting
|
||||
sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelPodName)
|
||||
// paging
|
||||
pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)
|
||||
|
||||
response.WriteAsJson(pagedMetrics)
|
||||
}
|
||||
}
|
||||
|
||||
func (u MonitorResource) monitorContainer(request *restful.Request, response *restful.Response) {
|
||||
metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ")
|
||||
promql := metrics.MakeContainerPromQL(request)
|
||||
res := client.SendPrometheusRequest(request, promql)
|
||||
cleanedJson := metrics.ReformatJson(res, metricsName)
|
||||
response.WriteAsJson(cleanedJson)
|
||||
func (u Monitor) monitorContainer(request *restful.Request, response *restful.Response) {
|
||||
requestParams := client.ParseMonitoringRequestParams(request)
|
||||
res := metrics.MonitorContainer(requestParams)
|
||||
|
||||
response.WriteAsJson(res)
|
||||
}
|
||||
|
||||
func (u MonitorResource) monitorWorkload(request *restful.Request, response *restful.Response) {
|
||||
wlKind := request.PathParameter("workload_kind")
|
||||
if strings.Trim(wlKind, " ") == "" {
|
||||
func (u Monitor) monitorWorkload(request *restful.Request, response *restful.Response) {
|
||||
requestParams := client.ParseMonitoringRequestParams(request)
|
||||
wlKind := requestParams.WorkloadKind
|
||||
if wlKind == "" {
|
||||
// count all workloads figure
|
||||
//metricName := "workload_count"
|
||||
res := metrics.MonitorWorkloadCount(request)
|
||||
res := metrics.MonitorWorkloadCount(requestParams.NsName)
|
||||
response.WriteAsJson(res)
|
||||
} else {
|
||||
res := metrics.MonitorAllMetrics(request)
|
||||
res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkload)
|
||||
response.WriteAsJson(res)
|
||||
}
|
||||
}
|
||||
|
||||
// merge multiple metric: all-devops, all-roles, all-projects...this api is designed for admin
|
||||
func (u MonitorResource) monitorWorkspaceUserInfo(request *restful.Request, response *restful.Response) {
|
||||
res := metrics.MonitorWorkspaceUserInfo(request)
|
||||
func (u Monitor) monitorAllWorkspacesStatistics(request *restful.Request, response *restful.Response) {
|
||||
res := metrics.MonitorAllWorkspacesStatistics()
|
||||
response.WriteAsJson(res)
|
||||
}
|
||||
|
||||
// merge multiple metric: devops, roles, projects...
|
||||
func (u MonitorResource) monitorWorkspaceResourceLevelMetrics(request *restful.Request, response *restful.Response) {
|
||||
res := metrics.MonitorWorkspaceResourceLevelMetrics(request)
|
||||
func (u Monitor) monitorOneWorkspaceStatistics(request *restful.Request, response *restful.Response) {
|
||||
requestParams := client.ParseMonitoringRequestParams(request)
|
||||
wsName := requestParams.WsName
|
||||
res := metrics.MonitorOneWorkspaceStatistics(wsName)
|
||||
response.WriteAsJson(res)
|
||||
}
|
||||
|
||||
func (u MonitorResource) monitorWorkspacePodLevelMetrics(request *restful.Request, response *restful.Response) {
|
||||
res := metrics.MonitorAllMetrics(request)
|
||||
response.WriteAsJson(res)
|
||||
func (u Monitor) monitorAllWorkspaces(request *restful.Request, response *restful.Response) {
|
||||
|
||||
requestParams := client.ParseMonitoringRequestParams(request)
|
||||
|
||||
rawMetrics := metrics.MonitorAllWorkspaces(requestParams)
|
||||
// sorting
|
||||
sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelWorkspace)
|
||||
// paging
|
||||
pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)
|
||||
|
||||
response.WriteAsJson(pagedMetrics)
|
||||
}
|
||||
|
||||
func (u MonitorResource) monitorNamespace(request *restful.Request, response *restful.Response) {
|
||||
nsName := strings.Trim(request.PathParameter("ns_name"), " ")
|
||||
func (u Monitor) monitorOneWorkspace(request *restful.Request, response *restful.Response) {
|
||||
requestParams := client.ParseMonitoringRequestParams(request)
|
||||
|
||||
tp := requestParams.Tp
|
||||
if tp != "" {
|
||||
// multiple
|
||||
rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkspace)
|
||||
// sorting
|
||||
sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelNamespace)
|
||||
// paging
|
||||
pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)
|
||||
|
||||
response.WriteAsJson(pagedMetrics)
|
||||
|
||||
} else {
|
||||
res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkspace)
|
||||
response.WriteAsJson(res)
|
||||
}
|
||||
}
|
||||
|
||||
func (u Monitor) monitorNamespace(request *restful.Request, response *restful.Response) {
|
||||
requestParams := client.ParseMonitoringRequestParams(request)
|
||||
metricName := requestParams.MetricsName
|
||||
nsName := requestParams.NsName
|
||||
if nsName != "" {
|
||||
// single
|
||||
metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ")
|
||||
res := metrics.MonitorNamespaceSingleMetric(request, metricsName)
|
||||
queryType, params := metrics.AssembleNamespaceMetricRequestInfo(requestParams, metricName)
|
||||
res := metrics.GetMetric(queryType, params, metricName)
|
||||
response.WriteAsJson(res)
|
||||
} else {
|
||||
// multiple
|
||||
res := metrics.MonitorAllMetrics(request)
|
||||
response.WriteAsJson(res)
|
||||
rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelNamespace)
|
||||
// sorting
|
||||
sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelNamespace)
|
||||
// paging
|
||||
pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)
|
||||
|
||||
response.WriteAsJson(pagedMetrics)
|
||||
}
|
||||
}
|
||||
|
||||
func (u MonitorResource) monitorNodeorCluster(request *restful.Request, response *restful.Response) {
|
||||
metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ")
|
||||
//var res *metrics.FormatedMetric
|
||||
if metricsName != "" {
|
||||
func (u Monitor) monitorCluster(request *restful.Request, response *restful.Response) {
|
||||
requestParams := client.ParseMonitoringRequestParams(request)
|
||||
|
||||
metricName := requestParams.MetricsName
|
||||
if metricName != "" {
|
||||
// single
|
||||
res := metrics.MonitorNodeorClusterSingleMetric(request, metricsName)
|
||||
queryType, params := metrics.AssembleClusterMetricRequestInfo(requestParams, metricName)
|
||||
res := metrics.GetMetric(queryType, params, metricName)
|
||||
|
||||
if metricName == metrics.MetricNameWorkspaceAllProjectCount {
|
||||
res = metrics.MonitorWorkspaceNamespaceHistory(res)
|
||||
}
|
||||
|
||||
response.WriteAsJson(res)
|
||||
} else {
|
||||
// multiple
|
||||
res := metrics.MonitorAllMetrics(request)
|
||||
res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelCluster)
|
||||
response.WriteAsJson(res)
|
||||
}
|
||||
}
|
||||
|
||||
type MonitorResource struct {
|
||||
func (u Monitor) monitorNode(request *restful.Request, response *restful.Response) {
|
||||
requestParams := client.ParseMonitoringRequestParams(request)
|
||||
|
||||
metricName := requestParams.MetricsName
|
||||
if metricName != "" {
|
||||
// single
|
||||
queryType, params := metrics.AssembleNodeMetricRequestInfo(requestParams, metricName)
|
||||
res := metrics.GetMetric(queryType, params, metricName)
|
||||
nodeAddress := metrics.GetNodeAddressInfo()
|
||||
metrics.AddNodeAddressMetric(res, nodeAddress)
|
||||
response.WriteAsJson(res)
|
||||
} else {
|
||||
// multiple
|
||||
rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelNode)
|
||||
nodeAddress := metrics.GetNodeAddressInfo()
|
||||
|
||||
for i := 0; i < len(rawMetrics.Results); i++ {
|
||||
metrics.AddNodeAddressMetric(&rawMetrics.Results[i], nodeAddress)
|
||||
}
|
||||
|
||||
// sorting
|
||||
sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelNode)
|
||||
// paging
|
||||
pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)
|
||||
|
||||
response.WriteAsJson(pagedMetrics)
|
||||
}
|
||||
}
|
||||
|
||||
// k8s component(controller, scheduler, etcd) status
|
||||
func (u Monitor) monitorComponentStatus(request *restful.Request, response *restful.Response) {
|
||||
requestParams := client.ParseMonitoringRequestParams(request)
|
||||
|
||||
status := metrics.MonitorComponentStatus(requestParams)
|
||||
response.WriteAsJson(status)
|
||||
}
|
||||
|
||||
func (u Monitor) monitorEvents(request *restful.Request, response *restful.Response) {
|
||||
// k8s component healthy status
|
||||
requestParams := client.ParseMonitoringRequestParams(request)
|
||||
|
||||
nsFilter := requestParams.NsFilter
|
||||
events := metrics.MonitorEvents(nsFilter)
|
||||
response.WriteAsJson(events)
|
||||
}
|
||||
|
||||
type Monitor struct {
|
||||
}
|
||||
|
||||
func Register(ws *restful.WebService, subPath string) {
|
||||
tags := []string{"monitoring apis"}
|
||||
u := MonitorResource{}
|
||||
u := Monitor{}
|
||||
|
||||
ws.Route(ws.GET(subPath+"/clusters").To(u.monitorNodeorCluster).
|
||||
ws.Route(ws.GET(subPath+"/clusters").To(u.monitorCluster).
|
||||
Filter(route.RouteLogging).
|
||||
Doc("monitor cluster level metrics").
|
||||
Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("cluster_cpu_utilisation")).
|
||||
@@ -118,16 +216,20 @@ func Register(ws *restful.WebService, subPath string) {
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
|
||||
ws.Route(ws.GET(subPath+"/nodes").To(u.monitorNodeorCluster).
|
||||
ws.Route(ws.GET(subPath+"/nodes").To(u.monitorNode).
|
||||
Filter(route.RouteLogging).
|
||||
Doc("monitor nodes level metrics").
|
||||
Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("node_cpu_utilisation")).
|
||||
Param(ws.QueryParameter("nodes_filter", "node re2 expression filter").DataType("string").Required(false).DefaultValue("")).
|
||||
Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)).
|
||||
Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)).
|
||||
Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")).
|
||||
Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")).
|
||||
Metadata(restfulspec.KeyOpenAPITags, tags)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
|
||||
ws.Route(ws.GET(subPath+"/nodes/{node_id}").To(u.monitorNodeorCluster).
|
||||
ws.Route(ws.GET(subPath+"/nodes/{node_id}").To(u.monitorNode).
|
||||
Filter(route.RouteLogging).
|
||||
Doc("monitor specific node level metrics").
|
||||
Param(ws.PathParameter("node_id", "specific node").DataType("string").Required(true).DefaultValue("")).
|
||||
@@ -141,6 +243,10 @@ func Register(ws *restful.WebService, subPath string) {
|
||||
Doc("monitor namespaces level metrics").
|
||||
Param(ws.QueryParameter("namespaces_filter", "namespaces re2 expression filter").DataType("string").Required(false).DefaultValue("")).
|
||||
Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("namespace_memory_utilisation")).
|
||||
Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)).
|
||||
Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)).
|
||||
Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")).
|
||||
Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")).
|
||||
Metadata(restfulspec.KeyOpenAPITags, tags)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
@@ -158,8 +264,12 @@ func Register(ws *restful.WebService, subPath string) {
|
||||
Filter(route.RouteLogging).
|
||||
Doc("monitor pods level metrics").
|
||||
Param(ws.PathParameter("ns_name", "specific namespace").DataType("string").Required(true).DefaultValue("monitoring")).
|
||||
Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("")).
|
||||
Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("pod_memory_utilisation_wo_cache")).
|
||||
Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("")).
|
||||
Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)).
|
||||
Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)).
|
||||
Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")).
|
||||
Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")).
|
||||
Metadata(restfulspec.KeyOpenAPITags, tags)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
@@ -178,8 +288,12 @@ func Register(ws *restful.WebService, subPath string) {
|
||||
Filter(route.RouteLogging).
|
||||
Doc("monitor pods level metrics by nodeid").
|
||||
Param(ws.PathParameter("node_id", "specific node").DataType("string").Required(true).DefaultValue("i-k89a62il")).
|
||||
Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("openpitrix.*")).
|
||||
Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("pod_memory_utilisation_wo_cache")).
|
||||
Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("openpitrix.*")).
|
||||
Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)).
|
||||
Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)).
|
||||
Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")).
|
||||
Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")).
|
||||
Metadata(restfulspec.KeyOpenAPITags, tags)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
@@ -236,27 +350,64 @@ func Register(ws *restful.WebService, subPath string) {
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
|
||||
ws.Route(ws.GET(subPath+"/workspaces/{workspace_name}/pods").To(u.monitorWorkspacePodLevelMetrics).
|
||||
// merge multiple metric: devops, roles, projects...
|
||||
ws.Route(ws.GET(subPath+"/cluster_workspaces/{workspace_name}/_statistics").To(u.monitorOneWorkspaceStatistics).
|
||||
Filter(route.RouteLogging).
|
||||
Doc("monitor specific workspace level metrics").
|
||||
Param(ws.PathParameter("workspace_name", "workspace name").DataType("string").Required(true)).
|
||||
Metadata(restfulspec.KeyOpenAPITags, tags)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
|
||||
// list all namespace in this workspace by selected metrics
|
||||
ws.Route(ws.GET(subPath+"/cluster_workspaces/{workspace_name}").To(u.monitorOneWorkspace).
|
||||
Filter(route.RouteLogging).
|
||||
Doc("monitor workspaces level metrics").
|
||||
Param(ws.PathParameter("workspace_name", "workspace name").DataType("string").Required(true)).
|
||||
Param(ws.QueryParameter("namespaces_filter", "namespaces filter").DataType("string").Required(false).DefaultValue("k.*")).
|
||||
Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...").DataType("string").Required(false).DefaultValue("tenant_memory_utilisation_wo_cache")).
|
||||
Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("namespace_memory_utilisation_wo_cache")).
|
||||
Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)).
|
||||
Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)).
|
||||
Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")).
|
||||
Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")).
|
||||
Param(ws.QueryParameter("type", "page number").DataType("string").Required(false).DefaultValue("1")).
|
||||
Metadata(restfulspec.KeyOpenAPITags, tags)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
|
||||
ws.Route(ws.GET(subPath+"/workspaces/{workspace_name}").To(u.monitorWorkspaceResourceLevelMetrics).
|
||||
// metrics from system users or projects, merge multiple metric: all-devops, all-roles, all-projects...
|
||||
ws.Route(ws.GET(subPath+"/cluster_workspaces/_statistics").To(u.monitorAllWorkspacesStatistics).
|
||||
Filter(route.RouteLogging).
|
||||
Doc("monitor specific workspace level metrics").
|
||||
Param(ws.PathParameter("workspace_name", "workspace name").DataType("string").Required(true)).
|
||||
Metadata(restfulspec.KeyOpenAPITags, tags)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
|
||||
ws.Route(ws.GET(subPath+"/workspaces").To(u.monitorWorkspaceUserInfo).
|
||||
// metrics from prometheus
|
||||
ws.Route(ws.GET(subPath+"/cluster_workspaces").To(u.monitorAllWorkspaces).
|
||||
Filter(route.RouteLogging).
|
||||
Doc("monitor specific workspace level metrics").
|
||||
Doc("monitor workspaces level metrics").
|
||||
Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("workspace_memory_utilisation")).
|
||||
Param(ws.QueryParameter("workspaces_filter", "workspaces re2 expression filter").DataType("string").Required(false).DefaultValue(".*")).
|
||||
Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)).
|
||||
Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)).
|
||||
Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")).
|
||||
Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")).
|
||||
Metadata(restfulspec.KeyOpenAPITags, tags)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
|
||||
ws.Route(ws.GET(subPath+"/events").To(u.monitorEvents).
|
||||
Filter(route.RouteLogging).
|
||||
Doc("monitor k8s events").
|
||||
Param(ws.QueryParameter("namespaces_filter", "namespaces filter").DataType("string").Required(false).DefaultValue(".*")).
|
||||
Metadata(restfulspec.KeyOpenAPITags, tags)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
|
||||
ws.Route(ws.GET(subPath+"/components").To(u.monitorComponentStatus).
|
||||
Filter(route.RouteLogging).
|
||||
Doc("monitor k8s components status").
|
||||
Metadata(restfulspec.KeyOpenAPITags, tags)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
|
||||
@@ -16,13 +16,12 @@ import (
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/emicklei/go-restful"
|
||||
"github.com/golang/glog"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -33,13 +32,38 @@ const (
|
||||
PrometheusEndpointUrl = DefaultScheme + "://" + DefaultPrometheusService + ":" + DefaultPrometheusPort + PrometheusApiPath
|
||||
DefaultQueryStep = "10m"
|
||||
DefaultQueryTimeout = "30s"
|
||||
RangeQueryType = "query_range?"
|
||||
DefaultQueryType = "query?"
|
||||
)
|
||||
|
||||
type MonitoringRequestParams struct {
|
||||
Params url.Values
|
||||
QueryType string
|
||||
SortMetricName string
|
||||
SortType string
|
||||
PageNum string
|
||||
LimitNum string
|
||||
Tp string
|
||||
MetricsFilter string
|
||||
NodesFilter string
|
||||
WsFilter string
|
||||
NsFilter string
|
||||
PodsFilter string
|
||||
ContainersFilter string
|
||||
MetricsName string
|
||||
WorkloadName string
|
||||
NodeId string
|
||||
WsName string
|
||||
NsName string
|
||||
PodName string
|
||||
ContainerName string
|
||||
WorkloadKind string
|
||||
}
|
||||
|
||||
var client = &http.Client{}
|
||||
|
||||
func SendRequest(postfix string, params string) string {
|
||||
epurl := PrometheusEndpointUrl + postfix + params
|
||||
//glog.Info("monitoring epurl:>", epurl)
|
||||
func SendMonitoringRequest(queryType string, params string) string {
|
||||
epurl := PrometheusEndpointUrl + queryType + params
|
||||
response, err := client.Get(epurl)
|
||||
if err != nil {
|
||||
glog.Error(err)
|
||||
@@ -56,34 +80,57 @@ func SendRequest(postfix string, params string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func SendPrometheusRequest(request *restful.Request, recordingRule string) string {
|
||||
paramsMap, bol, err := ParseRequestHeader(request)
|
||||
if err != nil {
|
||||
glog.Error(err)
|
||||
return ""
|
||||
}
|
||||
func ParseMonitoringRequestParams(request *restful.Request) *MonitoringRequestParams {
|
||||
instantTime := strings.Trim(request.QueryParameter("time"), " ")
|
||||
start := strings.Trim(request.QueryParameter("start"), " ")
|
||||
end := strings.Trim(request.QueryParameter("end"), " ")
|
||||
step := strings.Trim(request.QueryParameter("step"), " ")
|
||||
timeout := strings.Trim(request.QueryParameter("timeout"), " ")
|
||||
|
||||
var res = ""
|
||||
var postfix = ""
|
||||
if bol {
|
||||
// range query
|
||||
postfix = "query_range?"
|
||||
} else {
|
||||
// query
|
||||
postfix = "query?"
|
||||
}
|
||||
paramsMap.Set("query", recordingRule)
|
||||
params := paramsMap.Encode()
|
||||
res = SendRequest(postfix, params)
|
||||
return res
|
||||
}
|
||||
sortMetricName := strings.Trim(request.QueryParameter("sort_metric"), " ")
|
||||
sortType := strings.Trim(request.QueryParameter("sort_type"), " ")
|
||||
pageNum := strings.Trim(request.QueryParameter("page"), " ")
|
||||
limitNum := strings.Trim(request.QueryParameter("limit"), " ")
|
||||
tp := strings.Trim(request.QueryParameter("type"), " ")
|
||||
|
||||
func ParseRequestHeader(request *restful.Request) (url.Values, bool, error) {
|
||||
instantTime := request.QueryParameter("time")
|
||||
start := request.QueryParameter("start")
|
||||
end := request.QueryParameter("end")
|
||||
step := request.QueryParameter("step")
|
||||
timeout := request.QueryParameter("timeout")
|
||||
metricsFilter := strings.Trim(request.QueryParameter("metrics_filter"), " ")
|
||||
nodesFilter := strings.Trim(request.QueryParameter("nodes_filter"), " ")
|
||||
wsFilter := strings.Trim(request.QueryParameter("workspaces_filter"), " ")
|
||||
nsFilter := strings.Trim(request.QueryParameter("namespaces_filter"), " ")
|
||||
podsFilter := strings.Trim(request.QueryParameter("pods_filter"), " ")
|
||||
containersFilter := strings.Trim(request.QueryParameter("containers_filter"), " ")
|
||||
|
||||
metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ")
|
||||
workloadName := strings.Trim(request.QueryParameter("workload_name"), " ")
|
||||
|
||||
nodeId := strings.Trim(request.PathParameter("node_id"), " ")
|
||||
wsName := strings.Trim(request.PathParameter("workspace_name"), " ")
|
||||
nsName := strings.Trim(request.PathParameter("ns_name"), " ")
|
||||
podName := strings.Trim(request.PathParameter("pod_name"), " ")
|
||||
containerName := strings.Trim(request.PathParameter("container_name"), " ")
|
||||
workloadKind := strings.Trim(request.PathParameter("workload_kind"), " ")
|
||||
|
||||
var requestParams = MonitoringRequestParams{
|
||||
SortMetricName: sortMetricName,
|
||||
SortType: sortType,
|
||||
PageNum: pageNum,
|
||||
LimitNum: limitNum,
|
||||
Tp: tp,
|
||||
MetricsFilter: metricsFilter,
|
||||
NodesFilter: nodesFilter,
|
||||
WsFilter: wsFilter,
|
||||
NsFilter: nsFilter,
|
||||
PodsFilter: podsFilter,
|
||||
ContainersFilter: containersFilter,
|
||||
MetricsName: metricsName,
|
||||
WorkloadName: workloadName,
|
||||
NodeId: nodeId,
|
||||
WsName: wsName,
|
||||
NsName: nsName,
|
||||
PodName: podName,
|
||||
ContainerName: containerName,
|
||||
WorkloadKind: workloadKind,
|
||||
}
|
||||
|
||||
if timeout == "" {
|
||||
timeout = DefaultQueryTimeout
|
||||
@@ -93,25 +140,35 @@ func ParseRequestHeader(request *restful.Request) (url.Values, bool, error) {
|
||||
}
|
||||
// Whether query or query_range request
|
||||
u := url.Values{}
|
||||
|
||||
if start != "" && end != "" {
|
||||
u.Set("start", convertTimeGranularity(start))
|
||||
u.Set("end", convertTimeGranularity(end))
|
||||
u.Set("step", step)
|
||||
u.Set("timeout", timeout)
|
||||
return u, true, nil
|
||||
requestParams.QueryType = RangeQueryType
|
||||
requestParams.Params = u
|
||||
return &requestParams
|
||||
}
|
||||
if instantTime != "" {
|
||||
u.Set("time", instantTime)
|
||||
u.Set("timeout", timeout)
|
||||
return u, false, nil
|
||||
requestParams.QueryType = DefaultQueryType
|
||||
requestParams.Params = u
|
||||
return &requestParams
|
||||
} else {
|
||||
//u.Set("time", strconv.FormatInt(int64(time.Now().Unix()), 10))
|
||||
u.Set("timeout", timeout)
|
||||
return u, false, nil
|
||||
requestParams.QueryType = DefaultQueryType
|
||||
requestParams.Params = u
|
||||
return &requestParams
|
||||
}
|
||||
|
||||
glog.Errorln("Parse request %s failed", u)
|
||||
return u, false, errors.Errorf("Parse request time range %s failed", u)
|
||||
requestParams.QueryType = DefaultQueryType
|
||||
requestParams.Params = u
|
||||
|
||||
return &requestParams
|
||||
}
|
||||
|
||||
func convertTimeGranularity(ts string) string {
|
||||
|
||||
993
pkg/models/metrics/metrics.go
Normal file
993
pkg/models/metrics/metrics.go
Normal file
@@ -0,0 +1,993 @@
|
||||
/*
|
||||
Copyright 2018 The KubeSphere Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"github.com/pkg/errors"
|
||||
"k8s.io/api/core/v1"
|
||||
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"kubesphere.io/kubesphere/pkg/client"
|
||||
"kubesphere.io/kubesphere/pkg/models"
|
||||
"kubesphere.io/kubesphere/pkg/models/workspaces"
|
||||
)
|
||||
|
||||
var nodeStatusDelLables = []string{"endpoint", "instance", "job", "namespace", "pod", "service"}
|
||||
|
||||
const (
|
||||
ChannelMaxCapacityWorkspaceMetric = 400
|
||||
ChannelMaxCapacity = 100
|
||||
)
|
||||
|
||||
type PagedFormatedLevelMetric struct {
|
||||
CurrentPage int `json:"page"`
|
||||
TotalPage int `json:"total_page"`
|
||||
Message string `json:"msg"`
|
||||
Metric FormatedLevelMetric `json:"metrics"`
|
||||
}
|
||||
|
||||
type FormatedLevelMetric struct {
|
||||
MetricsLevel string `json:"metrics_level"`
|
||||
Results []FormatedMetric `json:"results"`
|
||||
}
|
||||
|
||||
type FormatedMetric struct {
|
||||
MetricName string `json:"metric_name, omitempty"`
|
||||
Status string `json:"status"`
|
||||
Data FormatedMetricData `json:"data, omitempty"`
|
||||
}
|
||||
|
||||
type FormatedMetricData struct {
|
||||
Result []map[string]interface{} `json:"result"`
|
||||
ResultType string `json:"resultType"`
|
||||
}
|
||||
|
||||
type MetricResultValues []MetricResultValue
|
||||
|
||||
type MetricResultValue struct {
|
||||
timestamp float64
|
||||
value string
|
||||
}
|
||||
|
||||
type MetricItem struct {
|
||||
MetricLabel map[string]string `json:"metric"`
|
||||
Value []interface{} `json:"value"`
|
||||
}
|
||||
|
||||
type CommonMetricsResult struct {
|
||||
Status string `json:"status"`
|
||||
Data CommonMetricsData `json:"data"`
|
||||
}
|
||||
|
||||
type CommonMetricsData struct {
|
||||
Result []CommonResultItem `json:"result"`
|
||||
ResultType string `json:"resultType"`
|
||||
}
|
||||
|
||||
type CommonResultItem struct {
|
||||
KubePodMetric KubePodMetric `json:"metric"`
|
||||
Value interface{} `json:"value"`
|
||||
}
|
||||
|
||||
type KubePodMetric struct {
|
||||
CreatedByKind string `json:"created_by_kind"`
|
||||
CreatedByName string `json:"created_by_name"`
|
||||
Namespace string `json:"namespace"`
|
||||
Pod string `json:"pod"`
|
||||
}
|
||||
|
||||
type ComponentStatus struct {
|
||||
Name string `json:"metric_name,omitempty"`
|
||||
Namespace string `json:"namespace,omitempty"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
ComponentStatus []OneComponentStatus `json:"component"`
|
||||
}
|
||||
|
||||
type OneComponentStatus struct {
|
||||
// Valid value: "Healthy"
|
||||
Type string `json:"type"`
|
||||
// Valid values for "Healthy": "True", "False", or "Unknown".
|
||||
Status string `json:"status"`
|
||||
// Message about the condition for a component.
|
||||
Message string `json:"message,omitempty"`
|
||||
// Condition error code for a component.
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func getPodNameRegexInWorkload(res string) string {
|
||||
|
||||
data := []byte(res)
|
||||
var dat CommonMetricsResult
|
||||
jsonErr := json.Unmarshal(data, &dat)
|
||||
if jsonErr != nil {
|
||||
glog.Errorln("json parse failed", jsonErr)
|
||||
}
|
||||
var podNames []string
|
||||
for _, item := range dat.Data.Result {
|
||||
podName := item.KubePodMetric.Pod
|
||||
podNames = append(podNames, podName)
|
||||
}
|
||||
podNamesFilter := "^(" + strings.Join(podNames, "|") + ")$"
|
||||
return podNamesFilter
|
||||
}
|
||||
|
||||
func AssembleWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
|
||||
rule := MakeWorkloadRule(monitoringRequest.WorkloadKind, monitoringRequest.WorkloadName, monitoringRequest.NsName)
|
||||
paramValues := monitoringRequest.Params
|
||||
params := makeRequestParamString(rule, paramValues)
|
||||
|
||||
res := client.SendMonitoringRequest(client.DefaultQueryType, params)
|
||||
|
||||
podNamesFilter := getPodNameRegexInWorkload(res)
|
||||
|
||||
queryType := monitoringRequest.QueryType
|
||||
rule = MakePodPromQL(metricName, monitoringRequest.NsName, "", "", podNamesFilter)
|
||||
params = makeRequestParamString(rule, paramValues)
|
||||
|
||||
return queryType, params
|
||||
}
|
||||
|
||||
func AssemblePodMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) {
|
||||
queryType := monitoringRequest.QueryType
|
||||
|
||||
paramValues := monitoringRequest.Params
|
||||
|
||||
rule := MakePodPromQL(metricName, monitoringRequest.NsName, monitoringRequest.NodeId, monitoringRequest.PodName, monitoringRequest.PodsFilter)
|
||||
params := makeRequestParamString(rule, paramValues)
|
||||
return queryType, params, rule == ""
|
||||
}
|
||||
|
||||
func GetMetric(queryType, params, metricName string) *FormatedMetric {
|
||||
res := client.SendMonitoringRequest(queryType, params)
|
||||
formatedMetric := ReformatJson(res, metricName)
|
||||
return formatedMetric
|
||||
}
|
||||
|
||||
func GetNodeAddressInfo() *map[string][]v1.NodeAddress {
|
||||
nodeList, _ := client.NewK8sClient().CoreV1().Nodes().List(metaV1.ListOptions{})
|
||||
var nodeAddress = make(map[string][]v1.NodeAddress)
|
||||
for _, node := range nodeList.Items {
|
||||
nodeAddress[node.Name] = node.Status.Addresses
|
||||
}
|
||||
return &nodeAddress
|
||||
}
|
||||
|
||||
func AddNodeAddressMetric(nodeMetric *FormatedMetric, nodeAddress *map[string][]v1.NodeAddress) {
|
||||
|
||||
for i := 0; i < len(nodeMetric.Data.Result); i++ {
|
||||
metricDesc := nodeMetric.Data.Result[i][ResultItemMetric]
|
||||
metricDescMap := metricDesc.(map[string]interface{})
|
||||
if nodeId, exist := metricDescMap["node"]; exist {
|
||||
addr, exist := (*nodeAddress)[nodeId.(string)]
|
||||
if exist {
|
||||
metricDescMap["address"] = addr
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func MonitorContainer(monitoringRequest *client.MonitoringRequestParams) *FormatedMetric {
|
||||
queryType := monitoringRequest.QueryType
|
||||
|
||||
paramValues := monitoringRequest.Params
|
||||
rule := MakeContainerPromQL(monitoringRequest.NsName, monitoringRequest.PodName, monitoringRequest.ContainerName, monitoringRequest.MetricsName, monitoringRequest.ContainersFilter)
|
||||
params := makeRequestParamString(rule, paramValues)
|
||||
|
||||
res := GetMetric(queryType, params, monitoringRequest.MetricsName)
|
||||
return res
|
||||
}
|
||||
|
||||
func AssembleNamespaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
|
||||
queryType := monitoringRequest.QueryType
|
||||
|
||||
paramValues := monitoringRequest.Params
|
||||
rule := MakeNamespacePromQL(monitoringRequest.NsName, monitoringRequest.NsFilter, metricName)
|
||||
params := makeRequestParamString(rule, paramValues)
|
||||
|
||||
return queryType, params
|
||||
}
|
||||
|
||||
func AssembleWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) {
|
||||
nsFilter := "^(" + strings.Join(namespaceList, "|") + ")$"
|
||||
|
||||
queryType := monitoringRequest.QueryType
|
||||
|
||||
rule := MakeWorkspacePromQL(metricName, nsFilter)
|
||||
paramValues := monitoringRequest.Params
|
||||
params := makeRequestParamString(rule, paramValues)
|
||||
return queryType, params
|
||||
}
|
||||
|
||||
func makeRequestParamString(rule string, paramValues url.Values) string {
|
||||
|
||||
var values = make(url.Values)
|
||||
for key, v := range paramValues {
|
||||
values.Set(key, v[0])
|
||||
}
|
||||
|
||||
values.Set("query", rule)
|
||||
|
||||
params := values.Encode()
|
||||
|
||||
return params
|
||||
}
|
||||
|
||||
func filterNamespace(nsFilter string, namespaceList []string) []string {
|
||||
var newNSlist []string
|
||||
if nsFilter == "" {
|
||||
nsFilter = ".*"
|
||||
}
|
||||
for _, ns := range namespaceList {
|
||||
bol, _ := regexp.MatchString(nsFilter, ns)
|
||||
if bol {
|
||||
newNSlist = append(newNSlist, ns)
|
||||
}
|
||||
}
|
||||
return newNSlist
|
||||
}
|
||||
|
||||
func MonitorAllWorkspaces(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
|
||||
metricsFilter := monitoringRequest.MetricsFilter
|
||||
if strings.Trim(metricsFilter, " ") == "" {
|
||||
metricsFilter = ".*"
|
||||
}
|
||||
var filterMetricsName []string
|
||||
for _, metricName := range WorkspaceMetricsNames {
|
||||
bol, err := regexp.MatchString(metricsFilter, metricName)
|
||||
if err == nil && bol {
|
||||
filterMetricsName = append(filterMetricsName, metricName)
|
||||
}
|
||||
}
|
||||
|
||||
var wgAll sync.WaitGroup
|
||||
var wsAllch = make(chan *[]FormatedMetric, ChannelMaxCapacityWorkspaceMetric)
|
||||
|
||||
workspaceNamespaceMap, _, _ := workspaces.GetAllOrgAndProjList()
|
||||
for ws, _ := range workspaceNamespaceMap {
|
||||
bol, err := regexp.MatchString(monitoringRequest.WsFilter, ws)
|
||||
if err == nil && bol {
|
||||
// a workspace
|
||||
wgAll.Add(1)
|
||||
go collectWorkspaceMetric(monitoringRequest, ws, filterMetricsName, &wgAll, wsAllch)
|
||||
}
|
||||
}
|
||||
|
||||
wgAll.Wait()
|
||||
close(wsAllch)
|
||||
|
||||
fmtMetricMap := make(map[string]FormatedMetric)
|
||||
for oneWsMetric := range wsAllch {
|
||||
if oneWsMetric != nil {
|
||||
// aggregate workspace metric
|
||||
for _, metric := range *oneWsMetric {
|
||||
fm, exist := fmtMetricMap[metric.MetricName]
|
||||
if exist {
|
||||
if metric.Status == "error" {
|
||||
fm.Status = metric.Status
|
||||
}
|
||||
fm.Data.Result = append(fm.Data.Result, metric.Data.Result...)
|
||||
fmtMetricMap[metric.MetricName] = fm
|
||||
} else {
|
||||
fmtMetricMap[metric.MetricName] = metric
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var metricArray = make([]FormatedMetric, 0)
|
||||
for _, metric := range fmtMetricMap {
|
||||
metricArray = append(metricArray, metric)
|
||||
}
|
||||
|
||||
return &FormatedLevelMetric{
|
||||
MetricsLevel: MetricLevelClusterWorkspace,
|
||||
Results: metricArray,
|
||||
}
|
||||
}
|
||||
|
||||
func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, ws string, filterMetricsName []string, wgAll *sync.WaitGroup, wsAllch chan *[]FormatedMetric) {
|
||||
defer wgAll.Done()
|
||||
var wg sync.WaitGroup
|
||||
var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
|
||||
namespaceArray, err := workspaces.WorkspaceNamespaces(ws)
|
||||
if err != nil {
|
||||
glog.Errorln(err.Error())
|
||||
}
|
||||
// add by namespace
|
||||
for _, metricName := range filterMetricsName {
|
||||
wg.Add(1)
|
||||
go func(metricName string) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
glog.Errorln(err)
|
||||
}
|
||||
}()
|
||||
|
||||
queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName)
|
||||
ch <- GetMetric(queryType, params, metricName)
|
||||
|
||||
wg.Done()
|
||||
}(metricName)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
|
||||
var metricsArray []FormatedMetric
|
||||
for oneMetric := range ch {
|
||||
if oneMetric != nil {
|
||||
// add "workspace" filed to oneMetric `metric` field
|
||||
for i := 0; i < len(oneMetric.Data.Result); i++ {
|
||||
tmap := oneMetric.Data.Result[i]["metric"].(map[string]interface{})
|
||||
tmap[MetricLevelWorkspace] = ws
|
||||
oneMetric.Data.Result[i]["metric"] = tmap
|
||||
}
|
||||
metricsArray = append(metricsArray, *oneMetric)
|
||||
}
|
||||
}
|
||||
|
||||
wsAllch <- &metricsArray
|
||||
}
|
||||
|
||||
func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resourceType string) *FormatedLevelMetric {
|
||||
metricsFilter := monitoringRequest.MetricsFilter
|
||||
if metricsFilter == "" {
|
||||
metricsFilter = ".*"
|
||||
}
|
||||
|
||||
var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
switch resourceType {
|
||||
case MetricLevelCluster:
|
||||
{
|
||||
for _, metricName := range ClusterMetricsNames {
|
||||
bol, err := regexp.MatchString(metricsFilter, metricName)
|
||||
if err == nil && bol {
|
||||
wg.Add(1)
|
||||
go func(metricName string) {
|
||||
queryType, params := AssembleClusterMetricRequestInfo(monitoringRequest, metricName)
|
||||
clusterMetrics := GetMetric(queryType, params, metricName)
|
||||
|
||||
// for this special case, get namespace history which in a workspace by determining namespace label
|
||||
if metricName == MetricNameWorkspaceAllProjectCount {
|
||||
clusterMetrics = MonitorWorkspaceNamespaceHistory(clusterMetrics)
|
||||
}
|
||||
|
||||
ch <- clusterMetrics
|
||||
|
||||
wg.Done()
|
||||
}(metricName)
|
||||
}
|
||||
}
|
||||
}
|
||||
case MetricLevelNode:
|
||||
{
|
||||
for _, metricName := range NodeMetricsNames {
|
||||
bol, err := regexp.MatchString(metricsFilter, metricName)
|
||||
if err == nil && bol {
|
||||
wg.Add(1)
|
||||
go func(metricName string) {
|
||||
queryType, params := AssembleNodeMetricRequestInfo(monitoringRequest, metricName)
|
||||
ch <- GetMetric(queryType, params, metricName)
|
||||
wg.Done()
|
||||
}(metricName)
|
||||
}
|
||||
}
|
||||
}
|
||||
case MetricLevelWorkspace:
|
||||
{
|
||||
namespaceArray, err := workspaces.WorkspaceNamespaces(monitoringRequest.WsName)
|
||||
if err != nil {
|
||||
glog.Errorln(err.Error())
|
||||
}
|
||||
namespaceArray = filterNamespace(monitoringRequest.NsFilter, namespaceArray)
|
||||
|
||||
if monitoringRequest.Tp == "" {
|
||||
for _, metricName := range WorkspaceMetricsNames {
|
||||
bol, err := regexp.MatchString(metricsFilter, metricName)
|
||||
if err == nil && bol {
|
||||
wg.Add(1)
|
||||
go func(metricName string) {
|
||||
queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName)
|
||||
ch <- GetMetric(queryType, params, metricName)
|
||||
wg.Done()
|
||||
}(metricName)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for _, metricName := range NamespaceMetricsNames {
|
||||
bol, err := regexp.MatchString(metricsFilter, metricName)
|
||||
ns := "^(" + strings.Join(namespaceArray, "|") + ")$"
|
||||
monitoringRequest.NsFilter = ns
|
||||
if err == nil && bol {
|
||||
wg.Add(1)
|
||||
go func(metricName string) {
|
||||
queryType, params := AssembleNamespaceMetricRequestInfo(monitoringRequest, metricName)
|
||||
ch <- GetMetric(queryType, params, metricName)
|
||||
wg.Done()
|
||||
}(metricName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
case MetricLevelNamespace:
|
||||
{
|
||||
for _, metricName := range NamespaceMetricsNames {
|
||||
bol, err := regexp.MatchString(metricsFilter, metricName)
|
||||
if err == nil && bol {
|
||||
wg.Add(1)
|
||||
go func(metricName string) {
|
||||
queryType, params := AssembleNamespaceMetricRequestInfo(monitoringRequest, metricName)
|
||||
ch <- GetMetric(queryType, params, metricName)
|
||||
wg.Done()
|
||||
}(metricName)
|
||||
}
|
||||
}
|
||||
}
|
||||
case MetricLevelWorkload:
|
||||
{
|
||||
for _, metricName := range WorkloadMetricsNames {
|
||||
bol, err := regexp.MatchString(metricsFilter, metricName)
|
||||
if err == nil && bol {
|
||||
wg.Add(1)
|
||||
go func(metricName string) {
|
||||
metricName = strings.TrimLeft(metricName, "workload_")
|
||||
queryType, params := AssembleWorkloadMetricRequestInfo(monitoringRequest, metricName)
|
||||
ch <- GetMetric(queryType, params, metricName)
|
||||
wg.Done()
|
||||
}(metricName)
|
||||
}
|
||||
}
|
||||
}
|
||||
case MetricLevelPod:
|
||||
{
|
||||
for _, metricName := range PodMetricsNames {
|
||||
bol, err := regexp.MatchString(metricsFilter, metricName)
|
||||
if err == nil && bol {
|
||||
wg.Add(1)
|
||||
go func(metricName string) {
|
||||
queryType, params, nullRule := AssemblePodMetricRequestInfo(monitoringRequest, metricName)
|
||||
if !nullRule {
|
||||
ch <- GetMetric(queryType, params, metricName)
|
||||
} else {
|
||||
ch <- nil
|
||||
}
|
||||
wg.Done()
|
||||
}(metricName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
|
||||
var metricsArray []FormatedMetric
|
||||
|
||||
for oneMetric := range ch {
|
||||
if oneMetric != nil {
|
||||
metricsArray = append(metricsArray, *oneMetric)
|
||||
}
|
||||
}
|
||||
|
||||
return &FormatedLevelMetric{
|
||||
MetricsLevel: resourceType,
|
||||
Results: metricsArray,
|
||||
}
|
||||
}
|
||||
|
||||
func MonitorWorkspaceNamespaceHistory(metric *FormatedMetric) *FormatedMetric {
|
||||
resultType := metric.Data.ResultType
|
||||
//metric.Status
|
||||
metricName := metric.MetricName
|
||||
|
||||
for i := 0; i < len(metric.Data.Result); i++ {
|
||||
metricItem := metric.Data.Result[i]
|
||||
|
||||
if resultType == ResultTypeVector {
|
||||
timeAndValue, sure := metricItem[ResultItemValue].([]interface{})
|
||||
if !sure {
|
||||
return metric
|
||||
}
|
||||
metric := getNamespaceHistoryMetric(timeAndValue[0].(float64), metricName)
|
||||
|
||||
workspaceNamespaceCount := calcWorkspaceNamespace(metric)
|
||||
|
||||
timeAndValue[1] = fmt.Sprintf("%d", workspaceNamespaceCount)
|
||||
|
||||
} else if resultType == ResultTypeMatrix {
|
||||
|
||||
values, sure := metricItem[ResultItemValues].([]interface{})
|
||||
if !sure {
|
||||
return metric
|
||||
}
|
||||
|
||||
for _, valueItem := range values {
|
||||
timeAndValue, sure := valueItem.([]interface{})
|
||||
if !sure {
|
||||
return metric
|
||||
}
|
||||
|
||||
metric := getNamespaceHistoryMetric(timeAndValue[0].(float64), metricName)
|
||||
|
||||
workspaceNamespaceCount := calcWorkspaceNamespace(metric)
|
||||
|
||||
timeAndValue[1] = fmt.Sprintf("%d", workspaceNamespaceCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return metric
|
||||
}
|
||||
|
||||
func getNamespaceHistoryMetric(timestamp float64, metricName string) *FormatedMetric {
|
||||
var timeRelatedParams = make(url.Values)
|
||||
timeRelatedParams.Set("time", fmt.Sprintf("%f", timestamp))
|
||||
timeRelatedParams.Set("query", NamespaceLabelRule)
|
||||
params := timeRelatedParams.Encode()
|
||||
metric := GetMetric(client.DefaultQueryType, params, metricName)
|
||||
return metric
|
||||
}
|
||||
|
||||
// calculate all namespace which belong to workspaces
|
||||
func calcWorkspaceNamespace(metric *FormatedMetric) int {
|
||||
if metric.Status == "error" {
|
||||
glog.Errorf("failed when retrive namespace history, the metric is %v", metric.Data.Result)
|
||||
return 0
|
||||
}
|
||||
|
||||
var workspaceNamespaceCount = 0
|
||||
|
||||
for _, result := range metric.Data.Result {
|
||||
tmpMap := result[ResultItemMetric].(map[string]interface{})
|
||||
wsName, exist := tmpMap[WorkspaceJoinedKey]
|
||||
|
||||
if exist && wsName != "" {
|
||||
workspaceNamespaceCount += 1
|
||||
}
|
||||
}
|
||||
|
||||
return workspaceNamespaceCount
|
||||
}
|
||||
|
||||
func MonitorAllWorkspacesStatistics() *FormatedLevelMetric {
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
var metricsArray []FormatedMetric
|
||||
timestamp := time.Now().Unix()
|
||||
|
||||
var orgResultItem *FormatedMetric
|
||||
var devopsResultItem *FormatedMetric
|
||||
var clusterProjResultItem *FormatedMetric
|
||||
var workspaceProjResultItem *FormatedMetric
|
||||
var accountResultItem *FormatedMetric
|
||||
|
||||
wsMap, projs, errProj := workspaces.GetAllOrgAndProjList()
|
||||
|
||||
wg.Add(5)
|
||||
|
||||
go func() {
|
||||
orgNums, errOrg := workspaces.GetAllOrgNums()
|
||||
if errOrg != nil {
|
||||
glog.Errorln(errOrg.Error())
|
||||
}
|
||||
orgResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllOrganizationCount, WorkspaceResourceKindOrganization, orgNums, errOrg)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
go func() {
|
||||
devOpsProjectNums, errDevops := workspaces.GetAllDevOpsProjectsNums()
|
||||
if errDevops != nil {
|
||||
glog.Errorln(errDevops.Error())
|
||||
}
|
||||
devopsResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllDevopsCount, WorkspaceResourceKindDevops, devOpsProjectNums, errDevops)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
go func() {
|
||||
var projNums = 0
|
||||
for _, v := range wsMap {
|
||||
projNums += len(v)
|
||||
}
|
||||
if errProj != nil {
|
||||
glog.Errorln(errProj.Error())
|
||||
}
|
||||
workspaceProjResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllProjectCount, WorkspaceResourceKindNamespace, projNums, errProj)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
go func() {
|
||||
projNums := len(projs)
|
||||
if errProj != nil {
|
||||
glog.Errorln(errProj.Error())
|
||||
}
|
||||
clusterProjResultItem = getSpecificMetricItem(timestamp, MetricNameClusterAllProjectCount, WorkspaceResourceKindNamespace, projNums, errProj)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
go func() {
|
||||
actNums, errAct := workspaces.GetAllAccountNums()
|
||||
if errAct != nil {
|
||||
glog.Errorln(errAct.Error())
|
||||
}
|
||||
accountResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllAccountCount, WorkspaceResourceKindAccount, actNums, errAct)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
metricsArray = append(metricsArray, *orgResultItem, *devopsResultItem, *workspaceProjResultItem, *accountResultItem, *clusterProjResultItem)
|
||||
|
||||
return &FormatedLevelMetric{
|
||||
MetricsLevel: MetricLevelWorkspace,
|
||||
Results: metricsArray,
|
||||
}
|
||||
}
|
||||
|
||||
func MonitorOneWorkspaceStatistics(wsName string) *FormatedLevelMetric {
|
||||
|
||||
var nsMetrics *FormatedMetric
|
||||
var devopsMetrics *FormatedMetric
|
||||
var memberMetrics *FormatedMetric
|
||||
var roleMetrics *FormatedMetric
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(4)
|
||||
|
||||
var fMetricsArray []FormatedMetric
|
||||
timestamp := int64(time.Now().Unix())
|
||||
|
||||
go func() {
|
||||
// add namespaces(project) metric
|
||||
namespaces, errNs := workspaces.WorkspaceNamespaces(wsName)
|
||||
namespaces, noneExistentNs := getExistingNamespace(namespaces)
|
||||
if len(noneExistentNs) != 0 {
|
||||
nsStr := strings.Join(noneExistentNs, "|")
|
||||
errStr := "the namespaces " + nsStr + " do not exist"
|
||||
if errNs == nil {
|
||||
errNs = errors.New(errStr)
|
||||
} else {
|
||||
errNs = errors.New(errNs.Error() + "\t" + errStr)
|
||||
}
|
||||
}
|
||||
if errNs != nil {
|
||||
glog.Errorln(errNs.Error())
|
||||
}
|
||||
nsMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceNamespaceCount, WorkspaceResourceKindNamespace, len(namespaces), errNs)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
go func() {
|
||||
devOpsProjects, errDevOps := workspaces.GetDevOpsProjects(wsName)
|
||||
if errDevOps != nil {
|
||||
glog.Errorln(errDevOps.Error())
|
||||
}
|
||||
// add devops metric
|
||||
devopsMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceDevopsCount, WorkspaceResourceKindDevops, len(devOpsProjects), errDevOps)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
go func() {
|
||||
members, errMemb := workspaces.GetOrgMembers(wsName)
|
||||
if errMemb != nil {
|
||||
glog.Errorln(errMemb.Error())
|
||||
}
|
||||
// add member metric
|
||||
memberMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceMemberCount, WorkspaceResourceKindMember, len(members), errMemb)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
go func() {
|
||||
roles, errRole := workspaces.GetOrgRoles(wsName)
|
||||
if errRole != nil {
|
||||
glog.Errorln(errRole.Error())
|
||||
}
|
||||
// add role metric
|
||||
roleMetrics = getSpecificMetricItem(timestamp, MetricNameWorkspaceRoleCount, WorkspaceResourceKindRole, len(roles), errRole)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
fMetricsArray = append(fMetricsArray, *nsMetrics, *devopsMetrics, *memberMetrics, *roleMetrics)
|
||||
|
||||
return &FormatedLevelMetric{
|
||||
MetricsLevel: MetricLevelWorkspace,
|
||||
Results: fMetricsArray,
|
||||
}
|
||||
}
|
||||
|
||||
func getSpecificMetricItem(timestamp int64, metricName string, resource string, count int, err error, resourceType ...string) *FormatedMetric {
|
||||
var nsMetrics FormatedMetric
|
||||
nsMetrics.MetricName = metricName
|
||||
nsMetrics.Data.ResultType = ResultTypeVector
|
||||
resultItem := make(map[string]interface{})
|
||||
tmp := make(map[string]string)
|
||||
|
||||
if len(resourceType) > 0 {
|
||||
tmp[resourceType[0]] = resource
|
||||
} else {
|
||||
tmp[ResultItemMetricResource] = resource
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
nsMetrics.Status = MetricStatusSuccess
|
||||
} else {
|
||||
nsMetrics.Status = MetricStatusError
|
||||
resultItem["errormsg"] = err.Error()
|
||||
}
|
||||
|
||||
resultItem[ResultItemMetric] = tmp
|
||||
resultItem[ResultItemValue] = []interface{}{timestamp, count}
|
||||
nsMetrics.Data.Result = make([]map[string]interface{}, 1)
|
||||
nsMetrics.Data.Result[0] = resultItem
|
||||
return &nsMetrics
|
||||
}
|
||||
|
||||
// k8s component(controller, scheduler, etcd) status
|
||||
func MonitorComponentStatus(monitoringRequest *client.MonitoringRequestParams) *[]interface{} {
|
||||
componentList, err := client.NewK8sClient().CoreV1().ComponentStatuses().List(metaV1.ListOptions{})
|
||||
if err != nil {
|
||||
glog.Errorln(err.Error())
|
||||
}
|
||||
|
||||
var componentStatusList []*ComponentStatus
|
||||
for _, item := range componentList.Items {
|
||||
var status []OneComponentStatus
|
||||
for _, cond := range item.Conditions {
|
||||
status = append(status, OneComponentStatus{
|
||||
Type: string(cond.Type),
|
||||
Status: string(cond.Status),
|
||||
Message: cond.Message,
|
||||
Error: cond.Error,
|
||||
})
|
||||
}
|
||||
|
||||
componentStatusList = append(componentStatusList, &ComponentStatus{
|
||||
Name: item.Name,
|
||||
Namespace: item.Namespace,
|
||||
Labels: item.Labels,
|
||||
ComponentStatus: status,
|
||||
})
|
||||
}
|
||||
|
||||
// node status
|
||||
queryType := monitoringRequest.QueryType
|
||||
paramValues := monitoringRequest.Params
|
||||
paramValues.Set("query", NodeStatusRule)
|
||||
params := paramValues.Encode()
|
||||
res := client.SendMonitoringRequest(queryType, params)
|
||||
|
||||
nodeStatusMetric := ReformatJson(res, "node_status", nodeStatusDelLables...)
|
||||
nodeStatusMetric = ReformatNodeStatusField(nodeStatusMetric)
|
||||
|
||||
var normalNodes []string
|
||||
var abnormalNodes []string
|
||||
for _, result := range nodeStatusMetric.Data.Result {
|
||||
tmap := result[ResultItemMetric].(map[string]interface{})
|
||||
if tmap[MetricStatus].(string) == "false" {
|
||||
abnormalNodes = append(abnormalNodes, tmap[MetricLevelNode].(string))
|
||||
} else {
|
||||
normalNodes = append(normalNodes, tmap[MetricLevelNode].(string))
|
||||
}
|
||||
}
|
||||
|
||||
Components, err := models.GetAllComponentsStatus()
|
||||
|
||||
if err != nil {
|
||||
glog.Error(err.Error())
|
||||
}
|
||||
|
||||
var namspaceComponentHealthyMap = make(map[string]int)
|
||||
var namspaceComponentTotalMap = make(map[string]int)
|
||||
|
||||
for _, ns := range models.SYSTEM_NAMESPACES {
|
||||
nsStatus, exist := Components[ns]
|
||||
if exist {
|
||||
for _, nsStatusItem := range nsStatus.(map[string]interface{}) {
|
||||
component := nsStatusItem.(models.Component)
|
||||
namspaceComponentTotalMap[ns] += 1
|
||||
if component.HealthyBackends == component.TotalBackends {
|
||||
namspaceComponentHealthyMap[ns] += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
timestamp := int64(time.Now().Unix())
|
||||
|
||||
onlineMetricItems := makeMetricItems(timestamp, namspaceComponentHealthyMap, MetricLevelNamespace)
|
||||
metricItems := makeMetricItems(timestamp, namspaceComponentTotalMap, MetricLevelNamespace)
|
||||
|
||||
var assembleList []interface{}
|
||||
assembleList = append(assembleList, nodeStatusMetric)
|
||||
|
||||
for _, statusItem := range componentStatusList {
|
||||
assembleList = append(assembleList, statusItem)
|
||||
}
|
||||
|
||||
assembleList = append(assembleList, FormatedMetric{
|
||||
Data: FormatedMetricData{
|
||||
Result: *onlineMetricItems,
|
||||
ResultType: ResultTypeVector,
|
||||
},
|
||||
MetricName: MetricNameComponentOnLine,
|
||||
Status: MetricStatusSuccess,
|
||||
})
|
||||
|
||||
assembleList = append(assembleList, FormatedMetric{
|
||||
Data: FormatedMetricData{
|
||||
Result: *metricItems,
|
||||
ResultType: ResultTypeVector,
|
||||
},
|
||||
MetricName: MetricNameComponentLine,
|
||||
Status: MetricStatusSuccess,
|
||||
})
|
||||
|
||||
return &assembleList
|
||||
}
|
||||
|
||||
func makeMetricItems(timestamp int64, statusMap map[string]int, resourceType string) *[]map[string]interface{} {
|
||||
var metricItems []map[string]interface{}
|
||||
|
||||
for ns, count := range statusMap {
|
||||
metricItems = append(metricItems, map[string]interface{}{
|
||||
ResultItemMetric: map[string]string{resourceType: ns},
|
||||
ResultItemValue: []interface{}{timestamp, fmt.Sprintf("%d", count)},
|
||||
})
|
||||
}
|
||||
return &metricItems
|
||||
}
|
||||
|
||||
// monitor k8s event, there are two status: Normal, Warning
|
||||
func MonitorEvents(nsFilter string) *[]v1.Event {
|
||||
namespaceMap, err := getAllNamespace()
|
||||
if err != nil {
|
||||
glog.Errorln(err.Error())
|
||||
}
|
||||
var nsList = make([]string, 0)
|
||||
for ns, _ := range namespaceMap {
|
||||
nsList = append(nsList, ns)
|
||||
}
|
||||
|
||||
filterNS := filterNamespace(nsFilter, nsList)
|
||||
var eventsList = make([]v1.Event, 0)
|
||||
|
||||
for _, ns := range filterNS {
|
||||
events, err := client.NewK8sClient().CoreV1().Events(ns).List(metaV1.ListOptions{})
|
||||
if err != nil {
|
||||
glog.Errorln(err.Error())
|
||||
} else {
|
||||
for _, item := range events.Items {
|
||||
eventsList = append(eventsList, item)
|
||||
}
|
||||
}
|
||||
}
|
||||
return &eventsList
|
||||
}
|
||||
|
||||
func AssembleClusterMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
|
||||
queryType := monitoringRequest.QueryType
|
||||
paramValues := monitoringRequest.Params
|
||||
rule := MakeClusterRule(metricName)
|
||||
params := makeRequestParamString(rule, paramValues)
|
||||
|
||||
return queryType, params
|
||||
}
|
||||
|
||||
func AssembleNodeMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
|
||||
queryType := monitoringRequest.QueryType
|
||||
paramValues := monitoringRequest.Params
|
||||
rule := MakeNodeRule(monitoringRequest.NodeId, monitoringRequest.NodesFilter, metricName)
|
||||
params := makeRequestParamString(rule, paramValues)
|
||||
|
||||
return queryType, params
|
||||
}
|
||||
|
||||
func getExistingNamespace(namespaces []string) ([]string, []string) {
|
||||
namespaceMap, err := getAllNamespace()
|
||||
var existedNS []string
|
||||
var noneExistedNS []string
|
||||
if err != nil {
|
||||
return namespaces, nil
|
||||
}
|
||||
for _, ns := range namespaces {
|
||||
if _, exist := namespaceMap[ns]; exist {
|
||||
existedNS = append(existedNS, ns)
|
||||
} else {
|
||||
noneExistedNS = append(noneExistedNS, ns)
|
||||
}
|
||||
}
|
||||
return existedNS, noneExistedNS
|
||||
}
|
||||
|
||||
func getAllNamespace() (map[string]int, error) {
|
||||
k8sClient := client.NewK8sClient()
|
||||
nsList, err := k8sClient.CoreV1().Namespaces().List(metaV1.ListOptions{})
|
||||
if err != nil {
|
||||
glog.Errorln(err.Error())
|
||||
return nil, err
|
||||
}
|
||||
namespaceMap := make(map[string]int)
|
||||
for _, item := range nsList.Items {
|
||||
namespaceMap[item.Name] = 0
|
||||
}
|
||||
return namespaceMap, nil
|
||||
}
|
||||
|
||||
func MonitorWorkloadCount(namespace string) *FormatedMetric {
|
||||
quotaMetric, err := models.GetNamespaceQuota(namespace)
|
||||
fMetric := convertQuota2MetricStruct(quotaMetric)
|
||||
|
||||
// whether the namespace in request parameters exists?
|
||||
namespaceMap, err := getAllNamespace()
|
||||
|
||||
_, exist := namespaceMap[namespace]
|
||||
if err != nil {
|
||||
exist = true
|
||||
}
|
||||
|
||||
if !exist || err != nil {
|
||||
fMetric.Status = MetricStatusError
|
||||
fMetric.Data.ResultType = ""
|
||||
errInfo := make(map[string]interface{})
|
||||
if err != nil {
|
||||
errInfo["errormsg"] = err.Error()
|
||||
} else {
|
||||
errInfo["errormsg"] = "namespace " + namespace + " does not exist"
|
||||
}
|
||||
fMetric.Data.Result = []map[string]interface{}{errInfo}
|
||||
}
|
||||
|
||||
return fMetric
|
||||
}
|
||||
|
||||
func convertQuota2MetricStruct(quotaMetric *models.ResourceQuota) *FormatedMetric {
|
||||
var fMetric FormatedMetric
|
||||
fMetric.MetricName = MetricNameWorkloadCount
|
||||
fMetric.Status = MetricStatusSuccess
|
||||
fMetric.Data.ResultType = ResultTypeVector
|
||||
timestamp := int64(time.Now().Unix())
|
||||
var resultItems []map[string]interface{}
|
||||
|
||||
hardMap := make(map[string]string)
|
||||
for resourceName, v := range quotaMetric.Data.Hard {
|
||||
hardMap[resourceName.String()] = v.String()
|
||||
}
|
||||
|
||||
for resourceName, v := range quotaMetric.Data.Used {
|
||||
resultItem := make(map[string]interface{})
|
||||
tmp := make(map[string]string)
|
||||
tmp[ResultItemMetricResource] = resourceName.String()
|
||||
resultItem[ResultItemMetric] = tmp
|
||||
resultItem[ResultItemValue] = []interface{}{timestamp, hardMap[resourceName.String()], v.String()}
|
||||
resultItems = append(resultItems, resultItem)
|
||||
}
|
||||
|
||||
fMetric.Data.Result = resultItems
|
||||
return &fMetric
|
||||
}
|
||||
@@ -1,539 +0,0 @@
|
||||
/*
|
||||
Copyright 2018 The KubeSphere Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/emicklei/go-restful"
|
||||
"github.com/golang/glog"
|
||||
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"k8s.io/api/core/v1"
|
||||
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"sync"
|
||||
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
v12 "k8s.io/client-go/listers/core/v1"
|
||||
|
||||
"kubesphere.io/kubesphere/pkg/client"
|
||||
"kubesphere.io/kubesphere/pkg/models"
|
||||
"kubesphere.io/kubesphere/pkg/models/controllers"
|
||||
"kubesphere.io/kubesphere/pkg/models/workspaces"
|
||||
)
|
||||
|
||||
func getPodNameRegexInWorkload(request *restful.Request) string {
|
||||
promql := MakeWorkloadRule(request)
|
||||
res := client.SendPrometheusRequest(request, promql)
|
||||
data := []byte(res)
|
||||
var dat CommonMetricsResult
|
||||
jsonErr := json.Unmarshal(data, &dat)
|
||||
if jsonErr != nil {
|
||||
glog.Errorln("json parse failed", jsonErr)
|
||||
}
|
||||
var podNames []string
|
||||
for _, x := range dat.Data.Result {
|
||||
podName := x.KubePodMetric.Pod
|
||||
podNames = append(podNames, podName)
|
||||
}
|
||||
podNamesFilter := "^(" + strings.Join(podNames, "|") + ")$"
|
||||
return podNamesFilter
|
||||
}
|
||||
|
||||
func MonitorWorkloadSingleMetric(request *restful.Request, metricsName string) *FormatedMetric {
|
||||
nsName := strings.Trim(request.PathParameter("ns_name"), " ")
|
||||
podNamesFilter := getPodNameRegexInWorkload(request)
|
||||
newPromql := MakePodPromQL(request, []string{metricsName, nsName, "", "", podNamesFilter})
|
||||
podMetrics := client.SendPrometheusRequest(request, newPromql)
|
||||
cleanedJson := ReformatJson(podMetrics, metricsName)
|
||||
return cleanedJson
|
||||
}
|
||||
|
||||
func MonitorPodSingleMetric(request *restful.Request, metricsName string) *FormatedMetric {
|
||||
nsName := strings.Trim(request.PathParameter("ns_name"), " ")
|
||||
nodeID := strings.Trim(request.PathParameter("node_id"), " ")
|
||||
podName := strings.Trim(request.PathParameter("pod_name"), " ")
|
||||
podFilter := strings.Trim(request.QueryParameter("pods_filter"), " ")
|
||||
params := []string{metricsName, nsName, nodeID, podName, podFilter}
|
||||
promql := MakePodPromQL(request, params)
|
||||
if promql != "" {
|
||||
res := client.SendPrometheusRequest(request, promql)
|
||||
cleanedJson := ReformatJson(res, metricsName)
|
||||
return cleanedJson
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func MonitorNamespaceSingleMetric(request *restful.Request, metricsName string) *FormatedMetric {
|
||||
recordingRule := MakeNamespacePromQL(request, metricsName)
|
||||
res := client.SendPrometheusRequest(request, recordingRule)
|
||||
cleanedJson := ReformatJson(res, metricsName)
|
||||
return cleanedJson
|
||||
}
|
||||
|
||||
// maybe this function is time consuming
|
||||
func ReformatJson(metric string, metricsName string) *FormatedMetric {
|
||||
var formatMetric FormatedMetric
|
||||
err := json.Unmarshal([]byte(metric), &formatMetric)
|
||||
if err != nil {
|
||||
glog.Errorln("Unmarshal metric json failed", err)
|
||||
}
|
||||
if formatMetric.MetricName == "" {
|
||||
formatMetric.MetricName = metricsName
|
||||
}
|
||||
// retrive metrics success
|
||||
if formatMetric.Status == MetricStatusSuccess {
|
||||
result := formatMetric.Data.Result
|
||||
for _, res := range result {
|
||||
metric, ok := res[ResultItemMetric]
|
||||
me := metric.(map[string]interface{})
|
||||
if ok {
|
||||
delete(me, "__name__")
|
||||
}
|
||||
}
|
||||
}
|
||||
return &formatMetric
|
||||
}
|
||||
|
||||
func collectNodeorClusterMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) {
|
||||
metric := MonitorNodeorClusterSingleMetric(request, metricsName)
|
||||
ch <- metric
|
||||
}
|
||||
|
||||
func collectNamespaceMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) {
|
||||
metric := MonitorNamespaceSingleMetric(request, metricsName)
|
||||
ch <- metric
|
||||
}
|
||||
|
||||
func collectWorkloadMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) {
|
||||
metricsName = strings.TrimLeft(metricsName, "workload_")
|
||||
metric := MonitorWorkloadSingleMetric(request, metricsName)
|
||||
ch <- metric
|
||||
}
|
||||
|
||||
func collectWorkspaceMetrics(request *restful.Request, metricsName string, namespaceList []string, ch chan<- *FormatedMetric) {
|
||||
mertic := monitorWorkspaceSingleMertic(request, metricsName, namespaceList)
|
||||
ch <- mertic
|
||||
}
|
||||
|
||||
func collectPodMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) {
|
||||
metric := MonitorPodSingleMetric(request, metricsName)
|
||||
ch <- metric
|
||||
}
|
||||
|
||||
func monitorWorkspaceSingleMertic(request *restful.Request, metricsName string, namespaceList []string) *FormatedMetric {
|
||||
namespaceRe2 := "^(" + strings.Join(namespaceList, "|") + ")$"
|
||||
newpromql := MakeWorkspacePromQL(metricsName, namespaceRe2)
|
||||
podMetrics := client.SendPrometheusRequest(request, newpromql)
|
||||
cleanedJson := ReformatJson(podMetrics, metricsName)
|
||||
return cleanedJson
|
||||
}
|
||||
|
||||
func filterNamespace(request *restful.Request, namespaceList []string) []string {
|
||||
var newNSlist []string
|
||||
nsFilter := strings.Trim(request.QueryParameter("namespaces_filter"), " ")
|
||||
if nsFilter == "" {
|
||||
nsFilter = ".*"
|
||||
}
|
||||
for _, ns := range namespaceList {
|
||||
bol, _ := regexp.MatchString(nsFilter, ns)
|
||||
if bol {
|
||||
newNSlist = append(newNSlist, ns)
|
||||
}
|
||||
}
|
||||
return newNSlist
|
||||
}
|
||||
|
||||
func MonitorAllMetrics(request *restful.Request) FormatedLevelMetric {
|
||||
metricsName := strings.Trim(request.QueryParameter("metrics_filter"), " ")
|
||||
if metricsName == "" {
|
||||
metricsName = ".*"
|
||||
}
|
||||
path := request.SelectedRoutePath()
|
||||
sourceType := path[strings.LastIndex(path, "/")+1 : len(path)-1]
|
||||
if strings.Contains(path, MetricLevelWorkload) {
|
||||
sourceType = MetricLevelWorkload
|
||||
} else if strings.Contains(path, MetricLevelWorkspace) {
|
||||
sourceType = MetricLevelWorkspace
|
||||
}
|
||||
var ch = make(chan *FormatedMetric, 10)
|
||||
for _, metricName := range MetricsNames {
|
||||
bol, err := regexp.MatchString(metricsName, metricName)
|
||||
if !bol {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
glog.Errorln("regex match failed", err)
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(metricName, sourceType) {
|
||||
if sourceType == MetricLevelCluster || sourceType == MetricLevelNode {
|
||||
go collectNodeorClusterMetrics(request, metricName, ch)
|
||||
} else if sourceType == MetricLevelNamespace {
|
||||
go collectNamespaceMetrics(request, metricName, ch)
|
||||
} else if sourceType == MetricLevelPod {
|
||||
go collectPodMetrics(request, metricName, ch)
|
||||
} else if sourceType == MetricLevelWorkload {
|
||||
go collectWorkloadMetrics(request, metricName, ch)
|
||||
} else if sourceType == MetricLevelWorkspace {
|
||||
name := request.PathParameter("workspace_name")
|
||||
namespaces, err := workspaces.WorkspaceNamespaces(name)
|
||||
|
||||
if err != nil {
|
||||
glog.Errorln(err)
|
||||
}
|
||||
namespaces = filterNamespace(request, namespaces)
|
||||
go collectWorkspaceMetrics(request, metricName, namespaces, ch)
|
||||
}
|
||||
}
|
||||
}
|
||||
var metricsArray []FormatedMetric
|
||||
var tempJson *FormatedMetric
|
||||
for _, k := range MetricsNames {
|
||||
bol, err := regexp.MatchString(metricsName, k)
|
||||
if !bol {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
glog.Errorln("regex match failed")
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(k, sourceType) {
|
||||
tempJson = <-ch
|
||||
if tempJson != nil {
|
||||
metricsArray = append(metricsArray, *tempJson)
|
||||
}
|
||||
}
|
||||
}
|
||||
return FormatedLevelMetric{
|
||||
MetricsLevel: sourceType,
|
||||
Results: metricsArray,
|
||||
}
|
||||
}
|
||||
|
||||
func MonitorWorkspaceUserInfo(req *restful.Request) FormatedLevelMetric {
|
||||
|
||||
var metricsArray []FormatedMetric
|
||||
timestamp := time.Now().Unix()
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
var orgResultItem FormatedMetric
|
||||
var dvpResultItem FormatedMetric
|
||||
var projResultItem FormatedMetric
|
||||
var actResultItem FormatedMetric
|
||||
|
||||
wg.Add(4)
|
||||
|
||||
go func() {
|
||||
orgNums, errOrg := workspaces.GetAllOrgNums()
|
||||
orgResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllOrganizationCount, WorkspaceResourceKindOrganization, orgNums, errOrg)
|
||||
wg.Done()
|
||||
}()
|
||||
go func() {
|
||||
devOpsProjectNums, errDevops := workspaces.GetAllDevOpsProjectsNums()
|
||||
dvpResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllDevopsCount, WorkspaceResourceKindDevops, devOpsProjectNums, errDevops)
|
||||
wg.Done()
|
||||
}()
|
||||
go func() {
|
||||
projNums, errProj := workspaces.GetAllProjectNums()
|
||||
projResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllProjectCount, WorkspaceResourceKindNamespace, projNums, errProj)
|
||||
wg.Done()
|
||||
}()
|
||||
go func() {
|
||||
actNums, errAct := workspaces.GetAllAccountNums()
|
||||
actResultItem = getSpecificMetricItem(timestamp, MetricNameWorkspaceAllAccountCount, WorkspaceResourceKindAccount, actNums, errAct)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
metricsArray = append(metricsArray, orgResultItem, dvpResultItem, projResultItem, actResultItem)
|
||||
|
||||
return FormatedLevelMetric{
|
||||
MetricsLevel: MetricLevelWorkspace,
|
||||
Results: metricsArray,
|
||||
}
|
||||
}
|
||||
|
||||
//func getWorkspaceMetricItem(timestamp int64, namespaceNums int64, resourceName string, err error) FormatedMetric {
|
||||
// var fMetric FormatedMetric
|
||||
// fMetric.Data.ResultType = ResultTypeVector
|
||||
// fMetric.MetricName = MetricNameWorkspaceInfoCount
|
||||
// fMetric.Status = MetricStatusSuccess
|
||||
// if err != nil {
|
||||
// fMetric.Status = MetricStatusError
|
||||
// }
|
||||
// resultItem := make(map[string]interface{})
|
||||
// tmp := make(map[string]string)
|
||||
// tmp[ResultItemMetricResource] = resourceName
|
||||
// resultItem[ResultItemMetric] = tmp
|
||||
// resultItem[ResultItemValue] = []interface{}{timestamp, strconv.FormatInt(namespaceNums, 10)}
|
||||
// return fMetric
|
||||
//}
|
||||
|
||||
func MonitorWorkspaceResourceLevelMetrics(request *restful.Request) FormatedLevelMetric {
|
||||
wsName := request.PathParameter("workspace_name")
|
||||
namspaces, errNs := workspaces.WorkspaceNamespaces(wsName)
|
||||
|
||||
devOpsProjects, errDevOps := workspaces.GetDevOpsProjects(wsName)
|
||||
members, errMemb := workspaces.GetOrgMembers(wsName)
|
||||
roles, errRole := workspaces.GetOrgRoles(wsName)
|
||||
|
||||
var fMetricsArray []FormatedMetric
|
||||
timestamp := int64(time.Now().Unix())
|
||||
namespaces, noneExistentNs := getExistingNamespace(namspaces)
|
||||
if len(noneExistentNs) != 0 {
|
||||
nsStr := strings.Join(noneExistentNs, "|")
|
||||
errStr := "the namespaces " + nsStr + " do not exist"
|
||||
if errNs == nil {
|
||||
errNs = errors.New(errStr)
|
||||
} else {
|
||||
errNs = errors.New(errNs.Error() + "\t" + errStr)
|
||||
}
|
||||
}
|
||||
|
||||
// add namespaces(project) metric
|
||||
nsMetrics := getSpecificMetricItem(timestamp, MetricNameWorkspaceNamespaceCount, WorkspaceResourceKindNamespace, len(namespaces), errNs)
|
||||
// add devops metric
|
||||
devopsMetrics := getSpecificMetricItem(timestamp, MetricNameWorkspaceDevopsCount, WorkspaceResourceKindDevops, len(devOpsProjects), errDevOps)
|
||||
// add member metric
|
||||
memberMetrics := getSpecificMetricItem(timestamp, MetricNameWorkspaceMemberCount, WorkspaceResourceKindMember, len(members), errMemb)
|
||||
// add role metric
|
||||
roleMetrics := getSpecificMetricItem(timestamp, MetricNameWorkspaceRoleCount, WorkspaceResourceKindRole, len(roles), errRole)
|
||||
// add workloads count metric
|
||||
wlMetrics := getWorkspaceWorkloadCountMetrics(namespaces)
|
||||
// add pods count metric
|
||||
podsCountMetrics := getWorkspacePodsCountMetrics(request, namespaces)
|
||||
fMetricsArray = append(fMetricsArray, nsMetrics, devopsMetrics, memberMetrics, roleMetrics, wlMetrics, *podsCountMetrics)
|
||||
|
||||
return FormatedLevelMetric{
|
||||
MetricsLevel: MetricLevelWorkspace,
|
||||
Results: fMetricsArray,
|
||||
}
|
||||
}
|
||||
|
||||
func getWorkspacePodsCountMetrics(request *restful.Request, namespaces []string) *FormatedMetric {
|
||||
metricName := MetricNameNamespacePodCount
|
||||
var recordingRule = RulePromQLTmplMap[metricName]
|
||||
nsFilter := "^(" + strings.Join(namespaces, "|") + ")$"
|
||||
recordingRule = strings.Replace(recordingRule, "$1", nsFilter, -1)
|
||||
res := client.SendPrometheusRequest(request, recordingRule)
|
||||
cleanedJson := ReformatJson(res, metricName)
|
||||
return cleanedJson
|
||||
}
|
||||
|
||||
func getWorkspaceWorkloadCountMetrics(namespaces []string) FormatedMetric {
|
||||
var wlQuotaMetrics models.ResourceQuota
|
||||
wlQuotaMetrics.NameSpace = strings.Join(namespaces, "|")
|
||||
wlQuotaMetrics.Data.Used = make(v1.ResourceList, 1)
|
||||
wlQuotaMetrics.Data.Hard = make(v1.ResourceList, 1)
|
||||
for _, ns := range namespaces {
|
||||
quotaMetric, err := models.GetNamespaceQuota(ns)
|
||||
if err != nil {
|
||||
glog.Errorln(err)
|
||||
continue
|
||||
}
|
||||
// sum all resources used along namespaces
|
||||
quotaUsed := quotaMetric.Data.Used
|
||||
for resourceName, quantity := range quotaUsed {
|
||||
if _, ok := wlQuotaMetrics.Data.Used[resourceName]; ok {
|
||||
tmpQuantity := wlQuotaMetrics.Data.Used[v1.ResourceName(resourceName)]
|
||||
tmpQuantity.Add(quantity)
|
||||
wlQuotaMetrics.Data.Used[v1.ResourceName(resourceName)] = tmpQuantity
|
||||
} else {
|
||||
wlQuotaMetrics.Data.Used[v1.ResourceName(resourceName)] = quantity.DeepCopy()
|
||||
}
|
||||
}
|
||||
|
||||
// sum all resources hard along namespaces
|
||||
quotaHard := quotaMetric.Data.Hard
|
||||
for resourceName, quantity := range quotaHard {
|
||||
if _, ok := wlQuotaMetrics.Data.Hard[resourceName]; ok {
|
||||
tmpQuantity := wlQuotaMetrics.Data.Hard[v1.ResourceName(resourceName)]
|
||||
tmpQuantity.Add(quantity)
|
||||
wlQuotaMetrics.Data.Hard[v1.ResourceName(resourceName)] = tmpQuantity
|
||||
} else {
|
||||
wlQuotaMetrics.Data.Hard[v1.ResourceName(resourceName)] = quantity.DeepCopy()
|
||||
}
|
||||
}
|
||||
}
|
||||
wlMetrics := convertQuota2MetricStruct(&wlQuotaMetrics)
|
||||
return wlMetrics
|
||||
}
|
||||
|
||||
func getSpecificMetricItem(timestamp int64, metricName string, kind string, count int, err error) FormatedMetric {
|
||||
var nsMetrics FormatedMetric
|
||||
nsMetrics.MetricName = metricName
|
||||
nsMetrics.Data.ResultType = ResultTypeVector
|
||||
resultItem := make(map[string]interface{})
|
||||
tmp := make(map[string]string)
|
||||
tmp[ResultItemMetricResource] = kind
|
||||
if err == nil {
|
||||
nsMetrics.Status = MetricStatusSuccess
|
||||
} else {
|
||||
nsMetrics.Status = MetricStatusError
|
||||
resultItem["errorinfo"] = err.Error()
|
||||
}
|
||||
|
||||
resultItem[ResultItemMetric] = tmp
|
||||
resultItem[ResultItemValue] = []interface{}{timestamp, count}
|
||||
nsMetrics.Data.Result = make([]map[string]interface{}, 1)
|
||||
nsMetrics.Data.Result[0] = resultItem
|
||||
return nsMetrics
|
||||
}
|
||||
|
||||
func MonitorNodeorClusterSingleMetric(request *restful.Request, metricsName string) *FormatedMetric {
|
||||
// support cluster node statistic, include healthy nodes and unhealthy nodes
|
||||
var res string
|
||||
var fMetric FormatedMetric
|
||||
timestamp := int64(time.Now().Unix())
|
||||
|
||||
if metricsName == "cluster_node_online" {
|
||||
onlineNodes, _ := getNodeHealthyConditionMetric()
|
||||
fMetric = getSpecificMetricItem(timestamp, MetricNameClusterHealthyNodeCount, "node_count", len(onlineNodes), nil)
|
||||
} else if metricsName == "cluster_node_offline" {
|
||||
_, offlineNodes := getNodeHealthyConditionMetric()
|
||||
fMetric = getSpecificMetricItem(timestamp, MetricNameClusterUnhealthyNodeCount, "node_count", len(offlineNodes), nil)
|
||||
} else if metricsName == "cluster_node_total" {
|
||||
onlineNodes, offlineNodes := getNodeHealthyConditionMetric()
|
||||
fMetric = getSpecificMetricItem(timestamp, MetricNameClusterNodeCount, "node_count", len(onlineNodes)+len(offlineNodes), nil)
|
||||
} else {
|
||||
recordingRule := MakeNodeorClusterRule(request, metricsName)
|
||||
res = client.SendPrometheusRequest(request, recordingRule)
|
||||
fMetric = *ReformatJson(res, metricsName)
|
||||
}
|
||||
return &fMetric
|
||||
}
|
||||
|
||||
func getNodeHealthyConditionMetric() ([]string, []string) {
|
||||
nodeList, err := client.NewK8sClient().CoreV1().Nodes().List(metaV1.ListOptions{})
|
||||
if err != nil {
|
||||
glog.Errorln(err)
|
||||
return nil, nil
|
||||
}
|
||||
var onlineNodes []string
|
||||
var offlineNodes []string
|
||||
for _, node := range nodeList.Items {
|
||||
nodeName := node.Labels["kubernetes.io/hostname"]
|
||||
nodeRole := node.Labels["role"]
|
||||
bol := true
|
||||
for _, cond := range node.Status.Conditions {
|
||||
if cond.Type == "Ready" && cond.Status == "Unknown" {
|
||||
bol = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if nodeRole != "log" {
|
||||
if bol {
|
||||
// reachable node
|
||||
onlineNodes = append(onlineNodes, nodeName)
|
||||
} else {
|
||||
// unreachable node
|
||||
offlineNodes = append(offlineNodes, nodeName)
|
||||
}
|
||||
}
|
||||
}
|
||||
return onlineNodes, offlineNodes
|
||||
}
|
||||
|
||||
func getExistingNamespace(namespaces []string) ([]string, []string) {
|
||||
namespaceMap, err := getAllNamespace()
|
||||
var existedNs []string
|
||||
var noneExistedNs []string
|
||||
if err != nil {
|
||||
return namespaces, nil
|
||||
}
|
||||
for _, ns := range namespaces {
|
||||
if _, ok := namespaceMap[ns]; ok {
|
||||
existedNs = append(existedNs, ns)
|
||||
} else {
|
||||
noneExistedNs = append(noneExistedNs, ns)
|
||||
}
|
||||
}
|
||||
return existedNs, noneExistedNs
|
||||
}
|
||||
|
||||
func getAllNamespace() (map[string]int, error) {
|
||||
lister := controllers.ResourceControllers.Controllers[controllers.Namespaces].Lister().(v12.NamespaceLister)
|
||||
nsList, err := lister.List(labels.Everything())
|
||||
if err != nil {
|
||||
glog.Errorln(err)
|
||||
return nil, err
|
||||
}
|
||||
namespaceMap := make(map[string]int)
|
||||
for _, item := range nsList {
|
||||
namespaceMap[item.Name] = 0
|
||||
}
|
||||
return namespaceMap, nil
|
||||
}
|
||||
|
||||
func MonitorWorkloadCount(request *restful.Request) FormatedMetric {
|
||||
namespace := strings.Trim(request.PathParameter("ns_name"), " ")
|
||||
|
||||
quotaMetric, err := models.GetNamespaceQuota(namespace)
|
||||
fMetric := convertQuota2MetricStruct(quotaMetric)
|
||||
|
||||
// whether the namespace in request parameters exists?
|
||||
namespaceMap, e := getAllNamespace()
|
||||
_, ok := namespaceMap[namespace]
|
||||
if e != nil {
|
||||
ok = true
|
||||
}
|
||||
if !ok || err != nil {
|
||||
fMetric.Status = MetricStatusError
|
||||
fMetric.Data.ResultType = ""
|
||||
errInfo := make(map[string]interface{})
|
||||
if err != nil {
|
||||
errInfo["errormsg"] = err.Error()
|
||||
} else {
|
||||
errInfo["errormsg"] = "namespace " + namespace + " does not exist"
|
||||
}
|
||||
fMetric.Data.Result = []map[string]interface{}{errInfo}
|
||||
}
|
||||
|
||||
return fMetric
|
||||
}
|
||||
|
||||
func convertQuota2MetricStruct(quotaMetric *models.ResourceQuota) FormatedMetric {
|
||||
var fMetric FormatedMetric
|
||||
fMetric.MetricName = MetricNameWorkloadCount
|
||||
fMetric.Status = MetricStatusSuccess
|
||||
fMetric.Data.ResultType = ResultTypeVector
|
||||
timestamp := int64(time.Now().Unix())
|
||||
var resultItems []map[string]interface{}
|
||||
|
||||
hardMap := make(map[string]string)
|
||||
for resourceName, v := range quotaMetric.Data.Hard {
|
||||
hardMap[resourceName.String()] = v.String()
|
||||
}
|
||||
|
||||
for resourceName, v := range quotaMetric.Data.Used {
|
||||
resultItem := make(map[string]interface{})
|
||||
tmp := make(map[string]string)
|
||||
tmp[ResultItemMetricResource] = resourceName.String()
|
||||
resultItem[ResultItemMetric] = tmp
|
||||
resultItem[ResultItemValue] = []interface{}{timestamp, hardMap[resourceName.String()], v.String()}
|
||||
resultItems = append(resultItems, resultItem)
|
||||
}
|
||||
|
||||
fMetric.Data.Result = resultItems
|
||||
return fMetric
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
/*
|
||||
Copyright 2018 The KubeSphere Authors.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package metrics
|
||||
|
||||
const (
|
||||
ResultTypeVector = "vector"
|
||||
ResultTypeMatrix = "matrix"
|
||||
MetricStatusError = "error"
|
||||
MetricStatusSuccess = "success"
|
||||
ResultItemMetric = "metric"
|
||||
ResultItemMetricResource = "resource"
|
||||
ResultItemValue = "value"
|
||||
)
|
||||
|
||||
const (
|
||||
MetricNameWorkloadCount = "workload_count"
|
||||
MetricNameNamespacePodCount = "namespace_pod_count"
|
||||
|
||||
MetricNameWorkspaceAllOrganizationCount = "workspace_all_organization_count"
|
||||
MetricNameWorkspaceAllAccountCount = "workspace_all_account_count"
|
||||
MetricNameWorkspaceAllProjectCount = "workspace_all_project_count"
|
||||
MetricNameWorkspaceAllDevopsCount = "workspace_all_devops_project_count"
|
||||
|
||||
MetricNameWorkspaceNamespaceCount = "workspace_namespace_count"
|
||||
MetricNameWorkspaceDevopsCount = "workspace_devops_project_count"
|
||||
MetricNameWorkspaceMemberCount = "workspace_member_count"
|
||||
MetricNameWorkspaceRoleCount = "workspace_role_count"
|
||||
|
||||
MetricNameClusterHealthyNodeCount = "cluster_node_online"
|
||||
MetricNameClusterUnhealthyNodeCount = "cluster_node_offline"
|
||||
MetricNameClusterNodeCount = "cluster_node_total"
|
||||
)
|
||||
|
||||
const (
|
||||
WorkspaceResourceKindOrganization = "organization"
|
||||
WorkspaceResourceKindAccount = "account"
|
||||
WorkspaceResourceKindNamespace = "namespace"
|
||||
WorkspaceResourceKindDevops = "devops"
|
||||
WorkspaceResourceKindMember = "member"
|
||||
WorkspaceResourceKindRole = "role"
|
||||
)
|
||||
|
||||
const (
|
||||
MetricLevelCluster = "cluster"
|
||||
MetricLevelNode = "node"
|
||||
MetricLevelWorkspace = "workspace"
|
||||
MetricLevelNamespace = "namespace"
|
||||
MetricLevelPod = "pod"
|
||||
MetricLevelContainer = "container"
|
||||
MetricLevelWorkload = "workload"
|
||||
)
|
||||
|
||||
type MetricMap map[string]string
|
||||
|
||||
var MetricsNames = []string{
|
||||
"cluster_cpu_utilisation",
|
||||
"cluster_cpu_usage",
|
||||
"cluster_cpu_total",
|
||||
"cluster_memory_utilisation",
|
||||
"cluster_pod_count",
|
||||
"cluster_memory_bytes_available",
|
||||
"cluster_memory_bytes_total",
|
||||
"cluster_memory_bytes_usage",
|
||||
"cluster_net_utilisation",
|
||||
"cluster_net_bytes_transmitted",
|
||||
"cluster_net_bytes_received",
|
||||
"cluster_disk_read_iops",
|
||||
"cluster_disk_write_iops",
|
||||
"cluster_disk_read_throughput",
|
||||
"cluster_disk_write_throughput",
|
||||
"cluster_disk_size_usage",
|
||||
"cluster_disk_size_utilisation",
|
||||
"cluster_disk_size_capacity",
|
||||
"cluster_disk_size_available",
|
||||
"cluster_node_online",
|
||||
"cluster_node_offline",
|
||||
"cluster_node_total",
|
||||
|
||||
"node_cpu_utilisation",
|
||||
"node_cpu_total",
|
||||
"node_cpu_usage",
|
||||
"node_memory_utilisation",
|
||||
"node_memory_bytes_usage",
|
||||
"node_memory_bytes_available",
|
||||
"node_memory_bytes_total",
|
||||
"node_net_utilisation",
|
||||
"node_net_bytes_transmitted",
|
||||
"node_net_bytes_received",
|
||||
"node_disk_read_iops",
|
||||
"node_disk_write_iops",
|
||||
"node_disk_read_throughput",
|
||||
"node_disk_write_throughput",
|
||||
"node_disk_size_capacity",
|
||||
"node_disk_size_available",
|
||||
"node_disk_size_usage",
|
||||
"node_disk_size_utilisation",
|
||||
"node_pod_count",
|
||||
"node_pod_quota",
|
||||
|
||||
"namespace_cpu_usage",
|
||||
"namespace_memory_usage",
|
||||
"namespace_memory_usage_wo_cache",
|
||||
"namespace_net_bytes_transmitted",
|
||||
"namespace_net_bytes_received",
|
||||
"namespace_pod_count",
|
||||
|
||||
"pod_cpu_usage",
|
||||
"pod_memory_usage",
|
||||
"pod_memory_usage_wo_cache",
|
||||
"pod_net_bytes_transmitted",
|
||||
"pod_net_bytes_received",
|
||||
|
||||
"workload_pod_cpu_usage",
|
||||
"workload_pod_memory_usage",
|
||||
"workload_pod_memory_usage_wo_cache",
|
||||
"workload_pod_net_bytes_transmitted",
|
||||
"workload_pod_net_bytes_received",
|
||||
//"container_cpu_usage",
|
||||
//"container_memory_usage_wo_cache",
|
||||
//"container_memory_usage",
|
||||
|
||||
"workspace_cpu_usage",
|
||||
"workspace_memory_usage",
|
||||
"workspace_memory_usage_wo_cache",
|
||||
"workspace_net_bytes_transmitted",
|
||||
"workspace_net_bytes_received",
|
||||
"workspace_pod_count",
|
||||
}
|
||||
|
||||
var RulePromQLTmplMap = MetricMap{
|
||||
//cluster
|
||||
"cluster_cpu_utilisation": ":node_cpu_utilisation:avg1m",
|
||||
"cluster_cpu_usage": `sum (irate(container_cpu_usage_seconds_total{job="kubelet", image!=""}[5m]))`,
|
||||
"cluster_cpu_total": "sum(node:node_num_cpu:sum)",
|
||||
"cluster_memory_utilisation": ":node_memory_utilisation:",
|
||||
"cluster_pod_count": `count(kube_pod_info unless on(pod) kube_pod_completion_time unless on(node) kube_node_labels{label_role="log"})`,
|
||||
"cluster_memory_bytes_available": "sum(node:node_memory_bytes_available:sum)",
|
||||
"cluster_memory_bytes_total": "sum(node:node_memory_bytes_total:sum)",
|
||||
"cluster_memory_bytes_usage": "sum(node:node_memory_bytes_total:sum) - sum(node:node_memory_bytes_available:sum)",
|
||||
"cluster_net_utilisation": ":node_net_utilisation:sum_irate",
|
||||
"cluster_net_bytes_transmitted": "sum(node:node_net_bytes_transmitted:sum_irate)",
|
||||
"cluster_net_bytes_received": "sum(node:node_net_bytes_received:sum_irate)",
|
||||
"cluster_disk_read_iops": "sum(node:data_volume_iops_reads:sum)",
|
||||
"cluster_disk_write_iops": "sum(node:data_volume_iops_writes:sum)",
|
||||
"cluster_disk_read_throughput": "sum(node:data_volume_throughput_bytes_read:sum)",
|
||||
"cluster_disk_write_throughput": "sum(node:data_volume_throughput_bytes_written:sum)",
|
||||
"cluster_disk_size_usage": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`,
|
||||
"cluster_disk_size_utilisation": `(sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))) / sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`,
|
||||
"cluster_disk_size_capacity": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`,
|
||||
"cluster_disk_size_available": `sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`,
|
||||
|
||||
//node
|
||||
"node_cpu_utilisation": "node:node_cpu_utilisation:avg1m",
|
||||
"node_cpu_total": "node:node_num_cpu:sum",
|
||||
"node_memory_utilisation": "node:node_memory_utilisation:",
|
||||
"node_memory_bytes_available": "node:node_memory_bytes_available:sum",
|
||||
"node_memory_bytes_total": "node:node_memory_bytes_total:sum",
|
||||
// Node network utilisation (bytes received + bytes transmitted per second)
|
||||
"node_net_utilisation": "node:node_net_utilisation:sum_irate",
|
||||
// Node network bytes transmitted per second
|
||||
"node_net_bytes_transmitted": "node:node_net_bytes_transmitted:sum_irate",
|
||||
// Node network bytes received per second
|
||||
"node_net_bytes_received": "node:node_net_bytes_received:sum_irate",
|
||||
|
||||
// node:data_volume_iops_reads:sum{node=~"i-5xcldxos|i-6soe9zl1"}
|
||||
"node_disk_read_iops": "node:data_volume_iops_reads:sum",
|
||||
// node:data_volume_iops_writes:sum{node=~"i-5xcldxos|i-6soe9zl1"}
|
||||
"node_disk_write_iops": "node:data_volume_iops_writes:sum",
|
||||
// node:data_volume_throughput_bytes_read:sum{node=~"i-5xcldxos|i-6soe9zl1"}
|
||||
"node_disk_read_throughput": "node:data_volume_throughput_bytes_read:sum",
|
||||
// node:data_volume_throughput_bytes_written:sum{node=~"i-5xcldxos|i-6soe9zl1"}
|
||||
"node_disk_write_throughput": "node:data_volume_throughput_bytes_written:sum",
|
||||
|
||||
"node_disk_size_capacity": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`,
|
||||
"node_disk_size_available": `sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`,
|
||||
"node_disk_size_usage": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) -sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`,
|
||||
"node_disk_size_utilisation": `sum by (node) (((node_filesystem_size{mountpoint="/", job="node-exporter"} - node_filesystem_avail{mountpoint="/", job="node-exporter"}) / node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`,
|
||||
"node_pod_count": `count(kube_pod_info$1 unless on(pod) kube_pod_completion_time) by (node)`,
|
||||
// without log node: unless on(node) kube_node_labels{label_role="log"}
|
||||
"node_pod_quota": `sum(kube_node_status_capacity_pods$1) by (node)`,
|
||||
"node_cpu_usage": `sum by (node) (label_join(irate(container_cpu_usage_seconds_total{job="kubelet", image!=""}[5m]), "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`,
|
||||
"node_memory_bytes_usage": "node:node_memory_bytes_total:sum$1 - node:node_memory_bytes_available:sum$1",
|
||||
|
||||
//namespace
|
||||
"namespace_cpu_usage": `namespace:container_cpu_usage_seconds_total:sum_rate{namespace=~"$1"}`,
|
||||
"namespace_memory_usage": `namespace:container_memory_usage_bytes:sum{namespace=~"$1"}`,
|
||||
"namespace_memory_usage_wo_cache": `namespace:container_memory_usage_bytes_wo_cache:sum{namespace=~"$1"}`,
|
||||
"namespace_net_bytes_transmitted": `sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m]))`,
|
||||
"namespace_net_bytes_received": `sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m]))`,
|
||||
"namespace_pod_count": `count(kube_pod_info{namespace=~"$1"} unless on(pod) kube_pod_completion_time) by (namespace)`,
|
||||
|
||||
// pod
|
||||
"pod_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name="$2", image!=""}[5m])) by (namespace, pod_name)`,
|
||||
"pod_memory_usage": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""}) by (namespace, pod_name)`,
|
||||
"pod_memory_usage_wo_cache": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name="$2",image!=""}) by (namespace, pod_name)`,
|
||||
"pod_net_bytes_transmitted": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[5m]))`,
|
||||
"pod_net_bytes_received": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[5m]))`,
|
||||
|
||||
"pod_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}[5m])) by (namespace, pod_name)`,
|
||||
"pod_memory_usage_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`,
|
||||
"pod_memory_usage_wo_cache_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`,
|
||||
"pod_net_bytes_transmitted_all": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`,
|
||||
"pod_net_bytes_received_all": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`,
|
||||
|
||||
"pod_cpu_usage_node": `sum by (node, pod) (label_join(irate(container_cpu_usage_seconds_total{job="kubelet",pod_name=~"$2", image!=""}[5m]), "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`,
|
||||
"pod_memory_usage_node": `sum by (node, pod) (label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`,
|
||||
"pod_memory_usage_wo_cache_node": `sum by (node, pod) ((label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") - label_join(container_memory_cache{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name")) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`,
|
||||
|
||||
// container
|
||||
"container_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name="$3"}[5m])) by (namespace, pod_name, container_name)`,
|
||||
"container_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}[5m])) by (namespace, pod_name, container_name)`,
|
||||
|
||||
"container_memory_usage_wo_cache": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name="$3"}`,
|
||||
"container_memory_usage_wo_cache_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`,
|
||||
"container_memory_usage": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"}`,
|
||||
"container_memory_usage_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`,
|
||||
|
||||
// enterprise
|
||||
"workspace_cpu_usage": `sum(namespace:container_cpu_usage_seconds_total:sum_rate{namespace =~"$1"})`,
|
||||
"workspace_memory_usage": `sum(namespace:container_memory_usage_bytes:sum{namespace =~"$1"})`,
|
||||
"workspace_memory_usage_wo_cache": `sum(namespace:container_memory_usage_bytes_wo_cache:sum{namespace =~"$1"})`,
|
||||
"workspace_net_bytes_transmitted": `sum(sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m])))`,
|
||||
"workspace_net_bytes_received": `sum(sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m])))`,
|
||||
"workspace_pod_count": `sum(count(kube_pod_info{namespace=~"$1"} unless on(pod) kube_pod_completion_time) by (namespace))`,
|
||||
}
|
||||
@@ -15,142 +15,119 @@ package metrics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/emicklei/go-restful"
|
||||
)
|
||||
|
||||
func MakeWorkloadRule(request *restful.Request) string {
|
||||
// kube_pod_info{created_by_kind="DaemonSet",created_by_name="fluent-bit",endpoint="https-main",
|
||||
// host_ip="192.168.0.14",instance="10.244.114.187:8443",job="kube-state-metrics",
|
||||
// namespace="kube-system",node="i-k89a62il",pod="fluent-bit-l5vxr",
|
||||
// pod_ip="10.244.114.175",service="kube-state-metrics"}
|
||||
rule := `kube_pod_info{created_by_kind="$1",created_by_name=$2,namespace="$3"}`
|
||||
kind := strings.Trim(request.PathParameter("workload_kind"), " ")
|
||||
name := strings.Trim(request.QueryParameter("workload_name"), " ")
|
||||
namespace := strings.Trim(request.PathParameter("ns_name"), " ")
|
||||
func MakeWorkloadRule(wkKind, wkName, namespace string) string {
|
||||
var rule = PodInfoRule
|
||||
if namespace == "" {
|
||||
namespace = ".*"
|
||||
}
|
||||
// alertnatives values: Deployment StatefulSet ReplicaSet DaemonSet
|
||||
wkKind = strings.ToLower(wkKind)
|
||||
|
||||
// kind alertnatives values: Deployment StatefulSet ReplicaSet DaemonSet
|
||||
kind = strings.ToLower(kind)
|
||||
|
||||
switch kind {
|
||||
switch wkKind {
|
||||
case "deployment":
|
||||
kind = "ReplicaSet"
|
||||
if name != "" {
|
||||
name = "~\"" + name + ".*\""
|
||||
wkKind = ReplicaSet
|
||||
if wkName != "" {
|
||||
wkName = "~\"" + wkName + ".*\""
|
||||
} else {
|
||||
name = "~\".*\""
|
||||
wkName = "~\".*\""
|
||||
}
|
||||
rule = strings.Replace(rule, "$1", kind, -1)
|
||||
rule = strings.Replace(rule, "$2", name, -1)
|
||||
rule = strings.Replace(rule, "$1", wkKind, -1)
|
||||
rule = strings.Replace(rule, "$2", wkName, -1)
|
||||
rule = strings.Replace(rule, "$3", namespace, -1)
|
||||
return rule
|
||||
case "replicaset":
|
||||
kind = "ReplicaSet"
|
||||
wkKind = ReplicaSet
|
||||
case "statefulset":
|
||||
kind = "StatefulSet"
|
||||
wkKind = StatefulSet
|
||||
case "daemonset":
|
||||
kind = "DaemonSet"
|
||||
wkKind = DaemonSet
|
||||
}
|
||||
|
||||
if name == "" {
|
||||
name = "~\".*\""
|
||||
if wkName == "" {
|
||||
wkName = "~\".*\""
|
||||
} else {
|
||||
name = "\"" + name + "\""
|
||||
wkName = "\"" + wkName + "\""
|
||||
}
|
||||
|
||||
rule = strings.Replace(rule, "$1", kind, -1)
|
||||
rule = strings.Replace(rule, "$2", name, -1)
|
||||
rule = strings.Replace(rule, "$1", wkKind, -1)
|
||||
rule = strings.Replace(rule, "$2", wkName, -1)
|
||||
rule = strings.Replace(rule, "$3", namespace, -1)
|
||||
return rule
|
||||
}
|
||||
|
||||
func MakeWorkspacePromQL(metricsName string, namespaceRe2 string) string {
|
||||
func MakeWorkspacePromQL(metricsName string, nsFilter string) string {
|
||||
promql := RulePromQLTmplMap[metricsName]
|
||||
promql = strings.Replace(promql, "$1", namespaceRe2, -1)
|
||||
promql = strings.Replace(promql, "$1", nsFilter, -1)
|
||||
return promql
|
||||
}
|
||||
|
||||
func MakeContainerPromQL(request *restful.Request) string {
|
||||
nsName := strings.Trim(request.PathParameter("ns_name"), " ")
|
||||
poName := strings.Trim(request.PathParameter("pod_name"), " ")
|
||||
containerName := strings.Trim(request.PathParameter("container_name"), " ")
|
||||
// metricType container_cpu_utilisation container_memory_utilisation container_memory_utilisation_wo_cache
|
||||
metricType := strings.Trim(request.QueryParameter("metrics_name"), " ")
|
||||
func MakeContainerPromQL(nsName, podName, containerName, metricName, containerFilter string) string {
|
||||
var promql = ""
|
||||
if containerName == "" {
|
||||
// all containers maybe use filter
|
||||
metricType += "_all"
|
||||
promql = RulePromQLTmplMap[metricType]
|
||||
metricName += "_all"
|
||||
promql = RulePromQLTmplMap[metricName]
|
||||
promql = strings.Replace(promql, "$1", nsName, -1)
|
||||
promql = strings.Replace(promql, "$2", poName, -1)
|
||||
containerFilter := strings.Trim(request.QueryParameter("containers_filter"), " ")
|
||||
promql = strings.Replace(promql, "$2", podName, -1)
|
||||
|
||||
if containerFilter == "" {
|
||||
containerFilter = ".*"
|
||||
}
|
||||
|
||||
promql = strings.Replace(promql, "$3", containerFilter, -1)
|
||||
return promql
|
||||
}
|
||||
promql = RulePromQLTmplMap[metricType]
|
||||
promql = RulePromQLTmplMap[metricName]
|
||||
|
||||
promql = strings.Replace(promql, "$1", nsName, -1)
|
||||
promql = strings.Replace(promql, "$2", poName, -1)
|
||||
promql = strings.Replace(promql, "$2", podName, -1)
|
||||
promql = strings.Replace(promql, "$3", containerName, -1)
|
||||
return promql
|
||||
}
|
||||
|
||||
func MakePodPromQL(request *restful.Request, params []string) string {
|
||||
metricType := params[0]
|
||||
nsName := params[1]
|
||||
nodeID := params[2]
|
||||
podName := params[3]
|
||||
podFilter := params[4]
|
||||
func MakePodPromQL(metricName, nsName, nodeID, podName, podFilter string) string {
|
||||
|
||||
if podFilter == "" {
|
||||
podFilter = ".*"
|
||||
}
|
||||
|
||||
var promql = ""
|
||||
if nsName != "" {
|
||||
// get pod metrics by namespace
|
||||
if podName != "" {
|
||||
// specific pod
|
||||
promql = RulePromQLTmplMap[metricType]
|
||||
promql = RulePromQLTmplMap[metricName]
|
||||
promql = strings.Replace(promql, "$1", nsName, -1)
|
||||
promql = strings.Replace(promql, "$2", podName, -1)
|
||||
|
||||
} else {
|
||||
// all pods
|
||||
metricType += "_all"
|
||||
promql = RulePromQLTmplMap[metricType]
|
||||
if podFilter == "" {
|
||||
podFilter = ".*"
|
||||
}
|
||||
metricName += "_all"
|
||||
promql = RulePromQLTmplMap[metricName]
|
||||
|
||||
promql = strings.Replace(promql, "$1", nsName, -1)
|
||||
promql = strings.Replace(promql, "$2", podFilter, -1)
|
||||
}
|
||||
} else if nodeID != "" {
|
||||
// get pod metrics by nodeid
|
||||
metricType += "_node"
|
||||
promql = RulePromQLTmplMap[metricType]
|
||||
metricName += "_node"
|
||||
promql = RulePromQLTmplMap[metricName]
|
||||
promql = strings.Replace(promql, "$3", nodeID, -1)
|
||||
if podName != "" {
|
||||
// specific pod
|
||||
promql = strings.Replace(promql, "$2", podName, -1)
|
||||
} else {
|
||||
// choose pod use re2 expression
|
||||
podFilter := strings.Trim(request.QueryParameter("pods_filter"), " ")
|
||||
if podFilter == "" {
|
||||
podFilter = ".*"
|
||||
}
|
||||
promql = strings.Replace(promql, "$2", podFilter, -1)
|
||||
}
|
||||
}
|
||||
return promql
|
||||
}
|
||||
|
||||
func MakeNamespacePromQL(request *restful.Request, metricsName string) string {
|
||||
nsName := strings.Trim(request.PathParameter("ns_name"), " ")
|
||||
metricType := metricsName
|
||||
var recordingRule = RulePromQLTmplMap[metricType]
|
||||
nsFilter := strings.Trim(request.QueryParameter("namespaces_filter"), " ")
|
||||
func MakeNamespacePromQL(nsName string, nsFilter string, metricsName string) string {
|
||||
var recordingRule = RulePromQLTmplMap[metricsName]
|
||||
|
||||
if nsName != "" {
|
||||
nsFilter = nsName
|
||||
} else {
|
||||
@@ -162,37 +139,37 @@ func MakeNamespacePromQL(request *restful.Request, metricsName string) string {
|
||||
return recordingRule
|
||||
}
|
||||
|
||||
func MakeNodeorClusterRule(request *restful.Request, metricsName string) string {
|
||||
nodeID := request.PathParameter("node_id")
|
||||
// cluster rule
|
||||
func MakeClusterRule(metricsName string) string {
|
||||
var rule = RulePromQLTmplMap[metricsName]
|
||||
|
||||
if strings.Contains(request.SelectedRoutePath(), "monitoring/cluster") {
|
||||
// cluster
|
||||
return rule
|
||||
} else {
|
||||
// node
|
||||
nodesFilter := strings.Trim(request.QueryParameter("nodes_filter"), " ")
|
||||
if nodesFilter == "" {
|
||||
nodesFilter = ".*"
|
||||
}
|
||||
if strings.Contains(metricsName, "disk_size") || strings.Contains(metricsName, "pod") || strings.Contains(metricsName, "usage") {
|
||||
// disk size promql
|
||||
if nodeID != "" {
|
||||
nodesFilter = "{" + "node" + "=" + "\"" + nodeID + "\"" + "}"
|
||||
} else {
|
||||
nodesFilter = "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}"
|
||||
}
|
||||
rule = strings.Replace(rule, "$1", nodesFilter, -1)
|
||||
} else {
|
||||
// cpu, memory, network, disk_iops rules
|
||||
if nodeID != "" {
|
||||
// specific node
|
||||
rule = rule + "{" + "node" + "=" + "\"" + nodeID + "\"" + "}"
|
||||
} else {
|
||||
// all nodes or specific nodes filted with re2 syntax
|
||||
rule = rule + "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}"
|
||||
}
|
||||
}
|
||||
}
|
||||
return rule
|
||||
}
|
||||
|
||||
// node rule
|
||||
func MakeNodeRule(nodeID string, nodesFilter string, metricsName string) string {
|
||||
var rule = RulePromQLTmplMap[metricsName]
|
||||
|
||||
if nodesFilter == "" {
|
||||
nodesFilter = ".*"
|
||||
}
|
||||
if strings.Contains(metricsName, "disk_size") || strings.Contains(metricsName, "pod") || strings.Contains(metricsName, "usage") || strings.Contains(metricsName, "inode") {
|
||||
// disk size promql
|
||||
if nodeID != "" {
|
||||
nodesFilter = "{" + "node" + "=" + "\"" + nodeID + "\"" + "}"
|
||||
} else {
|
||||
nodesFilter = "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}"
|
||||
}
|
||||
rule = strings.Replace(rule, "$1", nodesFilter, -1)
|
||||
} else {
|
||||
// cpu, memory, network, disk_iops rules
|
||||
if nodeID != "" {
|
||||
// specific node
|
||||
rule = rule + "{" + "node" + "=" + "\"" + nodeID + "\"" + "}"
|
||||
} else {
|
||||
// all nodes or specific nodes filted with re2 syntax
|
||||
rule = rule + "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}"
|
||||
}
|
||||
}
|
||||
|
||||
return rule
|
||||
}
|
||||
|
||||
560
pkg/models/metrics/metricsruleconst.go
Normal file
560
pkg/models/metrics/metricsruleconst.go
Normal file
@@ -0,0 +1,560 @@
|
||||
/*
|
||||
Copyright 2018 The KubeSphere Authors.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package metrics
|
||||
|
||||
const (
|
||||
ResultTypeVector = "vector"
|
||||
ResultTypeMatrix = "matrix"
|
||||
MetricStatus = "status"
|
||||
MetricStatusError = "error"
|
||||
MetricStatusSuccess = "success"
|
||||
ResultItemMetric = "metric"
|
||||
ResultItemMetricResource = "resource"
|
||||
ResultItemValue = "value"
|
||||
ResultItemValues = "values"
|
||||
ResultSortTypeDesc = "desc"
|
||||
ResultSortTypeAsce = "asce"
|
||||
)
|
||||
|
||||
const (
|
||||
MetricNameWorkloadCount = "workload_count"
|
||||
MetricNameNamespacePodCount = "namespace_pod_count"
|
||||
|
||||
MetricNameWorkspaceAllOrganizationCount = "workspace_all_organization_count"
|
||||
MetricNameWorkspaceAllAccountCount = "workspace_all_account_count"
|
||||
MetricNameWorkspaceAllProjectCount = "workspace_all_project_count"
|
||||
MetricNameWorkspaceAllDevopsCount = "workspace_all_devops_project_count"
|
||||
MetricNameClusterAllProjectCount = "cluster_namespace_count"
|
||||
|
||||
MetricNameWorkspaceNamespaceCount = "workspace_namespace_count"
|
||||
MetricNameWorkspaceDevopsCount = "workspace_devops_project_count"
|
||||
MetricNameWorkspaceMemberCount = "workspace_member_count"
|
||||
MetricNameWorkspaceRoleCount = "workspace_role_count"
|
||||
MetricNameComponentOnLine = "component_online_count"
|
||||
MetricNameComponentLine = "component_count"
|
||||
)
|
||||
|
||||
const (
|
||||
WorkspaceResourceKindOrganization = "organization"
|
||||
WorkspaceResourceKindAccount = "account"
|
||||
WorkspaceResourceKindNamespace = "namespace"
|
||||
WorkspaceResourceKindDevops = "devops"
|
||||
WorkspaceResourceKindMember = "member"
|
||||
WorkspaceResourceKindRole = "role"
|
||||
)
|
||||
|
||||
const (
|
||||
MetricLevelCluster = "cluster"
|
||||
MetricLevelClusterWorkspace = "cluster_workspace"
|
||||
MetricLevelNode = "node"
|
||||
MetricLevelWorkspace = "workspace"
|
||||
MetricLevelNamespace = "namespace"
|
||||
MetricLevelPod = "pod"
|
||||
MetricLevelPodName = "pod_name"
|
||||
MetricLevelContainer = "container"
|
||||
MetricLevelWorkload = "workload"
|
||||
)
|
||||
|
||||
const (
|
||||
ReplicaSet = "ReplicaSet"
|
||||
StatefulSet = "StatefulSet"
|
||||
DaemonSet = "DaemonSet"
|
||||
)
|
||||
|
||||
const (
|
||||
NodeStatusRule = `kube_node_status_condition{condition="Ready"} > 0`
|
||||
PodInfoRule = `kube_pod_info{created_by_kind="$1",created_by_name=$2,namespace="$3"}`
|
||||
NamespaceLabelRule = `kube_namespace_labels`
|
||||
)
|
||||
|
||||
const (
|
||||
WorkspaceJoinedKey = "label_kubesphere_io_workspace"
|
||||
)
|
||||
|
||||
type MetricMap map[string]string
|
||||
|
||||
var ClusterMetricsNames = []string{
|
||||
"cluster_cpu_utilisation",
|
||||
"cluster_cpu_usage",
|
||||
"cluster_cpu_total",
|
||||
"cluster_memory_utilisation",
|
||||
"cluster_memory_bytes_available",
|
||||
"cluster_memory_bytes_total",
|
||||
"cluster_memory_bytes_usage",
|
||||
"cluster_net_utilisation",
|
||||
"cluster_net_bytes_transmitted",
|
||||
"cluster_net_bytes_received",
|
||||
"cluster_disk_read_iops",
|
||||
"cluster_disk_write_iops",
|
||||
"cluster_disk_read_throughput",
|
||||
"cluster_disk_write_throughput",
|
||||
"cluster_disk_size_usage",
|
||||
"cluster_disk_size_utilisation",
|
||||
"cluster_disk_size_capacity",
|
||||
"cluster_disk_size_available",
|
||||
"cluster_disk_inode_total",
|
||||
"cluster_disk_inode_usage",
|
||||
"cluster_disk_inode_utilisation",
|
||||
|
||||
"cluster_node_online",
|
||||
"cluster_node_offline",
|
||||
"cluster_node_total",
|
||||
|
||||
"cluster_pod_count",
|
||||
"cluster_pod_quota",
|
||||
"cluster_pod_utilisation",
|
||||
"cluster_pod_running_count",
|
||||
"cluster_pod_succeeded_count",
|
||||
"cluster_pod_abnormal_count",
|
||||
"cluster_ingresses_extensions_count",
|
||||
"cluster_cronjob_count",
|
||||
"cluster_pvc_count",
|
||||
"cluster_daemonset_count",
|
||||
"cluster_deployment_count",
|
||||
"cluster_endpoint_count",
|
||||
"cluster_hpa_count",
|
||||
"cluster_job_count",
|
||||
"cluster_statefulset_count",
|
||||
"cluster_replicaset_count",
|
||||
"cluster_service_count",
|
||||
"cluster_secret_count",
|
||||
|
||||
"cluster_namespace_count",
|
||||
"workspace_all_project_count",
|
||||
}
|
||||
var NodeMetricsNames = []string{
|
||||
"node_cpu_utilisation",
|
||||
"node_cpu_total",
|
||||
"node_cpu_usage",
|
||||
"node_memory_utilisation",
|
||||
"node_memory_bytes_usage",
|
||||
"node_memory_bytes_available",
|
||||
"node_memory_bytes_total",
|
||||
"node_net_utilisation",
|
||||
"node_net_bytes_transmitted",
|
||||
"node_net_bytes_received",
|
||||
"node_disk_read_iops",
|
||||
"node_disk_write_iops",
|
||||
"node_disk_read_throughput",
|
||||
"node_disk_write_throughput",
|
||||
"node_disk_size_capacity",
|
||||
"node_disk_size_available",
|
||||
"node_disk_size_usage",
|
||||
"node_disk_size_utilisation",
|
||||
|
||||
"node_disk_inode_total",
|
||||
"node_disk_inode_usage",
|
||||
"node_disk_inode_utilisation",
|
||||
|
||||
"node_pod_count",
|
||||
"node_pod_quota",
|
||||
"node_pod_utilisation",
|
||||
"node_pod_running_count",
|
||||
"node_pod_succeeded_count",
|
||||
"node_pod_abnormal_count",
|
||||
}
|
||||
var WorkspaceMetricsNames = []string{
|
||||
"workspace_cpu_usage",
|
||||
"workspace_memory_usage",
|
||||
"workspace_memory_usage_wo_cache",
|
||||
"workspace_net_bytes_transmitted",
|
||||
"workspace_net_bytes_received",
|
||||
"workspace_pod_count",
|
||||
"workspace_pod_running_count",
|
||||
"workspace_pod_succeeded_count",
|
||||
"workspace_pod_abnormal_count",
|
||||
"workspace_ingresses_extensions_count",
|
||||
|
||||
"workspace_cronjob_count",
|
||||
"workspace_pvc_count",
|
||||
"workspace_daemonset_count",
|
||||
"workspace_deployment_count",
|
||||
"workspace_endpoint_count",
|
||||
"workspace_hpa_count",
|
||||
"workspace_job_count",
|
||||
"workspace_statefulset_count",
|
||||
"workspace_replicaset_count",
|
||||
"workspace_service_count",
|
||||
"workspace_secret_count",
|
||||
}
|
||||
var NamespaceMetricsNames = []string{
|
||||
"namespace_cpu_usage",
|
||||
"namespace_memory_usage",
|
||||
"namespace_memory_usage_wo_cache",
|
||||
"namespace_net_bytes_transmitted",
|
||||
"namespace_net_bytes_received",
|
||||
"namespace_pod_count",
|
||||
"namespace_pod_running_count",
|
||||
"namespace_pod_succeeded_count",
|
||||
"namespace_pod_abnormal_count",
|
||||
|
||||
"namespace_configmap_count_used",
|
||||
"namespace_jobs_batch_count_used",
|
||||
"namespace_roles_count_used",
|
||||
"namespace_memory_limit_used",
|
||||
"namespace_pvc_used",
|
||||
"namespace_memory_request_used",
|
||||
"namespace_pvc_count_used",
|
||||
"namespace_cronjobs_batch_count_used",
|
||||
"namespace_ingresses_extensions_count_used",
|
||||
"namespace_cpu_limit_used",
|
||||
"namespace_storage_request_used",
|
||||
"namespace_deployment_count_used",
|
||||
"namespace_pod_count_used",
|
||||
"namespace_statefulset_count_used",
|
||||
"namespace_daemonset_count_used",
|
||||
"namespace_secret_count_used",
|
||||
"namespace_service_count_used",
|
||||
"namespace_cpu_request_used",
|
||||
"namespace_service_loadbalancer_used",
|
||||
|
||||
"namespace_configmap_count_hard",
|
||||
"namespace_jobs_batch_count_hard",
|
||||
"namespace_roles_count_hard",
|
||||
"namespace_memory_limit_hard",
|
||||
"namespace_pvc_hard",
|
||||
"namespace_memory_request_hard",
|
||||
"namespace_pvc_count_hard",
|
||||
"namespace_cronjobs_batch_count_hard",
|
||||
"namespace_ingresses_extensions_count_hard",
|
||||
"namespace_cpu_limit_hard",
|
||||
"namespace_storage_request_hard",
|
||||
"namespace_deployment_count_hard",
|
||||
"namespace_pod_count_hard",
|
||||
"namespace_statefulset_count_hard",
|
||||
"namespace_daemonset_count_hard",
|
||||
"namespace_secret_count_hard",
|
||||
"namespace_service_count_hard",
|
||||
"namespace_cpu_request_hard",
|
||||
"namespace_service_loadbalancer_hard",
|
||||
|
||||
"namespace_cronjob_count",
|
||||
"namespace_pvc_count",
|
||||
"namespace_daemonset_count",
|
||||
"namespace_deployment_count",
|
||||
"namespace_endpoint_count",
|
||||
"namespace_hpa_count",
|
||||
"namespace_job_count",
|
||||
"namespace_statefulset_count",
|
||||
"namespace_replicaset_count",
|
||||
"namespace_service_count",
|
||||
"namespace_secret_count",
|
||||
}
|
||||
|
||||
var PodMetricsNames = []string{
|
||||
"pod_cpu_usage",
|
||||
"pod_memory_usage",
|
||||
"pod_memory_usage_wo_cache",
|
||||
"pod_net_bytes_transmitted",
|
||||
"pod_net_bytes_received",
|
||||
}
|
||||
|
||||
var WorkloadMetricsNames = []string{
|
||||
"workload_pod_cpu_usage",
|
||||
"workload_pod_memory_usage",
|
||||
"workload_pod_memory_usage_wo_cache",
|
||||
"workload_pod_net_bytes_transmitted",
|
||||
"workload_pod_net_bytes_received",
|
||||
}
|
||||
|
||||
var RulePromQLTmplMap = MetricMap{
|
||||
//cluster
|
||||
"cluster_cpu_utilisation": ":node_cpu_utilisation:avg1m",
|
||||
"cluster_cpu_usage": `:node_cpu_utilisation:avg1m * sum(node:node_num_cpu:sum)`,
|
||||
"cluster_cpu_total": "sum(node:node_num_cpu:sum)",
|
||||
"cluster_memory_utilisation": ":node_memory_utilisation:",
|
||||
"cluster_memory_bytes_available": "sum(node:node_memory_bytes_available:sum)",
|
||||
"cluster_memory_bytes_total": "sum(node:node_memory_bytes_total:sum)",
|
||||
"cluster_memory_bytes_usage": "sum(node:node_memory_bytes_total:sum) - sum(node:node_memory_bytes_available:sum)",
|
||||
"cluster_net_utilisation": ":node_net_utilisation:sum_irate",
|
||||
"cluster_net_bytes_transmitted": "sum(node:node_net_bytes_transmitted:sum_irate)",
|
||||
"cluster_net_bytes_received": "sum(node:node_net_bytes_received:sum_irate)",
|
||||
"cluster_disk_read_iops": "sum(node:data_volume_iops_reads:sum)",
|
||||
"cluster_disk_write_iops": "sum(node:data_volume_iops_writes:sum)",
|
||||
"cluster_disk_read_throughput": "sum(node:data_volume_throughput_bytes_read:sum)",
|
||||
"cluster_disk_write_throughput": "sum(node:data_volume_throughput_bytes_written:sum)",
|
||||
"cluster_disk_size_usage": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`,
|
||||
"cluster_disk_size_utilisation": `(sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:)) - sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))) / sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`,
|
||||
"cluster_disk_size_capacity": `sum(sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`,
|
||||
"cluster_disk_size_available": `sum(sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:))`,
|
||||
|
||||
"cluster_disk_inode_total": `sum(node:disk_inodes_total:)`,
|
||||
"cluster_disk_inode_usage": `sum(node:disk_inodes_total:) - sum(node:disk_inodes_free:)`,
|
||||
"cluster_disk_inode_utilisation": `1 - sum(node:disk_inodes_free:) / sum(node:disk_inodes_total:)`,
|
||||
|
||||
"cluster_namespace_count": `count(kube_namespace_annotations)`,
|
||||
"workspace_all_project_count": `count(kube_namespace_annotations)`,
|
||||
|
||||
// cluster_pod_count = cluster_pod_running_count + cluster_pod_succeeded_count + cluster_pod_abnormal_count
|
||||
"cluster_pod_count": `sum(kube_pod_status_phase)`,
|
||||
"cluster_pod_utilisation": `sum(kube_pod_status_phase) / sum(kube_node_status_capacity_pods)`,
|
||||
"cluster_pod_running_count": `sum(kube_pod_status_phase{phase="Running"})`,
|
||||
"cluster_pod_succeeded_count": `sum(kube_pod_status_phase{phase="Succeeded"})`,
|
||||
"cluster_pod_abnormal_count": `sum(kube_pod_status_phase{phase=~"Failed|Pending|Unknown"})`,
|
||||
|
||||
"cluster_node_online": `sum(kube_node_status_condition{condition="Ready",status="true"})`,
|
||||
"cluster_node_offline": `sum(kube_node_status_condition{condition="Ready",status=~"unknown|false"})`,
|
||||
"cluster_node_total": `sum(kube_node_status_condition{condition="Ready"})`,
|
||||
"cluster_pod_quota": `sum(kube_node_status_capacity_pods)`,
|
||||
"cluster_ingresses_extensions_count": `sum(kube_resourcequota{type="used", resource="count/ingresses.extensions"}) by (resource, type)`,
|
||||
|
||||
"cluster_configmap_count_used": `sum(kube_resourcequota{type="used", resource="count/configmaps"}) by (resource, type)`,
|
||||
"cluster_jobs_batch_count_used": `sum(kube_resourcequota{type="used", resource="count/jobs.batch"}) by (resource, type)`,
|
||||
"cluster_roles_count_used": `sum(kube_resourcequota{type="used", resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`,
|
||||
"cluster_memory_limit_used": `sum(kube_resourcequota{type="used", resource="limits.memory"}) by (resource, type)`,
|
||||
"cluster_pvc_used": `sum(kube_resourcequota{type="used", resource="persistentvolumeclaims"}) by (resource, type)`,
|
||||
"cluster_memory_request_used": `sum(kube_resourcequota{type="used", resource="requests.memory"}) by (resource, type)`,
|
||||
"cluster_pvc_count_used": `sum(kube_resourcequota{type="used", resource="count/persistentvolumeclaims"}) by (resource, type)`,
|
||||
"cluster_cronjobs_batch_count_used": `sum(kube_resourcequota{type="used", resource="count/cronjobs.batch"}) by (resource, type)`,
|
||||
"cluster_ingresses_extensions_count_used": `sum(kube_resourcequota{type="used", resource="count/ingresses.extensions"}) by (resource, type)`,
|
||||
"cluster_cpu_limit_used": `sum(kube_resourcequota{type="used", resource="limits.cpu"}) by (resource, type)`,
|
||||
"cluster_storage_request_used": `sum(kube_resourcequota{type="used", resource="requests.storage"}) by (resource, type)`,
|
||||
"cluster_deployment_count_used": `sum(kube_resourcequota{type="used", resource="count/deployments.apps"}) by (resource, type)`,
|
||||
"cluster_pod_count_used": `sum(kube_resourcequota{type="used", resource="count/pods"}) by (resource, type)`,
|
||||
"cluster_statefulset_count_used": `sum(kube_resourcequota{type="used", resource="count/statefulsets.apps"}) by (resource, type)`,
|
||||
"cluster_daemonset_count_used": `sum(kube_resourcequota{type="used", resource="count/daemonsets.apps"}) by (resource, type)`,
|
||||
"cluster_secret_count_used": `sum(kube_resourcequota{type="used", resource="count/secrets"}) by (resource, type)`,
|
||||
"cluster_service_count_used": `sum(kube_resourcequota{type="used", resource="count/services"}) by (resource, type)`,
|
||||
"cluster_cpu_request_used": `sum(kube_resourcequota{type="used", resource="requests.cpu"}) by (resource, type)`,
|
||||
"cluster_service_loadbalancer_used": `sum(kube_resourcequota{type="used", resource="services.loadbalancers"}) by (resource, type)`,
|
||||
|
||||
"cluster_configmap_count_hard": `sum(kube_resourcequota{type="hard", resource="count/configmaps"}) by (resource, type)`,
|
||||
"cluster_jobs_batch_count_hard": `sum(kube_resourcequota{type="hard", resource="count/jobs.batch"}) by (resource, type)`,
|
||||
"cluster_roles_count_hard": `sum(kube_resourcequota{type="hard", resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`,
|
||||
"cluster_memory_limit_hard": `sum(kube_resourcequota{type="hard", resource="limits.memory"}) by (resource, type)`,
|
||||
"cluster_pvc_hard": `sum(kube_resourcequota{type="hard", resource="persistentvolumeclaims"}) by (resource, type)`,
|
||||
"cluster_memory_request_hard": `sum(kube_resourcequota{type="hard", resource="requests.memory"}) by (resource, type)`,
|
||||
"cluster_pvc_count_hard": `sum(kube_resourcequota{type="hard", resource="count/persistentvolumeclaims"}) by (resource, type)`,
|
||||
"cluster_cronjobs_batch_count_hard": `sum(kube_resourcequota{type="hard", resource="count/cronjobs.batch"}) by (resource, type)`,
|
||||
"cluster_ingresses_extensions_count_hard": `sum(kube_resourcequota{type="hard", resource="count/ingresses.extensions"}) by (resource, type)`,
|
||||
"cluster_cpu_limit_hard": `sum(kube_resourcequota{type="hard", resource="limits.cpu"}) by (resource, type)`,
|
||||
"cluster_storage_request_hard": `sum(kube_resourcequota{type="hard", resource="requests.storage"}) by (resource, type)`,
|
||||
"cluster_deployment_count_hard": `sum(kube_resourcequota{type="hard", resource="count/deployments.apps"}) by (resource, type)`,
|
||||
"cluster_pod_count_hard": `sum(kube_resourcequota{type="hard", resource="count/pods"}) by (resource, type)`,
|
||||
"cluster_statefulset_count_hard": `sum(kube_resourcequota{type="hard", resource="count/statefulsets.apps"}) by (resource, type)`,
|
||||
"cluster_daemonset_count_hard": `sum(kube_resourcequota{type="hard", resource="count/daemonsets.apps"}) by (resource, type)`,
|
||||
"cluster_secret_count_hard": `sum(kube_resourcequota{type="hard", resource="count/secrets"}) by (resource, type)`,
|
||||
"cluster_service_count_hard": `sum(kube_resourcequota{type="hard", resource="count/services"}) by (resource, type)`,
|
||||
"cluster_cpu_request_hard": `sum(kube_resourcequota{type="hard", resource="requests.cpu"}) by (resource, type)`,
|
||||
"cluster_service_loadbalancer_hard": `sum(kube_resourcequota{type="hard", resource="services.loadbalancers"}) by (resource, type)`,
|
||||
|
||||
"cluster_cronjob_count": `sum(kube_cronjob_labels)`,
|
||||
"cluster_pvc_count": `sum(kube_persistentvolumeclaim_info)`,
|
||||
"cluster_daemonset_count": `sum(kube_daemonset_labels)`,
|
||||
"cluster_deployment_count": `sum(kube_deployment_labels)`,
|
||||
"cluster_endpoint_count": `sum(kube_endpoint_labels)`,
|
||||
"cluster_hpa_count": `sum(kube_hpa_labels)`,
|
||||
"cluster_job_count": `sum(kube_job_labels)`,
|
||||
"cluster_statefulset_count": `sum(kube_statefulset_labels)`,
|
||||
"cluster_replicaset_count": `count(kube_replicaset_created)`,
|
||||
"cluster_service_count": `sum(kube_service_info)`,
|
||||
"cluster_secret_count": `sum(kube_secret_info)`,
|
||||
"cluster_pv_count": `sum(kube_persistentvolume_labels)`,
|
||||
|
||||
//node
|
||||
"node_cpu_utilisation": "node:node_cpu_utilisation:avg1m",
|
||||
"node_cpu_total": "node:node_num_cpu:sum",
|
||||
"node_memory_utilisation": "node:node_memory_utilisation:",
|
||||
"node_memory_bytes_available": "node:node_memory_bytes_available:sum",
|
||||
"node_memory_bytes_total": "node:node_memory_bytes_total:sum",
|
||||
// Node network utilisation (bytes received + bytes transmitted per second)
|
||||
"node_net_utilisation": "node:node_net_utilisation:sum_irate",
|
||||
// Node network bytes transmitted per second
|
||||
"node_net_bytes_transmitted": "node:node_net_bytes_transmitted:sum_irate",
|
||||
// Node network bytes received per second
|
||||
"node_net_bytes_received": "node:node_net_bytes_received:sum_irate",
|
||||
|
||||
// node:data_volume_iops_reads:sum{node=~"i-5xcldxos|i-6soe9zl1"}
|
||||
"node_disk_read_iops": "node:data_volume_iops_reads:sum",
|
||||
// node:data_volume_iops_writes:sum{node=~"i-5xcldxos|i-6soe9zl1"}
|
||||
"node_disk_write_iops": "node:data_volume_iops_writes:sum",
|
||||
// node:data_volume_throughput_bytes_read:sum{node=~"i-5xcldxos|i-6soe9zl1"}
|
||||
"node_disk_read_throughput": "node:data_volume_throughput_bytes_read:sum",
|
||||
// node:data_volume_throughput_bytes_written:sum{node=~"i-5xcldxos|i-6soe9zl1"}
|
||||
"node_disk_write_throughput": "node:data_volume_throughput_bytes_written:sum",
|
||||
|
||||
"node_disk_size_capacity": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`,
|
||||
"node_disk_size_available": `sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`,
|
||||
"node_disk_size_usage": `sum by (node) ((node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) -sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`,
|
||||
"node_disk_size_utilisation": `sum by (node) (((node_filesystem_size{mountpoint="/", job="node-exporter"} - node_filesystem_avail{mountpoint="/", job="node-exporter"}) / node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`,
|
||||
|
||||
"node_disk_inode_total": `node:disk_inodes_total:$1`,
|
||||
"node_disk_inode_usage": `node:disk_inodes_total:$1 - node:disk_inodes_free:$1`,
|
||||
"node_disk_inode_utilisation": `(1 - (node:disk_inodes_free:$1 / node:disk_inodes_total:$1))`,
|
||||
|
||||
"node_pod_count": `count(kube_pod_info$1) by (node)`,
|
||||
"node_pod_quota": `sum(kube_node_status_capacity_pods$1) by (node)`,
|
||||
"node_pod_utilisation": `sum(kube_pod_info$1) by (node) / sum(kube_node_status_capacity_pods$1) by (node)`,
|
||||
"node_pod_running_count": `count(kube_pod_info$1 unless on (pod) (kube_pod_status_phase{phase=~"Failed|Pending|Unknown|Succeeded"} > 0)) by (node)`,
|
||||
"node_pod_succeeded_count": `count(kube_pod_info$1 unless on (pod) (kube_pod_status_phase{phase=~"Failed|Pending|Unknown|Running"} > 0)) by (node)`,
|
||||
"node_pod_abnormal_count": `count(kube_pod_info$1 unless on (pod) (kube_pod_status_phase{phase=~"Succeeded|Running"} > 0)) by (node)`,
|
||||
|
||||
// without log node: unless on(node) kube_node_labels{label_role="log"}
|
||||
"node_cpu_usage": `node:node_cpu_utilisation:avg1m$1 * node:node_num_cpu:sum$1`,
|
||||
"node_memory_bytes_usage": "node:node_memory_bytes_total:sum$1 - node:node_memory_bytes_available:sum$1",
|
||||
|
||||
//namespace
|
||||
"namespace_cpu_usage": `namespace:container_cpu_usage_seconds_total:sum_rate{namespace=~"$1"}`,
|
||||
"namespace_memory_usage": `namespace:container_memory_usage_bytes:sum{namespace=~"$1"}`,
|
||||
"namespace_memory_usage_wo_cache": `namespace:container_memory_usage_bytes_wo_cache:sum{namespace=~"$1"}`,
|
||||
"namespace_net_bytes_transmitted": `sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m]))`,
|
||||
"namespace_net_bytes_received": `sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m]))`,
|
||||
"namespace_pod_count": `sum(kube_pod_status_phase{namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_pod_running_count": `sum(kube_pod_status_phase{phase="Running", namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_pod_succeeded_count": `sum(kube_pod_status_phase{phase="Succeeded", namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_pod_abnormal_count": `sum(kube_pod_status_phase{phase=~"Failed|Pending|Unknown", namespace=~"$1"}) by (namespace)`,
|
||||
|
||||
"namespace_configmap_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/configmaps"}) by (namespace, resource, type)`,
|
||||
"namespace_jobs_batch_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/jobs.batch"}) by (namespace, resource, type)`,
|
||||
"namespace_roles_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/roles.rbac.authorization.k8s.io"}) by (namespace, resource, type)`,
|
||||
"namespace_memory_limit_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="limits.memory"}) by (namespace, resource, type)`,
|
||||
"namespace_pvc_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="persistentvolumeclaims"}) by (namespace, resource, type)`,
|
||||
"namespace_memory_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.memory"}) by (namespace, resource, type)`,
|
||||
"namespace_pvc_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/persistentvolumeclaims"}) by (namespace, resource, type)`,
|
||||
"namespace_cronjobs_batch_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/cronjobs.batch"}) by (namespace, resource, type)`,
|
||||
"namespace_ingresses_extensions_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/ingresses.extensions"}) by (namespace, resource, type)`,
|
||||
"namespace_cpu_limit_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="limits.cpu"}) by (namespace, resource, type)`,
|
||||
"namespace_storage_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.storage"}) by (namespace, resource, type)`,
|
||||
"namespace_deployment_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/deployments.apps"}) by (namespace, resource, type)`,
|
||||
"namespace_pod_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/pods"}) by (namespace, resource, type)`,
|
||||
"namespace_statefulset_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/statefulsets.apps"}) by (namespace, resource, type)`,
|
||||
"namespace_daemonset_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/daemonsets.apps"}) by (namespace, resource, type)`,
|
||||
"namespace_secret_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/secrets"}) by (namespace, resource, type)`,
|
||||
"namespace_service_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/services"}) by (namespace, resource, type)`,
|
||||
"namespace_cpu_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.cpu"}) by (namespace, resource, type)`,
|
||||
"namespace_service_loadbalancer_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="services.loadbalancers"}) by (namespace, resource, type)`,
|
||||
|
||||
"namespace_configmap_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/configmaps"}) by (namespace, resource, type)`,
|
||||
"namespace_jobs_batch_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/jobs.batch"}) by (namespace, resource, type)`,
|
||||
"namespace_roles_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/roles.rbac.authorization.k8s.io"}) by (namespace, resource, type)`,
|
||||
"namespace_memory_limit_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="limits.memory"}) by (namespace, resource, type)`,
|
||||
"namespace_pvc_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="persistentvolumeclaims"}) by (namespace, resource, type)`,
|
||||
"namespace_memory_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.memory"}) by (namespace, resource, type)`,
|
||||
"namespace_pvc_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/persistentvolumeclaims"}) by (namespace, resource, type)`,
|
||||
"namespace_cronjobs_batch_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/cronjobs.batch"}) by (namespace, resource, type)`,
|
||||
"namespace_ingresses_extensions_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/ingresses.extensions"}) by (namespace, resource, type)`,
|
||||
"namespace_cpu_limit_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="limits.cpu"}) by (namespace, resource, type)`,
|
||||
"namespace_storage_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.storage"}) by (namespace, resource, type)`,
|
||||
"namespace_deployment_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/deployments.apps"}) by (namespace, resource, type)`,
|
||||
"namespace_pod_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/pods"}) by (namespace, resource, type)`,
|
||||
"namespace_statefulset_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/statefulsets.apps"}) by (namespace, resource, type)`,
|
||||
"namespace_daemonset_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/daemonsets.apps"}) by (namespace, resource, type)`,
|
||||
"namespace_secret_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/secrets"}) by (namespace, resource, type)`,
|
||||
"namespace_service_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/services"}) by (namespace, resource, type)`,
|
||||
"namespace_cpu_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.cpu"}) by (namespace, resource, type)`,
|
||||
"namespace_service_loadbalancer_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="services.loadbalancers"}) by (namespace, resource, type)`,
|
||||
|
||||
"namespace_cronjob_count": `sum(kube_cronjob_labels{namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_pvc_count": `sum(kube_persistentvolumeclaim_info{namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_daemonset_count": `sum(kube_daemonset_labels{namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_deployment_count": `sum(kube_deployment_labels{namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_endpoint_count": `sum(kube_endpoint_labels{namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_hpa_count": `sum(kube_hpa_labels{namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_job_count": `sum(kube_job_labels{namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_statefulset_count": `sum(kube_statefulset_labels{namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_replicaset_count": `count(kube_replicaset_created{namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_service_count": `sum(kube_service_info{namespace=~"$1"}) by (namespace)`,
|
||||
"namespace_secret_count": `sum(kube_secret_info{namespace=~"$1"}) by (namespace)`,
|
||||
|
||||
// pod
|
||||
"pod_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name="$2", image!=""}[5m])) by (namespace, pod_name)`,
|
||||
"pod_memory_usage": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""}) by (namespace, pod_name)`,
|
||||
"pod_memory_usage_wo_cache": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name="$2",image!=""}) by (namespace, pod_name)`,
|
||||
"pod_net_bytes_transmitted": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[5m]))`,
|
||||
"pod_net_bytes_received": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[5m]))`,
|
||||
|
||||
"pod_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}[5m])) by (namespace, pod_name)`,
|
||||
"pod_memory_usage_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`,
|
||||
"pod_memory_usage_wo_cache_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`,
|
||||
"pod_net_bytes_transmitted_all": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`,
|
||||
"pod_net_bytes_received_all": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[5m]))`,
|
||||
|
||||
"pod_cpu_usage_node": `sum by (node, pod) (label_join(irate(container_cpu_usage_seconds_total{job="kubelet",pod_name=~"$2", image!=""}[5m]), "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`,
|
||||
"pod_memory_usage_node": `sum by (node, pod) (label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`,
|
||||
"pod_memory_usage_wo_cache_node": `sum by (node, pod) ((label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") - label_join(container_memory_cache{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name")) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`,
|
||||
|
||||
// container
|
||||
"container_cpu_usage": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name="$3"}[5m])) by (namespace, pod_name, container_name)`,
|
||||
"container_cpu_usage_all": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}[5m])) by (namespace, pod_name, container_name)`,
|
||||
|
||||
"container_memory_usage_wo_cache": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name="$3"}`,
|
||||
"container_memory_usage_wo_cache_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`,
|
||||
|
||||
"container_memory_usage": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"}`,
|
||||
"container_memory_usage_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`,
|
||||
|
||||
// workspace
|
||||
"workspace_cpu_usage": `sum(namespace:container_cpu_usage_seconds_total:sum_rate{namespace =~"$1"})`,
|
||||
"workspace_memory_usage": `sum(namespace:container_memory_usage_bytes:sum{namespace =~"$1"})`,
|
||||
"workspace_memory_usage_wo_cache": `sum(namespace:container_memory_usage_bytes_wo_cache:sum{namespace =~"$1"})`,
|
||||
"workspace_net_bytes_transmitted": `sum(sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m])))`,
|
||||
"workspace_net_bytes_received": `sum(sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[5m])))`,
|
||||
"workspace_pod_count": `sum(kube_pod_status_phase{namespace=~"$1"})`,
|
||||
"workspace_pod_running_count": `sum(kube_pod_status_phase{phase="Running", namespace=~"$1"})`,
|
||||
"workspace_pod_succeeded_count": `sum(kube_pod_status_phase{phase="Succeeded", namespace=~"$1"})`,
|
||||
"workspace_pod_abnormal_count": `sum(kube_pod_status_phase{phase=~"Failed|Pending|Unknown", namespace=~"$1"})`,
|
||||
|
||||
"workspace_configmap_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/configmaps"}) by (resource, type)`,
|
||||
"workspace_jobs_batch_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/jobs.batch"}) by (resource, type)`,
|
||||
"workspace_roles_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`,
|
||||
"workspace_memory_limit_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="limits.memory"}) by (resource, type)`,
|
||||
"workspace_pvc_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="persistentvolumeclaims"}) by (resource, type)`,
|
||||
"workspace_memory_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.memory"}) by (resource, type)`,
|
||||
"workspace_pvc_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/persistentvolumeclaims"}) by (resource, type)`,
|
||||
"workspace_cronjobs_batch_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/cronjobs.batch"}) by (resource, type)`,
|
||||
"workspace_ingresses_extensions_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/ingresses.extensions"}) by (resource, type)`,
|
||||
"workspace_cpu_limit_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="limits.cpu"}) by (resource, type)`,
|
||||
"workspace_storage_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.storage"}) by (resource, type)`,
|
||||
"workspace_deployment_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/deployments.apps"}) by (resource, type)`,
|
||||
"workspace_pod_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/pods"}) by (resource, type)`,
|
||||
"workspace_statefulset_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/statefulsets.apps"}) by (resource, type)`,
|
||||
"workspace_daemonset_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/daemonsets.apps"}) by (resource, type)`,
|
||||
"workspace_secret_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/secrets"}) by (resource, type)`,
|
||||
"workspace_service_count_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/services"}) by (resource, type)`,
|
||||
"workspace_cpu_request_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="requests.cpu"}) by (resource, type)`,
|
||||
"workspace_service_loadbalancer_used": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="services.loadbalancers"}) by (resource, type)`,
|
||||
|
||||
"workspace_configmap_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/configmaps"}) by (resource, type)`,
|
||||
"workspace_jobs_batch_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/jobs.batch"}) by (resource, type)`,
|
||||
"workspace_roles_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`,
|
||||
"workspace_memory_limit_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="limits.memory"}) by (resource, type)`,
|
||||
"workspace_pvc_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="persistentvolumeclaims"}) by (resource, type)`,
|
||||
"workspace_memory_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.memory"}) by (resource, type)`,
|
||||
"workspace_pvc_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/persistentvolumeclaims"}) by (resource, type)`,
|
||||
"workspace_cronjobs_batch_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/cronjobs.batch"}) by (resource, type)`,
|
||||
"workspace_ingresses_extensions_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/ingresses.extensions"}) by (resource, type)`,
|
||||
"workspace_cpu_limit_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="limits.cpu"}) by (resource, type)`,
|
||||
"workspace_storage_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.storage"}) by (resource, type)`,
|
||||
"workspace_deployment_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/deployments.apps"}) by (resource, type)`,
|
||||
"workspace_pod_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/pods"}) by (resource, type)`,
|
||||
"workspace_statefulset_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/statefulsets.apps"}) by (resource, type)`,
|
||||
"workspace_daemonset_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/daemonsets.apps"}) by (resource, type)`,
|
||||
"workspace_secret_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/secrets"}) by (resource, type)`,
|
||||
"workspace_service_count_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="count/services"}) by (resource, type)`,
|
||||
"workspace_cpu_request_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="requests.cpu"}) by (resource, type)`,
|
||||
"workspace_service_loadbalancer_hard": `sum(kube_resourcequota{type="hard", namespace=~"$1", resource="services.loadbalancers"}) by (resource, type)`,
|
||||
"workspace_ingresses_extensions_count": `sum(kube_resourcequota{type="used", namespace=~"$1", resource="count/ingresses.extensions"}) by (resource, type)`,
|
||||
|
||||
"workspace_cronjob_count": `sum(kube_cronjob_labels{namespace=~"$1"})`,
|
||||
"workspace_pvc_count": `sum(kube_persistentvolumeclaim_info{namespace=~"$1"})`,
|
||||
"workspace_daemonset_count": `sum(kube_daemonset_labels{namespace=~"$1"})`,
|
||||
"workspace_deployment_count": `sum(kube_deployment_labels{namespace=~"$1"})`,
|
||||
"workspace_endpoint_count": `sum(kube_endpoint_labels{namespace=~"$1"})`,
|
||||
"workspace_hpa_count": `sum(kube_hpa_labels{namespace=~"$1"})`,
|
||||
"workspace_job_count": `sum(kube_job_labels{namespace=~"$1"})`,
|
||||
"workspace_statefulset_count": `sum(kube_statefulset_labels{namespace=~"$1"})`,
|
||||
"workspace_replicaset_count": `count(kube_replicaset_created{namespace=~"$1"})`,
|
||||
"workspace_service_count": `sum(kube_service_info{namespace=~"$1"})`,
|
||||
"workspace_secret_count": `sum(kube_secret_info{namespace=~"$1"})`,
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
/*
|
||||
Copyright 2018 The KubeSphere Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package metrics
|
||||
|
||||
type FormatedLevelMetric struct {
|
||||
MetricsLevel string `json:"metrics_level"`
|
||||
Results []FormatedMetric `json:"results"`
|
||||
}
|
||||
|
||||
type FormatedMetric struct {
|
||||
MetricName string `json:"metric_name, omitempty"`
|
||||
Status string `json:"status"`
|
||||
Data FormatedMetricData `json:"data, omitempty"`
|
||||
}
|
||||
|
||||
type FormatedMetricData struct {
|
||||
Result []map[string]interface{} `json:"result"`
|
||||
ResultType string `json:"resultType"`
|
||||
}
|
||||
|
||||
type CommonMetricsResult struct {
|
||||
Status string `json:"status"`
|
||||
Data CommonMetricsData `json:"data"`
|
||||
}
|
||||
|
||||
type CommonMetricsData struct {
|
||||
Result []CommonResultItem `json:"result"`
|
||||
ResultType string `json:"resultType"`
|
||||
}
|
||||
|
||||
type CommonResultItem struct {
|
||||
KubePodMetric KubePodMetric `json:"metric"`
|
||||
Value interface{} `json:"value"`
|
||||
}
|
||||
|
||||
/**
|
||||
"__name__": "kube_pod_info",
|
||||
"created_by_kind": "\\u003cnone\\u003e",
|
||||
"created_by_name": "\\u003cnone\\u003e",
|
||||
"endpoint": "https-main",
|
||||
"host_ip": "192.168.0.13",
|
||||
"instance": "10.244.114.187:8443",
|
||||
"job": "kube-state-metrics",
|
||||
"namespace": "kube-system",
|
||||
"node": "i-39p7faw6",
|
||||
"pod": "cloud-controller-manager-i-39p7faw6",
|
||||
"pod_ip": "192.168.0.13",
|
||||
"service": "kube-state-metrics"
|
||||
*/
|
||||
type KubePodMetric struct {
|
||||
CreatedByKind string `json:"created_by_kind"`
|
||||
CreatedByName string `json:"created_by_name"`
|
||||
Namespace string `json:"namespace"`
|
||||
Pod string `json:"pod"`
|
||||
}
|
||||
268
pkg/models/metrics/util.go
Normal file
268
pkg/models/metrics/util.go
Normal file
@@ -0,0 +1,268 @@
|
||||
/*
|
||||
Copyright 2018 The KubeSphere Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"unicode"
|
||||
|
||||
"github.com/golang/glog"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultPageLimit = 5
|
||||
DefaultPage = 1
|
||||
)
|
||||
|
||||
type FormatedMetricDataWrapper struct {
|
||||
fmtMetricData FormatedMetricData
|
||||
by func(p, q *map[string]interface{}) bool
|
||||
}
|
||||
|
||||
func (wrapper FormatedMetricDataWrapper) Len() int {
|
||||
return len(wrapper.fmtMetricData.Result)
|
||||
}
|
||||
|
||||
func (wrapper FormatedMetricDataWrapper) Less(i, j int) bool {
|
||||
return wrapper.by(&wrapper.fmtMetricData.Result[i], &wrapper.fmtMetricData.Result[j])
|
||||
}
|
||||
|
||||
func (wrapper FormatedMetricDataWrapper) Swap(i, j int) {
|
||||
wrapper.fmtMetricData.Result[i], wrapper.fmtMetricData.Result[j] = wrapper.fmtMetricData.Result[j], wrapper.fmtMetricData.Result[i]
|
||||
}
|
||||
|
||||
// sorted metric by ascending or descending order
|
||||
func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelMetric, resourceType string) (*FormatedLevelMetric, int) {
|
||||
var maxLength = 0
|
||||
for _, metricItem := range fmtLevelMetric.Results {
|
||||
if metricItem.Data.ResultType == ResultTypeVector && metricItem.Status == MetricStatusSuccess {
|
||||
if maxLength < len(metricItem.Data.Result) {
|
||||
maxLength = len(metricItem.Data.Result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if sortMetricName == "" {
|
||||
return fmtLevelMetric, maxLength
|
||||
}
|
||||
|
||||
// default sort type is descending order
|
||||
if sortType == "" {
|
||||
sortType = ResultSortTypeDesc
|
||||
}
|
||||
|
||||
var currentResourceMap = make(map[string]int)
|
||||
|
||||
// indexMap store sorted index for each node/namespace/pod
|
||||
var indexMap = make(map[string]int)
|
||||
for _, metricItem := range fmtLevelMetric.Results {
|
||||
if metricItem.Data.ResultType == ResultTypeVector && metricItem.Status == MetricStatusSuccess {
|
||||
if metricItem.MetricName == sortMetricName {
|
||||
if sortType == ResultSortTypeAsce {
|
||||
// desc
|
||||
sort.Sort(FormatedMetricDataWrapper{metricItem.Data, func(p, q *map[string]interface{}) bool {
|
||||
value1 := (*p)[ResultItemValue].([]interface{})
|
||||
value2 := (*q)[ResultItemValue].([]interface{})
|
||||
v1, _ := strconv.ParseFloat(value1[len(value1)-1].(string), 64)
|
||||
v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64)
|
||||
return v1 < v2
|
||||
}})
|
||||
} else {
|
||||
// desc
|
||||
sort.Sort(FormatedMetricDataWrapper{metricItem.Data, func(p, q *map[string]interface{}) bool {
|
||||
value1 := (*p)[ResultItemValue].([]interface{})
|
||||
value2 := (*q)[ResultItemValue].([]interface{})
|
||||
v1, _ := strconv.ParseFloat(value1[len(value1)-1].(string), 64)
|
||||
v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64)
|
||||
return v1 > v2
|
||||
}})
|
||||
}
|
||||
|
||||
i := 0
|
||||
for _, r := range metricItem.Data.Result {
|
||||
// for some reasons, 'metric' may not contain `resourceType` field
|
||||
// example: {"metric":{},"value":[1541142931.731,"3"]}
|
||||
k, exist := r[ResultItemMetric].(map[string]interface{})[resourceType]
|
||||
if exist {
|
||||
indexMap[k.(string)] = i
|
||||
i = i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// iterator all metric to find max metricItems length
|
||||
for _, r := range metricItem.Data.Result {
|
||||
k, ok := r[ResultItemMetric].(map[string]interface{})[resourceType]
|
||||
if ok {
|
||||
currentResourceMap[k.(string)] = 1
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// sort other metric
|
||||
for i := 0; i < len(fmtLevelMetric.Results); i++ {
|
||||
re := fmtLevelMetric.Results[i]
|
||||
if re.MetricName != sortMetricName && re.Data.ResultType == ResultTypeVector && re.Status == MetricStatusSuccess {
|
||||
sortedMetric := make([]map[string]interface{}, len(indexMap))
|
||||
noneSortedMetric := make([]map[string]interface{}, 0)
|
||||
for j := 0; j < len(re.Data.Result); j++ {
|
||||
r := re.Data.Result[j]
|
||||
k, ok := r[ResultItemMetric].(map[string]interface{})[resourceType]
|
||||
if ok {
|
||||
index, exist := indexMap[k.(string)]
|
||||
if exist {
|
||||
sortedMetric[index] = r
|
||||
} else {
|
||||
noneSortedMetric = append(noneSortedMetric, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sortedMetric = append(sortedMetric, noneSortedMetric...)
|
||||
fmtLevelMetric.Results[i].Data.Result = sortedMetric
|
||||
}
|
||||
}
|
||||
|
||||
return fmtLevelMetric, maxLength
|
||||
}
|
||||
|
||||
func Page(pageNum string, limitNum string, fmtLevelMetric *FormatedLevelMetric, maxLength int) interface{} {
|
||||
// matrix type can not be sorted
|
||||
for _, metricItem := range fmtLevelMetric.Results {
|
||||
if metricItem.Data.ResultType != ResultTypeVector {
|
||||
return fmtLevelMetric
|
||||
}
|
||||
}
|
||||
|
||||
var page = DefaultPage
|
||||
|
||||
if pageNum != "" {
|
||||
p, err := strconv.Atoi(pageNum)
|
||||
if err != nil {
|
||||
glog.Errorln(err)
|
||||
} else {
|
||||
if p > 0 {
|
||||
page = p
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// the default mode is none paging
|
||||
return fmtLevelMetric
|
||||
}
|
||||
|
||||
var limit = DefaultPageLimit
|
||||
|
||||
if limitNum != "" {
|
||||
l, err := strconv.Atoi(limitNum)
|
||||
if err != nil {
|
||||
glog.Errorln(err)
|
||||
} else {
|
||||
if l > 0 {
|
||||
limit = l
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// the i page: [(page-1) * limit, (page) * limit - 1]
|
||||
start := (page - 1) * limit
|
||||
end := (page)*limit - 1
|
||||
|
||||
for i := 0; i < len(fmtLevelMetric.Results); i++ {
|
||||
// only pageing when result type is `vector` and result status is `success`
|
||||
if fmtLevelMetric.Results[i].Data.ResultType != ResultTypeVector || fmtLevelMetric.Results[i].Status != MetricStatusSuccess {
|
||||
continue
|
||||
}
|
||||
resultLen := len(fmtLevelMetric.Results[i].Data.Result)
|
||||
if start >= resultLen {
|
||||
fmtLevelMetric.Results[i].Data.Result = nil
|
||||
continue
|
||||
}
|
||||
if end >= resultLen {
|
||||
end = resultLen - 1
|
||||
}
|
||||
slice := fmtLevelMetric.Results[i].Data.Result[start : end+1]
|
||||
fmtLevelMetric.Results[i].Data.Result = slice
|
||||
}
|
||||
|
||||
allPage := int(math.Ceil(float64(maxLength) / float64(limit)))
|
||||
|
||||
return &PagedFormatedLevelMetric{
|
||||
Message: "paged",
|
||||
TotalPage: allPage,
|
||||
CurrentPage: page,
|
||||
Metric: *fmtLevelMetric,
|
||||
}
|
||||
}
|
||||
|
||||
// maybe this function is time consuming
|
||||
func ReformatJson(metric string, metricsName string, needDelParams ...string) *FormatedMetric {
|
||||
var formatMetric FormatedMetric
|
||||
err := json.Unmarshal([]byte(metric), &formatMetric)
|
||||
if err != nil {
|
||||
glog.Errorln("Unmarshal metric json failed", err)
|
||||
}
|
||||
if formatMetric.MetricName == "" {
|
||||
if metricsName != "" {
|
||||
formatMetric.MetricName = metricsName
|
||||
}
|
||||
}
|
||||
// retrive metrics success
|
||||
if formatMetric.Status == MetricStatusSuccess {
|
||||
result := formatMetric.Data.Result
|
||||
for _, res := range result {
|
||||
metric, exist := res[ResultItemMetric]
|
||||
metricMap := metric.(map[string]interface{})
|
||||
if exist {
|
||||
delete(metricMap, "__name__")
|
||||
}
|
||||
if len(needDelParams) > 0 {
|
||||
for _, p := range needDelParams {
|
||||
delete(metricMap, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &formatMetric
|
||||
}
|
||||
|
||||
func ReformatNodeStatusField(nodeMetric *FormatedMetric) *FormatedMetric {
|
||||
metricCount := len(nodeMetric.Data.Result)
|
||||
for i := 0; i < metricCount; i++ {
|
||||
metric, exist := nodeMetric.Data.Result[i][ResultItemMetric]
|
||||
if exist {
|
||||
status, exist := metric.(map[string]interface{})[MetricStatus]
|
||||
if exist {
|
||||
status = UpperFirstLetter(status.(string))
|
||||
metric.(map[string]interface{})[MetricStatus] = status
|
||||
}
|
||||
}
|
||||
}
|
||||
return nodeMetric
|
||||
}
|
||||
|
||||
func UpperFirstLetter(str string) string {
|
||||
for i, ch := range str {
|
||||
return string(unicode.ToUpper(ch)) + str[i+1:]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -24,6 +24,8 @@ import (
|
||||
clientV1 "k8s.io/client-go/listers/core/v1"
|
||||
"k8s.io/kubernetes/pkg/util/slice"
|
||||
|
||||
"github.com/golang/glog"
|
||||
|
||||
"kubesphere.io/kubesphere/pkg/client"
|
||||
"kubesphere.io/kubesphere/pkg/constants"
|
||||
"kubesphere.io/kubesphere/pkg/models/controllers"
|
||||
@@ -31,6 +33,10 @@ import (
|
||||
ksErr "kubesphere.io/kubesphere/pkg/util/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
WorkspaceKey = "kubesphere.io/workspace"
|
||||
)
|
||||
|
||||
var WorkSpaceRoles = []string{"admin", "operator", "viewer"}
|
||||
|
||||
func UnBindDevopsProject(workspace string, devops string) error {
|
||||
@@ -1307,3 +1313,37 @@ func GetAllAccountNums() (int, error) {
|
||||
}
|
||||
return 0, errors.New("not found")
|
||||
}
|
||||
|
||||
// get cluster organizations name which contains at least one namespace,
|
||||
func GetAllOrgAndProjList() (map[string][]string, map[string]string, error) {
|
||||
nsList, err := client.NewK8sClient().CoreV1().Namespaces().List(meta_v1.ListOptions{})
|
||||
if err != nil {
|
||||
glog.Errorln(err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
var workspaceNamespaceMap = make(map[string][]string)
|
||||
var namespaceWorkspaceMap = make(map[string]string)
|
||||
|
||||
for _, item := range nsList.Items {
|
||||
ws, exist := item.Labels[WorkspaceKey]
|
||||
ns := item.Name
|
||||
if exist {
|
||||
if nsArray, exist := workspaceNamespaceMap[ws]; exist {
|
||||
nsArray = append(nsArray, ns)
|
||||
workspaceNamespaceMap[ws] = nsArray
|
||||
} else {
|
||||
var nsArray []string
|
||||
nsArray = append(nsArray, ns)
|
||||
workspaceNamespaceMap[ws] = nsArray
|
||||
}
|
||||
|
||||
namespaceWorkspaceMap[ns] = ws
|
||||
} else {
|
||||
// this namespace do not belong to any workspaces
|
||||
namespaceWorkspaceMap[ns] = ""
|
||||
}
|
||||
}
|
||||
|
||||
return workspaceNamespaceMap, namespaceWorkspaceMap, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user