admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add feature: logging and monitoring

Browse Source 
Signed-off-by: huanggze <loganhuang@yunify.com>
This commit is contained in:
huanggze
2019-03-19 00:09:20 +08:00
parent 8e7a1b65bc
commit b48803a735
20 changed files with 2662 additions and 437 deletions
Download Patch File Download Diff File Expand all files Collapse all files

262
pkg/models/metrics/metrics.go
View File

@@ -1,42 +1,32 @@
/*
Copyright 2019 The KubeSphere Authors.
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package metrics
import (
"fmt"
"kubesphere.io/kubesphere/pkg/models"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"kubesphere.io/kubesphere/pkg/simple/client/prometheus"
"kubesphere.io/kubesphere/pkg/informers"
"net/url"
"regexp"
"strings"
"sync"
"time"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/models/components"
"kubesphere.io/kubesphere/pkg/models/workspaces"
"github.com/golang/glog"
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"runtime/debug"
"sort"
@@ -44,12 +34,12 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"kubesphere.io/kubesphere/pkg/models/workspaces"
client "kubesphere.io/kubesphere/pkg/simple/client/prometheus"
)
var (
jsonIter = jsoniter.ConfigCompatibleWithStandardLibrary
nodeStatusDelLabel = []string{"endpoint", "instance", "job", "namespace", "pod", "service"}
)
var jsonIter = jsoniter.ConfigCompatibleWithStandardLibrary
const (
ChannelMaxCapacityWorkspaceMetric = 800
@@ -145,9 +135,9 @@ func getAllWorkspaces() map[string]int {
paramValues := make(url.Values)
paramValues.Set("query", WorkspaceNamespaceLabelRule)
params := paramValues.Encode()
res := prometheus.SendMonitoringRequest(prometheus.DefaultQueryType, params)
res := client.SendMonitoringRequest(client.DefaultQueryType, params)
metric := ReformatJson(res, "")
metric := ReformatJson(res, "", map[string]string{"workspace": "workspace"})
return getAllWorkspaceNames(metric)
}
@@ -240,17 +230,17 @@ func unifyMetricHistoryTimeRange(fmtMetrics *FormatedMetric) {
}
}
func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *prometheus.MonitoringRequestParams, metricName string) (string, string, bool) {
func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) {
nsName := monitoringRequest.NsName
wkName := monitoringRequest.WorkloadName
podsFilter := monitoringRequest.PodsFilter
wlName := monitoringRequest.WorkloadName
podsFilter := monitoringRequest.ResourcesFilter
rule := MakeSpecificWorkloadRule(monitoringRequest.WorkloadKind, wkName, nsName)
rule := MakeSpecificWorkloadRule(monitoringRequest.WorkloadKind, wlName, nsName)
paramValues := monitoringRequest.Params
params := makeRequestParamString(rule, paramValues)
res := prometheus.SendMonitoringRequest(prometheus.DefaultQueryType, params)
res := client.SendMonitoringRequest(client.DefaultQueryType, params)
podNamesFilter := getPodNameRegexInWorkload(res, podsFilter)
@@ -261,32 +251,26 @@ func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *prometheus.Mon
return queryType, params, rule == ""
}
func AssembleAllWorkloadMetricRequestInfo(monitoringRequest *prometheus.MonitoringRequestParams, metricName string) (string, string) {
func AssembleAllWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
queryType := monitoringRequest.QueryType
paramValues := monitoringRequest.Params
rule := MakeWorkloadPromQL(metricName, monitoringRequest.NsName, monitoringRequest.WlFilter)
rule := MakeWorkloadPromQL(metricName, monitoringRequest.NsName, monitoringRequest.ResourcesFilter)
params := makeRequestParamString(rule, paramValues)
return queryType, params
}
func AssemblePodMetricRequestInfo(monitoringRequest *prometheus.MonitoringRequestParams, metricName string) (string, string, bool) {
func AssemblePodMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) {
queryType := monitoringRequest.QueryType
paramValues := monitoringRequest.Params
rule := MakePodPromQL(metricName, monitoringRequest.NsName, monitoringRequest.NodeId, monitoringRequest.PodName, monitoringRequest.PodsFilter)
rule := MakePodPromQL(metricName, monitoringRequest.NsName, monitoringRequest.NodeId, monitoringRequest.PodName, monitoringRequest.ResourcesFilter)
params := makeRequestParamString(rule, paramValues)
return queryType, params, rule == ""
}
func GetMetric(queryType, params, metricName string) *FormatedMetric {
res := prometheus.SendMonitoringRequest(queryType, params)
formatedMetric := ReformatJson(res, metricName)
return formatedMetric
}
func GetNodeAddressInfo() *map[string][]v1.NodeAddress {
nodeLister := informers.SharedInformerFactory().Core().V1().Nodes().Lister()
nodes, err := nodeLister.List(labels.Everything())
@@ -319,33 +303,34 @@ func AddNodeAddressMetric(nodeMetric *FormatedMetric, nodeAddress *map[string][]
}
}
func MonitorContainer(monitoringRequest *prometheus.MonitoringRequestParams, metricName string) *FormatedMetric {
func MonitorContainer(monitoringRequest *client.MonitoringRequestParams, metricName string) *FormatedMetric {
queryType, params := AssembleContainerMetricRequestInfo(monitoringRequest, metricName)
res := GetMetric(queryType, params, metricName)
metricsStr := client.SendMonitoringRequest(queryType, params)
res := ReformatJson(metricsStr, metricName, map[string]string{"container_name": ""})
return res
}
func AssembleContainerMetricRequestInfo(monitoringRequest *prometheus.MonitoringRequestParams, metricName string) (string, string) {
func AssembleContainerMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
queryType := monitoringRequest.QueryType
paramValues := monitoringRequest.Params
rule := MakeContainerPromQL(monitoringRequest.NsName, monitoringRequest.NodeId, monitoringRequest.PodName, monitoringRequest.ContainerName, metricName, monitoringRequest.ContainersFilter)
rule := MakeContainerPromQL(monitoringRequest.NsName, monitoringRequest.NodeId, monitoringRequest.PodName, monitoringRequest.ContainerName, metricName, monitoringRequest.ResourcesFilter)
params := makeRequestParamString(rule, paramValues)
return queryType, params
}
func AssembleNamespaceMetricRequestInfo(monitoringRequest *prometheus.MonitoringRequestParams, metricName string) (string, string) {
func AssembleNamespaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
queryType := monitoringRequest.QueryType
paramValues := monitoringRequest.Params
rule := MakeNamespacePromQL(monitoringRequest.NsName, monitoringRequest.NsFilter, metricName)
rule := MakeNamespacePromQL(monitoringRequest.NsName, monitoringRequest.ResourcesFilter, metricName)
params := makeRequestParamString(rule, paramValues)
return queryType, params
}
func AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest *prometheus.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) {
func AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) {
nsFilter := "^(" + strings.Join(namespaceList, "|") + ")$"
@@ -357,7 +342,7 @@ func AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest *prometheus.Mo
return queryType, params
}
func AssembleAllWorkspaceMetricRequestInfo(monitoringRequest *prometheus.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) {
func AssembleAllWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) {
var nsFilter = "^()$"
if namespaceList != nil {
@@ -407,7 +392,7 @@ func filterNamespace(nsFilter string, namespaceList []string) []string {
return newNSlist
}
func MonitorAllWorkspaces(monitoringRequest *prometheus.MonitoringRequestParams) *FormatedLevelMetric {
func MonitorAllWorkspaces(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
metricsFilter := monitoringRequest.MetricsFilter
if strings.Trim(metricsFilter, " ") == "" {
metricsFilter = ".*"
@@ -429,7 +414,7 @@ func MonitorAllWorkspaces(monitoringRequest *prometheus.MonitoringRequestParams)
wsMap := getAllWorkspaces()
for ws := range wsMap {
bol, err := regexp.MatchString(monitoringRequest.WsFilter, ws)
bol, err := regexp.MatchString(monitoringRequest.ResourcesFilter, ws)
if err == nil && bol {
// a workspace
wgAll.Add(1)
@@ -470,7 +455,7 @@ func MonitorAllWorkspaces(monitoringRequest *prometheus.MonitoringRequestParams)
}
}
func collectWorkspaceMetric(monitoringRequest *prometheus.MonitoringRequestParams, ws string, filterMetricsName []string, wgAll *sync.WaitGroup, wsAllch chan *[]FormatedMetric) {
func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, ws string, filterMetricsName []string, wgAll *sync.WaitGroup, wsAllch chan *[]FormatedMetric) {
defer wgAll.Done()
var wg sync.WaitGroup
var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
@@ -484,7 +469,8 @@ func collectWorkspaceMetric(monitoringRequest *prometheus.MonitoringRequestParam
go func(metricName string) {
queryType, params := AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName)
ch <- GetMetric(queryType, params, metricName)
metricsStr := client.SendMonitoringRequest(queryType, params)
ch <- ReformatJson(metricsStr, metricName, map[string]string{"namespace": ""})
wg.Done()
}(metricName)
@@ -511,7 +497,7 @@ func collectWorkspaceMetric(monitoringRequest *prometheus.MonitoringRequestParam
wsAllch <- &metricsArray
}
func MonitorAllMetrics(monitoringRequest *prometheus.MonitoringRequestParams, resourceType string) *FormatedLevelMetric {
func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resourceType string) *FormatedLevelMetric {
metricsFilter := monitoringRequest.MetricsFilter
if metricsFilter == "" {
metricsFilter = ".*"
@@ -529,10 +515,8 @@ func MonitorAllMetrics(monitoringRequest *prometheus.MonitoringRequestParams, re
wg.Add(1)
go func(metricName string) {
queryType, params := AssembleClusterMetricRequestInfo(monitoringRequest, metricName)
clusterMetrics := GetMetric(queryType, params, metricName)
ch <- clusterMetrics
metricsStr := client.SendMonitoringRequest(queryType, params)
ch <- ReformatJson(metricsStr, metricName, map[string]string{"cluster": "local"})
wg.Done()
}(metricName)
}
@@ -546,7 +530,8 @@ func MonitorAllMetrics(monitoringRequest *prometheus.MonitoringRequestParams, re
wg.Add(1)
go func(metricName string) {
queryType, params := AssembleNodeMetricRequestInfo(monitoringRequest, metricName)
ch <- GetMetric(queryType, params, metricName)
metricsStr := client.SendMonitoringRequest(queryType, params)
ch <- ReformatJson(metricsStr, metricName, map[string]string{"node": ""})
wg.Done()
}(metricName)
}
@@ -560,7 +545,7 @@ func MonitorAllMetrics(monitoringRequest *prometheus.MonitoringRequestParams, re
if err != nil {
glog.Errorln(err.Error())
}
namespaceArray = filterNamespace(monitoringRequest.NsFilter, namespaceArray)
namespaceArray = filterNamespace(monitoringRequest.ResourcesFilter, namespaceArray)
if monitoringRequest.Tp == "rank" {
for _, metricName := range NamespaceMetricsNames {
@@ -570,12 +555,13 @@ func MonitorAllMetrics(monitoringRequest *prometheus.MonitoringRequestParams, re
bol, err := regexp.MatchString(metricsFilter, metricName)
ns := "^(" + strings.Join(namespaceArray, "|") + ")$"
monitoringRequest.NsFilter = ns
monitoringRequest.ResourcesFilter = ns
if err == nil && bol {
wg.Add(1)
go func(metricName string) {
queryType, params := AssembleNamespaceMetricRequestInfo(monitoringRequest, metricName)
ch <- GetMetric(queryType, params, metricName)
metricsStr := client.SendMonitoringRequest(queryType, params)
ch <- ReformatJson(metricsStr, metricName, map[string]string{"workspace": "workspace"})
wg.Done()
}(metricName)
}
@@ -593,7 +579,8 @@ func MonitorAllMetrics(monitoringRequest *prometheus.MonitoringRequestParams, re
wg.Add(1)
go func(metricName string) {
queryType, params := AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName)
ch <- GetMetric(queryType, params, metricName)
metricsStr := client.SendMonitoringRequest(queryType, params)
ch <- ReformatJson(metricsStr, metricName, map[string]string{"workspace": "workspace"})
wg.Done()
}(metricName)
}
@@ -610,9 +597,8 @@ func MonitorAllMetrics(monitoringRequest *prometheus.MonitoringRequestParams, re
go func(metricName string) {
queryType, params := AssembleAllWorkspaceMetricRequestInfo(monitoringRequest, nil, metricName)
ch <- GetMetric(queryType, params, metricName)
metricsStr := client.SendMonitoringRequest(queryType, params)
ch <- ReformatJson(metricsStr, metricName, map[string]string{"workspace": "workspaces"})
wg.Done()
}(metricName)
}
@@ -627,7 +613,8 @@ func MonitorAllMetrics(monitoringRequest *prometheus.MonitoringRequestParams, re
wg.Add(1)
go func(metricName string) {
queryType, params := AssembleNamespaceMetricRequestInfo(monitoringRequest, metricName)
ch <- GetMetric(queryType, params, metricName)
metricsStr := client.SendMonitoringRequest(queryType, params)
ch <- ReformatJson(metricsStr, metricName, map[string]string{"namespace": ""})
wg.Done()
}(metricName)
}
@@ -635,15 +622,15 @@ func MonitorAllMetrics(monitoringRequest *prometheus.MonitoringRequestParams, re
}
case MetricLevelWorkload:
{
if monitoringRequest.Tp == "rank" {
if monitoringRequest.WorkloadName == "" {
for _, metricName := range WorkloadMetricsNames {
bol, err := regexp.MatchString(metricsFilter, metricName)
if err == nil && bol {
wg.Add(1)
go func(metricName string) {
queryType, params := AssembleAllWorkloadMetricRequestInfo(monitoringRequest, metricName)
fmtMetrics := GetMetric(queryType, params, metricName)
ch <- fmtMetrics
metricsStr := client.SendMonitoringRequest(queryType, params)
ch <- ReformatJson(metricsStr, metricName, map[string]string{"workload": ""})
wg.Done()
}(metricName)
}
@@ -657,7 +644,8 @@ func MonitorAllMetrics(monitoringRequest *prometheus.MonitoringRequestParams, re
metricName = strings.TrimLeft(metricName, "workload_")
queryType, params, nullRule := AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest, metricName)
if !nullRule {
fmtMetrics := GetMetric(queryType, params, metricName)
metricsStr := client.SendMonitoringRequest(queryType, params)
fmtMetrics := ReformatJson(metricsStr, metricName, map[string]string{"pod_name": ""})
unifyMetricHistoryTimeRange(fmtMetrics)
ch <- fmtMetrics
}
@@ -676,7 +664,8 @@ func MonitorAllMetrics(monitoringRequest *prometheus.MonitoringRequestParams, re
go func(metricName string) {
queryType, params, nullRule := AssemblePodMetricRequestInfo(monitoringRequest, metricName)
if !nullRule {
ch <- GetMetric(queryType, params, metricName)
metricsStr := client.SendMonitoringRequest(queryType, params)
ch <- ReformatJson(metricsStr, metricName, map[string]string{"pod_name": ""})
} else {
ch <- nil
}
@@ -693,7 +682,8 @@ func MonitorAllMetrics(monitoringRequest *prometheus.MonitoringRequestParams, re
wg.Add(1)
go func(metricName string) {
queryType, params := AssembleContainerMetricRequestInfo(monitoringRequest, metricName)
ch <- GetMetric(queryType, params, metricName)
metricsStr := client.SendMonitoringRequest(queryType, params)
ch <- ReformatJson(metricsStr, metricName, map[string]string{"container_name": ""})
wg.Done()
}(metricName)
}
@@ -868,125 +858,7 @@ func getSpecificMetricItem(timestamp int64, metricName string, resource string,
return &nsMetrics
}
// k8s component(controller, scheduler, etcd) status
func MonitorComponentStatus(monitoringRequest *prometheus.MonitoringRequestParams) *[]interface{} {
componentList, err := k8s.Client().CoreV1().ComponentStatuses().List(metaV1.ListOptions{})
if err != nil {
glog.Errorln(err.Error())
}
var componentStatusList []*ComponentStatus
for _, item := range componentList.Items {
var status []OneComponentStatus
for _, cond := range item.Conditions {
status = append(status, OneComponentStatus{
Type: string(cond.Type),
Status: string(cond.Status),
Message: cond.Message,
Error: cond.Error,
})
}
componentStatusList = append(componentStatusList, &ComponentStatus{
Name: item.Name,
Namespace: item.Namespace,
Labels: item.Labels,
ComponentStatus: status,
})
}
// node status
queryType := monitoringRequest.QueryType
paramValues := monitoringRequest.Params
paramValues.Set("query", NodeStatusRule)
params := paramValues.Encode()
res := prometheus.SendMonitoringRequest(queryType, params)
nodeStatusMetric := ReformatJson(res, "node_status", nodeStatusDelLabel...)
nodeStatusMetric = ReformatNodeStatusField(nodeStatusMetric)
var normalNodes []string
var abnormalNodes []string
for _, result := range nodeStatusMetric.Data.Result {
tmap, sure := result[ResultItemMetric].(map[string]interface{})
if sure {
if tmap[MetricStatus].(string) == "false" {
abnormalNodes = append(abnormalNodes, tmap[MetricLevelNode].(string))
} else {
normalNodes = append(normalNodes, tmap[MetricLevelNode].(string))
}
}
}
Components, err := components.GetAllComponentsStatus()
if err != nil {
glog.Error(err.Error())
}
var namespaceComponentHealthyMap = make(map[string]int)
var namespaceComponentTotalMap = make(map[string]int)
for _, ns := range constants.SystemNamespaces {
nsStatus, exist := Components[ns]
if exist {
for _, nsStatusItem := range nsStatus.(map[string]interface{}) {
component := nsStatusItem.(models.Component)
namespaceComponentTotalMap[ns] += 1
if component.HealthyBackends != 0 && component.HealthyBackends == component.TotalBackends {
namespaceComponentHealthyMap[ns] += 1
}
}
}
}
timestamp := int64(time.Now().Unix())
onlineMetricItems := makeMetricItems(timestamp, namespaceComponentHealthyMap, MetricLevelNamespace)
metricItems := makeMetricItems(timestamp, namespaceComponentTotalMap, MetricLevelNamespace)
var assembleList []interface{}
assembleList = append(assembleList, nodeStatusMetric)
for _, statusItem := range componentStatusList {
assembleList = append(assembleList, statusItem)
}
assembleList = append(assembleList, FormatedMetric{
Data: FormatedMetricData{
Result: *onlineMetricItems,
ResultType: ResultTypeVector,
},
MetricName: MetricNameComponentOnLine,
Status: MetricStatusSuccess,
})
assembleList = append(assembleList, FormatedMetric{
Data: FormatedMetricData{
Result: *metricItems,
ResultType: ResultTypeVector,
},
MetricName: MetricNameComponentLine,
Status: MetricStatusSuccess,
})
return &assembleList
}
func makeMetricItems(timestamp int64, statusMap map[string]int, resourceType string) *[]map[string]interface{} {
var metricItems []map[string]interface{}
for ns, count := range statusMap {
metricItems = append(metricItems, map[string]interface{}{
ResultItemMetric: map[string]string{resourceType: ns},
ResultItemValue: []interface{}{timestamp, fmt.Sprintf("%d", count)},
})
}
return &metricItems
}
func AssembleClusterMetricRequestInfo(monitoringRequest *prometheus.MonitoringRequestParams, metricName string) (string, string) {
func AssembleClusterMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
queryType := monitoringRequest.QueryType
paramValues := monitoringRequest.Params
rule := MakeClusterRule(metricName)
@@ -995,10 +867,10 @@ func AssembleClusterMetricRequestInfo(monitoringRequest *prometheus.MonitoringRe
return queryType, params
}
func AssembleNodeMetricRequestInfo(monitoringRequest *prometheus.MonitoringRequestParams, metricName string) (string, string) {
func AssembleNodeMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
queryType := monitoringRequest.QueryType
paramValues := monitoringRequest.Params
rule := MakeNodeRule(monitoringRequest.NodeId, monitoringRequest.NodesFilter, metricName)
rule := MakeNodeRule(monitoringRequest.NodeId, monitoringRequest.ResourcesFilter, metricName)
params := makeRequestParamString(rule, paramValues)
return queryType, params

28
pkg/models/metrics/metricsruleconst.go
View File

@@ -318,14 +318,14 @@ var RulePromQLTmplMap = MetricMap{
"cluster_disk_read_throughput": "sum(node:data_volume_throughput_bytes_read:sum)",
"cluster_disk_write_throughput": "sum(node:data_volume_throughput_bytes_written:sum)",
"cluster_disk_size_usage": `sum(max((node_filesystem_size{device=~"/dev/.+", job="node-exporter"} - node_filesystem_avail{device=~"/dev/.+", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:) by (node))`,
"cluster_disk_size_utilisation": `1 - sum(max(node_filesystem_avail{device=~"/dev/.+", job="node-exporter"} * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:) by (node)) / sum(max(node_filesystem_size{device=~"/dev/.+", job="node-exporter"} * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:) by (node))`,
"cluster_disk_size_capacity": `sum(max(node_filesystem_size{device=~"/dev/.+", job="node-exporter"} * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:) by (node))`,
"cluster_disk_size_available": `sum(max(node_filesystem_avail{device=~"/dev/.+", job="node-exporter"} * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:) by (node))`,
"cluster_disk_size_usage": `sum(max((node_filesystem_size_bytes{device=~"/dev/.+", job="node-exporter"} - node_filesystem_avail_bytes{device=~"/dev/.+", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:) by (node))`,
"cluster_disk_size_utilisation": `1 - sum(max(node_filesystem_avail_bytes{device=~"/dev/.+", job="node-exporter"} * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:) by (node)) / sum(max(node_filesystem_size_bytes{device=~"/dev/.+", job="node-exporter"} * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:) by (node))`,
"cluster_disk_size_capacity": `sum(max(node_filesystem_size_bytes{device=~"/dev/.+", job="node-exporter"} * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:) by (node))`,
"cluster_disk_size_available": `sum(max(node_filesystem_avail_bytes{device=~"/dev/.+", job="node-exporter"} * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:) by (node))`,
"cluster_disk_inode_total": `sum(node:disk_inodes_total:)`,
"cluster_disk_inode_usage": `sum(node:disk_inodes_total:) - sum(node:disk_inodes_free:)`,
"cluster_disk_inode_utilisation": `1 - sum(node:disk_inodes_free:) / sum(node:disk_inodes_total:)`,
"cluster_disk_inode_total": `sum(node:node_inodes_total:)`,
"cluster_disk_inode_usage": `sum(node:node_inodes_total:) - sum(node:node_inodes_free:)`,
"cluster_disk_inode_utilisation": `1 - sum(node:node_inodes_free:) / sum(node:node_inodes_total:)`,
"cluster_namespace_count": `count(kube_namespace_annotations)`,
@@ -396,14 +396,14 @@ var RulePromQLTmplMap = MetricMap{
"node_disk_read_throughput": "node:data_volume_throughput_bytes_read:sum",
"node_disk_write_throughput": "node:data_volume_throughput_bytes_written:sum",
"node_disk_size_capacity": `max(node_filesystem_size{device=~"/dev/.+", job="node-exporter"} * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) by (node)`,
"node_disk_size_available": `max(node_filesystem_avail{device=~"/dev/.+", job="node-exporter"} * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) by (node)`,
"node_disk_size_usage": `max((node_filesystem_size{device=~"/dev/.+", job="node-exporter"} - node_filesystem_avail{device=~"/dev/.+", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) by (node)`,
"node_disk_size_utilisation": `max(((node_filesystem_size{device=~"/dev/.+", job="node-exporter"} - node_filesystem_avail{device=~"/dev/.+", job="node-exporter"}) / node_filesystem_size{device=~"/dev/.+", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) by (node)`,
"node_disk_size_capacity": `max(node_filesystem_size_bytes{device=~"/dev/.+", job="node-exporter"} * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) by (node)`,
"node_disk_size_available": `max(node_filesystem_avail_bytes{device=~"/dev/.+", job="node-exporter"} * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) by (node)`,
"node_disk_size_usage": `max((node_filesystem_size_bytes{device=~"/dev/.+", job="node-exporter"} - node_filesystem_avail_bytes{device=~"/dev/.+", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) by (node)`,
"node_disk_size_utilisation": `max(((node_filesystem_size_bytes{device=~"/dev/.+", job="node-exporter"} - node_filesystem_avail_bytes{device=~"/dev/.+", job="node-exporter"}) / node_filesystem_size_bytes{device=~"/dev/.+", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1) by (node)`,
"node_disk_inode_total": `node:disk_inodes_total:$1`,
"node_disk_inode_usage": `node:disk_inodes_total:$1 - node:disk_inodes_free:$1`,
"node_disk_inode_utilisation": `(1 - (node:disk_inodes_free:$1 / node:disk_inodes_total:$1))`,
"node_disk_inode_total": `node:node_inodes_total:$1`,
"node_disk_inode_usage": `node:node_inodes_total:$1 - node:node_inodes_free:$1`,
"node_disk_inode_utilisation": `(1 - (node:node_inodes_free:$1 / node:node_inodes_total:$1))`,
"node_pod_count": `sum by (node) ((kube_pod_status_scheduled{condition="true"} > 0) * on (namespace, pod) group_left(node) kube_pod_info$1 unless on (node) (kube_node_status_condition{condition="Ready",status=~"unknown|false"} > 0))`,
"node_pod_quota": `sum(kube_node_status_capacity_pods$1) by (node) unless on (node) (kube_node_status_condition{condition="Ready",status=~"unknown|false"} > 0)`,

12
pkg/models/metrics/namespaces.go
View File

@@ -23,7 +23,7 @@ import (
"k8s.io/api/core/v1"
"kubesphere.io/kubesphere/pkg/simple/client/prometheus"
prom "kubesphere.io/kubesphere/pkg/simple/client/prometheus"
)
func GetNamespacesWithMetrics(namespaces []*v1.Namespace) []*v1.Namespace {
@@ -34,11 +34,11 @@ func GetNamespacesWithMetrics(namespaces []*v1.Namespace) []*v1.Namespace {
nsFilter := "^(" + strings.Join(nsNameList, "|") + ")$"
var timeRelateParams = make(url.Values)
params := prometheus.MonitoringRequestParams{
NsFilter: nsFilter,
Params: timeRelateParams,
QueryType: prometheus.DefaultQueryType,
MetricsFilter: "namespace_cpu_usage|namespace_memory_usage_wo_cache|namespace_pod_count",
params := prom.MonitoringRequestParams{
ResourcesFilter: nsFilter,
Params: timeRelateParams,
QueryType: prom.DefaultQueryType,
MetricsFilter: "namespace_cpu_usage|namespace_memory_usage_wo_cache|namespace_pod_count",
}
rawMetrics := MonitorAllMetrics(&params, MetricLevelNamespace)

31
pkg/models/metrics/util.go
View File

@@ -50,7 +50,7 @@ func (wrapper FormatedMetricDataWrapper) Swap(i, j int) {
}
// sorted metric by ascending or descending order
func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelMetric, resourceType string) (*FormatedLevelMetric, int) {
func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelMetric) (*FormatedLevelMetric, int) {
defer func() {
if err := recover(); err != nil {
glog.Errorln(err)
@@ -83,8 +83,8 @@ func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelM
v1, _ := strconv.ParseFloat(value1[len(value1)-1].(string), 64)
v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64)
if v1 == v2 {
resourceName1 := (*p)[ResultItemMetric].(map[string]interface{})[resourceType]
resourceName2 := (*q)[ResultItemMetric].(map[string]interface{})[resourceType]
resourceName1 := (*p)[ResultItemMetric].(map[string]interface{})["resource_name"]
resourceName2 := (*q)[ResultItemMetric].(map[string]interface{})["resource_name"]
return resourceName1.(string) < resourceName2.(string)
}
@@ -99,8 +99,8 @@ func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelM
v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64)
if v1 == v2 {
resourceName1 := (*p)[ResultItemMetric].(map[string]interface{})[resourceType]
resourceName2 := (*q)[ResultItemMetric].(map[string]interface{})[resourceType]
resourceName1 := (*p)[ResultItemMetric].(map[string]interface{})["resource_name"]
resourceName2 := (*q)[ResultItemMetric].(map[string]interface{})["resource_name"]
return resourceName1.(string) > resourceName2.(string)
}
@@ -111,7 +111,7 @@ func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelM
for _, r := range metricItem.Data.Result {
// for some reasons, 'metric' may not contain `resourceType` field
// example: {"metric":{},"value":[1541142931.731,"3"]}
k, exist := r[ResultItemMetric].(map[string]interface{})[resourceType]
k, exist := r[ResultItemMetric].(map[string]interface{})["resource_name"]
key := k.(string)
if exist {
if _, exist := indexMap[key]; !exist {
@@ -125,7 +125,7 @@ func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelM
// iterator all metric to find max metricItems length
for _, r := range metricItem.Data.Result {
k, ok := r[ResultItemMetric].(map[string]interface{})[resourceType]
k, ok := r[ResultItemMetric].(map[string]interface{})["resource_name"]
if ok {
currentResourceMap[k.(string)] = 1
}
@@ -154,7 +154,7 @@ func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelM
sortedMetric := make([]map[string]interface{}, len(indexMap))
for j := 0; j < len(re.Data.Result); j++ {
r := re.Data.Result[j]
k, exist := r[ResultItemMetric].(map[string]interface{})[resourceType]
k, exist := r[ResultItemMetric].(map[string]interface{})["resource_name"]
if exist {
index, exist := indexMap[k.(string)]
if exist {
@@ -176,7 +176,7 @@ func Page(pageNum string, limitNum string, fmtLevelMetric *FormatedLevelMetric,
}
// matrix type can not be sorted
for _, metricItem := range fmtLevelMetric.Results {
// if metric reterieved field, resultType is ""
// if metric reterieved field, resultType: ""
if metricItem.Data.ResultType == ResultTypeMatrix {
return fmtLevelMetric
}
@@ -251,7 +251,7 @@ func Page(pageNum string, limitNum string, fmtLevelMetric *FormatedLevelMetric,
}
// maybe this function is time consuming
func ReformatJson(metric string, metricsName string, needDelParams ...string) *FormatedMetric {
func ReformatJson(metric string, metricsName string, needAddParams map[string]string, needDelParams ...string) *FormatedMetric {
var formatMetric FormatedMetric
err := jsonIter.Unmarshal([]byte(metric), &formatMetric)
@@ -277,6 +277,17 @@ func ReformatJson(metric string, metricsName string, needDelParams ...string) *F
delete(metricMap, p)
}
}
if needAddParams != nil && len(needAddParams) > 0 {
for n := range needAddParams {
if v, ok := metricMap[n]; ok {
delete(metricMap, n)
metricMap["resource_name"] = v
} else {
metricMap["resource_name"] = needAddParams[n]
}
}
}
}
}