From 53dd54b163b7f1e9648d799d4706bfb988c693cd Mon Sep 17 00:00:00 2001 From: Carman Zhang Date: Thu, 27 Sep 2018 16:08:58 +0800 Subject: [PATCH] add monitoring apis --- pkg/apis/v1alpha/install.go | 2 + .../v1alpha/monitoring/monitor_handler.go | 205 ++++++++++++++++++ pkg/client/prometheusclient.go | 106 +++++++++ pkg/models/metrics/metrics_collector.go | 184 ++++++++++++++++ pkg/models/metrics/metrics_rule.go | 193 +++++++++++++++++ pkg/models/metrics/metrics_rule_tmpl.go | 147 +++++++++++++ pkg/models/metrics/metrics_struct.go | 69 ++++++ 7 files changed, 906 insertions(+) create mode 100644 pkg/apis/v1alpha/monitoring/monitor_handler.go create mode 100644 pkg/client/prometheusclient.go create mode 100644 pkg/models/metrics/metrics_collector.go create mode 100644 pkg/models/metrics/metrics_rule.go create mode 100644 pkg/models/metrics/metrics_rule_tmpl.go create mode 100644 pkg/models/metrics/metrics_struct.go diff --git a/pkg/apis/v1alpha/install.go b/pkg/apis/v1alpha/install.go index 48db15ea6..eeedb44c7 100644 --- a/pkg/apis/v1alpha/install.go +++ b/pkg/apis/v1alpha/install.go @@ -26,6 +26,7 @@ import ( "kubesphere.io/kubesphere/pkg/apis/v1alpha/hpa" "kubesphere.io/kubesphere/pkg/apis/v1alpha/iam" "kubesphere.io/kubesphere/pkg/apis/v1alpha/jobs" + "kubesphere.io/kubesphere/pkg/apis/v1alpha/monitoring" "kubesphere.io/kubesphere/pkg/apis/v1alpha/nodes" "kubesphere.io/kubesphere/pkg/apis/v1alpha/pods" "kubesphere.io/kubesphere/pkg/apis/v1alpha/quota" @@ -65,6 +66,7 @@ func init() { daemonsets.Register(ws, "/namespaces/{namespace}/daemonsets/{daemonset}/revisions/{revision}") statefulsets.Register(ws, "/namespaces/{namespace}/statefulsets/{statefulset}/revisions/{revision}") resources.Register(ws, "/resources") + monitoring.Register(ws, "/monitoring") // add webservice to default container restful.Add(ws) diff --git a/pkg/apis/v1alpha/monitoring/monitor_handler.go b/pkg/apis/v1alpha/monitoring/monitor_handler.go new file mode 100644 index 000000000..6c08836e4 --- /dev/null +++ b/pkg/apis/v1alpha/monitoring/monitor_handler.go @@ -0,0 +1,205 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package monitoring + +import ( + "strings" + + "github.com/emicklei/go-restful" + "github.com/emicklei/go-restful-openapi" + + "kubesphere.io/kubesphere/pkg/client" + "kubesphere.io/kubesphere/pkg/filter/route" + "kubesphere.io/kubesphere/pkg/models/metrics" +) + +func (u MonitorResource) monitorPod(request *restful.Request, response *restful.Response) { + podName := strings.Trim(request.PathParameter("pod_name"), " ") + if podName != "" { + // single pod single metric + metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") + res := metrics.MonitorPodSingleMetric(request, metricsName) + response.WriteAsJson(res) + } else { + // multiple pod multiple metric + res := metrics.MonitorAllMetrics(request) + response.WriteAsJson(res) + } +} + +func (u MonitorResource) monitorContainer(request *restful.Request, response *restful.Response) { + metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") + promql := metrics.MakeContainerPromQL(request) + res := client.SendPrometheusRequest(request, promql) + cleanedJson := metrics.ReformatJson(res, metricsName) + response.WriteAsJson(cleanedJson) +} + +func (u MonitorResource) monitorWorkload(request *restful.Request, response *restful.Response) { + res := metrics.MonitorAllMetrics(request) + response.WriteAsJson(res) +} + +func (u MonitorResource) monitorNamespace(request *restful.Request, response *restful.Response) { + nsName := strings.Trim(request.PathParameter("ns_name"), " ") + if nsName != "" { + // single + metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") + res := metrics.MonitorNamespaceSingleMetric(request, metricsName) + response.WriteAsJson(res) + } else { + // multiple + res := metrics.MonitorAllMetrics(request) + response.WriteAsJson(res) + } +} + +func (u MonitorResource) monitorNodeorCluster(request *restful.Request, response *restful.Response) { + metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") + //var res *metrics.FormatedMetric + if metricsName != "" { + // single + res := metrics.MonitorNodeorClusterSingleMetric(request, metricsName) + response.WriteAsJson(res) + } else { + // multiple + res := metrics.MonitorAllMetrics(request) + response.WriteAsJson(res) + } +} + +type MonitorResource struct { +} + +func Register(ws *restful.WebService, subPath string) { + tags := []string{"monitoring apis"} + u := MonitorResource{} + + ws.Route(ws.GET(subPath+"/clusters").To(u.monitorNodeorCluster). + Filter(route.RouteLogging). + Doc("monitor cluster level metrics"). + Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("cluster_cpu_utilisation")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/nodes").To(u.monitorNodeorCluster). + Filter(route.RouteLogging). + Doc("monitor nodes level metrics"). + Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("node_cpu_utilisation")). + Param(ws.QueryParameter("nodes_filter", "node re2 expression filter").DataType("string").Required(false).DefaultValue("")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/nodes/{node_id}").To(u.monitorNodeorCluster). + Filter(route.RouteLogging). + Doc("monitor specific node level metrics"). + Param(ws.PathParameter("node_id", "specific node").DataType("string").Required(true).DefaultValue("")). + Param(ws.QueryParameter("metrics_name", "metrics name cpu memory...").DataType("string").Required(true).DefaultValue("node_cpu_utilisation")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/namespaces").To(u.monitorNamespace). + Filter(route.RouteLogging). + Doc("monitor namespaces level metrics"). + Param(ws.QueryParameter("namespaces_filter", "namespaces re2 expression filter").DataType("string").Required(false).DefaultValue("")). + Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("namespace_memory_utilisation")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/namespaces/{ns_name}").To(u.monitorNamespace). + Filter(route.RouteLogging). + Doc("monitor specific namespace level metrics"). + Param(ws.PathParameter("ns_name", "specific namespace").DataType("string").Required(true).DefaultValue("monitoring")). + Param(ws.QueryParameter("metrics_name", "metrics name cpu memory...").DataType("string").Required(true).DefaultValue("namespace_memory_utilisation")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/namespaces/{ns_name}/pods").To(u.monitorPod). + Filter(route.RouteLogging). + Doc("monitor pods level metrics"). + Param(ws.PathParameter("ns_name", "specific namespace").DataType("string").Required(true).DefaultValue("monitoring")). + Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("")). + Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("pod_memory_utilisation_wo_cache")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/namespaces/{ns_name}/pods/{pod_name}").To(u.monitorPod). + Filter(route.RouteLogging). + Doc("monitor specific pod level metrics"). + Param(ws.PathParameter("ns_name", "specific namespace").DataType("string").Required(true).DefaultValue("monitoring")). + Param(ws.PathParameter("pod_name", "specific pod").DataType("string").Required(true).DefaultValue("")). + Param(ws.QueryParameter("metrics_name", "metrics name cpu memory...").DataType("string").Required(true).DefaultValue("pod_memory_utilisation_wo_cache")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/nodes/{node_id}/pods").To(u.monitorPod). + Filter(route.RouteLogging). + Doc("monitor pods level metrics by nodeid"). + Param(ws.PathParameter("node_id", "specific node").DataType("string").Required(true).DefaultValue("i-k89a62il")). + Param(ws.QueryParameter("pods_filter", "pod re2 expression filter").DataType("string").Required(false).DefaultValue("openpitrix.*")). + Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("pod_memory_utilisation_wo_cache")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/nodes/{node_id}/pods/{pod_name}").To(u.monitorPod). + Filter(route.RouteLogging). + Doc("monitor specific pod level metrics by nodeid"). + Param(ws.PathParameter("node_id", "specific node").DataType("string").Required(true).DefaultValue("i-k89a62il")). + Param(ws.PathParameter("pod_name", "specific pod").DataType("string").Required(true).DefaultValue("")). + Param(ws.QueryParameter("metrics_name", "metrics name cpu memory...").DataType("string").Required(true).DefaultValue("pod_memory_utilisation_wo_cache")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/namespaces/{ns_name}/pods/{pod_name}/containers").To(u.monitorContainer). + Filter(route.RouteLogging). + Doc("monitor containers level metrics"). + Param(ws.PathParameter("ns_name", "specific namespace").DataType("string").Required(true).DefaultValue("monitoring")). + Param(ws.PathParameter("pod_name", "specific pod").DataType("string").Required(true).DefaultValue("")). + Param(ws.QueryParameter("containers_filter", "container re2 expression filter").DataType("string").Required(false).DefaultValue("")). + Param(ws.QueryParameter("metrics_name", "metrics name cpu memory...").DataType("string").Required(true).DefaultValue("container_memory_utilisation_wo_cache")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/namespaces/{ns_name}/pods/{pod_name}/containers/{container_name}").To(u.monitorContainer). + Filter(route.RouteLogging). + Doc("monitor specific container level metrics"). + Param(ws.PathParameter("ns_name", "specific namespace").DataType("string").Required(true).DefaultValue("monitoring")). + Param(ws.PathParameter("pod_name", "specific pod").DataType("string").Required(true).DefaultValue("")). + Param(ws.PathParameter("container_name", "specific container").DataType("string").Required(true).DefaultValue("")). + Param(ws.QueryParameter("metrics_name", "metrics name cpu memory...").DataType("string").Required(true).DefaultValue("container_memory_utilisation_wo_cache")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET(subPath+"/namespaces/{ns_name}/workloads/{workload_kind}").To(u.monitorWorkload). + Filter(route.RouteLogging). + Doc("monitor specific workload level metrics"). + Param(ws.PathParameter("ns_name", "namespace").DataType("string").Required(true).DefaultValue("kube-system")). + Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...").DataType("string").Required(false)). + Param(ws.PathParameter("workload_kind", "workload kind").DataType("string").Required(true).DefaultValue("daemonset")). + Param(ws.QueryParameter("workload_name", "workload name").DataType("string").Required(true).DefaultValue("")). + Metadata(restfulspec.KeyOpenAPITags, tags)). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON) + +} diff --git a/pkg/client/prometheusclient.go b/pkg/client/prometheusclient.go new file mode 100644 index 000000000..9bfc1b056 --- /dev/null +++ b/pkg/client/prometheusclient.go @@ -0,0 +1,106 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package client + +import ( + "io/ioutil" + "net/http" + "net/url" + + "github.com/emicklei/go-restful" + "github.com/golang/glog" + "github.com/pkg/errors" +) + +const ( + DefaultScheme = "http" + DefaultPrometheusService = "prometheus-k8s.monitoring.svc" + DefaultPrometheusPort = "9090" + PrometheusApiPath = "/api/v1/" + PrometheusEndpointUrl = DefaultScheme + "://" + DefaultPrometheusService + ":" + DefaultPrometheusPort + PrometheusApiPath +) + +var client = &http.Client{} + +func SendRequest(postfix string, params string) string { + epurl := PrometheusEndpointUrl + postfix + params + //glog.Info("monitoring epurl:>", epurl) + response, err := client.Get(epurl) + if err != nil { + glog.Error(err) + } else { + defer response.Body.Close() + + contents, err := ioutil.ReadAll(response.Body) + + if err != nil { + glog.Error(err) + } + return string(contents) + } + return "" +} + +func SendPrometheusRequest(request *restful.Request, recordingRule string) string { + paramsMap, bol, err := ParseRequestHeader(request) + if err != nil { + glog.Error(err) + return "" + } + + var res = "" + var postfix = "" + if bol { + // range query + postfix = "query_range?" + } else { + // query + postfix = "query?" + } + paramsMap.Set("query", recordingRule) + params := paramsMap.Encode() + res = SendRequest(postfix, params) + return res +} + +func ParseRequestHeader(request *restful.Request) (url.Values, bool, error) { + instantTime := request.QueryParameter("time") + start := request.QueryParameter("start") + end := request.QueryParameter("end") + step := request.QueryParameter("step") + timeout := request.QueryParameter("timeout") + if timeout == "" { + timeout = "30s" + } + // Whether query or query_range request + u := url.Values{} + if start != "" && end != "" && step != "" { + u.Set("start", start) + u.Set("end", end) + u.Set("step", step) + u.Set("timeout", timeout) + return u, true, nil + } + if instantTime != "" { + u.Set("time", instantTime) + u.Set("timeout", timeout) + return u, false, nil + } else { + //u.Set("time", strconv.FormatInt(int64(time.Now().Unix()), 10)) + u.Set("timeout", timeout) + return u, false, nil + } + + glog.Error("Parse request failed", u) + return u, false, errors.Errorf("Parse request failed") +} diff --git a/pkg/models/metrics/metrics_collector.go b/pkg/models/metrics/metrics_collector.go new file mode 100644 index 000000000..8f5762cdc --- /dev/null +++ b/pkg/models/metrics/metrics_collector.go @@ -0,0 +1,184 @@ +/* +Copyright 2018 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "encoding/json" + "regexp" + "strings" + + "github.com/emicklei/go-restful" + "github.com/golang/glog" + + "kubesphere.io/kubesphere/pkg/client" +) + +func getPodNameRegexInWorkload(request *restful.Request) string { + promql := MakeWorkloadRule(request) + res := client.SendPrometheusRequest(request, promql) + data := []byte(res) + var dat CommonMetricsResult + jsonErr := json.Unmarshal(data, &dat) + if jsonErr != nil { + glog.Errorln("json parse failed", jsonErr) + } + var podNames []string + for _, x := range dat.Data.Result { + podName := x.KubePodMetric.Pod + podNames = append(podNames, podName) + } + podNamesFilter := "^(" + strings.Join(podNames, "|") + ")$" + return podNamesFilter +} + +func MonitorWorkloadSingleMetric(request *restful.Request, metricsName string) *FormatedMetric { + nsName := strings.Trim(request.PathParameter("ns_name"), " ") + podNamesFilter := getPodNameRegexInWorkload(request) + newPromql := MakePodPromQL(request, []string{metricsName, nsName, "", "", podNamesFilter}) + podMetrics := client.SendPrometheusRequest(request, newPromql) + cleanedJson := ReformatJson(podMetrics, metricsName) + return cleanedJson +} + +func MonitorPodSingleMetric(request *restful.Request, metricsName string) *FormatedMetric { + nsName := strings.Trim(request.PathParameter("ns_name"), " ") + nodeID := strings.Trim(request.PathParameter("node_id"), " ") + podName := strings.Trim(request.PathParameter("pod_name"), " ") + podFilter := strings.Trim(request.QueryParameter("pods_filter"), " ") + params := []string{metricsName, nsName, nodeID, podName, podFilter} + promql := MakePodPromQL(request, params) + if promql != "" { + res := client.SendPrometheusRequest(request, promql) + cleanedJson := ReformatJson(res, metricsName) + return cleanedJson + } + return nil +} + +func MonitorNamespaceSingleMetric(request *restful.Request, metricsName string) *FormatedMetric { + recordingRule := MakeNamespacePromQL(request, metricsName) + res := client.SendPrometheusRequest(request, recordingRule) + cleanedJson := ReformatJson(res, metricsName) + return cleanedJson +} + +func ReformatJson(metric string, metricsName string) *FormatedMetric { + var formatMetric FormatedMetric + err := json.Unmarshal([]byte(metric), &formatMetric) + if err != nil { + glog.Errorln("Unmarshal metric json failed", err) + } + if formatMetric.MetricName == "" { + formatMetric.MetricName = metricsName + } + // retrive metrics success + if formatMetric.Status == "success" { + result := formatMetric.Data.Result + for _, res := range result { + metric, ok := res["metric"] + me := metric.(map[string]interface{}) + if ok { + delete(me, "__name__") + } + } + } + return &formatMetric +} + +func collectNodeorClusterMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) { + metric := MonitorNodeorClusterSingleMetric(request, metricsName) + ch <- metric +} + +func collectNamespaceMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) { + metric := MonitorNamespaceSingleMetric(request, metricsName) + ch <- metric +} + +func collectWorkloadMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) { + metricsName = strings.TrimLeft(metricsName, "workload_") + metric := MonitorWorkloadSingleMetric(request, metricsName) + ch <- metric +} + +func collectPodMetrics(request *restful.Request, metricsName string, ch chan<- *FormatedMetric) { + metric := MonitorPodSingleMetric(request, metricsName) + ch <- metric +} + +func MonitorAllMetrics(request *restful.Request) FormatedLevelMetric { + metricsName := strings.Trim(request.QueryParameter("metrics_filter"), " ") + if metricsName == "" { + metricsName = ".*" + } + path := request.SelectedRoutePath() + sourceType := path[strings.LastIndex(path, "/")+1 : len(path)-1] + if strings.Contains(path, "workload") { + sourceType = "workload" + } + var ch = make(chan *FormatedMetric, 10) + for _, k := range MetricsNames { + bol, err := regexp.MatchString(metricsName, k) + if !bol { + continue + } + if err != nil { + glog.Errorln("regex match failed", err) + continue + } + if strings.HasPrefix(k, sourceType) { + if sourceType == "node" || sourceType == "cluster" { + go collectNodeorClusterMetrics(request, k, ch) + } else if sourceType == "namespace" { + go collectNamespaceMetrics(request, k, ch) + } else if sourceType == "pod" { + go collectPodMetrics(request, k, ch) + } else if sourceType == "workload" { + go collectWorkloadMetrics(request, k, ch) + } + } + } + var metricsArray []FormatedMetric + var tempJson *FormatedMetric + for _, k := range MetricsNames { + bol, err := regexp.MatchString(metricsName, k) + if !bol { + continue + } + if err != nil { + glog.Errorln("regex match failed") + continue + } + if strings.HasPrefix(k, sourceType) { + tempJson = <-ch + if tempJson != nil { + metricsArray = append(metricsArray, *tempJson) + } + } + } + return FormatedLevelMetric{ + MetricsLevel: sourceType, + Results: metricsArray, + } +} + +func MonitorNodeorClusterSingleMetric(request *restful.Request, metricsName string) *FormatedMetric { + recordingRule := MakeNodeorClusterRule(request, metricsName) + res := client.SendPrometheusRequest(request, recordingRule) + cleanedJson := ReformatJson(res, metricsName) + return cleanedJson +} diff --git a/pkg/models/metrics/metrics_rule.go b/pkg/models/metrics/metrics_rule.go new file mode 100644 index 000000000..14be69488 --- /dev/null +++ b/pkg/models/metrics/metrics_rule.go @@ -0,0 +1,193 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "strings" + + "github.com/emicklei/go-restful" +) + +func MakeWorkloadRule(request *restful.Request) string { + // kube_pod_info{created_by_kind="DaemonSet",created_by_name="fluent-bit",endpoint="https-main", + // host_ip="192.168.0.14",instance="10.244.114.187:8443",job="kube-state-metrics", + // namespace="kube-system",node="i-k89a62il",pod="fluent-bit-l5vxr", + // pod_ip="10.244.114.175",service="kube-state-metrics"} + rule := `kube_pod_info{created_by_kind="$1",created_by_name=$2,namespace="$3"}` + kind := strings.Trim(request.PathParameter("workload_kind"), " ") + name := strings.Trim(request.QueryParameter("workload_name"), " ") + namespace := strings.Trim(request.PathParameter("ns_name"), " ") + if namespace == "" { + namespace = ".*" + } + + // kind alertnatives values: Deployment StatefulSet ReplicaSet DaemonSet + kind = strings.ToLower(kind) + + switch kind { + case "deployment": + kind = "ReplicaSet" + if name != "" { + name = "~\"" + name + ".*\"" + } else { + name = "~\".*\"" + } + rule = strings.Replace(rule, "$1", kind, -1) + rule = strings.Replace(rule, "$2", name, -1) + rule = strings.Replace(rule, "$3", namespace, -1) + return rule + case "replicaset": + kind = "ReplicaSet" + case "statefulset": + kind = "StatefulSet" + case "daemonset": + kind = "DaemonSet" + } + + if name == "" { + name = "~\".*\"" + } else { + name = "\"" + name + "\"" + } + + rule = strings.Replace(rule, "$1", kind, -1) + rule = strings.Replace(rule, "$2", name, -1) + rule = strings.Replace(rule, "$3", namespace, -1) + return rule +} + +func MakeContainerPromQL(request *restful.Request) string { + nsName := strings.Trim(request.PathParameter("ns_name"), " ") + poName := strings.Trim(request.PathParameter("pod_name"), " ") + containerName := strings.Trim(request.PathParameter("container_name"), " ") + // metricType container_cpu_utilisation container_memory_utilisation container_memory_utilisation_wo_cache + metricType := strings.Trim(request.QueryParameter("metrics_name"), " ") + var promql = "" + if containerName == "" { + // all containers maybe use filter + metricType += "_all" + promql = RulePromQLTmplMap[metricType] + promql = strings.Replace(promql, "$1", nsName, -1) + promql = strings.Replace(promql, "$2", poName, -1) + containerFilter := strings.Trim(request.QueryParameter("containers_filter"), " ") + if containerFilter == "" { + containerFilter = ".*" + } + promql = strings.Replace(promql, "$3", containerFilter, -1) + return promql + } + promql = RulePromQLTmplMap[metricType] + + promql = strings.Replace(promql, "$1", nsName, -1) + promql = strings.Replace(promql, "$2", poName, -1) + promql = strings.Replace(promql, "$3", containerName, -1) + return promql +} + +func MakePodPromQL(request *restful.Request, params []string) string { + metricType := params[0] + nsName := params[1] + nodeID := params[2] + podName := params[3] + podFilter := params[4] + var promql = "" + if nsName != "" { + // get pod metrics by namespace + if podName != "" { + // specific pod + promql = RulePromQLTmplMap[metricType] + promql = strings.Replace(promql, "$1", nsName, -1) + promql = strings.Replace(promql, "$2", podName, -1) + + } else { + // all pods + metricType += "_all" + promql = RulePromQLTmplMap[metricType] + if podFilter == "" { + podFilter = ".*" + } + promql = strings.Replace(promql, "$1", nsName, -1) + promql = strings.Replace(promql, "$2", podFilter, -1) + } + } else if nodeID != "" { + // get pod metrics by nodeid + metricType += "_node" + promql = RulePromQLTmplMap[metricType] + promql = strings.Replace(promql, "$3", nodeID, -1) + if podName != "" { + // specific pod + promql = strings.Replace(promql, "$2", podName, -1) + } else { + // choose pod use re2 expression + podFilter := strings.Trim(request.QueryParameter("pods_filter"), " ") + if podFilter == "" { + podFilter = ".*" + } + promql = strings.Replace(promql, "$2", podFilter, -1) + } + } + return promql +} + +func MakeNamespacePromQL(request *restful.Request, metricsName string) string { + nsName := strings.Trim(request.PathParameter("ns_name"), " ") + metricType := metricsName + var recordingRule = RulePromQLTmplMap[metricType] + nsFilter := strings.Trim(request.QueryParameter("namespaces_filter"), " ") + if nsName != "" { + nsFilter = nsName + } else { + if nsFilter == "" { + nsFilter = ".*" + } + } + recordingRule = strings.Replace(recordingRule, "$1", nsFilter, -1) + return recordingRule +} + +func MakeNodeorClusterRule(request *restful.Request, metricsName string) string { + nodeID := request.PathParameter("node_id") + var rule = RulePromQLTmplMap[metricsName] + + if strings.Contains(request.SelectedRoutePath(), "monitoring/cluster") { + // cluster + return rule + } else { + // node + nodesFilter := strings.Trim(request.QueryParameter("nodes_filter"), " ") + if nodesFilter == "" { + nodesFilter = ".*" + } + if strings.Contains(metricsName, "disk") && (!(strings.Contains(metricsName, "read") || strings.Contains(metricsName, "write"))) { + // disk size promql + nodesFilter := "" + if nodeID != "" { + nodesFilter = "{" + "node" + "=" + "\"" + nodeID + "\"" + "}" + } else { + nodesFilter = "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}" + } + rule = strings.Replace(rule, "$1", nodesFilter, -1) + } else { + // cpu, memory, network, disk_iops rules + if nodeID != "" { + // specific node + rule = rule + "{" + "node" + "=" + "\"" + nodeID + "\"" + "}" + } else { + // all nodes or specific nodes filted with re2 syntax + rule = rule + "{" + "node" + "=~" + "\"" + nodesFilter + "\"" + "}" + } + } + } + return rule +} diff --git a/pkg/models/metrics/metrics_rule_tmpl.go b/pkg/models/metrics/metrics_rule_tmpl.go new file mode 100644 index 000000000..eae10835b --- /dev/null +++ b/pkg/models/metrics/metrics_rule_tmpl.go @@ -0,0 +1,147 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +type MetricMap map[string]string + +var MetricsNames = []string{ + "cluster_cpu_utilisation", + "cluster_memory_utilisation", + "cluster_net_utilisation", + "cluster_pod_count", + + "node_cpu_utilisation", + "node_memory_utilisation", + "node_memory_available", + "node_memory_total", + "node_net_utilisation", + "node_net_bytes_transmitted", + "node_net_bytes_received", + "node_disk_read_iops", + "node_disk_write_iops", + "node_disk_read_throughput", + "node_disk_write_throughput", + "node_disk_capacity", + "node_disk_available", + "node_disk_utilization", + + "namespace_cpu_utilisation", + "namespace_memory_utilisation", + "namespace_memory_utilisation_wo_cache", + "namespace_net_bytes_transmitted", + "namespace_net_bytes_received", + "namespace_pod_count", + + "pod_cpu_utilisation", + "pod_memory_utilisation", + "pod_memory_utilisation_wo_cache", + "pod_net_bytes_transmitted", + "pod_net_bytes_received", + + "workload_pod_cpu_utilisation", + "workload_pod_memory_utilisation", + "workload_pod_memory_utilisation_wo_cache", + "workload_pod_net_bytes_transmitted", + "workload_pod_net_bytes_received", + //"container_cpu_utilisation", + //"container_memory_utilisation_wo_cache", + //"container_memory_utilisation", + + "tenant_cpu_utilisation", + "tenant_memory_utilisation", + "tenant_memory_utilisation_wo_cache", + "tenant_net_bytes_transmitted", + "tenant_net_bytes_received", + "tenant_pod_count", +} + +var RulePromQLTmplMap = MetricMap{ + //cluster + "cluster_cpu_utilisation": ":node_cpu_utilisation:avg1m", + "cluster_memory_utilisation": ":node_memory_utilisation:", + // Cluster network utilisation (bytes received + bytes transmitted per second) + "cluster_net_utilisation": ":node_net_utilisation:sum_irate", + "cluster_pod_count": `count(kube_pod_info{job="kube-state-metrics"})`, + + //node + "node_cpu_utilisation": "node:node_cpu_utilisation:avg1m", + "node_memory_utilisation": "node:node_memory_utilisation:", + "node_memory_available": "node:node_memory_bytes_available:sum", + "node_memory_total": "node:node_memory_bytes_total:sum", + // Node network utilisation (bytes received + bytes transmitted per second) + "node_net_utilisation": "node:node_net_utilisation:sum_irate", + // Node network bytes transmitted per second + "node_net_bytes_transmitted": "node:node_net_bytes_transmitted:sum_irate", + // Node network bytes received per second + "node_net_bytes_received": "node:node_net_bytes_received:sum_irate", + + // node:data_volume_iops_reads:sum{node=~"i-5xcldxos|i-6soe9zl1"} + "node_disk_read_iops": "node:data_volume_iops_reads:sum", + // node:data_volume_iops_writes:sum{node=~"i-5xcldxos|i-6soe9zl1"} + "node_disk_write_iops": "node:data_volume_iops_writes:sum", + // node:data_volume_throughput_bytes_read:sum{node=~"i-5xcldxos|i-6soe9zl1"} + "node_disk_read_throughput": "node:data_volume_throughput_bytes_read:sum", + // node:data_volume_throughput_bytes_written:sum{node=~"i-5xcldxos|i-6soe9zl1"} + "node_disk_write_throughput": "node:data_volume_throughput_bytes_written:sum", + + "node_disk_capacity": `sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + "node_disk_available": `sum by (node) ((node_filesystem_avail{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + "node_disk_utilization": `sum by (node) (((node_filesystem_size{mountpoint="/", job="node-exporter"} - node_filesystem_avail{mountpoint="/", job="node-exporter"}) / node_filesystem_size{mountpoint="/", job="node-exporter"}) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:$1)`, + + //namespace + "namespace_cpu_utilisation": `namespace:container_cpu_usage_seconds_total:sum_rate{namespace=~"$1"}`, + "namespace_memory_utilisation": `namespace:container_memory_usage_bytes:sum{namespace=~"$1"}`, + "namespace_memory_utilisation_wo_cache": `namespace:container_memory_usage_bytes_wo_cache:sum{namespace=~"$1"}`, + "namespace_net_bytes_transmitted": `sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[2m]))`, + "namespace_net_bytes_received": `sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[2m]))`, + // count(kube_pod_info) by (namespace) namespace=~"monitoring|default|kube-system" + "namespace_pod_count": `count(kube_pod_info{job="kube-state-metrics", namespace=~"$1"}) by (namespace)`, + + // pod + "pod_cpu_utilisation": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name="$2", image!=""}[5m])) by (namespace, pod_name)`, + "pod_memory_utilisation": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""}) by (namespace, pod_name)`, + "pod_memory_utilisation_wo_cache": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name="$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name="$2",image!=""}) by (namespace, pod_name)`, + "pod_net_bytes_transmitted": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[2m]))`, + "pod_net_bytes_received": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name="$2", interface="eth0", job="kubelet"}[2m]))`, + + "pod_cpu_utilisation_all": `sum(irate(container_cpu_usage_seconds_total{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}[5m])) by (namespace, pod_name)`, + "pod_memory_utilisation_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`, + "pod_memory_utilisation_wo_cache_all": `sum(container_memory_usage_bytes{job="kubelet", namespace="$1", pod_name=~"$2", image!=""} - container_memory_cache{job="kubelet", namespace="$1", pod_name=~"$2", image!=""}) by (namespace, pod_name)`, + "pod_net_bytes_transmitted_all": `sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[2m]))`, + "pod_net_bytes_received_all": `sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{namespace="$1", pod_name!="", pod_name=~"$2", interface="eth0", job="kubelet"}[2m]))`, + + //"pod_cpu_utilisation_node": `sum by (node, pod) (label_join(irate(container_cpu_usage_seconds_total{job="kubelet", image!=""}[5m]), "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, + "pod_cpu_utilisation_node": `sum by (node, pod) (label_join(irate(container_cpu_usage_seconds_total{job="kubelet",pod_name=~"$2", image!=""}[5m]), "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, + "pod_memory_utilisation_node": `sum by (node, pod) (label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, + "pod_memory_utilisation_wo_cache_node": `sum by (node, pod) ((label_join(container_memory_usage_bytes{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name") - label_join(container_memory_cache{job="kubelet",pod_name=~"$2", image!=""}, "pod", " ", "pod_name")) * on (namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{node=~"$3"})`, + + // container + "container_cpu_utilisation": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name="$3"}[5m])) by (namespace, pod_name, container_name)`, + //"container_cpu_utilisation_wo_podname": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", container_name=~"$3"}[5m])) by (namespace, pod_name, container_name)`, + "container_cpu_utilisation_all": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}[5m])) by (namespace, pod_name, container_name)`, + //"container_cpu_utilisation_all_wo_podname": `sum(irate(container_cpu_usage_seconds_total{namespace="$1", container_name!="POD"}[5m])) by (namespace, pod_name, container_name)`, + + "container_memory_utilisation_wo_cache": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name="$3"}`, + "container_memory_utilisation_wo_cache_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"} - ignoring(id, image, endpoint, instance, job, name, service) container_memory_cache{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`, + "container_memory_utilisation": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name="$3"}`, + "container_memory_utilisation_all": `container_memory_usage_bytes{namespace="$1", pod_name="$2", container_name=~"$3", container_name!="POD"}`, + + // tenant + "tenant_cpu_utilisation": `sum(namespace:container_cpu_usage_seconds_total:sum_rate{namespace =~"$1"})`, + "tenant_memory_utilisation": `sum(namespace:container_memory_usage_bytes:sum{namespace =~"$1"})`, + "tenant_memory_utilisation_wo_cache": `sum(namespace:container_memory_usage_bytes_wo_cache:sum{namespace =~"$1"})`, + "tenant_net_bytes_transmitted": `sum(sum by (namespace) (irate(container_network_transmit_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[2m])))`, + "tenant_net_bytes_received": `sum(sum by (namespace) (irate(container_network_receive_bytes_total{namespace=~"$1", pod_name!="", interface="eth0", job="kubelet"}[2m])))`, + "tenant_pod_count": `sum(count(kube_pod_info{job="kube-state-metrics", namespace=~"$1"}) by (namespace))`, +} diff --git a/pkg/models/metrics/metrics_struct.go b/pkg/models/metrics/metrics_struct.go new file mode 100644 index 000000000..4c77783bc --- /dev/null +++ b/pkg/models/metrics/metrics_struct.go @@ -0,0 +1,69 @@ +/* +Copyright 2018 The KubeSphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +type FormatedLevelMetric struct { + MetricsLevel string `json:"metrics_level"` + Results []FormatedMetric `json:"results"` +} + +type FormatedMetric struct { + MetricName string `json:"metric_name, omitempty"` + Status string `json:"status"` + Data FormatedMetricData `json:"data, omitempty"` +} + +type FormatedMetricData struct { + Result []map[string]interface{} `json:"result"` + ResultType string `json:"resultType"` +} + +type CommonMetricsResult struct { + Status string `json:"status"` + Data CommonMetricsData `json:"data"` +} + +type CommonMetricsData struct { + Result []CommonResultItem `json:"result"` + ResultType string `json:"resultType"` +} + +type CommonResultItem struct { + KubePodMetric KubePodMetric `json:"metric"` + Value interface{} `json:"value"` +} + +/** +"__name__": "kube_pod_info", +"created_by_kind": "\\u003cnone\\u003e", +"created_by_name": "\\u003cnone\\u003e", +"endpoint": "https-main", +"host_ip": "192.168.0.13", +"instance": "10.244.114.187:8443", +"job": "kube-state-metrics", +"namespace": "kube-system", +"node": "i-39p7faw6", +"pod": "cloud-controller-manager-i-39p7faw6", +"pod_ip": "192.168.0.13", +"service": "kube-state-metrics" +*/ +type KubePodMetric struct { + CreatedByKind string `json:"created_by_kind"` + CreatedByName string `json:"created_by_name"` + Namespace string `json:"namespace"` + Pod string `json:"pod"` +}