diff --git a/cmd/ks-apiserver/app/options/options.go b/cmd/ks-apiserver/app/options/options.go index 2941ad10e..1e1a3c7b4 100644 --- a/cmd/ks-apiserver/app/options/options.go +++ b/cmd/ks-apiserver/app/options/options.go @@ -8,9 +8,9 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins" "kubesphere.io/kubesphere/pkg/simple/client/k8s" esclient "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus" "kubesphere.io/kubesphere/pkg/simple/client/mysql" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" - "kubesphere.io/kubesphere/pkg/simple/client/prometheus" "kubesphere.io/kubesphere/pkg/simple/client/s3" "kubesphere.io/kubesphere/pkg/simple/client/servicemesh" "kubesphere.io/kubesphere/pkg/simple/client/sonarqube" diff --git a/pkg/api/monitoring/v1alpha2/types.go b/pkg/api/monitoring/v1alpha2/types.go index 74700cb19..ee33074e3 100644 --- a/pkg/api/monitoring/v1alpha2/types.go +++ b/pkg/api/monitoring/v1alpha2/types.go @@ -1,23 +1,10 @@ package v1alpha2 -// Prometheus query api response +import "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + type APIResponse struct { - Status string `json:"status" description:"result status, one of error, success"` - Data QueryResult `json:"data" description:"actual metric result"` - ErrorType string `json:"errorType,omitempty"` - Error string `json:"error,omitempty"` - Warnings []string `json:"warnings,omitempty"` -} - -// QueryResult includes result data from a query. -type QueryResult struct { - ResultType string `json:"resultType" description:"result type, one of matrix, vector"` - Result []QueryValue `json:"result" description:"metric data including labels, time series and values"` -} - -// Time Series -type QueryValue struct { - Metric map[string]string `json:"metric,omitempty" description:"time series labels"` - Value []interface{} `json:"value,omitempty" description:"time series, values of vector type"` - Values [][]interface{} `json:"values,omitempty" description:"time series, values of matrix type"` + Results []monitoring.Metric `json:"results" description:"actual array of results"` + CurrentPage int `json:"page,omitempty" description:"current page returned"` + TotalPage int `json:"total_page,omitempty" description:"total number of pages"` + TotalItem int `json:"total_item,omitempty" description:"page size"` } diff --git a/pkg/apiserver/monitoring/monitoring.go b/pkg/apiserver/monitoring/monitoring.go deleted file mode 100644 index 446156fe8..000000000 --- a/pkg/apiserver/monitoring/monitoring.go +++ /dev/null @@ -1,203 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package monitoring - -import ( - "github.com/emicklei/go-restful" - "kubesphere.io/kubesphere/pkg/informers" - "kubesphere.io/kubesphere/pkg/models/metrics" - "net/url" - "strconv" - "strings" -) - -func MonitorCluster(request *restful.Request, response *restful.Response) { - r := ParseRequestParams(request) - - // TODO: expose kubesphere iam and devops statistics in prometheus format - var res *metrics.Response - if r.Type == "statistics" { - res = metrics.GetClusterStatistics() - } else { - res = metrics.GetClusterMetrics(r) - } - - response.WriteAsJson(res) -} - -func MonitorNode(request *restful.Request, response *restful.Response) { - r := ParseRequestParams(request) - res := metrics.GetNodeMetrics(r) - res, metricsNum := res.SortBy(r.SortMetric, r.SortType) - res = res.Page(r.PageNum, r.LimitNum, metricsNum) - response.WriteAsJson(res) -} - -func MonitorWorkspace(request *restful.Request, response *restful.Response) { - r := ParseRequestParams(request) - - // TODO: expose kubesphere iam and devops statistics in prometheus format - var res *metrics.Response - if r.Type == "statistics" && r.WorkspaceName != "" { - res = metrics.GetWorkspaceStatistics(r.WorkspaceName) - } else { - res = metrics.GetWorkspaceMetrics(r) - res, metricsNum := res.SortBy(r.SortMetric, r.SortType) - res = res.Page(r.PageNum, r.LimitNum, metricsNum) - } - - response.WriteAsJson(res) -} - -func MonitorNamespace(request *restful.Request, response *restful.Response) { - r := ParseRequestParams(request) - res := metrics.GetNamespaceMetrics(r) - res, metricsNum := res.SortBy(r.SortMetric, r.SortType) - res = res.Page(r.PageNum, r.LimitNum, metricsNum) - response.WriteAsJson(res) -} - -func MonitorWorkload(request *restful.Request, response *restful.Response) { - r := ParseRequestParams(request) - res := metrics.GetWorkloadMetrics(r) - res, metricsNum := res.SortBy(r.SortMetric, r.SortType) - res = res.Page(r.PageNum, r.LimitNum, metricsNum) - response.WriteAsJson(res) -} - -func MonitorPod(request *restful.Request, response *restful.Response) { - r := ParseRequestParams(request) - res := metrics.GetPodMetrics(r) - res, metricsNum := res.SortBy(r.SortMetric, r.SortType) - res = res.Page(r.PageNum, r.LimitNum, metricsNum) - response.WriteAsJson(res) -} - -func MonitorContainer(request *restful.Request, response *restful.Response) { - r := ParseRequestParams(request) - res := metrics.GetContainerMetrics(r) - res, metricsNum := res.SortBy(r.SortMetric, r.SortType) - res = res.Page(r.PageNum, r.LimitNum, metricsNum) - response.WriteAsJson(res) -} - -func MonitorPVC(request *restful.Request, response *restful.Response) { - r := ParseRequestParams(request) - res := metrics.GetPVCMetrics(r) - res, metricsNum := res.SortBy(r.SortMetric, r.SortType) - res = res.Page(r.PageNum, r.LimitNum, metricsNum) - response.WriteAsJson(res) -} - -func MonitorComponent(request *restful.Request, response *restful.Response) { - r := ParseRequestParams(request) - res := metrics.GetComponentMetrics(r) - response.WriteAsJson(res) -} - -func ParseRequestParams(request *restful.Request) metrics.RequestParams { - var requestParams metrics.RequestParams - - queryTime := strings.Trim(request.QueryParameter("time"), " ") - start := strings.Trim(request.QueryParameter("start"), " ") - end := strings.Trim(request.QueryParameter("end"), " ") - step := strings.Trim(request.QueryParameter("step"), " ") - sortMetric := strings.Trim(request.QueryParameter("sort_metric"), " ") - sortType := strings.Trim(request.QueryParameter("sort_type"), " ") - pageNum := strings.Trim(request.QueryParameter("page"), " ") - limitNum := strings.Trim(request.QueryParameter("limit"), " ") - tp := strings.Trim(request.QueryParameter("type"), " ") - metricsFilter := strings.Trim(request.QueryParameter("metrics_filter"), " ") - resourcesFilter := strings.Trim(request.QueryParameter("resources_filter"), " ") - nodeName := strings.Trim(request.PathParameter("node"), " ") - workspaceName := strings.Trim(request.PathParameter("workspace"), " ") - namespaceName := strings.Trim(request.PathParameter("namespace"), " ") - workloadKind := strings.Trim(request.PathParameter("kind"), " ") - workloadName := strings.Trim(request.PathParameter("workload"), " ") - podName := strings.Trim(request.PathParameter("pod"), " ") - containerName := strings.Trim(request.PathParameter("container"), " ") - pvcName := strings.Trim(request.PathParameter("pvc"), " ") - storageClassName := strings.Trim(request.PathParameter("storageclass"), " ") - componentName := strings.Trim(request.PathParameter("component"), " ") - - requestParams = metrics.RequestParams{ - SortMetric: sortMetric, - SortType: sortType, - PageNum: pageNum, - LimitNum: limitNum, - Type: tp, - MetricsFilter: metricsFilter, - ResourcesFilter: resourcesFilter, - NodeName: nodeName, - WorkspaceName: workspaceName, - NamespaceName: namespaceName, - WorkloadKind: workloadKind, - WorkloadName: workloadName, - PodName: podName, - ContainerName: containerName, - PVCName: pvcName, - StorageClassName: storageClassName, - ComponentName: componentName, - } - - if metricsFilter == "" { - requestParams.MetricsFilter = ".*" - } - if resourcesFilter == "" { - requestParams.ResourcesFilter = ".*" - } - - v := url.Values{} - - if start != "" && end != "" { // range query - - // metrics from a deleted namespace should be hidden - // therefore, for range query, if range query start time is less than the namespace creation time, set it to creation time - // it is the same with query at a fixed time point - if namespaceName != "" { - nsLister := informers.SharedInformerFactory().Core().V1().Namespaces().Lister() - ns, err := nsLister.Get(namespaceName) - if err == nil { - creationTime := ns.CreationTimestamp.Time.Unix() - queryStart, err := strconv.ParseInt(start, 10, 64) - if err == nil && queryStart < creationTime { - start = strconv.FormatInt(creationTime, 10) - } - } - } - - v.Set("start", start) - v.Set("end", end) - - if step == "" { - v.Set("step", metrics.DefaultQueryStep) - } else { - v.Set("step", step) - } - requestParams.QueryParams = v - requestParams.QueryType = metrics.RangeQuery - - return requestParams - } else if queryTime != "" { // query - v.Set("time", queryTime) - } - - requestParams.QueryParams = v - requestParams.QueryType = metrics.Query - return requestParams -} diff --git a/pkg/kapis/kapis.go b/pkg/kapis/kapis.go index fd2356ca4..1fb624b4c 100644 --- a/pkg/kapis/kapis.go +++ b/pkg/kapis/kapis.go @@ -17,15 +17,16 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/k8s" ldappool "kubesphere.io/kubesphere/pkg/simple/client/ldap" "kubesphere.io/kubesphere/pkg/simple/client/logging" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" "kubesphere.io/kubesphere/pkg/simple/client/mysql" op "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" ) -func InstallAPIs(container *restful.Container, client k8s.Client, op op.Client, db *mysql.Database, logging logging.Interface) { +func InstallAPIs(container *restful.Container, client k8s.Client, op op.Client, db *mysql.Database, logging logging.Interface, monitoring monitoring.Interface) { urlruntime.Must(servicemeshv1alpha2.AddToContainer(container)) urlruntime.Must(devopsv1alpha2.AddToContainer(container)) urlruntime.Must(loggingv1alpha2.AddToContainer(container, client, logging)) - urlruntime.Must(monitoringv1alpha2.AddToContainer(container)) + urlruntime.Must(monitoringv1alpha2.AddToContainer(container, client, monitoring)) urlruntime.Must(openpitrixv1.AddToContainer(container, client, op)) urlruntime.Must(operationsv1alpha2.AddToContainer(container, client)) urlruntime.Must(resourcesv1alpha2.AddToContainer(container, client)) diff --git a/pkg/kapis/monitoring/install/install.go b/pkg/kapis/monitoring/install/install.go deleted file mode 100644 index c1c15e8ae..000000000 --- a/pkg/kapis/monitoring/install/install.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package install - -import ( - "github.com/emicklei/go-restful" - urlruntime "k8s.io/apimachinery/pkg/util/runtime" - "kubesphere.io/kubesphere/pkg/apiserver/runtime" - "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha2" -) - -func init() { - Install(runtime.Container) -} - -func Install(container *restful.Container) { - urlruntime.Must(v1alpha2.AddToContainer(container)) -} diff --git a/pkg/kapis/monitoring/v1alpha2/handler.go b/pkg/kapis/monitoring/v1alpha2/handler.go new file mode 100644 index 000000000..962b4f2db --- /dev/null +++ b/pkg/kapis/monitoring/v1alpha2/handler.go @@ -0,0 +1,145 @@ +/* + + Copyright 2019 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +package v1alpha2 + +import ( + "github.com/emicklei/go-restful" + "kubesphere.io/kubesphere/pkg/api" + "kubesphere.io/kubesphere/pkg/api/monitoring/v1alpha2" + model "kubesphere.io/kubesphere/pkg/models/monitoring" + "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" +) + +type handler struct { + k k8s.Client + mo model.MonitoringOperator +} + +func newHandler(k k8s.Client, m monitoring.Interface) *handler { + return &handler{k, model.NewMonitoringOperator(m)} +} + +func (h handler) handleClusterMetricsQuery(req *restful.Request, resp *restful.Response) { + p, err := h.parseRequestParams(req, monitoring.LevelCluster) + if err != nil { + api.HandleBadRequest(resp, err) + return + } + h.handleNamedMetricsQuery(resp, p) +} + +func (h handler) handleNodeMetricsQuery(req *restful.Request, resp *restful.Response) { + p, err := h.parseRequestParams(req, monitoring.LevelNode) + if err != nil { + api.HandleBadRequest(resp, err) + return + } + h.handleNamedMetricsQuery(resp, p) +} + +func (h handler) handleWorkspaceMetricsQuery(req *restful.Request, resp *restful.Response) { + p, err := h.parseRequestParams(req, monitoring.LevelWorkspace) + if err != nil { + api.HandleBadRequest(resp, err) + return + } + h.handleNamedMetricsQuery(resp, p) +} + +func (h handler) handleNamespaceMetricsQuery(req *restful.Request, resp *restful.Response) { + p, err := h.parseRequestParams(req, monitoring.LevelNamespace) + if err != nil { + api.HandleBadRequest(resp, err) + return + } + h.handleNamedMetricsQuery(resp, p) +} + +func (h handler) handleWorkloadMetricsQuery(req *restful.Request, resp *restful.Response) { + p, err := h.parseRequestParams(req, monitoring.LevelWorkload) + if err != nil { + api.HandleBadRequest(resp, err) + return + } + h.handleNamedMetricsQuery(resp, p) +} + +func (h handler) handlePodMetricsQuery(req *restful.Request, resp *restful.Response) { + p, err := h.parseRequestParams(req, monitoring.LevelPod) + if err != nil { + api.HandleBadRequest(resp, err) + return + } + h.handleNamedMetricsQuery(resp, p) +} + +func (h handler) handleContainerMetricsQuery(req *restful.Request, resp *restful.Response) { + p, err := h.parseRequestParams(req, monitoring.LevelContainer) + if err != nil { + api.HandleBadRequest(resp, err) + return + } + h.handleNamedMetricsQuery(resp, p) +} + +func (h handler) handlePVCMetricsQuery(req *restful.Request, resp *restful.Response) { + p, err := h.parseRequestParams(req, monitoring.LevelPVC) + if err != nil { + api.HandleBadRequest(resp, err) + return + } + h.handleNamedMetricsQuery(resp, p) +} + +func (h handler) handleComponentMetricsQuery(req *restful.Request, resp *restful.Response) { + p, err := h.parseRequestParams(req, monitoring.LevelComponent) + if err != nil { + api.HandleBadRequest(resp, err) + return + } + h.handleNamedMetricsQuery(resp, p) +} + +func (h handler) handleNamedMetricsQuery(resp *restful.Response, p params) { + var res v1alpha2.APIResponse + var err error + + if p.isRangeQuery() { + res, err = h.mo.GetNamedMetricsOverTime(p.start, p.end, p.step, p.option) + if err != nil { + api.HandleInternalError(resp, err) + return + } + } else { + res, err = h.mo.GetNamedMetrics(p.time, p.option) + if err != nil { + api.HandleInternalError(resp, err) + return + } + + if p.shouldSort() { + var rows int + res, rows = h.mo.SortMetrics(res, p.target, p.order, p.identifier) + res = h.mo.PageMetrics(res, p.page, p.limit, rows) + } + } + + resp.WriteAsJson(res) +} diff --git a/pkg/kapis/monitoring/v1alpha2/helper.go b/pkg/kapis/monitoring/v1alpha2/helper.go new file mode 100644 index 000000000..1896c5e0b --- /dev/null +++ b/pkg/kapis/monitoring/v1alpha2/helper.go @@ -0,0 +1,217 @@ +package v1alpha2 + +import ( + "fmt" + "github.com/emicklei/go-restful" + "github.com/pkg/errors" + corev1 "k8s.io/apimachinery/pkg/apis/meta/v1" + model "kubesphere.io/kubesphere/pkg/models/monitoring" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + "strconv" + "time" +) + +const ( + DefaultStep = 10 * time.Minute + DefaultFilter = ".*" + DefaultOrder = model.OrderDescending + DefaultPage = 1 + DefaultLimit = 5 +) + +type params struct { + time time.Time + start, end time.Time + step time.Duration + + target string + identifier string + order string + page int + limit int + + option monitoring.QueryOption +} + +func (p params) isRangeQuery() bool { + return !p.time.IsZero() +} + +func (p params) shouldSort() bool { + return p.target != "" +} + +func (h handler) parseRequestParams(req *restful.Request, lvl monitoring.MonitoringLevel) (params, error) { + timestamp := req.QueryParameter("time") + start := req.QueryParameter("start") + end := req.QueryParameter("end") + step := req.QueryParameter("step") + target := req.QueryParameter("sort_metric") + order := req.QueryParameter("sort_type") + page := req.QueryParameter("page") + limit := req.QueryParameter("limit") + metricFilter := req.QueryParameter("metrics_filter") + resourceFilter := req.QueryParameter("resources_filter") + nodeName := req.PathParameter("node") + workspaceName := req.PathParameter("workspace") + namespaceName := req.PathParameter("namespace") + workloadKind := req.PathParameter("kind") + workloadName := req.PathParameter("workload") + podName := req.PathParameter("pod") + containerName := req.PathParameter("container") + pvcName := req.PathParameter("pvc") + storageClassName := req.PathParameter("storageclass") + componentType := req.PathParameter("component") + + var p params + var err error + if start != "" && end != "" { + p.start, err = time.Parse(time.RFC3339, start) + if err != nil { + return p, err + } + p.end, err = time.Parse(time.RFC3339, end) + if err != nil { + return p, err + } + if step == "" { + p.step = DefaultStep + } else { + p.step, err = time.ParseDuration(step) + if err != nil { + return p, err + } + } + } else if start == "" && end == "" { + if timestamp == "" { + p.time = time.Now() + } else { + p.time, err = time.Parse(time.RFC3339, req.QueryParameter("time")) + if err != nil { + return p, err + } + } + } else { + return p, errors.Errorf("'time' and the combination of 'start' and 'end' are mutually exclusive.") + } + + // hide metrics from a deleted namespace having the same name + namespace := req.QueryParameter("namespace") + if req.QueryParameter("namespace") != "" { + ns, err := h.k.Kubernetes().CoreV1().Namespaces().Get(namespace, corev1.GetOptions{}) + if err != nil { + return p, err + } + + cts := ns.CreationTimestamp.Time + if p.start.Before(cts) { + p.start = cts + } + if p.end.Before(cts) { + return p, errors.Errorf("End timestamp must not be before namespace creation time.") + } + } + + if resourceFilter == "" { + resourceFilter = DefaultFilter + } + + if metricFilter == "" { + metricFilter = DefaultFilter + } + if componentType != "" { + metricFilter = fmt.Sprintf("/^(?=.*%s)(?=.*%s)/s", componentType, metricFilter) + } + + // should sort + if target != "" { + p.page = DefaultPage + p.limit = DefaultLimit + if order != model.OrderAscending { + p.order = DefaultOrder + } + if page != "" { + p.page, err = strconv.Atoi(req.QueryParameter("page")) + if err != nil || p.page <= 0 { + return p, errors.Errorf("Invalid parameter 'page'.") + } + } + if limit != "" { + p.limit, err = strconv.Atoi(req.QueryParameter("limit")) + if err != nil || p.limit <= 0 { + return p, errors.Errorf("Invalid parameter 'limit'.") + } + } + } + + switch lvl { + case monitoring.LevelCluster: + p.option = monitoring.ClusterOption{MetricFilter: metricFilter} + case monitoring.LevelNode: + p.identifier = model.IdentifierNode + p.option = monitoring.NodeOption{ + MetricFilter: metricFilter, + ResourceFilter: resourceFilter, + NodeName: nodeName, + } + case monitoring.LevelWorkspace: + p.identifier = model.IdentifierWorkspace + p.option = monitoring.WorkspaceOption{ + MetricFilter: metricFilter, + ResourceFilter: resourceFilter, + WorkspaceName: workspaceName, + } + case monitoring.LevelNamespace: + p.identifier = model.IdentifierNamespace + p.option = monitoring.NamespaceOption{ + MetricFilter: metricFilter, + ResourceFilter: resourceFilter, + WorkspaceName: workspaceName, + NamespaceName: namespaceName, + } + case monitoring.LevelWorkload: + p.identifier = model.IdentifierWorkload + p.option = monitoring.WorkloadOption{ + MetricFilter: metricFilter, + ResourceFilter: resourceFilter, + NamespaceName: namespaceName, + WorkloadKind: workloadKind, + WorkloadName: workloadName, + } + case monitoring.LevelPod: + p.identifier = model.IdentifierPod + p.option = monitoring.PodOption{ + MetricFilter: metricFilter, + ResourceFilter: resourceFilter, + NodeName: nodeName, + NamespaceName: namespaceName, + WorkloadKind: workloadKind, + WorkloadName: workloadName, + PodName: podName, + } + case monitoring.LevelContainer: + p.identifier = model.IdentifierContainer + p.option = monitoring.ContainerOption{ + MetricFilter: metricFilter, + ResourceFilter: resourceFilter, + NamespaceName: namespaceName, + PodName: podName, + ContainerName: containerName, + } + case monitoring.LevelPVC: + p.identifier = model.IdentifierPVC + p.option = monitoring.PVCOption{ + MetricFilter: metricFilter, + ResourceFilter: resourceFilter, + NamespaceName: namespaceName, + StorageClassName: storageClassName, + PersistentVolumeClaimName: pvcName, + } + case monitoring.LevelComponent: + p.option = monitoring.ComponentOption{ + MetricFilter: metricFilter, + } + } + + return p, nil +} diff --git a/pkg/kapis/monitoring/v1alpha2/register.go b/pkg/kapis/monitoring/v1alpha2/register.go index ebc8a1fb1..3d0675d4e 100644 --- a/pkg/kapis/monitoring/v1alpha2/register.go +++ b/pkg/kapis/monitoring/v1alpha2/register.go @@ -21,10 +21,11 @@ import ( "github.com/emicklei/go-restful" "github.com/emicklei/go-restful-openapi" "k8s.io/apimachinery/pkg/runtime/schema" - "kubesphere.io/kubesphere/pkg/apiserver/monitoring" + "kubesphere.io/kubesphere/pkg/api/monitoring/v1alpha2" "kubesphere.io/kubesphere/pkg/apiserver/runtime" "kubesphere.io/kubesphere/pkg/constants" - "kubesphere.io/kubesphere/pkg/models/metrics" + "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" "net/http" ) @@ -35,29 +36,26 @@ const ( var GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha2"} -var ( - WebServiceBuilder = runtime.NewContainerBuilder(addWebService) - AddToContainer = WebServiceBuilder.AddToContainer -) - -func addWebService(c *restful.Container) error { +func AddToContainer(c *restful.Container, k8sClient k8s.Client, monitoringClient monitoring.Interface) error { ws := runtime.NewWebService(GroupVersion) - ws.Route(ws.GET("/cluster").To(monitoring.MonitorCluster). + h := newHandler(k8sClient, monitoringClient) + + ws.Route(ws.GET("/cluster"). + To(h.handleClusterMetricsQuery). Doc("Get cluster-level metric data."). Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both cluster CPU usage and disk usage: `cluster_cpu_usage|cluster_disk_size_usage`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). - Param(ws.QueryParameter("type", "Additional operations. Currently available types is statistics. It retrieves the total number of workspaces, devops projects, namespaces, accounts in the cluster at the moment.").DataType("string").Required(false)). Metadata(restfulspec.KeyOpenAPITags, []string{constants.ClusterMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/nodes").To(monitoring.MonitorNode). + ws.Route(ws.GET("/nodes"). + To(h.handleNodeMetricsQuery). Doc("Get node-level metric data of all nodes."). Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both node CPU usage and disk usage: `node_cpu_usage|node_disk_size_usage`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). Param(ws.QueryParameter("resources_filter", "The node filter consists of a regexp pattern. It specifies which node data to return. For example, the following filter matches both node i-caojnter and i-cmu82ogj: `i-caojnter|i-cmu82ogj`.").DataType("string").Required(false)). @@ -70,12 +68,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.NodeMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/nodes/{node}").To(monitoring.MonitorNode). + ws.Route(ws.GET("/nodes/{node}"). + To(h.handleNodeMetricsQuery). Doc("Get node-level metric data of the specific node."). Param(ws.PathParameter("node", "Node name.").DataType("string").Required(true)). Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both node CPU usage and disk usage: `node_cpu_usage|node_disk_size_usage`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). @@ -84,12 +82,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). Metadata(restfulspec.KeyOpenAPITags, []string{constants.NodeMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/workspaces").To(monitoring.MonitorWorkspace). + ws.Route(ws.GET("/workspaces"). + To(h.handleWorkspaceMetricsQuery). Doc("Get workspace-level metric data of all workspaces."). Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workspace CPU usage and memory usage: `workspace_cpu_usage|workspace_memory_usage`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). Param(ws.QueryParameter("resources_filter", "The workspace filter consists of a regexp pattern. It specifies which workspace data to return.").DataType("string").Required(false)). @@ -102,12 +100,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkspaceMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/workspaces/{workspace}").To(monitoring.MonitorWorkspace). + ws.Route(ws.GET("/workspaces/{workspace}"). + To(h.handleWorkspaceMetricsQuery). Doc("Get workspace-level metric data of a specific workspace."). Param(ws.PathParameter("workspace", "Workspace name.").DataType("string").Required(true)). Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workspace CPU usage and memory usage: `workspace_cpu_usage|workspace_memory_usage`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). @@ -115,14 +113,13 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). - Param(ws.QueryParameter("type", "Additional operations. Currently available types is statistics. It retrieves the total number of namespaces, devops projects, members and roles in this workspace at the moment.").DataType("string").Required(false)). Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkspaceMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/workspaces/{workspace}/namespaces").To(monitoring.MonitorNamespace). + ws.Route(ws.GET("/workspaces/{workspace}/namespaces"). + To(h.handleNamespaceMetricsQuery). Doc("Get namespace-level metric data of a specific workspace."). Param(ws.PathParameter("workspace", "Workspace name.").DataType("string").Required(true)). Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both namespace CPU usage and memory usage: `namespace_cpu_usage|namespace_memory_usage`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). @@ -136,12 +133,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/namespaces").To(monitoring.MonitorNamespace). + ws.Route(ws.GET("/namespaces"). + To(h.handleNamespaceMetricsQuery). Doc("Get namespace-level metric data of all namespaces."). Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both namespace CPU usage and memory usage: `namespace_cpu_usage|namespace_memory_usage`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). Param(ws.QueryParameter("resources_filter", "The namespace filter consists of a regexp pattern. It specifies which namespace data to return. For example, the following filter matches both namespace test and kube-system: `test|kube-system`.").DataType("string").Required(false)). @@ -154,12 +151,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/namespaces/{namespace}").To(monitoring.MonitorNamespace). + ws.Route(ws.GET("/namespaces/{namespace}"). + To(h.handleNamespaceMetricsQuery). Doc("Get namespace-level metric data of the specific namespace."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both namespace CPU usage and memory usage: `namespace_cpu_usage|namespace_memory_usage`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). @@ -168,12 +165,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/namespaces/{namespace}/workloads").To(monitoring.MonitorWorkload). + ws.Route(ws.GET("/namespaces/{namespace}/workloads"). + To(h.handleWorkloadMetricsQuery). Doc("Get workload-level metric data of a specific namespace's workloads."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workload CPU usage and memory usage: `workload_cpu_usage|workload_memory_usage`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). @@ -187,12 +184,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkloadMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/namespaces/{namespace}/workloads/{kind}").To(monitoring.MonitorWorkload). + ws.Route(ws.GET("/namespaces/{namespace}/workloads/{kind}"). + To(h.handleWorkloadMetricsQuery). Doc("Get workload-level metric data of all workloads which belongs to a specific kind."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.PathParameter("kind", "Workload kind. One of deployment, daemonset, statefulset.").DataType("string").Required(true)). @@ -207,12 +204,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkloadMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/namespaces/{namespace}/pods").To(monitoring.MonitorPod). + ws.Route(ws.GET("/namespaces/{namespace}/pods"). + To(h.handlePodMetricsQuery). Doc("Get pod-level metric data of the specific namespace's pods."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `pod_cpu_usage|pod_memory_usage`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). @@ -226,12 +223,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}").To(monitoring.MonitorPod). + ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}"). + To(h.handlePodMetricsQuery). Doc("Get pod-level metric data of a specific pod. Navigate to the pod by the pod's namespace."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)). @@ -241,12 +238,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/namespaces/{namespace}/workloads/{kind}/{workload}/pods").To(monitoring.MonitorPod). + ws.Route(ws.GET("/namespaces/{namespace}/workloads/{kind}/{workload}/pods"). + To(h.handlePodMetricsQuery). Doc("Get pod-level metric data of a specific workload's pods. Navigate to the workload by the namespace."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.PathParameter("kind", "Workload kind. One of deployment, daemonset, statefulset.").DataType("string").Required(true)). @@ -262,12 +259,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/nodes/{node}/pods").To(monitoring.MonitorPod). + ws.Route(ws.GET("/nodes/{node}/pods"). + To(h.handlePodMetricsQuery). Doc("Get pod-level metric data of all pods on a specific node."). Param(ws.PathParameter("node", "Node name.").DataType("string").Required(true)). Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `pod_cpu_usage|pod_memory_usage`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). @@ -281,12 +278,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/nodes/{node}/pods/{pod}").To(monitoring.MonitorPod). + ws.Route(ws.GET("/nodes/{node}/pods/{pod}"). + To(h.handlePodMetricsQuery). Doc("Get pod-level metric data of a specific pod. Navigate to the pod by the node where it is scheduled."). Param(ws.PathParameter("node", "Node name.").DataType("string").Required(true)). Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)). @@ -296,12 +293,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}/containers").To(monitoring.MonitorContainer). + ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}/containers"). + To(h.handleContainerMetricsQuery). Doc("Get container-level metric data of a specific pod's containers. Navigate to the pod by the pod's namespace."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)). @@ -316,12 +313,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.ContainerMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}/containers/{container}").To(monitoring.MonitorContainer). + ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}/containers/{container}"). + To(h.handleContainerMetricsQuery). Doc("Get container-level metric data of a specific container. Navigate to the container by the pod name and the namespace."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)). @@ -332,12 +329,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). Metadata(restfulspec.KeyOpenAPITags, []string{constants.ContainerMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/storageclasses/{storageclass}/persistentvolumeclaims").To(monitoring.MonitorPVC). + ws.Route(ws.GET("/storageclasses/{storageclass}/persistentvolumeclaims"). + To(h.handlePVCMetricsQuery). Doc("Get PVC-level metric data of the specific storageclass's PVCs."). Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(true)). Param(ws.PathParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both PVC available and used inodes: `pvc_inodes_available|pvc_inodes_used`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). @@ -351,12 +348,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.PVCMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/namespaces/{namespace}/persistentvolumeclaims").To(monitoring.MonitorPVC). + ws.Route(ws.GET("/namespaces/{namespace}/persistentvolumeclaims"). + To(h.handlePVCMetricsQuery). Doc("Get PVC-level metric data of the specific namespace's PVCs."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.PathParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both PVC available and used inodes: `pvc_inodes_available|pvc_inodes_used`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). @@ -370,12 +367,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.PVCMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/namespaces/{namespace}/persistentvolumeclaims/{pvc}").To(monitoring.MonitorPVC). + ws.Route(ws.GET("/namespaces/{namespace}/persistentvolumeclaims/{pvc}"). + To(h.handlePVCMetricsQuery). Doc("Get PVC-level metric data of a specific PVC. Navigate to the PVC by the PVC's namespace."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.PathParameter("pvc", "PVC name.").DataType("string").Required(true)). @@ -385,12 +382,12 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). Metadata(restfulspec.KeyOpenAPITags, []string{constants.PVCMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) - ws.Route(ws.GET("/components/{component}").To(monitoring.MonitorComponent). + ws.Route(ws.GET("/components/{component}"). + To(h.handleComponentMetricsQuery). Doc("Get component-level metric data of the specific system component."). Param(ws.PathParameter("component", "system component to monitor. One of etcd, apiserver, scheduler, controller_manager, coredns, prometheus.").DataType("string").Required(true)). Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both etcd server list and total size of the underlying database: `etcd_server_list|etcd_mvcc_db_size`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). @@ -399,9 +396,8 @@ func addWebService(c *restful.Container) error { Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). Metadata(restfulspec.KeyOpenAPITags, []string{constants.ComponentMetricsTag}). - Writes(metrics.Response{}). - Returns(http.StatusOK, RespOK, metrics.Response{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). + Writes(v1alpha2.APIResponse{}). + Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). Produces(restful.MIME_JSON) c.Add(ws) diff --git a/pkg/kapis/tenant/v1alpha2/handler.go b/pkg/kapis/tenant/v1alpha2/handler.go index f1e973ee5..9a551ab43 100644 --- a/pkg/kapis/tenant/v1alpha2/handler.go +++ b/pkg/kapis/tenant/v1alpha2/handler.go @@ -11,7 +11,7 @@ import ( "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/models/iam" - "kubesphere.io/kubesphere/pkg/models/metrics" + "kubesphere.io/kubesphere/pkg/models/monitoring" "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2" "kubesphere.io/kubesphere/pkg/models/tenant" apierr "kubesphere.io/kubesphere/pkg/server/errors" @@ -127,7 +127,7 @@ func (h *tenantHandler) ListNamespaces(req *restful.Request, resp *restful.Respo namespaces = append(namespaces, item.(*v1.Namespace).DeepCopy()) } - namespaces = metrics.GetNamespacesWithMetrics(namespaces) + namespaces = monitoring.GetNamespacesWithMetrics(namespaces) items := make([]interface{}, 0) diff --git a/pkg/models/metrics/constants.go b/pkg/models/metrics/constants.go deleted file mode 100644 index d20edcb7d..000000000 --- a/pkg/models/metrics/constants.go +++ /dev/null @@ -1,41 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package metrics - -const ( - MonitorLevelCluster = "cluster" - MonitorLevelNode = "node" - MonitorLevelWorkspace = "workspace" - MonitorLevelNamespace = "namespace" - MonitorLevelPod = "pod" - MonitorLevelContainer = "container" - MonitorLevelPVC = "pvc" - MonitorLevelWorkload = "workload" - MonitorLevelComponent = "component" - - ChannelMaxCapacity = 100 - - // prometheus query type - RangeQuery = "query_range" - Query = "query" - DefaultQueryStep = "10m" - - StatefulSet = "StatefulSet" - DaemonSet = "DaemonSet" - Deployment = "Deployment" -) diff --git a/pkg/models/metrics/metrics.go b/pkg/models/metrics/metrics.go deleted file mode 100644 index be6453fd3..000000000 --- a/pkg/models/metrics/metrics.go +++ /dev/null @@ -1,802 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ - -package metrics - -import ( - "fmt" - "github.com/json-iterator/go" - "k8s.io/klog" - "kubesphere.io/kubesphere/pkg/api/monitoring/v1alpha2" - cs "kubesphere.io/kubesphere/pkg/simple/client" - "net/url" - "regexp" - "strings" - "sync" - "time" -) - -var jsonIter = jsoniter.ConfigCompatibleWithStandardLibrary - -func GetClusterMetrics(params RequestParams) *Response { - client, err := cs.ClientSets().Prometheus() - if err != nil { - klog.Error(err) - return nil - } - - ch := make(chan APIResponse, ChannelMaxCapacity) - var wg sync.WaitGroup - - // for each metric, make PromQL expression and send the request to Prometheus servers - for _, metricName := range clusterMetrics { - matched, _ := regexp.MatchString(params.MetricsFilter, metricName) - if matched { - wg.Add(1) - go func(metricName string, params RequestParams) { - exp := makePromqlForCluster(metricName, params) - v := url.Values{} - for key, value := range params.QueryParams { - v[key] = value - } - v.Set("query", exp) - response := client.QueryToK8SPrometheus(params.QueryType, v.Encode()) - ch <- APIResponse{ - MetricName: metricName, - APIResponse: response, - } - wg.Done() - }(metricName, params) - } - } - wg.Wait() - close(ch) - - var apiResponse []APIResponse - for e := range ch { - apiResponse = append(apiResponse, e) - } - - return &Response{ - MetricsLevel: MonitorLevelCluster, - Results: apiResponse, - } -} - -func GetNodeMetrics(params RequestParams) *Response { - client, err := cs.ClientSets().Prometheus() - if err != nil { - klog.Error(err) - return nil - } - - ch := make(chan APIResponse, ChannelMaxCapacity) - var wg sync.WaitGroup - - // for each metric, make PromQL expression and send the request to Prometheus servers - for _, metricName := range nodeMetrics { - matched, _ := regexp.MatchString(params.MetricsFilter, metricName) - if matched { - wg.Add(1) - go func(metricName string, params RequestParams) { - exp := makePromqlForNode(metricName, params) - v := url.Values{} - for key, value := range params.QueryParams { - v[key] = value - } - v.Set("query", exp) - response := client.QueryToK8SPrometheus(params.QueryType, v.Encode()) - - // add label resouce_name, node_ip, node_role to each metric result item - // resouce_name serves as a unique identifier for the monitored resource - // it will be used during metrics sorting - for _, item := range response.Data.Result { - nodeName := item.Metric["node"] - item.Metric["resource_name"] = nodeName - item.Metric["node_ip"], item.Metric["node_role"] = getNodeAddressAndRole(nodeName) - } - - ch <- APIResponse{ - MetricName: metricName, - APIResponse: response, - } - wg.Done() - }(metricName, params) - } - } - wg.Wait() - close(ch) - - var apiResponse []APIResponse - for e := range ch { - apiResponse = append(apiResponse, e) - } - - return &Response{ - MetricsLevel: MonitorLevelNode, - Results: apiResponse, - } -} - -func GetWorkspaceMetrics(params RequestParams) *Response { - client, err := cs.ClientSets().Prometheus() - if err != nil { - klog.Error(err) - return nil - } - - ch := make(chan APIResponse, ChannelMaxCapacity) - var wg sync.WaitGroup - - // for each metric, make PromQL expression and send the request to Prometheus servers - for _, metricName := range workspaceMetrics { - matched, _ := regexp.MatchString(params.MetricsFilter, metricName) - if matched { - wg.Add(1) - go func(metricName string, params RequestParams) { - exp := makePromqlForWorkspace(metricName, params) - v := url.Values{} - for key, value := range params.QueryParams { - v[key] = value - } - v.Set("query", exp) - response := client.QueryToK8SPrometheus(params.QueryType, v.Encode()) - - // add label resouce_name - for _, item := range response.Data.Result { - item.Metric["resource_name"] = item.Metric["label_kubesphere_io_workspace"] - } - - ch <- APIResponse{ - MetricName: metricName, - APIResponse: response, - } - wg.Done() - }(metricName, params) - } - } - wg.Wait() - close(ch) - - var apiResponse []APIResponse - for e := range ch { - apiResponse = append(apiResponse, e) - } - - return &Response{ - MetricsLevel: MonitorLevelWorkspace, - Results: apiResponse, - } -} - -func GetNamespaceMetrics(params RequestParams) *Response { - client, err := cs.ClientSets().Prometheus() - if err != nil { - klog.Error(err) - return nil - } - - ch := make(chan APIResponse, ChannelMaxCapacity) - var wg sync.WaitGroup - - // for each metric, make PromQL expression and send the request to Prometheus servers - for _, metricName := range namespaceMetrics { - matched, _ := regexp.MatchString(params.MetricsFilter, metricName) - if matched { - wg.Add(1) - go func(metricName string, params RequestParams) { - exp := makePromqlForNamespace(metricName, params) - v := url.Values{} - for key, value := range params.QueryParams { - v[key] = value - } - v.Set("query", exp) - response := client.QueryToK8SPrometheus(params.QueryType, v.Encode()) - - // add label resouce_name - for _, item := range response.Data.Result { - item.Metric["resource_name"] = item.Metric["namespace"] - } - - ch <- APIResponse{ - MetricName: metricName, - APIResponse: response, - } - wg.Done() - }(metricName, params) - } - } - wg.Wait() - close(ch) - - var apiResponse []APIResponse - for e := range ch { - apiResponse = append(apiResponse, e) - } - - return &Response{ - MetricsLevel: MonitorLevelNamespace, - Results: apiResponse, - } -} - -func GetWorkloadMetrics(params RequestParams) *Response { - client, err := cs.ClientSets().Prometheus() - if err != nil { - klog.Error(err) - return nil - } - - ch := make(chan APIResponse, ChannelMaxCapacity) - var wg sync.WaitGroup - - // for each metric, make PromQL expression and send the request to Prometheus servers - for _, metricName := range workloadMetrics { - matched, _ := regexp.MatchString(params.MetricsFilter, metricName) - if matched { - wg.Add(1) - go func(metricName string, params RequestParams) { - exp := makePromqlForWorkload(metricName, params) - v := url.Values{} - for key, value := range params.QueryParams { - v[key] = value - } - v.Set("query", exp) - response := client.QueryToK8SPrometheus(params.QueryType, v.Encode()) - - // add label resouce_name - for _, item := range response.Data.Result { - item.Metric["resource_name"] = item.Metric["workload"] - } - - ch <- APIResponse{ - MetricName: metricName, - APIResponse: response, - } - wg.Done() - }(metricName, params) - } - } - wg.Wait() - close(ch) - - var apiResponse []APIResponse - for e := range ch { - apiResponse = append(apiResponse, e) - } - - return &Response{ - MetricsLevel: MonitorLevelWorkload, - Results: apiResponse, - } -} - -func GetPodMetrics(params RequestParams) *Response { - client, err := cs.ClientSets().Prometheus() - if err != nil { - klog.Error(err) - return nil - } - - ch := make(chan APIResponse, ChannelMaxCapacity) - var wg sync.WaitGroup - - // for each metric, make PromQL expression and send the request to Prometheus servers - for _, metricName := range podMetrics { - matched, _ := regexp.MatchString(params.MetricsFilter, metricName) - if matched { - wg.Add(1) - go func(metricName string, params RequestParams) { - exp := makePromqlForPod(metricName, params) - v := url.Values{} - for key, value := range params.QueryParams { - v[key] = value - } - v.Set("query", exp) - response := client.QueryToK8SPrometheus(params.QueryType, v.Encode()) - - // add label resouce_name - for _, item := range response.Data.Result { - item.Metric["resource_name"] = item.Metric["pod_name"] - } - - ch <- APIResponse{ - MetricName: metricName, - APIResponse: response, - } - wg.Done() - }(metricName, params) - } - } - wg.Wait() - close(ch) - - var apiResponse []APIResponse - for e := range ch { - apiResponse = append(apiResponse, e) - } - - return &Response{ - MetricsLevel: MonitorLevelPod, - Results: apiResponse, - } -} - -func GetContainerMetrics(params RequestParams) *Response { - client, err := cs.ClientSets().Prometheus() - if err != nil { - klog.Error(err) - return nil - } - - ch := make(chan APIResponse, ChannelMaxCapacity) - var wg sync.WaitGroup - - // for each metric, make PromQL expression and send the request to Prometheus servers - for _, metricName := range containerMetrics { - matched, _ := regexp.MatchString(params.MetricsFilter, metricName) - if matched { - wg.Add(1) - go func(metricName string, params RequestParams) { - exp := makePromqlForContainer(metricName, params) - v := url.Values{} - for key, value := range params.QueryParams { - v[key] = value - } - v.Set("query", exp) - response := client.QueryToK8SPrometheus(params.QueryType, v.Encode()) - - // add label resouce_name - for _, item := range response.Data.Result { - item.Metric["resource_name"] = item.Metric["container_name"] - } - - ch <- APIResponse{ - MetricName: metricName, - APIResponse: response, - } - wg.Done() - }(metricName, params) - } - } - wg.Wait() - close(ch) - - var apiResponse []APIResponse - for e := range ch { - apiResponse = append(apiResponse, e) - } - - return &Response{ - MetricsLevel: MonitorLevelContainer, - Results: apiResponse, - } -} - -func GetPVCMetrics(params RequestParams) *Response { - client, err := cs.ClientSets().Prometheus() - if err != nil { - klog.Error(err) - return nil - } - - ch := make(chan APIResponse, ChannelMaxCapacity) - var wg sync.WaitGroup - - // for each metric, make PromQL expression and send the request to Prometheus servers - for _, metricName := range pvcMetrics { - matched, _ := regexp.MatchString(params.MetricsFilter, metricName) - if matched { - wg.Add(1) - go func(metricName string, params RequestParams) { - exp := makePromqlForPVC(metricName, params) - v := url.Values{} - for key, value := range params.QueryParams { - v[key] = value - } - v.Set("query", exp) - response := client.QueryToK8SPrometheus(params.QueryType, v.Encode()) - - // add label resouce_name - for _, item := range response.Data.Result { - item.Metric["resource_name"] = item.Metric["persistentvolumeclaim"] - } - - ch <- APIResponse{ - MetricName: metricName, - APIResponse: response, - } - wg.Done() - }(metricName, params) - } - } - wg.Wait() - close(ch) - - var apiResponse []APIResponse - for e := range ch { - apiResponse = append(apiResponse, e) - } - - return &Response{ - MetricsLevel: MonitorLevelPVC, - Results: apiResponse, - } -} - -func GetComponentMetrics(params RequestParams) *Response { - client, err := cs.ClientSets().Prometheus() - if err != nil { - klog.Error(err) - return nil - } - - ch := make(chan APIResponse, ChannelMaxCapacity) - var wg sync.WaitGroup - - // for each metric, make PromQL expression and send the request to Prometheus servers - for _, metricName := range componentMetrics { - matchComponentName, _ := regexp.MatchString(params.ComponentName, metricName) - matchMetricsFilter, _ := regexp.MatchString(params.MetricsFilter, metricName) - if matchComponentName && matchMetricsFilter { - wg.Add(1) - go func(metricName string, params RequestParams) { - exp := makePromqlForComponent(metricName, params) - v := url.Values{} - for key, value := range params.QueryParams { - v[key] = value - } - v.Set("query", exp) - response := client.QueryToK8SSystemPrometheus(params.QueryType, v.Encode()) - - // add node address when queried metric is etcd_server_list - if metricName == "etcd_server_list" { - for _, item := range response.Data.Result { - item.Metric["node_name"] = getNodeName(item.Metric["node_ip"]) - } - } - - ch <- APIResponse{ - MetricName: metricName, - APIResponse: response, - } - wg.Done() - }(metricName, params) - } - } - wg.Wait() - close(ch) - - var apiResponse []APIResponse - for e := range ch { - apiResponse = append(apiResponse, e) - } - - return &Response{ - MetricsLevel: MonitorLevelComponent, - Results: apiResponse, - } -} - -func makePromqlForCluster(metricName string, _ RequestParams) string { - return metricsPromqlMap[metricName] -} - -func makePromqlForNode(metricName string, params RequestParams) string { - var rule = metricsPromqlMap[metricName] - var nodeSelector string - - if params.NodeName != "" { - nodeSelector = fmt.Sprintf(`node="%s"`, params.NodeName) - } else { - nodeSelector = fmt.Sprintf(`node=~"%s"`, params.ResourcesFilter) - } - - return strings.Replace(rule, "$1", nodeSelector, -1) -} - -func makePromqlForWorkspace(metricName string, params RequestParams) string { - var exp = metricsPromqlMap[metricName] - var workspaceSelector string - - if params.WorkspaceName != "" { - workspaceSelector = fmt.Sprintf(`label_kubesphere_io_workspace="%s"`, params.WorkspaceName) - } else { - workspaceSelector = fmt.Sprintf(`label_kubesphere_io_workspace=~"%s", label_kubesphere_io_workspace!=""`, params.ResourcesFilter) - } - - return strings.Replace(exp, "$1", workspaceSelector, -1) -} - -func makePromqlForNamespace(metricName string, params RequestParams) string { - var exp = metricsPromqlMap[metricName] - var namespaceSelector string - - // For monitoring namespaces in the specific workspace - // GET /workspaces/{workspace}/namespaces - if params.WorkspaceName != "" { - namespaceSelector = fmt.Sprintf(`label_kubesphere_io_workspace="%s", namespace=~"%s"`, params.WorkspaceName, params.ResourcesFilter) - return strings.Replace(exp, "$1", namespaceSelector, -1) - } - - // For monitoring the specific namespaces - // GET /namespaces/{namespace} or - // GET /namespaces - if params.NamespaceName != "" { - namespaceSelector = fmt.Sprintf(`namespace="%s"`, params.NamespaceName) - } else { - namespaceSelector = fmt.Sprintf(`namespace=~"%s"`, params.ResourcesFilter) - } - return strings.Replace(exp, "$1", namespaceSelector, -1) -} - -func makePromqlForWorkload(metricName string, params RequestParams) string { - var exp = metricsPromqlMap[metricName] - var kind, kindSelector, workloadSelector string - - switch params.WorkloadKind { - case "deployment": - kind = Deployment - kindSelector = fmt.Sprintf(`namespace="%s", deployment!="", deployment=~"%s"`, params.NamespaceName, params.ResourcesFilter) - case "statefulset": - kind = StatefulSet - kindSelector = fmt.Sprintf(`namespace="%s", statefulset!="", statefulset=~"%s"`, params.NamespaceName, params.ResourcesFilter) - case "daemonset": - kind = DaemonSet - kindSelector = fmt.Sprintf(`namespace="%s", daemonset!="", daemonset=~"%s"`, params.NamespaceName, params.ResourcesFilter) - default: - kind = ".*" - kindSelector = fmt.Sprintf(`namespace="%s"`, params.NamespaceName) - } - - workloadSelector = fmt.Sprintf(`namespace="%s", workload=~"%s:%s"`, params.NamespaceName, kind, params.ResourcesFilter) - return strings.NewReplacer("$1", workloadSelector, "$2", kindSelector).Replace(exp) -} - -func makePromqlForPod(metricName string, params RequestParams) string { - var exp = metricsPromqlMap[metricName] - var podSelector, workloadSelector string - - // For monitoriong pods of the specific workload - // GET /namespaces/{namespace}/workloads/{kind}/{workload}/pods - if params.WorkloadName != "" { - switch params.WorkloadKind { - case "deployment": - workloadSelector = fmt.Sprintf(`owner_kind="ReplicaSet", owner_name=~"^%s-[^-]{1,10}$"`, params.WorkloadName) - case "statefulset": - workloadSelector = fmt.Sprintf(`owner_kind="StatefulSet", owner_name="%s"`, params.WorkloadName) - case "daemonset": - workloadSelector = fmt.Sprintf(`owner_kind="DaemonSet", owner_name="%s"`, params.WorkloadName) - } - } - - // For monitoring pods in the specific namespace - // GET /namespaces/{namespace}/workloads/{kind}/{workload}/pods or - // GET /namespaces/{namespace}/pods/{pod} or - // GET /namespaces/{namespace}/pods - if params.NamespaceName != "" { - if params.PodName != "" { - podSelector = fmt.Sprintf(`pod="%s", namespace="%s"`, params.PodName, params.NamespaceName) - } else { - podSelector = fmt.Sprintf(`pod=~"%s", namespace="%s"`, params.ResourcesFilter, params.NamespaceName) - } - } - - // For monitoring pods on the specific node - // GET /nodes/{node}/pods/{pod} - if params.NodeName != "" { - if params.PodName != "" { - podSelector = fmt.Sprintf(`pod="%s", node="%s"`, params.PodName, params.NodeName) - } else { - podSelector = fmt.Sprintf(`pod=~"%s", node="%s"`, params.ResourcesFilter, params.NodeName) - } - } - - return strings.NewReplacer("$1", workloadSelector, "$2", podSelector).Replace(exp) -} - -func makePromqlForContainer(metricName string, params RequestParams) string { - var exp = metricsPromqlMap[metricName] - var containerSelector string - - if params.ContainerName != "" { - containerSelector = fmt.Sprintf(`pod_name="%s", namespace="%s", container_name="%s"`, params.PodName, params.NamespaceName, params.ContainerName) - } else { - containerSelector = fmt.Sprintf(`pod_name="%s", namespace="%s", container_name=~"%s"`, params.PodName, params.NamespaceName, params.ResourcesFilter) - } - - return strings.Replace(exp, "$1", containerSelector, -1) -} - -func makePromqlForPVC(metricName string, params RequestParams) string { - var exp = metricsPromqlMap[metricName] - var pvcSelector string - - // For monitoring persistentvolumeclaims in the specific namespace - // GET /namespaces/{namespace}/persistentvolumeclaims/{persistentvolumeclaim} or - // GET /namespaces/{namespace}/persistentvolumeclaims - if params.NamespaceName != "" { - if params.PVCName != "" { - pvcSelector = fmt.Sprintf(`namespace="%s", persistentvolumeclaim="%s"`, params.NamespaceName, params.PVCName) - } else { - pvcSelector = fmt.Sprintf(`namespace="%s", persistentvolumeclaim=~"%s"`, params.NamespaceName, params.ResourcesFilter) - } - return strings.Replace(exp, "$1", pvcSelector, -1) - } - - // For monitoring persistentvolumeclaims of the specific storageclass - // GET /storageclasses/{storageclass}/persistentvolumeclaims - if params.StorageClassName != "" { - pvcSelector = fmt.Sprintf(`storageclass="%s", persistentvolumeclaim=~"%s"`, params.StorageClassName, params.ResourcesFilter) - } - return strings.Replace(exp, "$1", pvcSelector, -1) -} - -func makePromqlForComponent(metricName string, _ RequestParams) string { - return metricsPromqlMap[metricName] -} - -func GetClusterStatistics() *Response { - - now := time.Now().Unix() - - var metricsArray []APIResponse - workspaceStats := APIResponse{MetricName: MetricClusterWorkspaceCount} - devopsStats := APIResponse{MetricName: MetricClusterDevopsCount} - namespaceStats := APIResponse{MetricName: MetricClusterNamespaceCount} - accountStats := APIResponse{MetricName: MetricClusterAccountCount} - - wg := sync.WaitGroup{} - wg.Add(4) - - //go func() { - // num, err := workspaces.WorkspaceCount() - // if err != nil { - // klog.Errorln(err) - // workspaceStats.Status = "error" - // } else { - // workspaceStats.withMetricResult(now, num) - // } - // wg.Done() - //}() - - //go func() { - //num, err := workspaces.GetAllDevOpsProjectsNums() - //if err != nil { - // if _, notEnabled := err.(cs.ClientSetNotEnabledError); !notEnabled { - // klog.Errorln(err) - // } - // devopsStats.Status = "error" - //} else { - // devopsStats.withMetricResult(now, num) - //} - // wg.Done() - //}() - - //go func() { - //num, err := workspaces.GetAllProjectNums() - //if err != nil { - // klog.Errorln(err) - // namespaceStats.Status = "error" - //} else { - // namespaceStats.withMetricResult(now, num) - //} - // wg.Done() - //}() - - go func() { - ret, err := cs.ClientSets().KubeSphere().ListUsers() - if err != nil { - klog.Errorln(err) - accountStats.Status = "error" - } else { - accountStats.withMetricResult(now, ret.TotalCount) - } - wg.Done() - }() - - wg.Wait() - - metricsArray = append(metricsArray, workspaceStats, devopsStats, namespaceStats, accountStats) - - return &Response{ - MetricsLevel: MonitorLevelCluster, - Results: metricsArray, - } -} - -func GetWorkspaceStatistics(workspaceName string) *Response { - - //now := time.Now().Unix() - - var metricsArray []APIResponse - namespaceStats := APIResponse{MetricName: MetricWorkspaceNamespaceCount} - devopsStats := APIResponse{MetricName: MetricWorkspaceDevopsCount} - memberStats := APIResponse{MetricName: MetricWorkspaceMemberCount} - roleStats := APIResponse{MetricName: MetricWorkspaceRoleCount} - - wg := sync.WaitGroup{} - wg.Add(4) - - //go func() { - // num, err := workspaces.WorkspaceNamespaceCount(workspaceName) - // if err != nil { - // klog.Errorln(err) - // namespaceStats.Status = "error" - // } else { - // namespaceStats.withMetricResult(now, num) - // } - // wg.Done() - //}() - - //go func() { - // num, err := workspaces.GetDevOpsProjectsCount(workspaceName) - // if err != nil { - // if _, notEnabled := err.(cs.ClientSetNotEnabledError); !notEnabled { - // klog.Errorln(err) - // } - // devopsStats.Status = "error" - // } else { - // devopsStats.withMetricResult(now, num) - // } - // wg.Done() - //}() - - //go func() { - //num, err := workspaces.WorkspaceUserCount(workspaceName) - //if err != nil { - // klog.Errorln(err) - // memberStats.Status = "error" - //} else { - // memberStats.withMetricResult(now, num) - //} - // wg.Done() - //}() - - //go func() { - //num, err := workspaces.GetOrgRolesCount(workspaceName) - // if err != nil { - // klog.Errorln(err) - // roleStats.Status = "error" - // } else { - // roleStats.withMetricResult(now, num) - // } - // wg.Done() - //}() - - wg.Wait() - - metricsArray = append(metricsArray, namespaceStats, devopsStats, memberStats, roleStats) - - return &Response{ - MetricsLevel: MonitorLevelWorkspace, - Results: metricsArray, - } -} - -func (response *APIResponse) withMetricResult(time int64, value int) { - response.Status = "success" - response.Data = v1alpha2.QueryResult{ - ResultType: "vector", - Result: []v1alpha2.QueryValue{ - { - Value: []interface{}{time, value}, - }, - }, - } -} diff --git a/pkg/models/metrics/namespaces.go b/pkg/models/metrics/namespaces.go deleted file mode 100644 index 5125cc168..000000000 --- a/pkg/models/metrics/namespaces.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package metrics - -import ( - "net/url" - "strings" - - "k8s.io/api/core/v1" -) - -func GetNamespacesWithMetrics(namespaces []*v1.Namespace) []*v1.Namespace { - var nsNameList []string - for i := range namespaces { - nsNameList = append(nsNameList, namespaces[i].Name) - } - nsFilter := "^(" + strings.Join(nsNameList, "|") + ")$" - var timeRelateParams = make(url.Values) - - params := RequestParams{ - ResourcesFilter: nsFilter, - QueryParams: timeRelateParams, - QueryType: Query, - MetricsFilter: "namespace_cpu_usage|namespace_memory_usage_wo_cache|namespace_pod_count", - } - - rawMetrics := GetNamespaceMetrics(params) - if rawMetrics == nil { - return namespaces - } - - for _, result := range rawMetrics.Results { - for _, data := range result.Data.Result { - - ns, exist := data.Metric["namespace"] - - if !exist || len(data.Value) != 2 { - continue - } - - for _, item := range namespaces { - if item.Name == ns { - if item.Annotations == nil { - item.Annotations = make(map[string]string, 0) - } - item.Annotations[result.MetricName] = data.Value[1].(string) - } - } - } - } - - return namespaces -} diff --git a/pkg/models/metrics/types.go b/pkg/models/metrics/types.go deleted file mode 100644 index 1b8023da6..000000000 --- a/pkg/models/metrics/types.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ - -package metrics - -import ( - "kubesphere.io/kubesphere/pkg/api/monitoring/v1alpha2" - "net/url" -) - -type RequestParams struct { - QueryParams url.Values - QueryType string - SortMetric string - SortType string - PageNum string - LimitNum string - Type string - MetricsFilter string - ResourcesFilter string - NodeName string - WorkspaceName string - NamespaceName string - WorkloadKind string - WorkloadName string - PodName string - ContainerName string - PVCName string - StorageClassName string - ComponentName string -} - -type APIResponse struct { - MetricName string `json:"metric_name,omitempty" description:"metric name, eg. scheduler_up_sum"` - v1alpha2.APIResponse -} - -type Response struct { - MetricsLevel string `json:"metrics_level" description:"metric level, eg. cluster"` - Results []APIResponse `json:"results" description:"actual array of results"` - CurrentPage int `json:"page,omitempty" description:"current page returned"` - TotalPage int `json:"total_page,omitempty" description:"total number of pages"` - TotalItem int `json:"total_item,omitempty" description:"page size"` -} diff --git a/pkg/models/metrics/util.go b/pkg/models/metrics/util.go deleted file mode 100644 index bead04ff4..000000000 --- a/pkg/models/metrics/util.go +++ /dev/null @@ -1,293 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ - -package metrics - -import ( - "k8s.io/apimachinery/pkg/labels" - "k8s.io/klog" - "kubesphere.io/kubesphere/pkg/api/monitoring/v1alpha2" - "kubesphere.io/kubesphere/pkg/informers" - "math" - "sort" - "strconv" - - "runtime/debug" -) - -const ( - DefaultPageLimit = 5 - DefaultPage = 1 - - ResultTypeVector = "vector" - ResultTypeMatrix = "matrix" - MetricStatusSuccess = "success" - ResultItemMetricResourceName = "resource_name" - ResultSortTypeDesc = "desc" - ResultSortTypeAsc = "asc" -) - -type FormatedMetricDataWrapper struct { - fmtMetricData v1alpha2.QueryResult - by func(p, q *v1alpha2.QueryValue) bool -} - -func (wrapper FormatedMetricDataWrapper) Len() int { - return len(wrapper.fmtMetricData.Result) -} - -func (wrapper FormatedMetricDataWrapper) Less(i, j int) bool { - return wrapper.by(&wrapper.fmtMetricData.Result[i], &wrapper.fmtMetricData.Result[j]) -} - -func (wrapper FormatedMetricDataWrapper) Swap(i, j int) { - wrapper.fmtMetricData.Result[i], wrapper.fmtMetricData.Result[j] = wrapper.fmtMetricData.Result[j], wrapper.fmtMetricData.Result[i] -} - -// sorted metric by ascending or descending order -func (rawMetrics *Response) SortBy(sortMetricName string, sortType string) (*Response, int) { - defer func() { - if err := recover(); err != nil { - klog.Errorln(err) - debug.PrintStack() - } - }() - - if sortMetricName == "" || rawMetrics == nil { - return rawMetrics, -1 - } - - // default sort type is descending order - if sortType == "" { - sortType = ResultSortTypeDesc - } - - var currentResourceMap = make(map[string]int) - - // {: } - var indexMap = make(map[string]int) - i := 0 - - // each metricItem is the result for a specific metric name - // so we find the metricItem with sortMetricName, and sort it - for _, metricItem := range rawMetrics.Results { - // only vector type result can be sorted - if metricItem.Data.ResultType == ResultTypeVector && metricItem.Status == MetricStatusSuccess { - if metricItem.MetricName == sortMetricName { - if sortType == ResultSortTypeAsc { - // asc - sort.Sort(FormatedMetricDataWrapper{metricItem.Data, func(p, q *v1alpha2.QueryValue) bool { - value1 := p.Value - value2 := q.Value - v1, _ := strconv.ParseFloat(value1[len(value1)-1].(string), 64) - v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64) - if v1 == v2 { - resourceName1 := p.Metric[ResultItemMetricResourceName] - resourceName2 := q.Metric[ResultItemMetricResourceName] - return resourceName1 < resourceName2 - } - - return v1 < v2 - }}) - } else { - // desc - sort.Sort(FormatedMetricDataWrapper{metricItem.Data, func(p, q *v1alpha2.QueryValue) bool { - value1 := p.Value - value2 := q.Value - v1, _ := strconv.ParseFloat(value1[len(value1)-1].(string), 64) - v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64) - - if v1 == v2 { - resourceName1 := p.Metric[ResultItemMetricResourceName] - resourceName2 := q.Metric[ResultItemMetricResourceName] - return resourceName1 > resourceName2 - } - - return v1 > v2 - }}) - } - - for _, r := range metricItem.Data.Result { - // record the ordering of resource_name to indexMap - // example: {"metric":{ResultItemMetricResourceName: "Deployment:xxx"},"value":[1541142931.731,"3"]} - resourceName, exist := r.Metric[ResultItemMetricResourceName] - if exist { - if _, exist := indexMap[resourceName]; !exist { - indexMap[resourceName] = i - i = i + 1 - } - } - } - } - - // iterator all metric to find max metricItems length - for _, r := range metricItem.Data.Result { - k, ok := r.Metric[ResultItemMetricResourceName] - if ok { - currentResourceMap[k] = 1 - } - } - - } - } - - var keys []string - for k := range currentResourceMap { - keys = append(keys, k) - } - sort.Strings(keys) - - for _, resource := range keys { - if _, exist := indexMap[resource]; !exist { - indexMap[resource] = i - i = i + 1 - } - } - - // sort other metric - for i := 0; i < len(rawMetrics.Results); i++ { - re := rawMetrics.Results[i] - if re.Data.ResultType == ResultTypeVector && re.Status == MetricStatusSuccess { - sortedMetric := make([]v1alpha2.QueryValue, len(indexMap)) - for j := 0; j < len(re.Data.Result); j++ { - r := re.Data.Result[j] - k, exist := r.Metric[ResultItemMetricResourceName] - if exist { - index, exist := indexMap[k] - if exist { - sortedMetric[index] = r - } - } - } - - rawMetrics.Results[i].Data.Result = sortedMetric - } - } - - return rawMetrics, len(indexMap) -} - -func (fmtLevelMetric *Response) Page(pageNum string, limitNum string, maxLength int) *Response { - if maxLength <= 0 { - return fmtLevelMetric - } - // matrix type can not be sorted - for _, metricItem := range fmtLevelMetric.Results { - // if metric reterieved field, resultType: "" - if metricItem.Data.ResultType == ResultTypeMatrix { - return fmtLevelMetric - } - } - - var page = DefaultPage - - if pageNum != "" { - p, err := strconv.Atoi(pageNum) - if err != nil { - klog.Errorln(err) - } else { - if p > 0 { - page = p - } - } - } else { - // the default mode is none paging - return fmtLevelMetric - } - - var limit = DefaultPageLimit - - if limitNum != "" { - l, err := strconv.Atoi(limitNum) - if err != nil { - klog.Errorln(err) - } else { - if l > 0 { - limit = l - } - } - } - - // the i page: [(page-1) * limit, (page) * limit - 1] - start := (page - 1) * limit - end := (page)*limit - 1 - - for i := 0; i < len(fmtLevelMetric.Results); i++ { - // only pageing when result type is `vector` and result status is `success` - if fmtLevelMetric.Results[i].Data.ResultType != ResultTypeVector || fmtLevelMetric.Results[i].Status != MetricStatusSuccess { - continue - } - resultLen := len(fmtLevelMetric.Results[i].Data.Result) - if start >= resultLen { - fmtLevelMetric.Results[i].Data.Result = nil - continue - } - if end >= resultLen { - end = resultLen - 1 - } - slice := fmtLevelMetric.Results[i].Data.Result[start : end+1] - fmtLevelMetric.Results[i].Data.Result = slice - } - - allPage := int(math.Ceil(float64(maxLength) / float64(limit))) - - // add page fields - fmtLevelMetric.CurrentPage = page - fmtLevelMetric.TotalItem = maxLength - fmtLevelMetric.TotalPage = allPage - - return fmtLevelMetric -} - -func getNodeAddressAndRole(nodeName string) (string, string) { - nodeLister := informers.SharedInformerFactory().Core().V1().Nodes().Lister() - node, err := nodeLister.Get(nodeName) - if err != nil { - return "", "" - } - - var addr string - for _, address := range node.Status.Addresses { - if address.Type == "InternalIP" { - addr = address.Address - break - } - } - - role := "node" - _, exists := node.Labels["node-role.kubernetes.io/master"] - if exists { - role = "master" - } - return addr, role -} - -func getNodeName(nodeIp string) string { - nodeLister := informers.SharedInformerFactory().Core().V1().Nodes().Lister() - nodes, _ := nodeLister.List(labels.Everything()) - - for _, node := range nodes { - for _, address := range node.Status.Addresses { - if address.Type == "InternalIP" && address.Address == nodeIp { - return node.Name - } - } - } - - return "" -} diff --git a/pkg/models/monitoring/monitoring.go b/pkg/models/monitoring/monitoring.go new file mode 100644 index 000000000..0f24484af --- /dev/null +++ b/pkg/models/monitoring/monitoring.go @@ -0,0 +1,69 @@ +/* + + Copyright 2019 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +package monitoring + +import ( + "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/api/monitoring/v1alpha2" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + "time" +) + +type MonitoringOperator interface { + GetMetrics(stmts []string, time time.Time) (v1alpha2.APIResponse, error) + GetMetricsOverTime(stmts []string, start, end time.Time, step time.Duration) (v1alpha2.APIResponse, error) + GetNamedMetrics(time time.Time, opt monitoring.QueryOption) (v1alpha2.APIResponse, error) + GetNamedMetricsOverTime(start, end time.Time, step time.Duration, opt monitoring.QueryOption) (v1alpha2.APIResponse, error) + SortMetrics(raw v1alpha2.APIResponse, target, order, identifier string) (v1alpha2.APIResponse, int) + PageMetrics(raw v1alpha2.APIResponse, page, limit, rows int) v1alpha2.APIResponse +} + +type monitoringOperator struct { + c monitoring.Interface +} + +func NewMonitoringOperator(client monitoring.Interface) MonitoringOperator { + return &monitoringOperator{client} +} + +// TODO(huanggze): reserve for custom monitoring +func (mo monitoringOperator) GetMetrics(stmts []string, time time.Time) (v1alpha2.APIResponse, error) { + panic("implement me") +} + +// TODO(huanggze): reserve for custom monitoring +func (mo monitoringOperator) GetMetricsOverTime(stmts []string, start, end time.Time, step time.Duration) (v1alpha2.APIResponse, error) { + panic("implement me") +} + +func (mo monitoringOperator) GetNamedMetrics(time time.Time, opt monitoring.QueryOption) (v1alpha2.APIResponse, error) { + metrics, err := mo.c.GetNamedMetrics(time, opt) + if err != nil { + klog.Error(err) + } + return v1alpha2.APIResponse{Results: metrics}, err +} + +func (mo monitoringOperator) GetNamedMetricsOverTime(start, end time.Time, step time.Duration, opt monitoring.QueryOption) (v1alpha2.APIResponse, error) { + metrics, err := mo.c.GetNamedMetricsOverTime(start, end, step, opt) + if err != nil { + klog.Error(err) + } + return v1alpha2.APIResponse{Results: metrics}, err +} diff --git a/pkg/models/monitoring/namespaces.go b/pkg/models/monitoring/namespaces.go new file mode 100644 index 000000000..44a809bc9 --- /dev/null +++ b/pkg/models/monitoring/namespaces.go @@ -0,0 +1,64 @@ +/* + + Copyright 2019 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ +package monitoring + +import "k8s.io/api/core/v1" + +// TODO(wansir): Can we decouple this part from monitoring module, since the project structure has been changed +func GetNamespacesWithMetrics(namespaces []*v1.Namespace) []*v1.Namespace { + // var nsNameList []string + // for i := range namespaces { + // nsNameList = append(nsNameList, namespaces[i].Name) + // } + // nsFilter := "^(" + strings.Join(nsNameList, "|") + ")$" + // + // now := time.Now() + // opt := &monitoring.QueryOptions{ + // Level: monitoring.MetricsLevelNamespace, + // ResourcesFilter: nsFilter, + // Start: now, + // End: now, + // MetricsFilter: "namespace_cpu_usage|namespace_memory_usage_wo_cache|namespace_pod_count", + // } + // + // gm, err := monitoring.Get(opt) + // if err != nil { + // klog.Error(err) + // return namespaces + // } + // + // for _, m := range gm.Results { + // for _, v := range m.Data.MetricsValues { + // ns, exist := v.Metadata["namespace"] + // if !exist { + // continue + // } + // + // for _, item := range namespaces { + // if item.Name == ns { + // if item.Annotations == nil { + // item.Annotations = make(map[string]string, 0) + // } + // item.Annotations[m.MetricsName] = strconv.FormatFloat(v.Sample[1], 'f', -1, 64) + // } + // } + // } + // } + // + return namespaces +} diff --git a/pkg/models/monitoring/sort_page.go b/pkg/models/monitoring/sort_page.go new file mode 100644 index 000000000..1d280c177 --- /dev/null +++ b/pkg/models/monitoring/sort_page.go @@ -0,0 +1,204 @@ +/* + + Copyright 2019 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +package monitoring + +import ( + "kubesphere.io/kubesphere/pkg/api/monitoring/v1alpha2" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + "math" + "sort" +) + +// TODO(huanggze): the id value is dependent of Prometheus label-value pair (i.e. label_kubesphere_io_workspace). We should regulate the naming convention. +const ( + IdentifierNode = "node" + IdentifierWorkspace = "label_kubesphere_io_workspace" + IdentifierNamespace = "namespace" + IdentifierWorkload = "workload" + IdentifierPod = "pod" + IdentifierContainer = "container" + IdentifierPVC = "persistentvolumeclaim" + + OrderAscending = "asc" + OrderDescending = "desc" +) + +type wrapper struct { + monitoring.MetricData + by func(p, q *monitoring.MetricValue) bool +} + +func (w wrapper) Len() int { + return len(w.MetricValues) +} + +func (w wrapper) Less(i, j int) bool { + return w.by(&w.MetricValues[i], &w.MetricValues[j]) +} + +func (w wrapper) Swap(i, j int) { + w.MetricValues[i], w.MetricValues[j] = w.MetricValues[j], w.MetricValues[i] +} + +// The sortMetrics sorts a group of resources by a given metric +// Example: +// +// before sorting +// |------| Metric 1 | Metric 2 | Metric 3 | +// | ID a | 1 | XL | | +// | ID b | 1 | S | | +// | ID c | 3 | M | | +// +// sort by metrics_2 +// |------| Metric 1 | Metric 2 (asc) | Metric 3 | +// | ID a | 1 | XL | | +// | ID c | 3 | M | | +// | ID b | 1 | S | | +// +// ranking can only be applied to instant query results, not range query +func (mo monitoringOperator) SortMetrics(raw v1alpha2.APIResponse, target, order, identifier string) (v1alpha2.APIResponse, int) { + if target == "" || len(raw.Results) == 0 { + return raw, -1 + } + + if order == "" { + order = OrderDescending + } + + var currentResourceMap = make(map[string]int) + + // resource-ordinal map + var indexMap = make(map[string]int) + i := 0 + + for _, item := range raw.Results { + if item.MetricType == monitoring.MetricTypeVector && item.Status == monitoring.StatusSuccess { + if item.MetricName == target { + if order == OrderAscending { + sort.Sort(wrapper{item.MetricData, func(p, q *monitoring.MetricValue) bool { + if p.Sample[1] == q.Sample[1] { + return p.Metadata[identifier] < q.Metadata[identifier] + } + return p.Sample[1] < q.Sample[1] + }}) + } else { + sort.Sort(wrapper{item.MetricData, func(p, q *monitoring.MetricValue) bool { + if p.Sample[1] == q.Sample[1] { + return p.Metadata[identifier] > q.Metadata[identifier] + } + return p.Sample[1] > q.Sample[1] + }}) + } + + for _, r := range item.MetricValues { + // record the ordinal of resource to indexMap + resourceName, exist := r.Metadata[identifier] + if exist { + if _, exist := indexMap[resourceName]; !exist { + indexMap[resourceName] = i + i = i + 1 + } + } + } + } + + // get total number of rows + for _, r := range item.MetricValues { + k, ok := r.Metadata[identifier] + if ok { + currentResourceMap[k] = 1 + } + } + + } + } + + var keys []string + for k := range currentResourceMap { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, resource := range keys { + if _, exist := indexMap[resource]; !exist { + indexMap[resource] = i + i = i + 1 + } + } + + // sort other metrics + for i := 0; i < len(raw.Results); i++ { + item := raw.Results[i] + if item.MetricType == monitoring.MetricTypeVector && item.Status == monitoring.StatusSuccess { + sortedMetric := make([]monitoring.MetricValue, len(indexMap)) + for j := 0; j < len(item.MetricValues); j++ { + r := item.MetricValues[j] + k, exist := r.Metadata[identifier] + if exist { + index, exist := indexMap[k] + if exist { + sortedMetric[index] = r + } + } + } + + raw.Results[i].MetricValues = sortedMetric + } + } + + return raw, len(indexMap) +} + +func (mo monitoringOperator) PageMetrics(raw v1alpha2.APIResponse, page, limit, rows int) v1alpha2.APIResponse { + if page <= 0 || limit <= 0 || rows <= 0 || len(raw.Results) == 0 { + return raw + } + + // matrix type can not be sorted + for _, item := range raw.Results { + if item.MetricType != monitoring.MetricTypeVector { + return raw + } + } + + // the i page: [(page-1) * limit, (page) * limit - 1] + start := (page - 1) * limit + end := (page)*limit - 1 + + for i := 0; i < len(raw.Results); i++ { + if raw.Results[i].MetricType != monitoring.MetricTypeVector || raw.Results[i].Status != monitoring.StatusSuccess { + continue + } + resultLen := len(raw.Results[i].MetricValues) + if start >= resultLen { + raw.Results[i].MetricValues = nil + continue + } + if end >= resultLen { + end = resultLen - 1 + } + slice := raw.Results[i].MetricValues[start : end+1] + raw.Results[i].MetricValues = slice + } + + raw.CurrentPage = page + raw.TotalPage = int(math.Ceil(float64(rows) / float64(limit))) + raw.TotalItem = rows + return raw +} diff --git a/pkg/server/config/config.go b/pkg/server/config/config.go index bc65fbc5e..38c441ee9 100644 --- a/pkg/server/config/config.go +++ b/pkg/server/config/config.go @@ -14,10 +14,10 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/kubesphere" "kubesphere.io/kubesphere/pkg/simple/client/ldap" "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus" "kubesphere.io/kubesphere/pkg/simple/client/mysql" "kubesphere.io/kubesphere/pkg/simple/client/notification" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" - "kubesphere.io/kubesphere/pkg/simple/client/prometheus" "kubesphere.io/kubesphere/pkg/simple/client/s3" "kubesphere.io/kubesphere/pkg/simple/client/servicemesh" "kubesphere.io/kubesphere/pkg/simple/client/sonarqube" diff --git a/pkg/server/config/config_test.go b/pkg/server/config/config_test.go index 346d9ab08..3fe663d20 100644 --- a/pkg/server/config/config_test.go +++ b/pkg/server/config/config_test.go @@ -11,10 +11,10 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/kubesphere" "kubesphere.io/kubesphere/pkg/simple/client/ldap" "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus" "kubesphere.io/kubesphere/pkg/simple/client/mysql" "kubesphere.io/kubesphere/pkg/simple/client/notification" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" - "kubesphere.io/kubesphere/pkg/simple/client/prometheus" "kubesphere.io/kubesphere/pkg/simple/client/s3" "kubesphere.io/kubesphere/pkg/simple/client/servicemesh" "kubesphere.io/kubesphere/pkg/simple/client/sonarqube" diff --git a/pkg/simple/client/factory.go b/pkg/simple/client/factory.go index 400da6d6f..612ba14e1 100644 --- a/pkg/simple/client/factory.go +++ b/pkg/simple/client/factory.go @@ -9,9 +9,10 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/kubesphere" "kubesphere.io/kubesphere/pkg/simple/client/ldap" "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus" "kubesphere.io/kubesphere/pkg/simple/client/mysql" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" - "kubesphere.io/kubesphere/pkg/simple/client/prometheus" "kubesphere.io/kubesphere/pkg/simple/client/s3" "kubesphere.io/kubesphere/pkg/simple/client/sonarqube" "sync" @@ -119,7 +120,7 @@ type ClientSet struct { sonarQubeClient *sonarqube.Client redisClient cache.Interface s3Client s3.Interface - prometheusClient *prometheus.Client + prometheusClient monitoring.Interface openpitrixClient openpitrix.Client kubesphereClient *kubesphere.Client elasticSearchClient *elasticsearch.Elasticsearch @@ -320,9 +321,7 @@ func (cs *ClientSet) OpenPitrix() (openpitrix.Client, error) { } } -func (cs *ClientSet) Prometheus() (*prometheus.Client, error) { - var err error - +func (cs *ClientSet) MonitoringClient() (monitoring.Interface, error) { if cs.csoptions.prometheusOptions == nil || cs.csoptions.prometheusOptions.Endpoint == "" { return nil, ErrClientSetNotEnabled } @@ -334,10 +333,7 @@ func (cs *ClientSet) Prometheus() (*prometheus.Client, error) { defer mutex.Unlock() if cs.prometheusClient == nil { - cs.prometheusClient, err = prometheus.NewPrometheusClient(cs.csoptions.prometheusOptions) - if err != nil { - return nil, err - } + cs.prometheusClient = prometheus.NewPrometheus(cs.csoptions.prometheusOptions) } return cs.prometheusClient, nil } diff --git a/pkg/simple/client/monitoring/interface.go b/pkg/simple/client/monitoring/interface.go index 8230be3a2..2f135b5dc 100644 --- a/pkg/simple/client/monitoring/interface.go +++ b/pkg/simple/client/monitoring/interface.go @@ -1,32 +1,41 @@ package monitoring -type ClusterQuery struct { +import "time" + +const ( + StatusSuccess = "success" + StatusError = "error" + MetricTypeMatrix = "matrix" + MetricTypeVector = "vector" +) + +type Metric struct { + MetricName string `json:"metric_name,omitempty" description:"metric name, eg. scheduler_up_sum"` + Status string `json:"status" description:"result status, one of error, success"` + MetricData `json:"data" description:"actual metric result"` + ErrorType string `json:"errorType,omitempty"` + Error string `json:"error,omitempty"` } -type ClusterMetrics struct { +type MetricData struct { + MetricType string `json:"resultType" description:"result type, one of matrix, vector"` + MetricValues []MetricValue `json:"result" description:"metric data including labels, time series and values"` } -type WorkspaceQuery struct { +type Point [2]float64 + +type MetricValue struct { + Metadata map[string]string `json:"metric,omitempty" description:"time series labels"` + Sample Point `json:"value,omitempty" description:"time series, values of vector type"` + Series []Point `json:"values,omitempty" description:"time series, values of matrix type"` } -type WorkspaceMetrics struct { -} - -type NamespaceQuery struct { -} - -type NamespaceMetrics struct { -} - -// Interface defines all the abstract behaviors of monitoring type Interface interface { + // The `stmts` defines statements, expressions or rules (eg. promql in Prometheus) for querying specific metrics. + GetMetrics(stmts []string, time time.Time) ([]Metric, error) + GetMetricsOverTime(stmts []string, start, end time.Time, step time.Duration) ([]Metric, error) - // Get - GetClusterMetrics(query ClusterQuery) ClusterMetrics - - // - GetWorkspaceMetrics(query WorkspaceQuery) WorkspaceMetrics - - // - GetNamespaceMetrics(query NamespaceQuery) NamespaceMetrics + // Get named metrics (eg. node_cpu_usage) + GetNamedMetrics(time time.Time, opt QueryOption) ([]Metric, error) + GetNamedMetricsOverTime(start, end time.Time, step time.Duration, opt QueryOption) ([]Metric, error) } diff --git a/pkg/simple/client/monitoring/named_metrics.go b/pkg/simple/client/monitoring/named_metrics.go new file mode 100644 index 000000000..882b12e66 --- /dev/null +++ b/pkg/simple/client/monitoring/named_metrics.go @@ -0,0 +1,252 @@ +package monitoring + +type MonitoringLevel int + +const ( + LevelCluster = MonitoringLevel(1) << iota + LevelNode + LevelWorkspace + LevelNamespace + LevelWorkload + LevelPod + LevelContainer + LevelPVC + LevelComponent +) + +var ClusterMetrics = []string{ + "cluster_cpu_utilisation", + "cluster_cpu_usage", + "cluster_cpu_total", + "cluster_memory_utilisation", + "cluster_memory_available", + "cluster_memory_total", + "cluster_memory_usage_wo_cache", + "cluster_net_utilisation", + "cluster_net_bytes_transmitted", + "cluster_net_bytes_received", + "cluster_disk_read_iops", + "cluster_disk_write_iops", + "cluster_disk_read_throughput", + "cluster_disk_write_throughput", + "cluster_disk_size_usage", + "cluster_disk_size_utilisation", + "cluster_disk_size_capacity", + "cluster_disk_size_available", + "cluster_disk_inode_total", + "cluster_disk_inode_usage", + "cluster_disk_inode_utilisation", + "cluster_namespace_count", + "cluster_pod_count", + "cluster_pod_quota", + "cluster_pod_utilisation", + "cluster_pod_running_count", + "cluster_pod_succeeded_count", + "cluster_pod_abnormal_count", + "cluster_node_online", + "cluster_node_offline", + "cluster_node_total", + "cluster_cronjob_count", + "cluster_pvc_count", + "cluster_daemonset_count", + "cluster_deployment_count", + "cluster_endpoint_count", + "cluster_hpa_count", + "cluster_job_count", + "cluster_statefulset_count", + "cluster_replicaset_count", + "cluster_service_count", + "cluster_secret_count", + "cluster_pv_count", + "cluster_ingresses_extensions_count", + "cluster_load1", + "cluster_load5", + "cluster_load15", + "cluster_pod_abnormal_ratio", + "cluster_node_offline_ratio", +} + +var NodeMetrics = []string{ + "node_cpu_utilisation", + "node_cpu_total", + "node_cpu_usage", + "node_memory_utilisation", + "node_memory_usage_wo_cache", + "node_memory_available", + "node_memory_total", + "node_net_utilisation", + "node_net_bytes_transmitted", + "node_net_bytes_received", + "node_disk_read_iops", + "node_disk_write_iops", + "node_disk_read_throughput", + "node_disk_write_throughput", + "node_disk_size_capacity", + "node_disk_size_available", + "node_disk_size_usage", + "node_disk_size_utilisation", + "node_disk_inode_total", + "node_disk_inode_usage", + "node_disk_inode_utilisation", + "node_pod_count", + "node_pod_quota", + "node_pod_utilisation", + "node_pod_running_count", + "node_pod_succeeded_count", + "node_pod_abnormal_count", + "node_load1", + "node_load5", + "node_load15", + "node_pod_abnormal_ratio", +} + +var WorkspaceMetrics = []string{ + "workspace_cpu_usage", + "workspace_memory_usage", + "workspace_memory_usage_wo_cache", + "workspace_net_bytes_transmitted", + "workspace_net_bytes_received", + "workspace_pod_count", + "workspace_pod_running_count", + "workspace_pod_succeeded_count", + "workspace_pod_abnormal_count", + "workspace_ingresses_extensions_count", + "workspace_cronjob_count", + "workspace_pvc_count", + "workspace_daemonset_count", + "workspace_deployment_count", + "workspace_endpoint_count", + "workspace_hpa_count", + "workspace_job_count", + "workspace_statefulset_count", + "workspace_replicaset_count", + "workspace_service_count", + "workspace_secret_count", + "workspace_pod_abnormal_ratio", +} + +var NamespaceMetrics = []string{ + "namespace_cpu_usage", + "namespace_memory_usage", + "namespace_memory_usage_wo_cache", + "namespace_net_bytes_transmitted", + "namespace_net_bytes_received", + "namespace_pod_count", + "namespace_pod_running_count", + "namespace_pod_succeeded_count", + "namespace_pod_abnormal_count", + "namespace_pod_abnormal_ratio", + "namespace_memory_limit_hard", + "namespace_cpu_limit_hard", + "namespace_pod_count_hard", + "namespace_cronjob_count", + "namespace_pvc_count", + "namespace_daemonset_count", + "namespace_deployment_count", + "namespace_endpoint_count", + "namespace_hpa_count", + "namespace_job_count", + "namespace_statefulset_count", + "namespace_replicaset_count", + "namespace_service_count", + "namespace_secret_count", + "namespace_configmap_count", + "namespace_ingresses_extensions_count", + "namespace_s2ibuilder_count", +} + +var WorkloadMetrics = []string{ + "workload_cpu_usage", + "workload_memory_usage", + "workload_memory_usage_wo_cache", + "workload_net_bytes_transmitted", + "workload_net_bytes_received", + + "workload_deployment_replica", + "workload_deployment_replica_available", + "workload_statefulset_replica", + "workload_statefulset_replica_available", + "workload_daemonset_replica", + "workload_daemonset_replica_available", + "workload_deployment_unavailable_replicas_ratio", + "workload_daemonset_unavailable_replicas_ratio", + "workload_statefulset_unavailable_replicas_ratio", +} + +var PodMetrics = []string{ + "pod_cpu_usage", + "pod_memory_usage", + "pod_memory_usage_wo_cache", + "pod_net_bytes_transmitted", + "pod_net_bytes_received", +} + +var ContainerMetrics = []string{ + "container_cpu_usage", + "container_memory_usage", + "container_memory_usage_wo_cache", +} + +var PVCMetrics = []string{ + "pvc_inodes_available", + "pvc_inodes_used", + "pvc_inodes_total", + "pvc_inodes_utilisation", + "pvc_bytes_available", + "pvc_bytes_used", + "pvc_bytes_total", + "pvc_bytes_utilisation", +} + +var ComponentMetrics = []string{ + "etcd_server_list", + "etcd_server_total", + "etcd_server_up_total", + "etcd_server_has_leader", + "etcd_server_leader_changes", + "etcd_server_proposals_failed_rate", + "etcd_server_proposals_applied_rate", + "etcd_server_proposals_committed_rate", + "etcd_server_proposals_pending_count", + "etcd_mvcc_db_size", + "etcd_network_client_grpc_received_bytes", + "etcd_network_client_grpc_sent_bytes", + "etcd_grpc_call_rate", + "etcd_grpc_call_failed_rate", + "etcd_grpc_server_msg_received_rate", + "etcd_grpc_server_msg_sent_rate", + "etcd_disk_wal_fsync_duration", + "etcd_disk_wal_fsync_duration_quantile", + "etcd_disk_backend_commit_duration", + "etcd_disk_backend_commit_duration_quantile", + + "apiserver_up_sum", + "apiserver_request_rate", + "apiserver_request_by_verb_rate", + "apiserver_request_latencies", + "apiserver_request_by_verb_latencies", + + "scheduler_up_sum", + "scheduler_schedule_attempts", + "scheduler_schedule_attempt_rate", + "scheduler_e2e_scheduling_latency", + "scheduler_e2e_scheduling_latency_quantile", + + "controller_manager_up_sum", + + "coredns_up_sum", + "coredns_cache_hits", + "coredns_cache_misses", + "coredns_dns_request_rate", + "coredns_dns_request_duration", + "coredns_dns_request_duration_quantile", + "coredns_dns_request_by_type_rate", + "coredns_dns_request_by_rcode_rate", + "coredns_panic_rate", + "coredns_proxy_request_rate", + "coredns_proxy_request_duration", + "coredns_proxy_request_duration_quantile", + + "prometheus_up_sum", + "prometheus_tsdb_head_samples_appended_rate", +} diff --git a/pkg/simple/client/monitoring/prometheus.go b/pkg/simple/client/monitoring/prometheus.go deleted file mode 100644 index 5b82066e6..000000000 --- a/pkg/simple/client/monitoring/prometheus.go +++ /dev/null @@ -1,31 +0,0 @@ -package monitoring - -import ( - "net/http" - "time" -) - -// prometheus implements monitoring interface backed by Prometheus -type prometheus struct { - options *Options - client *http.Client -} - -func NewPrometheus(options *Options) Interface { - return &prometheus{ - options: options, - client: &http.Client{Timeout: 10 * time.Second}, - } -} - -func (p prometheus) GetClusterMetrics(query ClusterQuery) ClusterMetrics { - panic("implement me") -} - -func (p prometheus) GetWorkspaceMetrics(query WorkspaceQuery) WorkspaceMetrics { - panic("implement me") -} - -func (p prometheus) GetNamespaceMetrics(query NamespaceQuery) NamespaceMetrics { - panic("implement me") -} diff --git a/pkg/simple/client/monitoring/prometheus/prometheus.go b/pkg/simple/client/monitoring/prometheus/prometheus.go new file mode 100644 index 000000000..d4a02320f --- /dev/null +++ b/pkg/simple/client/monitoring/prometheus/prometheus.go @@ -0,0 +1,178 @@ +package prometheus + +import ( + "fmt" + "github.com/json-iterator/go" + "io/ioutil" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + "net/http" + "net/url" + "regexp" + "sync" + "time" +) + +var json = jsoniter.ConfigCompatibleWithStandardLibrary + +// prometheus implements monitoring interface backed by Prometheus +type prometheus struct { + options *Options + client *http.Client +} + +func NewPrometheus(options *Options) monitoring.Interface { + return &prometheus{ + options: options, + client: &http.Client{Timeout: 10 * time.Second}, + } +} + +// TODO(huanggze): reserve for custom monitoring +func (p *prometheus) GetMetrics(stmts []string, time time.Time) ([]monitoring.Metric, error) { + panic("implement me") +} + +// TODO(huanggze): reserve for custom monitoring +func (p *prometheus) GetMetricsOverTime(stmts []string, start, end time.Time, step time.Duration) ([]monitoring.Metric, error) { + panic("implement me") +} + +func (p *prometheus) GetNamedMetrics(ts time.Time, o monitoring.QueryOption) ([]monitoring.Metric, error) { + metrics := make([]monitoring.Metric, 0) + var mtx sync.Mutex // guard metrics + var wg sync.WaitGroup + + opts := monitoring.NewQueryOptions() + o.Apply(opts) + + errCh := make(chan error) + for _, metric := range opts.NamedMetrics { + matched, _ := regexp.MatchString(opts.MetricFilter, metric) + if matched { + exp := makeExpression(metric, *opts) + wg.Add(1) + go func(metric, exp string) { + res, err := p.query(exp, ts) + if err != nil { + select { + case errCh <- err: // Record error once + default: + } + } else { + res.MetricName = metric // Add metric name + mtx.Lock() + metrics = append(metrics, res) + mtx.Unlock() + } + wg.Done() + }(metric, exp) + } + } + + wg.Wait() + + select { + case err := <-errCh: + return nil, err + default: + return metrics, nil + } +} + +func (p *prometheus) GetNamedMetricsOverTime(start, end time.Time, step time.Duration, o monitoring.QueryOption) ([]monitoring.Metric, error) { + metrics := make([]monitoring.Metric, 0) + var mtx sync.Mutex // guard metrics + var wg sync.WaitGroup + + opts := monitoring.NewQueryOptions() + o.Apply(opts) + + errCh := make(chan error) + for _, metric := range opts.NamedMetrics { + matched, _ := regexp.MatchString(opts.MetricFilter, metric) + if matched { + exp := makeExpression(metric, *opts) + wg.Add(1) + go func(metric, exp string) { + res, err := p.rangeQuery(exp, start, end, step) + if err != nil { + select { + case errCh <- err: // Record error once + default: + } + } else { + res.MetricName = metric // Add metric name + mtx.Lock() + metrics = append(metrics, res) + mtx.Unlock() + } + wg.Done() + }(metric, exp) + } + } + + wg.Wait() + + select { + case err := <-errCh: + return nil, err + default: + return metrics, nil + } +} + +func (p prometheus) query(exp string, ts time.Time) (monitoring.Metric, error) { + params := &url.Values{} + params.Set("time", ts.Format(time.RFC3339)) + params.Set("query", exp) + + u := fmt.Sprintf("%s/api/v1/query?%s", p.options.Endpoint, params.Encode()) + + var m monitoring.Metric + response, err := p.client.Get(u) + if err != nil { + return monitoring.Metric{}, err + } + + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return monitoring.Metric{}, err + } + defer response.Body.Close() + + err = json.Unmarshal(body, m) + if err != nil { + return monitoring.Metric{}, err + } + + return m, nil +} + +func (p prometheus) rangeQuery(exp string, start, end time.Time, step time.Duration) (monitoring.Metric, error) { + params := &url.Values{} + params.Set("start", start.Format(time.RFC3339)) + params.Set("end", end.Format(time.RFC3339)) + params.Set("step", step.String()) + params.Set("query", exp) + + u := fmt.Sprintf("%s/api/v1/query?%s", p.options.Endpoint, params.Encode()) + + var m monitoring.Metric + response, err := p.client.Get(u) + if err != nil { + return monitoring.Metric{}, err + } + + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return monitoring.Metric{}, err + } + defer response.Body.Close() + + err = json.Unmarshal(body, m) + if err != nil { + return monitoring.Metric{}, err + } + + return m, nil +} diff --git a/pkg/simple/client/prometheus/options.go b/pkg/simple/client/monitoring/prometheus/prometheus_options.go similarity index 100% rename from pkg/simple/client/prometheus/options.go rename to pkg/simple/client/monitoring/prometheus/prometheus_options.go diff --git a/pkg/models/metrics/metrics_rules.go b/pkg/simple/client/monitoring/prometheus/promql_templates.go similarity index 77% rename from pkg/models/metrics/metrics_rules.go rename to pkg/simple/client/monitoring/prometheus/promql_templates.go index 8879a5af6..5ec826ac2 100644 --- a/pkg/models/metrics/metrics_rules.go +++ b/pkg/simple/client/monitoring/prometheus/promql_templates.go @@ -11,266 +11,22 @@ See the License for the specific language governing permissions and limitations under the License. */ -package metrics +package prometheus -const ( - // TODO: expose the following metrics in prometheus format - MetricClusterWorkspaceCount = "cluster_workspace_count" - MetricClusterAccountCount = "cluster_account_count" - MetricClusterNamespaceCount = "cluster_namespace_count" - MetricClusterDevopsCount = "cluster_devops_project_count" - - MetricWorkspaceNamespaceCount = "workspace_namespace_count" - MetricWorkspaceDevopsCount = "workspace_devops_project_count" - MetricWorkspaceMemberCount = "workspace_member_count" - MetricWorkspaceRoleCount = "workspace_role_count" +import ( + "fmt" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + "strings" ) -var clusterMetrics = []string{ - "cluster_cpu_utilisation", - "cluster_cpu_usage", - "cluster_cpu_total", - "cluster_memory_utilisation", - "cluster_memory_available", - "cluster_memory_total", - "cluster_memory_usage_wo_cache", - "cluster_net_utilisation", - "cluster_net_bytes_transmitted", - "cluster_net_bytes_received", - "cluster_disk_read_iops", - "cluster_disk_write_iops", - "cluster_disk_read_throughput", - "cluster_disk_write_throughput", - "cluster_disk_size_usage", - "cluster_disk_size_utilisation", - "cluster_disk_size_capacity", - "cluster_disk_size_available", - "cluster_disk_inode_total", - "cluster_disk_inode_usage", - "cluster_disk_inode_utilisation", - "cluster_namespace_count", - "cluster_pod_count", - "cluster_pod_quota", - "cluster_pod_utilisation", - "cluster_pod_running_count", - "cluster_pod_succeeded_count", - "cluster_pod_abnormal_count", - "cluster_node_online", - "cluster_node_offline", - "cluster_node_total", - "cluster_cronjob_count", - "cluster_pvc_count", - "cluster_daemonset_count", - "cluster_deployment_count", - "cluster_endpoint_count", - "cluster_hpa_count", - "cluster_job_count", - "cluster_statefulset_count", - "cluster_replicaset_count", - "cluster_service_count", - "cluster_secret_count", - "cluster_pv_count", - "cluster_ingresses_extensions_count", - "cluster_load1", - "cluster_load5", - "cluster_load15", - "cluster_pod_abnormal_ratio", - "cluster_node_offline_ratio", -} +const ( + StatefulSet = "StatefulSet" + DaemonSet = "DaemonSet" + Deployment = "Deployment" +) -var nodeMetrics = []string{ - "node_cpu_utilisation", - "node_cpu_total", - "node_cpu_usage", - "node_memory_utilisation", - "node_memory_usage_wo_cache", - "node_memory_available", - "node_memory_total", - "node_net_utilisation", - "node_net_bytes_transmitted", - "node_net_bytes_received", - "node_disk_read_iops", - "node_disk_write_iops", - "node_disk_read_throughput", - "node_disk_write_throughput", - "node_disk_size_capacity", - "node_disk_size_available", - "node_disk_size_usage", - "node_disk_size_utilisation", - "node_disk_inode_total", - "node_disk_inode_usage", - "node_disk_inode_utilisation", - "node_pod_count", - "node_pod_quota", - "node_pod_utilisation", - "node_pod_running_count", - "node_pod_succeeded_count", - "node_pod_abnormal_count", - "node_load1", - "node_load5", - "node_load15", - "node_pod_abnormal_ratio", -} - -var workspaceMetrics = []string{ - "workspace_cpu_usage", - "workspace_memory_usage", - "workspace_memory_usage_wo_cache", - "workspace_net_bytes_transmitted", - "workspace_net_bytes_received", - "workspace_pod_count", - "workspace_pod_running_count", - "workspace_pod_succeeded_count", - "workspace_pod_abnormal_count", - "workspace_ingresses_extensions_count", - "workspace_cronjob_count", - "workspace_pvc_count", - "workspace_daemonset_count", - "workspace_deployment_count", - "workspace_endpoint_count", - "workspace_hpa_count", - "workspace_job_count", - "workspace_statefulset_count", - "workspace_replicaset_count", - "workspace_service_count", - "workspace_secret_count", - "workspace_pod_abnormal_ratio", -} - -var namespaceMetrics = []string{ - "namespace_cpu_usage", - "namespace_memory_usage", - "namespace_memory_usage_wo_cache", - "namespace_net_bytes_transmitted", - "namespace_net_bytes_received", - "namespace_pod_count", - "namespace_pod_running_count", - "namespace_pod_succeeded_count", - "namespace_pod_abnormal_count", - "namespace_pod_abnormal_ratio", - "namespace_memory_limit_hard", - "namespace_cpu_limit_hard", - "namespace_pod_count_hard", - "namespace_cronjob_count", - "namespace_pvc_count", - "namespace_daemonset_count", - "namespace_deployment_count", - "namespace_endpoint_count", - "namespace_hpa_count", - "namespace_job_count", - "namespace_statefulset_count", - "namespace_replicaset_count", - "namespace_service_count", - "namespace_secret_count", - "namespace_configmap_count", - "namespace_ingresses_extensions_count", - "namespace_s2ibuilder_count", -} - -var workloadMetrics = []string{ - // TODO: the following five metrics are deprecated. - "workload_pod_cpu_usage", - "workload_pod_memory_usage", - "workload_pod_memory_usage_wo_cache", - "workload_pod_net_bytes_transmitted", - "workload_pod_net_bytes_received", - - "workload_cpu_usage", - "workload_memory_usage", - "workload_memory_usage_wo_cache", - "workload_net_bytes_transmitted", - "workload_net_bytes_received", - - "workload_deployment_replica", - "workload_deployment_replica_available", - "workload_statefulset_replica", - "workload_statefulset_replica_available", - "workload_daemonset_replica", - "workload_daemonset_replica_available", - "workload_deployment_unavailable_replicas_ratio", - "workload_daemonset_unavailable_replicas_ratio", - "workload_statefulset_unavailable_replicas_ratio", -} - -var podMetrics = []string{ - "pod_cpu_usage", - "pod_memory_usage", - "pod_memory_usage_wo_cache", - "pod_net_bytes_transmitted", - "pod_net_bytes_received", -} - -var containerMetrics = []string{ - "container_cpu_usage", - "container_memory_usage", - "container_memory_usage_wo_cache", -} - -var pvcMetrics = []string{ - "pvc_inodes_available", - "pvc_inodes_used", - "pvc_inodes_total", - "pvc_inodes_utilisation", - "pvc_bytes_available", - "pvc_bytes_used", - "pvc_bytes_total", - "pvc_bytes_utilisation", -} - -var componentMetrics = []string{ - "etcd_server_list", - "etcd_server_total", - "etcd_server_up_total", - "etcd_server_has_leader", - "etcd_server_leader_changes", - "etcd_server_proposals_failed_rate", - "etcd_server_proposals_applied_rate", - "etcd_server_proposals_committed_rate", - "etcd_server_proposals_pending_count", - "etcd_mvcc_db_size", - "etcd_network_client_grpc_received_bytes", - "etcd_network_client_grpc_sent_bytes", - "etcd_grpc_call_rate", - "etcd_grpc_call_failed_rate", - "etcd_grpc_server_msg_received_rate", - "etcd_grpc_server_msg_sent_rate", - "etcd_disk_wal_fsync_duration", - "etcd_disk_wal_fsync_duration_quantile", - "etcd_disk_backend_commit_duration", - "etcd_disk_backend_commit_duration_quantile", - - "apiserver_up_sum", - "apiserver_request_rate", - "apiserver_request_by_verb_rate", - "apiserver_request_latencies", - "apiserver_request_by_verb_latencies", - - "scheduler_up_sum", - "scheduler_schedule_attempts", - "scheduler_schedule_attempt_rate", - "scheduler_e2e_scheduling_latency", - "scheduler_e2e_scheduling_latency_quantile", - - "controller_manager_up_sum", - - "coredns_up_sum", - "coredns_cache_hits", - "coredns_cache_misses", - "coredns_dns_request_rate", - "coredns_dns_request_duration", - "coredns_dns_request_duration_quantile", - "coredns_dns_request_by_type_rate", - "coredns_dns_request_by_rcode_rate", - "coredns_panic_rate", - "coredns_proxy_request_rate", - "coredns_proxy_request_duration", - "coredns_proxy_request_duration_quantile", - - "prometheus_up_sum", - "prometheus_tsdb_head_samples_appended_rate", -} - -var metricsPromqlMap = map[string]string{ +//TODO(huanggze): move this part to a ConfigMap +var promQLTemplates = map[string]string{ //cluster "cluster_cpu_utilisation": ":node_cpu_utilisation:avg1m", "cluster_cpu_usage": `round(:node_cpu_utilisation:avg1m * sum(node:node_num_cpu:sum), 0.001)`, @@ -293,7 +49,7 @@ var metricsPromqlMap = map[string]string{ "cluster_disk_inode_total": `sum(node:node_inodes_total:)`, "cluster_disk_inode_usage": `sum(node:node_inodes_total:) - sum(node:node_inodes_free:)`, "cluster_disk_inode_utilisation": `cluster:disk_inode_utilization:ratio`, - "cluster_namespace_count": `count(kube_namespace_annotations)`, + "cluster_namespace_count": `count(kube_namespace_labels)`, "cluster_pod_count": `cluster:pod:sum`, "cluster_pod_quota": `sum(max(kube_node_status_capacity_pods) by (node) unless on (node) (kube_node_status_condition{condition="Ready",status=~"unknown|false"} > 0))`, "cluster_pod_utilisation": `cluster:pod_utilization:ratio`, @@ -311,7 +67,7 @@ var metricsPromqlMap = map[string]string{ "cluster_hpa_count": `sum(kube_hpa_labels)`, "cluster_job_count": `sum(kube_job_labels)`, "cluster_statefulset_count": `sum(kube_statefulset_labels)`, - "cluster_replicaset_count": `count(kube_replicaset_created)`, + "cluster_replicaset_count": `count(kube_replicaset_labels)`, "cluster_service_count": `sum(kube_service_info)`, "cluster_secret_count": `sum(kube_secret_info)`, "cluster_pv_count": `sum(kube_persistentvolume_labels)`, @@ -374,7 +130,7 @@ var metricsPromqlMap = map[string]string{ "workspace_hpa_count": `sum by (label_kubesphere_io_workspace) (kube_hpa_labels{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{$1}))`, "workspace_job_count": `sum by (label_kubesphere_io_workspace) (kube_job_labels{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{$1}))`, "workspace_statefulset_count": `sum by (label_kubesphere_io_workspace) (kube_statefulset_labels{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{$1}))`, - "workspace_replicaset_count": `count by (label_kubesphere_io_workspace) (kube_replicaset_created{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{$1}))`, + "workspace_replicaset_count": `count by (label_kubesphere_io_workspace) (kube_replicaset_labels{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{$1}))`, "workspace_service_count": `sum by (label_kubesphere_io_workspace) (kube_service_info{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{$1}))`, "workspace_secret_count": `sum by (label_kubesphere_io_workspace) (kube_secret_info{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{$1}))`, "workspace_pod_abnormal_ratio": `count by (label_kubesphere_io_workspace) ((kube_pod_info{node!=""} unless on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Succeeded"}>0) unless on (pod, namespace) ((kube_pod_status_ready{job="kube-state-metrics", condition="true"}>0) and on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Running"}>0)) unless on (pod, namespace) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", reason="ContainerCreating"}>0)) * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1}) / sum by (label_kubesphere_io_workspace) (kube_pod_status_phase{phase!="Succeeded", namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{$1}))`, @@ -401,7 +157,7 @@ var metricsPromqlMap = map[string]string{ "namespace_hpa_count": `sum by (namespace) (kube_hpa_labels{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})`, "namespace_job_count": `sum by (namespace) (kube_job_labels{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})`, "namespace_statefulset_count": `sum by (namespace) (kube_statefulset_labels{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})`, - "namespace_replicaset_count": `count by (namespace) (kube_replicaset_created{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})`, + "namespace_replicaset_count": `count by (namespace) (kube_replicaset_labels{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})`, "namespace_service_count": `sum by (namespace) (kube_service_info{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})`, "namespace_secret_count": `sum by (namespace) (kube_secret_info{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})`, "namespace_configmap_count": `sum by (namespace) (kube_configmap_info{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})`, @@ -409,13 +165,6 @@ var metricsPromqlMap = map[string]string{ "namespace_s2ibuilder_count": `sum by (namespace) (s2i_s2ibuilder_created{namespace!=""} * on (namespace) group_left(label_kubesphere_io_workspace) kube_namespace_labels{$1})`, // workload - // TODO: the following five metrics are deprecated. - "workload_pod_cpu_usage": `round(namespace:workload_cpu_usage:sum{$1}, 0.001)`, - "workload_pod_memory_usage": `namespace:workload_memory_usage:sum{$1}`, - "workload_pod_memory_usage_wo_cache": `namespace:workload_memory_usage_wo_cache:sum{$1}`, - "workload_pod_net_bytes_transmitted": `namespace:workload_net_bytes_transmitted:sum_irate{$1}`, - "workload_pod_net_bytes_received": `namespace:workload_net_bytes_received:sum_irate{$1}`, - "workload_cpu_usage": `round(namespace:workload_cpu_usage:sum{$1}, 0.001)`, "workload_memory_usage": `namespace:workload_memory_usage:sum{$1}`, "workload_memory_usage_wo_cache": `namespace:workload_memory_usage_wo_cache:sum{$1}`, @@ -435,14 +184,14 @@ var metricsPromqlMap = map[string]string{ // pod "pod_cpu_usage": `round(label_join(sum by (namespace, pod_name) (irate(container_cpu_usage_seconds_total{job="kubelet", pod_name!="", image!=""}[5m])), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}, 0.001)`, "pod_memory_usage": `label_join(sum by (namespace, pod_name) (container_memory_usage_bytes{job="kubelet", pod_name!="", image!=""}), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}`, - "pod_memory_usage_wo_cache": `label_join(sum by (namespace, pod_name) (container_memory_usage_bytes{job="kubelet", pod_name!="", image!=""} - container_memory_cache{job="kubelet", pod_name!="", image!=""}), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}`, + "pod_memory_usage_wo_cache": `label_join(sum by (namespace, pod_name) (container_memory_working_set_bytes{job="kubelet", pod_name!="", image!=""}), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}`, "pod_net_bytes_transmitted": `label_join(sum by (namespace, pod_name) (irate(container_network_transmit_bytes_total{pod_name!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet"}[5m])), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}`, "pod_net_bytes_received": `label_join(sum by (namespace, pod_name) (irate(container_network_receive_bytes_total{pod_name!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet"}[5m])), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} * on (namespace, pod) group_left(node) kube_pod_info{$2}`, // container "container_cpu_usage": `round(sum by (namespace, pod_name, container_name) (irate(container_cpu_usage_seconds_total{job="kubelet", container_name!="POD", container_name!="", image!="", $1}[5m])), 0.001)`, "container_memory_usage": `sum by (namespace, pod_name, container_name) (container_memory_usage_bytes{job="kubelet", container_name!="POD", container_name!="", image!="", $1})`, - "container_memory_usage_wo_cache": `sum by (namespace, pod_name, container_name) (container_memory_usage_bytes{job="kubelet", container_name!="POD", container_name!="", image!="", $1} - container_memory_cache{job="kubelet", container_name!="POD", container_name!="", image!="", $1})`, + "container_memory_usage_wo_cache": `sum by (namespace, pod_name, container_name) (container_memory_working_set_bytes{job="kubelet", container_name!="POD", container_name!="", image!="", $1})`, // pvc "pvc_inodes_available": `max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_inodes_free) * on (namespace, persistentvolumeclaim) group_left (storageclass) kube_persistentvolumeclaim_info{$1}`, @@ -506,3 +255,161 @@ var metricsPromqlMap = map[string]string{ "prometheus_up_sum": `prometheus:up:sum`, "prometheus_tsdb_head_samples_appended_rate": `prometheus:prometheus_tsdb_head_samples_appended:sum_rate`, } + +func makeExpression(metric string, opt monitoring.QueryOptions) string { + tmpl := promQLTemplates[metric] + switch opt.Level { + case monitoring.LevelCluster: + case monitoring.LevelNode: + makeNodeMetricExpression(tmpl, opt) + case monitoring.LevelWorkspace: + makeWorkspaceMetricExpression(tmpl, opt) + case monitoring.LevelNamespace: + makeNamespaceMetricExpression(tmpl, opt) + case monitoring.LevelWorkload: + makeWorkloadMetricExpression(tmpl, opt) + case monitoring.LevelPod: + makePodMetricExpression(tmpl, opt) + case monitoring.LevelContainer: + makeContainerMetricExpression(tmpl, opt) + case monitoring.LevelPVC: + makePVCMetricExpression(tmpl, opt) + case monitoring.LevelComponent: + default: + } + return tmpl +} + +func makeNodeMetricExpression(tmpl string, o monitoring.QueryOptions) string { + var nodeSelector string + if o.NodeName != "" { + nodeSelector = fmt.Sprintf(`node="%s"`, o.NodeName) + } else { + nodeSelector = fmt.Sprintf(`node=~"%s"`, o.ResourceFilter) + } + return strings.Replace(tmpl, "$1", nodeSelector, -1) +} + +func makeWorkspaceMetricExpression(tmpl string, o monitoring.QueryOptions) string { + var workspaceSelector string + if o.WorkspaceName != "" { + workspaceSelector = fmt.Sprintf(`label_kubesphere_io_workspace="%s"`, o.WorkspaceName) + } else { + workspaceSelector = fmt.Sprintf(`label_kubesphere_io_workspace=~"%s", label_kubesphere_io_workspace!=""`, o.ResourceFilter) + } + return strings.Replace(tmpl, "$1", workspaceSelector, -1) +} + +func makeNamespaceMetricExpression(tmpl string, o monitoring.QueryOptions) string { + var namespaceSelector string + + // For monitoring namespaces in the specific workspace + // GET /workspaces/{workspace}/namespaces + if o.WorkspaceName != "" { + namespaceSelector = fmt.Sprintf(`label_kubesphere_io_workspace="%s", namespace=~"%s"`, o.WorkspaceName, o.ResourceFilter) + return strings.Replace(tmpl, "$1", namespaceSelector, -1) + } + + // For monitoring the specific namespaces + // GET /namespaces/{namespace} or + // GET /namespaces + if o.NamespaceName != "" { + namespaceSelector = fmt.Sprintf(`namespace="%s"`, o.NamespaceName) + } else { + namespaceSelector = fmt.Sprintf(`namespace=~"%s"`, o.ResourceFilter) + } + return strings.Replace(tmpl, "$1", namespaceSelector, -1) +} + +func makeWorkloadMetricExpression(tmpl string, o monitoring.QueryOptions) string { + var kindSelector, workloadSelector string + switch o.WorkloadKind { + case "deployment": + o.WorkloadKind = Deployment + kindSelector = fmt.Sprintf(`namespace="%s", deployment!="", deployment=~"%s"`, o.NamespaceName, o.ResourceFilter) + case "statefulset": + o.WorkloadKind = StatefulSet + kindSelector = fmt.Sprintf(`namespace="%s", statefulset!="", statefulset=~"%s"`, o.NamespaceName, o.ResourceFilter) + case "daemonset": + o.WorkloadKind = DaemonSet + kindSelector = fmt.Sprintf(`namespace="%s", daemonset!="", daemonset=~"%s"`, o.NamespaceName, o.ResourceFilter) + default: + o.WorkloadKind = ".*" + kindSelector = fmt.Sprintf(`namespace="%s"`, o.NamespaceName) + } + workloadSelector = fmt.Sprintf(`namespace="%s", workload=~"%s:%s"`, o.NamespaceName, o.WorkloadKind, o.ResourceFilter) + return strings.NewReplacer("$1", workloadSelector, "$2", kindSelector).Replace(tmpl) +} + +func makePodMetricExpression(tmpl string, o monitoring.QueryOptions) string { + var podSelector, workloadSelector string + + // For monitoriong pods of the specific workload + // GET /namespaces/{namespace}/workloads/{kind}/{workload}/pods + if o.WorkloadName != "" { + switch o.WorkloadKind { + case "deployment": + workloadSelector = fmt.Sprintf(`owner_kind="ReplicaSet", owner_name=~"^%s-[^-]{1,10}$"`, o.WorkloadKind) + case "statefulset": + workloadSelector = fmt.Sprintf(`owner_kind="StatefulSet", owner_name="%s"`, o.WorkloadKind) + case "daemonset": + workloadSelector = fmt.Sprintf(`owner_kind="DaemonSet", owner_name="%s"`, o.WorkloadKind) + } + } + + // For monitoring pods in the specific namespace + // GET /namespaces/{namespace}/workloads/{kind}/{workload}/pods or + // GET /namespaces/{namespace}/pods/{pod} or + // GET /namespaces/{namespace}/pods + if o.NamespaceName != "" { + if o.PodName != "" { + podSelector = fmt.Sprintf(`pod="%s", namespace="%s"`, o.PodName, o.NamespaceName) + } else { + podSelector = fmt.Sprintf(`pod=~"%s", namespace="%s"`, o.ResourceFilter, o.NamespaceName) + } + } + + // For monitoring pods on the specific node + // GET /nodes/{node}/pods/{pod} + if o.PodName != "" { + if o.PodName != "" { + podSelector = fmt.Sprintf(`pod="%s", node="%s"`, o.PodName, o.NodeName) + } else { + podSelector = fmt.Sprintf(`pod=~"%s", node="%s"`, o.ResourceFilter, o.NodeName) + } + } + return strings.NewReplacer("$1", workloadSelector, "$2", podSelector).Replace(tmpl) +} + +func makeContainerMetricExpression(tmpl string, o monitoring.QueryOptions) string { + var containerSelector string + if o.ContainerName != "" { + containerSelector = fmt.Sprintf(`pod_name="%s", namespace="%s", container_name="%s"`, o.PodName, o.NamespaceName, o.ContainerName) + } else { + containerSelector = fmt.Sprintf(`pod_name="%s", namespace="%s", container_name=~"%s"`, o.PodName, o.NamespaceName, o.ResourceFilter) + } + return strings.Replace(tmpl, "$1", containerSelector, -1) +} + +func makePVCMetricExpression(tmpl string, o monitoring.QueryOptions) string { + var pvcSelector string + + // For monitoring persistentvolumeclaims in the specific namespace + // GET /namespaces/{namespace}/persistentvolumeclaims/{persistentvolumeclaim} or + // GET /namespaces/{namespace}/persistentvolumeclaims + if o.NamespaceName != "" { + if o.PersistentVolumeClaimName != "" { + pvcSelector = fmt.Sprintf(`namespace="%s", persistentvolumeclaim="%s"`, o.NamespaceName, o.PersistentVolumeClaimName) + } else { + pvcSelector = fmt.Sprintf(`namespace="%s", persistentvolumeclaim=~"%s"`, o.NamespaceName, o.ResourceFilter) + } + return strings.Replace(tmpl, "$1", pvcSelector, -1) + } + + // For monitoring persistentvolumeclaims of the specific storageclass + // GET /storageclasses/{storageclass}/persistentvolumeclaims + if o.StorageClassName != "" { + pvcSelector = fmt.Sprintf(`storageclass="%s", persistentvolumeclaim=~"%s"`, o.StorageClassName, o.ResourceFilter) + } + return strings.Replace(tmpl, "$1", pvcSelector, -1) +} diff --git a/pkg/simple/client/monitoring/prometheus_options.go b/pkg/simple/client/monitoring/prometheus_options.go deleted file mode 100644 index 0aed413c9..000000000 --- a/pkg/simple/client/monitoring/prometheus_options.go +++ /dev/null @@ -1,41 +0,0 @@ -package monitoring - -import ( - "github.com/spf13/pflag" -) - -type Options struct { - Endpoint string `json:"endpoint,omitempty" yaml:"endpoint"` - SecondaryEndpoint string `json:"secondaryEndpoint,omitempty" yaml:"secondaryEndpoint"` -} - -func NewPrometheusOptions() *Options { - return &Options{ - Endpoint: "", - SecondaryEndpoint: "", - } -} - -func (s *Options) Validate() []error { - var errs []error - return errs -} - -func (s *Options) ApplyTo(options *Options) { - if s.Endpoint != "" { - options.Endpoint = s.Endpoint - } - - if s.SecondaryEndpoint != "" { - options.SecondaryEndpoint = s.SecondaryEndpoint - } -} - -func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) { - fs.StringVar(&s.Endpoint, "prometheus-endpoint", c.Endpoint, ""+ - "Prometheus service endpoint which stores KubeSphere monitoring data, if left "+ - "blank, will use builtin metrics-server as data source.") - - fs.StringVar(&s.SecondaryEndpoint, "prometheus-secondary-endpoint", c.SecondaryEndpoint, ""+ - "Prometheus secondary service endpoint, if left empty and endpoint is set, will use endpoint instead.") -} diff --git a/pkg/simple/client/monitoring/prometheus_test.go b/pkg/simple/client/monitoring/prometheus_test.go deleted file mode 100644 index 2f2912ae1..000000000 --- a/pkg/simple/client/monitoring/prometheus_test.go +++ /dev/null @@ -1 +0,0 @@ -package monitoring diff --git a/pkg/simple/client/monitoring/query_options.go b/pkg/simple/client/monitoring/query_options.go new file mode 100644 index 000000000..4acb05f51 --- /dev/null +++ b/pkg/simple/client/monitoring/query_options.go @@ -0,0 +1,162 @@ +package monitoring + +type QueryOption interface { + Apply(*QueryOptions) +} + +type QueryOptions struct { + Level MonitoringLevel + NamedMetrics []string + + MetricFilter string + ResourceFilter string + NodeName string + WorkspaceName string + NamespaceName string + WorkloadKind string + WorkloadName string + PodName string + ContainerName string + StorageClassName string + PersistentVolumeClaimName string +} + +func NewQueryOptions() *QueryOptions { + return &QueryOptions{} +} + +type ClusterOption struct { + MetricFilter string +} + +func (co ClusterOption) Apply(o *QueryOptions) { + o.Level = LevelCluster + o.NamedMetrics = ClusterMetrics +} + +type NodeOption struct { + MetricFilter string + ResourceFilter string + NodeName string +} + +func (no NodeOption) Apply(o *QueryOptions) { + o.Level = LevelNode + o.NamedMetrics = NodeMetrics + o.ResourceFilter = no.ResourceFilter + o.NodeName = no.NodeName +} + +type WorkspaceOption struct { + MetricFilter string + ResourceFilter string + WorkspaceName string +} + +func (wo WorkspaceOption) Apply(o *QueryOptions) { + o.Level = LevelWorkspace + o.NamedMetrics = WorkspaceMetrics + o.MetricFilter = wo.MetricFilter + o.ResourceFilter = wo.ResourceFilter + o.WorkspaceName = wo.WorkspaceName +} + +type NamespaceOption struct { + MetricFilter string + ResourceFilter string + WorkspaceName string + NamespaceName string +} + +func (no NamespaceOption) Apply(o *QueryOptions) { + o.Level = LevelNamespace + o.NamedMetrics = NamespaceMetrics + o.MetricFilter = no.MetricFilter + o.ResourceFilter = no.ResourceFilter + o.WorkspaceName = no.WorkspaceName + o.NamespaceName = no.NamespaceName +} + +type WorkloadOption struct { + MetricFilter string + ResourceFilter string + NamespaceName string + WorkloadKind string + WorkloadName string +} + +func (wo WorkloadOption) Apply(o *QueryOptions) { + o.Level = LevelWorkload + o.NamedMetrics = WorkspaceMetrics + o.MetricFilter = wo.MetricFilter + o.ResourceFilter = wo.ResourceFilter + o.NamespaceName = wo.NamespaceName + o.WorkloadKind = wo.WorkloadKind + o.WorkloadName = wo.WorkloadName +} + +type PodOption struct { + MetricFilter string + ResourceFilter string + NodeName string + NamespaceName string + WorkloadKind string + WorkloadName string + PodName string +} + +func (po PodOption) Apply(o *QueryOptions) { + o.Level = LevelPod + o.NamedMetrics = PodMetrics + o.MetricFilter = po.MetricFilter + o.ResourceFilter = po.ResourceFilter + o.NamespaceName = po.NamespaceName + o.WorkloadKind = po.WorkloadKind + o.WorkloadName = po.WorkloadName +} + +type ContainerOption struct { + MetricFilter string + ResourceFilter string + NamespaceName string + PodName string + ContainerName string +} + +func (co ContainerOption) Apply(o *QueryOptions) { + o.Level = LevelContainer + o.NamedMetrics = ContainerMetrics + o.MetricFilter = co.MetricFilter + o.ResourceFilter = co.ResourceFilter + o.NamespaceName = co.NamespaceName + o.PodName = co.PodName + o.ContainerName = co.ContainerName +} + +type PVCOption struct { + MetricFilter string + ResourceFilter string + NamespaceName string + StorageClassName string + PersistentVolumeClaimName string +} + +func (po PVCOption) Apply(o *QueryOptions) { + o.Level = LevelPVC + o.NamedMetrics = PVCMetrics + o.MetricFilter = po.MetricFilter + o.ResourceFilter = po.ResourceFilter + o.NamespaceName = po.NamespaceName + o.StorageClassName = po.StorageClassName + o.PersistentVolumeClaimName = po.PersistentVolumeClaimName +} + +type ComponentOption struct { + MetricFilter string +} + +func (co ComponentOption) Apply(o *QueryOptions) { + o.Level = LevelComponent + o.NamedMetrics = ComponentMetrics + o.MetricFilter = co.MetricFilter +} diff --git a/pkg/simple/client/prometheus/prometheus.go b/pkg/simple/client/prometheus/prometheus.go deleted file mode 100644 index a24cb1607..000000000 --- a/pkg/simple/client/prometheus/prometheus.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package prometheus - -import ( - "fmt" - jsoniter "github.com/json-iterator/go" - "io/ioutil" - "k8s.io/klog" - "kubesphere.io/kubesphere/pkg/api/monitoring/v1alpha2" - "net/http" - "time" -) - -type Client struct { - client *http.Client - endpoint string - secondaryEndpoint string -} - -func NewPrometheusClient(options *Options) (*Client, error) { - return &Client{ - client: &http.Client{ - Timeout: 10 * time.Second, - }, - endpoint: options.Endpoint, - secondaryEndpoint: options.SecondaryEndpoint, - }, nil -} - -func (c *Client) QueryToK8SPrometheus(queryType string, params string) (apiResponse v1alpha2.APIResponse) { - return c.query(c.endpoint, queryType, params) -} - -func (c *Client) QueryToK8SSystemPrometheus(queryType string, params string) (apiResponse v1alpha2.APIResponse) { - return c.query(c.secondaryEndpoint, queryType, params) -} - -var jsonIter = jsoniter.ConfigCompatibleWithStandardLibrary - -func (c *Client) query(endpoint string, queryType string, params string) (apiResponse v1alpha2.APIResponse) { - url := fmt.Sprintf("%s/api/v1/%s?%s", endpoint, queryType, params) - - response, err := c.client.Get(url) - if err != nil { - klog.Error(err) - apiResponse.Status = "error" - return apiResponse - } - defer response.Body.Close() - - body, err := ioutil.ReadAll(response.Body) - if err != nil { - klog.Error(err) - apiResponse.Status = "error" - return apiResponse - } - - err = jsonIter.Unmarshal(body, &apiResponse) - if err != nil { - klog.Errorf("fail to unmarshal prometheus query result: %s", err.Error()) - apiResponse.Status = "error" - return apiResponse - } - - return apiResponse -}