diff --git a/pkg/api/metering/v1alpha1/types.go b/pkg/api/metering/v1alpha1/types.go new file mode 100644 index 000000000..dffb67c97 --- /dev/null +++ b/pkg/api/metering/v1alpha1/types.go @@ -0,0 +1,89 @@ +package v1alpha1 + +import ( + "time" + + "github.com/emicklei/go-restful" + "kubesphere.io/kubesphere/pkg/apiserver/query" + model "kubesphere.io/kubesphere/pkg/models/monitoring" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" +) + +const ( + DefaultStep = 10 * time.Minute + DefaultFilter = ".*" + DefaultOrder = model.OrderDescending + DefaultPage = 1 + DefaultLimit = 5 + + ErrNoHit = "'end' or 'time' must be after the namespace creation time." + ErrParamConflict = "'time' and the combination of 'start' and 'end' are mutually exclusive." + ErrInvalidStartEnd = "'start' must be before 'end'." + ErrInvalidPage = "Invalid parameter 'page'." + ErrInvalidLimit = "Invalid parameter 'limit'." + ErrParameterNotfound = "Parmameter [%s] not found" + ErrResourceNotfound = "resource not found" + ErrScopeNotAllowed = "scope [%s] not allowed" +) + +type Query struct { + Level monitoring.Level + Operation string + LabelSelector string + Time string + Start string + End string + Step string + Target string + Order string + Page string + Limit string + MetricFilter string + ResourceFilter string + NodeName string + WorkspaceName string + NamespaceName string + WorkloadKind string + WorkloadName string + PodName string + Applications string + Services string + StorageClassName string + PVCFilter string +} + +func ParseQueryParameter(req *restful.Request) *Query { + var q Query + + q.LabelSelector = req.QueryParameter(query.ParameterLabelSelector) + + q.Level = monitoring.Level(monitoring.MeteringLevelMap[req.QueryParameter("level")]) + q.Operation = req.QueryParameter("operation") + q.Time = req.QueryParameter("time") + q.Start = req.QueryParameter("start") + q.End = req.QueryParameter("end") + q.Step = req.QueryParameter("step") + q.Target = req.QueryParameter("sort_metric") + q.Order = req.QueryParameter("sort_type") + q.Page = req.QueryParameter("page") + q.Limit = req.QueryParameter("limit") + q.MetricFilter = req.QueryParameter("metrics_filter") + q.ResourceFilter = req.QueryParameter("resources_filter") + q.WorkspaceName = req.QueryParameter("workspace") + + q.NamespaceName = req.QueryParameter("namespace") + if q.NamespaceName == "" { + q.NamespaceName = req.PathParameter("namespace") + } + + q.NodeName = req.QueryParameter("node") + q.WorkloadKind = req.QueryParameter("kind") + q.WorkloadName = req.QueryParameter("workload") + q.PodName = req.QueryParameter("pod") + q.Applications = req.QueryParameter("applications") + q.Services = req.QueryParameter("services") + q.StorageClassName = req.QueryParameter("storageclass") + q.PVCFilter = req.QueryParameter("pvc_filter") + + return &q +} diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 5f4aaf33c..da8879a74 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -61,6 +61,7 @@ import ( devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2" devopsv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha3" iamapi "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2" + meteringv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/metering/v1alpha1" monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3" networkv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/network/v1alpha2" notificationv1 "kubesphere.io/kubesphere/pkg/kapis/notification/v1" @@ -215,12 +216,13 @@ func (s *APIServer) installKubeSphereAPIs() { urlruntime.Must(configv1alpha2.AddToContainer(s.container, s.Config)) urlruntime.Must(resourcev1alpha3.AddToContainer(s.container, s.InformerFactory, s.RuntimeCache)) urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.MetricsClient, s.InformerFactory, s.OpenpitrixClient)) + urlruntime.Must(meteringv1alpha1.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.InformerFactory, s.OpenpitrixClient, s.RuntimeCache)) urlruntime.Must(openpitrixv1.AddToContainer(s.container, s.InformerFactory, s.OpenpitrixClient)) urlruntime.Must(operationsv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes())) urlruntime.Must(resourcesv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.InformerFactory, s.KubernetesClient.Master())) urlruntime.Must(tenantv1alpha2.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(), - s.KubernetesClient.KubeSphere(), s.EventsClient, s.LoggingClient, s.AuditingClient, amOperator, rbacAuthorizer)) + s.KubernetesClient.KubeSphere(), s.EventsClient, s.LoggingClient, s.AuditingClient, amOperator, rbacAuthorizer, s.MonitoringClient, s.OpenpitrixClient, s.RuntimeCache)) urlruntime.Must(terminalv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.KubernetesClient.Config())) urlruntime.Must(clusterkapisv1alpha1.AddToContainer(s.container, s.InformerFactory.KubernetesSharedInformerFactory(), diff --git a/pkg/apiserver/request/requestinfo.go b/pkg/apiserver/request/requestinfo.go index b07e78326..77f3da048 100644 --- a/pkg/apiserver/request/requestinfo.go +++ b/pkg/apiserver/request/requestinfo.go @@ -31,12 +31,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" + k8srequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/klog" "kubesphere.io/kubesphere/pkg/api" "kubesphere.io/kubesphere/pkg/constants" netutils "kubesphere.io/kubesphere/pkg/utils/net" - - k8srequest "k8s.io/apiserver/pkg/endpoints/request" ) type RequestInfoResolver interface { diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index e759ee989..ea1dd471d 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -105,7 +105,20 @@ const ( EventsQueryTag = "Events Query" AuditingQueryTag = "Auditing Query" - AlertingTag = "Alerting" + ClusterMetersTag = "Cluster Meters" + NodeMetersTag = "Node Meters" + WorkspaceMetersTag = "Workspace Meters" + NamespaceMetersTag = "Namespace Meters" + WorkloadMetersTag = "Workload Meters" + PodMetersTag = "Pod Meters" + ServiceMetricsTag = "ServiceName Meters" + + ApplicationReleaseName = "meta.helm.sh/release-name" + ApplicationReleaseNS = "meta.helm.sh/release-namespace" + + ApplicationName = "app.kubernetes.io/name" + ApplicationVersion = "app.kubernetes.io/version" + AlertingTag = "Alerting" ) var ( diff --git a/pkg/kapis/metering/group.go b/pkg/kapis/metering/group.go new file mode 100644 index 000000000..8abb1bc95 --- /dev/null +++ b/pkg/kapis/metering/group.go @@ -0,0 +1,18 @@ +/* +Copyright 2019 The KubeSphere authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package metering contains monitoring API versions +package metering diff --git a/pkg/kapis/metering/v1alpha1/handler.go b/pkg/kapis/metering/v1alpha1/handler.go new file mode 100644 index 000000000..da722a97e --- /dev/null +++ b/pkg/kapis/metering/v1alpha1/handler.go @@ -0,0 +1,45 @@ +/* + + Copyright 2019 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +package v1alpha1 + +import ( + "github.com/emicklei/go-restful" + "k8s.io/client-go/kubernetes" + "kubesphere.io/kubesphere/pkg/informers" + monitorhle "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3" + resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" +) + +type meterHandler interface { + HandleClusterMetersQuery(req *restful.Request, resp *restful.Response) + HandleNodeMetersQuery(req *restful.Request, resp *restful.Response) + HandleWorkspaceMetersQuery(req *restful.Request, resp *restful.Response) + HandleNamespaceMetersQuery(re *restful.Request, resp *restful.Response) + HandleWorkloadMetersQuery(req *restful.Request, resp *restful.Response) + HandleApplicationMetersQuery(req *restful.Request, resp *restful.Response) + HandlePodMetersQuery(req *restful.Request, resp *restful.Response) + HandleServiceMetersQuery(req *restful.Request, resp *restful.Response) + HandlePVCMetersQuery(req *restful.Request, resp *restful.Response) +} + +func newHandler(k kubernetes.Interface, m monitoring.Interface, f informers.InformerFactory, o openpitrix.Client, resourceGetter *resourcev1alpha3.ResourceGetter) meterHandler { + return monitorhle.NewHandler(k, m, nil, f, o, resourceGetter) +} diff --git a/pkg/kapis/metering/v1alpha1/register.go b/pkg/kapis/metering/v1alpha1/register.go new file mode 100644 index 000000000..fb00da15e --- /dev/null +++ b/pkg/kapis/metering/v1alpha1/register.go @@ -0,0 +1,357 @@ +/* + + Copyright 2019 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ +package v1alpha1 + +import ( + "net/http" + + "github.com/emicklei/go-restful" + "github.com/emicklei/go-restful-openapi" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + "kubesphere.io/kubesphere/pkg/apiserver/runtime" + "kubesphere.io/kubesphere/pkg/constants" + "kubesphere.io/kubesphere/pkg/informers" + monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3" + model "kubesphere.io/kubesphere/pkg/models/monitoring" + resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" + "sigs.k8s.io/controller-runtime/pkg/cache" +) + +const ( + groupName = "metering.kubesphere.io" + respOK = "ok" +) + +var GroupVersion = schema.GroupVersion{Group: groupName, Version: "v1alpha1"} + +func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, meteringClient monitoring.Interface, factory informers.InformerFactory, opClient openpitrix.Client, cache cache.Cache) error { + ws := runtime.NewWebService(GroupVersion) + + h := newHandler(k8sClient, meteringClient, factory, opClient, resourcev1alpha3.NewResourceGetter(factory, cache)) + + ws.Route(ws.GET("/cluster"). + To(h.HandleClusterMetersQuery). + Doc("Get cluster-level meter data."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which meter data to return. For example, the following filter matches both cluster CPU usage and disk usage: `meter_cluster_cpu_usage|meter_cluster_memory_usage`.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.ClusterMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/nodes"). + To(h.HandleNodeMetersQuery). + Doc("Get node-level meter data of all nodes."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which meter data to return. For example, the following filter matches both node CPU usage and disk usage: `meter_node_cpu_usage|meter_node_memory_usage`.").DataType("string").Required(false)). + Param(ws.QueryParameter("resources_filter", "The node filter consists of a regexp pattern. It specifies which node data to return. For example, the following filter matches both node i-caojnter and i-cmu82ogj: `i-caojnter|i-cmu82ogj`.").DataType("string").Required(false)). + Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)). + Param(ws.QueryParameter("pvc_filter", "The PVCs filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_metric", "Sort nodes by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). + Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.NodeMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/nodes/{node}"). + To(h.HandleNodeMetersQuery). + Doc("Get node-level meter data of the specific node."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.PathParameter("node", "Node name.").DataType("string").Required(true)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which meter data to return. For example, the following filter matches both node CPU usage and disk usage: `meter_node_cpu_usage|meter_node_memory_usage`.").DataType("string").Required(false)). + Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)). + Param(ws.QueryParameter("pvc_filter", "The PVCs filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.NodeMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/workspaces"). + To(h.HandleWorkspaceMetersQuery). + Doc("Get workspace-level meter data of all workspaces."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workspace CPU usage and memory usage: `meter_workspace_cpu_usage|meter_workspace_memory_usage`.").DataType("string").Required(false)). + Param(ws.QueryParameter("resources_filter", "The workspace filter consists of a regexp pattern. It specifies which workspace data to return.").DataType("string").Required(false)). + Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)). + Param(ws.QueryParameter("pvc_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_metric", "Sort workspaces by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). + Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkspaceMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/workspaces/{workspace}"). + To(h.HandleWorkspaceMetersQuery). + Doc("Get workspace-level meter data of a specific workspace."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.PathParameter("workspace", "Workspace name.").DataType("string").Required(true)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workspace CPU usage and memory usage: `meter_workspace_cpu_usage|meter_workspace_memory_usage`.").DataType("string").Required(false)). + Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)). + Param(ws.QueryParameter("pvc_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("type", "Additional operations. Currently available types is statistics. It retrieves the total number of namespaces, devops projects, members and roles in this workspace at the moment.").DataType("string").Required(false)). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkspaceMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/workspaces/{workspace}/namespaces"). + To(h.HandleNamespaceMetersQuery). + Doc("Get namespace-level meter data of a specific workspace."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.PathParameter("workspace", "Workspace name.").DataType("string").Required(true)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both namespace CPU usage and memory usage: `meter_namespace_cpu_usage|meter_namespace_memory_usage_wo_cache`.").DataType("string").Required(false)). + Param(ws.QueryParameter("resources_filter", "The namespace filter consists of a regexp pattern. It specifies which namespace data to return. For example, the following filter matches both namespace test and kube-system: `test|kube-system`.").DataType("string").Required(false)). + Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)). + Param(ws.QueryParameter("pvc_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_metric", "Sort namespaces by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). + Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/namespaces"). + To(h.HandleNamespaceMetersQuery). + Doc("Get namespace-level meter data of all namespaces."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both namespace CPU usage and memory usage: `meter_namespace_cpu_usage|meter_namespace_memory_usage_wo_cache`.").DataType("string").Required(false)). + Param(ws.QueryParameter("resources_filter", "The namespace filter consists of a regexp pattern. It specifies which namespace data to return. For example, the following filter matches both namespace test and kube-system: `test|kube-system`.").DataType("string").Required(false)). + Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)). + Param(ws.QueryParameter("pvc_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_metric", "Sort namespaces by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). + Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/namespaces/{namespace}"). + To(h.HandleNamespaceMetersQuery). + Doc("Get namespace-level meter data of the specific namespace."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both namespace CPU usage and memory usage: `meter_namespace_cpu_usage|meter_namespace_memory_usage_wo_cache`.").DataType("string").Required(false)). + Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)). + Param(ws.QueryParameter("pvc_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/namespaces/{namespace}/workloads"). + To(h.HandleWorkloadMetersQuery). + Doc("Get workload-level meter data of all workloads which belongs to a specific kind."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). + Param(ws.QueryParameter("kind", "Workload kind. One of deployment, daemonset, statefulset.").DataType("string").Required(true)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workload CPU usage and memory usage: `meter_workload_cpu_usage|meter_workload_memory_usage_wo_cache`.").DataType("string").Required(false)). + Param(ws.QueryParameter("resources_filter", "The workload filter consists of a regexp pattern. It specifies which workload data to return. For example, the following filter matches any workload whose name begins with prometheus: `prometheus.*`.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_metric", "Sort workloads by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). + Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkloadMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/namespaces/{namespace}/applications"). + To(h.HandleApplicationMetersQuery). + Doc("Get app-level meter data of a specific application. Navigate to the app by the app's namespace."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). + Param(ws.QueryParameter("applications", "Appliction names, format app_name[:app_version](such as nginx:v1, nignx) which are joined by \"|\" ").DataType("string").Required(false)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_application_cpu_usage|meter_application_memory_usage_wo_cache`.").DataType("string").Required(false)). + Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_metric", "Sort pods by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/namespaces/{namespace}/pods"). + To(h.HandlePodMetersQuery). + Doc("Get pod-level meter data of the specific namespace's pods."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_pod_cpu_usage|meter_pod_memory_usage`.").DataType("string").Required(false)). + Param(ws.QueryParameter("resources_filter", "The pod filter consists of a regexp pattern. It specifies which pod data to return. For example, the following filter matches any pod whose name begins with redis: `redis.*`.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_metric", "Sort pods by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). + Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}"). + To(h.HandlePodMetersQuery). + Doc("Get pod-level meter data of a specific pod. Navigate to the pod by the pod's namespace."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). + Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_pod_cpu_usage|_meter_pod_memory_usage`.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/namespaces/{namespace}/workloads/{workload}/pods"). + To(h.HandlePodMetersQuery). + Doc("Get pod-level meter data of a specific workload's pods. Navigate to the workload by the namespace."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). + Param(ws.PathParameter("workload", "Workload name.").DataType("string").Required(true)). + Param(ws.QueryParameter("kind", "Workload kind. One of deployment, daemonset, statefulset.").DataType("string").Required(true)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_pod_cpu_usage|meter_pod_memory_usage`.").DataType("string").Required(false)). + Param(ws.QueryParameter("resources_filter", "The pod filter consists of a regexp pattern. It specifies which pod data to return. For example, the following filter matches any pod whose name begins with redis: `redis.*`.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_metric", "Sort pods by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). + Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/nodes/{node}/pods"). + To(h.HandlePodMetersQuery). + Doc("Get pod-level meter data of all pods on a specific node."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.PathParameter("node", "Node name.").DataType("string").Required(true)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_pod_cpu_usage|meter_pod_memory_usage`.").DataType("string").Required(false)). + Param(ws.QueryParameter("resources_filter", "The pod filter consists of a regexp pattern. It specifies which pod data to return. For example, the following filter matches any pod whose name begins with redis: `redis.*`.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_metric", "Sort pods by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). + Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/nodes/{node}/pods/{pod}"). + To(h.HandlePodMetersQuery). + Doc("Get pod-level meter data of a specific pod. Navigate to the pod by the node where it is scheduled."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.PathParameter("node", "Node name.").DataType("string").Required(true)). + Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_pod_cpu_usage|meter_pod_memory_usage`.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetersTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/namespaces/{namespace}/services"). + To(h.HandleServiceMetersQuery). + Doc("Get service-level meter data of the specific namespace's pods."). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_pod_cpu_usage|meter_pod_memory_usage`.").DataType("string").Required(false)). + Param(ws.QueryParameter("services", "Services which are joined by \"|\".").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_metric", "Sort pods by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). + Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.ServiceMetricsTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + c.Add(ws) + return nil +} diff --git a/pkg/kapis/monitoring/v1alpha3/handler.go b/pkg/kapis/monitoring/v1alpha3/handler.go index 4dd53a15d..191ba1bb2 100644 --- a/pkg/kapis/monitoring/v1alpha3/handler.go +++ b/pkg/kapis/monitoring/v1alpha3/handler.go @@ -20,14 +20,17 @@ package v1alpha3 import ( "errors" + "regexp" + "strings" + "github.com/emicklei/go-restful" "k8s.io/client-go/kubernetes" "kubesphere.io/kubesphere/pkg/api" "kubesphere.io/kubesphere/pkg/informers" model "kubesphere.io/kubesphere/pkg/models/monitoring" + resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource" "kubesphere.io/kubesphere/pkg/simple/client/monitoring" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" - "regexp" ) type handler struct { @@ -35,8 +38,8 @@ type handler struct { mo model.MonitoringOperator } -func newHandler(k kubernetes.Interface, monitoringClient monitoring.Interface, metricsClient monitoring.Interface, f informers.InformerFactory, o openpitrix.Client) *handler { - return &handler{k, model.NewMonitoringOperator(monitoringClient, metricsClient, k, f, o)} +func NewHandler(k kubernetes.Interface, monitoringClient monitoring.Interface, metricsClient monitoring.Interface, f informers.InformerFactory, o openpitrix.Client, resourceGetter *resourcev1alpha3.ResourceGetter) *handler { + return &handler{k, model.NewMonitoringOperator(monitoringClient, metricsClient, k, f, o, resourceGetter)} } func (h handler) handleKubeSphereMetricsQuery(req *restful.Request, resp *restful.Response) { @@ -186,6 +189,10 @@ func (h handler) handleNamedMetricsQuery(resp *restful.Response, q queryOptions) var metrics []string for _, metric := range q.namedMetrics { + if strings.HasPrefix(metric, model.MetricMeterPrefix) { + // skip meter metric + continue + } ok, _ := regexp.MatchString(q.metricFilter, metric) if ok { metrics = append(metrics, metric) diff --git a/pkg/kapis/monitoring/v1alpha3/helper.go b/pkg/kapis/monitoring/v1alpha3/helper.go index 69d6b28a0..7fee0477a 100644 --- a/pkg/kapis/monitoring/v1alpha3/helper.go +++ b/pkg/kapis/monitoring/v1alpha3/helper.go @@ -17,14 +17,21 @@ limitations under the License. package v1alpha3 import ( + "bytes" "context" + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + "time" + "github.com/emicklei/go-restful" "github.com/pkg/errors" corev1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "kubesphere.io/kubesphere/pkg/api" model "kubesphere.io/kubesphere/pkg/models/monitoring" "kubesphere.io/kubesphere/pkg/simple/client/monitoring" - "strconv" - "time" ) const ( @@ -34,47 +41,56 @@ const ( DefaultPage = 1 DefaultLimit = 5 + OperationQuery = "query" + OperationExport = "export" + ComponentEtcd = "etcd" ComponentAPIServer = "apiserver" ComponentScheduler = "scheduler" - ErrNoHit = "'end' or 'time' must be after the namespace creation time." - ErrParamConflict = "'time' and the combination of 'start' and 'end' are mutually exclusive." - ErrInvalidStartEnd = "'start' must be before 'end'." - ErrInvalidPage = "Invalid parameter 'page'." - ErrInvalidLimit = "Invalid parameter 'limit'." + ErrNoHit = "'end' or 'time' must be after the namespace creation time." + ErrParamConflict = "'time' and the combination of 'start' and 'end' are mutually exclusive." + ErrInvalidStartEnd = "'start' must be before 'end'." + ErrInvalidPage = "Invalid parameter 'page'." + ErrInvalidLimit = "Invalid parameter 'limit'." + ErrParameterNotfound = "Parmameter [%s] not found" ) type reqParams struct { - time string - start string - end string - step string - target string - order string - page string - limit string - metricFilter string - namespacedResourcesFilter string - resourceFilter string - nodeName string - workspaceName string - namespaceName string - workloadKind string - workloadName string - podName string - containerName string - pvcName string - storageClassName string - componentType string - expression string - metric string + operation string + time string + start string + end string + step string + target string + order string + page string + limit string + metricFilter string + resourceFilter string + nodeName string + workspaceName string + namespaceName string + workloadKind string + workloadName string + podName string + containerName string + pvcName string + storageClassName string + componentType string + expression string + metric string + applications string + services string + pvcFilter string } type queryOptions struct { metricFilter string namedMetrics []string + Operation string + start time.Time end time.Time time time.Time @@ -99,6 +115,7 @@ func (q queryOptions) shouldSort() bool { func parseRequestParams(req *restful.Request) reqParams { var r reqParams + r.operation = req.QueryParameter("operation") r.time = req.QueryParameter("time") r.start = req.QueryParameter("start") r.end = req.QueryParameter("end") @@ -108,12 +125,24 @@ func parseRequestParams(req *restful.Request) reqParams { r.page = req.QueryParameter("page") r.limit = req.QueryParameter("limit") r.metricFilter = req.QueryParameter("metrics_filter") - r.namespacedResourcesFilter = req.QueryParameter("namespaced_resources_filter") r.resourceFilter = req.QueryParameter("resources_filter") - r.nodeName = req.PathParameter("node") r.workspaceName = req.PathParameter("workspace") r.namespaceName = req.PathParameter("namespace") - r.workloadKind = req.PathParameter("kind") + + if req.QueryParameter("node") != "" { + r.nodeName = req.QueryParameter("node") + } else { + // compatible with monitoring request + r.nodeName = req.PathParameter("node") + } + + if req.QueryParameter("kind") != "" { + r.workloadKind = req.QueryParameter("kind") + } else { + // compatible with monitoring request + r.workloadKind = req.PathParameter("kind") + } + r.workloadName = req.PathParameter("workload") r.podName = req.PathParameter("pod") r.containerName = req.PathParameter("container") @@ -122,6 +151,10 @@ func parseRequestParams(req *restful.Request) reqParams { r.componentType = req.PathParameter("component") r.expression = req.QueryParameter("expr") r.metric = req.QueryParameter("metric") + r.applications = req.QueryParameter("applications") + r.services = req.QueryParameter("services") + r.pvcFilter = req.QueryParameter("pvc_filter") + return r } @@ -135,70 +168,109 @@ func (h handler) makeQueryOptions(r reqParams, lvl monitoring.Level) (q queryOpt q.metricFilter = DefaultFilter } + q.Operation = r.operation + if r.operation == "" { + q.Operation = OperationQuery + } + switch lvl { case monitoring.LevelCluster: q.option = monitoring.ClusterOption{} q.namedMetrics = model.ClusterMetrics + case monitoring.LevelNode: q.identifier = model.IdentifierNode - q.namedMetrics = model.NodeMetrics q.option = monitoring.NodeOption{ - ResourceFilter: r.resourceFilter, - NodeName: r.nodeName, + ResourceFilter: r.resourceFilter, + NodeName: r.nodeName, + PVCFilter: r.pvcFilter, // metering pvc + StorageClassName: r.storageClassName, // metering pvc } + q.namedMetrics = model.NodeMetrics + case monitoring.LevelWorkspace: q.identifier = model.IdentifierWorkspace - q.namedMetrics = model.WorkspaceMetrics q.option = monitoring.WorkspaceOption{ - ResourceFilter: r.resourceFilter, - WorkspaceName: r.workspaceName, + ResourceFilter: r.resourceFilter, + WorkspaceName: r.workspaceName, + PVCFilter: r.pvcFilter, // metering pvc + StorageClassName: r.storageClassName, // metering pvc } + q.namedMetrics = model.WorkspaceMetrics + case monitoring.LevelNamespace: q.identifier = model.IdentifierNamespace - q.namedMetrics = model.NamespaceMetrics q.option = monitoring.NamespaceOption{ - ResourceFilter: r.resourceFilter, - WorkspaceName: r.workspaceName, - NamespaceName: r.namespaceName, + ResourceFilter: r.resourceFilter, + WorkspaceName: r.workspaceName, + NamespaceName: r.namespaceName, + PVCFilter: r.pvcFilter, // metering pvc + StorageClassName: r.storageClassName, // metering pvc } + q.namedMetrics = model.NamespaceMetrics + + case monitoring.LevelApplication: + q.identifier = model.IdentifierApplication + if r.namespaceName == "" { + return q, errors.New(fmt.Sprintf(ErrParameterNotfound, "namespace")) + } + + q.option = monitoring.ApplicationsOption{ + NamespaceName: r.namespaceName, + Applications: strings.Split(r.applications, "|"), + StorageClassName: r.storageClassName, // metering pvc + } + q.namedMetrics = model.ApplicationMetrics + case monitoring.LevelWorkload: q.identifier = model.IdentifierWorkload - q.namedMetrics = model.WorkloadMetrics q.option = monitoring.WorkloadOption{ ResourceFilter: r.resourceFilter, NamespaceName: r.namespaceName, WorkloadKind: r.workloadKind, } + q.namedMetrics = model.WorkloadMetrics + case monitoring.LevelPod: q.identifier = model.IdentifierPod - q.namedMetrics = model.PodMetrics q.option = monitoring.PodOption{ - NamespacedResourcesFilter: r.namespacedResourcesFilter, - ResourceFilter: r.resourceFilter, - NodeName: r.nodeName, - NamespaceName: r.namespaceName, - WorkloadKind: r.workloadKind, - WorkloadName: r.workloadName, - PodName: r.podName, + ResourceFilter: r.resourceFilter, + NodeName: r.nodeName, + NamespaceName: r.namespaceName, + WorkloadKind: r.workloadKind, + WorkloadName: r.workloadName, + PodName: r.podName, } + q.namedMetrics = model.PodMetrics + + case monitoring.LevelService: + q.identifier = model.IdentifierService + q.option = monitoring.ServicesOption{ + NamespaceName: r.namespaceName, + Services: strings.Split(r.services, "|"), + } + q.namedMetrics = model.ServiceMetrics + case monitoring.LevelContainer: q.identifier = model.IdentifierContainer - q.namedMetrics = model.ContainerMetrics q.option = monitoring.ContainerOption{ ResourceFilter: r.resourceFilter, NamespaceName: r.namespaceName, PodName: r.podName, ContainerName: r.containerName, } + q.namedMetrics = model.ContainerMetrics + case monitoring.LevelPVC: q.identifier = model.IdentifierPVC - q.namedMetrics = model.PVCMetrics q.option = monitoring.PVCOption{ ResourceFilter: r.resourceFilter, NamespaceName: r.namespaceName, StorageClassName: r.storageClassName, PersistentVolumeClaimName: r.pvcName, } + q.namedMetrics = model.PVCMetrics + case monitoring.LevelComponent: q.option = monitoring.ComponentOption{} switch r.componentType { @@ -304,3 +376,35 @@ func (h handler) makeQueryOptions(r reqParams, lvl monitoring.Level) (q queryOpt return q, nil } + +func ExportMetrics(resp *restful.Response, metrics model.Metrics) { + resp.Header().Set(restful.HEADER_ContentType, "text/plain") + resp.Header().Set("Content-Disposition", "attachment") + + for i, _ := range metrics.Results { + ret := metrics.Results[i] + for j, _ := range ret.MetricValues { + ret.MetricValues[j].TransferToExportedMetricValue() + } + } + + resBytes, err := json.MarshalIndent(metrics, "", " ") + if err != nil { + api.HandleBadRequest(resp, nil, err) + return + } + + output := new(bytes.Buffer) + _, err = output.Write(resBytes) + if err != nil { + api.HandleBadRequest(resp, nil, err) + return + } + + _, err = io.Copy(resp, output) + if err != nil { + api.HandleBadRequest(resp, nil, err) + return + } + return +} diff --git a/pkg/kapis/monitoring/v1alpha3/helper_test.go b/pkg/kapis/monitoring/v1alpha3/helper_test.go index 334c0c33f..97eaaa016 100644 --- a/pkg/kapis/monitoring/v1alpha3/helper_test.go +++ b/pkg/kapis/monitoring/v1alpha3/helper_test.go @@ -18,6 +18,9 @@ package v1alpha3 import ( "fmt" + "testing" + "time" + "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -25,8 +28,6 @@ import ( "kubesphere.io/kubesphere/pkg/informers" model "kubesphere.io/kubesphere/pkg/models/monitoring" "kubesphere.io/kubesphere/pkg/simple/client/monitoring" - "testing" - "time" ) func TestIsRangeQuery(t *testing.T) { @@ -84,6 +85,7 @@ func TestParseRequestParams(t *testing.T) { metricFilter: ".*", namedMetrics: model.ClusterMetrics, option: monitoring.ClusterOption{}, + Operation: OperationQuery, }, expectedErr: false, }, @@ -114,6 +116,7 @@ func TestParseRequestParams(t *testing.T) { ResourceFilter: ".*", NamespaceName: "default", }, + Operation: OperationQuery, }, expectedErr: false, }, @@ -181,6 +184,7 @@ func TestParseRequestParams(t *testing.T) { metricFilter: "etcd_server_list", namedMetrics: model.EtcdMetrics, option: monitoring.ComponentOption{}, + Operation: OperationQuery, }, expectedErr: false, }, @@ -208,6 +212,7 @@ func TestParseRequestParams(t *testing.T) { order: "desc", page: 1, limit: 10, + Operation: OperationQuery, }, expectedErr: false, }, @@ -217,7 +222,7 @@ func TestParseRequestParams(t *testing.T) { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { client := fake.NewSimpleClientset(&tt.namespace) fakeInformerFactory := informers.NewInformerFactories(client, nil, nil, nil, nil, nil) - handler := newHandler(client, nil, nil, fakeInformerFactory, nil) + handler := NewHandler(client, nil, nil, fakeInformerFactory, nil, nil) result, err := handler.makeQueryOptions(tt.params, tt.lvl) if err != nil { diff --git a/pkg/kapis/monitoring/v1alpha3/meter.go b/pkg/kapis/monitoring/v1alpha3/meter.go new file mode 100644 index 000000000..5c837d2b1 --- /dev/null +++ b/pkg/kapis/monitoring/v1alpha3/meter.go @@ -0,0 +1,327 @@ +package v1alpha3 + +import ( + "regexp" + "strings" + + "github.com/emicklei/go-restful" + "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/api" + model "kubesphere.io/kubesphere/pkg/models/monitoring" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" +) + +func (h handler) HandleClusterMetersQuery(req *restful.Request, resp *restful.Response) { + params := parseRequestParams(req) + opt, err := h.makeQueryOptions(params, monitoring.LevelCluster) + if err != nil { + api.HandleBadRequest(resp, nil, err) + return + } + h.handleNamedMetersQuery(resp, opt) +} + +func getMetricPosMap(metrics []monitoring.Metric) map[string]int { + var metricMap = make(map[string]int) + + for i, m := range metrics { + metricMap[m.MetricName] = i + } + + return metricMap +} + +func (h handler) handleApplicationMetersQuery(meters []string, resp *restful.Response, q queryOptions) { + var metricMap = make(map[string]int) + var res model.Metrics + var current_res model.Metrics + var err error + + aso, ok := q.option.(monitoring.ApplicationsOption) + if !ok { + klog.Error("invalid application option") + return + } + componentsMap := h.mo.GetAppComponentsMap(aso.NamespaceName, aso.Applications) + + for k, _ := range componentsMap { + opt := monitoring.ApplicationOption{ + NamespaceName: aso.NamespaceName, + Application: k, + ApplicationComponents: componentsMap[k], + StorageClassName: aso.StorageClassName, + } + + if q.isRangeQuery() { + current_res, err = h.mo.GetNamedMetersOverTime(meters, q.start, q.end, q.step, opt) + } else { + current_res, err = h.mo.GetNamedMeters(meters, q.time, opt) + } + if err != nil { + api.HandleBadRequest(resp, nil, err) + return + } + + if res.Results == nil { + res = current_res + metricMap = getMetricPosMap(res.Results) + } else { + for _, cur_res := range current_res.Results { + pos, ok := metricMap[cur_res.MetricName] + if ok { + res.Results[pos].MetricValues = append(res.Results[pos].MetricValues, cur_res.MetricValues...) + } else { + res.Results = append(res.Results, cur_res) + } + } + } + } + + if !q.isRangeQuery() && q.shouldSort() { + res = *res.Sort(q.target, q.order, q.identifier).Page(q.page, q.limit) + } + + if q.Operation == OperationExport { + ExportMetrics(resp, res) + return + } + + resp.WriteAsJson(res) +} + +func (h handler) handleServiceMetersQuery(meters []string, resp *restful.Response, q queryOptions) { + var metricMap = make(map[string]int) + var res model.Metrics + var current_res model.Metrics + var err error + + sso, ok := q.option.(monitoring.ServicesOption) + if !ok { + klog.Error("invalid service option") + return + } + svcPodsMap := h.mo.GetSerivePodsMap(sso.NamespaceName, sso.Services) + + for k, _ := range svcPodsMap { + opt := monitoring.ServiceOption{ + NamespaceName: sso.NamespaceName, + ServiceName: k, + PodNames: svcPodsMap[k], + } + + if q.isRangeQuery() { + current_res, err = h.mo.GetNamedMetersOverTime(meters, q.start, q.end, q.step, opt) + } else { + current_res, err = h.mo.GetNamedMeters(meters, q.time, opt) + } + if err != nil { + api.HandleBadRequest(resp, nil, err) + return + } + + if res.Results == nil { + res = current_res + metricMap = getMetricPosMap(res.Results) + } else { + for _, cur_res := range current_res.Results { + pos, ok := metricMap[cur_res.MetricName] + if ok { + res.Results[pos].MetricValues = append(res.Results[pos].MetricValues, cur_res.MetricValues...) + } else { + res.Results = append(res.Results, cur_res) + } + } + } + } + + if !q.isRangeQuery() && q.shouldSort() { + res = *res.Sort(q.target, q.order, q.identifier).Page(q.page, q.limit) + } + + if q.Operation == OperationExport { + ExportMetrics(resp, res) + return + } + + resp.WriteAsJson(res) +} + +func (h handler) handleNamedMetersQuery(resp *restful.Response, q queryOptions) { + var res model.Metrics + var err error + + var meters []string + for _, meter := range q.namedMetrics { + if !strings.HasPrefix(meter, model.MetricMeterPrefix) { + // skip non-meter metric + continue + } + + ok, _ := regexp.MatchString(q.metricFilter, meter) + if ok { + meters = append(meters, meter) + } + } + + if len(meters) == 0 { + klog.Info("no meters found") + resp.WriteAsJson(res) + return + } + + _, ok := q.option.(monitoring.ApplicationsOption) + if ok { + h.handleApplicationMetersQuery(meters, resp, q) + return + } + + _, ok = q.option.(monitoring.ServicesOption) + if ok { + h.handleServiceMetersQuery(meters, resp, q) + return + } + + if q.isRangeQuery() { + res, err = h.mo.GetNamedMetersOverTime(meters, q.start, q.end, q.step, q.option) + if err != nil { + api.HandleBadRequest(resp, nil, err) + return + } + } else { + res, err = h.mo.GetNamedMeters(meters, q.time, q.option) + if err != nil { + api.HandleBadRequest(resp, nil, err) + return + } + + if q.shouldSort() { + res = *res.Sort(q.target, q.order, q.identifier).Page(q.page, q.limit) + } + } + + if q.Operation == OperationExport { + ExportMetrics(resp, res) + return + } + + resp.WriteAsJson(res) +} + +func (h handler) HandleNodeMetersQuery(req *restful.Request, resp *restful.Response) { + params := parseRequestParams(req) + opt, err := h.makeQueryOptions(params, monitoring.LevelNode) + if err != nil { + api.HandleBadRequest(resp, nil, err) + return + } + h.handleNamedMetersQuery(resp, opt) +} + +func (h handler) HandleWorkspaceMetersQuery(req *restful.Request, resp *restful.Response) { + params := parseRequestParams(req) + opt, err := h.makeQueryOptions(params, monitoring.LevelWorkspace) + if err != nil { + api.HandleBadRequest(resp, nil, err) + return + } + + h.handleNamedMetersQuery(resp, opt) +} + +func (h handler) HandleNamespaceMetersQuery(req *restful.Request, resp *restful.Response) { + params := parseRequestParams(req) + opt, err := h.makeQueryOptions(params, monitoring.LevelNamespace) + if err != nil { + if err.Error() == ErrNoHit { + res := handleNoHit(opt.namedMetrics) + resp.WriteAsJson(res) + return + } + + api.HandleBadRequest(resp, nil, err) + return + } + + h.handleNamedMetersQuery(resp, opt) +} + +func (h handler) HandleWorkloadMetersQuery(req *restful.Request, resp *restful.Response) { + params := parseRequestParams(req) + opt, err := h.makeQueryOptions(params, monitoring.LevelWorkload) + if err != nil { + if err.Error() == ErrNoHit { + res := handleNoHit(opt.namedMetrics) + resp.WriteAsJson(res) + return + } + + api.HandleBadRequest(resp, nil, err) + return + } + h.handleNamedMetersQuery(resp, opt) +} + +func (h handler) HandleApplicationMetersQuery(req *restful.Request, resp *restful.Response) { + params := parseRequestParams(req) + opt, err := h.makeQueryOptions(params, monitoring.LevelApplication) + if err != nil { + if err.Error() == ErrNoHit { + res := handleNoHit(opt.namedMetrics) + resp.WriteAsJson(res) + return + } + + api.HandleBadRequest(resp, nil, err) + return + } + h.handleNamedMetersQuery(resp, opt) +} + +func (h handler) HandlePodMetersQuery(req *restful.Request, resp *restful.Response) { + params := parseRequestParams(req) + opt, err := h.makeQueryOptions(params, monitoring.LevelPod) + if err != nil { + if err.Error() == ErrNoHit { + res := handleNoHit(opt.namedMetrics) + resp.WriteAsJson(res) + return + } + + api.HandleBadRequest(resp, nil, err) + return + } + h.handleNamedMetersQuery(resp, opt) +} + +func (h handler) HandleServiceMetersQuery(req *restful.Request, resp *restful.Response) { + params := parseRequestParams(req) + opt, err := h.makeQueryOptions(params, monitoring.LevelService) + if err != nil { + if err.Error() == ErrNoHit { + res := handleNoHit(opt.namedMetrics) + resp.WriteAsJson(res) + return + } + + api.HandleBadRequest(resp, nil, err) + return + } + + h.handleNamedMetersQuery(resp, opt) +} + +func (h handler) HandlePVCMetersQuery(req *restful.Request, resp *restful.Response) { + params := parseRequestParams(req) + opt, err := h.makeQueryOptions(params, monitoring.LevelPVC) + if err != nil { + if err.Error() == ErrNoHit { + res := handleNoHit(opt.namedMetrics) + resp.WriteAsJson(res) + return + } + + api.HandleBadRequest(resp, nil, err) + return + } + h.handleNamedMetersQuery(resp, opt) +} diff --git a/pkg/kapis/monitoring/v1alpha3/register.go b/pkg/kapis/monitoring/v1alpha3/register.go index 44e05cac9..51e143a1a 100644 --- a/pkg/kapis/monitoring/v1alpha3/register.go +++ b/pkg/kapis/monitoring/v1alpha3/register.go @@ -42,7 +42,7 @@ var GroupVersion = schema.GroupVersion{Group: groupName, Version: "v1alpha3"} func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, monitoringClient monitoring.Interface, metricsClient monitoring.Interface, factory informers.InformerFactory, opClient openpitrix.Client) error { ws := runtime.NewWebService(GroupVersion) - h := newHandler(k8sClient, monitoringClient, metricsClient, factory, opClient) + h := NewHandler(k8sClient, monitoringClient, metricsClient, factory, opClient, nil) ws.Route(ws.GET("/kubesphere"). To(h.handleKubeSphereMetricsQuery). diff --git a/pkg/kapis/tenant/v1alpha2/handler.go b/pkg/kapis/tenant/v1alpha2/handler.go index 65bddea45..e2e085b92 100644 --- a/pkg/kapis/tenant/v1alpha2/handler.go +++ b/pkg/kapis/tenant/v1alpha2/handler.go @@ -19,6 +19,7 @@ package v1alpha2 import ( "encoding/json" "fmt" + "github.com/emicklei/go-restful" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -36,11 +37,14 @@ import ( kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned" "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/models/iam/am" + resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource" "kubesphere.io/kubesphere/pkg/models/tenant" servererr "kubesphere.io/kubesphere/pkg/server/errors" "kubesphere.io/kubesphere/pkg/simple/client/auditing" "kubesphere.io/kubesphere/pkg/simple/client/events" "kubesphere.io/kubesphere/pkg/simple/client/logging" + monitoringclient "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + opclient "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" ) type tenantHandler struct { @@ -49,10 +53,12 @@ type tenantHandler struct { func newTenantHandler(factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client, loggingClient logging.Client, auditingclient auditing.Client, - am am.AccessManagementInterface, authorizer authorizer.Authorizer) *tenantHandler { + am am.AccessManagementInterface, authorizer authorizer.Authorizer, + monitoringclient monitoringclient.Interface, opClient opclient.Client, + resourceGetter *resourcev1alpha3.ResourceGetter) *tenantHandler { return &tenantHandler{ - tenant: tenant.New(factory, k8sclient, ksclient, evtsClient, loggingClient, auditingclient, am, authorizer), + tenant: tenant.New(factory, k8sclient, ksclient, evtsClient, loggingClient, auditingclient, am, authorizer, monitoringclient, opClient, resourceGetter), } } diff --git a/pkg/kapis/tenant/v1alpha2/metering.go b/pkg/kapis/tenant/v1alpha2/metering.go new file mode 100644 index 000000000..48060ea1b --- /dev/null +++ b/pkg/kapis/tenant/v1alpha2/metering.go @@ -0,0 +1,86 @@ +package v1alpha2 + +import ( + "fmt" + + "github.com/emicklei/go-restful" + "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/api" + meteringv1alpha1 "kubesphere.io/kubesphere/pkg/api/metering/v1alpha1" + "kubesphere.io/kubesphere/pkg/apiserver/request" + monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3" + "kubesphere.io/kubesphere/pkg/models/metering" + "kubesphere.io/kubesphere/pkg/models/monitoring" + monitoringclient "kubesphere.io/kubesphere/pkg/simple/client/monitoring" +) + +func (h *tenantHandler) QueryMeterings(req *restful.Request, resp *restful.Response) { + + u, ok := request.UserFrom(req.Request.Context()) + if !ok { + err := fmt.Errorf("cannot obtain user info") + klog.Errorln(err) + api.HandleForbidden(resp, req, err) + return + } + + q := meteringv1alpha1.ParseQueryParameter(req) + + res, err := h.tenant.Metering(u, q) + if err != nil { + api.HandleBadRequest(resp, nil, err) + return + } + + if q.Operation == monitoringv1alpha3.OperationExport { + monitoringv1alpha3.ExportMetrics(resp, res) + return + } + + resp.WriteAsJson(res) +} + +func (h *tenantHandler) QueryMeteringsHierarchy(req *restful.Request, resp *restful.Response) { + u, ok := request.UserFrom(req.Request.Context()) + if !ok { + err := fmt.Errorf("cannot obtain user info") + klog.Errorln(err) + api.HandleForbidden(resp, req, err) + return + } + + q := meteringv1alpha1.ParseQueryParameter(req) + q.Level = monitoringclient.LevelPod + + resourceStats, err := h.tenant.MeteringHierarchy(u, q) + if err != nil { + api.HandleBadRequest(resp, nil, err) + return + } + + resp.WriteAsJson(resourceStats) +} + +func (h *tenantHandler) HandlePriceInfoQuery(req *restful.Request, resp *restful.Response) { + + var priceInfoResponse metering.PriceInfo + + meterConfig, err := monitoring.LoadYaml() + if err != nil { + klog.Error(err) + resp.WriteAsJson(priceInfoResponse) + return + } + + priceInfo := meterConfig.GetPriceInfo() + priceInfoResponse.Currency = "CNY" + priceInfoResponse.CpuPerCorePerHour = priceInfo.CpuPerCorePerHour + priceInfoResponse.MemPerGigabytesPerHour = priceInfo.MemPerGigabytesPerHour + priceInfoResponse.IngressNetworkTrafficPerGiagabytesPerHour = priceInfo.IngressNetworkTrafficPerGiagabytesPerHour + priceInfoResponse.EgressNetworkTrafficPerGiagabytesPerHour = priceInfo.EgressNetworkTrafficPerGigabytesPerHour + priceInfoResponse.PvcPerGigabytesPerHour = priceInfo.PvcPerGigabytesPerHour + + resp.WriteAsJson(priceInfoResponse) + + return +} diff --git a/pkg/kapis/tenant/v1alpha2/register.go b/pkg/kapis/tenant/v1alpha2/register.go index cf295e5e4..e2abd4a1e 100644 --- a/pkg/kapis/tenant/v1alpha2/register.go +++ b/pkg/kapis/tenant/v1alpha2/register.go @@ -17,6 +17,10 @@ limitations under the License. package v1alpha2 import ( + "kubesphere.io/kubesphere/pkg/models/metering" + "net/http" + "sigs.k8s.io/controller-runtime/pkg/cache" + "github.com/emicklei/go-restful" "github.com/emicklei/go-restful-openapi" corev1 "k8s.io/api/core/v1" @@ -32,13 +36,17 @@ import ( kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned" "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/informers" + monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3" "kubesphere.io/kubesphere/pkg/models" "kubesphere.io/kubesphere/pkg/models/iam/am" + "kubesphere.io/kubesphere/pkg/models/monitoring" + resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource" "kubesphere.io/kubesphere/pkg/server/errors" "kubesphere.io/kubesphere/pkg/simple/client/auditing" "kubesphere.io/kubesphere/pkg/simple/client/events" "kubesphere.io/kubesphere/pkg/simple/client/logging" - "net/http" + monitoringclient "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + opclient "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" ) const ( @@ -53,11 +61,12 @@ func Resource(resource string) schema.GroupResource { func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client, loggingClient logging.Client, - auditingclient auditing.Client, am am.AccessManagementInterface, authorizer authorizer.Authorizer) error { + auditingclient auditing.Client, am am.AccessManagementInterface, authorizer authorizer.Authorizer, + monitoringclient monitoringclient.Interface, opClient opclient.Client, cache cache.Cache) error { mimePatch := []string{restful.MIME_JSON, runtime.MimeMergePatchJson, runtime.MimeJsonPatchJson} ws := runtime.NewWebService(GroupVersion) - handler := newTenantHandler(factory, k8sclient, ksclient, evtsClient, loggingClient, auditingclient, am, authorizer) + handler := newTenantHandler(factory, k8sclient, ksclient, evtsClient, loggingClient, auditingclient, am, authorizer, monitoringclient, opClient, resourcev1alpha3.NewResourceGetter(factory, cache)) ws.Route(ws.GET("/clusters"). To(handler.ListClusters). @@ -288,6 +297,50 @@ func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8s Writes(auditingv1alpha1.APIResponse{}). Returns(http.StatusOK, api.StatusOK, auditingv1alpha1.APIResponse{})) + ws.Route(ws.GET("/meterings"). + To(handler.QueryMeterings). + Doc("Get meterings against the cluster."). + Param(ws.QueryParameter("level", "Metering level.").DataType("string").Required(true)). + Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)). + Param(ws.QueryParameter("node", "Node name.").DataType("string").Required(false)). + Param(ws.QueryParameter("workspace", "Workspace name.").DataType("string").Required(false)). + Param(ws.QueryParameter("namespace", "Namespace name.").DataType("string").Required(false)). + Param(ws.QueryParameter("kind", "Workload kind. One of deployment, daemonset, statefulset.").DataType("string").Required(false)). + Param(ws.QueryParameter("workload", "Workload name.").DataType("string").Required(false)). + Param(ws.QueryParameter("pod", "Pod name.").DataType("string").Required(false)). + Param(ws.QueryParameter("applications", "Appliction names, format app_name[:app_version](such as nginx:v1, nignx) which are joined by \"|\" ").DataType("string").Required(false)). + Param(ws.QueryParameter("services", "Services which are joined by \"|\".").DataType("string").Required(false)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workspace CPU usage and memory usage: `meter_workspace_cpu_usage|meter_workspace_memory_usage`.").DataType("string").Required(false)). + Param(ws.QueryParameter("resources_filter", "The workspace filter consists of a regexp pattern. It specifies which workspace data to return.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_metric", "Sort workspaces by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). + Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). + Param(ws.QueryParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)). + Param(ws.QueryParameter("pvc_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkspaceMetersTag}). + Writes(monitoring.Metrics{}). + Returns(http.StatusOK, api.StatusOK, monitoring.Metrics{})) + + ws.Route(ws.GET("/namespaces/{namespace}/metering_hierarchy"). + To(handler.QueryMeteringsHierarchy). + Param(ws.PathParameter("namespace", "Namespace name.").DataType("string").Required(false)). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workspace CPU usage and memory usage: `meter_pod_cpu_usage|meter_pod_memory_usage_wo_cache`.").DataType("string").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Doc("get current metering hierarchies info in last one hour"). + Writes(metering.ResourceStatistic{}). + Returns(http.StatusOK, api.StatusOK, metering.ResourceStatistic{})) + + ws.Route(ws.GET("/metering/price_info"). + To(handler.HandlePriceInfoQuery). + Doc("Get resoure price info."). + Writes(metering.PriceInfo{}). + Returns(http.StatusOK, api.StatusOK, metering.PriceInfo{})) + c.Add(ws) return nil } diff --git a/pkg/models/metering/type.go b/pkg/models/metering/type.go new file mode 100644 index 000000000..ef6c228fc --- /dev/null +++ b/pkg/models/metering/type.go @@ -0,0 +1,299 @@ +package metering + +type PriceInfo struct { + Currency string `json:"currency" description:"currency"` + CpuPerCorePerHour float64 `json:"cpu_per_core_per_hour,omitempty" description:"cpu price"` + MemPerGigabytesPerHour float64 `json:"mem_per_gigabytes_per_hour,omitempty" description:"mem price"` + IngressNetworkTrafficPerGiagabytesPerHour float64 `json:"ingress_network_traffic_per_giagabytes_per_hour,omitempty" description:"ingress price"` + EgressNetworkTrafficPerGiagabytesPerHour float64 `json:"egress_network_traffic_per_gigabytes_per_hour,omitempty" description:"egress price"` + PvcPerGigabytesPerHour float64 `json:"pvc_per_gigabytes_per_hour,omitempty" description:"pvc price"` +} + +type PodStatistic struct { + CPUUsage float64 `json:"cpu_usage" description:"cpu_usage"` + MemoryUsageWoCache float64 `json:"memory_usage_wo_cache" description:"memory_usage_wo_cache"` + NetBytesTransmitted float64 `json:"net_bytes_transmitted" desription:"net_bytes_transmitted"` + NetBytesReceived float64 `json:"net_bytes_received" description:"net_bytes_received"` + PVCBytesTotal float64 `json:"pvc_bytes_total" description:"pvc_bytes_total"` +} + +type PodsStats map[string]*PodStatistic + +func (ps *PodsStats) Set(podName, meterName string, value float64) { + if _, ok := (*ps)[podName]; !ok { + (*ps)[podName] = &PodStatistic{} + } + switch meterName { + case "meter_pod_cpu_usage": + (*ps)[podName].CPUUsage = value + case "meter_pod_memory_usage_wo_cache": + (*ps)[podName].MemoryUsageWoCache = value + case "meter_pod_net_bytes_transmitted": + (*ps)[podName].NetBytesTransmitted = value + case "meter_pod_net_bytes_received": + (*ps)[podName].NetBytesReceived = value + case "meter_pod_pvc_bytes_total": + (*ps)[podName].PVCBytesTotal = value + } +} + +type AppStatistic struct { + CPUUsage float64 `json:"cpu_usage" description:"cpu_usage"` + MemoryUsageWoCache float64 `json:"memory_usage_wo_cache" description:"memory_usage_wo_cache"` + NetBytesTransmitted float64 `json:"net_bytes_transmitted" description:"net_bytes_transmitted"` + NetBytesReceived float64 `json:"net_bytes_received" description:"net_bytes_received"` + PVCBytesTotal float64 `json:"pvc_bytes_total" description:"pvc_bytes_total"` + Services map[string]*ServiceStatistic `json:"services" description:"services"` +} + +func (as *AppStatistic) GetServiceStats(name string) *ServiceStatistic { + if as.Services == nil { + as.Services = make(map[string]*ServiceStatistic) + } + if as.Services[name] == nil { + as.Services[name] = &ServiceStatistic{} + } + return as.Services[name] +} + +func (as *AppStatistic) Aggregate() { + if as.Services == nil { + return + } + + // remove duplicate pods which were selected by different svc + podsMap := make(map[string]struct{}) + for _, svcObj := range as.Services { + for podName, podObj := range svcObj.Pods { + if _, ok := podsMap[podName]; ok { + continue + } else { + podsMap[podName] = struct{}{} + } + as.CPUUsage += podObj.CPUUsage + as.MemoryUsageWoCache += podObj.MemoryUsageWoCache + as.NetBytesTransmitted += podObj.NetBytesTransmitted + as.NetBytesReceived += podObj.NetBytesReceived + as.PVCBytesTotal += podObj.PVCBytesTotal + } + } +} + +type ServiceStatistic struct { + CPUUsage float64 `json:"cpu_usage" description:"cpu_usage"` + MemoryUsageWoCache float64 `json:"memory_usage_wo_cache" desription:"memory_usage_wo_cache"` + NetBytesTransmitted float64 `json:"net_bytes_transmitted" description:"net_bytes_transmitted"` + NetBytesReceived float64 `json:"net_bytes_received" description:"net_bytes_received"` + Pods map[string]*PodStatistic `json:"pods" description:"pod statistic"` +} + +func (ss *ServiceStatistic) SetPodStats(name string, podStat *PodStatistic) error { + if ss.Pods == nil { + ss.Pods = make(map[string]*PodStatistic) + } + ss.Pods[name] = podStat + return nil +} + +func (ss *ServiceStatistic) GetPodStats(name string) *PodStatistic { + if ss.Pods == nil { + ss.Pods = make(map[string]*PodStatistic) + } + if ss.Pods[name] == nil { + ss.Pods[name] = &PodStatistic{} + } + return ss.Pods[name] +} + +func (ss *ServiceStatistic) Aggregate() { + if ss.Pods == nil { + return + } + + for key := range ss.Pods { + ss.CPUUsage += ss.GetPodStats(key).CPUUsage + ss.MemoryUsageWoCache += ss.GetPodStats(key).MemoryUsageWoCache + ss.NetBytesTransmitted += ss.GetPodStats(key).NetBytesTransmitted + ss.NetBytesReceived += ss.GetPodStats(key).NetBytesReceived + } +} + +type DeploymentStatistic struct { + CPUUsage float64 `json:"cpu_usage" description:"cpu_usage"` + MemoryUsageWoCache float64 `json:"memory_usage_wo_cache" description:"memory_usage_wo_cache"` + NetBytesTransmitted float64 `json:"net_bytes_transmitted" desciption:"net_bytes_transmitted"` + NetBytesReceived float64 `json:"net_bytes_received" description:"net_bytes_received"` + PVCBytesTotal float64 `json:"pvc_bytes_total" description:"pvc_bytes_total"` + Pods map[string]*PodStatistic `json:"pods" description:"pod statistic"` +} + +func (ds *DeploymentStatistic) GetPodStats(name string) *PodStatistic { + if ds.Pods == nil { + ds.Pods = make(map[string]*PodStatistic) + } + if ds.Pods[name] == nil { + ds.Pods[name] = &PodStatistic{} + } + return ds.Pods[name] +} + +func (ds *DeploymentStatistic) SetPodStats(name string, podStat *PodStatistic) error { + if ds.Pods == nil { + ds.Pods = make(map[string]*PodStatistic) + } + ds.Pods[name] = podStat + return nil +} + +func (ds *DeploymentStatistic) Aggregate() { + if ds.Pods == nil { + return + } + + for key := range ds.Pods { + ds.CPUUsage += ds.GetPodStats(key).CPUUsage + ds.MemoryUsageWoCache += ds.GetPodStats(key).MemoryUsageWoCache + ds.NetBytesTransmitted += ds.GetPodStats(key).NetBytesTransmitted + ds.NetBytesReceived += ds.GetPodStats(key).NetBytesReceived + ds.PVCBytesTotal += ds.GetPodStats(key).PVCBytesTotal + } +} + +type StatefulsetStatistic struct { + CPUUsage float64 `json:"cpu_usage" description:"cpu_usage"` + MemoryUsageWoCache float64 `json:"memory_usage_wo_cache" description:"memory_usage_wo_cache"` + NetBytesTransmitted float64 `json:"net_bytes_transmitted" description:"net_bytes_transmitted"` + NetBytesReceived float64 `json:"net_bytes_received" description:"net_bytes_received"` + PVCBytesTotal float64 `json:"pvc_bytes_total" description:"pvc_bytes_total"` + Pods map[string]*PodStatistic `json:"pods" description:"pod statistic"` +} + +func (ss *StatefulsetStatistic) GetPodStats(name string) *PodStatistic { + if ss.Pods == nil { + ss.Pods = make(map[string]*PodStatistic) + } + if ss.Pods[name] == nil { + ss.Pods[name] = &PodStatistic{} + } + return ss.Pods[name] +} + +func (ss *StatefulsetStatistic) SetPodStats(name string, podStat *PodStatistic) error { + if ss.Pods == nil { + ss.Pods = make(map[string]*PodStatistic) + } + ss.Pods[name] = podStat + return nil +} + +func (ss *StatefulsetStatistic) Aggregate() { + if ss.Pods == nil { + return + } + + for key := range ss.Pods { + ss.CPUUsage += ss.GetPodStats(key).CPUUsage + ss.MemoryUsageWoCache += ss.GetPodStats(key).MemoryUsageWoCache + ss.NetBytesTransmitted += ss.GetPodStats(key).NetBytesTransmitted + ss.NetBytesReceived += ss.GetPodStats(key).NetBytesReceived + ss.PVCBytesTotal += ss.GetPodStats(key).PVCBytesTotal + } +} + +type DaemonsetStatistic struct { + CPUUsage float64 `json:"cpu_usage" description:"cpu_usage"` + MemoryUsageWoCache float64 `json:"memory_usage_wo_cache" description:"memory_usage_wo_cache"` + NetBytesTransmitted float64 `json:"net_bytes_transmitted" description:"net_bytes_transmitted"` + NetBytesReceived float64 `json:"net_bytes_received" description:"net_bytes_received"` + PVCBytesTotal float64 `json:"pvc_bytes_total" description:"pvc_bytes_total"` + Pods map[string]*PodStatistic `json:"pods" description:"pod statistic"` +} + +func (ds *DaemonsetStatistic) GetPodStats(name string) *PodStatistic { + if ds.Pods == nil { + ds.Pods = make(map[string]*PodStatistic) + } + if ds.Pods[name] == nil { + ds.Pods[name] = &PodStatistic{} + } + return ds.Pods[name] +} + +func (ds *DaemonsetStatistic) SetPodStats(name string, podStat *PodStatistic) error { + if ds.Pods == nil { + ds.Pods = make(map[string]*PodStatistic) + } + ds.Pods[name] = podStat + return nil +} + +func (ds *DaemonsetStatistic) Aggregate() { + if ds.Pods == nil { + return + } + for key := range ds.Pods { + ds.CPUUsage += ds.GetPodStats(key).CPUUsage + ds.MemoryUsageWoCache += ds.GetPodStats(key).MemoryUsageWoCache + ds.NetBytesTransmitted += ds.GetPodStats(key).NetBytesTransmitted + ds.NetBytesReceived += ds.GetPodStats(key).NetBytesReceived + ds.PVCBytesTotal += ds.GetPodStats(key).PVCBytesTotal + } +} + +type ResourceStatistic struct { + Apps map[string]*AppStatistic `json:"apps" description:"app statistic"` + Services map[string]*ServiceStatistic `json:"services" description:"service statistic"` + Deploys map[string]*DeploymentStatistic `json:"deployments" description:"deployment statistic"` + Statefulsets map[string]*StatefulsetStatistic `json:"statefulsets" description:"statefulset statistic"` + Daemonsets map[string]*DaemonsetStatistic `json:"daemonsets" description:"daemonsets statistics"` +} + +func (rs *ResourceStatistic) GetAppStats(name string) *AppStatistic { + if rs.Apps == nil { + rs.Apps = make(map[string]*AppStatistic) + } + if rs.Apps[name] == nil { + rs.Apps[name] = &AppStatistic{} + } + return rs.Apps[name] +} + +func (rs *ResourceStatistic) GetServiceStats(name string) *ServiceStatistic { + if rs.Services == nil { + rs.Services = make(map[string]*ServiceStatistic) + } + if rs.Services[name] == nil { + rs.Services[name] = &ServiceStatistic{} + } + return rs.Services[name] +} + +func (rs *ResourceStatistic) GetDeployStats(name string) *DeploymentStatistic { + if rs.Deploys == nil { + rs.Deploys = make(map[string]*DeploymentStatistic) + } + if rs.Deploys[name] == nil { + rs.Deploys[name] = &DeploymentStatistic{} + } + return rs.Deploys[name] +} + +func (rs *ResourceStatistic) GetStatefulsetStats(name string) *StatefulsetStatistic { + if rs.Statefulsets == nil { + rs.Statefulsets = make(map[string]*StatefulsetStatistic) + } + if rs.Statefulsets[name] == nil { + rs.Statefulsets[name] = &StatefulsetStatistic{} + } + return rs.Statefulsets[name] +} + +func (rs *ResourceStatistic) GetDaemonsetStats(name string) *DaemonsetStatistic { + if rs.Daemonsets == nil { + rs.Daemonsets = make(map[string]*DaemonsetStatistic) + } + if rs.Daemonsets[name] == nil { + rs.Daemonsets[name] = &DaemonsetStatistic{} + } + return rs.Daemonsets[name] +} diff --git a/pkg/models/monitoring/monitoring.go b/pkg/models/monitoring/monitoring.go index 9237a6251..d93447602 100644 --- a/pkg/models/monitoring/monitoring.go +++ b/pkg/models/monitoring/monitoring.go @@ -18,6 +18,9 @@ package monitoring import ( "context" + "fmt" + "math" + "strings" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,14 +29,19 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/klog" "kubesphere.io/kubesphere/pkg/apis/iam/v1alpha2" + "kubesphere.io/kubesphere/pkg/apiserver/query" ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions" "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/models/monitoring/expressions" "kubesphere.io/kubesphere/pkg/models/openpitrix" + resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource" + "kubesphere.io/kubesphere/pkg/server/errors" "kubesphere.io/kubesphere/pkg/server/params" "kubesphere.io/kubesphere/pkg/simple/client/monitoring" opclient "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" + "sigs.k8s.io/application/api/v1beta1" + appv1beta1 "sigs.k8s.io/application/api/v1beta1" ) type MonitoringOperator interface { @@ -47,23 +55,31 @@ type MonitoringOperator interface { // TODO: expose KubeSphere self metrics in Prometheus format GetKubeSphereStats() Metrics GetWorkspaceStats(workspace string) Metrics + + // meter + GetNamedMetersOverTime(metrics []string, start, end time.Time, step time.Duration, opt monitoring.QueryOption) (Metrics, error) + GetNamedMeters(metrics []string, time time.Time, opt monitoring.QueryOption) (Metrics, error) + GetAppComponentsMap(ns string, apps []string) map[string][]string + GetSerivePodsMap(ns string, services []string) map[string][]string } type monitoringOperator struct { - prometheus monitoring.Interface - metricsserver monitoring.Interface - k8s kubernetes.Interface - ks ksinformers.SharedInformerFactory - op openpitrix.Interface + prometheus monitoring.Interface + metricsserver monitoring.Interface + k8s kubernetes.Interface + ks ksinformers.SharedInformerFactory + op openpitrix.Interface + resourceGetter *resourcev1alpha3.ResourceGetter } -func NewMonitoringOperator(monitoringClient monitoring.Interface, metricsClient monitoring.Interface, k8s kubernetes.Interface, factory informers.InformerFactory, opClient opclient.Client) MonitoringOperator { +func NewMonitoringOperator(monitoringClient monitoring.Interface, metricsClient monitoring.Interface, k8s kubernetes.Interface, factory informers.InformerFactory, opClient opclient.Client, resourceGetter *resourcev1alpha3.ResourceGetter) MonitoringOperator { return &monitoringOperator{ - prometheus: monitoringClient, - metricsserver: metricsClient, - k8s: k8s, - ks: factory.KubeSphereSharedInformerFactory(), - op: openpitrix.NewOpenpitrixOperator(factory.KubernetesSharedInformerFactory(), opClient), + prometheus: monitoringClient, + metricsserver: metricsClient, + k8s: k8s, + ks: factory.KubeSphereSharedInformerFactory(), + op: openpitrix.NewOpenpitrixOperator(factory.KubernetesSharedInformerFactory(), opClient), + resourceGetter: resourceGetter, } } @@ -354,3 +370,225 @@ func (mo monitoringOperator) GetWorkspaceStats(workspace string) Metrics { return res } + +/* + meter related methods +*/ + +func (mo monitoringOperator) getNamedMetersWithHourInterval(meters []string, t time.Time, opt monitoring.QueryOption) Metrics { + + var opts []monitoring.QueryOption + + opts = append(opts, opt) + opts = append(opts, monitoring.MeterOption{ + Step: 1 * time.Hour, + }) + + ress := mo.prometheus.GetNamedMeters(meters, t, opts) + + return Metrics{Results: ress} +} + +func generateScalingFactorMap(step time.Duration) map[string]float64 { + scalingMap := make(map[string]float64) + + for k := range MeterResourceMap { + scalingMap[k] = step.Hours() + } + return scalingMap +} + +func (mo monitoringOperator) GetNamedMetersOverTime(meters []string, start, end time.Time, step time.Duration, opt monitoring.QueryOption) (metrics Metrics, err error) { + + if step.Hours() < 1 { + klog.Warning("step should be longer than one hour") + step = 1 * time.Hour + } + if end.Sub(start).Hours() > 30*24 { + if step.Hours() < 24 { + err = errors.New("step should be larger than 24 hours") + return + } + } + if math.Mod(step.Hours(), 1.0) > 0 { + err = errors.New("step should be integer hours") + return + } + + // query time range: (start, end], so here we need to exclude start itself. + if start.Add(step).After(end) { + start = end + } else { + start = start.Add(step) + } + + var opts []monitoring.QueryOption + + opts = append(opts, opt) + opts = append(opts, monitoring.MeterOption{ + Start: start, + End: end, + Step: step, + }) + + ress := mo.prometheus.GetNamedMetersOverTime(meters, start, end, step, opts) + sMap := generateScalingFactorMap(step) + + for i, _ := range ress { + ress[i].MetricData = updateMetricStatData(ress[i], sMap) + } + + return Metrics{Results: ress}, nil +} + +func (mo monitoringOperator) GetNamedMeters(meters []string, time time.Time, opt monitoring.QueryOption) (Metrics, error) { + + metersPerHour := mo.getNamedMetersWithHourInterval(meters, time, opt) + + for metricIndex, _ := range metersPerHour.Results { + + res := metersPerHour.Results[metricIndex] + + metersPerHour.Results[metricIndex].MetricData = updateMetricStatData(res, nil) + } + + return metersPerHour, nil +} + +func (mo monitoringOperator) GetAppComponentsMap(ns string, apps []string) map[string][]string { + + componentsMap := make(map[string][]string) + applicationList := []*appv1beta1.Application{} + + result, err := mo.resourceGetter.List("applications", ns, query.New()) + if err != nil { + klog.Error(err) + return nil + } + + for _, obj := range result.Items { + app, ok := obj.(*appv1beta1.Application) + if !ok { + continue + } + + applicationList = append(applicationList, app) + } + + getAppFullName := func(appObject *v1beta1.Application) (name string) { + name = appObject.Labels[constants.ApplicationName] + if appObject.Labels[constants.ApplicationVersion] != "" { + name += fmt.Sprintf(":%v", appObject.Labels[constants.ApplicationVersion]) + } + return + } + + appFilter := func(appObject *v1beta1.Application) bool { + + for _, app := range apps { + var applicationName, applicationVersion string + tmp := strings.Split(app, ":") + + if len(tmp) >= 1 { + applicationName = tmp[0] + } + if len(tmp) == 2 { + applicationVersion = tmp[1] + } + + if applicationName != "" && appObject.Labels[constants.ApplicationName] != applicationName { + return false + } + if applicationVersion != "" && appObject.Labels[constants.ApplicationVersion] != applicationVersion { + return false + } + return true + } + + return true + } + + for _, appObj := range applicationList { + if appFilter(appObj) { + for _, com := range appObj.Status.ComponentList.Objects { + kind := strings.Title(com.Kind) + name := com.Name + componentsMap[getAppFullName((appObj))] = append(componentsMap[getAppFullName(appObj)], kind+":"+name) + } + } + } + + return componentsMap +} + +func (mo monitoringOperator) getApplicationPVCs(appObject *v1beta1.Application) []string { + + var pvcList []string + + ns := appObject.Namespace + for _, com := range appObject.Status.ComponentList.Objects { + + switch strings.Title(com.Kind) { + case "Deployment": + deployObj, err := mo.k8s.AppsV1().Deployments(ns).Get(context.Background(), com.Name, metav1.GetOptions{}) + if err != nil { + klog.Error(err.Error()) + return nil + } + + for _, vol := range deployObj.Spec.Template.Spec.Volumes { + pvcList = append(pvcList, vol.PersistentVolumeClaim.ClaimName) + } + case "Statefulset": + stsObj, err := mo.k8s.AppsV1().StatefulSets(ns).Get(context.Background(), com.Name, metav1.GetOptions{}) + if err != nil { + klog.Error(err.Error()) + return nil + } + for _, vol := range stsObj.Spec.Template.Spec.Volumes { + pvcList = append(pvcList, vol.PersistentVolumeClaim.ClaimName) + } + } + + } + + return pvcList + +} + +func (mo monitoringOperator) GetSerivePodsMap(ns string, services []string) map[string][]string { + var svcPodsMap = make(map[string][]string) + + for _, svc := range services { + svcObj, err := mo.k8s.CoreV1().Services(ns).Get(context.Background(), svc, metav1.GetOptions{}) + if err != nil { + klog.Error(err.Error()) + return svcPodsMap + } + + svcSelector := svcObj.Spec.Selector + if len(svcSelector) == 0 { + return svcPodsMap + } + + svcLabels := labels.Set{} + for key, value := range svcSelector { + svcLabels[key] = value + } + + selector := labels.SelectorFromSet(svcLabels) + opt := metav1.ListOptions{LabelSelector: selector.String()} + + podList, err := mo.k8s.CoreV1().Pods(ns).List(context.Background(), opt) + if err != nil { + klog.Error(err.Error()) + return svcPodsMap + } + + for _, pod := range podList.Items { + svcPodsMap[svc] = append(svcPodsMap[svc], pod.Name) + } + + } + return svcPodsMap +} diff --git a/pkg/models/monitoring/named_metrics.go b/pkg/models/monitoring/named_metrics.go index 9ab38381a..5c01a8e6f 100644 --- a/pkg/models/monitoring/named_metrics.go +++ b/pkg/models/monitoring/named_metrics.go @@ -26,6 +26,8 @@ const ( WorkspaceDevopsCount = "workspace_devops_project_count" WorkspaceMemberCount = "workspace_member_count" WorkspaceRoleCount = "workspace_role_count" + + MetricMeterPrefix = "meter_" ) var ClusterMetrics = []string{ @@ -78,6 +80,13 @@ var ClusterMetrics = []string{ "cluster_load15", "cluster_pod_abnormal_ratio", "cluster_node_offline_ratio", + + // meter + "meter_cluster_cpu_usage", + "meter_cluster_memory_usage", + "meter_cluster_net_bytes_transmitted", + "meter_cluster_net_bytes_received", + "meter_cluster_pvc_bytes_total", } var NodeMetrics = []string{ @@ -113,6 +122,13 @@ var NodeMetrics = []string{ "node_load15", "node_pod_abnormal_ratio", "node_pleg_quantile", + + // meter + "meter_node_cpu_usage", + "meter_node_memory_usage_wo_cache", + "meter_node_net_bytes_transmitted", + "meter_node_net_bytes_received", + "meter_node_pvc_bytes_total", } var WorkspaceMetrics = []string{ @@ -138,6 +154,13 @@ var WorkspaceMetrics = []string{ "workspace_service_count", "workspace_secret_count", "workspace_pod_abnormal_ratio", + + // meter + "meter_workspace_cpu_usage", + "meter_workspace_memory_usage", + "meter_workspace_net_bytes_transmitted", + "meter_workspace_net_bytes_received", + "meter_workspace_pvc_bytes_total", } var NamespaceMetrics = []string{ @@ -168,6 +191,23 @@ var NamespaceMetrics = []string{ "namespace_configmap_count", "namespace_ingresses_extensions_count", "namespace_s2ibuilder_count", + + // meter + "meter_namespace_cpu_usage", + "meter_namespace_memory_usage_wo_cache", + "meter_namespace_net_bytes_transmitted", + "meter_namespace_net_bytes_received", + "meter_namespace_pvc_bytes_total", +} + +var ApplicationMetrics = []string{ + + // meter + "meter_application_cpu_usage", + "meter_application_memory_usage_wo_cache", + "meter_application_net_bytes_transmitted", + "meter_application_net_bytes_received", + "meter_application_pvc_bytes_total", } var WorkloadMetrics = []string{ @@ -185,6 +225,21 @@ var WorkloadMetrics = []string{ "workload_deployment_unavailable_replicas_ratio", "workload_daemonset_unavailable_replicas_ratio", "workload_statefulset_unavailable_replicas_ratio", + + // meter + "meter_workload_cpu_usage", + "meter_workload_memory_usage_wo_cache", + "meter_workload_net_bytes_transmitted", + "meter_workload_net_bytes_received", + "meter_workload_pvc_bytes_total", +} + +var ServiceMetrics = []string{ + // meter + "meter_service_cpu_usage", + "meter_service_memory_usage_wo_cache", + "meter_service_net_bytes_transmitted", + "meter_service_net_bytes_received", } var PodMetrics = []string{ @@ -193,6 +248,13 @@ var PodMetrics = []string{ "pod_memory_usage_wo_cache", "pod_net_bytes_transmitted", "pod_net_bytes_received", + + // meter + "meter_pod_cpu_usage", + "meter_pod_memory_usage_wo_cache", + "meter_pod_net_bytes_transmitted", + "meter_pod_net_bytes_received", + "meter_pod_pvc_bytes_total", } var ContainerMetrics = []string{ diff --git a/pkg/models/monitoring/sort_page.go b/pkg/models/monitoring/sort_page.go index 97d0a76f5..814f1a453 100644 --- a/pkg/models/monitoring/sort_page.go +++ b/pkg/models/monitoring/sort_page.go @@ -23,13 +23,15 @@ import ( ) const ( - IdentifierNode = "node" - IdentifierWorkspace = "workspace" - IdentifierNamespace = "namespace" - IdentifierWorkload = "workload" - IdentifierPod = "pod" - IdentifierContainer = "container" - IdentifierPVC = "persistentvolumeclaim" + IdentifierNode = "node" + IdentifierWorkspace = "workspace" + IdentifierNamespace = "namespace" + IdentifierWorkload = "workload" + IdentifierPod = "pod" + IdentifierContainer = "container" + IdentifierPVC = "persistentvolumeclaim" + IdentifierService = "service" + IdentifierApplication = "application" OrderAscending = "asc" OrderDescending = "desc" diff --git a/pkg/models/monitoring/types.go b/pkg/models/monitoring/types.go index 3adde7363..b6fe2882c 100644 --- a/pkg/models/monitoring/types.go +++ b/pkg/models/monitoring/types.go @@ -16,7 +16,9 @@ limitations under the License. package monitoring -import "kubesphere.io/kubesphere/pkg/simple/client/monitoring" +import ( + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" +) type Metrics struct { Results []monitoring.Metric `json:"results" description:"actual array of results"` diff --git a/pkg/models/monitoring/utils.go b/pkg/models/monitoring/utils.go new file mode 100644 index 000000000..daf12409e --- /dev/null +++ b/pkg/models/monitoring/utils.go @@ -0,0 +1,222 @@ +package monitoring + +import ( + "math" + "os" + + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" +) + +const ( + METER_RESOURCE_TYPE_CPU = iota + METER_RESOURCE_TYPE_MEM + METER_RESOURCE_TYPE_NET_INGRESS + METER_RESOURCE_TYPE_NET_EGRESS + METER_RESOURCE_TYPE_PVC + + meteringConfig = "/etc/kubesphere/metering/ks-metering.yaml" +) + +var MeterResourceMap = map[string]int{ + "meter_cluster_cpu_usage": METER_RESOURCE_TYPE_CPU, + "meter_cluster_memory_usage": METER_RESOURCE_TYPE_MEM, + "meter_cluster_net_bytes_transmitted": METER_RESOURCE_TYPE_NET_EGRESS, + "meter_cluster_net_bytes_received": METER_RESOURCE_TYPE_NET_INGRESS, + "meter_cluster_pvc_bytes_total": METER_RESOURCE_TYPE_PVC, + "meter_node_cpu_usage": METER_RESOURCE_TYPE_CPU, + "meter_node_memory_usage_wo_cache": METER_RESOURCE_TYPE_MEM, + "meter_node_net_bytes_transmitted": METER_RESOURCE_TYPE_NET_EGRESS, + "meter_node_net_bytes_received": METER_RESOURCE_TYPE_NET_INGRESS, + "meter_node_pvc_bytes_total": METER_RESOURCE_TYPE_PVC, + "meter_workspace_cpu_usage": METER_RESOURCE_TYPE_CPU, + "meter_workspace_memory_usage": METER_RESOURCE_TYPE_MEM, + "meter_workspace_net_bytes_transmitted": METER_RESOURCE_TYPE_NET_EGRESS, + "meter_workspace_net_bytes_received": METER_RESOURCE_TYPE_NET_INGRESS, + "meter_workspace_pvc_bytes_total": METER_RESOURCE_TYPE_PVC, + "meter_namespace_cpu_usage": METER_RESOURCE_TYPE_CPU, + "meter_namespace_memory_usage_wo_cache": METER_RESOURCE_TYPE_MEM, + "meter_namespace_net_bytes_transmitted": METER_RESOURCE_TYPE_NET_EGRESS, + "meter_namespace_net_bytes_received": METER_RESOURCE_TYPE_NET_INGRESS, + "meter_namespace_pvc_bytes_total": METER_RESOURCE_TYPE_PVC, + "meter_application_cpu_usage": METER_RESOURCE_TYPE_CPU, + "meter_application_memory_usage_wo_cache": METER_RESOURCE_TYPE_MEM, + "meter_application_net_bytes_transmitted": METER_RESOURCE_TYPE_NET_EGRESS, + "meter_application_net_bytes_received": METER_RESOURCE_TYPE_NET_INGRESS, + "meter_application_pvc_bytes_total": METER_RESOURCE_TYPE_PVC, + "meter_workload_cpu_usage": METER_RESOURCE_TYPE_CPU, + "meter_workload_memory_usage_wo_cache": METER_RESOURCE_TYPE_MEM, + "meter_workload_net_bytes_transmitted": METER_RESOURCE_TYPE_NET_EGRESS, + "meter_workload_net_bytes_received": METER_RESOURCE_TYPE_NET_INGRESS, + "meter_workload_pvc_bytes_total": METER_RESOURCE_TYPE_PVC, + "meter_service_cpu_usage": METER_RESOURCE_TYPE_CPU, + "meter_service_memory_usage_wo_cache": METER_RESOURCE_TYPE_MEM, + "meter_service_net_bytes_transmitted": METER_RESOURCE_TYPE_NET_EGRESS, + "meter_service_net_bytes_received": METER_RESOURCE_TYPE_NET_INGRESS, + "meter_pod_cpu_usage": METER_RESOURCE_TYPE_CPU, + "meter_pod_memory_usage_wo_cache": METER_RESOURCE_TYPE_MEM, + "meter_pod_net_bytes_transmitted": METER_RESOURCE_TYPE_NET_EGRESS, + "meter_pod_net_bytes_received": METER_RESOURCE_TYPE_NET_INGRESS, + "meter_pod_pvc_bytes_total": METER_RESOURCE_TYPE_PVC, +} + +type PriceInfo struct { + CpuPerCorePerHour float64 `json:"cpuPerCorePerHour" yaml:"cpuPerCorePerHour"` + MemPerGigabytesPerHour float64 `json:"memPerGigabytesPerHour" yaml:"memPerGigabytesPerHour"` + IngressNetworkTrafficPerGiagabytesPerHour float64 `json:"ingressNetworkTrafficPerGiagabytesPerHour" yaml:"ingressNetworkTrafficPerGiagabytesPerHour"` + EgressNetworkTrafficPerGigabytesPerHour float64 `json:"egressNetworkTrafficPerGigabytesPerHour" yaml:"egressNetworkTrafficPerGigabytesPerHour"` + PvcPerGigabytesPerHour float64 `json:"pvcPerGigabytesPerHour" yaml:"pvcPerGigabytesPerHour"` +} + +type Billing struct { + PriceInfo PriceInfo `json:"priceInfo" yaml:"priceInfo"` +} + +type MeterConfig struct { + Billing Billing `json:"billing" yaml:"billing"` +} + +func (mc MeterConfig) GetPriceInfo() PriceInfo { + return mc.Billing.PriceInfo +} + +func LoadYaml() (*MeterConfig, error) { + + var meterConfig MeterConfig + + mf, err := os.Open(meteringConfig) + if err != nil { + klog.Error(err) + return nil, err + } + + if err = yaml.NewYAMLOrJSONDecoder(mf, 1024).Decode(&meterConfig); err != nil { + klog.Error(err) + return nil, err + } + + return &meterConfig, nil +} + +func getMaxPointValue(points []monitoring.Point) float64 { + var max float64 + for i, p := range points { + if i == 0 { + max = p.Value() + } + + if p.Value() > max { + max = p.Value() + } + } + + return max +} + +func getMinPointValue(points []monitoring.Point) float64 { + var min float64 + for i, p := range points { + if i == 0 { + min = p.Value() + } + + if p.Value() < min { + min = p.Value() + } + } + + return min +} + +func getSumPointValue(points []monitoring.Point) float64 { + avg := 0.0 + + for _, p := range points { + avg += p.Value() + } + + return avg +} + +func getAvgPointValue(points []monitoring.Point) float64 { + return getSumPointValue(points) / float64(len(points)) +} + +func getFeeWithMeterName(meterName string, sum float64) float64 { + + meterConfig, err := LoadYaml() + if err != nil { + klog.Error(err) + return -1 + } + priceInfo := meterConfig.GetPriceInfo() + + if resourceType, ok := MeterResourceMap[meterName]; !ok { + klog.Errorf("invlaid meter %v", meterName) + return -1 + } else { + switch resourceType { + case METER_RESOURCE_TYPE_CPU: + // unit: core, precision: 0.001 + sum = math.Round(sum*1000) / 1000 + return priceInfo.CpuPerCorePerHour * sum + case METER_RESOURCE_TYPE_MEM: + // unit: Gigabyte, precision: 0.1 + sum = math.Round(sum/1073741824*10) / 10 + return priceInfo.MemPerGigabytesPerHour * sum + case METER_RESOURCE_TYPE_NET_INGRESS: + // unit: Megabyte, precision: 1 + sum = math.Round(sum / 1048576) + return priceInfo.IngressNetworkTrafficPerGiagabytesPerHour * sum + case METER_RESOURCE_TYPE_NET_EGRESS: + // unit: Megabyte, precision: + sum = math.Round(sum / 1048576) + return priceInfo.EgressNetworkTrafficPerGigabytesPerHour * sum + case METER_RESOURCE_TYPE_PVC: + // unit: Gigabyte, precision: 0.1 + sum = math.Round(sum/1073741824*10) / 10 + return priceInfo.PvcPerGigabytesPerHour * sum + } + + return -1 + } +} + +func updateMetricStatData(metric monitoring.Metric, scalingMap map[string]float64) monitoring.MetricData { + metricName := metric.MetricName + metricData := metric.MetricData + for index, metricValue := range metricData.MetricValues { + + var points []monitoring.Point + if metricData.MetricType == monitoring.MetricTypeMatrix { + points = metricValue.Series + } else { + points = append(points, *metricValue.Sample) + } + + var factor float64 = 1 + if scalingMap != nil { + factor = scalingMap[metricName] + } + + if len(points) == 1 { + sample := points[0] + sum := sample[1] * factor + metricData.MetricValues[index].MinValue = sample[1] + metricData.MetricValues[index].MaxValue = sample[1] + metricData.MetricValues[index].AvgValue = sample[1] + metricData.MetricValues[index].SumValue = sum + metricData.MetricValues[index].Fee = getFeeWithMeterName(metricName, sum) + } else { + sum := getSumPointValue(points) * factor + metricData.MetricValues[index].MinValue = getMinPointValue(points) + metricData.MetricValues[index].MaxValue = getMaxPointValue(points) + metricData.MetricValues[index].AvgValue = getAvgPointValue(points) + metricData.MetricValues[index].SumValue = sum + metricData.MetricValues[index].Fee = getFeeWithMeterName(metricName, sum) + } + + } + return metricData +} diff --git a/pkg/models/tenant/metering.go b/pkg/models/tenant/metering.go new file mode 100644 index 000000000..0094362d5 --- /dev/null +++ b/pkg/models/tenant/metering.go @@ -0,0 +1,1117 @@ +package tenant + +import ( + "context" + "fmt" + "kubesphere.io/kubesphere/pkg/constants" + "kubesphere.io/kubesphere/pkg/models/metering" + "regexp" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + appv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/api" + meteringv1alpha1 "kubesphere.io/kubesphere/pkg/api/metering/v1alpha1" + tenantv1alpha2 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha2" + "kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer" + "kubesphere.io/kubesphere/pkg/apiserver/query" + "kubesphere.io/kubesphere/pkg/apiserver/request" + monitoringmodel "kubesphere.io/kubesphere/pkg/models/monitoring" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" +) + +type QueryOptions struct { + MetricFilter string + NamedMetrics []string + + Start time.Time + End time.Time + Time time.Time + Step time.Duration + + Target string + Identifier string + Order string + Page int + Limit int + + Option monitoring.QueryOption +} + +func (q QueryOptions) isRangeQuery() bool { + return q.Time.IsZero() +} + +func (q QueryOptions) shouldSort() bool { + return q.Target != "" && q.Identifier != "" +} + +func (t *tenantOperator) makeQueryOptions(user user.Info, q meteringv1alpha1.Query, lvl monitoring.Level) (qo QueryOptions, err error) { + if q.ResourceFilter == "" { + q.ResourceFilter = meteringv1alpha1.DefaultFilter + } + + qo.MetricFilter = q.MetricFilter + if q.MetricFilter == "" { + qo.MetricFilter = meteringv1alpha1.DefaultFilter + } + + var decision authorizer.Decision + switch lvl { + case monitoring.LevelCluster: + clusterOption := monitoring.ClusterOption{} + qo.NamedMetrics = monitoringmodel.ClusterMetrics + + listPods := authorizer.AttributesRecord{ + User: user, + Verb: "list", + APIGroup: "", + APIVersion: "v1", + Resource: "pods", + ResourceRequest: true, + ResourceScope: request.ClusterScope, + } + decision, _, err = t.authorizer.Authorize(listPods) + if err != nil { + klog.Error(err) + return + } + + // only cluster admin is allowed + if decision != authorizer.DecisionAllow { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrScopeNotAllowed, request.ClusterScope)) + } + qo.Option = clusterOption + + case monitoring.LevelNode: + qo.Identifier = monitoringmodel.IdentifierNode + nodeOption := monitoring.NodeOption{ + ResourceFilter: q.ResourceFilter, + NodeName: q.NodeName, + PVCFilter: q.PVCFilter, + StorageClassName: q.StorageClassName, + } + qo.NamedMetrics = monitoringmodel.NodeMetrics + + listPods := authorizer.AttributesRecord{ + User: user, + Verb: "list", + APIGroup: "", + APIVersion: "v1", + Resource: "pods", + ResourceRequest: true, + ResourceScope: request.ClusterScope, + } + decision, _, err = t.authorizer.Authorize(listPods) + if err != nil { + klog.Error(err) + return + } + + // only cluster admin is allowed + if decision != authorizer.DecisionAllow { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrScopeNotAllowed, request.ClusterScope)) + } + qo.Option = nodeOption + + case monitoring.LevelWorkspace: + qo.Identifier = monitoringmodel.IdentifierWorkspace + + // at least one of WorkspaceName, ResourceFilter isn't empty + wsOption := monitoring.WorkspaceOption{ + ResourceFilter: q.ResourceFilter, // ws filter + WorkspaceName: q.WorkspaceName, + PVCFilter: q.PVCFilter, + StorageClassName: q.StorageClassName, + } + qo.NamedMetrics = monitoringmodel.WorkspaceMetrics + + wsScope := request.ClusterScope + if q.WorkspaceName != "" { + wsScope = request.WorkspaceScope + } + + listPods := authorizer.AttributesRecord{ + User: user, + Verb: "list", + Resource: "pods", + Workspace: q.WorkspaceName, + ResourceScope: wsScope, + ResourceRequest: true, + } + decision, _, err = t.authorizer.Authorize(listPods) + if err != nil { + klog.Error(err) + return + } + if decision != authorizer.DecisionAllow { + // specified by WorkspaceName and not allowed + if q.WorkspaceName != "" { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrScopeNotAllowed, wsScope)) + } + + // not specified by ResourceFilter or WorkspaceName + if q.ResourceFilter == "" { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrScopeNotAllowed, wsScope)) + } + } + + // apply ResourceFilter if necessary + if q.ResourceFilter != "" { + var wsList *api.ListResult + qu := query.New() + qu.LabelSelector = q.LabelSelector + wsList, err = t.ListWorkspaces(user, qu) + if err != nil { + return qo, err + } + + targetWs := []string{} + for _, item := range wsList.Items { + ws := item.(*tenantv1alpha2.WorkspaceTemplate) + if ok, _ := regexp.MatchString(q.ResourceFilter, ws.ObjectMeta.GetName()); ok { + listPods = authorizer.AttributesRecord{ + User: user, + Verb: "list", + Resource: "pods", + Workspace: ws.ObjectMeta.GetName(), + ResourceScope: request.WorkspaceScope, + ResourceRequest: true, + } + decision, _, err = t.authorizer.Authorize(listPods) + if err != nil { + klog.Error(err) + return + } + if decision == authorizer.DecisionAllow { + targetWs = append(targetWs, ws.ObjectMeta.GetName()) + } + } + } + wsOption.ResourceFilter = strings.Join(targetWs, "|") + } + + qo.Option = wsOption + + case monitoring.LevelNamespace: + qo.Identifier = monitoringmodel.IdentifierNamespace + nsOption := monitoring.NamespaceOption{ + ResourceFilter: q.ResourceFilter, // ns filter + WorkspaceName: q.WorkspaceName, + NamespaceName: q.NamespaceName, + PVCFilter: q.PVCFilter, + StorageClassName: q.StorageClassName, + } + qo.NamedMetrics = monitoringmodel.NamespaceMetrics + + nsScope := request.ClusterScope + if q.WorkspaceName != "" { + nsScope = request.WorkspaceScope + } + if q.NamespaceName != "" { + nsScope = request.NamespaceScope + } + + listPods := authorizer.AttributesRecord{ + User: user, + Verb: "list", + APIGroup: "", + APIVersion: "v1", + Resource: "pods", + ResourceRequest: true, + Workspace: q.WorkspaceName, + Namespace: q.NamespaceName, + ResourceScope: nsScope, + } + decision, _, err = t.authorizer.Authorize(listPods) + if err != nil { + klog.Error(err) + return + } + if decision != authorizer.DecisionAllow { + if q.WorkspaceName != "" { + // specified by WorkspaceName & NamespaceName and not allowed + if q.NamespaceName != "" { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrScopeNotAllowed, nsScope)) + } + } else { + // specified by NamespaceName & NamespaceName and not allowed + if q.NamespaceName != "" { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrScopeNotAllowed, nsScope)) + } + } + + if q.ResourceFilter == "" { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrScopeNotAllowed, nsScope)) + } + } + + if q.NamespaceName != "" { + nsOption.ResourceFilter = q.NamespaceName + } else { + var nsList *api.ListResult + qu := query.New() + qu.LabelSelector = q.LabelSelector + nsList, err = t.ListNamespaces(user, q.WorkspaceName, qu) + if err != nil { + return qo, err + } + + targetNs := []string{} + for _, item := range nsList.Items { + ns := item.(*corev1.Namespace) + if ok, _ := regexp.MatchString(q.ResourceFilter, ns.ObjectMeta.GetName()); ok { + listPods = authorizer.AttributesRecord{ + User: user, + Verb: "list", + Resource: "pods", + Namespace: ns.ObjectMeta.GetName(), + ResourceScope: request.NamespaceScope, + ResourceRequest: true, + } + decision, _, err = t.authorizer.Authorize(listPods) + if err != nil { + klog.Error(err) + return + } + if decision == authorizer.DecisionAllow { + targetNs = append(targetNs, ns.ObjectMeta.GetName()) + } + } + } + nsOption.ResourceFilter = strings.Join(targetNs, "|") + } + + qo.Option = nsOption + + case monitoring.LevelApplication: + qo.Identifier = monitoringmodel.IdentifierApplication + if q.NamespaceName == "" { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrParameterNotfound, "namespace")) + } + + appScope := request.NamespaceScope + + listPods := authorizer.AttributesRecord{ + User: user, + Verb: "list", + APIGroup: "", + APIVersion: "v1", + Resource: "pods", + ResourceRequest: true, + Namespace: q.NamespaceName, + ResourceScope: appScope, + } + decision, _, err = t.authorizer.Authorize(listPods) + if err != nil { + klog.Error(err) + return + } + if decision != authorizer.DecisionAllow { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrScopeNotAllowed, appScope)) + } + + qo.Option = monitoring.ApplicationsOption{ + NamespaceName: q.NamespaceName, + Applications: strings.Split(q.Applications, "|"), + StorageClassName: q.StorageClassName, + } + qo.NamedMetrics = monitoringmodel.ApplicationMetrics + + case monitoring.LevelWorkload: + qo.Identifier = monitoringmodel.IdentifierWorkload + if q.NamespaceName == "" { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrParameterNotfound, "namespace")) + } + + qo.Option = monitoring.WorkloadOption{ + ResourceFilter: q.ResourceFilter, // workload filter + NamespaceName: q.NamespaceName, + WorkloadKind: q.WorkloadKind, + } + + workloadScope := request.NamespaceScope + + listPods := authorizer.AttributesRecord{ + User: user, + Verb: "list", + APIGroup: "", + APIVersion: "v1", + Resource: "pods", + ResourceRequest: true, + Namespace: q.NamespaceName, + ResourceScope: workloadScope, + } + decision, _, err = t.authorizer.Authorize(listPods) + if err != nil { + klog.Error(err) + return + } + if decision != authorizer.DecisionAllow { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrScopeNotAllowed, workloadScope)) + } + + qo.NamedMetrics = monitoringmodel.WorkloadMetrics + + case monitoring.LevelPod: + qo.Identifier = monitoringmodel.IdentifierPod + qo.Option = monitoring.PodOption{ + ResourceFilter: q.ResourceFilter, + NodeName: q.NodeName, + NamespaceName: q.NamespaceName, + WorkloadKind: q.WorkloadKind, + WorkloadName: q.WorkloadName, + PodName: q.PodName, + } + + podScope := request.NamespaceScope + + listPods := authorizer.AttributesRecord{ + User: user, + Verb: "list", + APIGroup: "", + APIVersion: "v1", + Resource: "pods", + ResourceRequest: true, + Namespace: q.NamespaceName, + ResourceScope: podScope, + } + decision, _, err = t.authorizer.Authorize(listPods) + if err != nil { + klog.Error(err) + return + } + if decision != authorizer.DecisionAllow { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrScopeNotAllowed, podScope)) + } + + qo.NamedMetrics = monitoringmodel.PodMetrics + + case monitoring.LevelService: + qo.Identifier = monitoringmodel.IdentifierService + qo.Option = monitoring.ServicesOption{ + NamespaceName: q.NamespaceName, + Services: strings.Split(q.Services, "|"), + } + + if q.NamespaceName == "" { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrParameterNotfound, "namespace")) + } + serviceScope := request.NamespaceScope + + listPods := authorizer.AttributesRecord{ + User: user, + Verb: "list", + APIGroup: "", + APIVersion: "v1", + Resource: "pods", + ResourceRequest: true, + Namespace: q.NamespaceName, + ResourceScope: serviceScope, + } + decision, _, err = t.authorizer.Authorize(listPods) + if err != nil { + klog.Error(err) + return + } + if decision != authorizer.DecisionAllow { + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrScopeNotAllowed, serviceScope)) + } + + // TODO: list services if q.Services are empty + + qo.NamedMetrics = monitoringmodel.ServiceMetrics + + default: + return qo, errors.New(fmt.Sprintf(meteringv1alpha1.ErrParameterNotfound, "level")) + } + + // Parse time params + if q.Start != "" && q.End != "" { + startInt, err := strconv.ParseInt(q.Start, 10, 64) + if err != nil { + return qo, err + } + qo.Start = time.Unix(startInt, 0) + + endInt, err := strconv.ParseInt(q.End, 10, 64) + if err != nil { + return qo, err + } + qo.End = time.Unix(endInt, 0) + + if q.Step == "" { + qo.Step = meteringv1alpha1.DefaultStep + } else { + qo.Step, err = time.ParseDuration(q.Step) + if err != nil { + return qo, err + } + } + + if qo.Start.After(qo.End) { + return qo, errors.New(meteringv1alpha1.ErrInvalidStartEnd) + } + } else if q.Start == "" && q.End == "" { + if q.Time == "" { + qo.Time = time.Now() + } else { + timeInt, err := strconv.ParseInt(q.Time, 10, 64) + if err != nil { + return qo, err + } + qo.Time = time.Unix(timeInt, 0) + } + } else { + return qo, errors.Errorf(meteringv1alpha1.ErrParamConflict) + } + + if q.NamespaceName != "" { + + queryParameter := query.New() + queryParameter.Filters[query.FieldName] = query.Value(q.NamespaceName) + + listResult, err := t.ListNamespaces(user, q.WorkspaceName, queryParameter) + if err != nil { + return qo, err + } + if listResult.TotalItems == 0 { + return qo, errors.New(meteringv1alpha1.ErrResourceNotfound) + } + ns := listResult.Items[0].(*corev1.Namespace) + cts := ns.CreationTimestamp.Time + + // Query should happen no earlier than namespace's creation time. + // For range query, check and mutate `start`. For instant query, check `time`. + // In range query, if `start` and `end` are both before namespace's creation time, it causes no hit. + if !qo.isRangeQuery() { + if qo.Time.Before(cts) { + return qo, errors.New(meteringv1alpha1.ErrNoHit) + } + } else { + if qo.End.Before(cts) { + return qo, errors.New(meteringv1alpha1.ErrNoHit) + } + if qo.Start.Before(cts) { + qo.Start = qo.End + for qo.Start.Add(-qo.Step).After(cts) { + qo.Start = qo.Start.Add(-qo.Step) + } + } + } + } + + // Parse sorting and paging params + if q.Target != "" { + qo.Target = q.Target + qo.Page = meteringv1alpha1.DefaultPage + qo.Limit = meteringv1alpha1.DefaultLimit + qo.Order = q.Order + if q.Order != monitoringmodel.OrderAscending { + qo.Order = meteringv1alpha1.DefaultOrder + } + if q.Page != "" { + qo.Page, err = strconv.Atoi(q.Page) + if err != nil || qo.Page <= 0 { + return qo, errors.New(meteringv1alpha1.ErrInvalidPage) + } + } + if q.Limit != "" { + qo.Limit, err = strconv.Atoi(q.Limit) + if err != nil || qo.Limit <= 0 { + return qo, errors.New(meteringv1alpha1.ErrInvalidLimit) + } + } + } + + return qo, nil +} + +func (t *tenantOperator) ProcessNamedMetersQuery(q QueryOptions) (metrics monitoringmodel.Metrics, err error) { + var meters []string + for _, meter := range q.NamedMetrics { + if !strings.HasPrefix(meter, monitoringmodel.MetricMeterPrefix) { + // skip non-meter metric + continue + } + + ok, _ := regexp.MatchString(q.MetricFilter, meter) + if ok { + meters = append(meters, meter) + } + } + + if len(meters) == 0 { + klog.Info("no meters found") + return + } + + _, ok := q.Option.(monitoring.ApplicationsOption) + if ok { + metrics, err = t.processApplicationMetersQuery(meters, q) + return + } + + _, ok = q.Option.(monitoring.ServicesOption) + if ok { + metrics, err = t.processServiceMetersQuery(meters, q) + return + } + + if q.isRangeQuery() { + metrics, err = t.mo.GetNamedMetersOverTime(meters, q.Start, q.End, q.Step, q.Option) + } else { + metrics, err = t.mo.GetNamedMeters(meters, q.Time, q.Option) + if q.shouldSort() { + metrics = *metrics.Sort(q.Target, q.Order, q.Identifier).Page(q.Page, q.Limit) + } + } + + return +} + +func getMetricPosMap(metrics []monitoring.Metric) map[string]int { + var metricMap = make(map[string]int) + + for i, m := range metrics { + metricMap[m.MetricName] = i + } + + return metricMap +} + +func (t *tenantOperator) processApplicationMetersQuery(meters []string, q QueryOptions) (res monitoringmodel.Metrics, err error) { + var metricMap = make(map[string]int) + var current_res monitoringmodel.Metrics + + aso, ok := q.Option.(monitoring.ApplicationsOption) + if !ok { + err = errors.New("invalid application option") + klog.Error(err.Error()) + return + } + componentsMap := t.mo.GetAppComponentsMap(aso.NamespaceName, aso.Applications) + + for k, _ := range componentsMap { + opt := monitoring.ApplicationOption{ + NamespaceName: aso.NamespaceName, + Application: k, + ApplicationComponents: componentsMap[k], + StorageClassName: aso.StorageClassName, + } + + if q.isRangeQuery() { + current_res, err = t.mo.GetNamedMetersOverTime(meters, q.Start, q.End, q.Step, opt) + } else { + current_res, err = t.mo.GetNamedMeters(meters, q.Time, opt) + } + + if res.Results == nil { + res = current_res + metricMap = getMetricPosMap(res.Results) + } else { + for _, cur_res := range current_res.Results { + pos, ok := metricMap[cur_res.MetricName] + if ok { + res.Results[pos].MetricValues = append(res.Results[pos].MetricValues, cur_res.MetricValues...) + } else { + res.Results = append(res.Results, cur_res) + } + } + } + } + + if !q.isRangeQuery() && q.shouldSort() { + res = *res.Sort(q.Target, q.Order, q.Identifier).Page(q.Page, q.Limit) + } + + return +} + +func (t *tenantOperator) processServiceMetersQuery(meters []string, q QueryOptions) (res monitoringmodel.Metrics, err error) { + var metricMap = make(map[string]int) + var current_res monitoringmodel.Metrics + + sso, ok := q.Option.(monitoring.ServicesOption) + if !ok { + err = errors.New("invalid service option") + klog.Error(err.Error()) + return + } + svcPodsMap := t.mo.GetSerivePodsMap(sso.NamespaceName, sso.Services) + + for k, _ := range svcPodsMap { + opt := monitoring.ServiceOption{ + NamespaceName: sso.NamespaceName, + ServiceName: k, + PodNames: svcPodsMap[k], + } + + if q.isRangeQuery() { + current_res, err = t.mo.GetNamedMetersOverTime(meters, q.Start, q.End, q.Step, opt) + } else { + current_res, err = t.mo.GetNamedMeters(meters, q.Time, opt) + } + + if res.Results == nil { + res = current_res + metricMap = getMetricPosMap(res.Results) + } else { + for _, cur_res := range current_res.Results { + pos, ok := metricMap[cur_res.MetricName] + if ok { + res.Results[pos].MetricValues = append(res.Results[pos].MetricValues, cur_res.MetricValues...) + } else { + res.Results = append(res.Results, cur_res) + } + } + } + } + + if !q.isRangeQuery() && q.shouldSort() { + res = *res.Sort(q.Target, q.Order, q.Identifier).Page(q.Page, q.Limit) + } + + return +} + +func (t *tenantOperator) transformMetricData(metrics monitoringmodel.Metrics) metering.PodsStats { + podsStats := make(metering.PodsStats) + + for _, metric := range metrics.Results { + metricName := metric.MetricName + for _, metricValue := range metric.MetricValues { + //metricValue.SumValue + podName := metricValue.Metadata["pod"] + podsStats.Set(podName, metricName, metricValue.SumValue) + } + } + + return podsStats +} + +func (t *tenantOperator) classifyPodStats(user user.Info, ns string, podsStats metering.PodsStats) (resourceStats metering.ResourceStatistic, err error) { + + if err = t.updateServicesStats(user, ns, podsStats, &resourceStats); err != nil { + return + } + + if err = t.updateDeploysStats(user, ns, podsStats, &resourceStats); err != nil { + return + } + + if err = t.updateDaemonsetsStats(user, ns, podsStats, &resourceStats); err != nil { + return + } + + if err = t.updateStatefulsetsStats(user, ns, podsStats, &resourceStats); err != nil { + return + } + + return +} + +func (t *tenantOperator) updateServicesStats(user user.Info, ns string, podsStats metering.PodsStats, resourceStats *metering.ResourceStatistic) error { + + svcList, err := t.listServices(user, ns) + if err != nil { + return err + } + + for _, svc := range svcList.Items { + if svc.Annotations[constants.ApplicationReleaseName] != "" && + svc.Annotations[constants.ApplicationReleaseNS] != "" && + t.isOpNamespace(ns) { + // for op svc + // currently we do NOT include op svc + continue + } else { + appName, nameOK := svc.Labels[constants.ApplicationName] + appVersion, versionOK := svc.Labels[constants.ApplicationVersion] + + svcPodsMap := t.mo.GetSerivePodsMap(ns, []string{svc.Name}) + pods := svcPodsMap[svc.Name] + + if nameOK && versionOK { + // for app crd svc + for _, pod := range pods { + podStat := podsStats[pod] + if podStat == nil { + klog.Warningf("%v not found", pod) + continue + } + + appFullName := appName + ":" + appVersion + if err := resourceStats.GetAppStats(appFullName).GetServiceStats(svc.Name).SetPodStats(pod, podsStats[pod]); err != nil { + klog.Error(err) + return err + } + } + } else { + // for k8s svc + for _, pod := range pods { + if err := resourceStats.GetServiceStats(svc.Name).SetPodStats(pod, podsStats[pod]); err != nil { + klog.Error(err) + return err + } + } + } + } + } + + // aggregate svc data + for _, app := range resourceStats.Apps { + for _, svc := range app.Services { + svc.Aggregate() + } + app.Aggregate() + } + + for _, svc := range resourceStats.Services { + svc.Aggregate() + } + + return nil +} + +func (t *tenantOperator) listServices(user user.Info, ns string) (*corev1.ServiceList, error) { + + svcScope := request.NamespaceScope + + listSvc := authorizer.AttributesRecord{ + User: user, + Verb: "list", + Resource: "services", + Namespace: ns, + ResourceRequest: true, + ResourceScope: svcScope, + } + + decision, _, err := t.authorizer.Authorize(listSvc) + if err != nil { + klog.Error(err) + return nil, err + } + + if decision != authorizer.DecisionAllow { + _, err := t.am.ListRoleBindings(user.GetName(), nil, ns) + if err != nil { + klog.Error(err) + return nil, err + } + } + + svcs, err := t.k8sclient.CoreV1().Services(ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + return svcs, nil +} + +func (t *tenantOperator) updateDeploysStats(user user.Info, ns string, podsStats metering.PodsStats, resourceStats *metering.ResourceStatistic) error { + deployList, err := t.listDeploys(user, ns) + if err != nil { + return err + } + + for _, deploy := range deployList.Items { + + if deploy.Annotations[constants.ApplicationReleaseName] != "" && + deploy.Annotations[constants.ApplicationReleaseNS] != "" && + t.isOpNamespace(ns) { + // for op deploy + // currently we do NOT include op deploy + continue + } else { + _, appNameOK := deploy.Labels[constants.ApplicationName] + _, appVersionOK := deploy.Labels[constants.ApplicationVersion] + + pods, err := t.listPods(user, ns, deploy.Spec.Selector) + if err != nil { + klog.Error(err) + return err + } + + if appNameOK && appVersionOK { + // for app crd svc + continue + } else { + // for k8s svc + for _, pod := range pods { + if err := resourceStats.GetDeployStats(deploy.Name).SetPodStats(pod, podsStats[pod]); err != nil { + klog.Error(err) + return err + } + } + } + } + } + + for _, deploy := range resourceStats.Deploys { + deploy.Aggregate() + } + return nil +} + +func (t *tenantOperator) updateDaemonsetsStats(user user.Info, ns string, podsStats metering.PodsStats, resourceStats *metering.ResourceStatistic) error { + daemonsetList, err := t.listDaemonsets(user, ns) + if err != nil { + return err + } + + for _, daemonset := range daemonsetList.Items { + + if daemonset.Annotations["meta.helm.sh/release-name"] != "" && + daemonset.Annotations["meta.helm.sh/release-namespace"] != "" && + t.isOpNamespace(ns) { + // for op deploy + // currently we do NOT include op deploy + continue + } else { + appName := daemonset.Labels[constants.ApplicationName] + appVersion := daemonset.Labels[constants.ApplicationVersion] + + pods, err := t.listPods(user, ns, daemonset.Spec.Selector) + if err != nil { + klog.Error(err) + return err + } + + if appName != "" && appVersion != "" { + // for app crd svc + continue + } else { + // for k8s svc + for _, pod := range pods { + if err := resourceStats.GetDaemonsetStats(daemonset.Name).SetPodStats(pod, podsStats[pod]); err != nil { + klog.Error(err) + return err + } + } + } + } + } + + for _, daemonset := range resourceStats.Daemonsets { + daemonset.Aggregate() + } + return nil +} + +func (t *tenantOperator) isOpNamespace(ns string) bool { + + nsObj, err := t.k8sclient.CoreV1().Namespaces().Get(context.Background(), ns, metav1.GetOptions{}) + if err != nil { + return false + } + + ws := nsObj.Labels[constants.WorkspaceLabelKey] + + if len(ws) != 0 && ws != "system-workspace" { + return true + } + return false +} + +func (t *tenantOperator) updateStatefulsetsStats(user user.Info, ns string, podsStats metering.PodsStats, resourceStats *metering.ResourceStatistic) error { + statefulsetsList, err := t.listStatefulsets(user, ns) + if err != nil { + return err + } + + for _, statefulset := range statefulsetsList.Items { + + if statefulset.Annotations[constants.ApplicationReleaseName] != "" && + statefulset.Annotations[constants.ApplicationReleaseNS] != "" && + t.isOpNamespace(ns) { + // for op deploy + // currently we do NOT include op deploy + continue + } else { + appName := statefulset.Labels[constants.ApplicationName] + appVersion := statefulset.Labels[constants.ApplicationVersion] + + pods, err := t.listPods(user, ns, statefulset.Spec.Selector) + if err != nil { + klog.Error(err) + return err + } + + if appName != "" && appVersion != "" { + // for app crd svc + continue + } else { + // for k8s svc + for _, pod := range pods { + if err := resourceStats.GetStatefulsetStats(statefulset.Name).SetPodStats(pod, podsStats[pod]); err != nil { + klog.Error(err) + return err + } + } + } + } + } + + for _, statefulset := range resourceStats.Statefulsets { + statefulset.Aggregate() + } + return nil +} + +func (t *tenantOperator) listPods(user user.Info, ns string, selector *metav1.LabelSelector) ([]string, error) { + podScope := request.NamespaceScope + + listPods := authorizer.AttributesRecord{ + User: user, + Verb: "list", + Resource: "pods", + ResourceRequest: true, + Namespace: ns, + ResourceScope: podScope, + } + + decision, _, err := t.authorizer.Authorize(listPods) + if err != nil { + klog.Error(err) + return nil, err + } + + if decision != authorizer.DecisionAllow { + _, err := t.am.ListRoleBindings(user.GetName(), nil, ns) + if err != nil { + klog.Error(err) + return nil, err + } + } + + var labelFilter []string + for k, v := range selector.MatchLabels { + labelFilter = append(labelFilter, fmt.Sprintf("%v=%v", k, v)) + } + + opt := metav1.ListOptions{LabelSelector: strings.Join(labelFilter, ",")} + + pods, err := t.k8sclient.CoreV1().Pods(ns).List(context.Background(), opt) + if err != nil { + return nil, err + } + + ret := []string{} + for _, pod := range pods.Items { + ret = append(ret, pod.Name) + } + + return ret, nil +} + +func (t *tenantOperator) listDeploys(user user.Info, ns string) (*appv1.DeploymentList, error) { + + deployScope := request.NamespaceScope + + listSvc := authorizer.AttributesRecord{ + User: user, + Verb: "list", + Resource: "deployments", + ResourceRequest: true, + Namespace: ns, + ResourceScope: deployScope, + } + + decision, _, err := t.authorizer.Authorize(listSvc) + if err != nil { + klog.Error(err) + return nil, err + } + + if decision != authorizer.DecisionAllow { + _, err := t.am.ListRoleBindings(user.GetName(), nil, ns) + if err != nil { + klog.Error(err) + return nil, err + } + } + + deploys, err := t.k8sclient.AppsV1().Deployments(ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + return deploys, nil +} + +func (t *tenantOperator) listDaemonsets(user user.Info, ns string) (*appv1.DaemonSetList, error) { + + dsScope := request.NamespaceScope + + listSvc := authorizer.AttributesRecord{ + User: user, + Verb: "list", + Resource: "daemonsets", + ResourceRequest: true, + Namespace: ns, + ResourceScope: dsScope, + } + + decision, _, err := t.authorizer.Authorize(listSvc) + if err != nil { + klog.Error(err) + return nil, err + } + + if decision != authorizer.DecisionAllow { + _, err := t.am.ListRoleBindings(user.GetName(), nil, ns) + if err != nil { + klog.Error(err) + return nil, err + } + } + + ds, err := t.k8sclient.AppsV1().DaemonSets(ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + return ds, nil +} + +func (t *tenantOperator) listStatefulsets(user user.Info, ns string) (*appv1.StatefulSetList, error) { + + stsScope := request.NamespaceScope + + listSvc := authorizer.AttributesRecord{ + User: user, + Verb: "list", + Resource: "statefulsets", + Namespace: ns, + ResourceRequest: true, + ResourceScope: stsScope, + } + + decision, _, err := t.authorizer.Authorize(listSvc) + if err != nil { + klog.Error(err) + return nil, err + } + + if decision != authorizer.DecisionAllow { + _, err := t.am.ListRoleBindings(user.GetName(), nil, ns) + if err != nil { + klog.Error(err) + return nil, err + } + } + + stss, err := t.k8sclient.AppsV1().StatefulSets(ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + return stss, nil +} diff --git a/pkg/models/tenant/tenant.go b/pkg/models/tenant/tenant.go index 86e1ffd76..6a7ae1014 100644 --- a/pkg/models/tenant/tenant.go +++ b/pkg/models/tenant/tenant.go @@ -37,6 +37,7 @@ import ( auditingv1alpha1 "kubesphere.io/kubesphere/pkg/api/auditing/v1alpha1" eventsv1alpha1 "kubesphere.io/kubesphere/pkg/api/events/v1alpha1" loggingv1alpha2 "kubesphere.io/kubesphere/pkg/api/logging/v1alpha2" + meteringv1alpha1 "kubesphere.io/kubesphere/pkg/api/metering/v1alpha1" clusterv1alpha1 "kubesphere.io/kubesphere/pkg/apis/cluster/v1alpha1" tenantv1alpha1 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha1" tenantv1alpha2 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha2" @@ -50,11 +51,16 @@ import ( "kubesphere.io/kubesphere/pkg/models/events" "kubesphere.io/kubesphere/pkg/models/iam/am" "kubesphere.io/kubesphere/pkg/models/logging" + "kubesphere.io/kubesphere/pkg/models/metering" + "kubesphere.io/kubesphere/pkg/models/monitoring" resources "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3" resourcesv1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource" + resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource" auditingclient "kubesphere.io/kubesphere/pkg/simple/client/auditing" eventsclient "kubesphere.io/kubesphere/pkg/simple/client/events" loggingclient "kubesphere.io/kubesphere/pkg/simple/client/logging" + monitoringclient "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + opclient "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" "kubesphere.io/kubesphere/pkg/utils/stringutils" ) @@ -79,6 +85,8 @@ type Interface interface { PatchNamespace(workspace string, namespace *corev1.Namespace) (*corev1.Namespace, error) PatchWorkspace(workspace string, data json.RawMessage) (*tenantv1alpha2.WorkspaceTemplate, error) ListClusters(info user.Info) (*api.ListResult, error) + Metering(user user.Info, queryParam *meteringv1alpha1.Query) (monitoring.Metrics, error) + MeteringHierarchy(user user.Info, queryParam *meteringv1alpha1.Query) (metering.ResourceStatistic, error) } type tenantOperator struct { @@ -90,9 +98,10 @@ type tenantOperator struct { events events.Interface lo logging.LoggingOperator auditing auditing.Interface + mo monitoring.MonitoringOperator } -func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient eventsclient.Client, loggingClient loggingclient.Client, auditingclient auditingclient.Client, am am.AccessManagementInterface, authorizer authorizer.Authorizer) Interface { +func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient eventsclient.Client, loggingClient loggingclient.Client, auditingclient auditingclient.Client, am am.AccessManagementInterface, authorizer authorizer.Authorizer, monitoringclient monitoringclient.Interface, opClient opclient.Client, resourceGetter *resourcev1alpha3.ResourceGetter) Interface { return &tenantOperator{ am: am, authorizer: authorizer, @@ -102,6 +111,7 @@ func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ks events: events.NewEventsOperator(evtsClient), lo: logging.NewLoggingOperator(loggingClient), auditing: auditing.NewEventsOperator(auditingclient), + mo: monitoring.NewMonitoringOperator(monitoringclient, nil, k8sclient, informers, opClient, resourceGetter), } } @@ -951,6 +961,38 @@ func (t *tenantOperator) Auditing(user user.Info, queryParam *auditingv1alpha1.Q }) } +func (t *tenantOperator) Metering(user user.Info, query *meteringv1alpha1.Query) (metrics monitoring.Metrics, err error) { + + var opt QueryOptions + + opt, err = t.makeQueryOptions(user, *query, query.Level) + if err != nil { + return + } + metrics, err = t.ProcessNamedMetersQuery(opt) + + return +} + +func (t *tenantOperator) MeteringHierarchy(user user.Info, queryParam *meteringv1alpha1.Query) (metering.ResourceStatistic, error) { + res, err := t.Metering(user, queryParam) + if err != nil { + return metering.ResourceStatistic{}, err + } + + // get pods stat info under ns + podsStats := t.transformMetricData(res) + + // classify pods stats + resourceStats, err := t.classifyPodStats(user, queryParam.NamespaceName, podsStats) + if err != nil { + klog.Error(err) + return metering.ResourceStatistic{}, err + } + + return resourceStats, nil +} + func contains(objects []runtime.Object, object runtime.Object) bool { for _, item := range objects { if item == object { diff --git a/pkg/models/tenant/tenent_test.go b/pkg/models/tenant/tenent_test.go index d2a6311af..a9fad144a 100644 --- a/pkg/models/tenant/tenent_test.go +++ b/pkg/models/tenant/tenent_test.go @@ -541,5 +541,5 @@ func prepare() Interface { amOperator := am.NewOperator(ksClient, k8sClient, fakeInformerFactory) authorizer := rbac.NewRBACAuthorizer(amOperator) - return New(fakeInformerFactory, k8sClient, ksClient, nil, nil, nil, amOperator, authorizer) + return New(fakeInformerFactory, k8sClient, ksClient, nil, nil, nil, amOperator, authorizer, nil, nil, nil) } diff --git a/pkg/simple/client/monitoring/interface.go b/pkg/simple/client/monitoring/interface.go index 800033265..1e82eb221 100644 --- a/pkg/simple/client/monitoring/interface.go +++ b/pkg/simple/client/monitoring/interface.go @@ -25,4 +25,8 @@ type Interface interface { GetNamedMetricsOverTime(metrics []string, start, end time.Time, step time.Duration, opt QueryOption) []Metric GetMetadata(namespace string) []Metadata GetMetricLabelSet(expr string, start, end time.Time) []map[string]string + + // meter + GetNamedMeters(meters []string, time time.Time, opts []QueryOption) []Metric + GetNamedMetersOverTime(metrics []string, start, end time.Time, step time.Duration, opts []QueryOption) []Metric } diff --git a/pkg/simple/client/monitoring/metricsserver/metricsserver.go b/pkg/simple/client/monitoring/metricsserver/metricsserver.go index b7efbb2c2..7096a235d 100644 --- a/pkg/simple/client/monitoring/metricsserver/metricsserver.go +++ b/pkg/simple/client/monitoring/metricsserver/metricsserver.go @@ -445,3 +445,11 @@ func (m metricsServer) GetMetricLabelSet(expr string, start, end time.Time) []ma return res } + +// meter +func (m metricsServer) GetNamedMeters(meters []string, time time.Time, opts []monitoring.QueryOption) []monitoring.Metric { + return nil +} +func (m metricsServer) GetNamedMetersOverTime(metrics []string, start, end time.Time, step time.Duration, opts []monitoring.QueryOption) []monitoring.Metric { + return nil +} diff --git a/pkg/simple/client/monitoring/prometheus/prometheus.go b/pkg/simple/client/monitoring/prometheus/prometheus.go index 7bd841113..d02ba509b 100644 --- a/pkg/simple/client/monitoring/prometheus/prometheus.go +++ b/pkg/simple/client/monitoring/prometheus/prometheus.go @@ -30,6 +30,8 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/monitoring" ) +const MeteringDefaultTimeout = 20 * time.Second + // prometheus implements monitoring interface backed by Prometheus type prometheus struct { client apiv1.API @@ -147,6 +149,108 @@ func (p prometheus) GetNamedMetricsOverTime(metrics []string, start, end time.Ti return res } +func (p prometheus) GetNamedMeters(meters []string, ts time.Time, opts []monitoring.QueryOption) []monitoring.Metric { + var res []monitoring.Metric + var wg sync.WaitGroup + var mtx sync.Mutex + + queryOptions := monitoring.NewQueryOptions() + + for _, opt := range opts { + opt.Apply(queryOptions) + } + + prometheusCtx, cancel := context.WithTimeout(context.Background(), MeteringDefaultTimeout) + defer cancel() + + for _, meter := range meters { + + wg.Add(1) + + go func(metric string) { + parsedResp := monitoring.Metric{MetricName: metric} + + begin := time.Now() + value, _, err := p.client.Query(prometheusCtx, makeMeterExpr(metric, *queryOptions), ts) + end := time.Now() + timeElapsed := end.Unix() - begin.Unix() + if timeElapsed > int64(MeteringDefaultTimeout.Seconds())/2 { + klog.Warningf("long time query[cost %v seconds], expr: %v", timeElapsed, makeMeterExpr(metric, *queryOptions)) + } + + if err != nil { + parsedResp.Error = err.Error() + } else { + parsedResp.MetricData = parseQueryResp(value, nil) + } + + mtx.Lock() + res = append(res, parsedResp) + mtx.Unlock() + + wg.Done() + }(meter) + + } + + wg.Wait() + + return res +} + +func (p prometheus) GetNamedMetersOverTime(meters []string, start, end time.Time, step time.Duration, opts []monitoring.QueryOption) []monitoring.Metric { + var res []monitoring.Metric + var wg sync.WaitGroup + var mtx sync.Mutex + + queryOptions := monitoring.NewQueryOptions() + + for _, opt := range opts { + opt.Apply(queryOptions) + } + + timeRange := apiv1.Range{ + Start: start, + End: end, + Step: step, + } + + prometheusCtx, cancel := context.WithTimeout(context.Background(), MeteringDefaultTimeout) + defer cancel() + + for _, meter := range meters { + + wg.Add(1) + + go func(metric string) { + parsedResp := monitoring.Metric{MetricName: metric} + begin := time.Now() + value, _, err := p.client.QueryRange(prometheusCtx, makeMeterExpr(metric, *queryOptions), timeRange) + end := time.Now() + timeElapsed := end.Unix() - begin.Unix() + if timeElapsed > int64(MeteringDefaultTimeout.Seconds())/2 { + klog.Warningf("long time query[cost %v seconds], expr: %v", timeElapsed, makeMeterExpr(metric, *queryOptions)) + } + + if err != nil { + parsedResp.Error = err.Error() + } else { + parsedResp.MetricData = parseQueryRangeResp(value, nil) + } + + mtx.Lock() + res = append(res, parsedResp) + mtx.Unlock() + + wg.Done() + }(meter) + } + + wg.Wait() + + return res +} + func (p prometheus) GetMetadata(namespace string) []monitoring.Metadata { var meta []monitoring.Metadata var matchTarget string diff --git a/pkg/simple/client/monitoring/prometheus/promql_meter.go b/pkg/simple/client/monitoring/prometheus/promql_meter.go new file mode 100644 index 000000000..8eab0453f --- /dev/null +++ b/pkg/simple/client/monitoring/prometheus/promql_meter.go @@ -0,0 +1,1304 @@ +/* +Copyright 2019 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "fmt" + "strconv" + "strings" + + "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/simple/client/monitoring" +) + +var promQLMeterTemplates = map[string]string{ + // cluster + "meter_cluster_cpu_usage": ` +round( + ( + sum( + avg_over_time(kube_pod_container_resource_requests{resource="cpu",unit="core"}[$step]) + ) >= + ( + avg_over_time(:node_cpu_utilisation:avg1m[$step]) * + sum( + avg_over_time(node:node_num_cpu:sum[$step]) + ) + ) + ) + or + ( + ( + avg_over_time(:node_cpu_utilisation:avg1m[$step]) * + sum( + avg_over_time(node:node_num_cpu:sum[$step]) + ) + ) > + sum( + avg_over_time(kube_pod_container_resource_requests{resource="cpu",unit="core"}[$step]) + ) + ), + 0.001 +)`, + + "meter_cluster_memory_usage": ` +round( + ( + sum( + avg_over_time(kube_pod_container_resource_requests{resource="memory",unit="byte"}[$step]) + ) >= + ( + avg_over_time(:node_memory_utilisation:[$step]) * + sum( + avg_over_time(node:node_memory_bytes_total:sum[$step]) + ) + ) + ) + or + ( + ( + avg_over_time(:node_memory_utilisation:[$step]) * + sum( + avg_over_time(node:node_memory_bytes_total:sum[$step]) + ) + ) > + sum( + avg_over_time(kube_pod_container_resource_requests{resource="memory",unit="byte"}[$step]) + ) + ), + 1 +)`, + + // avg over time manually + "meter_cluster_net_bytes_transmitted": ` +round( + sum( + increase( + node_network_transmit_bytes_total{ + job="node-exporter", + device!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)" + }[$step] + ) + ) / $factor, + 1 +)`, + + // avg over time manually + "meter_cluster_net_bytes_received": ` +round( + sum( + increase( + node_network_receive_bytes_total{ + job="node-exporter", + device!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)" + }[$step] + ) + ) / $factor, + 1 +)`, + + "meter_cluster_pvc_bytes_total": ` +sum( + topk(1, avg_over_time(namespace:pvc_bytes_total:sum{}[$step])) by (persistentvolumeclaim) +)`, + + // node + "meter_node_cpu_usage": ` +round( + ( + sum( + avg_over_time(kube_pod_container_resource_requests{$nodeSelector, resource="cpu",unit="core"}[$step]) + ) by (node) >= + sum( + avg_over_time(node:node_cpu_utilisation:avg1m{$nodeSelector}[$step]) * + avg_over_time(node:node_num_cpu:sum{$nodeSelector}[$step]) + ) by (node) + ) + or + ( + sum( + avg_over_time(node:node_cpu_utilisation:avg1m{$nodeSelector}[$step]) * + avg_over_time(node:node_num_cpu:sum{$nodeSelector}[$step]) + ) by (node) > + sum( + avg_over_time(kube_pod_container_resource_requests{$nodeSelector, resource="cpu",unit="core"}[$step]) + ) by (node) + ) + or + ( + sum( + avg_over_time(node:node_cpu_utilisation:avg1m{$nodeSelector}[$step]) * + avg_over_time(node:node_num_cpu:sum{$nodeSelector}[$step]) + ) by (node) + ), + 0.001 +)`, + + "meter_node_memory_usage_wo_cache": ` +round( + ( + sum( + avg_over_time(kube_pod_container_resource_requests{$nodeSelector, resource="memory",unit="byte"}[$step]) + ) by (node) >= + sum( + avg_over_time(node:node_memory_bytes_total:sum{$nodeSelector}[$step]) - + avg_over_time(node:node_memory_bytes_available:sum{$nodeSelector}[$step]) + ) by (node) + ) + or + ( + sum( + avg_over_time(node:node_memory_bytes_total:sum{$nodeSelector}[$step]) - + avg_over_time(node:node_memory_bytes_available:sum{$nodeSelector}[$step]) + ) by (node) > + sum( + avg_over_time(kube_pod_container_resource_requests{$nodeSelector, resource="memory",unit="byte"}[$step]) + ) by (node) + ) + or + ( + sum( + avg_over_time(node:node_memory_bytes_total:sum{$nodeSelector}[$step]) - + avg_over_time(node:node_memory_bytes_available:sum{$nodeSelector}[$step]) + ) by (node) + ), + 0.001 +)`, + + // avg over time manually + "meter_node_net_bytes_transmitted": ` +round( + sum by (node) ( + sum without (instance) ( + label_replace( + increase( + node_network_transmit_bytes_total{ + job="node-exporter", + device!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", + $instanceSelector + }[$step] + ), + "node", + "$1", + "instance", + "(.*)" + ) + ) + ) / $factor, + 1 +)`, + + // avg over time manually + "meter_node_net_bytes_received": ` +round( + sum by (node) ( + sum without (instance) ( + label_replace( + increase( + node_network_receive_bytes_total{ + job="node-exporter", + device!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", + $instanceSelector + }[$step] + ), + "node", " + $1", + "instance", + "(.*)" + ) + ) + ) / $factor, + 1 +)`, + + "meter_node_pvc_bytes_total": ` +sum( + topk( + 1, + avg_over_time( + namespace:pvc_bytes_total:sum{$nodeSelector}[$step] + ) + ) by (persistentvolumeclaim, node) +) by (node)`, + + // workspace + "meter_workspace_cpu_usage": ` +round( + ( + sum by (workspace) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{ + owner_kind!="Job", + namespace!="", + resource="cpu", + $1 + }[$step] + ) + ) >= + sum by (workspace) ( + avg_over_time(namespace:container_cpu_usage_seconds_total:sum_rate{namespace!="", $1}[$step]) + ) + ) + or + ( + sum by (workspace) ( + avg_over_time(namespace:container_cpu_usage_seconds_total:sum_rate{namespace!="", $1}[$step]) + ) > + sum by (workspace) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{ + owner_kind!="Job", + namespace!="", + resource="cpu", + $1 + }[$step] + ) + ) + ) + or + ( + sum by (workspace) ( + avg_over_time(namespace:container_cpu_usage_seconds_total:sum_rate{namespace!="", $1}[$step]) + ) + ), + 0.001 +)`, + + "meter_workspace_memory_usage": ` +round( + ( + sum by (workspace) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{ + owner_kind!="Job", + namespace!="", + resource="memory", + $1 + }[$step] + ) + ) >= + sum by (workspace) ( + avg_over_time(namespace:container_memory_usage_bytes:sum{namespace!="", $1}[$step]) + ) + ) + or + ( + sum by (workspace) ( + avg_over_time(namespace:container_memory_usage_bytes:sum{namespace!="", $1}[$step]) + ) > + sum by (workspace) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{ + owner_kind!="Job", namespace!="", resource="memory", $1 + }[$step] + ) + ) + ) + or + ( + sum by (workspace) ( + avg_over_time(namespace:container_memory_usage_bytes:sum{namespace!="", $1}[$step]) + ) + ), + 1 +)`, + + // avg over time manually + "meter_workspace_net_bytes_transmitted": ` +round( + ( + sum by (workspace) ( + sum by (namespace) ( + increase( + container_network_transmit_bytes_total{ + namespace!="", + pod!="", + interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", + job="kubelet" + }[$step] + ) + ) * on (namespace) group_left(workspace) + kube_namespace_labels{$1} + ) or on(workspace) max by(workspace) (kube_namespace_labels{$1} * 0) + ) / $factor, + 1 +)`, + + "meter_workspace_net_bytes_received": ` +round( + ( + sum by (workspace) ( + sum by (namespace) ( + increase( + container_network_receive_bytes_total{ + namespace!="", + pod!="", + interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", + job="kubelet" + }[$step] + ) + ) * on (namespace) group_left(workspace) + kube_namespace_labels{$1} + ) or on(workspace) max by(workspace) (kube_namespace_labels{$1} * 0) + ) / $factor, + 1 +)`, + + "meter_workspace_pvc_bytes_total": ` +sum ( + topk( + 1, + avg_over_time(namespace:pvc_bytes_total:sum{$1}[$step]) + ) by (persistentvolumeclaim, workspace) +) by (workspace)`, + + // namespace + "meter_namespace_cpu_usage": ` +round( + ( + sum by (namespace) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{ + owner_kind!="Job", + namespace!="", + resource="cpu", + $1 + }[$step] + ) + ) >= + sum by (namespace) ( + avg_over_time(namespace:container_cpu_usage_seconds_total:sum_rate{namespace!="", $1}[$step]) + ) + ) + or + ( + sum by (namespace) ( + avg_over_time(namespace:container_cpu_usage_seconds_total:sum_rate{namespace!="", $1}[$step]) + ) > + sum by (namespace) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{owner_kind!="Job", namespace!="", resource="cpu", $1}[$step] + ) + ) + ) + or + ( + sum by (namespace) ( + avg_over_time( + namespace:container_cpu_usage_seconds_total:sum_rate{namespace!="", $1}[$step] + ) + ) + ), + 0.001 +)`, + + "meter_namespace_memory_usage_wo_cache": ` +round( + ( + sum by (namespace) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{ + owner_kind!="Job", + namespace!="", + resource="memory", + $1 + }[$step] + ) + ) >= + sum by (namespace) ( + avg_over_time(namespace:container_memory_usage_bytes_wo_cache:sum{namespace!="", $1}[$step]) + ) + ) + or + ( + sum by (namespace) ( + avg_over_time(namespace:container_memory_usage_bytes_wo_cache:sum{namespace!="", $1}[$step]) + ) > + sum by (namespace) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{ + owner_kind!="Job", namespace!="", resource="memory", $1 + }[$step] + ) + ) + ) + or + ( + sum by (namespace) ( + avg_over_time( + namespace:container_memory_usage_bytes_wo_cache:sum{namespace!="", $1}[$step] + ) + ) + ), + 1 +)`, + + "meter_namespace_net_bytes_transmitted": ` +round( + ( + sum by (namespace) ( + increase( + container_network_transmit_bytes_total{ + namespace!="", + pod!="", + interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", + job="kubelet" + }[$step] + ) + * on (namespace) group_left(workspace) + kube_namespace_labels{$1} + ) + or on(namespace) max by(namespace) (kube_namespace_labels{$1} * 0) + ) / $factor, + 1 +)`, + + "meter_namespace_net_bytes_received": ` +round( + ( + sum by (namespace) ( + increase( + container_network_receive_bytes_total{ + namespace!="", + pod!="", + interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", + job="kubelet" + }[$step] + ) + * on (namespace) group_left(workspace) + kube_namespace_labels{$1} + ) + or on(namespace) max by(namespace) (kube_namespace_labels{$1} * 0) + ) / $factor, + 1 +)`, + + "meter_namespace_pvc_bytes_total": ` +sum ( + topk( + 1, + avg_over_time(namespace:pvc_bytes_total:sum{$1}[$step]) + ) by (persistentvolumeclaim, namespace) +) by (namespace)`, + + // application + "meter_application_cpu_usage": ` +round( + ( + sum by (namespace, application) ( + label_replace( + avg_over_time( + namespace:kube_workload_resource_request:sum{workload!~"Job:.+", resource="cpu", $1}[$step] + ), + "application", + "$app", + "", + "" + ) + ) >= + sum by (namespace, application) ( + label_replace( + avg_over_time(namespace:workload_cpu_usage:sum{$1}[$step]), + "application", + "$app", + "", + "" + ) + ) + ) + or + ( + sum by (namespace, application) ( + label_replace( + avg_over_time(namespace:workload_cpu_usage:sum{$1}[$step]), + "application", + "$app", + "", + "" + ) + ) > + sum by (namespace, application) ( + label_replace( + avg_over_time( + namespace:kube_workload_resource_request:sum{workload!~"Job:.+", resource="cpu", $1}[$step] + ), + "application", + "$app", + "", + "" + ) + ) + ) + or + ( + sum by (namespace, application) ( + label_replace( + avg_over_time(namespace:workload_cpu_usage:sum{$1}[$step]), + "application", + "$app", + "", + "" + ) + ) + ), + 0.001 +)`, + + "meter_application_memory_usage_wo_cache": ` +round( + ( + sum by (namespace, application) ( + label_replace( + avg_over_time( + namespace:kube_workload_resource_request:sum{workload!~"Job:.+", resource="memory", $1}[$step] + ), + "application", + "$app", + "", + "" + ) + ) >= + sum by (namespace, application) ( + label_replace( + avg_over_time(namespace:workload_memory_usage_wo_cache:sum{$1}[$step]), + "application", + "$app", + "", + "" + ) + ) + ) + or + ( + sum by (namespace, application) ( + label_replace( + avg_over_time(namespace:workload_memory_usage_wo_cache:sum{$1}[$step]), + "application", + "$app", + "", + "" + ) + ) > + sum by (namespace, application) ( + label_replace( + avg_over_time( + namespace:kube_workload_resource_request:sum{workload!~"Job:.+", resource="memory", $1}[$step] + ), + "application", + "$app", + "", + "" + ) + ) + ) + or + ( + sum by (namespace, application) ( + label_replace( + avg_over_time(namespace:workload_memory_usage_wo_cache:sum{$1}[$step]), + "application", + "$app", + "", + "" + ) + ) + ), + 1 +)`, + + "meter_application_net_bytes_transmitted": ` +round( + sum by (namespace, application) ( + label_replace( + increase( + namespace:workload_net_bytes_transmitted:sum{$1}[$step] + ), + "application", + "$app", + "", + "" + ) + ) / $factor, + 1 +)`, + + "meter_application_net_bytes_received": ` +round( + sum by (namespace, application) ( + label_replace( + increase( + namespace:workload_net_bytes_received:sum{$1}[$step] + ), + "application", + "$app", + "", + "" + ) + ) / $factor, + 1 +)`, + + "meter_application_pvc_bytes_total": ` +sum by (namespace, application) ( + label_replace( + topk(1, avg_over_time(namespace:pvc_bytes_total:sum{$1}[$step])) by (persistentvolumeclaim), + "application", + "$app", + "", + "" + ) +)`, + + // workload + "meter_workload_cpu_usage": ` +round( + ( + sum by (namespace, workload) ( + avg_over_time( + namespace:kube_workload_resource_request:sum{ + workload!~"Job:.+", resource="cpu", $1 + }[$step] + ) + ) >= + sum by (namespace, workload) ( + avg_over_time(namespace:workload_cpu_usage:sum{$1}[$step]) + ) + ) + or + ( + sum by (namespace, workload) ( + avg_over_time(namespace:workload_cpu_usage:sum{$1}[$step]) + ) > + sum by (namespace, workload) ( + avg_over_time( + namespace:kube_workload_resource_request:sum{ + workload!~"Job:.+", resource="cpu", $1 + }[$step] + ) + ) + ) + or + ( + sum by (namespace, workload) ( + avg_over_time(namespace:workload_cpu_usage:sum{$1}[$step]) + ) + ), + 0.001 +)`, + + "meter_workload_memory_usage_wo_cache": ` +round( + ( + sum by (namespace, workload) ( + avg_over_time( + namespace:kube_workload_resource_request:sum{ + workload!~"Job:.+", resource="memory", $1 + }[$step] + ) + ) >= + sum by (namespace, workload) ( + avg_over_time(namespace:workload_memory_usage_wo_cache:sum{$1}[$step]) + ) + ) + or + ( + sum by (namespace, workload) ( + avg_over_time(namespace:workload_memory_usage_wo_cache:sum{$1}[$step]) + ) > + sum by (namespace, workload) ( + avg_over_time( + namespace:kube_workload_resource_request:sum{ + workload!~"Job:.+", resource="memory", $1 + }[$step] + ) + ) + ) + or + ( + sum by (namespace, workload) ( + avg_over_time(namespace:workload_memory_usage_wo_cache:sum{$1}[$step]) + ) + ), + 1 +)`, + + "meter_workload_net_bytes_transmitted": ` +round( + increase( + namespace:workload_net_bytes_transmitted:sum{$1}[$step] + ) / $factor, + 1 +)`, + + "meter_workload_net_bytes_received": ` +round( + increase( + namespace:workload_net_bytes_received:sum{$1}[$step] + ) / $factor, + 1 +)`, + + "meter_workload_pvc_bytes_total": ` +sum by (namespace, workload) ( + topk( + 1, + avg_over_time(namespace:pvc_bytes_total:sum{$1}[$step]) + ) by (persistentvolumeclaim, namespace, workload) +)`, + + // service + "meter_service_cpu_usage": ` +round( + sum by (namespace, service) ( + label_replace( + sum by (namespace, pod) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{owner_kind!="Job", resource="cpu", $1}[$step] + ) + ) >= + sum by (namespace, pod) ( + sum by (namespace, pod) ( + irate( + container_cpu_usage_seconds_total{job="kubelet", pod!="", image!=""}[$step] + ) + ) * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{} * on (namespace, pod) group_left(node) + kube_pod_info{$1} + ), + "service", + "$svc", + "", + "" + ) + ) + or + sum by (namespace, service) ( + label_replace( + sum by (namespace, pod) ( + sum by (namespace, pod) ( + irate( + container_cpu_usage_seconds_total{job="kubelet", pod!="", image!=""}[$step] + ) + ) * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{} * on (namespace, pod) group_left(node) + kube_pod_info{$1} + ) > + sum by (namespace, pod) ( + avg_over_time(namespace:kube_pod_resource_request:sum{owner_kind!="Job", resource="cpu", $1}[$step]) + ), + "service", + "$svc", + "", + "" + ) + ) + or + sum by (namespace, service) ( + label_replace( + sum by (namespace, pod) ( + sum by (namespace, pod) ( + irate( + container_cpu_usage_seconds_total{job="kubelet", pod!="", image!=""}[$step] + ) + ) * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{} * on (namespace, pod) group_left(node) + kube_pod_info{$1} + ), + "service", + "$svc", + "", + "" + ) + ), + 0.001 +)`, + + "meter_service_memory_usage_wo_cache": ` +round( + ( + sum by (namespace, service) ( + label_replace( + sum by (namespace, pod) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{owner_kind!="Job", resource="memory", $1}[$step] + ) + ) >= + sum by (namespace, pod) ( + sum by (namespace, pod) ( + avg_over_time( + container_memory_working_set_bytes{job="kubelet", pod!="", image!=""}[$step] + ) + ) * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{} * on (namespace, pod) group_left(node) + kube_pod_info{$1} + ), + "service", + "$svc", + "", + "" + ) + ) + ) + or + ( + sum by (namespace, service) ( + label_replace( + sum by (namespace, pod) ( + sum by (namespace, pod) ( + avg_over_time( + container_memory_working_set_bytes{job="kubelet", pod!="", image!=""}[$step] + ) + ) * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{} * on (namespace, pod) group_left(node) + kube_pod_info{$1} + ) > + sum by (namespace, pod) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{owner_kind!="Job", resource="memory", $1}[$step] + ) + ), + "service", + "$svc", + "", + "" + ) + ) + ) + or + ( + sum by (namespace, service) ( + label_replace( + sum by (namespace, pod) ( + sum by (namespace, pod) ( + avg_over_time( + container_memory_working_set_bytes{job="kubelet", pod!="", image!=""}[$step] + ) + ) * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{} * on (namespace, pod) group_left(node) + kube_pod_info{$1} + ), + "service", + "$svc", + "", + "" + ) + ) + ), + 1 +)`, + + "meter_service_net_bytes_transmitted": ` +round( + sum by (namespace, service) ( + label_replace( + sum by (namespace, pod) ( + sum by (namespace, pod) ( + increase( + container_network_transmit_bytes_total{ + pod!="", + interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", + job="kubelet" + }[$step] + ) + ) * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{} * on (namespace, pod) group_left(node) + kube_pod_info{$1} + ), + "service", + "$svc", + "", + "" + ) + ) / $factor, + 1 +)`, + + "meter_service_net_bytes_received": ` +round( + sum by (namespace, service) ( + label_replace( + sum by (namepace, pod) ( + sum by (namespace, pod) ( + increase( + container_network_receive_bytes_total{ + pod!="", + interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", + job="kubelet" + }[$step] + ) + ) * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{} * on (namespace, pod) group_left(node) + kube_pod_info{$1} + ), + "service", + "$svc", + "", + "" + ) + ) / $factor, + 1 +)`, + + // pod + "meter_pod_cpu_usage": ` +round( + ( + sum by (namespace, pod) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{ + owner_kind!="Job", + resource="cpu", + $internalPodSelector + }[$step] + ) + ) + * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{$1} + * on (namespace, pod) group_left(node) + kube_pod_info{$2} >= + sum by (namespace, pod) ( + irate(container_cpu_usage_seconds_total{job="kubelet",pod!="",image!="", $internalPodSelector}[$step]) + ) + * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{$1} + * on (namespace, pod) group_left(node) + kube_pod_info{$2} + ) + or + ( + sum by (namespace, pod) ( + irate(container_cpu_usage_seconds_total{job="kubelet",pod!="",image!="", $internalPodSelector}[$step]) + ) + * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{$1} + * on (namespace, pod) group_left(node) + kube_pod_info{$2} > + sum by (namespace, pod) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{ + owner_kind!="Job", + resource="cpu", + $internalPodSelector + }[$step] + ) + ) + * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{$1} + * on (namespace, pod) group_left(node) + kube_pod_info{$2} + ) + or + ( + sum by (namespace, pod) ( + irate(container_cpu_usage_seconds_total{job="kubelet",pod!="",image!="", $internalPodSelector}[$step]) + ) + * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{$1} + * on (namespace, pod) group_left(node) + kube_pod_info{$2} + ), + 0.001 +)`, + + "meter_pod_memory_usage_wo_cache": ` +round( + ( + sum by (namespace, pod) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{ + owner_kind!="Job", + resource="memory", + $internalPodSelector + }[$step] + ) + ) + * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{$1} + * on (namespace, pod) group_left(node) + kube_pod_info{$2} >= + sum by (namespace, pod) ( + avg_over_time(container_memory_working_set_bytes{job="kubelet", pod!="", image!="", $internalPodSelector}[$step]) + ) + * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{$1} + * on (namespace, pod) group_left(node) + kube_pod_info{$2} + ) + or + ( + sum by (namespace, pod) ( + avg_over_time(container_memory_working_set_bytes{job="kubelet", pod!="", image!="", $internalPodSelector}[$step]) + ) + * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{$1} + * on (namespace, pod) group_left(node) + kube_pod_info{$2} > + sum by (namespace, pod) ( + avg_over_time( + namespace:kube_pod_resource_request:sum{ + owner_kind!="Job", + resource="memory", + $internalPodSelector + }[$step] + ) + * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{$1} + * on (namespace, pod) group_left(node) + kube_pod_info{$2} + ) + ) + or + ( + sum by (namespace, pod) ( + avg_over_time(container_memory_working_set_bytes{job="kubelet", pod!="", image!="", $internalPodSelector}[$step]) + ) + * on (namespace, pod) group_left(owner_kind, owner_name) + kube_pod_owner{$1} + * on (namespace, pod) group_left(node) + kube_pod_info{$2} + ), + 0.001 +)`, + + "meter_pod_net_bytes_transmitted": ` +sum by (namespace, pod) ( + increase( + container_network_transmit_bytes_total{ + pod!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet", $internalPodSelector + }[$step] + ) / $factor +) +* on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} +* on (namespace, pod) group_left(node) kube_pod_info{$2}`, + + "meter_pod_net_bytes_received": ` +sum by (namespace, pod) ( + increase( + container_network_receive_bytes_total{ + pod!="", interface!~"^(cali.+|tunl.+|dummy.+|kube.+|flannel.+|cni.+|docker.+|veth.+|lo.*)", job="kubelet", $internalPodSelector + }[$step] + ) / $factor +) +* on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} +* on (namespace, pod) group_left(node) kube_pod_info{$2}`, + + "meter_pod_pvc_bytes_total": ` +sum by (namespace, pod) ( + avg_over_time(namespace:pvc_bytes_total:sum{$internalPodSelector}[$step]) +) +* on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{$1} +* on (namespace, pod) group_left(node) kube_pod_info{$2}`, +} + +func makeMeterExpr(meter string, o monitoring.QueryOptions) string { + + var tmpl string + if tmpl = getMeterTemplate(meter); len(tmpl) == 0 { + klog.Errorf("invalid meter %s", meter) + return "" + } + tmpl = renderMeterTemplate(tmpl, o) + + switch o.Level { + case monitoring.LevelCluster: + return makeClusterMeterExpr(tmpl, o) + case monitoring.LevelNode: + return makeNodeMeterExpr(tmpl, o) + case monitoring.LevelWorkspace: + return makeWorkspaceMeterExpr(tmpl, o) + case monitoring.LevelNamespace: + return makeNamespaceMeterExpr(tmpl, o) + case monitoring.LevelApplication: + return makeApplicationMeterExpr(tmpl, o) + case monitoring.LevelWorkload: + return makeWorkloadMeterExpr(meter, tmpl, o) + case monitoring.LevelService: + return makeServiceMeterExpr(tmpl, o) + case monitoring.LevelPod: + return makePodMeterExpr(tmpl, o) + default: + return "" + } + +} + +func getMeterTemplate(meter string) string { + if tmpl, ok := promQLMeterTemplates[meter]; !ok { + klog.Errorf("invalid meter %s", meter) + return "" + } else { + return strings.Join(strings.Fields(strings.TrimSpace(tmpl)), " ") + } +} + +func renderMeterTemplate(tmpl string, o monitoring.QueryOptions) string { + if o.MeterOptions == nil { + klog.Error("meter options not found") + return "" + } + + tmpl = replaceStepSelector(tmpl, o) + tmpl = replacePVCSelector(tmpl, o) + tmpl = replaceNodeSelector(tmpl, o) + tmpl = replaceInstanceSelector(tmpl, o) + tmpl = replaceAppSelector(tmpl, o) + tmpl = replaceSvcSelector(tmpl, o) + tmpl = replaceFactor(tmpl, o) + + return tmpl +} + +func makeClusterMeterExpr(tmpl string, o monitoring.QueryOptions) string { + return tmpl +} + +func makeNodeMeterExpr(tmpl string, o monitoring.QueryOptions) string { + return tmpl +} + +func makeWorkspaceMeterExpr(tmpl string, o monitoring.QueryOptions) string { + return makeWorkspaceMetricExpr(tmpl, o) +} + +func makeNamespaceMeterExpr(tmpl string, o monitoring.QueryOptions) string { + return makeNamespaceMetricExpr(tmpl, o) +} + +func makeApplicationMeterExpr(tmpl string, o monitoring.QueryOptions) string { + return strings.NewReplacer("$1", o.ResourceFilter).Replace(tmpl) +} + +func makeWorkloadMeterExpr(meter string, tmpl string, o monitoring.QueryOptions) string { + return makeWorkloadMetricExpr(meter, tmpl, o) +} + +func makeServiceMeterExpr(tmpl string, o monitoring.QueryOptions) string { + return strings.Replace(tmpl, "$1", o.ResourceFilter, -1) +} + +func makePodMeterExpr(tmpl string, o monitoring.QueryOptions) string { + + // here we support internal pod selector to accelerate metering pod filter operation, otherwise we will iterate + // pod in cluster scope which required longer time + var internalPodSelector string + if o.PodName != "" { + internalPodSelector += fmt.Sprintf(`pod="%s", `, o.PodName) + } + if o.ResourceFilter != "" { + internalPodSelector += fmt.Sprintf(`pod=~"%s", `, o.ResourceFilter) + } + if o.NamespaceName != "" { + internalPodSelector += fmt.Sprintf(`namespace="%s"`, o.NamespaceName) + } + + tmpl = strings.NewReplacer("$internalPodSelector", internalPodSelector).Replace(tmpl) + + return makePodMetricExpr(tmpl, o) +} + +func replacePVCSelector(tmpl string, o monitoring.QueryOptions) string { + var filterConditions []string + + switch o.Level { + case monitoring.LevelCluster: + break + case monitoring.LevelNode: + if o.NodeName != "" { + filterConditions = append(filterConditions, fmt.Sprintf(`node="%s"`, o.NodeName)) + } else if o.ResourceFilter != "" { + filterConditions = append(filterConditions, fmt.Sprintf(`node=~"%s"`, o.ResourceFilter)) + } + if o.PVCFilter != "" { + filterConditions = append(filterConditions, fmt.Sprintf(`persistentvolumeclaim=~"%s"`, o.PVCFilter)) + } + if o.StorageClassName != "" { + filterConditions = append(filterConditions, fmt.Sprintf(`storageclass="%s"`, o.StorageClassName)) + } + case monitoring.LevelWorkspace: + if o.WorkspaceName != "" { + filterConditions = append(filterConditions, fmt.Sprintf(`workspace="%s"`, o.WorkspaceName)) + } + if o.PVCFilter != "" { + filterConditions = append(filterConditions, fmt.Sprintf(`persistentvolumeclaim=~"%s"`, o.PVCFilter)) + } + if o.StorageClassName != "" { + filterConditions = append(filterConditions, fmt.Sprintf(`storageclass="%s"`, o.StorageClassName)) + } + case monitoring.LevelNamespace: + if o.NamespaceName != "" { + filterConditions = append(filterConditions, fmt.Sprintf(`namespace="%s"`, o.NamespaceName)) + } + if o.PVCFilter != "" { + filterConditions = append(filterConditions, fmt.Sprintf(`persistentvolumeclaim=~"%s"`, o.PVCFilter)) + } + if o.StorageClassName != "" { + filterConditions = append(filterConditions, fmt.Sprintf(`storageclass="%s"`, o.StorageClassName)) + } + case monitoring.LevelApplication: + if o.NamespaceName != "" { + filterConditions = append(filterConditions, fmt.Sprintf(`namespace="%s"`, o.NamespaceName)) + } + // o.PVCFilter is required and is filled with application volumes else just empty string which means that + // there won't be any match application volumes + filterConditions = append(filterConditions, fmt.Sprintf(`persistentvolumeclaim=~"%s"`, o.PVCFilter)) + + if o.StorageClassName != "" { + filterConditions = append(filterConditions, fmt.Sprintf(`storageclass="%s"`, o.StorageClassName)) + } + default: + return tmpl + } + return strings.Replace(tmpl, "$pvc", strings.Join(filterConditions, ","), -1) +} + +func replaceFactor(tmpl string, o monitoring.QueryOptions) string { + stepStr := strconv.Itoa(int(o.MeterOptions.Step.Hours())) + + return strings.Replace(tmpl, "$factor", stepStr, -1) +} + +func replaceStepSelector(tmpl string, o monitoring.QueryOptions) string { + stepStr := strconv.Itoa(int(o.MeterOptions.Step.Hours())) + "h" + + return strings.Replace(tmpl, "$step", stepStr, -1) +} + +func replaceNodeSelector(tmpl string, o monitoring.QueryOptions) string { + + var nodeSelector string + if o.NodeName != "" { + nodeSelector = fmt.Sprintf(`node="%s"`, o.NodeName) + } else { + nodeSelector = fmt.Sprintf(`node=~"%s"`, o.ResourceFilter) + } + return strings.Replace(tmpl, "$nodeSelector", nodeSelector, -1) +} + +func replaceInstanceSelector(tmpl string, o monitoring.QueryOptions) string { + var instanceSelector string + if o.NodeName != "" { + instanceSelector = fmt.Sprintf(`instance="%s"`, o.NodeName) + } else { + instanceSelector = fmt.Sprintf(`instance=~"%s"`, o.ResourceFilter) + } + return strings.Replace(tmpl, "$instanceSelector", instanceSelector, -1) +} + +func replaceAppSelector(tmpl string, o monitoring.QueryOptions) string { + return strings.Replace(tmpl, "$app", o.ApplicationName, -1) +} + +func replaceSvcSelector(tmpl string, o monitoring.QueryOptions) string { + return strings.Replace(tmpl, "$svc", o.ServiceName, -1) +} diff --git a/pkg/simple/client/monitoring/query_options.go b/pkg/simple/client/monitoring/query_options.go index 37f77b543..75b9afeb3 100644 --- a/pkg/simple/client/monitoring/query_options.go +++ b/pkg/simple/client/monitoring/query_options.go @@ -16,6 +16,12 @@ limitations under the License. package monitoring +import ( + "fmt" + "strings" + "time" +) + type Level int const ( @@ -23,17 +29,39 @@ const ( LevelNode LevelWorkspace LevelNamespace + LevelApplication LevelWorkload + LevelService LevelPod LevelContainer LevelPVC LevelComponent ) +var MeteringLevelMap = map[string]int{ + "LevelCluster": LevelCluster, + "LevelNode": LevelNode, + "LevelWorkspace": LevelWorkspace, + "LevelNamespace": LevelNamespace, + "LevelApplication": LevelApplication, + "LevelWorkload": LevelWorkload, + "LevelService": LevelService, + "LevelPod": LevelPod, + "LevelContainer": LevelContainer, + "LevelPVC": LevelPVC, + "LevelComponent": LevelComponent, +} + type QueryOption interface { Apply(*QueryOptions) } +type Meteroptions struct { + Start time.Time + End time.Time + Step time.Duration +} + type QueryOptions struct { Level Level @@ -48,6 +76,10 @@ type QueryOptions struct { ContainerName string StorageClassName string PersistentVolumeClaimName string + PVCFilter string + ApplicationName string + ServiceName string + MeterOptions *Meteroptions } func NewQueryOptions() *QueryOptions { @@ -61,31 +93,41 @@ func (_ ClusterOption) Apply(o *QueryOptions) { } type NodeOption struct { - ResourceFilter string - NodeName string + ResourceFilter string + NodeName string + PVCFilter string + StorageClassName string } func (no NodeOption) Apply(o *QueryOptions) { o.Level = LevelNode o.ResourceFilter = no.ResourceFilter o.NodeName = no.NodeName + o.PVCFilter = no.PVCFilter + o.StorageClassName = no.StorageClassName } type WorkspaceOption struct { - ResourceFilter string - WorkspaceName string + ResourceFilter string + WorkspaceName string + PVCFilter string + StorageClassName string } func (wo WorkspaceOption) Apply(o *QueryOptions) { o.Level = LevelWorkspace o.ResourceFilter = wo.ResourceFilter o.WorkspaceName = wo.WorkspaceName + o.PVCFilter = wo.PVCFilter + o.StorageClassName = wo.StorageClassName } type NamespaceOption struct { - ResourceFilter string - WorkspaceName string - NamespaceName string + ResourceFilter string + WorkspaceName string + NamespaceName string + PVCFilter string + StorageClassName string } func (no NamespaceOption) Apply(o *QueryOptions) { @@ -93,6 +135,41 @@ func (no NamespaceOption) Apply(o *QueryOptions) { o.ResourceFilter = no.ResourceFilter o.WorkspaceName = no.WorkspaceName o.NamespaceName = no.NamespaceName + o.PVCFilter = no.PVCFilter + o.StorageClassName = no.StorageClassName +} + +type ApplicationsOption struct { + NamespaceName string + Applications []string + StorageClassName string +} + +func (aso ApplicationsOption) Apply(o *QueryOptions) { + // nothing should be done + return +} + +type ApplicationOption struct { + NamespaceName string + Application string + ApplicationComponents []string + StorageClassName string +} + +func (ao ApplicationOption) Apply(o *QueryOptions) { + o.Level = LevelApplication + o.NamespaceName = ao.NamespaceName + o.ApplicationName = ao.Application + o.StorageClassName = ao.StorageClassName + + app_components := strings.Join(ao.ApplicationComponents[:], "|") + + if len(app_components) > 0 { + o.ResourceFilter = fmt.Sprintf(`namespace="%s", workload=~"%s"`, o.NamespaceName, app_components) + } else { + o.ResourceFilter = fmt.Sprintf(`namespace="%s", workload=~"%s"`, o.NamespaceName, ".*") + } } type WorkloadOption struct { @@ -108,6 +185,37 @@ func (wo WorkloadOption) Apply(o *QueryOptions) { o.WorkloadKind = wo.WorkloadKind } +type ServicesOption struct { + NamespaceName string + Services []string +} + +func (sso ServicesOption) Apply(o *QueryOptions) { + // nothing should be done + return +} + +type ServiceOption struct { + ResourceFilter string + NamespaceName string + ServiceName string + PodNames []string +} + +func (so ServiceOption) Apply(o *QueryOptions) { + o.Level = LevelService + o.NamespaceName = so.NamespaceName + o.ServiceName = so.ServiceName + + pod_names := strings.Join(so.PodNames, "|") + + if len(pod_names) > 0 { + o.ResourceFilter = fmt.Sprintf(`pod=~"%s", namespace="%s"`, pod_names, o.NamespaceName) + } else { + o.ResourceFilter = fmt.Sprintf(`pod=~"%s", namespace="%s"`, ".*", o.NamespaceName) + } +} + type PodOption struct { NamespacedResourcesFilter string ResourceFilter string @@ -157,6 +265,9 @@ func (po PVCOption) Apply(o *QueryOptions) { o.NamespaceName = po.NamespaceName o.StorageClassName = po.StorageClassName o.PersistentVolumeClaimName = po.PersistentVolumeClaimName + + // for meter + o.PVCFilter = po.PersistentVolumeClaimName } type ComponentOption struct{} @@ -164,3 +275,17 @@ type ComponentOption struct{} func (_ ComponentOption) Apply(o *QueryOptions) { o.Level = LevelComponent } + +type MeterOption struct { + Start time.Time + End time.Time + Step time.Duration +} + +func (mo MeterOption) Apply(o *QueryOptions) { + o.MeterOptions = &Meteroptions{ + Start: mo.Start, + End: mo.End, + Step: mo.Step, + } +} diff --git a/pkg/simple/client/monitoring/types.go b/pkg/simple/client/monitoring/types.go index fd970bf7b..3def940d3 100644 --- a/pkg/simple/client/monitoring/types.go +++ b/pkg/simple/client/monitoring/types.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/json-iterator/go" "strconv" + "time" ) const ( @@ -54,8 +55,32 @@ type MetricValue struct { // The type of Point is a float64 array with fixed length of 2. // So Point will always be initialized as [0, 0], rather than nil. // To allow empty Sample, we should declare Sample to type *Point - Sample *Point `json:"value,omitempty" description:"time series, values of vector type"` - Series []Point `json:"values,omitempty" description:"time series, values of matrix type"` + Sample *Point `json:"value,omitempty" description:"time series, values of vector type"` + Series []Point `json:"values,omitempty" description:"time series, values of matrix type"` + ExportSample *ExportPoint `json:"exported_value,omitempty" description:"exported time series, values of vector type"` + ExportedSeries []ExportPoint `json:"exported_values,omitempty" description:"exported time series, values of matrix type"` + + MinValue float64 `json:"min_value" description:"minimum value from monitor points"` + MaxValue float64 `json:"max_value" description:"maximum value from monitor points"` + AvgValue float64 `json:"avg_value" description:"average value from monitor points"` + SumValue float64 `json:"sum_value" description:"sum value from monitor points"` + Fee float64 `json:"fee" description:"resource fee"` +} + +func (mv *MetricValue) TransferToExportedMetricValue() { + + if mv.Sample != nil { + sample := mv.Sample.transferToExported() + mv.ExportSample = &sample + mv.Sample = nil + } + + for _, item := range mv.Series { + mv.ExportedSeries = append(mv.ExportedSeries, item.transferToExported()) + } + mv.Series = nil + + return } func (p Point) Timestamp() float64 { @@ -66,6 +91,10 @@ func (p Point) Value() float64 { return p[1] } +func (p Point) transferToExported() ExportPoint { + return ExportPoint{p[0], p[1]} +} + // MarshalJSON implements json.Marshaler. It will be called when writing JSON to HTTP response // Inspired by prometheus/client_golang func (p Point) MarshalJSON() ([]byte, error) { @@ -112,3 +141,27 @@ func (p *Point) UnmarshalJSON(b []byte) error { p[1] = valf return nil } + +type ExportPoint [2]float64 + +func (p ExportPoint) Timestamp() string { + return time.Unix(int64(p[0]), 0).Format("2006-01-02 03:04:05 PM") +} + +func (p ExportPoint) Value() float64 { + return p[1] +} + +// MarshalJSON implements json.Marshaler. It will be called when writing JSON to HTTP response +// Inspired by prometheus/client_golang +func (p ExportPoint) MarshalJSON() ([]byte, error) { + t, err := jsoniter.Marshal(p.Timestamp()) + if err != nil { + return nil, err + } + v, err := jsoniter.Marshal(strconv.FormatFloat(p.Value(), 'f', -1, 64)) + if err != nil { + return nil, err + } + return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil +} diff --git a/tools/cmd/doc-gen/main.go b/tools/cmd/doc-gen/main.go index edb8f21eb..8173feba9 100644 --- a/tools/cmd/doc-gen/main.go +++ b/tools/cmd/doc-gen/main.go @@ -125,7 +125,7 @@ func generateSwaggerJson() []byte { urlruntime.Must(operationsv1alpha2.AddToContainer(container, clientsets.Kubernetes())) urlruntime.Must(resourcesv1alpha2.AddToContainer(container, clientsets.Kubernetes(), informerFactory, "")) urlruntime.Must(resourcesv1alpha3.AddToContainer(container, informerFactory, nil)) - urlruntime.Must(tenantv1alpha2.AddToContainer(container, informerFactory, nil, nil, nil, nil, nil, nil, nil)) + urlruntime.Must(tenantv1alpha2.AddToContainer(container, informerFactory, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)) urlruntime.Must(terminalv1alpha2.AddToContainer(container, clientsets.Kubernetes(), nil)) urlruntime.Must(metricsv1alpha2.AddToContainer(container)) urlruntime.Must(networkv1alpha2.AddToContainer(container, ""))