Merge pull request #3018 from yunkunrao/metering

Add metering api

go.mod

@@ -45,6 +45,7 @@ require (
 	github.com/gorilla/websocket v1.4.1
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/json-iterator/go v1.1.10
+	github.com/jszwec/csvutil v1.5.0
 	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kubernetes-csi/external-snapshotter/client/v3 v3.0.0

go.sum

@@ -440,6 +440,8 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o=
+github.com/jszwec/csvutil v1.5.0 h1:ErLnF1Qzzt9svk8CUY7CyLl/W9eET+KWPIZWkE1o6JM=
+github.com/jszwec/csvutil v1.5.0/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=

pkg/api/metering/v1alpha1/types.go

@@ -0,0 +1,89 @@
+package v1alpha1
+
+import (
+	"time"
+
+	"github.com/emicklei/go-restful"
+	"kubesphere.io/kubesphere/pkg/apiserver/query"
+	model "kubesphere.io/kubesphere/pkg/models/monitoring"
+	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+)
+
+const (
+	DefaultStep   = 10 * time.Minute
+	DefaultFilter = ".*"
+	DefaultOrder  = model.OrderDescending
+	DefaultPage   = 1
+	DefaultLimit  = 5
+
+	ErrNoHit             = "'end' or 'time' must be after the namespace creation time."
+	ErrParamConflict     = "'time' and the combination of 'start' and 'end' are mutually exclusive."
+	ErrInvalidStartEnd   = "'start' must be before 'end'."
+	ErrInvalidPage       = "Invalid parameter 'page'."
+	ErrInvalidLimit      = "Invalid parameter 'limit'."
+	ErrParameterNotfound = "Parmameter [%s] not found"
+	ErrResourceNotfound  = "resource not found"
+	ErrScopeNotAllowed   = "scope [%s] not allowed"
+)
+
+type Query struct {
+	Level            monitoring.Level
+	Operation        string
+	LabelSelector    string
+	Time             string
+	Start            string
+	End              string
+	Step             string
+	Target           string
+	Order            string
+	Page             string
+	Limit            string
+	MetricFilter     string
+	ResourceFilter   string
+	NodeName         string
+	WorkspaceName    string
+	NamespaceName    string
+	WorkloadKind     string
+	WorkloadName     string
+	PodName          string
+	Applications     string
+	Services         string
+	StorageClassName string
+	PVCFilter        string
+}
+
+func ParseQueryParameter(req *restful.Request) *Query {
+	var q Query
+
+	q.LabelSelector = req.QueryParameter(query.ParameterLabelSelector)
+
+	q.Level = monitoring.Level(monitoring.MeteringLevelMap[req.QueryParameter("level")])
+	q.Operation = req.QueryParameter("operation")
+	q.Time = req.QueryParameter("time")
+	q.Start = req.QueryParameter("start")
+	q.End = req.QueryParameter("end")
+	q.Step = req.QueryParameter("step")
+	q.Target = req.QueryParameter("sort_metric")
+	q.Order = req.QueryParameter("sort_type")
+	q.Page = req.QueryParameter("page")
+	q.Limit = req.QueryParameter("limit")
+	q.MetricFilter = req.QueryParameter("metrics_filter")
+	q.ResourceFilter = req.QueryParameter("resources_filter")
+	q.WorkspaceName = req.QueryParameter("workspace")
+
+	q.NamespaceName = req.QueryParameter("namespace")
+	if q.NamespaceName == "" {
+		q.NamespaceName = req.PathParameter("namespace")
+	}
+
+	q.NodeName = req.QueryParameter("node")
+	q.WorkloadKind = req.QueryParameter("kind")
+	q.WorkloadName = req.QueryParameter("workload")
+	q.PodName = req.QueryParameter("pod")
+	q.Applications = req.QueryParameter("applications")
+	q.Services = req.QueryParameter("services")
+	q.StorageClassName = req.QueryParameter("storageclass")
+	q.PVCFilter = req.QueryParameter("pvc_filter")
+
+	return &q
+}

pkg/apiserver/apiserver.go

@@ -61,6 +61,7 @@ import (
 	devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2"
 	devopsv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha3"
 	iamapi "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2"
+	meteringv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/metering/v1alpha1"
 	monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3"
 	networkv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/network/v1alpha2"
 	notificationv1 "kubesphere.io/kubesphere/pkg/kapis/notification/v1"
@@ -215,12 +216,13 @@ func (s *APIServer) installKubeSphereAPIs() {
 	urlruntime.Must(configv1alpha2.AddToContainer(s.container, s.Config))
 	urlruntime.Must(resourcev1alpha3.AddToContainer(s.container, s.InformerFactory, s.RuntimeCache))
 	urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.MetricsClient, s.InformerFactory, s.OpenpitrixClient))
+	urlruntime.Must(meteringv1alpha1.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.InformerFactory, s.OpenpitrixClient, s.RuntimeCache))
 	urlruntime.Must(openpitrixv1.AddToContainer(s.container, s.InformerFactory, s.OpenpitrixClient))
 	urlruntime.Must(operationsv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes()))
 	urlruntime.Must(resourcesv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.InformerFactory,
 		s.KubernetesClient.Master()))
 	urlruntime.Must(tenantv1alpha2.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(),
-		s.KubernetesClient.KubeSphere(), s.EventsClient, s.LoggingClient, s.AuditingClient, amOperator, rbacAuthorizer))
+		s.KubernetesClient.KubeSphere(), s.EventsClient, s.LoggingClient, s.AuditingClient, amOperator, rbacAuthorizer, s.MonitoringClient, s.OpenpitrixClient, s.RuntimeCache))
 	urlruntime.Must(terminalv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.KubernetesClient.Config()))
 	urlruntime.Must(clusterkapisv1alpha1.AddToContainer(s.container,
 		s.InformerFactory.KubernetesSharedInformerFactory(),

pkg/apiserver/request/requestinfo.go

@@ -31,12 +31,11 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
+	k8srequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/klog"
 	"kubesphere.io/kubesphere/pkg/api"
 	"kubesphere.io/kubesphere/pkg/constants"
 	netutils "kubesphere.io/kubesphere/pkg/utils/net"
-
-	k8srequest "k8s.io/apiserver/pkg/endpoints/request"
 )
 
 type RequestInfoResolver interface {

pkg/constants/constants.go

@@ -105,7 +105,20 @@ const (
 	EventsQueryTag   = "Events Query"
 	AuditingQueryTag = "Auditing Query"
 
-	AlertingTag = "Alerting"
+	ClusterMetersTag   = "Cluster Meters"
+	NodeMetersTag      = "Node Meters"
+	WorkspaceMetersTag = "Workspace Meters"
+	NamespaceMetersTag = "Namespace Meters"
+	WorkloadMetersTag  = "Workload Meters"
+	PodMetersTag       = "Pod Meters"
+	ServiceMetricsTag  = "ServiceName Meters"
+
+	ApplicationReleaseName = "meta.helm.sh/release-name"
+	ApplicationReleaseNS   = "meta.helm.sh/release-namespace"
+
+	ApplicationName    = "app.kubernetes.io/name"
+	ApplicationVersion = "app.kubernetes.io/version"
+	AlertingTag        = "Alerting"
 )
 
 var (

pkg/kapis/metering/group.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2019 The KubeSphere authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package metering contains monitoring API versions
+package metering

pkg/kapis/metering/v1alpha1/handler.go

@@ -0,0 +1,45 @@
+/*
+
+ Copyright 2019 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+*/
+
+package v1alpha1
+
+import (
+	"github.com/emicklei/go-restful"
+	"k8s.io/client-go/kubernetes"
+	"kubesphere.io/kubesphere/pkg/informers"
+	monitorhle "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3"
+	resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
+	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
+)
+
+type meterHandler interface {
+	HandleClusterMetersQuery(req *restful.Request, resp *restful.Response)
+	HandleNodeMetersQuery(req *restful.Request, resp *restful.Response)
+	HandleWorkspaceMetersQuery(req *restful.Request, resp *restful.Response)
+	HandleNamespaceMetersQuery(re *restful.Request, resp *restful.Response)
+	HandleWorkloadMetersQuery(req *restful.Request, resp *restful.Response)
+	HandleApplicationMetersQuery(req *restful.Request, resp *restful.Response)
+	HandlePodMetersQuery(req *restful.Request, resp *restful.Response)
+	HandleServiceMetersQuery(req *restful.Request, resp *restful.Response)
+	HandlePVCMetersQuery(req *restful.Request, resp *restful.Response)
+}
+
+func newHandler(k kubernetes.Interface, m monitoring.Interface, f informers.InformerFactory, o openpitrix.Client, resourceGetter *resourcev1alpha3.ResourceGetter) meterHandler {
+	return monitorhle.NewHandler(k, m, nil, f, o, resourceGetter)
+}

pkg/kapis/metering/v1alpha1/register.go

@@ -0,0 +1,357 @@
+/*
+
+ Copyright 2019 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+*/
+package v1alpha1
+
+import (
+	"net/http"
+
+	"github.com/emicklei/go-restful"
+	"github.com/emicklei/go-restful-openapi"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes"
+	"kubesphere.io/kubesphere/pkg/apiserver/runtime"
+	"kubesphere.io/kubesphere/pkg/constants"
+	"kubesphere.io/kubesphere/pkg/informers"
+	monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3"
+	model "kubesphere.io/kubesphere/pkg/models/monitoring"
+	resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
+	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+)
+
+const (
+	groupName = "metering.kubesphere.io"
+	respOK    = "ok"
+)
+
+var GroupVersion = schema.GroupVersion{Group: groupName, Version: "v1alpha1"}
+
+func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, meteringClient monitoring.Interface, factory informers.InformerFactory, opClient openpitrix.Client, cache cache.Cache) error {
+	ws := runtime.NewWebService(GroupVersion)
+
+	h := newHandler(k8sClient, meteringClient, factory, opClient, resourcev1alpha3.NewResourceGetter(factory, cache))
+
+	ws.Route(ws.GET("/cluster").
+		To(h.HandleClusterMetersQuery).
+		Doc("Get cluster-level meter data.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which meter data to return. For example, the following filter matches both cluster CPU usage and disk usage: `meter_cluster_cpu_usage|meter_cluster_memory_usage`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.ClusterMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/nodes").
+		To(h.HandleNodeMetersQuery).
+		Doc("Get node-level meter data of all nodes.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which meter data to return. For example, the following filter matches both node CPU usage and disk usage: `meter_node_cpu_usage|meter_node_memory_usage`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("resources_filter", "The node filter consists of a regexp pattern. It specifies which node data to return. For example, the following filter matches both node i-caojnter and i-cmu82ogj: `i-caojnter|i-cmu82ogj`.").DataType("string").Required(false)).
+		Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("pvc_filter", "The PVCs filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort nodes by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
+		Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NodeMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/nodes/{node}").
+		To(h.HandleNodeMetersQuery).
+		Doc("Get node-level meter data of the specific node.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.PathParameter("node", "Node name.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which meter data to return. For example, the following filter matches both node CPU usage and disk usage: `meter_node_cpu_usage|meter_node_memory_usage`.").DataType("string").Required(false)).
+		Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("pvc_filter", "The PVCs filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NodeMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/workspaces").
+		To(h.HandleWorkspaceMetersQuery).
+		Doc("Get workspace-level meter data of all workspaces.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workspace CPU usage and memory usage: `meter_workspace_cpu_usage|meter_workspace_memory_usage`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("resources_filter", "The workspace filter consists of a regexp pattern. It specifies which workspace data to return.").DataType("string").Required(false)).
+		Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("pvc_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort workspaces by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
+		Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkspaceMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/workspaces/{workspace}").
+		To(h.HandleWorkspaceMetersQuery).
+		Doc("Get workspace-level meter data of a specific workspace.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.PathParameter("workspace", "Workspace name.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workspace CPU usage and memory usage: `meter_workspace_cpu_usage|meter_workspace_memory_usage`.").DataType("string").Required(false)).
+		Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("pvc_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("type", "Additional operations. Currently available types is statistics. It retrieves the total number of namespaces, devops projects, members and roles in this workspace at the moment.").DataType("string").Required(false)).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkspaceMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/workspaces/{workspace}/namespaces").
+		To(h.HandleNamespaceMetersQuery).
+		Doc("Get namespace-level meter data of a specific workspace.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.PathParameter("workspace", "Workspace name.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both namespace CPU usage and memory usage: `meter_namespace_cpu_usage|meter_namespace_memory_usage_wo_cache`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("resources_filter", "The namespace filter consists of a regexp pattern. It specifies which namespace data to return. For example, the following filter matches both namespace test and kube-system: `test|kube-system`.").DataType("string").Required(false)).
+		Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("pvc_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort namespaces by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
+		Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/namespaces").
+		To(h.HandleNamespaceMetersQuery).
+		Doc("Get namespace-level meter data of all namespaces.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both namespace CPU usage and memory usage: `meter_namespace_cpu_usage|meter_namespace_memory_usage_wo_cache`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("resources_filter", "The namespace filter consists of a regexp pattern. It specifies which namespace data to return. For example, the following filter matches both namespace test and kube-system: `test|kube-system`.").DataType("string").Required(false)).
+		Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("pvc_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort namespaces by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
+		Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/namespaces/{namespace}").
+		To(h.HandleNamespaceMetersQuery).
+		Doc("Get namespace-level meter data of the specific namespace.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both namespace CPU usage and memory usage: `meter_namespace_cpu_usage|meter_namespace_memory_usage_wo_cache`.").DataType("string").Required(false)).
+		Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("pvc_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/namespaces/{namespace}/workloads").
+		To(h.HandleWorkloadMetersQuery).
+		Doc("Get workload-level meter data of all workloads which belongs to a specific kind.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("kind", "Workload kind. One of deployment, daemonset, statefulset.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workload CPU usage and memory usage: `meter_workload_cpu_usage|meter_workload_memory_usage_wo_cache`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("resources_filter", "The workload filter consists of a regexp pattern. It specifies which workload data to return. For example, the following filter matches any workload whose name begins with prometheus: `prometheus.*`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort workloads by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
+		Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkloadMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/namespaces/{namespace}/applications").
+		To(h.HandleApplicationMetersQuery).
+		Doc("Get app-level meter data of a specific application. Navigate to the app by the app's namespace.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("applications", "Appliction names, format app_name[:app_version](such as nginx:v1, nignx) which are joined by \"|\" ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_application_cpu_usage|meter_application_memory_usage_wo_cache`.").DataType("string").Required(false)).
+		Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort pods by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/namespaces/{namespace}/pods").
+		To(h.HandlePodMetersQuery).
+		Doc("Get pod-level meter data of the specific namespace's pods.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_pod_cpu_usage|meter_pod_memory_usage`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("resources_filter", "The pod filter consists of a regexp pattern. It specifies which pod data to return. For example, the following filter matches any pod whose name begins with redis: `redis.*`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort pods by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
+		Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}").
+		To(h.HandlePodMetersQuery).
+		Doc("Get pod-level meter data of a specific pod. Navigate to the pod by the pod's namespace.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
+		Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_pod_cpu_usage|_meter_pod_memory_usage`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/namespaces/{namespace}/workloads/{workload}/pods").
+		To(h.HandlePodMetersQuery).
+		Doc("Get pod-level meter data of a specific workload's pods. Navigate to the workload by the namespace.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
+		Param(ws.PathParameter("workload", "Workload name.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("kind", "Workload kind. One of deployment, daemonset, statefulset.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_pod_cpu_usage|meter_pod_memory_usage`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("resources_filter", "The pod filter consists of a regexp pattern. It specifies which pod data to return. For example, the following filter matches any pod whose name begins with redis: `redis.*`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort pods by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
+		Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/nodes/{node}/pods").
+		To(h.HandlePodMetersQuery).
+		Doc("Get pod-level meter data of all pods on a specific node.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.PathParameter("node", "Node name.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_pod_cpu_usage|meter_pod_memory_usage`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("resources_filter", "The pod filter consists of a regexp pattern. It specifies which pod data to return. For example, the following filter matches any pod whose name begins with redis: `redis.*`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort pods by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
+		Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/nodes/{node}/pods/{pod}").
+		To(h.HandlePodMetersQuery).
+		Doc("Get pod-level meter data of a specific pod. Navigate to the pod by the node where it is scheduled.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.PathParameter("node", "Node name.").DataType("string").Required(true)).
+		Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_pod_cpu_usage|meter_pod_memory_usage`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.PodMetersTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/namespaces/{namespace}/services").
+		To(h.HandleServiceMetersQuery).
+		Doc("Get service-level meter data of the specific namespace's pods.").
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both pod CPU usage and memory usage: `meter_pod_cpu_usage|meter_pod_memory_usage`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("services", "Services which are joined by \"|\".").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort pods by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
+		Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.ServiceMetricsTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, respOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
+
+	c.Add(ws)
+	return nil
+}

pkg/kapis/monitoring/v1alpha3/handler.go

@@ -20,14 +20,17 @@ package v1alpha3
 
 import (
 	"errors"
+	"regexp"
+	"strings"
+
 	"github.com/emicklei/go-restful"
 	"k8s.io/client-go/kubernetes"
 	"kubesphere.io/kubesphere/pkg/api"
 	"kubesphere.io/kubesphere/pkg/informers"
 	model "kubesphere.io/kubesphere/pkg/models/monitoring"
+	resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
 	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
-	"regexp"
 )
 
 type handler struct {
@@ -35,8 +38,8 @@ type handler struct {
 	mo model.MonitoringOperator
 }
 
-func newHandler(k kubernetes.Interface, monitoringClient monitoring.Interface, metricsClient monitoring.Interface, f informers.InformerFactory, o openpitrix.Client) *handler {
-	return &handler{k, model.NewMonitoringOperator(monitoringClient, metricsClient, k, f, o)}
+func NewHandler(k kubernetes.Interface, monitoringClient monitoring.Interface, metricsClient monitoring.Interface, f informers.InformerFactory, o openpitrix.Client, resourceGetter *resourcev1alpha3.ResourceGetter) *handler {
+	return &handler{k, model.NewMonitoringOperator(monitoringClient, metricsClient, k, f, o, resourceGetter)}
 }
 
 func (h handler) handleKubeSphereMetricsQuery(req *restful.Request, resp *restful.Response) {
@@ -186,6 +189,10 @@ func (h handler) handleNamedMetricsQuery(resp *restful.Response, q queryOptions)
 
 	var metrics []string
 	for _, metric := range q.namedMetrics {
+		if strings.HasPrefix(metric, model.MetricMeterPrefix) {
+			// skip meter metric
+			continue
+		}
 		ok, _ := regexp.MatchString(q.metricFilter, metric)
 		if ok {
 			metrics = append(metrics, metric)

pkg/kapis/monitoring/v1alpha3/helper.go

@@ -17,14 +17,21 @@ limitations under the License.
 package v1alpha3
 
 import (
+	"bytes"
 	"context"
+	"fmt"
+	"github.com/jszwec/csvutil"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/emicklei/go-restful"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"kubesphere.io/kubesphere/pkg/api"
 	model "kubesphere.io/kubesphere/pkg/models/monitoring"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
-	"strconv"
-	"time"
 )
 
 const (
@@ -34,47 +41,56 @@ const (
 	DefaultPage   = 1
 	DefaultLimit  = 5
 
+	OperationQuery  = "query"
+	OperationExport = "export"
+
 	ComponentEtcd      = "etcd"
 	ComponentAPIServer = "apiserver"
 	ComponentScheduler = "scheduler"
 
-	ErrNoHit           = "'end' or 'time' must be after the namespace creation time."
-	ErrParamConflict   = "'time' and the combination of 'start' and 'end' are mutually exclusive."
-	ErrInvalidStartEnd = "'start' must be before 'end'."
-	ErrInvalidPage     = "Invalid parameter 'page'."
-	ErrInvalidLimit    = "Invalid parameter 'limit'."
+	ErrNoHit             = "'end' or 'time' must be after the namespace creation time."
+	ErrParamConflict     = "'time' and the combination of 'start' and 'end' are mutually exclusive."
+	ErrInvalidStartEnd   = "'start' must be before 'end'."
+	ErrInvalidPage       = "Invalid parameter 'page'."
+	ErrInvalidLimit      = "Invalid parameter 'limit'."
+	ErrParameterNotfound = "Parmameter [%s] not found"
 )
 
 type reqParams struct {
-	time                      string
-	start                     string
-	end                       string
-	step                      string
-	target                    string
-	order                     string
-	page                      string
-	limit                     string
-	metricFilter              string
-	namespacedResourcesFilter string
-	resourceFilter            string
-	nodeName                  string
-	workspaceName             string
-	namespaceName             string
-	workloadKind              string
-	workloadName              string
-	podName                   string
-	containerName             string
-	pvcName                   string
-	storageClassName          string
-	componentType             string
-	expression                string
-	metric                    string
+	operation        string
+	time             string
+	start            string
+	end              string
+	step             string
+	target           string
+	order            string
+	page             string
+	limit            string
+	metricFilter     string
+	resourceFilter   string
+	nodeName         string
+	workspaceName    string
+	namespaceName    string
+	workloadKind     string
+	workloadName     string
+	podName          string
+	containerName    string
+	pvcName          string
+	storageClassName string
+	componentType    string
+	expression       string
+	metric           string
+	applications     string
+	services         string
+	pvcFilter        string
 }
 
 type queryOptions struct {
 	metricFilter string
 	namedMetrics []string
 
+	Operation string
+
 	start time.Time
 	end   time.Time
 	time  time.Time
@@ -99,6 +115,7 @@ func (q queryOptions) shouldSort() bool {
 
 func parseRequestParams(req *restful.Request) reqParams {
 	var r reqParams
+	r.operation = req.QueryParameter("operation")
 	r.time = req.QueryParameter("time")
 	r.start = req.QueryParameter("start")
 	r.end = req.QueryParameter("end")
@@ -108,12 +125,24 @@ func parseRequestParams(req *restful.Request) reqParams {
 	r.page = req.QueryParameter("page")
 	r.limit = req.QueryParameter("limit")
 	r.metricFilter = req.QueryParameter("metrics_filter")
-	r.namespacedResourcesFilter = req.QueryParameter("namespaced_resources_filter")
 	r.resourceFilter = req.QueryParameter("resources_filter")
-	r.nodeName = req.PathParameter("node")
 	r.workspaceName = req.PathParameter("workspace")
 	r.namespaceName = req.PathParameter("namespace")
-	r.workloadKind = req.PathParameter("kind")
+
+	if req.QueryParameter("node") != "" {
+		r.nodeName = req.QueryParameter("node")
+	} else {
+		// compatible with monitoring request
+		r.nodeName = req.PathParameter("node")
+	}
+
+	if req.QueryParameter("kind") != "" {
+		r.workloadKind = req.QueryParameter("kind")
+	} else {
+		// compatible with monitoring request
+		r.workloadKind = req.PathParameter("kind")
+	}
+
 	r.workloadName = req.PathParameter("workload")
 	r.podName = req.PathParameter("pod")
 	r.containerName = req.PathParameter("container")
@@ -122,6 +151,10 @@ func parseRequestParams(req *restful.Request) reqParams {
 	r.componentType = req.PathParameter("component")
 	r.expression = req.QueryParameter("expr")
 	r.metric = req.QueryParameter("metric")
+	r.applications = req.QueryParameter("applications")
+	r.services = req.QueryParameter("services")
+	r.pvcFilter = req.QueryParameter("pvc_filter")
+
 	return r
 }
 
@@ -135,70 +168,109 @@ func (h handler) makeQueryOptions(r reqParams, lvl monitoring.Level) (q queryOpt
 		q.metricFilter = DefaultFilter
 	}
 
+	q.Operation = r.operation
+	if r.operation == "" {
+		q.Operation = OperationQuery
+	}
+
 	switch lvl {
 	case monitoring.LevelCluster:
 		q.option = monitoring.ClusterOption{}
 		q.namedMetrics = model.ClusterMetrics
+
 	case monitoring.LevelNode:
 		q.identifier = model.IdentifierNode
-		q.namedMetrics = model.NodeMetrics
 		q.option = monitoring.NodeOption{
-			ResourceFilter: r.resourceFilter,
-			NodeName:       r.nodeName,
+			ResourceFilter:   r.resourceFilter,
+			NodeName:         r.nodeName,
+			PVCFilter:        r.pvcFilter,        // metering pvc
+			StorageClassName: r.storageClassName, // metering pvc
 		}
+		q.namedMetrics = model.NodeMetrics
+
 	case monitoring.LevelWorkspace:
 		q.identifier = model.IdentifierWorkspace
-		q.namedMetrics = model.WorkspaceMetrics
 		q.option = monitoring.WorkspaceOption{
-			ResourceFilter: r.resourceFilter,
-			WorkspaceName:  r.workspaceName,
+			ResourceFilter:   r.resourceFilter,
+			WorkspaceName:    r.workspaceName,
+			PVCFilter:        r.pvcFilter,        // metering pvc
+			StorageClassName: r.storageClassName, // metering pvc
 		}
+		q.namedMetrics = model.WorkspaceMetrics
+
 	case monitoring.LevelNamespace:
 		q.identifier = model.IdentifierNamespace
-		q.namedMetrics = model.NamespaceMetrics
 		q.option = monitoring.NamespaceOption{
-			ResourceFilter: r.resourceFilter,
-			WorkspaceName:  r.workspaceName,
-			NamespaceName:  r.namespaceName,
+			ResourceFilter:   r.resourceFilter,
+			WorkspaceName:    r.workspaceName,
+			NamespaceName:    r.namespaceName,
+			PVCFilter:        r.pvcFilter,        // metering pvc
+			StorageClassName: r.storageClassName, // metering pvc
 		}
+		q.namedMetrics = model.NamespaceMetrics
+
+	case monitoring.LevelApplication:
+		q.identifier = model.IdentifierApplication
+		if r.namespaceName == "" {
+			return q, errors.New(fmt.Sprintf(ErrParameterNotfound, "namespace"))
+		}
+
+		q.option = monitoring.ApplicationsOption{
+			NamespaceName:    r.namespaceName,
+			Applications:     strings.Split(r.applications, "|"),
+			StorageClassName: r.storageClassName, // metering pvc
+		}
+		q.namedMetrics = model.ApplicationMetrics
+
 	case monitoring.LevelWorkload:
 		q.identifier = model.IdentifierWorkload
-		q.namedMetrics = model.WorkloadMetrics
 		q.option = monitoring.WorkloadOption{
 			ResourceFilter: r.resourceFilter,
 			NamespaceName:  r.namespaceName,
 			WorkloadKind:   r.workloadKind,
 		}
+		q.namedMetrics = model.WorkloadMetrics
+
 	case monitoring.LevelPod:
 		q.identifier = model.IdentifierPod
-		q.namedMetrics = model.PodMetrics
 		q.option = monitoring.PodOption{
-			NamespacedResourcesFilter: r.namespacedResourcesFilter,
-			ResourceFilter:            r.resourceFilter,
-			NodeName:                  r.nodeName,
-			NamespaceName:             r.namespaceName,
-			WorkloadKind:              r.workloadKind,
-			WorkloadName:              r.workloadName,
-			PodName:                   r.podName,
+			ResourceFilter: r.resourceFilter,
+			NodeName:       r.nodeName,
+			NamespaceName:  r.namespaceName,
+			WorkloadKind:   r.workloadKind,
+			WorkloadName:   r.workloadName,
+			PodName:        r.podName,
 		}
+		q.namedMetrics = model.PodMetrics
+
+	case monitoring.LevelService:
+		q.identifier = model.IdentifierService
+		q.option = monitoring.ServicesOption{
+			NamespaceName: r.namespaceName,
+			Services:      strings.Split(r.services, "|"),
+		}
+		q.namedMetrics = model.ServiceMetrics
+
 	case monitoring.LevelContainer:
 		q.identifier = model.IdentifierContainer
-		q.namedMetrics = model.ContainerMetrics
 		q.option = monitoring.ContainerOption{
 			ResourceFilter: r.resourceFilter,
 			NamespaceName:  r.namespaceName,
 			PodName:        r.podName,
 			ContainerName:  r.containerName,
 		}
+		q.namedMetrics = model.ContainerMetrics
+
 	case monitoring.LevelPVC:
 		q.identifier = model.IdentifierPVC
-		q.namedMetrics = model.PVCMetrics
 		q.option = monitoring.PVCOption{
 			ResourceFilter:            r.resourceFilter,
 			NamespaceName:             r.namespaceName,
 			StorageClassName:          r.storageClassName,
 			PersistentVolumeClaimName: r.pvcName,
 		}
+		q.namedMetrics = model.PVCMetrics
+
 	case monitoring.LevelComponent:
 		q.option = monitoring.ComponentOption{}
 		switch r.componentType {
@@ -304,3 +376,35 @@ func (h handler) makeQueryOptions(r reqParams, lvl monitoring.Level) (q queryOpt
 
 	return q, nil
 }
+
+func ExportMetrics(resp *restful.Response, metrics model.Metrics) {
+	resp.Header().Set(restful.HEADER_ContentType, "text/plain")
+	resp.Header().Set("Content-Disposition", "attachment")
+
+	for i, _ := range metrics.Results {
+		ret := metrics.Results[i]
+		for j, _ := range ret.MetricValues {
+			ret.MetricValues[j].TransferToExportedMetricValue()
+		}
+	}
+
+	resBytes, err := csvutil.Marshal(metrics.Results)
+	if err != nil {
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	output := new(bytes.Buffer)
+	_, err = output.Write(resBytes)
+	if err != nil {
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	_, err = io.Copy(resp, output)
+	if err != nil {
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+	return
+}

pkg/kapis/monitoring/v1alpha3/helper_test.go

@@ -18,6 +18,9 @@ package v1alpha3
 
 import (
 	"fmt"
+	"testing"
+	"time"
+
 	"github.com/google/go-cmp/cmp"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -25,8 +28,6 @@ import (
 	"kubesphere.io/kubesphere/pkg/informers"
 	model "kubesphere.io/kubesphere/pkg/models/monitoring"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
-	"testing"
-	"time"
 )
 
 func TestIsRangeQuery(t *testing.T) {
@@ -84,6 +85,7 @@ func TestParseRequestParams(t *testing.T) {
 				metricFilter: ".*",
 				namedMetrics: model.ClusterMetrics,
 				option:       monitoring.ClusterOption{},
+				Operation:    OperationQuery,
 			},
 			expectedErr: false,
 		},
@@ -114,6 +116,7 @@ func TestParseRequestParams(t *testing.T) {
 					ResourceFilter: ".*",
 					NamespaceName:  "default",
 				},
+				Operation: OperationQuery,
 			},
 			expectedErr: false,
 		},
@@ -181,6 +184,7 @@ func TestParseRequestParams(t *testing.T) {
 				metricFilter: "etcd_server_list",
 				namedMetrics: model.EtcdMetrics,
 				option:       monitoring.ComponentOption{},
+				Operation:    OperationQuery,
 			},
 			expectedErr: false,
 		},
@@ -208,6 +212,7 @@ func TestParseRequestParams(t *testing.T) {
 				order:      "desc",
 				page:       1,
 				limit:      10,
+				Operation:  OperationQuery,
 			},
 			expectedErr: false,
 		},
@@ -217,7 +222,7 @@ func TestParseRequestParams(t *testing.T) {
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
 			client := fake.NewSimpleClientset(&tt.namespace)
 			fakeInformerFactory := informers.NewInformerFactories(client, nil, nil, nil, nil, nil)
-			handler := newHandler(client, nil, nil, fakeInformerFactory, nil)
+			handler := NewHandler(client, nil, nil, fakeInformerFactory, nil, nil)
 
 			result, err := handler.makeQueryOptions(tt.params, tt.lvl)
 			if err != nil {

pkg/kapis/monitoring/v1alpha3/meter.go

@@ -0,0 +1,327 @@
+package v1alpha3
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/emicklei/go-restful"
+	"k8s.io/klog"
+	"kubesphere.io/kubesphere/pkg/api"
+	model "kubesphere.io/kubesphere/pkg/models/monitoring"
+	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+)
+
+func (h handler) HandleClusterMetersQuery(req *restful.Request, resp *restful.Response) {
+	params := parseRequestParams(req)
+	opt, err := h.makeQueryOptions(params, monitoring.LevelCluster)
+	if err != nil {
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+	h.handleNamedMetersQuery(resp, opt)
+}
+
+func getMetricPosMap(metrics []monitoring.Metric) map[string]int {
+	var metricMap = make(map[string]int)
+
+	for i, m := range metrics {
+		metricMap[m.MetricName] = i
+	}
+
+	return metricMap
+}
+
+func (h handler) handleApplicationMetersQuery(meters []string, resp *restful.Response, q queryOptions) {
+	var metricMap = make(map[string]int)
+	var res model.Metrics
+	var current_res model.Metrics
+	var err error
+
+	aso, ok := q.option.(monitoring.ApplicationsOption)
+	if !ok {
+		klog.Error("invalid application option")
+		return
+	}
+	componentsMap := h.mo.GetAppComponentsMap(aso.NamespaceName, aso.Applications)
+
+	for k, _ := range componentsMap {
+		opt := monitoring.ApplicationOption{
+			NamespaceName:         aso.NamespaceName,
+			Application:           k,
+			ApplicationComponents: componentsMap[k],
+			StorageClassName:      aso.StorageClassName,
+		}
+
+		if q.isRangeQuery() {
+			current_res, err = h.mo.GetNamedMetersOverTime(meters, q.start, q.end, q.step, opt)
+		} else {
+			current_res, err = h.mo.GetNamedMeters(meters, q.time, opt)
+		}
+		if err != nil {
+			api.HandleBadRequest(resp, nil, err)
+			return
+		}
+
+		if res.Results == nil {
+			res = current_res
+			metricMap = getMetricPosMap(res.Results)
+		} else {
+			for _, cur_res := range current_res.Results {
+				pos, ok := metricMap[cur_res.MetricName]
+				if ok {
+					res.Results[pos].MetricValues = append(res.Results[pos].MetricValues, cur_res.MetricValues...)
+				} else {
+					res.Results = append(res.Results, cur_res)
+				}
+			}
+		}
+	}
+
+	if !q.isRangeQuery() && q.shouldSort() {
+		res = *res.Sort(q.target, q.order, q.identifier).Page(q.page, q.limit)
+	}
+
+	if q.Operation == OperationExport {
+		ExportMetrics(resp, res)
+		return
+	}
+
+	resp.WriteAsJson(res)
+}
+
+func (h handler) handleServiceMetersQuery(meters []string, resp *restful.Response, q queryOptions) {
+	var metricMap = make(map[string]int)
+	var res model.Metrics
+	var current_res model.Metrics
+	var err error
+
+	sso, ok := q.option.(monitoring.ServicesOption)
+	if !ok {
+		klog.Error("invalid service option")
+		return
+	}
+	svcPodsMap := h.mo.GetSerivePodsMap(sso.NamespaceName, sso.Services)
+
+	for k, _ := range svcPodsMap {
+		opt := monitoring.ServiceOption{
+			NamespaceName: sso.NamespaceName,
+			ServiceName:   k,
+			PodNames:      svcPodsMap[k],
+		}
+
+		if q.isRangeQuery() {
+			current_res, err = h.mo.GetNamedMetersOverTime(meters, q.start, q.end, q.step, opt)
+		} else {
+			current_res, err = h.mo.GetNamedMeters(meters, q.time, opt)
+		}
+		if err != nil {
+			api.HandleBadRequest(resp, nil, err)
+			return
+		}
+
+		if res.Results == nil {
+			res = current_res
+			metricMap = getMetricPosMap(res.Results)
+		} else {
+			for _, cur_res := range current_res.Results {
+				pos, ok := metricMap[cur_res.MetricName]
+				if ok {
+					res.Results[pos].MetricValues = append(res.Results[pos].MetricValues, cur_res.MetricValues...)
+				} else {
+					res.Results = append(res.Results, cur_res)
+				}
+			}
+		}
+	}
+
+	if !q.isRangeQuery() && q.shouldSort() {
+		res = *res.Sort(q.target, q.order, q.identifier).Page(q.page, q.limit)
+	}
+
+	if q.Operation == OperationExport {
+		ExportMetrics(resp, res)
+		return
+	}
+
+	resp.WriteAsJson(res)
+}
+
+func (h handler) handleNamedMetersQuery(resp *restful.Response, q queryOptions) {
+	var res model.Metrics
+	var err error
+
+	var meters []string
+	for _, meter := range q.namedMetrics {
+		if !strings.HasPrefix(meter, model.MetricMeterPrefix) {
+			// skip non-meter metric
+			continue
+		}
+
+		ok, _ := regexp.MatchString(q.metricFilter, meter)
+		if ok {
+			meters = append(meters, meter)
+		}
+	}
+
+	if len(meters) == 0 {
+		klog.Info("no meters found")
+		resp.WriteAsJson(res)
+		return
+	}
+
+	_, ok := q.option.(monitoring.ApplicationsOption)
+	if ok {
+		h.handleApplicationMetersQuery(meters, resp, q)
+		return
+	}
+
+	_, ok = q.option.(monitoring.ServicesOption)
+	if ok {
+		h.handleServiceMetersQuery(meters, resp, q)
+		return
+	}
+
+	if q.isRangeQuery() {
+		res, err = h.mo.GetNamedMetersOverTime(meters, q.start, q.end, q.step, q.option)
+		if err != nil {
+			api.HandleBadRequest(resp, nil, err)
+			return
+		}
+	} else {
+		res, err = h.mo.GetNamedMeters(meters, q.time, q.option)
+		if err != nil {
+			api.HandleBadRequest(resp, nil, err)
+			return
+		}
+
+		if q.shouldSort() {
+			res = *res.Sort(q.target, q.order, q.identifier).Page(q.page, q.limit)
+		}
+	}
+
+	if q.Operation == OperationExport {
+		ExportMetrics(resp, res)
+		return
+	}
+
+	resp.WriteAsJson(res)
+}
+
+func (h handler) HandleNodeMetersQuery(req *restful.Request, resp *restful.Response) {
+	params := parseRequestParams(req)
+	opt, err := h.makeQueryOptions(params, monitoring.LevelNode)
+	if err != nil {
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+	h.handleNamedMetersQuery(resp, opt)
+}
+
+func (h handler) HandleWorkspaceMetersQuery(req *restful.Request, resp *restful.Response) {
+	params := parseRequestParams(req)
+	opt, err := h.makeQueryOptions(params, monitoring.LevelWorkspace)
+	if err != nil {
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	h.handleNamedMetersQuery(resp, opt)
+}
+
+func (h handler) HandleNamespaceMetersQuery(req *restful.Request, resp *restful.Response) {
+	params := parseRequestParams(req)
+	opt, err := h.makeQueryOptions(params, monitoring.LevelNamespace)
+	if err != nil {
+		if err.Error() == ErrNoHit {
+			res := handleNoHit(opt.namedMetrics)
+			resp.WriteAsJson(res)
+			return
+		}
+
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	h.handleNamedMetersQuery(resp, opt)
+}
+
+func (h handler) HandleWorkloadMetersQuery(req *restful.Request, resp *restful.Response) {
+	params := parseRequestParams(req)
+	opt, err := h.makeQueryOptions(params, monitoring.LevelWorkload)
+	if err != nil {
+		if err.Error() == ErrNoHit {
+			res := handleNoHit(opt.namedMetrics)
+			resp.WriteAsJson(res)
+			return
+		}
+
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+	h.handleNamedMetersQuery(resp, opt)
+}
+
+func (h handler) HandleApplicationMetersQuery(req *restful.Request, resp *restful.Response) {
+	params := parseRequestParams(req)
+	opt, err := h.makeQueryOptions(params, monitoring.LevelApplication)
+	if err != nil {
+		if err.Error() == ErrNoHit {
+			res := handleNoHit(opt.namedMetrics)
+			resp.WriteAsJson(res)
+			return
+		}
+
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+	h.handleNamedMetersQuery(resp, opt)
+}
+
+func (h handler) HandlePodMetersQuery(req *restful.Request, resp *restful.Response) {
+	params := parseRequestParams(req)
+	opt, err := h.makeQueryOptions(params, monitoring.LevelPod)
+	if err != nil {
+		if err.Error() == ErrNoHit {
+			res := handleNoHit(opt.namedMetrics)
+			resp.WriteAsJson(res)
+			return
+		}
+
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+	h.handleNamedMetersQuery(resp, opt)
+}
+
+func (h handler) HandleServiceMetersQuery(req *restful.Request, resp *restful.Response) {
+	params := parseRequestParams(req)
+	opt, err := h.makeQueryOptions(params, monitoring.LevelService)
+	if err != nil {
+		if err.Error() == ErrNoHit {
+			res := handleNoHit(opt.namedMetrics)
+			resp.WriteAsJson(res)
+			return
+		}
+
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	h.handleNamedMetersQuery(resp, opt)
+}
+
+func (h handler) HandlePVCMetersQuery(req *restful.Request, resp *restful.Response) {
+	params := parseRequestParams(req)
+	opt, err := h.makeQueryOptions(params, monitoring.LevelPVC)
+	if err != nil {
+		if err.Error() == ErrNoHit {
+			res := handleNoHit(opt.namedMetrics)
+			resp.WriteAsJson(res)
+			return
+		}
+
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+	h.handleNamedMetersQuery(resp, opt)
+}

pkg/kapis/monitoring/v1alpha3/register.go

@@ -42,7 +42,7 @@ var GroupVersion = schema.GroupVersion{Group: groupName, Version: "v1alpha3"}
 func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, monitoringClient monitoring.Interface, metricsClient monitoring.Interface, factory informers.InformerFactory, opClient openpitrix.Client) error {
 	ws := runtime.NewWebService(GroupVersion)
 
-	h := newHandler(k8sClient, monitoringClient, metricsClient, factory, opClient)
+	h := NewHandler(k8sClient, monitoringClient, metricsClient, factory, opClient, nil)
 
 	ws.Route(ws.GET("/kubesphere").
 		To(h.handleKubeSphereMetricsQuery).

pkg/kapis/tenant/v1alpha2/handler.go

@@ -19,6 +19,7 @@ package v1alpha2
 import (
 	"encoding/json"
 	"fmt"
+
 	"github.com/emicklei/go-restful"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -37,11 +38,14 @@ import (
 	kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
 	"kubesphere.io/kubesphere/pkg/informers"
 	"kubesphere.io/kubesphere/pkg/models/iam/am"
+	resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
 	"kubesphere.io/kubesphere/pkg/models/tenant"
 	servererr "kubesphere.io/kubesphere/pkg/server/errors"
 	"kubesphere.io/kubesphere/pkg/simple/client/auditing"
 	"kubesphere.io/kubesphere/pkg/simple/client/events"
 	"kubesphere.io/kubesphere/pkg/simple/client/logging"
+	monitoringclient "kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+	opclient "kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 )
 
 type tenantHandler struct {
@@ -50,10 +54,12 @@ type tenantHandler struct {
 
 func newTenantHandler(factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface,
 	evtsClient events.Client, loggingClient logging.Client, auditingclient auditing.Client,
-	am am.AccessManagementInterface, authorizer authorizer.Authorizer) *tenantHandler {
+	am am.AccessManagementInterface, authorizer authorizer.Authorizer,
+	monitoringclient monitoringclient.Interface, opClient opclient.Client,
+	resourceGetter *resourcev1alpha3.ResourceGetter) *tenantHandler {
 
 	return &tenantHandler{
-		tenant: tenant.New(factory, k8sclient, ksclient, evtsClient, loggingClient, auditingclient, am, authorizer),
+		tenant: tenant.New(factory, k8sclient, ksclient, evtsClient, loggingClient, auditingclient, am, authorizer, monitoringclient, opClient, resourceGetter),
 	}
 }
 

pkg/kapis/tenant/v1alpha2/metering.go

@@ -0,0 +1,87 @@
+package v1alpha2
+
+import (
+	"fmt"
+
+	"github.com/emicklei/go-restful"
+	"k8s.io/klog"
+	"kubesphere.io/kubesphere/pkg/api"
+	meteringv1alpha1 "kubesphere.io/kubesphere/pkg/api/metering/v1alpha1"
+	"kubesphere.io/kubesphere/pkg/apiserver/request"
+	monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3"
+	"kubesphere.io/kubesphere/pkg/models/metering"
+	"kubesphere.io/kubesphere/pkg/models/monitoring"
+	monitoringclient "kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+)
+
+func (h *tenantHandler) QueryMeterings(req *restful.Request, resp *restful.Response) {
+
+	u, ok := request.UserFrom(req.Request.Context())
+	if !ok {
+		err := fmt.Errorf("cannot obtain user info")
+		klog.Errorln(err)
+		api.HandleForbidden(resp, req, err)
+		return
+	}
+
+	q := meteringv1alpha1.ParseQueryParameter(req)
+
+	res, err := h.tenant.Metering(u, q)
+	if err != nil {
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	if q.Operation == monitoringv1alpha3.OperationExport {
+		monitoringv1alpha3.ExportMetrics(resp, res)
+		return
+	}
+
+	resp.WriteAsJson(res)
+}
+
+func (h *tenantHandler) QueryMeteringsHierarchy(req *restful.Request, resp *restful.Response) {
+	u, ok := request.UserFrom(req.Request.Context())
+	if !ok {
+		err := fmt.Errorf("cannot obtain user info")
+		klog.Errorln(err)
+		api.HandleForbidden(resp, req, err)
+		return
+	}
+
+	q := meteringv1alpha1.ParseQueryParameter(req)
+	q.Level = monitoringclient.LevelPod
+
+	resourceStats, err := h.tenant.MeteringHierarchy(u, q)
+	if err != nil {
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	resp.WriteAsJson(resourceStats)
+}
+
+func (h *tenantHandler) HandlePriceInfoQuery(req *restful.Request, resp *restful.Response) {
+
+	var priceInfoResponse metering.PriceInfo
+	priceInfoResponse.Init()
+
+	meterConfig, err := monitoring.LoadYaml()
+	if err != nil {
+		klog.Warning(err)
+		resp.WriteAsJson(priceInfoResponse)
+		return
+	}
+
+	priceInfo := meterConfig.GetPriceInfo()
+	priceInfoResponse.Currency = priceInfo.CurrencyUnit
+	priceInfoResponse.CpuPerCorePerHour = priceInfo.CpuPerCorePerHour
+	priceInfoResponse.MemPerGigabytesPerHour = priceInfo.MemPerGigabytesPerHour
+	priceInfoResponse.IngressNetworkTrafficPerGiagabytesPerHour = priceInfo.IngressNetworkTrafficPerGiagabytesPerHour
+	priceInfoResponse.EgressNetworkTrafficPerGiagabytesPerHour = priceInfo.EgressNetworkTrafficPerGigabytesPerHour
+	priceInfoResponse.PvcPerGigabytesPerHour = priceInfo.PvcPerGigabytesPerHour
+
+	resp.WriteAsJson(priceInfoResponse)
+
+	return
+}

pkg/kapis/tenant/v1alpha2/register.go

@@ -17,6 +17,11 @@ limitations under the License.
 package v1alpha2
 
 import (
+	"net/http"
+
+	"kubesphere.io/kubesphere/pkg/models/metering"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+
 	"github.com/emicklei/go-restful"
 	"github.com/emicklei/go-restful-openapi"
 	corev1 "k8s.io/api/core/v1"
@@ -33,13 +38,17 @@ import (
 	kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
 	"kubesphere.io/kubesphere/pkg/constants"
 	"kubesphere.io/kubesphere/pkg/informers"
+	monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3"
 	"kubesphere.io/kubesphere/pkg/models"
 	"kubesphere.io/kubesphere/pkg/models/iam/am"
+	"kubesphere.io/kubesphere/pkg/models/monitoring"
+	resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
 	"kubesphere.io/kubesphere/pkg/server/errors"
 	"kubesphere.io/kubesphere/pkg/simple/client/auditing"
 	"kubesphere.io/kubesphere/pkg/simple/client/events"
 	"kubesphere.io/kubesphere/pkg/simple/client/logging"
-	"net/http"
+	monitoringclient "kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+	opclient "kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 )
 
 const (
@@ -54,11 +63,12 @@ func Resource(resource string) schema.GroupResource {
 
 func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8sclient kubernetes.Interface,
 	ksclient kubesphere.Interface, evtsClient events.Client, loggingClient logging.Client,
-	auditingclient auditing.Client, am am.AccessManagementInterface, authorizer authorizer.Authorizer) error {
+	auditingclient auditing.Client, am am.AccessManagementInterface, authorizer authorizer.Authorizer,
+	monitoringclient monitoringclient.Interface, opClient opclient.Client, cache cache.Cache) error {
 	mimePatch := []string{restful.MIME_JSON, runtime.MimeMergePatchJson, runtime.MimeJsonPatchJson}
 
 	ws := runtime.NewWebService(GroupVersion)
-	handler := newTenantHandler(factory, k8sclient, ksclient, evtsClient, loggingClient, auditingclient, am, authorizer)
+	handler := newTenantHandler(factory, k8sclient, ksclient, evtsClient, loggingClient, auditingclient, am, authorizer, monitoringclient, opClient, resourcev1alpha3.NewResourceGetter(factory, cache))
 
 	ws.Route(ws.GET("/clusters").
 		To(handler.ListClusters).
@@ -289,6 +299,49 @@ func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8s
 		Writes(auditingv1alpha1.APIResponse{}).
 		Returns(http.StatusOK, api.StatusOK, auditingv1alpha1.APIResponse{}))
 
+	ws.Route(ws.GET("/metering").
+		To(handler.QueryMeterings).
+		Doc("Get meterings against the cluster.").
+		Param(ws.QueryParameter("level", "Metering level.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("operation", "Metering operation.").DataType("string").Required(false).DefaultValue(monitoringv1alpha3.OperationQuery)).
+		Param(ws.QueryParameter("node", "Node name.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("workspace", "Workspace name.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("namespace", "Namespace name.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("kind", "Workload kind. One of deployment, daemonset, statefulset.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("workload", "Workload name.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("pod", "Pod name.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("applications", "Appliction names, format app_name[:app_version](such as nginx:v1, nignx) which are joined by \"|\" ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("services", "Services which are joined by \"|\".").DataType("string").Required(false)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workspace CPU usage and memory usage: `meter_workspace_cpu_usage|meter_workspace_memory_usage`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("resources_filter", "The workspace filter consists of a regexp pattern. It specifies which workspace data to return.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort workspaces by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
+		Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
+		Param(ws.QueryParameter("storageclass", "The name of the storageclass.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("pvc_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return.").DataType("string").Required(false)).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkspaceMetersTag}).
+		Writes(monitoring.Metrics{}).
+		Returns(http.StatusOK, api.StatusOK, monitoring.Metrics{}))
+
+	ws.Route(ws.GET("/namespaces/{namespace}/metering/hierarchy").
+		To(handler.QueryMeteringsHierarchy).
+		Param(ws.PathParameter("namespace", "Namespace name.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both workspace CPU usage and memory usage: `meter_pod_cpu_usage|meter_pod_memory_usage_wo_cache`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Doc("get current metering hierarchies info in last one hour").
+		Writes(metering.ResourceStatistic{}).
+		Returns(http.StatusOK, api.StatusOK, metering.ResourceStatistic{}))
+
+	ws.Route(ws.GET("/metering/price").
+		To(handler.HandlePriceInfoQuery).
+		Doc("Get resoure price.").
+		Writes(metering.PriceInfo{}).
+		Returns(http.StatusOK, api.StatusOK, metering.PriceInfo{}))
 	ws.Route(ws.POST("/workspaces/{workspace}/resourcequotas").
 		To(handler.CreateWorkspaceResourceQuota).
 		Reads(quotav1alpha2.ResourceQuota{}).

pkg/models/metering/type.go

@@ -0,0 +1,309 @@
+package metering
+
+type PriceInfo struct {
+	Currency                                  string  `json:"currency" description:"currency"`
+	CpuPerCorePerHour                         float64 `json:"cpu_per_core_per_hour,omitempty" description:"cpu price"`
+	MemPerGigabytesPerHour                    float64 `json:"mem_per_gigabytes_per_hour,omitempty" description:"mem price"`
+	IngressNetworkTrafficPerGiagabytesPerHour float64 `json:"ingress_network_traffic_per_giagabytes_per_hour,omitempty" description:"ingress price"`
+	EgressNetworkTrafficPerGiagabytesPerHour  float64 `json:"egress_network_traffic_per_gigabytes_per_hour,omitempty" description:"egress price"`
+	PvcPerGigabytesPerHour                    float64 `json:"pvc_per_gigabytes_per_hour,omitempty" description:"pvc price"`
+}
+
+// currently init method fill illegal value to hint that metering config file was not mounted yet
+func (p *PriceInfo) Init() {
+	p.Currency = ""
+	p.CpuPerCorePerHour = -1
+	p.MemPerGigabytesPerHour = -1
+	p.IngressNetworkTrafficPerGiagabytesPerHour = -1
+	p.EgressNetworkTrafficPerGiagabytesPerHour = -1
+	p.PvcPerGigabytesPerHour = -1
+}
+
+type PodStatistic struct {
+	CPUUsage            float64 `json:"cpu_usage" description:"cpu_usage"`
+	MemoryUsageWoCache  float64 `json:"memory_usage_wo_cache" description:"memory_usage_wo_cache"`
+	NetBytesTransmitted float64 `json:"net_bytes_transmitted" desription:"net_bytes_transmitted"`
+	NetBytesReceived    float64 `json:"net_bytes_received" description:"net_bytes_received"`
+	PVCBytesTotal       float64 `json:"pvc_bytes_total" description:"pvc_bytes_total"`
+}
+
+type PodsStats map[string]*PodStatistic
+
+func (ps *PodsStats) Set(podName, meterName string, value float64) {
+	if _, ok := (*ps)[podName]; !ok {
+		(*ps)[podName] = &PodStatistic{}
+	}
+	switch meterName {
+	case "meter_pod_cpu_usage":
+		(*ps)[podName].CPUUsage = value
+	case "meter_pod_memory_usage_wo_cache":
+		(*ps)[podName].MemoryUsageWoCache = value
+	case "meter_pod_net_bytes_transmitted":
+		(*ps)[podName].NetBytesTransmitted = value
+	case "meter_pod_net_bytes_received":
+		(*ps)[podName].NetBytesReceived = value
+	case "meter_pod_pvc_bytes_total":
+		(*ps)[podName].PVCBytesTotal = value
+	}
+}
+
+type AppStatistic struct {
+	CPUUsage            float64                      `json:"cpu_usage" description:"cpu_usage"`
+	MemoryUsageWoCache  float64                      `json:"memory_usage_wo_cache" description:"memory_usage_wo_cache"`
+	NetBytesTransmitted float64                      `json:"net_bytes_transmitted" description:"net_bytes_transmitted"`
+	NetBytesReceived    float64                      `json:"net_bytes_received" description:"net_bytes_received"`
+	PVCBytesTotal       float64                      `json:"pvc_bytes_total" description:"pvc_bytes_total"`
+	Services            map[string]*ServiceStatistic `json:"services" description:"services"`
+}
+
+func (as *AppStatistic) GetServiceStats(name string) *ServiceStatistic {
+	if as.Services == nil {
+		as.Services = make(map[string]*ServiceStatistic)
+	}
+	if as.Services[name] == nil {
+		as.Services[name] = &ServiceStatistic{}
+	}
+	return as.Services[name]
+}
+
+func (as *AppStatistic) Aggregate() {
+	if as.Services == nil {
+		return
+	}
+
+	// remove duplicate pods which were selected by different svc
+	podsMap := make(map[string]struct{})
+	for _, svcObj := range as.Services {
+		for podName, podObj := range svcObj.Pods {
+			if _, ok := podsMap[podName]; ok {
+				continue
+			} else {
+				podsMap[podName] = struct{}{}
+			}
+			as.CPUUsage += podObj.CPUUsage
+			as.MemoryUsageWoCache += podObj.MemoryUsageWoCache
+			as.NetBytesTransmitted += podObj.NetBytesTransmitted
+			as.NetBytesReceived += podObj.NetBytesReceived
+			as.PVCBytesTotal += podObj.PVCBytesTotal
+		}
+	}
+}
+
+type ServiceStatistic struct {
+	CPUUsage            float64                  `json:"cpu_usage" description:"cpu_usage"`
+	MemoryUsageWoCache  float64                  `json:"memory_usage_wo_cache" desription:"memory_usage_wo_cache"`
+	NetBytesTransmitted float64                  `json:"net_bytes_transmitted" description:"net_bytes_transmitted"`
+	NetBytesReceived    float64                  `json:"net_bytes_received" description:"net_bytes_received"`
+	Pods                map[string]*PodStatistic `json:"pods" description:"pod statistic"`
+}
+
+func (ss *ServiceStatistic) SetPodStats(name string, podStat *PodStatistic) error {
+	if ss.Pods == nil {
+		ss.Pods = make(map[string]*PodStatistic)
+	}
+	ss.Pods[name] = podStat
+	return nil
+}
+
+func (ss *ServiceStatistic) GetPodStats(name string) *PodStatistic {
+	if ss.Pods == nil {
+		ss.Pods = make(map[string]*PodStatistic)
+	}
+	if ss.Pods[name] == nil {
+		ss.Pods[name] = &PodStatistic{}
+	}
+	return ss.Pods[name]
+}
+
+func (ss *ServiceStatistic) Aggregate() {
+	if ss.Pods == nil {
+		return
+	}
+
+	for key := range ss.Pods {
+		ss.CPUUsage += ss.GetPodStats(key).CPUUsage
+		ss.MemoryUsageWoCache += ss.GetPodStats(key).MemoryUsageWoCache
+		ss.NetBytesTransmitted += ss.GetPodStats(key).NetBytesTransmitted
+		ss.NetBytesReceived += ss.GetPodStats(key).NetBytesReceived
+	}
+}
+
+type DeploymentStatistic struct {
+	CPUUsage            float64                  `json:"cpu_usage" description:"cpu_usage"`
+	MemoryUsageWoCache  float64                  `json:"memory_usage_wo_cache" description:"memory_usage_wo_cache"`
+	NetBytesTransmitted float64                  `json:"net_bytes_transmitted" desciption:"net_bytes_transmitted"`
+	NetBytesReceived    float64                  `json:"net_bytes_received" description:"net_bytes_received"`
+	PVCBytesTotal       float64                  `json:"pvc_bytes_total" description:"pvc_bytes_total"`
+	Pods                map[string]*PodStatistic `json:"pods" description:"pod statistic"`
+}
+
+func (ds *DeploymentStatistic) GetPodStats(name string) *PodStatistic {
+	if ds.Pods == nil {
+		ds.Pods = make(map[string]*PodStatistic)
+	}
+	if ds.Pods[name] == nil {
+		ds.Pods[name] = &PodStatistic{}
+	}
+	return ds.Pods[name]
+}
+
+func (ds *DeploymentStatistic) SetPodStats(name string, podStat *PodStatistic) error {
+	if ds.Pods == nil {
+		ds.Pods = make(map[string]*PodStatistic)
+	}
+	ds.Pods[name] = podStat
+	return nil
+}
+
+func (ds *DeploymentStatistic) Aggregate() {
+	if ds.Pods == nil {
+		return
+	}
+
+	for key := range ds.Pods {
+		ds.CPUUsage += ds.GetPodStats(key).CPUUsage
+		ds.MemoryUsageWoCache += ds.GetPodStats(key).MemoryUsageWoCache
+		ds.NetBytesTransmitted += ds.GetPodStats(key).NetBytesTransmitted
+		ds.NetBytesReceived += ds.GetPodStats(key).NetBytesReceived
+		ds.PVCBytesTotal += ds.GetPodStats(key).PVCBytesTotal
+	}
+}
+
+type StatefulsetStatistic struct {
+	CPUUsage            float64                  `json:"cpu_usage" description:"cpu_usage"`
+	MemoryUsageWoCache  float64                  `json:"memory_usage_wo_cache" description:"memory_usage_wo_cache"`
+	NetBytesTransmitted float64                  `json:"net_bytes_transmitted" description:"net_bytes_transmitted"`
+	NetBytesReceived    float64                  `json:"net_bytes_received" description:"net_bytes_received"`
+	PVCBytesTotal       float64                  `json:"pvc_bytes_total" description:"pvc_bytes_total"`
+	Pods                map[string]*PodStatistic `json:"pods" description:"pod statistic"`
+}
+
+func (ss *StatefulsetStatistic) GetPodStats(name string) *PodStatistic {
+	if ss.Pods == nil {
+		ss.Pods = make(map[string]*PodStatistic)
+	}
+	if ss.Pods[name] == nil {
+		ss.Pods[name] = &PodStatistic{}
+	}
+	return ss.Pods[name]
+}
+
+func (ss *StatefulsetStatistic) SetPodStats(name string, podStat *PodStatistic) error {
+	if ss.Pods == nil {
+		ss.Pods = make(map[string]*PodStatistic)
+	}
+	ss.Pods[name] = podStat
+	return nil
+}
+
+func (ss *StatefulsetStatistic) Aggregate() {
+	if ss.Pods == nil {
+		return
+	}
+
+	for key := range ss.Pods {
+		ss.CPUUsage += ss.GetPodStats(key).CPUUsage
+		ss.MemoryUsageWoCache += ss.GetPodStats(key).MemoryUsageWoCache
+		ss.NetBytesTransmitted += ss.GetPodStats(key).NetBytesTransmitted
+		ss.NetBytesReceived += ss.GetPodStats(key).NetBytesReceived
+		ss.PVCBytesTotal += ss.GetPodStats(key).PVCBytesTotal
+	}
+}
+
+type DaemonsetStatistic struct {
+	CPUUsage            float64                  `json:"cpu_usage" description:"cpu_usage"`
+	MemoryUsageWoCache  float64                  `json:"memory_usage_wo_cache" description:"memory_usage_wo_cache"`
+	NetBytesTransmitted float64                  `json:"net_bytes_transmitted" description:"net_bytes_transmitted"`
+	NetBytesReceived    float64                  `json:"net_bytes_received" description:"net_bytes_received"`
+	PVCBytesTotal       float64                  `json:"pvc_bytes_total" description:"pvc_bytes_total"`
+	Pods                map[string]*PodStatistic `json:"pods" description:"pod statistic"`
+}
+
+func (ds *DaemonsetStatistic) GetPodStats(name string) *PodStatistic {
+	if ds.Pods == nil {
+		ds.Pods = make(map[string]*PodStatistic)
+	}
+	if ds.Pods[name] == nil {
+		ds.Pods[name] = &PodStatistic{}
+	}
+	return ds.Pods[name]
+}
+
+func (ds *DaemonsetStatistic) SetPodStats(name string, podStat *PodStatistic) error {
+	if ds.Pods == nil {
+		ds.Pods = make(map[string]*PodStatistic)
+	}
+	ds.Pods[name] = podStat
+	return nil
+}
+
+func (ds *DaemonsetStatistic) Aggregate() {
+	if ds.Pods == nil {
+		return
+	}
+	for key := range ds.Pods {
+		ds.CPUUsage += ds.GetPodStats(key).CPUUsage
+		ds.MemoryUsageWoCache += ds.GetPodStats(key).MemoryUsageWoCache
+		ds.NetBytesTransmitted += ds.GetPodStats(key).NetBytesTransmitted
+		ds.NetBytesReceived += ds.GetPodStats(key).NetBytesReceived
+		ds.PVCBytesTotal += ds.GetPodStats(key).PVCBytesTotal
+	}
+}
+
+type ResourceStatistic struct {
+	Apps         map[string]*AppStatistic         `json:"apps" description:"app statistic"`
+	Services     map[string]*ServiceStatistic     `json:"services" description:"service statistic"`
+	Deploys      map[string]*DeploymentStatistic  `json:"deployments" description:"deployment statistic"`
+	Statefulsets map[string]*StatefulsetStatistic `json:"statefulsets" description:"statefulset statistic"`
+	Daemonsets   map[string]*DaemonsetStatistic   `json:"daemonsets" description:"daemonsets statistics"`
+}
+
+func (rs *ResourceStatistic) GetAppStats(name string) *AppStatistic {
+	if rs.Apps == nil {
+		rs.Apps = make(map[string]*AppStatistic)
+	}
+	if rs.Apps[name] == nil {
+		rs.Apps[name] = &AppStatistic{}
+	}
+	return rs.Apps[name]
+}
+
+func (rs *ResourceStatistic) GetServiceStats(name string) *ServiceStatistic {
+	if rs.Services == nil {
+		rs.Services = make(map[string]*ServiceStatistic)
+	}
+	if rs.Services[name] == nil {
+		rs.Services[name] = &ServiceStatistic{}
+	}
+	return rs.Services[name]
+}
+
+func (rs *ResourceStatistic) GetDeployStats(name string) *DeploymentStatistic {
+	if rs.Deploys == nil {
+		rs.Deploys = make(map[string]*DeploymentStatistic)
+	}
+	if rs.Deploys[name] == nil {
+		rs.Deploys[name] = &DeploymentStatistic{}
+	}
+	return rs.Deploys[name]
+}
+
+func (rs *ResourceStatistic) GetStatefulsetStats(name string) *StatefulsetStatistic {
+	if rs.Statefulsets == nil {
+		rs.Statefulsets = make(map[string]*StatefulsetStatistic)
+	}
+	if rs.Statefulsets[name] == nil {
+		rs.Statefulsets[name] = &StatefulsetStatistic{}
+	}
+	return rs.Statefulsets[name]
+}
+
+func (rs *ResourceStatistic) GetDaemonsetStats(name string) *DaemonsetStatistic {
+	if rs.Daemonsets == nil {
+		rs.Daemonsets = make(map[string]*DaemonsetStatistic)
+	}
+	if rs.Daemonsets[name] == nil {
+		rs.Daemonsets[name] = &DaemonsetStatistic{}
+	}
+	return rs.Daemonsets[name]
+}

pkg/models/monitoring/monitoring.go

@@ -18,6 +18,9 @@ package monitoring
 
 import (
 	"context"
+	"fmt"
+	"math"
+	"strings"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -26,14 +29,19 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
 	"kubesphere.io/kubesphere/pkg/apis/iam/v1alpha2"
+	"kubesphere.io/kubesphere/pkg/apiserver/query"
 	ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
 	"kubesphere.io/kubesphere/pkg/constants"
 	"kubesphere.io/kubesphere/pkg/informers"
 	"kubesphere.io/kubesphere/pkg/models/monitoring/expressions"
 	"kubesphere.io/kubesphere/pkg/models/openpitrix"
+	resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
+	"kubesphere.io/kubesphere/pkg/server/errors"
 	"kubesphere.io/kubesphere/pkg/server/params"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
 	opclient "kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
+	"sigs.k8s.io/application/api/v1beta1"
+	appv1beta1 "sigs.k8s.io/application/api/v1beta1"
 )
 
 type MonitoringOperator interface {
@@ -47,23 +55,31 @@ type MonitoringOperator interface {
 	// TODO: expose KubeSphere self metrics in Prometheus format
 	GetKubeSphereStats() Metrics
 	GetWorkspaceStats(workspace string) Metrics
+
+	// meter
+	GetNamedMetersOverTime(metrics []string, start, end time.Time, step time.Duration, opt monitoring.QueryOption) (Metrics, error)
+	GetNamedMeters(metrics []string, time time.Time, opt monitoring.QueryOption) (Metrics, error)
+	GetAppComponentsMap(ns string, apps []string) map[string][]string
+	GetSerivePodsMap(ns string, services []string) map[string][]string
 }
 
 type monitoringOperator struct {
-	prometheus    monitoring.Interface
-	metricsserver monitoring.Interface
-	k8s           kubernetes.Interface
-	ks            ksinformers.SharedInformerFactory
-	op            openpitrix.Interface
+	prometheus     monitoring.Interface
+	metricsserver  monitoring.Interface
+	k8s            kubernetes.Interface
+	ks             ksinformers.SharedInformerFactory
+	op             openpitrix.Interface
+	resourceGetter *resourcev1alpha3.ResourceGetter
 }
 
-func NewMonitoringOperator(monitoringClient monitoring.Interface, metricsClient monitoring.Interface, k8s kubernetes.Interface, factory informers.InformerFactory, opClient opclient.Client) MonitoringOperator {
+func NewMonitoringOperator(monitoringClient monitoring.Interface, metricsClient monitoring.Interface, k8s kubernetes.Interface, factory informers.InformerFactory, opClient opclient.Client, resourceGetter *resourcev1alpha3.ResourceGetter) MonitoringOperator {
 	return &monitoringOperator{
-		prometheus:    monitoringClient,
-		metricsserver: metricsClient,
-		k8s:           k8s,
-		ks:            factory.KubeSphereSharedInformerFactory(),
-		op:            openpitrix.NewOpenpitrixOperator(factory.KubernetesSharedInformerFactory(), opClient),
+		prometheus:     monitoringClient,
+		metricsserver:  metricsClient,
+		k8s:            k8s,
+		ks:             factory.KubeSphereSharedInformerFactory(),
+		op:             openpitrix.NewOpenpitrixOperator(factory.KubernetesSharedInformerFactory(), opClient),
+		resourceGetter: resourceGetter,
 	}
 }
 
@@ -354,3 +370,225 @@ func (mo monitoringOperator) GetWorkspaceStats(workspace string) Metrics {
 
 	return res
 }
+
+/*
+	meter related methods
+*/
+
+func (mo monitoringOperator) getNamedMetersWithHourInterval(meters []string, t time.Time, opt monitoring.QueryOption) Metrics {
+
+	var opts []monitoring.QueryOption
+
+	opts = append(opts, opt)
+	opts = append(opts, monitoring.MeterOption{
+		Step: 1 * time.Hour,
+	})
+
+	ress := mo.prometheus.GetNamedMeters(meters, t, opts)
+
+	return Metrics{Results: ress}
+}
+
+func generateScalingFactorMap(step time.Duration) map[string]float64 {
+	scalingMap := make(map[string]float64)
+
+	for k := range MeterResourceMap {
+		scalingMap[k] = step.Hours()
+	}
+	return scalingMap
+}
+
+func (mo monitoringOperator) GetNamedMetersOverTime(meters []string, start, end time.Time, step time.Duration, opt monitoring.QueryOption) (metrics Metrics, err error) {
+
+	if step.Hours() < 1 {
+		klog.Warning("step should be longer than one hour")
+		step = 1 * time.Hour
+	}
+	if end.Sub(start).Hours() > 30*24 {
+		if step.Hours() < 24 {
+			err = errors.New("step should be larger than 24 hours")
+			return
+		}
+	}
+	if math.Mod(step.Hours(), 1.0) > 0 {
+		err = errors.New("step should be integer hours")
+		return
+	}
+
+	// query time range: (start, end], so here we need to exclude start itself.
+	if start.Add(step).After(end) {
+		start = end
+	} else {
+		start = start.Add(step)
+	}
+
+	var opts []monitoring.QueryOption
+
+	opts = append(opts, opt)
+	opts = append(opts, monitoring.MeterOption{
+		Start: start,
+		End:   end,
+		Step:  step,
+	})
+
+	ress := mo.prometheus.GetNamedMetersOverTime(meters, start, end, step, opts)
+	sMap := generateScalingFactorMap(step)
+
+	for i, _ := range ress {
+		ress[i].MetricData = updateMetricStatData(ress[i], sMap)
+	}
+
+	return Metrics{Results: ress}, nil
+}
+
+func (mo monitoringOperator) GetNamedMeters(meters []string, time time.Time, opt monitoring.QueryOption) (Metrics, error) {
+
+	metersPerHour := mo.getNamedMetersWithHourInterval(meters, time, opt)
+
+	for metricIndex, _ := range metersPerHour.Results {
+
+		res := metersPerHour.Results[metricIndex]
+
+		metersPerHour.Results[metricIndex].MetricData = updateMetricStatData(res, nil)
+	}
+
+	return metersPerHour, nil
+}
+
+func (mo monitoringOperator) GetAppComponentsMap(ns string, apps []string) map[string][]string {
+
+	componentsMap := make(map[string][]string)
+	applicationList := []*appv1beta1.Application{}
+
+	result, err := mo.resourceGetter.List("applications", ns, query.New())
+	if err != nil {
+		klog.Error(err)
+		return nil
+	}
+
+	for _, obj := range result.Items {
+		app, ok := obj.(*appv1beta1.Application)
+		if !ok {
+			continue
+		}
+
+		applicationList = append(applicationList, app)
+	}
+
+	getAppFullName := func(appObject *v1beta1.Application) (name string) {
+		name = appObject.Labels[constants.ApplicationName]
+		if appObject.Labels[constants.ApplicationVersion] != "" {
+			name += fmt.Sprintf(":%v", appObject.Labels[constants.ApplicationVersion])
+		}
+		return
+	}
+
+	appFilter := func(appObject *v1beta1.Application) bool {
+
+		for _, app := range apps {
+			var applicationName, applicationVersion string
+			tmp := strings.Split(app, ":")
+
+			if len(tmp) >= 1 {
+				applicationName = tmp[0]
+			}
+			if len(tmp) == 2 {
+				applicationVersion = tmp[1]
+			}
+
+			if applicationName != "" && appObject.Labels[constants.ApplicationName] != applicationName {
+				return false
+			}
+			if applicationVersion != "" && appObject.Labels[constants.ApplicationVersion] != applicationVersion {
+				return false
+			}
+			return true
+		}
+
+		return true
+	}
+
+	for _, appObj := range applicationList {
+		if appFilter(appObj) {
+			for _, com := range appObj.Status.ComponentList.Objects {
+				kind := strings.Title(com.Kind)
+				name := com.Name
+				componentsMap[getAppFullName((appObj))] = append(componentsMap[getAppFullName(appObj)], kind+":"+name)
+			}
+		}
+	}
+
+	return componentsMap
+}
+
+func (mo monitoringOperator) getApplicationPVCs(appObject *v1beta1.Application) []string {
+
+	var pvcList []string
+
+	ns := appObject.Namespace
+	for _, com := range appObject.Status.ComponentList.Objects {
+
+		switch strings.Title(com.Kind) {
+		case "Deployment":
+			deployObj, err := mo.k8s.AppsV1().Deployments(ns).Get(context.Background(), com.Name, metav1.GetOptions{})
+			if err != nil {
+				klog.Error(err.Error())
+				return nil
+			}
+
+			for _, vol := range deployObj.Spec.Template.Spec.Volumes {
+				pvcList = append(pvcList, vol.PersistentVolumeClaim.ClaimName)
+			}
+		case "Statefulset":
+			stsObj, err := mo.k8s.AppsV1().StatefulSets(ns).Get(context.Background(), com.Name, metav1.GetOptions{})
+			if err != nil {
+				klog.Error(err.Error())
+				return nil
+			}
+			for _, vol := range stsObj.Spec.Template.Spec.Volumes {
+				pvcList = append(pvcList, vol.PersistentVolumeClaim.ClaimName)
+			}
+		}
+
+	}
+
+	return pvcList
+
+}
+
+func (mo monitoringOperator) GetSerivePodsMap(ns string, services []string) map[string][]string {
+	var svcPodsMap = make(map[string][]string)
+
+	for _, svc := range services {
+		svcObj, err := mo.k8s.CoreV1().Services(ns).Get(context.Background(), svc, metav1.GetOptions{})
+		if err != nil {
+			klog.Error(err.Error())
+			return svcPodsMap
+		}
+
+		svcSelector := svcObj.Spec.Selector
+		if len(svcSelector) == 0 {
+			return svcPodsMap
+		}
+
+		svcLabels := labels.Set{}
+		for key, value := range svcSelector {
+			svcLabels[key] = value
+		}
+
+		selector := labels.SelectorFromSet(svcLabels)
+		opt := metav1.ListOptions{LabelSelector: selector.String()}
+
+		podList, err := mo.k8s.CoreV1().Pods(ns).List(context.Background(), opt)
+		if err != nil {
+			klog.Error(err.Error())
+			return svcPodsMap
+		}
+
+		for _, pod := range podList.Items {
+			svcPodsMap[svc] = append(svcPodsMap[svc], pod.Name)
+		}
+
+	}
+	return svcPodsMap
+}

pkg/models/monitoring/named_metrics.go

@@ -26,6 +26,8 @@ const (
 	WorkspaceDevopsCount    = "workspace_devops_project_count"
 	WorkspaceMemberCount    = "workspace_member_count"
 	WorkspaceRoleCount      = "workspace_role_count"
+
+	MetricMeterPrefix = "meter_"
 )
 
 var ClusterMetrics = []string{
@@ -78,6 +80,13 @@ var ClusterMetrics = []string{
 	"cluster_load15",
 	"cluster_pod_abnormal_ratio",
 	"cluster_node_offline_ratio",
+
+	// meter
+	"meter_cluster_cpu_usage",
+	"meter_cluster_memory_usage",
+	"meter_cluster_net_bytes_transmitted",
+	"meter_cluster_net_bytes_received",
+	"meter_cluster_pvc_bytes_total",
 }
 
 var NodeMetrics = []string{
@@ -113,6 +122,13 @@ var NodeMetrics = []string{
 	"node_load15",
 	"node_pod_abnormal_ratio",
 	"node_pleg_quantile",
+
+	// meter
+	"meter_node_cpu_usage",
+	"meter_node_memory_usage_wo_cache",
+	"meter_node_net_bytes_transmitted",
+	"meter_node_net_bytes_received",
+	"meter_node_pvc_bytes_total",
 }
 
 var WorkspaceMetrics = []string{
@@ -138,6 +154,13 @@ var WorkspaceMetrics = []string{
 	"workspace_service_count",
 	"workspace_secret_count",
 	"workspace_pod_abnormal_ratio",
+
+	// meter
+	"meter_workspace_cpu_usage",
+	"meter_workspace_memory_usage",
+	"meter_workspace_net_bytes_transmitted",
+	"meter_workspace_net_bytes_received",
+	"meter_workspace_pvc_bytes_total",
 }
 
 var NamespaceMetrics = []string{
@@ -168,6 +191,23 @@ var NamespaceMetrics = []string{
 	"namespace_configmap_count",
 	"namespace_ingresses_extensions_count",
 	"namespace_s2ibuilder_count",
+
+	// meter
+	"meter_namespace_cpu_usage",
+	"meter_namespace_memory_usage_wo_cache",
+	"meter_namespace_net_bytes_transmitted",
+	"meter_namespace_net_bytes_received",
+	"meter_namespace_pvc_bytes_total",
+}
+
+var ApplicationMetrics = []string{
+
+	// meter
+	"meter_application_cpu_usage",
+	"meter_application_memory_usage_wo_cache",
+	"meter_application_net_bytes_transmitted",
+	"meter_application_net_bytes_received",
+	"meter_application_pvc_bytes_total",
 }
 
 var WorkloadMetrics = []string{
@@ -185,6 +225,21 @@ var WorkloadMetrics = []string{
 	"workload_deployment_unavailable_replicas_ratio",
 	"workload_daemonset_unavailable_replicas_ratio",
 	"workload_statefulset_unavailable_replicas_ratio",
+
+	// meter
+	"meter_workload_cpu_usage",
+	"meter_workload_memory_usage_wo_cache",
+	"meter_workload_net_bytes_transmitted",
+	"meter_workload_net_bytes_received",
+	"meter_workload_pvc_bytes_total",
+}
+
+var ServiceMetrics = []string{
+	// meter
+	"meter_service_cpu_usage",
+	"meter_service_memory_usage_wo_cache",
+	"meter_service_net_bytes_transmitted",
+	"meter_service_net_bytes_received",
 }
 
 var PodMetrics = []string{
@@ -193,6 +248,13 @@ var PodMetrics = []string{
 	"pod_memory_usage_wo_cache",
 	"pod_net_bytes_transmitted",
 	"pod_net_bytes_received",
+
+	// meter
+	"meter_pod_cpu_usage",
+	"meter_pod_memory_usage_wo_cache",
+	"meter_pod_net_bytes_transmitted",
+	"meter_pod_net_bytes_received",
+	"meter_pod_pvc_bytes_total",
 }
 
 var ContainerMetrics = []string{

pkg/models/monitoring/sort_page.go

@@ -23,13 +23,15 @@ import (
 )
 
 const (
-	IdentifierNode      = "node"
-	IdentifierWorkspace = "workspace"
-	IdentifierNamespace = "namespace"
-	IdentifierWorkload  = "workload"
-	IdentifierPod       = "pod"
-	IdentifierContainer = "container"
-	IdentifierPVC       = "persistentvolumeclaim"
+	IdentifierNode        = "node"
+	IdentifierWorkspace   = "workspace"
+	IdentifierNamespace   = "namespace"
+	IdentifierWorkload    = "workload"
+	IdentifierPod         = "pod"
+	IdentifierContainer   = "container"
+	IdentifierPVC         = "persistentvolumeclaim"
+	IdentifierService     = "service"
+	IdentifierApplication = "application"
 
 	OrderAscending  = "asc"
 	OrderDescending = "desc"

pkg/models/monitoring/types.go

@@ -16,7 +16,9 @@ limitations under the License.
 
 package monitoring
 
-import "kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+import (
+	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+)
 
 type Metrics struct {
 	Results     []monitoring.Metric `json:"results" description:"actual array of results"`

pkg/models/monitoring/utils.go

@@ -0,0 +1,252 @@
+package monitoring
+
+import (
+	"math"
+	"os"
+
+	"k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/klog"
+	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+)
+
+const (
+	METER_RESOURCE_TYPE_CPU = iota
+	METER_RESOURCE_TYPE_MEM
+	METER_RESOURCE_TYPE_NET_INGRESS
+	METER_RESOURCE_TYPE_NET_EGRESS
+	METER_RESOURCE_TYPE_PVC
+
+	meteringConfig = "/etc/kubesphere/metering/ks-metering.yaml"
+)
+
+var meterResourceUnitMap = map[int]string{
+	METER_RESOURCE_TYPE_CPU:         "cores",
+	METER_RESOURCE_TYPE_MEM:         "bytes",
+	METER_RESOURCE_TYPE_NET_INGRESS: "bytes",
+	METER_RESOURCE_TYPE_NET_EGRESS:  "bytes",
+	METER_RESOURCE_TYPE_PVC:         "bytes",
+}
+
+var MeterResourceMap = map[string]int{
+	"meter_cluster_cpu_usage":                 METER_RESOURCE_TYPE_CPU,
+	"meter_cluster_memory_usage":              METER_RESOURCE_TYPE_MEM,
+	"meter_cluster_net_bytes_transmitted":     METER_RESOURCE_TYPE_NET_EGRESS,
+	"meter_cluster_net_bytes_received":        METER_RESOURCE_TYPE_NET_INGRESS,
+	"meter_cluster_pvc_bytes_total":           METER_RESOURCE_TYPE_PVC,
+	"meter_node_cpu_usage":                    METER_RESOURCE_TYPE_CPU,
+	"meter_node_memory_usage_wo_cache":        METER_RESOURCE_TYPE_MEM,
+	"meter_node_net_bytes_transmitted":        METER_RESOURCE_TYPE_NET_EGRESS,
+	"meter_node_net_bytes_received":           METER_RESOURCE_TYPE_NET_INGRESS,
+	"meter_node_pvc_bytes_total":              METER_RESOURCE_TYPE_PVC,
+	"meter_workspace_cpu_usage":               METER_RESOURCE_TYPE_CPU,
+	"meter_workspace_memory_usage":            METER_RESOURCE_TYPE_MEM,
+	"meter_workspace_net_bytes_transmitted":   METER_RESOURCE_TYPE_NET_EGRESS,
+	"meter_workspace_net_bytes_received":      METER_RESOURCE_TYPE_NET_INGRESS,
+	"meter_workspace_pvc_bytes_total":         METER_RESOURCE_TYPE_PVC,
+	"meter_namespace_cpu_usage":               METER_RESOURCE_TYPE_CPU,
+	"meter_namespace_memory_usage_wo_cache":   METER_RESOURCE_TYPE_MEM,
+	"meter_namespace_net_bytes_transmitted":   METER_RESOURCE_TYPE_NET_EGRESS,
+	"meter_namespace_net_bytes_received":      METER_RESOURCE_TYPE_NET_INGRESS,
+	"meter_namespace_pvc_bytes_total":         METER_RESOURCE_TYPE_PVC,
+	"meter_application_cpu_usage":             METER_RESOURCE_TYPE_CPU,
+	"meter_application_memory_usage_wo_cache": METER_RESOURCE_TYPE_MEM,
+	"meter_application_net_bytes_transmitted": METER_RESOURCE_TYPE_NET_EGRESS,
+	"meter_application_net_bytes_received":    METER_RESOURCE_TYPE_NET_INGRESS,
+	"meter_application_pvc_bytes_total":       METER_RESOURCE_TYPE_PVC,
+	"meter_workload_cpu_usage":                METER_RESOURCE_TYPE_CPU,
+	"meter_workload_memory_usage_wo_cache":    METER_RESOURCE_TYPE_MEM,
+	"meter_workload_net_bytes_transmitted":    METER_RESOURCE_TYPE_NET_EGRESS,
+	"meter_workload_net_bytes_received":       METER_RESOURCE_TYPE_NET_INGRESS,
+	"meter_workload_pvc_bytes_total":          METER_RESOURCE_TYPE_PVC,
+	"meter_service_cpu_usage":                 METER_RESOURCE_TYPE_CPU,
+	"meter_service_memory_usage_wo_cache":     METER_RESOURCE_TYPE_MEM,
+	"meter_service_net_bytes_transmitted":     METER_RESOURCE_TYPE_NET_EGRESS,
+	"meter_service_net_bytes_received":        METER_RESOURCE_TYPE_NET_INGRESS,
+	"meter_pod_cpu_usage":                     METER_RESOURCE_TYPE_CPU,
+	"meter_pod_memory_usage_wo_cache":         METER_RESOURCE_TYPE_MEM,
+	"meter_pod_net_bytes_transmitted":         METER_RESOURCE_TYPE_NET_EGRESS,
+	"meter_pod_net_bytes_received":            METER_RESOURCE_TYPE_NET_INGRESS,
+	"meter_pod_pvc_bytes_total":               METER_RESOURCE_TYPE_PVC,
+}
+
+type PriceInfo struct {
+	CpuPerCorePerHour                         float64 `json:"cpuPerCorePerHour" yaml:"cpuPerCorePerHour"`
+	MemPerGigabytesPerHour                    float64 `json:"memPerGigabytesPerHour" yaml:"memPerGigabytesPerHour"`
+	IngressNetworkTrafficPerGiagabytesPerHour float64 `json:"ingressNetworkTrafficPerGiagabytesPerHour" yaml:"ingressNetworkTrafficPerGiagabytesPerHour"`
+	EgressNetworkTrafficPerGigabytesPerHour   float64 `json:"egressNetworkTrafficPerGigabytesPerHour" yaml:"egressNetworkTrafficPerGigabytesPerHour"`
+	PvcPerGigabytesPerHour                    float64 `json:"pvcPerGigabytesPerHour" yaml:"pvcPerGigabytesPerHour"`
+	CurrencyUnit                              string  `json:"currencyUnit" yaml:"currencyUnit"`
+}
+
+type Billing struct {
+	PriceInfo PriceInfo `json:"priceInfo" yaml:"priceInfo"`
+}
+
+type MeterConfig struct {
+	Billing Billing `json:"billing" yaml:"billing"`
+}
+
+func (mc MeterConfig) GetPriceInfo() PriceInfo {
+	return mc.Billing.PriceInfo
+}
+
+func LoadYaml() (*MeterConfig, error) {
+
+	var meterConfig MeterConfig
+
+	mf, err := os.Open(meteringConfig)
+	if err != nil {
+		klog.Error(err)
+		return nil, err
+	}
+
+	if err = yaml.NewYAMLOrJSONDecoder(mf, 1024).Decode(&meterConfig); err != nil {
+		klog.Error(err)
+		return nil, err
+	}
+
+	return &meterConfig, nil
+}
+
+func getMaxPointValue(points []monitoring.Point) float64 {
+	var max float64
+	for i, p := range points {
+		if i == 0 {
+			max = p.Value()
+		}
+
+		if p.Value() > max {
+			max = p.Value()
+		}
+	}
+
+	return max
+}
+
+func getMinPointValue(points []monitoring.Point) float64 {
+	var min float64
+	for i, p := range points {
+		if i == 0 {
+			min = p.Value()
+		}
+
+		if p.Value() < min {
+			min = p.Value()
+		}
+	}
+
+	return min
+}
+
+func getSumPointValue(points []monitoring.Point) float64 {
+	avg := 0.0
+
+	for _, p := range points {
+		avg += p.Value()
+	}
+
+	return avg
+}
+
+func getAvgPointValue(points []monitoring.Point) float64 {
+	return getSumPointValue(points) / float64(len(points))
+}
+
+func getCurrencyUnit() string {
+	meterConfig, err := LoadYaml()
+	if err != nil {
+		klog.Error(err)
+		return ""
+	}
+
+	return meterConfig.GetPriceInfo().CurrencyUnit
+}
+
+func getResourceUnit(meterName string) string {
+	if resourceType, ok := MeterResourceMap[meterName]; !ok {
+		klog.Errorf("invlaid meter %v", meterName)
+		return ""
+	} else {
+		return meterResourceUnitMap[resourceType]
+	}
+}
+
+func getFeeWithMeterName(meterName string, sum float64) float64 {
+
+	meterConfig, err := LoadYaml()
+	if err != nil {
+		klog.Error(err)
+		return -1
+	}
+	priceInfo := meterConfig.GetPriceInfo()
+
+	if resourceType, ok := MeterResourceMap[meterName]; !ok {
+		klog.Errorf("invlaid meter %v", meterName)
+		return -1
+	} else {
+		switch resourceType {
+		case METER_RESOURCE_TYPE_CPU:
+			// unit: core, precision: 0.001
+			sum = math.Round(sum*1000) / 1000
+			return priceInfo.CpuPerCorePerHour * sum
+		case METER_RESOURCE_TYPE_MEM:
+			// unit: Gigabyte, precision: 0.1
+			sum = math.Round(sum/1073741824*10) / 10
+			return priceInfo.MemPerGigabytesPerHour * sum
+		case METER_RESOURCE_TYPE_NET_INGRESS:
+			// unit: Megabyte, precision: 1
+			sum = math.Round(sum / 1048576)
+			return priceInfo.IngressNetworkTrafficPerGiagabytesPerHour * sum
+		case METER_RESOURCE_TYPE_NET_EGRESS:
+			// unit: Megabyte, precision:
+			sum = math.Round(sum / 1048576)
+			return priceInfo.EgressNetworkTrafficPerGigabytesPerHour * sum
+		case METER_RESOURCE_TYPE_PVC:
+			// unit: Gigabyte, precision: 0.1
+			sum = math.Round(sum/1073741824*10) / 10
+			return priceInfo.PvcPerGigabytesPerHour * sum
+		}
+
+		return -1
+	}
+}
+
+func updateMetricStatData(metric monitoring.Metric, scalingMap map[string]float64) monitoring.MetricData {
+	metricName := metric.MetricName
+	metricData := metric.MetricData
+	for index, metricValue := range metricData.MetricValues {
+
+		var points []monitoring.Point
+		if metricData.MetricType == monitoring.MetricTypeMatrix {
+			points = metricValue.Series
+		} else {
+			points = append(points, *metricValue.Sample)
+		}
+
+		var factor float64 = 1
+		if scalingMap != nil {
+			factor = scalingMap[metricName]
+		}
+
+		if len(points) == 1 {
+			sample := points[0]
+			sum := sample[1] * factor
+			metricData.MetricValues[index].MinValue = sample[1]
+			metricData.MetricValues[index].MaxValue = sample[1]
+			metricData.MetricValues[index].AvgValue = sample[1]
+			metricData.MetricValues[index].SumValue = sum
+			metricData.MetricValues[index].Fee = getFeeWithMeterName(metricName, sum)
+		} else {
+			sum := getSumPointValue(points) * factor
+			metricData.MetricValues[index].MinValue = getMinPointValue(points)
+			metricData.MetricValues[index].MaxValue = getMaxPointValue(points)
+			metricData.MetricValues[index].AvgValue = getAvgPointValue(points)
+			metricData.MetricValues[index].SumValue = sum
+			metricData.MetricValues[index].Fee = getFeeWithMeterName(metricName, sum)
+		}
+		metricData.MetricValues[index].CurrencyUnit = getCurrencyUnit()
+		metricData.MetricValues[index].ResourceUnit = getResourceUnit(metricName)
+
+	}
+	return metricData
+}

pkg/models/tenant/tenant.go

@@ -37,6 +37,7 @@ import (
 	auditingv1alpha1 "kubesphere.io/kubesphere/pkg/api/auditing/v1alpha1"
 	eventsv1alpha1 "kubesphere.io/kubesphere/pkg/api/events/v1alpha1"
 	loggingv1alpha2 "kubesphere.io/kubesphere/pkg/api/logging/v1alpha2"
+	meteringv1alpha1 "kubesphere.io/kubesphere/pkg/api/metering/v1alpha1"
 	clusterv1alpha1 "kubesphere.io/kubesphere/pkg/apis/cluster/v1alpha1"
 	quotav1alpha2 "kubesphere.io/kubesphere/pkg/apis/quota/v1alpha2"
 	tenantv1alpha1 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha1"
@@ -51,11 +52,16 @@ import (
 	"kubesphere.io/kubesphere/pkg/models/events"
 	"kubesphere.io/kubesphere/pkg/models/iam/am"
 	"kubesphere.io/kubesphere/pkg/models/logging"
+	"kubesphere.io/kubesphere/pkg/models/metering"
+	"kubesphere.io/kubesphere/pkg/models/monitoring"
 	resources "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3"
 	resourcesv1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
+	resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
 	auditingclient "kubesphere.io/kubesphere/pkg/simple/client/auditing"
 	eventsclient "kubesphere.io/kubesphere/pkg/simple/client/events"
 	loggingclient "kubesphere.io/kubesphere/pkg/simple/client/logging"
+	monitoringclient "kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+	opclient "kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/utils/stringutils"
 )
 
@@ -80,6 +86,8 @@ type Interface interface {
 	PatchNamespace(workspace string, namespace *corev1.Namespace) (*corev1.Namespace, error)
 	PatchWorkspace(workspace string, data json.RawMessage) (*tenantv1alpha2.WorkspaceTemplate, error)
 	ListClusters(info user.Info) (*api.ListResult, error)
+	Metering(user user.Info, queryParam *meteringv1alpha1.Query) (monitoring.Metrics, error)
+	MeteringHierarchy(user user.Info, queryParam *meteringv1alpha1.Query) (metering.ResourceStatistic, error)
 	CreateWorkspaceResourceQuota(workspace string, resourceQuota *quotav1alpha2.ResourceQuota) (*quotav1alpha2.ResourceQuota, error)
 	DeleteWorkspaceResourceQuota(workspace string, resourceQuotaName string) error
 	UpdateWorkspaceResourceQuota(workspace string, resourceQuota *quotav1alpha2.ResourceQuota) (*quotav1alpha2.ResourceQuota, error)
@@ -95,9 +103,10 @@ type tenantOperator struct {
 	events         events.Interface
 	lo             logging.LoggingOperator
 	auditing       auditing.Interface
+	mo             monitoring.MonitoringOperator
 }
 
-func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient eventsclient.Client, loggingClient loggingclient.Client, auditingclient auditingclient.Client, am am.AccessManagementInterface, authorizer authorizer.Authorizer) Interface {
+func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient eventsclient.Client, loggingClient loggingclient.Client, auditingclient auditingclient.Client, am am.AccessManagementInterface, authorizer authorizer.Authorizer, monitoringclient monitoringclient.Interface, opClient opclient.Client, resourceGetter *resourcev1alpha3.ResourceGetter) Interface {
 	return &tenantOperator{
 		am:             am,
 		authorizer:     authorizer,
@@ -107,6 +116,7 @@ func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ks
 		events:         events.NewEventsOperator(evtsClient),
 		lo:             logging.NewLoggingOperator(loggingClient),
 		auditing:       auditing.NewEventsOperator(auditingclient),
+		mo:             monitoring.NewMonitoringOperator(monitoringclient, nil, k8sclient, informers, opClient, resourceGetter),
 	}
 }
 
@@ -956,6 +966,38 @@ func (t *tenantOperator) Auditing(user user.Info, queryParam *auditingv1alpha1.Q
 	})
 }
 
+func (t *tenantOperator) Metering(user user.Info, query *meteringv1alpha1.Query) (metrics monitoring.Metrics, err error) {
+
+	var opt QueryOptions
+
+	opt, err = t.makeQueryOptions(user, *query, query.Level)
+	if err != nil {
+		return
+	}
+	metrics, err = t.ProcessNamedMetersQuery(opt)
+
+	return
+}
+
+func (t *tenantOperator) MeteringHierarchy(user user.Info, queryParam *meteringv1alpha1.Query) (metering.ResourceStatistic, error) {
+	res, err := t.Metering(user, queryParam)
+	if err != nil {
+		return metering.ResourceStatistic{}, err
+	}
+
+	// get pods stat info under ns
+	podsStats := t.transformMetricData(res)
+
+	// classify pods stats
+	resourceStats, err := t.classifyPodStats(user, queryParam.NamespaceName, podsStats)
+	if err != nil {
+		klog.Error(err)
+		return metering.ResourceStatistic{}, err
+	}
+
+	return resourceStats, nil
+}
+
 func contains(objects []runtime.Object, object runtime.Object) bool {
 	for _, item := range objects {
 		if item == object {

pkg/models/tenant/tenent_test.go

@@ -541,5 +541,5 @@ func prepare() Interface {
 	amOperator := am.NewOperator(ksClient, k8sClient, fakeInformerFactory)
 	authorizer := rbac.NewRBACAuthorizer(amOperator)
 
-	return New(fakeInformerFactory, k8sClient, ksClient, nil, nil, nil, amOperator, authorizer)
+	return New(fakeInformerFactory, k8sClient, ksClient, nil, nil, nil, amOperator, authorizer, nil, nil, nil)
 }

pkg/simple/client/monitoring/interface.go

@@ -25,4 +25,8 @@ type Interface interface {
 	GetNamedMetricsOverTime(metrics []string, start, end time.Time, step time.Duration, opt QueryOption) []Metric
 	GetMetadata(namespace string) []Metadata
 	GetMetricLabelSet(expr string, start, end time.Time) []map[string]string
+
+	// meter
+	GetNamedMeters(meters []string, time time.Time, opts []QueryOption) []Metric
+	GetNamedMetersOverTime(metrics []string, start, end time.Time, step time.Duration, opts []QueryOption) []Metric
 }

pkg/simple/client/monitoring/metricsserver/metricsserver.go

@@ -445,3 +445,11 @@ func (m metricsServer) GetMetricLabelSet(expr string, start, end time.Time) []ma
 
 	return res
 }
+
+// meter
+func (m metricsServer) GetNamedMeters(meters []string, time time.Time, opts []monitoring.QueryOption) []monitoring.Metric {
+	return nil
+}
+func (m metricsServer) GetNamedMetersOverTime(metrics []string, start, end time.Time, step time.Duration, opts []monitoring.QueryOption) []monitoring.Metric {
+	return nil
+}

pkg/simple/client/monitoring/prometheus/prometheus.go

@@ -30,6 +30,8 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
 )
 
+const MeteringDefaultTimeout = 20 * time.Second
+
 // prometheus implements monitoring interface backed by Prometheus
 type prometheus struct {
 	client apiv1.API
@@ -147,6 +149,108 @@ func (p prometheus) GetNamedMetricsOverTime(metrics []string, start, end time.Ti
 	return res
 }
 
+func (p prometheus) GetNamedMeters(meters []string, ts time.Time, opts []monitoring.QueryOption) []monitoring.Metric {
+	var res []monitoring.Metric
+	var wg sync.WaitGroup
+	var mtx sync.Mutex
+
+	queryOptions := monitoring.NewQueryOptions()
+
+	for _, opt := range opts {
+		opt.Apply(queryOptions)
+	}
+
+	prometheusCtx, cancel := context.WithTimeout(context.Background(), MeteringDefaultTimeout)
+	defer cancel()
+
+	for _, meter := range meters {
+
+		wg.Add(1)
+
+		go func(metric string) {
+			parsedResp := monitoring.Metric{MetricName: metric}
+
+			begin := time.Now()
+			value, _, err := p.client.Query(prometheusCtx, makeMeterExpr(metric, *queryOptions), ts)
+			end := time.Now()
+			timeElapsed := end.Unix() - begin.Unix()
+			if timeElapsed > int64(MeteringDefaultTimeout.Seconds())/2 {
+				klog.Warningf("long time query[cost %v seconds], expr: %v", timeElapsed, makeMeterExpr(metric, *queryOptions))
+			}
+
+			if err != nil {
+				parsedResp.Error = err.Error()
+			} else {
+				parsedResp.MetricData = parseQueryResp(value, nil)
+			}
+
+			mtx.Lock()
+			res = append(res, parsedResp)
+			mtx.Unlock()
+
+			wg.Done()
+		}(meter)
+
+	}
+
+	wg.Wait()
+
+	return res
+}
+
+func (p prometheus) GetNamedMetersOverTime(meters []string, start, end time.Time, step time.Duration, opts []monitoring.QueryOption) []monitoring.Metric {
+	var res []monitoring.Metric
+	var wg sync.WaitGroup
+	var mtx sync.Mutex
+
+	queryOptions := monitoring.NewQueryOptions()
+
+	for _, opt := range opts {
+		opt.Apply(queryOptions)
+	}
+
+	timeRange := apiv1.Range{
+		Start: start,
+		End:   end,
+		Step:  step,
+	}
+
+	prometheusCtx, cancel := context.WithTimeout(context.Background(), MeteringDefaultTimeout)
+	defer cancel()
+
+	for _, meter := range meters {
+
+		wg.Add(1)
+
+		go func(metric string) {
+			parsedResp := monitoring.Metric{MetricName: metric}
+			begin := time.Now()
+			value, _, err := p.client.QueryRange(prometheusCtx, makeMeterExpr(metric, *queryOptions), timeRange)
+			end := time.Now()
+			timeElapsed := end.Unix() - begin.Unix()
+			if timeElapsed > int64(MeteringDefaultTimeout.Seconds())/2 {
+				klog.Warningf("long time query[cost %v seconds], expr: %v", timeElapsed, makeMeterExpr(metric, *queryOptions))
+			}
+
+			if err != nil {
+				parsedResp.Error = err.Error()
+			} else {
+				parsedResp.MetricData = parseQueryRangeResp(value, nil)
+			}
+
+			mtx.Lock()
+			res = append(res, parsedResp)
+			mtx.Unlock()
+
+			wg.Done()
+		}(meter)
+	}
+
+	wg.Wait()
+
+	return res
+}
+
 func (p prometheus) GetMetadata(namespace string) []monitoring.Metadata {
 	var meta []monitoring.Metadata
 	var matchTarget string

pkg/simple/client/monitoring/query_options.go

@@ -16,6 +16,12 @@ limitations under the License.
 
 package monitoring
 
+import (
+	"fmt"
+	"strings"
+	"time"
+)
+
 type Level int
 
 const (
@@ -23,17 +29,39 @@ const (
 	LevelNode
 	LevelWorkspace
 	LevelNamespace
+	LevelApplication
 	LevelWorkload
+	LevelService
 	LevelPod
 	LevelContainer
 	LevelPVC
 	LevelComponent
 )
 
+var MeteringLevelMap = map[string]int{
+	"LevelCluster":     LevelCluster,
+	"LevelNode":        LevelNode,
+	"LevelWorkspace":   LevelWorkspace,
+	"LevelNamespace":   LevelNamespace,
+	"LevelApplication": LevelApplication,
+	"LevelWorkload":    LevelWorkload,
+	"LevelService":     LevelService,
+	"LevelPod":         LevelPod,
+	"LevelContainer":   LevelContainer,
+	"LevelPVC":         LevelPVC,
+	"LevelComponent":   LevelComponent,
+}
+
 type QueryOption interface {
 	Apply(*QueryOptions)
 }
 
+type Meteroptions struct {
+	Start time.Time
+	End   time.Time
+	Step  time.Duration
+}
+
 type QueryOptions struct {
 	Level Level
 
@@ -48,6 +76,10 @@ type QueryOptions struct {
 	ContainerName             string
 	StorageClassName          string
 	PersistentVolumeClaimName string
+	PVCFilter                 string
+	ApplicationName           string
+	ServiceName               string
+	MeterOptions              *Meteroptions
 }
 
 func NewQueryOptions() *QueryOptions {
@@ -61,31 +93,41 @@ func (_ ClusterOption) Apply(o *QueryOptions) {
 }
 
 type NodeOption struct {
-	ResourceFilter string
-	NodeName       string
+	ResourceFilter   string
+	NodeName         string
+	PVCFilter        string
+	StorageClassName string
 }
 
 func (no NodeOption) Apply(o *QueryOptions) {
 	o.Level = LevelNode
 	o.ResourceFilter = no.ResourceFilter
 	o.NodeName = no.NodeName
+	o.PVCFilter = no.PVCFilter
+	o.StorageClassName = no.StorageClassName
 }
 
 type WorkspaceOption struct {
-	ResourceFilter string
-	WorkspaceName  string
+	ResourceFilter   string
+	WorkspaceName    string
+	PVCFilter        string
+	StorageClassName string
 }
 
 func (wo WorkspaceOption) Apply(o *QueryOptions) {
 	o.Level = LevelWorkspace
 	o.ResourceFilter = wo.ResourceFilter
 	o.WorkspaceName = wo.WorkspaceName
+	o.PVCFilter = wo.PVCFilter
+	o.StorageClassName = wo.StorageClassName
 }
 
 type NamespaceOption struct {
-	ResourceFilter string
-	WorkspaceName  string
-	NamespaceName  string
+	ResourceFilter   string
+	WorkspaceName    string
+	NamespaceName    string
+	PVCFilter        string
+	StorageClassName string
 }
 
 func (no NamespaceOption) Apply(o *QueryOptions) {
@@ -93,6 +135,41 @@ func (no NamespaceOption) Apply(o *QueryOptions) {
 	o.ResourceFilter = no.ResourceFilter
 	o.WorkspaceName = no.WorkspaceName
 	o.NamespaceName = no.NamespaceName
+	o.PVCFilter = no.PVCFilter
+	o.StorageClassName = no.StorageClassName
+}
+
+type ApplicationsOption struct {
+	NamespaceName    string
+	Applications     []string
+	StorageClassName string
+}
+
+func (aso ApplicationsOption) Apply(o *QueryOptions) {
+	// nothing should be done
+	return
+}
+
+type ApplicationOption struct {
+	NamespaceName         string
+	Application           string
+	ApplicationComponents []string
+	StorageClassName      string
+}
+
+func (ao ApplicationOption) Apply(o *QueryOptions) {
+	o.Level = LevelApplication
+	o.NamespaceName = ao.NamespaceName
+	o.ApplicationName = ao.Application
+	o.StorageClassName = ao.StorageClassName
+
+	app_components := strings.Join(ao.ApplicationComponents[:], "|")
+
+	if len(app_components) > 0 {
+		o.ResourceFilter = fmt.Sprintf(`namespace="%s", workload=~"%s"`, o.NamespaceName, app_components)
+	} else {
+		o.ResourceFilter = fmt.Sprintf(`namespace="%s", workload=~"%s"`, o.NamespaceName, ".*")
+	}
 }
 
 type WorkloadOption struct {
@@ -108,6 +185,37 @@ func (wo WorkloadOption) Apply(o *QueryOptions) {
 	o.WorkloadKind = wo.WorkloadKind
 }
 
+type ServicesOption struct {
+	NamespaceName string
+	Services      []string
+}
+
+func (sso ServicesOption) Apply(o *QueryOptions) {
+	// nothing should be done
+	return
+}
+
+type ServiceOption struct {
+	ResourceFilter string
+	NamespaceName  string
+	ServiceName    string
+	PodNames       []string
+}
+
+func (so ServiceOption) Apply(o *QueryOptions) {
+	o.Level = LevelService
+	o.NamespaceName = so.NamespaceName
+	o.ServiceName = so.ServiceName
+
+	pod_names := strings.Join(so.PodNames, "|")
+
+	if len(pod_names) > 0 {
+		o.ResourceFilter = fmt.Sprintf(`pod=~"%s", namespace="%s"`, pod_names, o.NamespaceName)
+	} else {
+		o.ResourceFilter = fmt.Sprintf(`pod=~"%s", namespace="%s"`, ".*", o.NamespaceName)
+	}
+}
+
 type PodOption struct {
 	NamespacedResourcesFilter string
 	ResourceFilter            string
@@ -157,6 +265,9 @@ func (po PVCOption) Apply(o *QueryOptions) {
 	o.NamespaceName = po.NamespaceName
 	o.StorageClassName = po.StorageClassName
 	o.PersistentVolumeClaimName = po.PersistentVolumeClaimName
+
+	// for meter
+	o.PVCFilter = po.PersistentVolumeClaimName
 }
 
 type ComponentOption struct{}
@@ -164,3 +275,17 @@ type ComponentOption struct{}
 func (_ ComponentOption) Apply(o *QueryOptions) {
 	o.Level = LevelComponent
 }
+
+type MeterOption struct {
+	Start time.Time
+	End   time.Time
+	Step  time.Duration
+}
+
+func (mo MeterOption) Apply(o *QueryOptions) {
+	o.MeterOptions = &Meteroptions{
+		Start: mo.Start,
+		End:   mo.End,
+		Step:  mo.Step,
+	}
+}

pkg/simple/client/monitoring/types.go

@@ -20,7 +20,10 @@ import (
 	"errors"
 	"fmt"
 	"github.com/json-iterator/go"
+	"github.com/jszwec/csvutil"
 	"strconv"
+	"strings"
+	"time"
 )
 
 const (
@@ -35,14 +38,46 @@ type Metadata struct {
 }
 
 type Metric struct {
-	MetricName string `json:"metric_name,omitempty" description:"metric name, eg. scheduler_up_sum"`
+	MetricName string `json:"metric_name,omitempty" description:"metric name, eg. scheduler_up_sum" csv:"metric_name"`
 	MetricData `json:"data,omitempty" description:"actual metric result"`
-	Error      string `json:"error,omitempty"`
+	Error      string `json:"error,omitempty" csv:"-"`
+}
+
+type MetricValues []MetricValue
+
+func (m MetricValues) MarshalCSV() ([]byte, error) {
+
+	var ret []string
+	for _, v := range m {
+		tmp, err := v.MarshalCSV()
+		if err != nil {
+			return nil, err
+		}
+
+		ret = append(ret, string(tmp))
+	}
+
+	return []byte(strings.Join(ret, "||")), nil
 }
 
 type MetricData struct {
-	MetricType   string        `json:"resultType,omitempty" description:"result type, one of matrix, vector"`
-	MetricValues []MetricValue `json:"result,omitempty" description:"metric data including labels, time series and values"`
+	MetricType   string `json:"resultType,omitempty" description:"result type, one of matrix, vector" csv:"metric_type"`
+	MetricValues `json:"result,omitempty" description:"metric data including labels, time series and values" csv:"metric_values"`
+}
+
+func (m MetricData) MarshalCSV() ([]byte, error) {
+	var ret []byte
+
+	for _, v := range m.MetricValues {
+		tmp, err := csvutil.Marshal(&v)
+		if err != nil {
+			return nil, err
+		}
+
+		ret = append(ret, tmp...)
+	}
+
+	return ret, nil
 }
 
 // The first element is the timestamp, the second is the metric value.
@@ -54,8 +89,69 @@ type MetricValue struct {
 	// The type of Point is a float64 array with fixed length of 2.
 	// So Point will always be initialized as [0, 0], rather than nil.
 	// To allow empty Sample, we should declare Sample to type *Point
-	Sample *Point  `json:"value,omitempty" description:"time series, values of vector type"`
-	Series []Point `json:"values,omitempty" description:"time series, values of matrix type"`
+	Sample         *Point        `json:"value,omitempty" description:"time series, values of vector type"`
+	Series         []Point       `json:"values,omitempty" description:"time series, values of matrix type"`
+	ExportSample   *ExportPoint  `json:"exported_value,omitempty" description:"exported time series, values of vector type"`
+	ExportedSeries []ExportPoint `json:"exported_values,omitempty" description:"exported time series, values of matrix type"`
+
+	MinValue     float64 `json:"min_value" description:"minimum value from monitor points"`
+	MaxValue     float64 `json:"max_value" description:"maximum value from monitor points"`
+	AvgValue     float64 `json:"avg_value" description:"average value from monitor points"`
+	SumValue     float64 `json:"sum_value" description:"sum value from monitor points"`
+	Fee          float64 `json:"fee" description:"resource fee"`
+	ResourceUnit string  `json:"resource_unit"`
+	CurrencyUnit string  `json:"currency_unit"`
+}
+
+func (mv MetricValue) MarshalCSV() ([]byte, error) {
+	// metric value format:
+	// 	target,stats value(include fees),exported_value,exported_values
+	// for example:
+	// 	{workspace:demo-ws},,2021-02-23 01:00:00 AM 0|2021-02-23 02:00:00 AM 0|...
+	var metricValueCSVTemplate = "{%s},unit:%s|min:%.3f|max:%.3f|avg:%.3f|sum:%.3f|fee:%.2f %s,%s,%s"
+
+	var targetList []string
+	for k, v := range mv.Metadata {
+		targetList = append(targetList, fmt.Sprintf("%s=%s", k, v))
+	}
+
+	exportedSampleStr := ""
+	if mv.ExportSample != nil {
+		exportedSampleStr = mv.ExportSample.Format()
+	}
+
+	exportedSeriesStrList := []string{}
+	for _, v := range mv.ExportedSeries {
+		exportedSeriesStrList = append(exportedSeriesStrList, v.Format())
+	}
+
+	return []byte(fmt.Sprintf(metricValueCSVTemplate,
+		strings.Join(targetList, "|"),
+		mv.ResourceUnit,
+		mv.MinValue,
+		mv.MaxValue,
+		mv.AvgValue,
+		mv.SumValue,
+		mv.Fee,
+		mv.CurrencyUnit,
+		exportedSampleStr,
+		exportedSeriesStrList)), nil
+}
+
+func (mv *MetricValue) TransferToExportedMetricValue() {
+
+	if mv.Sample != nil {
+		sample := mv.Sample.transferToExported()
+		mv.ExportSample = &sample
+		mv.Sample = nil
+	}
+
+	for _, item := range mv.Series {
+		mv.ExportedSeries = append(mv.ExportedSeries, item.transferToExported())
+	}
+	mv.Series = nil
+
+	return
 }
 
 func (p Point) Timestamp() float64 {
@@ -66,6 +162,10 @@ func (p Point) Value() float64 {
 	return p[1]
 }
 
+func (p Point) transferToExported() ExportPoint {
+	return ExportPoint{p[0], p[1]}
+}
+
 // MarshalJSON implements json.Marshaler. It will be called when writing JSON to HTTP response
 // Inspired by prometheus/client_golang
 func (p Point) MarshalJSON() ([]byte, error) {
@@ -112,3 +212,17 @@ func (p *Point) UnmarshalJSON(b []byte) error {
 	p[1] = valf
 	return nil
 }
+
+type ExportPoint [2]float64
+
+func (p ExportPoint) Timestamp() string {
+	return time.Unix(int64(p[0]), 0).Format("2006-01-02 03:04:05 PM")
+}
+
+func (p ExportPoint) Value() float64 {
+	return p[1]
+}
+
+func (p ExportPoint) Format() string {
+	return p.Timestamp() + " " + strconv.FormatFloat(p.Value(), 'f', -1, 64)
+}

tools/cmd/doc-gen/main.go

@@ -125,7 +125,7 @@ func generateSwaggerJson() []byte {
 	urlruntime.Must(operationsv1alpha2.AddToContainer(container, clientsets.Kubernetes()))
 	urlruntime.Must(resourcesv1alpha2.AddToContainer(container, clientsets.Kubernetes(), informerFactory, ""))
 	urlruntime.Must(resourcesv1alpha3.AddToContainer(container, informerFactory, nil))
-	urlruntime.Must(tenantv1alpha2.AddToContainer(container, informerFactory, nil, nil, nil, nil, nil, nil, nil))
+	urlruntime.Must(tenantv1alpha2.AddToContainer(container, informerFactory, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil))
 	urlruntime.Must(terminalv1alpha2.AddToContainer(container, clientsets.Kubernetes(), nil))
 	urlruntime.Must(metricsv1alpha2.AddToContainer(container))
 	urlruntime.Must(networkv1alpha2.AddToContainer(container, ""))