add pvc stats api

Signed-off-by: Xin Wang <wileywang@yunify.com>
This commit is contained in:
Xin Wang
2019-09-02 20:43:50 +08:00
parent 76800b435c
commit 87865f9416
7 changed files with 235 additions and 34 deletions

View File

@@ -154,6 +154,59 @@ func addWebService(c *restful.Container) error {
Consumes(restful.MIME_JSON, restful.MIME_XML).
Produces(restful.MIME_JSON)
ws.Route(ws.GET("/storageclasses/{storageclass}/persistentvolumeclaims").To(monitoring.MonitorAllPVCsOfSpecificStorageClass).
Doc("Get PVC-level metric data of the specific storageclass's PVCs.").
Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(true)).
Param(ws.PathParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both PVC available and used inodes: `pvc_inodes_available|pvc_inodes_used`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)).
Param(ws.QueryParameter("resources_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return. For example, the following filter matches any pod whose name begins with redis: `redis.*`.").DataType("string").Required(false)).
Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
Param(ws.QueryParameter("sort_metric", "Sort PVCs by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
Metadata(restfulspec.KeyOpenAPITags, []string{constants.PVCMetricsTag}).
Writes(metrics.FormatedLevelMetric{}).
Returns(http.StatusOK, RespOK, metrics.FormatedLevelMetric{})).
Consumes(restful.MIME_JSON, restful.MIME_XML).
Produces(restful.MIME_JSON)
ws.Route(ws.GET("/namespaces/{namespace}/persistentvolumeclaims").To(monitoring.MonitorAllPVCsOfSpecificNamespace).
Doc("Get PVC-level metric data of the specific namespace's PVCs.").
Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
Param(ws.PathParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both PVC available and used inodes: `pvc_inodes_available|pvc_inodes_used`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)).
Param(ws.QueryParameter("resources_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return. For example, the following filter matches any pod whose name begins with redis: `redis.*`.").DataType("string").Required(false)).
Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
Param(ws.QueryParameter("sort_metric", "Sort PVCs by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
Metadata(restfulspec.KeyOpenAPITags, []string{constants.PVCMetricsTag}).
Writes(metrics.FormatedLevelMetric{}).
Returns(http.StatusOK, RespOK, metrics.FormatedLevelMetric{})).
Consumes(restful.MIME_JSON, restful.MIME_XML).
Produces(restful.MIME_JSON)
ws.Route(ws.GET("/namespaces/{namespace}/persistentvolumeclaims/{pvc}").To(monitoring.MonitorSpecificPVCofSpecificNamespace).
Doc("Get PVC-level metric data of a specific PVC. Navigate to the PVC by the PVC's namespace.").
Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
Param(ws.PathParameter("pvc", "PVC name.").DataType("string").Required(true)).
Param(ws.PathParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both PVC available and used inodes: `pvc_inodes_available|pvc_inodes_used`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)).
Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
Metadata(restfulspec.KeyOpenAPITags, []string{constants.PVCMetricsTag}).
Writes(metrics.FormatedLevelMetric{}).
Returns(http.StatusOK, RespOK, metrics.FormatedLevelMetric{})).
Consumes(restful.MIME_JSON, restful.MIME_XML).
Produces(restful.MIME_JSON)
ws.Route(ws.GET("/nodes/{node}/pods").To(monitoring.MonitorAllPodsOnSpecificNode).
Doc("Get pod-level metric data of all pods on a specific node.").
Param(ws.PathParameter("node", "Node name.").DataType("string").Required(true)).

View File

@@ -235,6 +235,33 @@ func MonitorNode(request *restful.Request, response *restful.Response) {
}
}
func MonitorAllPVCsOfSpecificNamespace(request *restful.Request, response *restful.Response) {
MonitorPVC(request, response)
}
func MonitorAllPVCsOfSpecificStorageClass(request *restful.Request, response *restful.Response) {
MonitorPVC(request, response)
}
func MonitorSpecificPVCofSpecificNamespace(request *restful.Request, response *restful.Response) {
MonitorPVC(request, response)
}
func MonitorPVC(request *restful.Request, response *restful.Response) {
requestParams := prometheus.ParseMonitoringRequestParams(request)
pvcName := requestParams.PVCName
if pvcName != "" {
requestParams.ResourcesFilter = fmt.Sprintf("^%s$", requestParams.PVCName)
}
rawMetrics := metrics.GetPVCLevelMetrics(requestParams)
// sorting
sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
// paging
pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)
response.WriteAsJson(pagedMetrics)
}
func MonitorComponent(request *restful.Request, response *restful.Response) {
requestParams := prometheus.ParseMonitoringRequestParams(request)

View File

@@ -68,6 +68,7 @@ const (
NodeMetricsTag = "Node Metrics"
NamespaceMetricsTag = "Namespace Metrics"
PodMetricsTag = "Pod Metrics"
PVCMetricsTag = "PVC Metrics"
ContainerMetricsTag = "Container Metrics"
WorkloadMetricsTag = "Workload Metrics"
WorkspaceMetricsTag = "Workspace Metrics"

View File

@@ -276,6 +276,16 @@ func AssemblePodMetricRequestInfo(monitoringRequest *client.MonitoringRequestPar
return queryType, params, rule == ""
}
func AssemblePVCMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) {
queryType := monitoringRequest.QueryType
paramValues := monitoringRequest.Params
rule := MakePVCPromQL(metricName, monitoringRequest.NsName, monitoringRequest.PVCName, monitoringRequest.StorageClassName, monitoringRequest.ResourcesFilter)
params := makeRequestParamString(rule, paramValues)
return queryType, params, rule == ""
}
func GetNodeAddressInfo() *map[string][]v1.NodeAddress {
nodeLister := informers.SharedInformerFactory().Core().V1().Nodes().Lister()
nodes, err := nodeLister.List(labels.Everything())
@@ -901,6 +911,49 @@ func GetContainerLevelMetrics(monitoringRequest *client.MonitoringRequestParams)
}
}
func GetPVCLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
metricsFilter := monitoringRequest.MetricsFilter
if metricsFilter == "" {
metricsFilter = ".*"
}
var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
var wg sync.WaitGroup
for _, metricName := range PVCMetricsNames {
matched, err := regexp.MatchString(metricsFilter, metricName)
if err == nil && matched {
wg.Add(1)
go func(metricName string) {
queryType, params, nullRule := AssemblePVCMetricRequestInfo(monitoringRequest, metricName)
if !nullRule {
metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelPVC: ""})
} else {
ch <- nil
}
wg.Done()
}(metricName)
}
}
wg.Wait()
close(ch)
var metricsArray []FormatedMetric
for oneMetric := range ch {
if oneMetric != nil {
metricsArray = append(metricsArray, *oneMetric)
}
}
return &FormatedLevelMetric{
MetricsLevel: MetricLevelPVC,
Results: metricsArray,
}
}
func GetComponentLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
metricsFilter := monitoringRequest.MetricsFilter
if metricsFilter == "" {

View File

@@ -198,6 +198,37 @@ func MakePodPromQL(metricName, nsName, nodeID, podName, podFilter string) string
return promql
}
func MakePVCPromQL(metricName, nsName, pvcName, scName, pvcFilter string) string {
if pvcFilter == "" {
pvcFilter = ".*"
}
var promql = ""
if nsName != "" {
// get pvc metrics by namespace
if pvcName != "" {
// specific pvc
promql = RulePromQLTmplMap[metricName]
promql = strings.Replace(promql, "$1", nsName, -1)
promql = strings.Replace(promql, "$2", pvcName, -1)
} else {
// all pvc in a specific namespace
metricName += "_ns"
promql = RulePromQLTmplMap[metricName]
promql = strings.Replace(promql, "$1", nsName, -1)
promql = strings.Replace(promql, "$2", pvcFilter, -1)
}
} else {
if scName != "" {
// all pvc in a specific storageclass
metricName += "_sc"
promql = RulePromQLTmplMap[metricName]
promql = strings.Replace(promql, "$1", scName, -1)
}
}
return promql
}
func MakeNamespacePromQL(nsName string, nsFilter string, metricsName string) string {
var recordingRule = RulePromQLTmplMap[metricsName]

View File

@@ -67,6 +67,7 @@ const (
MetricLevelPodName = "pod_name"
MetricLevelContainer = "container"
MetricLevelContainerName = "container_name"
MetricLevelPVC = "persistentvolumeclaim"
MetricLevelWorkload = "workload"
MetricLevelComponent = "component"
)
@@ -330,6 +331,15 @@ var ContainerMetricsNames = []string{
//"container_net_bytes_received",
}
var PVCMetricsNames = []string{
"pvc_inodes_available",
"pvc_inodes_used",
"pvc_inodes_total",
"pvc_bytes_available",
"pvc_bytes_used",
"pvc_bytes_total",
}
var ComponentMetricsNames = []string{
"etcd_server_list",
"etcd_server_total",
@@ -684,6 +694,26 @@ var RulePromQLTmplMap = MetricMap{
// New in ks 2.0
"workspace_pod_abnormal_ratio": `count((kube_pod_info{node!="", namespace$1} unless on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Succeeded"}>0) unless on (pod, namespace) ((kube_pod_status_ready{job="kube-state-metrics", condition="true"}>0) and on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Running"}>0)) unless on (pod, namespace) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", reason="ContainerCreating"}>0)) / sum(kube_pod_status_phase{phase!~"Succeeded", namespace!="", namespace$1}) * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`,
// PVC
"pvc_inodes_available": `max (kubelet_volume_stats_inodes_free{namespace="$1",persistentvolumeclaim="$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
"pvc_inodes_used": `max (kubelet_volume_stats_inodes_used{namespace="$1", persistentvolumeclaim="$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
"pvc_inodes_total": `max (kubelet_volume_stats_inodes{namespace="$1", persistentvolumeclaim="$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
"pvc_bytes_available": `max (kubelet_volume_stats_available_bytes{namespace="$1", persistentvolumeclaim="$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
"pvc_bytes_used": `max (kubelet_volume_stats_used_bytes{namespace="$1", persistentvolumeclaim="$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
"pvc_bytes_total": `max (kubelet_volume_stats_capacity_bytes{namespace="$1", persistentvolumeclaim="$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
"pvc_inodes_available_ns": `max (kubelet_volume_stats_inodes_free{namespace="$1",persistentvolumeclaim=~"$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
"pvc_inodes_used_ns": `max (kubelet_volume_stats_inodes_used{namespace="$1",persistentvolumeclaim=~"$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
"pvc_inodes_total_ns": `max (kubelet_volume_stats_inodes{namespace="$1",persistentvolumeclaim=~"$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
"pvc_bytes_available_ns": `max (kubelet_volume_stats_available_bytes{namespace="$1",persistentvolumeclaim=~"$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
"pvc_bytes_used_ns": `max (kubelet_volume_stats_used_bytes{namespace="$1",persistentvolumeclaim=~"$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
"pvc_bytes_total_ns": `max (kubelet_volume_stats_capacity_bytes{namespace="$1",persistentvolumeclaim=~"$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
"pvc_inodes_available_sc": `max (kubelet_volume_stats_inodes_free)by(namespace,persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info{storageclass="$1"}`,
"pvc_inodes_used_sc": `max (kubelet_volume_stats_inodes_used)by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info{storageclass="$1"}`,
"pvc_inodes_total_sc": `max (kubelet_volume_stats_inodes)by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info{storageclass="$1"}`,
"pvc_bytes_available_sc": `max (kubelet_volume_stats_available_bytes)by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info{storageclass="$1"}`,
"pvc_bytes_used_sc": `max (kubelet_volume_stats_used_bytes)by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info{storageclass="$1"}`,
"pvc_bytes_total_sc": `max (kubelet_volume_stats_capacity_bytes)by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info{storageclass="$1"}`,
// component
"etcd_server_list": `label_replace(up{job="etcd"}, "node_ip", "$1", "instance", "(.*):.*")`,
"etcd_server_total": `count(up{job="etcd"})`,

View File

@@ -50,24 +50,26 @@ func init() {
}
type MonitoringRequestParams struct {
Params url.Values
QueryType string
SortMetricName string
SortType string
PageNum string
LimitNum string
Tp string
MetricsFilter string
ResourcesFilter string
MetricsName string
WorkloadName string
NodeId string
WsName string
NsName string
PodName string
ContainerName string
WorkloadKind string
ComponentName string
Params url.Values
QueryType string
SortMetricName string
SortType string
PageNum string
LimitNum string
Tp string
MetricsFilter string
ResourcesFilter string
MetricsName string
WorkloadName string
NodeId string
WsName string
NsName string
PodName string
PVCName string
StorageClassName string
ContainerName string
WorkloadKind string
ComponentName string
}
var client = &http.Client{}
@@ -113,27 +115,31 @@ func ParseMonitoringRequestParams(request *restful.Request) *MonitoringRequestPa
wsName := strings.Trim(request.PathParameter("workspace"), " ")
nsName := strings.Trim(request.PathParameter("namespace"), " ")
podName := strings.Trim(request.PathParameter("pod"), " ")
pvcName := strings.Trim(request.PathParameter("pvc"), " ")
storageClassName := strings.Trim(request.PathParameter("storageclass"), " ")
containerName := strings.Trim(request.PathParameter("container"), " ")
workloadKind := strings.Trim(request.PathParameter("kind"), " ")
componentName := strings.Trim(request.PathParameter("component"), " ")
var requestParams = MonitoringRequestParams{
SortMetricName: sortMetricName,
SortType: sortType,
PageNum: pageNum,
LimitNum: limitNum,
Tp: tp,
MetricsFilter: metricsFilter,
ResourcesFilter: resourcesFilter,
MetricsName: metricsName,
WorkloadName: workloadName,
NodeId: nodeId,
WsName: wsName,
NsName: nsName,
PodName: podName,
ContainerName: containerName,
WorkloadKind: workloadKind,
ComponentName: componentName,
SortMetricName: sortMetricName,
SortType: sortType,
PageNum: pageNum,
LimitNum: limitNum,
Tp: tp,
MetricsFilter: metricsFilter,
ResourcesFilter: resourcesFilter,
MetricsName: metricsName,
WorkloadName: workloadName,
NodeId: nodeId,
WsName: wsName,
NsName: nsName,
PodName: podName,
PVCName: pvcName,
StorageClassName: storageClassName,
ContainerName: containerName,
WorkloadKind: workloadKind,
ComponentName: componentName,
}
if timeout == "" {