From 1ee5e49ac053bac7b3e12fecbd553600e25b702d Mon Sep 17 00:00:00 2001 From: "Roland.Ma" Date: Fri, 10 Sep 2021 02:13:22 +0000 Subject: [PATCH] feat: ingress metrics query apis Signed-off-by: Roland.Ma --- pkg/constants/constants.go | 1 + pkg/kapis/monitoring/v1alpha3/handler.go | 16 ++++++ pkg/kapis/monitoring/v1alpha3/helper.go | 26 ++++++++++ pkg/kapis/monitoring/v1alpha3/helper_test.go | 42 +++++++++++++++ pkg/kapis/monitoring/v1alpha3/register.go | 38 ++++++++++++++ pkg/models/monitoring/named_metrics.go | 16 ++++++ pkg/models/monitoring/sort_page.go | 1 + .../client/monitoring/prometheus/promql.go | 51 +++++++++++++++++++ .../monitoring/prometheus/promql_test.go | 10 ++++ .../monitoring/prometheus/testdata/promqls.go | 1 + pkg/simple/client/monitoring/query_options.go | 26 ++++++++++ 11 files changed, 228 insertions(+) diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 77f5647fa..5d6169186 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -107,6 +107,7 @@ const ( NamespaceMetricsTag = "Namespace Metrics" PodMetricsTag = "Pod Metrics" PVCMetricsTag = "PVC Metrics" + IngressMetricsTag = "Ingress Metrics" ContainerMetricsTag = "Container Metrics" WorkloadMetricsTag = "Workload Metrics" WorkspaceMetricsTag = "Workspace Metrics" diff --git a/pkg/kapis/monitoring/v1alpha3/handler.go b/pkg/kapis/monitoring/v1alpha3/handler.go index e0d94ef9c..8da172dc0 100644 --- a/pkg/kapis/monitoring/v1alpha3/handler.go +++ b/pkg/kapis/monitoring/v1alpha3/handler.go @@ -205,6 +205,22 @@ func (h handler) handlePVCMetricsQuery(req *restful.Request, resp *restful.Respo h.handleNamedMetricsQuery(resp, opt) } +func (h handler) handleIngressMetricsQuery(req *restful.Request, resp *restful.Response) { + params := parseRequestParams(req) + opt, err := h.makeQueryOptions(params, monitoring.LevelIngress) + if err != nil { + if err.Error() == ErrNoHit { + res := handleNoHit(opt.namedMetrics) + resp.WriteAsJson(res) + return + } + + api.HandleBadRequest(resp, nil, err) + return + } + h.handleNamedMetricsQuery(resp, opt) +} + func (h handler) handleComponentMetricsQuery(req *restful.Request, resp *restful.Response) { params := parseRequestParams(req) opt, err := h.makeQueryOptions(params, monitoring.LevelComponent) diff --git a/pkg/kapis/monitoring/v1alpha3/helper.go b/pkg/kapis/monitoring/v1alpha3/helper.go index 92e861aea..a2802de57 100644 --- a/pkg/kapis/monitoring/v1alpha3/helper.go +++ b/pkg/kapis/monitoring/v1alpha3/helper.go @@ -88,6 +88,8 @@ type reqParams struct { applications string openpitrixs string cluster string + ingress string + job string services string pvcFilter string queryType string @@ -141,11 +143,15 @@ func parseRequestParams(req *restful.Request) reqParams { r.workloadKind = req.PathParameter("kind") r.nodeName = req.PathParameter("node") r.workloadName = req.PathParameter("workload") + //will be overide if "pod" in the path parameter. + r.podName = req.QueryParameter("pod") r.podName = req.PathParameter("pod") r.containerName = req.PathParameter("container") r.pvcName = req.PathParameter("pvc") r.storageClassName = req.PathParameter("storageclass") r.componentType = req.PathParameter("component") + r.ingress = req.PathParameter("ingress") + r.job = req.QueryParameter("job") r.expression = req.QueryParameter("expr") r.metric = req.QueryParameter("metric") r.queryType = req.QueryParameter("type") @@ -337,6 +343,26 @@ func (h handler) makeQueryOptions(r reqParams, lvl monitoring.Level) (q queryOpt } q.namedMetrics = model.PVCMetrics + case monitoring.LevelIngress: + q.identifier = model.IdentifierIngress + var step *time.Duration + // step param is reused in none Range Query to pass vector's time duration. + if r.time != "" { + s, err := time.ParseDuration(r.step) + if err == nil { + step = &s + } + } + q.option = monitoring.IngressOption{ + ResourceFilter: r.resourceFilter, + NamespaceName: r.namespaceName, + Ingress: r.ingress, + Job: r.job, + Pod: r.podName, + Step: step, + } + q.namedMetrics = model.IngressMetrics + case monitoring.LevelComponent: q.option = monitoring.ComponentOption{} switch r.componentType { diff --git a/pkg/kapis/monitoring/v1alpha3/helper_test.go b/pkg/kapis/monitoring/v1alpha3/helper_test.go index c9882f756..84d2d7a23 100644 --- a/pkg/kapis/monitoring/v1alpha3/helper_test.go +++ b/pkg/kapis/monitoring/v1alpha3/helper_test.go @@ -221,6 +221,48 @@ func TestParseRequestParams(t *testing.T) { }, expectedErr: false, }, + { + namespace: corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + CreationTimestamp: metav1.Time{ + Time: time.Unix(1585836666, 0), + }, + }, + }, + params: reqParams{ + time: "1585839999", + metricFilter: "ingress_request_count", + page: "1", + limit: "10", + order: "desc", + target: "ingress_request_count", + job: "job-1", + podName: "pod-1", + namespaceName: "default", + ingress: "ingress-1", + }, + lvl: monitoring.LevelIngress, + expected: queryOptions{ + time: time.Unix(1585839999, 0), + metricFilter: "ingress_request_count", + namedMetrics: model.IngressMetrics, + option: monitoring.IngressOption{ + ResourceFilter: ".*", + NamespaceName: "default", + Ingress: "ingress-1", + Job: "job-1", + Pod: "pod-1", + }, + target: "ingress_request_count", + identifier: "ingress", + order: "desc", + page: 1, + limit: 10, + Operation: OperationQuery, + }, + expectedErr: false, + }, { params: reqParams{ time: "1585830000", diff --git a/pkg/kapis/monitoring/v1alpha3/register.go b/pkg/kapis/monitoring/v1alpha3/register.go index 4858ddb73..4369a4443 100644 --- a/pkg/kapis/monitoring/v1alpha3/register.go +++ b/pkg/kapis/monitoring/v1alpha3/register.go @@ -425,6 +425,44 @@ func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, monito Returns(http.StatusOK, respOK, model.Metrics{})). Produces(restful.MIME_JSON) + ws.Route(ws.GET("/namespaces/{namespace}/ingresses"). + To(h.handleIngressMetricsQuery). + Doc("Get Ingress-level metric data of the specific namespace's Ingresses."). + Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). + Param(ws.QueryParameter("job", "The job name filter. Ingress could be served by multi Ingress controllers, The job filters metric from a specific controller.").DataType("string").Required(true)). + Param(ws.QueryParameter("pod", "The pod name filter.").DataType("string")). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both PVC available and used inodes: `pvc_inodes_available|pvc_inodes_used`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). + Param(ws.QueryParameter("resources_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return. For example, the following filter matches any pod whose name begins with redis: `redis.*`.").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_metric", "Sort PVCs by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)). + Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)). + Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.IngressMetricsTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/namespaces/{namespace}/ingresses/{ingress}"). + To(h.handleIngressMetricsQuery). + Doc("Get Ingress-level metric data of a specific Ingress. Navigate to the Ingress by the Ingress's namespace."). + Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). + Param(ws.PathParameter("ingress", "ingress name.").DataType("string").Required(true)). + Param(ws.QueryParameter("job", "The job name filter. Ingress could be served by multi Ingress controllers, The job filters metric from a specific controller.").DataType("string").Required(true)). + Param(ws.QueryParameter("pod", "The pod filter.").DataType("string")). + Param(ws.QueryParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both PVC available and used inodes: `pvc_inodes_available|pvc_inodes_used`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)). + Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). + Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). + Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.IngressMetricsTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, respOK, model.Metrics{})). + Produces(restful.MIME_JSON) + ws.Route(ws.GET("/components/{component}"). To(h.handleComponentMetricsQuery). Doc("Get component-level metric data of the specific system component."). diff --git a/pkg/models/monitoring/named_metrics.go b/pkg/models/monitoring/named_metrics.go index 5c01a8e6f..3fb075ae4 100644 --- a/pkg/models/monitoring/named_metrics.go +++ b/pkg/models/monitoring/named_metrics.go @@ -274,6 +274,22 @@ var PVCMetrics = []string{ "pvc_bytes_utilisation", } +var IngressMetrics = []string{ + "ingress_request_count", + "ingress_request_5xx_count", + "ingress_request_4xx_count", + "ingress_active_connections", + "ingress_success_rate", + "ingress_request_duration_average", + "ingress_request_duration_50percentage", + "ingress_request_duration_95percentage", + "ingress_request_duration_99percentage", + "ingress_request_volume", + "ingress_request_volume_by_ingress", + "ingress_request_network_sent", + "ingress_request_network_received", +} + var EtcdMetrics = []string{ "etcd_server_list", "etcd_server_total", diff --git a/pkg/models/monitoring/sort_page.go b/pkg/models/monitoring/sort_page.go index fba5d2259..6926225f2 100644 --- a/pkg/models/monitoring/sort_page.go +++ b/pkg/models/monitoring/sort_page.go @@ -33,6 +33,7 @@ const ( IdentifierPVC = "persistentvolumeclaim" IdentifierService = "service" IdentifierApplication = "application" + IdentifierIngress = "ingress" OrderAscending = "asc" OrderDescending = "desc" diff --git a/pkg/simple/client/monitoring/prometheus/promql.go b/pkg/simple/client/monitoring/prometheus/promql.go index 8849cf3a8..9dd236a4a 100644 --- a/pkg/simple/client/monitoring/prometheus/promql.go +++ b/pkg/simple/client/monitoring/prometheus/promql.go @@ -165,6 +165,21 @@ var promQLTemplates = map[string]string{ "namespace_ingresses_extensions_count": `sum by (namespace) (kube_ingress_labels{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`, "namespace_s2ibuilder_count": `sum by (namespace) (s2i_s2ibuilder_created{namespace!=""} * on (namespace) group_left(workspace) kube_namespace_labels{$1})`, + // ingress + "ingress_request_count": `round(sum(increase(nginx_ingress_controller_requests{$1,$2}[$3])))`, + "ingress_request_4xx_count": `round(sum(increase(nginx_ingress_controller_requests{$1,$2,status="[4].*"}[$3])))`, + "ingress_request_5xx_count": `round(sum(increase(nginx_ingress_controller_requests{$1,$2,status="[5].*"}[$3])))`, + "ingress_active_connections": `sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{$2,state="active"}[$3]))`, + "ingress_success_rate": `sum(rate(nginx_ingress_controller_requests{$1,$2,status!~"[4-5].*"}[$3])) / sum(rate(nginx_ingress_controller_requests{$1,$2}[$3]))`, + "ingress_request_duration_average": `sum_over_time(nginx_ingress_controller_request_duration_seconds_sum{$1,$2}[$3])/sum_over_time(nginx_ingress_controller_request_duration_seconds_count{$1,$2}[$3])`, + "ingress_request_duration_50percentage": `histogram_quantile(0.50, sum by (le) (rate(nginx_ingress_controller_request_duration_seconds_bucket{$1,$2}[$3])))`, + "ingress_request_duration_95percentage": `histogram_quantile(0.90, sum by (le) (rate(nginx_ingress_controller_request_duration_seconds_bucket{$1,$2}[$3])))`, + "ingress_request_duration_99percentage": `histogram_quantile(0.99, sum by (le) (rate(nginx_ingress_controller_request_duration_seconds_bucket{$1,$2}[$3])))`, + "ingress_request_volume": `round(sum(irate(nginx_ingress_controller_requests{$1,$2}[$3])), 0.001)`, + "ingress_request_volume_by_ingress": `round(sum(irate(nginx_ingress_controller_requests{$1,$2}[$3])) by (ingress), 0.001)`, + "ingress_request_network_sent": `sum(irate(nginx_ingress_controller_response_size_sum{$1,$2}[$3]))`, + "ingress_request_network_received": `sum(irate(nginx_ingress_controller_request_size_sum{$1,$2}[$3]))`, + // workload "workload_cpu_usage": `round(namespace:workload_cpu_usage:sum{$1}, 0.001)`, "workload_memory_usage": `namespace:workload_memory_usage:sum{$1}`, @@ -258,6 +273,8 @@ func makeExpr(metric string, opts monitoring.QueryOptions) string { return makeContainerMetricExpr(tmpl, opts) case monitoring.LevelPVC: return makePVCMetricExpr(tmpl, opts) + case monitoring.LevelIngress: + return makeIngressMetricExpr(tmpl, opts) case monitoring.LevelComponent: return tmpl default: @@ -448,3 +465,37 @@ func makePVCMetricExpr(tmpl string, o monitoring.QueryOptions) string { } return strings.Replace(tmpl, "$1", pvcSelector, -1) } + +func makeIngressMetricExpr(tmpl string, o monitoring.QueryOptions) string { + var ingressSelector string + var jobSelector string + duration := "5m" + + // parse Range Vector Selectors metric{key=value}[duration] + if o.MeterOptions != nil { + duration = o.MeterOptions.Step.String() + } + + // For monitoring ingress in the specific namespace + // GET /namespaces/{namespace}/ingress/{ingress} or + // GET /namespaces/{namespace}/ingress + if o.NamespaceName != "" { + if o.Ingress != "" { + ingressSelector = fmt.Sprintf(`exported_namespace="%s", ingress="%s"`, o.NamespaceName, o.Ingress) + } else { + ingressSelector = fmt.Sprintf(`exported_namespace="%s", ingress=~"%s"`, o.NamespaceName, o.ResourceFilter) + } + } + // job is a reqiuried filter + // GET /namespaces/{namespace}/ingress?job=xxx&pod=xxx + if o.Job != "" { + jobSelector = fmt.Sprintf(`job="%s"`, o.Job) + if o.PodName != "" { + jobSelector = fmt.Sprintf(`%s,controller_pod="%s"`, jobSelector, o.PodName) + } + } + + tmpl = strings.Replace(tmpl, "$1", ingressSelector, -1) + tmpl = strings.Replace(tmpl, "$2", jobSelector, -1) + return strings.Replace(tmpl, "$3", duration, -1) +} diff --git a/pkg/simple/client/monitoring/prometheus/promql_test.go b/pkg/simple/client/monitoring/prometheus/promql_test.go index 1a3f1a658..0e69165e2 100644 --- a/pkg/simple/client/monitoring/prometheus/promql_test.go +++ b/pkg/simple/client/monitoring/prometheus/promql_test.go @@ -186,6 +186,16 @@ func TestMakeExpr(t *testing.T) { ResourceFilter: "db-123", }, }, + { + name: "ingress_request_count", + opts: monitoring.QueryOptions{ + Level: monitoring.LevelIngress, + NamespaceName: "default", + Ingress: "ingress-1", + Job: "job-1", + PodName: "pod-1", + }, + }, { name: "etcd_server_list", opts: monitoring.QueryOptions{ diff --git a/pkg/simple/client/monitoring/prometheus/testdata/promqls.go b/pkg/simple/client/monitoring/prometheus/testdata/promqls.go index 030217ecc..79658c7f4 100644 --- a/pkg/simple/client/monitoring/prometheus/testdata/promqls.go +++ b/pkg/simple/client/monitoring/prometheus/testdata/promqls.go @@ -37,5 +37,6 @@ var PromQLs = map[string]string{ "pvc_inodes_available": `max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_inodes_free) * on (namespace, persistentvolumeclaim) group_left (storageclass) kube_persistentvolumeclaim_info{namespace="default", persistentvolumeclaim="db-123"}`, "pvc_inodes_used": `max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_inodes_used) * on (namespace, persistentvolumeclaim) group_left (storageclass) kube_persistentvolumeclaim_info{namespace="default", persistentvolumeclaim=~"db-123"}`, "pvc_inodes_total": `max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_inodes) * on (namespace, persistentvolumeclaim) group_left (storageclass) kube_persistentvolumeclaim_info{storageclass="default", persistentvolumeclaim=~"db-123"}`, + "ingress_request_count": `round(sum(increase(nginx_ingress_controller_requests{exported_namespace="default", ingress="ingress-1",job="job-1",controller_pod="pod-1"}[5m])))`, "etcd_server_list": `label_replace(up{job="etcd"}, "node_ip", "$1", "instance", "(.*):.*")`, } diff --git a/pkg/simple/client/monitoring/query_options.go b/pkg/simple/client/monitoring/query_options.go index 4b5b49a8f..dc3178765 100644 --- a/pkg/simple/client/monitoring/query_options.go +++ b/pkg/simple/client/monitoring/query_options.go @@ -37,6 +37,7 @@ const ( LevelContainer LevelPVC LevelComponent + LevelIngress ) var MeteringLevelMap = map[string]int{ @@ -81,6 +82,8 @@ type QueryOptions struct { PVCFilter string ApplicationName string ServiceName string + Ingress string + Job string MeterOptions *Meteroptions } @@ -287,6 +290,29 @@ func (po PVCOption) Apply(o *QueryOptions) { o.PVCFilter = po.PersistentVolumeClaimName } +type IngressOption struct { + ResourceFilter string + NamespaceName string + Ingress string + Job string + Pod string + Step *time.Duration +} + +func (no IngressOption) Apply(o *QueryOptions) { + o.Level = LevelIngress + o.ResourceFilter = no.ResourceFilter + o.NamespaceName = no.NamespaceName + o.Ingress = no.Ingress + o.Job = no.Job + o.PodName = no.Pod + if no.Step != nil { + o.MeterOptions = &Meteroptions{ + Step: *no.Step, + } + } +} + type ComponentOption struct{} func (_ ComponentOption) Apply(o *QueryOptions) {