Merge pull request #3220 from xyz-li/master

add metrics

pkg/apiserver/apiserver.go

@@ -82,10 +82,12 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 	"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
+	"kubesphere.io/kubesphere/pkg/utils/metrics"
 	utilnet "kubesphere.io/kubesphere/pkg/utils/net"
 	"net/http"
 	rt "runtime"
 	runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
+	"strconv"
 	"time"
 )
 
@@ -149,7 +151,6 @@ type APIServer struct {
 }
 
 func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
-
 	s.container = restful.NewContainer()
 	s.container.Filter(logRequestAndResponse)
 	s.container.Router(restful.CurlyRouter{})
@@ -159,6 +160,9 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
 
 	s.installKubeSphereAPIs()
 
+	s.installMetricsAPI()
+	s.container.Filter(monitorRequest)
+
 	for _, ws := range s.container.RegisteredWebServices() {
 		klog.V(2).Infof("%s", ws.RootPath())
 	}
@@ -170,6 +174,22 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
 	return nil
 }
 
+func monitorRequest(r *restful.Request, response *restful.Response, chain *restful.FilterChain) {
+	start := time.Now()
+	chain.ProcessFilter(r, response)
+	reqInfo, exists := request.RequestInfoFrom(r.Request.Context())
+	if exists && reqInfo.APIGroup != "" {
+		RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc()
+		elapsedSeconds := time.Now().Sub(start).Seconds()
+		RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds)
+	}
+}
+
+func (s *APIServer) installMetricsAPI() {
+	registerMetrics()
+	metrics.Defaults.Install(s.container)
+}
+
 // Install all kubesphere api groups
 // Installation happens before all informers start to cache objects, so
 //   any attempt to list objects using listers will get empty results.
@@ -297,7 +317,7 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) {
 	default:
 		fallthrough
 	case authorizationoptions.RBAC:
-		excludedPaths := []string{"/oauth/*", "/kapis/config.kubesphere.io/*", "/kapis/version"}
+		excludedPaths := []string{"/oauth/*", "/kapis/config.kubesphere.io/*", "/kapis/version", "/kapis/metrics"}
 		pathAuthorizer, _ := path.NewAuthorizer(excludedPaths)
 		amOperator := am.NewReadOnlyOperator(s.InformerFactory)
 		authorizers = unionauthorizer.New(pathAuthorizer, rbac.NewRBACAuthorizer(amOperator))
@@ -321,6 +341,7 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) {
 			s.InformerFactory.KubeSphereSharedInformerFactory().Iam().V1alpha2().Users().Lister())))
 	handler = filters.WithAuthentication(handler, authn)
 	handler = filters.WithRequestInfo(handler, requestInfoResolver)
+
 	s.Server.Handler = handler
 }
 

pkg/apiserver/metric.go

@@ -0,0 +1,42 @@
+package apiserver
+
+import (
+	compbasemetrics "k8s.io/component-base/metrics"
+	"kubesphere.io/kubesphere/pkg/utils/metrics"
+)
+
+var (
+	RequestCounter = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Name:           "ks_server_request_total",
+			Help:           "Counter of ks_server requests broken out for each verb, group, version, resource and HTTP response code.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"verb", "group", "version", "resource", "code"},
+	)
+
+	RequestLatencies = compbasemetrics.NewHistogramVec(
+		&compbasemetrics.HistogramOpts{
+			Name: "ks_server_request_duration_seconds",
+			Help: "Response latency distribution in seconds for each verb, group, version, resource",
+			// This metric is used for verifying api call latencies SLO,
+			// as well as tracking regressions in this aspects.
+			// Thus we customize buckets significantly, to empower both usecases.
+			Buckets: []float64{0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
+				1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60},
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"verb", "group", "version", "resource"},
+	)
+
+	metricsList = []compbasemetrics.Registerable{
+		RequestCounter,
+		RequestLatencies,
+	}
+)
+
+func registerMetrics() {
+	for _, m := range metricsList {
+		metrics.MustRegister(m)
+	}
+}