add metrics to controller-manager

Signed-off-by: LiHui <andrewli@yunify.com>

pkg/apiserver/apiserver.go

@@ -179,14 +179,15 @@ func monitorRequest(r *restful.Request, response *restful.Response, chain *restf
 	chain.ProcessFilter(r, response)
 	reqInfo, exists := request.RequestInfoFrom(r.Request.Context())
 	if exists && reqInfo.APIGroup != "" {
-		metrics.RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc()
+		RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc()
 		elapsedSeconds := time.Now().Sub(start).Seconds()
-		metrics.RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds)
+		RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds)
 	}
 }
 
 func (s *APIServer) installAPI() {
 	if s.Config.MetricsOptions != nil && s.Config.MetricsOptions.Enable {
+		register()
 		metrics.Defaults.Install(s.container)
 	}
 }

pkg/apiserver/metric.go

@@ -1,7 +1,8 @@
-package metrics
+package apiserver
 
 import (
 	compbasemetrics "k8s.io/component-base/metrics"
+	"kubesphere.io/kubesphere/pkg/utils/metrics"
 )
 
 var (
@@ -28,8 +29,14 @@ var (
 		[]string{"verb", "group", "version", "resource"},
 	)
 
-	metrics = []compbasemetrics.Registerable{
+	metricsList = []compbasemetrics.Registerable{
 		RequestCounter,
 		RequestLatencies,
 	}
 )
+
+func register() {
+	for _, m := range metricsList {
+		metrics.MustRegister(m)
+	}
+}

pkg/controller/workspace/metrics.go

@@ -0,0 +1,24 @@
+package workspace
+
+import (
+	compbasemetrics "k8s.io/component-base/metrics"
+	"kubesphere.io/kubesphere/pkg/utils/metrics"
+)
+
+var (
+	workspaceOperation = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Name: "ks_controller_manager_workspace_operation",
+			Help: "Counter of ks controller manager workspace operation broken out for each operation, name",
+			// This metric is used for verifying api call latencies SLO,
+			// as well as tracking regressions in this aspects.
+			// Thus we customize buckets significantly, to empower both usecases.
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"operation", "name"},
+	)
+)
+
+func init() {
+	metrics.MustRegister(workspaceOperation)
+}

pkg/controller/workspace/workspace_controller.go

@@ -98,6 +98,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			if err := r.Update(rootCtx, workspace); err != nil {
 				return ctrl.Result{}, err
 			}
+			workspaceOperation.WithLabelValues("create", instance.Name).Inc()
 		}
 	} else {
 		// The object is being deleted
@@ -111,6 +112,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 				logger.Error(err, "update workspace failed")
 				return ctrl.Result{}, err
 			}
+			workspaceOperation.WithLabelValues("delete", instance.Name).Inc()
 		}
 		// Our finalizer has finished, so the reconciler can do nothing.
 		return ctrl.Result{}, nil

pkg/simple/client/metrics/metrics.go

@@ -20,5 +20,5 @@ func (s *Options) ApplyTo(options *Options) {
 }
 
 func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) {
-	fs.BoolVar(&s.Enable, "enable-metric", c.Enable, "If true, allow metric. [default=false]")
+	fs.BoolVar(&s.Enable, "enable-metrics", c.Enable, "If true, allow metrics. [default=false]")
 }

pkg/utils/metrics/metrics.go

@@ -8,25 +8,28 @@ import (
 	compbasemetrics "k8s.io/component-base/metrics"
 	ksVersion "kubesphere.io/kubesphere/pkg/version"
 	"net/http"
-	"sync"
 )
 
 var (
-	Defaults DefaultMetrics
-	//registerMetrics sync.Once
-	defaultRegistry = compbasemetrics.NewKubeRegistry()
+	Defaults        DefaultMetrics
+	defaultRegistry compbasemetrics.KubeRegistry
 	// MustRegister registers registerable metrics but uses the defaultRegistry, panic upon the first registration that causes an error
-	MustRegister = defaultRegistry.MustRegister
+	MustRegister func(...compbasemetrics.Registerable)
 	// Register registers a collectable metric but uses the defaultRegistry
-	Register = defaultRegistry.Register
+	Register func(compbasemetrics.Registerable) error
 )
 
+func init() {
+	defaultRegistry = compbasemetrics.NewKubeRegistry()
+	MustRegister = defaultRegistry.MustRegister
+	Register = defaultRegistry.Register
+}
+
 // DefaultMetrics installs the default prometheus metrics handler
 type DefaultMetrics struct{}
 
 // Install adds the DefaultMetrics handler
 func (m DefaultMetrics) Install(c *restful.Container) {
-	register()
 	c.Handle("/kapis/metrics", Handler())
 }
 
@@ -51,15 +54,3 @@ func versionGet() apimachineryversion.Info {
 func Handler() http.Handler {
 	return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, promhttp.HandlerFor(defaultRegistry, promhttp.HandlerOpts{}))
 }
-
-var registerMetrics sync.Once
-
-func register() {
-	registerMetrics.Do(func() {
-		defaultRegistry.RawMustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
-		defaultRegistry.RawMustRegister(prometheus.NewGoCollector())
-		for _, metric := range metrics {
-			MustRegister(metric)
-		}
-	})
-}