add metrics to controller-manager

Signed-off-by: LiHui <andrewli@yunify.com>

cmd/controller-manager/app/options/options.go

@@ -29,6 +29,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
 	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
 	ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap"
+	"kubesphere.io/kubesphere/pkg/simple/client/metrics"
 	"kubesphere.io/kubesphere/pkg/simple/client/multicluster"
 	"kubesphere.io/kubesphere/pkg/simple/client/network"
 	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
@@ -45,6 +46,7 @@ type KubeSphereControllerManagerOptions struct {
 	OpenPitrixOptions     *openpitrix.Options
 	NetworkOptions        *network.Options
 	MultiClusterOptions   *multicluster.Options
+	MetricsOptions        *metrics.Options
 	ServiceMeshOptions    *servicemesh.Options
 	LeaderElect           bool
 	LeaderElection        *leaderelection.LeaderElectionConfig
@@ -60,6 +62,7 @@ func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions
 		OpenPitrixOptions:     openpitrix.NewOptions(),
 		NetworkOptions:        network.NewNetworkOptions(),
 		MultiClusterOptions:   multicluster.NewOptions(),
+		MetricsOptions:        metrics.NewMetricsOptions(),
 		ServiceMeshOptions:    servicemesh.NewServiceMeshOptions(),
 		AuthenticationOptions: authoptions.NewAuthenticateOptions(),
 		LeaderElection: &leaderelection.LeaderElectionConfig{
@@ -86,6 +89,7 @@ func (s *KubeSphereControllerManagerOptions) Flags() cliflag.NamedFlagSets {
 	s.NetworkOptions.AddFlags(fss.FlagSet("network"), s.NetworkOptions)
 	s.MultiClusterOptions.AddFlags(fss.FlagSet("multicluster"), s.MultiClusterOptions)
 	s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"), s.ServiceMeshOptions)
+	s.MetricsOptions.AddFlags(fss.FlagSet("metrics"), s.MetricsOptions)
 
 	fs := fss.FlagSet("leaderelection")
 	s.bindLeaderElectionFlags(s.LeaderElection, fs)

cmd/controller-manager/app/server.go

@@ -18,6 +18,7 @@ package app
 
 import (
 	"fmt"
+	"kubesphere.io/kubesphere/pkg/utils/metrics"
 	"github.com/spf13/cobra"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	cliflag "k8s.io/component-base/cli/flag"
@@ -64,6 +65,7 @@ func NewControllerManagerCommand() *cobra.Command {
 			OpenPitrixOptions:     conf.OpenPitrixOptions,
 			NetworkOptions:        conf.NetworkOptions,
 			MultiClusterOptions:   conf.MultiClusterOptions,
+			MetricsOptions:        conf.MetricsOptions,
 			ServiceMeshOptions:    conf.ServiceMeshOptions,
 			LeaderElection:        s.LeaderElection,
 			LeaderElect:           s.LeaderElect,
@@ -261,6 +263,11 @@ func run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{})
 	hookServer.Register("/validate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.ValidatingHandler{C: mgr.GetClient()}})
 	hookServer.Register("/mutate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.MutatingHandler{C: mgr.GetClient()}})
 
+	if s.MetricsOptions != nil && s.MetricsOptions.Enable {
+		klog.V(2).Info("registering metrics to then webhook server")
+		hookServer.Register("/metrics", metrics.Handler())
+	}
+
 	klog.V(0).Info("Starting the controllers.")
 	if err = mgr.Start(stopCh); err != nil {
 		klog.Fatalf("unable to run the manager: %v", err)

pkg/apiserver/apiserver.go

@@ -179,14 +179,15 @@ func monitorRequest(r *restful.Request, response *restful.Response, chain *restf
 	chain.ProcessFilter(r, response)
 	reqInfo, exists := request.RequestInfoFrom(r.Request.Context())
 	if exists && reqInfo.APIGroup != "" {
-		metrics.RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc()
+		RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc()
 		elapsedSeconds := time.Now().Sub(start).Seconds()
-		metrics.RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds)
+		RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds)
 	}
 }
 
 func (s *APIServer) installAPI() {
 	if s.Config.MetricsOptions != nil && s.Config.MetricsOptions.Enable {
+		register()
 		metrics.Defaults.Install(s.container)
 	}
 }

pkg/apiserver/metric.go

@@ -1,7 +1,8 @@
-package metrics
+package apiserver
 
 import (
 	compbasemetrics "k8s.io/component-base/metrics"
+	"kubesphere.io/kubesphere/pkg/utils/metrics"
 )
 
 var (
@@ -28,8 +29,14 @@ var (
 		[]string{"verb", "group", "version", "resource"},
 	)
 
-	metrics = []compbasemetrics.Registerable{
+	metricsList = []compbasemetrics.Registerable{
 		RequestCounter,
 		RequestLatencies,
 	}
 )
+
+func register() {
+	for _, m := range metricsList {
+		metrics.MustRegister(m)
+	}
+}

pkg/controller/workspace/metrics.go

@@ -0,0 +1,24 @@
+package workspace
+
+import (
+	compbasemetrics "k8s.io/component-base/metrics"
+	"kubesphere.io/kubesphere/pkg/utils/metrics"
+)
+
+var (
+	workspaceOperation = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Name: "ks_controller_manager_workspace_operation",
+			Help: "Counter of ks controller manager workspace operation broken out for each operation, name",
+			// This metric is used for verifying api call latencies SLO,
+			// as well as tracking regressions in this aspects.
+			// Thus we customize buckets significantly, to empower both usecases.
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"operation", "name"},
+	)
+)
+
+func init() {
+	metrics.MustRegister(workspaceOperation)
+}

pkg/controller/workspace/workspace_controller.go

@@ -98,6 +98,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			if err := r.Update(rootCtx, workspace); err != nil {
 				return ctrl.Result{}, err
 			}
+			workspaceOperation.WithLabelValues("create", instance.Name).Inc()
 		}
 	} else {
 		// The object is being deleted
@@ -111,6 +112,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 				logger.Error(err, "update workspace failed")
 				return ctrl.Result{}, err
 			}
+			workspaceOperation.WithLabelValues("delete", instance.Name).Inc()
 		}
 		// Our finalizer has finished, so the reconciler can do nothing.
 		return ctrl.Result{}, nil

pkg/simple/client/metrics/metrics.go

@@ -20,5 +20,5 @@ func (s *Options) ApplyTo(options *Options) {
 }
 
 func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) {
-	fs.BoolVar(&s.Enable, "enable-metric", c.Enable, "If true, allow metric. [default=false]")
+	fs.BoolVar(&s.Enable, "enable-metrics", c.Enable, "If true, allow metrics. [default=false]")
 }

pkg/utils/metrics/metrics.go

@@ -8,25 +8,28 @@ import (
 	compbasemetrics "k8s.io/component-base/metrics"
 	ksVersion "kubesphere.io/kubesphere/pkg/version"
 	"net/http"
-	"sync"
 )
 
 var (
-	Defaults DefaultMetrics
-	//registerMetrics sync.Once
-	defaultRegistry = compbasemetrics.NewKubeRegistry()
+	Defaults        DefaultMetrics
+	defaultRegistry compbasemetrics.KubeRegistry
 	// MustRegister registers registerable metrics but uses the defaultRegistry, panic upon the first registration that causes an error
-	MustRegister = defaultRegistry.MustRegister
+	MustRegister func(...compbasemetrics.Registerable)
 	// Register registers a collectable metric but uses the defaultRegistry
-	Register = defaultRegistry.Register
+	Register func(compbasemetrics.Registerable) error
 )
 
+func init() {
+	defaultRegistry = compbasemetrics.NewKubeRegistry()
+	MustRegister = defaultRegistry.MustRegister
+	Register = defaultRegistry.Register
+}
+
 // DefaultMetrics installs the default prometheus metrics handler
 type DefaultMetrics struct{}
 
 // Install adds the DefaultMetrics handler
 func (m DefaultMetrics) Install(c *restful.Container) {
-	register()
 	c.Handle("/kapis/metrics", Handler())
 }
 
@@ -51,15 +54,3 @@ func versionGet() apimachineryversion.Info {
 func Handler() http.Handler {
 	return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, promhttp.HandlerFor(defaultRegistry, promhttp.HandlerOpts{}))
 }
-
-var registerMetrics sync.Once
-
-func register() {
-	registerMetrics.Do(func() {
-		defaultRegistry.RawMustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
-		defaultRegistry.RawMustRegister(prometheus.NewGoCollector())
-		for _, metric := range metrics {
-			MustRegister(metric)
-		}
-	})
-}