diff --git a/cmd/controller-manager/app/options/options.go b/cmd/controller-manager/app/options/options.go index 2d98ef8b4..c6e57d422 100644 --- a/cmd/controller-manager/app/options/options.go +++ b/cmd/controller-manager/app/options/options.go @@ -29,6 +29,7 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins" "kubesphere.io/kubesphere/pkg/simple/client/k8s" ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap" + "kubesphere.io/kubesphere/pkg/simple/client/metrics" "kubesphere.io/kubesphere/pkg/simple/client/multicluster" "kubesphere.io/kubesphere/pkg/simple/client/network" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" @@ -45,6 +46,7 @@ type KubeSphereControllerManagerOptions struct { OpenPitrixOptions *openpitrix.Options NetworkOptions *network.Options MultiClusterOptions *multicluster.Options + MetricsOptions *metrics.Options ServiceMeshOptions *servicemesh.Options LeaderElect bool LeaderElection *leaderelection.LeaderElectionConfig @@ -60,6 +62,7 @@ func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions OpenPitrixOptions: openpitrix.NewOptions(), NetworkOptions: network.NewNetworkOptions(), MultiClusterOptions: multicluster.NewOptions(), + MetricsOptions: metrics.NewMetricsOptions(), ServiceMeshOptions: servicemesh.NewServiceMeshOptions(), AuthenticationOptions: authoptions.NewAuthenticateOptions(), LeaderElection: &leaderelection.LeaderElectionConfig{ @@ -86,6 +89,7 @@ func (s *KubeSphereControllerManagerOptions) Flags() cliflag.NamedFlagSets { s.NetworkOptions.AddFlags(fss.FlagSet("network"), s.NetworkOptions) s.MultiClusterOptions.AddFlags(fss.FlagSet("multicluster"), s.MultiClusterOptions) s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"), s.ServiceMeshOptions) + s.MetricsOptions.AddFlags(fss.FlagSet("metrics"), s.MetricsOptions) fs := fss.FlagSet("leaderelection") s.bindLeaderElectionFlags(s.LeaderElection, fs) diff --git a/cmd/controller-manager/app/server.go b/cmd/controller-manager/app/server.go index 6c9c7fdfb..a0a950f1b 100644 --- a/cmd/controller-manager/app/server.go +++ b/cmd/controller-manager/app/server.go @@ -18,6 +18,7 @@ package app import ( "fmt" + "kubesphere.io/kubesphere/pkg/utils/metrics" "github.com/spf13/cobra" utilerrors "k8s.io/apimachinery/pkg/util/errors" cliflag "k8s.io/component-base/cli/flag" @@ -64,6 +65,7 @@ func NewControllerManagerCommand() *cobra.Command { OpenPitrixOptions: conf.OpenPitrixOptions, NetworkOptions: conf.NetworkOptions, MultiClusterOptions: conf.MultiClusterOptions, + MetricsOptions: conf.MetricsOptions, ServiceMeshOptions: conf.ServiceMeshOptions, LeaderElection: s.LeaderElection, LeaderElect: s.LeaderElect, @@ -261,6 +263,11 @@ func run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{}) hookServer.Register("/validate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.ValidatingHandler{C: mgr.GetClient()}}) hookServer.Register("/mutate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.MutatingHandler{C: mgr.GetClient()}}) + if s.MetricsOptions != nil && s.MetricsOptions.Enable { + klog.V(2).Info("registering metrics to then webhook server") + hookServer.Register("/metrics", metrics.Handler()) + } + klog.V(0).Info("Starting the controllers.") if err = mgr.Start(stopCh); err != nil { klog.Fatalf("unable to run the manager: %v", err) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 11f0131dd..a0f02d67a 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -179,14 +179,15 @@ func monitorRequest(r *restful.Request, response *restful.Response, chain *restf chain.ProcessFilter(r, response) reqInfo, exists := request.RequestInfoFrom(r.Request.Context()) if exists && reqInfo.APIGroup != "" { - metrics.RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc() + RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc() elapsedSeconds := time.Now().Sub(start).Seconds() - metrics.RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds) + RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds) } } func (s *APIServer) installAPI() { if s.Config.MetricsOptions != nil && s.Config.MetricsOptions.Enable { + register() metrics.Defaults.Install(s.container) } } diff --git a/pkg/utils/metrics/endpoint.go b/pkg/apiserver/metric.go similarity index 86% rename from pkg/utils/metrics/endpoint.go rename to pkg/apiserver/metric.go index 73c4ded83..1365202fb 100644 --- a/pkg/utils/metrics/endpoint.go +++ b/pkg/apiserver/metric.go @@ -1,7 +1,8 @@ -package metrics +package apiserver import ( compbasemetrics "k8s.io/component-base/metrics" + "kubesphere.io/kubesphere/pkg/utils/metrics" ) var ( @@ -28,8 +29,14 @@ var ( []string{"verb", "group", "version", "resource"}, ) - metrics = []compbasemetrics.Registerable{ + metricsList = []compbasemetrics.Registerable{ RequestCounter, RequestLatencies, } ) + +func register() { + for _, m := range metricsList { + metrics.MustRegister(m) + } +} diff --git a/pkg/controller/workspace/metrics.go b/pkg/controller/workspace/metrics.go new file mode 100644 index 000000000..b3e5c0a63 --- /dev/null +++ b/pkg/controller/workspace/metrics.go @@ -0,0 +1,24 @@ +package workspace + +import ( + compbasemetrics "k8s.io/component-base/metrics" + "kubesphere.io/kubesphere/pkg/utils/metrics" +) + +var ( + workspaceOperation = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "ks_controller_manager_workspace_operation", + Help: "Counter of ks controller manager workspace operation broken out for each operation, name", + // This metric is used for verifying api call latencies SLO, + // as well as tracking regressions in this aspects. + // Thus we customize buckets significantly, to empower both usecases. + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"operation", "name"}, + ) +) + +func init() { + metrics.MustRegister(workspaceOperation) +} diff --git a/pkg/controller/workspace/workspace_controller.go b/pkg/controller/workspace/workspace_controller.go index f412c13a3..357d480ac 100644 --- a/pkg/controller/workspace/workspace_controller.go +++ b/pkg/controller/workspace/workspace_controller.go @@ -98,6 +98,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { if err := r.Update(rootCtx, workspace); err != nil { return ctrl.Result{}, err } + workspaceOperation.WithLabelValues("create", instance.Name).Inc() } } else { // The object is being deleted @@ -111,6 +112,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { logger.Error(err, "update workspace failed") return ctrl.Result{}, err } + workspaceOperation.WithLabelValues("delete", instance.Name).Inc() } // Our finalizer has finished, so the reconciler can do nothing. return ctrl.Result{}, nil diff --git a/pkg/simple/client/metrics/metrics.go b/pkg/simple/client/metrics/metrics.go index 34989e635..96cf93a6e 100644 --- a/pkg/simple/client/metrics/metrics.go +++ b/pkg/simple/client/metrics/metrics.go @@ -20,5 +20,5 @@ func (s *Options) ApplyTo(options *Options) { } func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) { - fs.BoolVar(&s.Enable, "enable-metric", c.Enable, "If true, allow metric. [default=false]") + fs.BoolVar(&s.Enable, "enable-metrics", c.Enable, "If true, allow metrics. [default=false]") } diff --git a/pkg/utils/metrics/metrics.go b/pkg/utils/metrics/metrics.go index 19cecb3dd..447549886 100644 --- a/pkg/utils/metrics/metrics.go +++ b/pkg/utils/metrics/metrics.go @@ -8,25 +8,28 @@ import ( compbasemetrics "k8s.io/component-base/metrics" ksVersion "kubesphere.io/kubesphere/pkg/version" "net/http" - "sync" ) var ( - Defaults DefaultMetrics - //registerMetrics sync.Once - defaultRegistry = compbasemetrics.NewKubeRegistry() + Defaults DefaultMetrics + defaultRegistry compbasemetrics.KubeRegistry // MustRegister registers registerable metrics but uses the defaultRegistry, panic upon the first registration that causes an error - MustRegister = defaultRegistry.MustRegister + MustRegister func(...compbasemetrics.Registerable) // Register registers a collectable metric but uses the defaultRegistry - Register = defaultRegistry.Register + Register func(compbasemetrics.Registerable) error ) +func init() { + defaultRegistry = compbasemetrics.NewKubeRegistry() + MustRegister = defaultRegistry.MustRegister + Register = defaultRegistry.Register +} + // DefaultMetrics installs the default prometheus metrics handler type DefaultMetrics struct{} // Install adds the DefaultMetrics handler func (m DefaultMetrics) Install(c *restful.Container) { - register() c.Handle("/kapis/metrics", Handler()) } @@ -51,15 +54,3 @@ func versionGet() apimachineryversion.Info { func Handler() http.Handler { return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, promhttp.HandlerFor(defaultRegistry, promhttp.HandlerOpts{})) } - -var registerMetrics sync.Once - -func register() { - registerMetrics.Do(func() { - defaultRegistry.RawMustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) - defaultRegistry.RawMustRegister(prometheus.NewGoCollector()) - for _, metric := range metrics { - MustRegister(metric) - } - }) -}