From 5c8ac10d26c5d3b6c82589132fc064e405660735 Mon Sep 17 00:00:00 2001 From: LiHui Date: Tue, 22 Dec 2020 16:19:25 +0800 Subject: [PATCH 1/6] add metrics Signed-off-by: LiHui --- cmd/ks-apiserver/app/options/options.go | 1 + pkg/apiserver/apiserver.go | 26 +++++++++- pkg/apiserver/config/config.go | 3 ++ pkg/models/openpitrix/apps.go | 3 ++ pkg/models/openpitrix/metric.go | 25 ++++++++++ pkg/simple/client/metrics/metrics.go | 24 +++++++++ pkg/utils/metrics/endpoint.go | 35 +++++++++++++ pkg/utils/metrics/metrics.go | 65 +++++++++++++++++++++++++ 8 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 pkg/models/openpitrix/metric.go create mode 100644 pkg/simple/client/metrics/metrics.go create mode 100644 pkg/utils/metrics/endpoint.go create mode 100644 pkg/utils/metrics/metrics.go diff --git a/cmd/ks-apiserver/app/options/options.go b/cmd/ks-apiserver/app/options/options.go index 2745b6c25..873b4224e 100644 --- a/cmd/ks-apiserver/app/options/options.go +++ b/cmd/ks-apiserver/app/options/options.go @@ -82,6 +82,7 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) { s.MultiClusterOptions.AddFlags(fss.FlagSet("multicluster"), s.MultiClusterOptions) s.EventsOptions.AddFlags(fss.FlagSet("events"), s.EventsOptions) s.AuditingOptions.AddFlags(fss.FlagSet("auditing"), s.AuditingOptions) + s.MetricsOptions.AddFlags(fss.FlagSet("metric"), s.MetricsOptions) fs = fss.FlagSet("klog") local := flag.NewFlagSet("klog", flag.ExitOnError) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index ab928e54d..11f0131dd 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -149,7 +149,6 @@ type APIServer struct { } func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error { - s.container = restful.NewContainer() s.container.Filter(logRequestAndResponse) s.container.Router(restful.CurlyRouter{}) @@ -159,6 +158,11 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error { s.installKubeSphereAPIs() + s.installAPI() + if s.Config.MetricsOptions != nil && s.Config.MetricsOptions.Enable { + s.container.Filter(monitorRequest) + } + for _, ws := range s.container.RegisteredWebServices() { klog.V(2).Infof("%s", ws.RootPath()) } @@ -170,6 +174,23 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error { return nil } +func monitorRequest(r *restful.Request, response *restful.Response, chain *restful.FilterChain) { + start := time.Now() + chain.ProcessFilter(r, response) + reqInfo, exists := request.RequestInfoFrom(r.Request.Context()) + if exists && reqInfo.APIGroup != "" { + metrics.RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc() + elapsedSeconds := time.Now().Sub(start).Seconds() + metrics.RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds) + } +} + +func (s *APIServer) installAPI() { + if s.Config.MetricsOptions != nil && s.Config.MetricsOptions.Enable { + metrics.Defaults.Install(s.container) + } +} + // Install all kubesphere api groups // Installation happens before all informers start to cache objects, so // any attempt to list objects using listers will get empty results. @@ -297,7 +318,7 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) { default: fallthrough case authorizationoptions.RBAC: - excludedPaths := []string{"/oauth/*", "/kapis/config.kubesphere.io/*", "/kapis/version"} + excludedPaths := []string{"/oauth/*", "/kapis/config.kubesphere.io/*", "/kapis/version", "/kapis/metrics"} pathAuthorizer, _ := path.NewAuthorizer(excludedPaths) amOperator := am.NewReadOnlyOperator(s.InformerFactory) authorizers = unionauthorizer.New(pathAuthorizer, rbac.NewRBACAuthorizer(amOperator)) @@ -321,6 +342,7 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) { s.InformerFactory.KubeSphereSharedInformerFactory().Iam().V1alpha2().Users().Lister()))) handler = filters.WithAuthentication(handler, authn) handler = filters.WithRequestInfo(handler, requestInfoResolver) + s.Server.Handler = handler } diff --git a/pkg/apiserver/config/config.go b/pkg/apiserver/config/config.go index 7255a3c8b..8bdbfd4fe 100644 --- a/pkg/apiserver/config/config.go +++ b/pkg/apiserver/config/config.go @@ -30,6 +30,7 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/k8s" "kubesphere.io/kubesphere/pkg/simple/client/ldap" "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch" + "kubesphere.io/kubesphere/pkg/simple/client/metrics" "kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus" "kubesphere.io/kubesphere/pkg/simple/client/multicluster" "kubesphere.io/kubesphere/pkg/simple/client/network" @@ -97,6 +98,7 @@ type Config struct { AuditingOptions *auditingclient.Options `json:"auditing,omitempty" yaml:"auditing,omitempty" mapstructure:"auditing"` AlertingOptions *alerting.Options `json:"alerting,omitempty" yaml:"alerting,omitempty" mapstructure:"alerting"` NotificationOptions *notification.Options `json:"notification,omitempty" yaml:"notification,omitempty" mapstructure:"notification"` + MetricsOptions *metrics.Options `json:"metrics,omitempty" yaml:"metrics,omitempty" mapstructure:"metrics"` } // newConfig creates a default non-empty Config @@ -120,6 +122,7 @@ func New() *Config { MultiClusterOptions: multicluster.NewOptions(), EventsOptions: eventsclient.NewElasticSearchOptions(), AuditingOptions: auditingclient.NewElasticSearchOptions(), + MetricsOptions: metrics.NewMetricsOptions(), } } diff --git a/pkg/models/openpitrix/apps.go b/pkg/models/openpitrix/apps.go index 1f5c040cf..342723459 100644 --- a/pkg/models/openpitrix/apps.go +++ b/pkg/models/openpitrix/apps.go @@ -154,7 +154,10 @@ func (c *appTemplateOperator) CreateApp(request *CreateAppRequest) (*CreateAppRe resp, err := c.opClient.CreateApp(openpitrix.ContextWithUsername(request.Username), createAppRequest) if err != nil { klog.Error(err) + appTemplateCreationCounter.WithLabelValues(request.Isv, request.Name, "failed").Inc() return nil, err + } else { + appTemplateCreationCounter.WithLabelValues(request.Isv, request.Name, "success").Inc() } return &CreateAppResponse{ AppID: resp.GetAppId().GetValue(), diff --git a/pkg/models/openpitrix/metric.go b/pkg/models/openpitrix/metric.go new file mode 100644 index 000000000..307099322 --- /dev/null +++ b/pkg/models/openpitrix/metric.go @@ -0,0 +1,25 @@ +package openpitrix + +import ( + compbasemetrics "k8s.io/component-base/metrics" + "kubesphere.io/kubesphere/pkg/utils/metrics" +) + +var ( + appTemplateCreationCounter = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "application_template_creation", + Help: "Counter of application template creation broken out for each workspace, name and create state", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"workspace", "name", "state"}, + ) +) + +func init() { + Register() +} + +func Register() { + metrics.MustRegister(appTemplateCreationCounter) +} diff --git a/pkg/simple/client/metrics/metrics.go b/pkg/simple/client/metrics/metrics.go new file mode 100644 index 000000000..34989e635 --- /dev/null +++ b/pkg/simple/client/metrics/metrics.go @@ -0,0 +1,24 @@ +package metrics + +import "github.com/spf13/pflag" + +type Options struct { + Enable bool `json:"enable,omitempty" description:"enable metric"` +} + +func NewMetricsOptions() *Options { + return &Options{ + Enable: false, + } +} + +func (s *Options) ApplyTo(options *Options) { + if options == nil { + options = s + return + } +} + +func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) { + fs.BoolVar(&s.Enable, "enable-metric", c.Enable, "If true, allow metric. [default=false]") +} diff --git a/pkg/utils/metrics/endpoint.go b/pkg/utils/metrics/endpoint.go new file mode 100644 index 000000000..73c4ded83 --- /dev/null +++ b/pkg/utils/metrics/endpoint.go @@ -0,0 +1,35 @@ +package metrics + +import ( + compbasemetrics "k8s.io/component-base/metrics" +) + +var ( + RequestCounter = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "ks_server_request_total", + Help: "Counter of ks_server requests broken out for each verb, group, version, resource and HTTP response code.", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"verb", "group", "version", "resource", "code"}, + ) + + RequestLatencies = compbasemetrics.NewHistogramVec( + &compbasemetrics.HistogramOpts{ + Name: "ks_server_request_duration_seconds", + Help: "Response latency distribution in seconds for each verb, group, version, resource", + // This metric is used for verifying api call latencies SLO, + // as well as tracking regressions in this aspects. + // Thus we customize buckets significantly, to empower both usecases. + Buckets: []float64{0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, + 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60}, + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"verb", "group", "version", "resource"}, + ) + + metrics = []compbasemetrics.Registerable{ + RequestCounter, + RequestLatencies, + } +) diff --git a/pkg/utils/metrics/metrics.go b/pkg/utils/metrics/metrics.go new file mode 100644 index 000000000..19cecb3dd --- /dev/null +++ b/pkg/utils/metrics/metrics.go @@ -0,0 +1,65 @@ +package metrics + +import ( + "github.com/emicklei/go-restful" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + apimachineryversion "k8s.io/apimachinery/pkg/version" + compbasemetrics "k8s.io/component-base/metrics" + ksVersion "kubesphere.io/kubesphere/pkg/version" + "net/http" + "sync" +) + +var ( + Defaults DefaultMetrics + //registerMetrics sync.Once + defaultRegistry = compbasemetrics.NewKubeRegistry() + // MustRegister registers registerable metrics but uses the defaultRegistry, panic upon the first registration that causes an error + MustRegister = defaultRegistry.MustRegister + // Register registers a collectable metric but uses the defaultRegistry + Register = defaultRegistry.Register +) + +// DefaultMetrics installs the default prometheus metrics handler +type DefaultMetrics struct{} + +// Install adds the DefaultMetrics handler +func (m DefaultMetrics) Install(c *restful.Container) { + register() + c.Handle("/kapis/metrics", Handler()) +} + +func versionGet() apimachineryversion.Info { + info := ksVersion.Get() + return apimachineryversion.Info{ + Major: info.GitMajor, + Minor: info.GitMinor, + GitVersion: info.GitVersion, + GitCommit: info.GitCommit, + GitTreeState: info.GitTreeState, + BuildDate: info.BuildDate, + GoVersion: info.GoVersion, + Compiler: info.Compiler, + Platform: info.Platform, + } +} + +// Handler returns an HTTP handler for the DefaultGatherer. It is +// already instrumented with InstrumentHandler (using "prometheus" as handler +// name). +func Handler() http.Handler { + return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, promhttp.HandlerFor(defaultRegistry, promhttp.HandlerOpts{})) +} + +var registerMetrics sync.Once + +func register() { + registerMetrics.Do(func() { + defaultRegistry.RawMustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) + defaultRegistry.RawMustRegister(prometheus.NewGoCollector()) + for _, metric := range metrics { + MustRegister(metric) + } + }) +} From 056bebdec6ebc48a83eb31a3cba2a3d2838ffbc6 Mon Sep 17 00:00:00 2001 From: LiHui Date: Tue, 22 Dec 2020 17:50:22 +0800 Subject: [PATCH 2/6] fix test Signed-off-by: LiHui --- pkg/apiserver/config/config_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/apiserver/config/config_test.go b/pkg/apiserver/config/config_test.go index 4d6c7e1d5..d581cf387 100644 --- a/pkg/apiserver/config/config_test.go +++ b/pkg/apiserver/config/config_test.go @@ -33,6 +33,7 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/k8s" "kubesphere.io/kubesphere/pkg/simple/client/ldap" "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch" + "kubesphere.io/kubesphere/pkg/simple/client/metrics" "kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus" "kubesphere.io/kubesphere/pkg/simple/client/multicluster" "kubesphere.io/kubesphere/pkg/simple/client/network" @@ -156,6 +157,9 @@ func newTestConfig() (*Config, error) { IndexPrefix: "ks-logstash-auditing", Version: "6", }, + MetricsOptions: &metrics.Options{ + Enable: false, + }, } return conf, nil } From 611f1c699135227a7c9beca50ad16e7f5e4b839c Mon Sep 17 00:00:00 2001 From: LiHui Date: Wed, 23 Dec 2020 09:44:30 +0800 Subject: [PATCH 3/6] add metrics to controller-manager Signed-off-by: LiHui --- cmd/controller-manager/app/options/options.go | 4 +++ cmd/controller-manager/app/server.go | 7 +++++ pkg/apiserver/apiserver.go | 5 ++-- .../endpoint.go => apiserver/metric.go} | 11 +++++-- pkg/controller/workspace/metrics.go | 24 +++++++++++++++ .../workspace/workspace_controller.go | 2 ++ pkg/simple/client/metrics/metrics.go | 2 +- pkg/utils/metrics/metrics.go | 29 +++++++------------ 8 files changed, 60 insertions(+), 24 deletions(-) rename pkg/{utils/metrics/endpoint.go => apiserver/metric.go} (86%) create mode 100644 pkg/controller/workspace/metrics.go diff --git a/cmd/controller-manager/app/options/options.go b/cmd/controller-manager/app/options/options.go index 2d98ef8b4..c6e57d422 100644 --- a/cmd/controller-manager/app/options/options.go +++ b/cmd/controller-manager/app/options/options.go @@ -29,6 +29,7 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins" "kubesphere.io/kubesphere/pkg/simple/client/k8s" ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap" + "kubesphere.io/kubesphere/pkg/simple/client/metrics" "kubesphere.io/kubesphere/pkg/simple/client/multicluster" "kubesphere.io/kubesphere/pkg/simple/client/network" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" @@ -45,6 +46,7 @@ type KubeSphereControllerManagerOptions struct { OpenPitrixOptions *openpitrix.Options NetworkOptions *network.Options MultiClusterOptions *multicluster.Options + MetricsOptions *metrics.Options ServiceMeshOptions *servicemesh.Options LeaderElect bool LeaderElection *leaderelection.LeaderElectionConfig @@ -60,6 +62,7 @@ func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions OpenPitrixOptions: openpitrix.NewOptions(), NetworkOptions: network.NewNetworkOptions(), MultiClusterOptions: multicluster.NewOptions(), + MetricsOptions: metrics.NewMetricsOptions(), ServiceMeshOptions: servicemesh.NewServiceMeshOptions(), AuthenticationOptions: authoptions.NewAuthenticateOptions(), LeaderElection: &leaderelection.LeaderElectionConfig{ @@ -86,6 +89,7 @@ func (s *KubeSphereControllerManagerOptions) Flags() cliflag.NamedFlagSets { s.NetworkOptions.AddFlags(fss.FlagSet("network"), s.NetworkOptions) s.MultiClusterOptions.AddFlags(fss.FlagSet("multicluster"), s.MultiClusterOptions) s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"), s.ServiceMeshOptions) + s.MetricsOptions.AddFlags(fss.FlagSet("metrics"), s.MetricsOptions) fs := fss.FlagSet("leaderelection") s.bindLeaderElectionFlags(s.LeaderElection, fs) diff --git a/cmd/controller-manager/app/server.go b/cmd/controller-manager/app/server.go index 6c9c7fdfb..a0a950f1b 100644 --- a/cmd/controller-manager/app/server.go +++ b/cmd/controller-manager/app/server.go @@ -18,6 +18,7 @@ package app import ( "fmt" + "kubesphere.io/kubesphere/pkg/utils/metrics" "github.com/spf13/cobra" utilerrors "k8s.io/apimachinery/pkg/util/errors" cliflag "k8s.io/component-base/cli/flag" @@ -64,6 +65,7 @@ func NewControllerManagerCommand() *cobra.Command { OpenPitrixOptions: conf.OpenPitrixOptions, NetworkOptions: conf.NetworkOptions, MultiClusterOptions: conf.MultiClusterOptions, + MetricsOptions: conf.MetricsOptions, ServiceMeshOptions: conf.ServiceMeshOptions, LeaderElection: s.LeaderElection, LeaderElect: s.LeaderElect, @@ -261,6 +263,11 @@ func run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{}) hookServer.Register("/validate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.ValidatingHandler{C: mgr.GetClient()}}) hookServer.Register("/mutate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.MutatingHandler{C: mgr.GetClient()}}) + if s.MetricsOptions != nil && s.MetricsOptions.Enable { + klog.V(2).Info("registering metrics to then webhook server") + hookServer.Register("/metrics", metrics.Handler()) + } + klog.V(0).Info("Starting the controllers.") if err = mgr.Start(stopCh); err != nil { klog.Fatalf("unable to run the manager: %v", err) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 11f0131dd..a0f02d67a 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -179,14 +179,15 @@ func monitorRequest(r *restful.Request, response *restful.Response, chain *restf chain.ProcessFilter(r, response) reqInfo, exists := request.RequestInfoFrom(r.Request.Context()) if exists && reqInfo.APIGroup != "" { - metrics.RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc() + RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc() elapsedSeconds := time.Now().Sub(start).Seconds() - metrics.RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds) + RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds) } } func (s *APIServer) installAPI() { if s.Config.MetricsOptions != nil && s.Config.MetricsOptions.Enable { + register() metrics.Defaults.Install(s.container) } } diff --git a/pkg/utils/metrics/endpoint.go b/pkg/apiserver/metric.go similarity index 86% rename from pkg/utils/metrics/endpoint.go rename to pkg/apiserver/metric.go index 73c4ded83..1365202fb 100644 --- a/pkg/utils/metrics/endpoint.go +++ b/pkg/apiserver/metric.go @@ -1,7 +1,8 @@ -package metrics +package apiserver import ( compbasemetrics "k8s.io/component-base/metrics" + "kubesphere.io/kubesphere/pkg/utils/metrics" ) var ( @@ -28,8 +29,14 @@ var ( []string{"verb", "group", "version", "resource"}, ) - metrics = []compbasemetrics.Registerable{ + metricsList = []compbasemetrics.Registerable{ RequestCounter, RequestLatencies, } ) + +func register() { + for _, m := range metricsList { + metrics.MustRegister(m) + } +} diff --git a/pkg/controller/workspace/metrics.go b/pkg/controller/workspace/metrics.go new file mode 100644 index 000000000..b3e5c0a63 --- /dev/null +++ b/pkg/controller/workspace/metrics.go @@ -0,0 +1,24 @@ +package workspace + +import ( + compbasemetrics "k8s.io/component-base/metrics" + "kubesphere.io/kubesphere/pkg/utils/metrics" +) + +var ( + workspaceOperation = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "ks_controller_manager_workspace_operation", + Help: "Counter of ks controller manager workspace operation broken out for each operation, name", + // This metric is used for verifying api call latencies SLO, + // as well as tracking regressions in this aspects. + // Thus we customize buckets significantly, to empower both usecases. + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"operation", "name"}, + ) +) + +func init() { + metrics.MustRegister(workspaceOperation) +} diff --git a/pkg/controller/workspace/workspace_controller.go b/pkg/controller/workspace/workspace_controller.go index f412c13a3..357d480ac 100644 --- a/pkg/controller/workspace/workspace_controller.go +++ b/pkg/controller/workspace/workspace_controller.go @@ -98,6 +98,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { if err := r.Update(rootCtx, workspace); err != nil { return ctrl.Result{}, err } + workspaceOperation.WithLabelValues("create", instance.Name).Inc() } } else { // The object is being deleted @@ -111,6 +112,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { logger.Error(err, "update workspace failed") return ctrl.Result{}, err } + workspaceOperation.WithLabelValues("delete", instance.Name).Inc() } // Our finalizer has finished, so the reconciler can do nothing. return ctrl.Result{}, nil diff --git a/pkg/simple/client/metrics/metrics.go b/pkg/simple/client/metrics/metrics.go index 34989e635..96cf93a6e 100644 --- a/pkg/simple/client/metrics/metrics.go +++ b/pkg/simple/client/metrics/metrics.go @@ -20,5 +20,5 @@ func (s *Options) ApplyTo(options *Options) { } func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) { - fs.BoolVar(&s.Enable, "enable-metric", c.Enable, "If true, allow metric. [default=false]") + fs.BoolVar(&s.Enable, "enable-metrics", c.Enable, "If true, allow metrics. [default=false]") } diff --git a/pkg/utils/metrics/metrics.go b/pkg/utils/metrics/metrics.go index 19cecb3dd..447549886 100644 --- a/pkg/utils/metrics/metrics.go +++ b/pkg/utils/metrics/metrics.go @@ -8,25 +8,28 @@ import ( compbasemetrics "k8s.io/component-base/metrics" ksVersion "kubesphere.io/kubesphere/pkg/version" "net/http" - "sync" ) var ( - Defaults DefaultMetrics - //registerMetrics sync.Once - defaultRegistry = compbasemetrics.NewKubeRegistry() + Defaults DefaultMetrics + defaultRegistry compbasemetrics.KubeRegistry // MustRegister registers registerable metrics but uses the defaultRegistry, panic upon the first registration that causes an error - MustRegister = defaultRegistry.MustRegister + MustRegister func(...compbasemetrics.Registerable) // Register registers a collectable metric but uses the defaultRegistry - Register = defaultRegistry.Register + Register func(compbasemetrics.Registerable) error ) +func init() { + defaultRegistry = compbasemetrics.NewKubeRegistry() + MustRegister = defaultRegistry.MustRegister + Register = defaultRegistry.Register +} + // DefaultMetrics installs the default prometheus metrics handler type DefaultMetrics struct{} // Install adds the DefaultMetrics handler func (m DefaultMetrics) Install(c *restful.Container) { - register() c.Handle("/kapis/metrics", Handler()) } @@ -51,15 +54,3 @@ func versionGet() apimachineryversion.Info { func Handler() http.Handler { return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, promhttp.HandlerFor(defaultRegistry, promhttp.HandlerOpts{})) } - -var registerMetrics sync.Once - -func register() { - registerMetrics.Do(func() { - defaultRegistry.RawMustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) - defaultRegistry.RawMustRegister(prometheus.NewGoCollector()) - for _, metric := range metrics { - MustRegister(metric) - } - }) -} From 1ecec78379dc581fea0ffcfa81190e58cef691ea Mon Sep 17 00:00:00 2001 From: LiHui Date: Mon, 4 Jan 2021 12:11:38 +0800 Subject: [PATCH 4/6] enable metrics by default Signed-off-by: LiHui --- cmd/controller-manager/app/options/options.go | 4 ---- cmd/controller-manager/app/server.go | 9 +++---- cmd/ks-apiserver/app/options/options.go | 1 - pkg/apiserver/apiserver.go | 14 ++++------- pkg/apiserver/config/config.go | 3 --- pkg/apiserver/config/config_test.go | 4 ---- .../workspace/workspace_controller.go | 4 ++-- pkg/simple/client/metrics/metrics.go | 24 ------------------- 8 files changed, 10 insertions(+), 53 deletions(-) delete mode 100644 pkg/simple/client/metrics/metrics.go diff --git a/cmd/controller-manager/app/options/options.go b/cmd/controller-manager/app/options/options.go index c6e57d422..2d98ef8b4 100644 --- a/cmd/controller-manager/app/options/options.go +++ b/cmd/controller-manager/app/options/options.go @@ -29,7 +29,6 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins" "kubesphere.io/kubesphere/pkg/simple/client/k8s" ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap" - "kubesphere.io/kubesphere/pkg/simple/client/metrics" "kubesphere.io/kubesphere/pkg/simple/client/multicluster" "kubesphere.io/kubesphere/pkg/simple/client/network" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" @@ -46,7 +45,6 @@ type KubeSphereControllerManagerOptions struct { OpenPitrixOptions *openpitrix.Options NetworkOptions *network.Options MultiClusterOptions *multicluster.Options - MetricsOptions *metrics.Options ServiceMeshOptions *servicemesh.Options LeaderElect bool LeaderElection *leaderelection.LeaderElectionConfig @@ -62,7 +60,6 @@ func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions OpenPitrixOptions: openpitrix.NewOptions(), NetworkOptions: network.NewNetworkOptions(), MultiClusterOptions: multicluster.NewOptions(), - MetricsOptions: metrics.NewMetricsOptions(), ServiceMeshOptions: servicemesh.NewServiceMeshOptions(), AuthenticationOptions: authoptions.NewAuthenticateOptions(), LeaderElection: &leaderelection.LeaderElectionConfig{ @@ -89,7 +86,6 @@ func (s *KubeSphereControllerManagerOptions) Flags() cliflag.NamedFlagSets { s.NetworkOptions.AddFlags(fss.FlagSet("network"), s.NetworkOptions) s.MultiClusterOptions.AddFlags(fss.FlagSet("multicluster"), s.MultiClusterOptions) s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"), s.ServiceMeshOptions) - s.MetricsOptions.AddFlags(fss.FlagSet("metrics"), s.MetricsOptions) fs := fss.FlagSet("leaderelection") s.bindLeaderElectionFlags(s.LeaderElection, fs) diff --git a/cmd/controller-manager/app/server.go b/cmd/controller-manager/app/server.go index a0a950f1b..e40e7a02d 100644 --- a/cmd/controller-manager/app/server.go +++ b/cmd/controller-manager/app/server.go @@ -18,7 +18,6 @@ package app import ( "fmt" - "kubesphere.io/kubesphere/pkg/utils/metrics" "github.com/spf13/cobra" utilerrors "k8s.io/apimachinery/pkg/util/errors" cliflag "k8s.io/component-base/cli/flag" @@ -42,6 +41,7 @@ import ( ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" "kubesphere.io/kubesphere/pkg/simple/client/s3" + "kubesphere.io/kubesphere/pkg/utils/metrics" "kubesphere.io/kubesphere/pkg/utils/term" "os" application "sigs.k8s.io/application/controllers" @@ -65,7 +65,6 @@ func NewControllerManagerCommand() *cobra.Command { OpenPitrixOptions: conf.OpenPitrixOptions, NetworkOptions: conf.NetworkOptions, MultiClusterOptions: conf.MultiClusterOptions, - MetricsOptions: conf.MetricsOptions, ServiceMeshOptions: conf.ServiceMeshOptions, LeaderElection: s.LeaderElection, LeaderElect: s.LeaderElect, @@ -263,10 +262,8 @@ func run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{}) hookServer.Register("/validate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.ValidatingHandler{C: mgr.GetClient()}}) hookServer.Register("/mutate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.MutatingHandler{C: mgr.GetClient()}}) - if s.MetricsOptions != nil && s.MetricsOptions.Enable { - klog.V(2).Info("registering metrics to then webhook server") - hookServer.Register("/metrics", metrics.Handler()) - } + klog.V(2).Info("registering metrics to the webhook server") + hookServer.Register("/metrics", metrics.Handler()) klog.V(0).Info("Starting the controllers.") if err = mgr.Start(stopCh); err != nil { diff --git a/cmd/ks-apiserver/app/options/options.go b/cmd/ks-apiserver/app/options/options.go index 873b4224e..2745b6c25 100644 --- a/cmd/ks-apiserver/app/options/options.go +++ b/cmd/ks-apiserver/app/options/options.go @@ -82,7 +82,6 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) { s.MultiClusterOptions.AddFlags(fss.FlagSet("multicluster"), s.MultiClusterOptions) s.EventsOptions.AddFlags(fss.FlagSet("events"), s.EventsOptions) s.AuditingOptions.AddFlags(fss.FlagSet("auditing"), s.AuditingOptions) - s.MetricsOptions.AddFlags(fss.FlagSet("metric"), s.MetricsOptions) fs = fss.FlagSet("klog") local := flag.NewFlagSet("klog", flag.ExitOnError) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index a0f02d67a..010b2687b 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -158,10 +158,8 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error { s.installKubeSphereAPIs() - s.installAPI() - if s.Config.MetricsOptions != nil && s.Config.MetricsOptions.Enable { - s.container.Filter(monitorRequest) - } + s.installMetricsAPI() + s.container.Filter(monitorRequest) for _, ws := range s.container.RegisteredWebServices() { klog.V(2).Infof("%s", ws.RootPath()) @@ -185,11 +183,9 @@ func monitorRequest(r *restful.Request, response *restful.Response, chain *restf } } -func (s *APIServer) installAPI() { - if s.Config.MetricsOptions != nil && s.Config.MetricsOptions.Enable { - register() - metrics.Defaults.Install(s.container) - } +func (s *APIServer) installMetricsAPI() { + register() + metrics.Defaults.Install(s.container) } // Install all kubesphere api groups diff --git a/pkg/apiserver/config/config.go b/pkg/apiserver/config/config.go index 8bdbfd4fe..7255a3c8b 100644 --- a/pkg/apiserver/config/config.go +++ b/pkg/apiserver/config/config.go @@ -30,7 +30,6 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/k8s" "kubesphere.io/kubesphere/pkg/simple/client/ldap" "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch" - "kubesphere.io/kubesphere/pkg/simple/client/metrics" "kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus" "kubesphere.io/kubesphere/pkg/simple/client/multicluster" "kubesphere.io/kubesphere/pkg/simple/client/network" @@ -98,7 +97,6 @@ type Config struct { AuditingOptions *auditingclient.Options `json:"auditing,omitempty" yaml:"auditing,omitempty" mapstructure:"auditing"` AlertingOptions *alerting.Options `json:"alerting,omitempty" yaml:"alerting,omitempty" mapstructure:"alerting"` NotificationOptions *notification.Options `json:"notification,omitempty" yaml:"notification,omitempty" mapstructure:"notification"` - MetricsOptions *metrics.Options `json:"metrics,omitempty" yaml:"metrics,omitempty" mapstructure:"metrics"` } // newConfig creates a default non-empty Config @@ -122,7 +120,6 @@ func New() *Config { MultiClusterOptions: multicluster.NewOptions(), EventsOptions: eventsclient.NewElasticSearchOptions(), AuditingOptions: auditingclient.NewElasticSearchOptions(), - MetricsOptions: metrics.NewMetricsOptions(), } } diff --git a/pkg/apiserver/config/config_test.go b/pkg/apiserver/config/config_test.go index d581cf387..4d6c7e1d5 100644 --- a/pkg/apiserver/config/config_test.go +++ b/pkg/apiserver/config/config_test.go @@ -33,7 +33,6 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/k8s" "kubesphere.io/kubesphere/pkg/simple/client/ldap" "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch" - "kubesphere.io/kubesphere/pkg/simple/client/metrics" "kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus" "kubesphere.io/kubesphere/pkg/simple/client/multicluster" "kubesphere.io/kubesphere/pkg/simple/client/network" @@ -157,9 +156,6 @@ func newTestConfig() (*Config, error) { IndexPrefix: "ks-logstash-auditing", Version: "6", }, - MetricsOptions: &metrics.Options{ - Enable: false, - }, } return conf, nil } diff --git a/pkg/controller/workspace/workspace_controller.go b/pkg/controller/workspace/workspace_controller.go index 357d480ac..e800267f0 100644 --- a/pkg/controller/workspace/workspace_controller.go +++ b/pkg/controller/workspace/workspace_controller.go @@ -98,7 +98,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { if err := r.Update(rootCtx, workspace); err != nil { return ctrl.Result{}, err } - workspaceOperation.WithLabelValues("create", instance.Name).Inc() + workspaceOperation.WithLabelValues("create", workspace.Name).Inc() } } else { // The object is being deleted @@ -112,7 +112,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { logger.Error(err, "update workspace failed") return ctrl.Result{}, err } - workspaceOperation.WithLabelValues("delete", instance.Name).Inc() + workspaceOperation.WithLabelValues("delete", workspace.Name).Inc() } // Our finalizer has finished, so the reconciler can do nothing. return ctrl.Result{}, nil diff --git a/pkg/simple/client/metrics/metrics.go b/pkg/simple/client/metrics/metrics.go deleted file mode 100644 index 96cf93a6e..000000000 --- a/pkg/simple/client/metrics/metrics.go +++ /dev/null @@ -1,24 +0,0 @@ -package metrics - -import "github.com/spf13/pflag" - -type Options struct { - Enable bool `json:"enable,omitempty" description:"enable metric"` -} - -func NewMetricsOptions() *Options { - return &Options{ - Enable: false, - } -} - -func (s *Options) ApplyTo(options *Options) { - if options == nil { - options = s - return - } -} - -func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) { - fs.BoolVar(&s.Enable, "enable-metrics", c.Enable, "If true, allow metrics. [default=false]") -} From bc9fe7f656cbaacbf9bd62a54ec65b2ffe6e0453 Mon Sep 17 00:00:00 2001 From: LiHui Date: Mon, 4 Jan 2021 15:36:30 +0800 Subject: [PATCH 5/6] add process and goroutine metrics Signed-off-by: LiHui --- pkg/apiserver/apiserver.go | 2 ++ pkg/utils/metrics/metrics.go | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 010b2687b..d3a47c7a3 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -82,10 +82,12 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" "kubesphere.io/kubesphere/pkg/simple/client/s3" "kubesphere.io/kubesphere/pkg/simple/client/sonarqube" + "kubesphere.io/kubesphere/pkg/utils/metrics" utilnet "kubesphere.io/kubesphere/pkg/utils/net" "net/http" rt "runtime" runtimecache "sigs.k8s.io/controller-runtime/pkg/cache" + "strconv" "time" ) diff --git a/pkg/utils/metrics/metrics.go b/pkg/utils/metrics/metrics.go index 447549886..b18a1dd37 100644 --- a/pkg/utils/metrics/metrics.go +++ b/pkg/utils/metrics/metrics.go @@ -17,12 +17,18 @@ var ( MustRegister func(...compbasemetrics.Registerable) // Register registers a collectable metric but uses the defaultRegistry Register func(compbasemetrics.Registerable) error + + RawMustRegister func(...prometheus.Collector) ) func init() { defaultRegistry = compbasemetrics.NewKubeRegistry() MustRegister = defaultRegistry.MustRegister Register = defaultRegistry.Register + RawMustRegister = defaultRegistry.RawMustRegister + + RawMustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) + RawMustRegister(prometheus.NewGoCollector()) } // DefaultMetrics installs the default prometheus metrics handler @@ -33,6 +39,7 @@ func (m DefaultMetrics) Install(c *restful.Container) { c.Handle("/kapis/metrics", Handler()) } +//Overwrite version.Get func versionGet() apimachineryversion.Info { info := ksVersion.Get() return apimachineryversion.Info{ From e57e602b69a51b6bffaec105b1c2aca6e0ab2af4 Mon Sep 17 00:00:00 2001 From: LiHui Date: Fri, 8 Jan 2021 14:11:41 +0800 Subject: [PATCH 6/6] change method name Signed-off-by: LiHui --- pkg/apiserver/apiserver.go | 2 +- pkg/apiserver/metric.go | 2 +- pkg/models/openpitrix/metric.go | 4 ---- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index d3a47c7a3..440cacee5 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -186,7 +186,7 @@ func monitorRequest(r *restful.Request, response *restful.Response, chain *restf } func (s *APIServer) installMetricsAPI() { - register() + registerMetrics() metrics.Defaults.Install(s.container) } diff --git a/pkg/apiserver/metric.go b/pkg/apiserver/metric.go index 1365202fb..27906950b 100644 --- a/pkg/apiserver/metric.go +++ b/pkg/apiserver/metric.go @@ -35,7 +35,7 @@ var ( } ) -func register() { +func registerMetrics() { for _, m := range metricsList { metrics.MustRegister(m) } diff --git a/pkg/models/openpitrix/metric.go b/pkg/models/openpitrix/metric.go index 307099322..794668f09 100644 --- a/pkg/models/openpitrix/metric.go +++ b/pkg/models/openpitrix/metric.go @@ -17,9 +17,5 @@ var ( ) func init() { - Register() -} - -func Register() { metrics.MustRegister(appTemplateCreationCounter) }