diff --git a/cmd/controller-manager/app/server.go b/cmd/controller-manager/app/server.go index 6c9c7fdfb..e40e7a02d 100644 --- a/cmd/controller-manager/app/server.go +++ b/cmd/controller-manager/app/server.go @@ -41,6 +41,7 @@ import ( ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" "kubesphere.io/kubesphere/pkg/simple/client/s3" + "kubesphere.io/kubesphere/pkg/utils/metrics" "kubesphere.io/kubesphere/pkg/utils/term" "os" application "sigs.k8s.io/application/controllers" @@ -261,6 +262,9 @@ func run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{}) hookServer.Register("/validate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.ValidatingHandler{C: mgr.GetClient()}}) hookServer.Register("/mutate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.MutatingHandler{C: mgr.GetClient()}}) + klog.V(2).Info("registering metrics to the webhook server") + hookServer.Register("/metrics", metrics.Handler()) + klog.V(0).Info("Starting the controllers.") if err = mgr.Start(stopCh); err != nil { klog.Fatalf("unable to run the manager: %v", err) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index ab928e54d..440cacee5 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -82,10 +82,12 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" "kubesphere.io/kubesphere/pkg/simple/client/s3" "kubesphere.io/kubesphere/pkg/simple/client/sonarqube" + "kubesphere.io/kubesphere/pkg/utils/metrics" utilnet "kubesphere.io/kubesphere/pkg/utils/net" "net/http" rt "runtime" runtimecache "sigs.k8s.io/controller-runtime/pkg/cache" + "strconv" "time" ) @@ -149,7 +151,6 @@ type APIServer struct { } func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error { - s.container = restful.NewContainer() s.container.Filter(logRequestAndResponse) s.container.Router(restful.CurlyRouter{}) @@ -159,6 +160,9 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error { s.installKubeSphereAPIs() + s.installMetricsAPI() + s.container.Filter(monitorRequest) + for _, ws := range s.container.RegisteredWebServices() { klog.V(2).Infof("%s", ws.RootPath()) } @@ -170,6 +174,22 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error { return nil } +func monitorRequest(r *restful.Request, response *restful.Response, chain *restful.FilterChain) { + start := time.Now() + chain.ProcessFilter(r, response) + reqInfo, exists := request.RequestInfoFrom(r.Request.Context()) + if exists && reqInfo.APIGroup != "" { + RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc() + elapsedSeconds := time.Now().Sub(start).Seconds() + RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds) + } +} + +func (s *APIServer) installMetricsAPI() { + registerMetrics() + metrics.Defaults.Install(s.container) +} + // Install all kubesphere api groups // Installation happens before all informers start to cache objects, so // any attempt to list objects using listers will get empty results. @@ -297,7 +317,7 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) { default: fallthrough case authorizationoptions.RBAC: - excludedPaths := []string{"/oauth/*", "/kapis/config.kubesphere.io/*", "/kapis/version"} + excludedPaths := []string{"/oauth/*", "/kapis/config.kubesphere.io/*", "/kapis/version", "/kapis/metrics"} pathAuthorizer, _ := path.NewAuthorizer(excludedPaths) amOperator := am.NewReadOnlyOperator(s.InformerFactory) authorizers = unionauthorizer.New(pathAuthorizer, rbac.NewRBACAuthorizer(amOperator)) @@ -321,6 +341,7 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) { s.InformerFactory.KubeSphereSharedInformerFactory().Iam().V1alpha2().Users().Lister()))) handler = filters.WithAuthentication(handler, authn) handler = filters.WithRequestInfo(handler, requestInfoResolver) + s.Server.Handler = handler } diff --git a/pkg/apiserver/metric.go b/pkg/apiserver/metric.go new file mode 100644 index 000000000..27906950b --- /dev/null +++ b/pkg/apiserver/metric.go @@ -0,0 +1,42 @@ +package apiserver + +import ( + compbasemetrics "k8s.io/component-base/metrics" + "kubesphere.io/kubesphere/pkg/utils/metrics" +) + +var ( + RequestCounter = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "ks_server_request_total", + Help: "Counter of ks_server requests broken out for each verb, group, version, resource and HTTP response code.", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"verb", "group", "version", "resource", "code"}, + ) + + RequestLatencies = compbasemetrics.NewHistogramVec( + &compbasemetrics.HistogramOpts{ + Name: "ks_server_request_duration_seconds", + Help: "Response latency distribution in seconds for each verb, group, version, resource", + // This metric is used for verifying api call latencies SLO, + // as well as tracking regressions in this aspects. + // Thus we customize buckets significantly, to empower both usecases. + Buckets: []float64{0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, + 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60}, + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"verb", "group", "version", "resource"}, + ) + + metricsList = []compbasemetrics.Registerable{ + RequestCounter, + RequestLatencies, + } +) + +func registerMetrics() { + for _, m := range metricsList { + metrics.MustRegister(m) + } +} diff --git a/pkg/controller/workspace/metrics.go b/pkg/controller/workspace/metrics.go new file mode 100644 index 000000000..b3e5c0a63 --- /dev/null +++ b/pkg/controller/workspace/metrics.go @@ -0,0 +1,24 @@ +package workspace + +import ( + compbasemetrics "k8s.io/component-base/metrics" + "kubesphere.io/kubesphere/pkg/utils/metrics" +) + +var ( + workspaceOperation = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "ks_controller_manager_workspace_operation", + Help: "Counter of ks controller manager workspace operation broken out for each operation, name", + // This metric is used for verifying api call latencies SLO, + // as well as tracking regressions in this aspects. + // Thus we customize buckets significantly, to empower both usecases. + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"operation", "name"}, + ) +) + +func init() { + metrics.MustRegister(workspaceOperation) +} diff --git a/pkg/controller/workspace/workspace_controller.go b/pkg/controller/workspace/workspace_controller.go index f412c13a3..e800267f0 100644 --- a/pkg/controller/workspace/workspace_controller.go +++ b/pkg/controller/workspace/workspace_controller.go @@ -98,6 +98,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { if err := r.Update(rootCtx, workspace); err != nil { return ctrl.Result{}, err } + workspaceOperation.WithLabelValues("create", workspace.Name).Inc() } } else { // The object is being deleted @@ -111,6 +112,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { logger.Error(err, "update workspace failed") return ctrl.Result{}, err } + workspaceOperation.WithLabelValues("delete", workspace.Name).Inc() } // Our finalizer has finished, so the reconciler can do nothing. return ctrl.Result{}, nil diff --git a/pkg/models/openpitrix/apps.go b/pkg/models/openpitrix/apps.go index 1f5c040cf..342723459 100644 --- a/pkg/models/openpitrix/apps.go +++ b/pkg/models/openpitrix/apps.go @@ -154,7 +154,10 @@ func (c *appTemplateOperator) CreateApp(request *CreateAppRequest) (*CreateAppRe resp, err := c.opClient.CreateApp(openpitrix.ContextWithUsername(request.Username), createAppRequest) if err != nil { klog.Error(err) + appTemplateCreationCounter.WithLabelValues(request.Isv, request.Name, "failed").Inc() return nil, err + } else { + appTemplateCreationCounter.WithLabelValues(request.Isv, request.Name, "success").Inc() } return &CreateAppResponse{ AppID: resp.GetAppId().GetValue(), diff --git a/pkg/models/openpitrix/metric.go b/pkg/models/openpitrix/metric.go new file mode 100644 index 000000000..794668f09 --- /dev/null +++ b/pkg/models/openpitrix/metric.go @@ -0,0 +1,21 @@ +package openpitrix + +import ( + compbasemetrics "k8s.io/component-base/metrics" + "kubesphere.io/kubesphere/pkg/utils/metrics" +) + +var ( + appTemplateCreationCounter = compbasemetrics.NewCounterVec( + &compbasemetrics.CounterOpts{ + Name: "application_template_creation", + Help: "Counter of application template creation broken out for each workspace, name and create state", + StabilityLevel: compbasemetrics.ALPHA, + }, + []string{"workspace", "name", "state"}, + ) +) + +func init() { + metrics.MustRegister(appTemplateCreationCounter) +} diff --git a/pkg/utils/metrics/metrics.go b/pkg/utils/metrics/metrics.go new file mode 100644 index 000000000..b18a1dd37 --- /dev/null +++ b/pkg/utils/metrics/metrics.go @@ -0,0 +1,63 @@ +package metrics + +import ( + "github.com/emicklei/go-restful" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + apimachineryversion "k8s.io/apimachinery/pkg/version" + compbasemetrics "k8s.io/component-base/metrics" + ksVersion "kubesphere.io/kubesphere/pkg/version" + "net/http" +) + +var ( + Defaults DefaultMetrics + defaultRegistry compbasemetrics.KubeRegistry + // MustRegister registers registerable metrics but uses the defaultRegistry, panic upon the first registration that causes an error + MustRegister func(...compbasemetrics.Registerable) + // Register registers a collectable metric but uses the defaultRegistry + Register func(compbasemetrics.Registerable) error + + RawMustRegister func(...prometheus.Collector) +) + +func init() { + defaultRegistry = compbasemetrics.NewKubeRegistry() + MustRegister = defaultRegistry.MustRegister + Register = defaultRegistry.Register + RawMustRegister = defaultRegistry.RawMustRegister + + RawMustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) + RawMustRegister(prometheus.NewGoCollector()) +} + +// DefaultMetrics installs the default prometheus metrics handler +type DefaultMetrics struct{} + +// Install adds the DefaultMetrics handler +func (m DefaultMetrics) Install(c *restful.Container) { + c.Handle("/kapis/metrics", Handler()) +} + +//Overwrite version.Get +func versionGet() apimachineryversion.Info { + info := ksVersion.Get() + return apimachineryversion.Info{ + Major: info.GitMajor, + Minor: info.GitMinor, + GitVersion: info.GitVersion, + GitCommit: info.GitCommit, + GitTreeState: info.GitTreeState, + BuildDate: info.BuildDate, + GoVersion: info.GoVersion, + Compiler: info.Compiler, + Platform: info.Platform, + } +} + +// Handler returns an HTTP handler for the DefaultGatherer. It is +// already instrumented with InstrumentHandler (using "prometheus" as handler +// name). +func Handler() http.Handler { + return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, promhttp.HandlerFor(defaultRegistry, promhttp.HandlerOpts{})) +}