add metrics

Signed-off-by: LiHui <andrewli@yunify.com>

cmd/ks-apiserver/app/options/options.go

@@ -82,6 +82,7 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
 	s.MultiClusterOptions.AddFlags(fss.FlagSet("multicluster"), s.MultiClusterOptions)
 	s.EventsOptions.AddFlags(fss.FlagSet("events"), s.EventsOptions)
 	s.AuditingOptions.AddFlags(fss.FlagSet("auditing"), s.AuditingOptions)
+	s.MetricsOptions.AddFlags(fss.FlagSet("metric"), s.MetricsOptions)
 
 	fs = fss.FlagSet("klog")
 	local := flag.NewFlagSet("klog", flag.ExitOnError)

pkg/apiserver/apiserver.go

@@ -149,7 +149,6 @@ type APIServer struct {
 }
 
 func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
-
 	s.container = restful.NewContainer()
 	s.container.Filter(logRequestAndResponse)
 	s.container.Router(restful.CurlyRouter{})
@@ -159,6 +158,11 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
 
 	s.installKubeSphereAPIs()
 
+	s.installAPI()
+	if s.Config.MetricsOptions != nil && s.Config.MetricsOptions.Enable {
+		s.container.Filter(monitorRequest)
+	}
+
 	for _, ws := range s.container.RegisteredWebServices() {
 		klog.V(2).Infof("%s", ws.RootPath())
 	}
@@ -170,6 +174,23 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
 	return nil
 }
 
+func monitorRequest(r *restful.Request, response *restful.Response, chain *restful.FilterChain) {
+	start := time.Now()
+	chain.ProcessFilter(r, response)
+	reqInfo, exists := request.RequestInfoFrom(r.Request.Context())
+	if exists && reqInfo.APIGroup != "" {
+		metrics.RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc()
+		elapsedSeconds := time.Now().Sub(start).Seconds()
+		metrics.RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds)
+	}
+}
+
+func (s *APIServer) installAPI() {
+	if s.Config.MetricsOptions != nil && s.Config.MetricsOptions.Enable {
+		metrics.Defaults.Install(s.container)
+	}
+}
+
 // Install all kubesphere api groups
 // Installation happens before all informers start to cache objects, so
 //   any attempt to list objects using listers will get empty results.
@@ -297,7 +318,7 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) {
 	default:
 		fallthrough
 	case authorizationoptions.RBAC:
-		excludedPaths := []string{"/oauth/*", "/kapis/config.kubesphere.io/*", "/kapis/version"}
+		excludedPaths := []string{"/oauth/*", "/kapis/config.kubesphere.io/*", "/kapis/version", "/kapis/metrics"}
 		pathAuthorizer, _ := path.NewAuthorizer(excludedPaths)
 		amOperator := am.NewReadOnlyOperator(s.InformerFactory)
 		authorizers = unionauthorizer.New(pathAuthorizer, rbac.NewRBACAuthorizer(amOperator))
@@ -321,6 +342,7 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) {
 			s.InformerFactory.KubeSphereSharedInformerFactory().Iam().V1alpha2().Users().Lister())))
 	handler = filters.WithAuthentication(handler, authn)
 	handler = filters.WithRequestInfo(handler, requestInfoResolver)
+
 	s.Server.Handler = handler
 }
 

pkg/apiserver/config/config.go

@@ -30,6 +30,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
 	"kubesphere.io/kubesphere/pkg/simple/client/ldap"
 	"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch"
+	"kubesphere.io/kubesphere/pkg/simple/client/metrics"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
 	"kubesphere.io/kubesphere/pkg/simple/client/multicluster"
 	"kubesphere.io/kubesphere/pkg/simple/client/network"
@@ -97,6 +98,7 @@ type Config struct {
 	AuditingOptions       *auditingclient.Options                    `json:"auditing,omitempty" yaml:"auditing,omitempty" mapstructure:"auditing"`
 	AlertingOptions       *alerting.Options                          `json:"alerting,omitempty" yaml:"alerting,omitempty" mapstructure:"alerting"`
 	NotificationOptions   *notification.Options                      `json:"notification,omitempty" yaml:"notification,omitempty" mapstructure:"notification"`
+	MetricsOptions        *metrics.Options                           `json:"metrics,omitempty" yaml:"metrics,omitempty" mapstructure:"metrics"`
 }
 
 // newConfig creates a default non-empty Config
@@ -120,6 +122,7 @@ func New() *Config {
 		MultiClusterOptions:   multicluster.NewOptions(),
 		EventsOptions:         eventsclient.NewElasticSearchOptions(),
 		AuditingOptions:       auditingclient.NewElasticSearchOptions(),
+		MetricsOptions:        metrics.NewMetricsOptions(),
 	}
 }
 

pkg/models/openpitrix/apps.go

@@ -154,7 +154,10 @@ func (c *appTemplateOperator) CreateApp(request *CreateAppRequest) (*CreateAppRe
 	resp, err := c.opClient.CreateApp(openpitrix.ContextWithUsername(request.Username), createAppRequest)
 	if err != nil {
 		klog.Error(err)
+		appTemplateCreationCounter.WithLabelValues(request.Isv, request.Name, "failed").Inc()
 		return nil, err
+	} else {
+		appTemplateCreationCounter.WithLabelValues(request.Isv, request.Name, "success").Inc()
 	}
 	return &CreateAppResponse{
 		AppID:     resp.GetAppId().GetValue(),

pkg/models/openpitrix/metric.go

@@ -0,0 +1,25 @@
+package openpitrix
+
+import (
+	compbasemetrics "k8s.io/component-base/metrics"
+	"kubesphere.io/kubesphere/pkg/utils/metrics"
+)
+
+var (
+	appTemplateCreationCounter = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Name:           "application_template_creation",
+			Help:           "Counter of application template creation broken out for each workspace, name and create state",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"workspace", "name", "state"},
+	)
+)
+
+func init() {
+	Register()
+}
+
+func Register() {
+	metrics.MustRegister(appTemplateCreationCounter)
+}

pkg/simple/client/metrics/metrics.go

@@ -0,0 +1,24 @@
+package metrics
+
+import "github.com/spf13/pflag"
+
+type Options struct {
+	Enable bool `json:"enable,omitempty" description:"enable metric"`
+}
+
+func NewMetricsOptions() *Options {
+	return &Options{
+		Enable: false,
+	}
+}
+
+func (s *Options) ApplyTo(options *Options) {
+	if options == nil {
+		options = s
+		return
+	}
+}
+
+func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) {
+	fs.BoolVar(&s.Enable, "enable-metric", c.Enable, "If true, allow metric. [default=false]")
+}

pkg/utils/metrics/endpoint.go

@@ -0,0 +1,35 @@
+package metrics
+
+import (
+	compbasemetrics "k8s.io/component-base/metrics"
+)
+
+var (
+	RequestCounter = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Name:           "ks_server_request_total",
+			Help:           "Counter of ks_server requests broken out for each verb, group, version, resource and HTTP response code.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"verb", "group", "version", "resource", "code"},
+	)
+
+	RequestLatencies = compbasemetrics.NewHistogramVec(
+		&compbasemetrics.HistogramOpts{
+			Name: "ks_server_request_duration_seconds",
+			Help: "Response latency distribution in seconds for each verb, group, version, resource",
+			// This metric is used for verifying api call latencies SLO,
+			// as well as tracking regressions in this aspects.
+			// Thus we customize buckets significantly, to empower both usecases.
+			Buckets: []float64{0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
+				1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60},
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"verb", "group", "version", "resource"},
+	)
+
+	metrics = []compbasemetrics.Registerable{
+		RequestCounter,
+		RequestLatencies,
+	}
+)

pkg/utils/metrics/metrics.go

@@ -0,0 +1,65 @@
+package metrics
+
+import (
+	"github.com/emicklei/go-restful"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	apimachineryversion "k8s.io/apimachinery/pkg/version"
+	compbasemetrics "k8s.io/component-base/metrics"
+	ksVersion "kubesphere.io/kubesphere/pkg/version"
+	"net/http"
+	"sync"
+)
+
+var (
+	Defaults DefaultMetrics
+	//registerMetrics sync.Once
+	defaultRegistry = compbasemetrics.NewKubeRegistry()
+	// MustRegister registers registerable metrics but uses the defaultRegistry, panic upon the first registration that causes an error
+	MustRegister = defaultRegistry.MustRegister
+	// Register registers a collectable metric but uses the defaultRegistry
+	Register = defaultRegistry.Register
+)
+
+// DefaultMetrics installs the default prometheus metrics handler
+type DefaultMetrics struct{}
+
+// Install adds the DefaultMetrics handler
+func (m DefaultMetrics) Install(c *restful.Container) {
+	register()
+	c.Handle("/kapis/metrics", Handler())
+}
+
+func versionGet() apimachineryversion.Info {
+	info := ksVersion.Get()
+	return apimachineryversion.Info{
+		Major:        info.GitMajor,
+		Minor:        info.GitMinor,
+		GitVersion:   info.GitVersion,
+		GitCommit:    info.GitCommit,
+		GitTreeState: info.GitTreeState,
+		BuildDate:    info.BuildDate,
+		GoVersion:    info.GoVersion,
+		Compiler:     info.Compiler,
+		Platform:     info.Platform,
+	}
+}
+
+// Handler returns an HTTP handler for the DefaultGatherer. It is
+// already instrumented with InstrumentHandler (using "prometheus" as handler
+// name).
+func Handler() http.Handler {
+	return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, promhttp.HandlerFor(defaultRegistry, promhttp.HandlerOpts{}))
+}
+
+var registerMetrics sync.Once
+
+func register() {
+	registerMetrics.Do(func() {
+		defaultRegistry.RawMustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
+		defaultRegistry.RawMustRegister(prometheus.NewGoCollector())
+		for _, metric := range metrics {
+			MustRegister(metric)
+		}
+	})
+}