admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3220 from xyz-li/master

Browse Source 
add metrics
This commit is contained in:
KubeSphere CI Bot
2021-01-15 10:16:37 +08:00
committed by GitHub
parent ff0861bd1c e57e602b69
commit f5e2b05240
8 changed files with 182 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/controller-manager/app/server.go
View File

@@ -41,6 +41,7 @@ import (
ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap"
"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
"kubesphere.io/kubesphere/pkg/simple/client/s3"
"kubesphere.io/kubesphere/pkg/utils/metrics"
"kubesphere.io/kubesphere/pkg/utils/term"
"os"
application "sigs.k8s.io/application/controllers"
@@ -261,6 +262,9 @@ func run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{})
hookServer.Register("/validate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.ValidatingHandler{C: mgr.GetClient()}})
hookServer.Register("/mutate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.MutatingHandler{C: mgr.GetClient()}})
klog.V(2).Info("registering metrics to the webhook server")
hookServer.Register("/metrics", metrics.Handler())
klog.V(0).Info("Starting the controllers.")
if err = mgr.Start(stopCh); err != nil {
klog.Fatalf("unable to run the manager: %v", err)

25
pkg/apiserver/apiserver.go
View File

@@ -82,10 +82,12 @@ import (
"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
"kubesphere.io/kubesphere/pkg/simple/client/s3"
"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
"kubesphere.io/kubesphere/pkg/utils/metrics"
utilnet "kubesphere.io/kubesphere/pkg/utils/net"
"net/http"
rt "runtime"
runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
"strconv"
"time"
)
@@ -149,7 +151,6 @@ type APIServer struct {
}
func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
s.container = restful.NewContainer()
s.container.Filter(logRequestAndResponse)
s.container.Router(restful.CurlyRouter{})
@@ -159,6 +160,9 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
s.installKubeSphereAPIs()
s.installMetricsAPI()
s.container.Filter(monitorRequest)
for _, ws := range s.container.RegisteredWebServices() {
klog.V(2).Infof("%s", ws.RootPath())
}
@@ -170,6 +174,22 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
return nil
}
func monitorRequest(r *restful.Request, response *restful.Response, chain *restful.FilterChain) {
start := time.Now()
chain.ProcessFilter(r, response)
reqInfo, exists := request.RequestInfoFrom(r.Request.Context())
if exists && reqInfo.APIGroup != "" {
RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc()
elapsedSeconds := time.Now().Sub(start).Seconds()
RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds)
}
}
func (s *APIServer) installMetricsAPI() {
registerMetrics()
metrics.Defaults.Install(s.container)
}
// Install all kubesphere api groups
// Installation happens before all informers start to cache objects, so
// any attempt to list objects using listers will get empty results.
@@ -297,7 +317,7 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) {
default:
fallthrough
case authorizationoptions.RBAC:
excludedPaths := []string{"/oauth/*", "/kapis/config.kubesphere.io/*", "/kapis/version"}
excludedPaths := []string{"/oauth/*", "/kapis/config.kubesphere.io/*", "/kapis/version", "/kapis/metrics"}
pathAuthorizer, _ := path.NewAuthorizer(excludedPaths)
amOperator := am.NewReadOnlyOperator(s.InformerFactory)
authorizers = unionauthorizer.New(pathAuthorizer, rbac.NewRBACAuthorizer(amOperator))
@@ -321,6 +341,7 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) {
s.InformerFactory.KubeSphereSharedInformerFactory().Iam().V1alpha2().Users().Lister())))
handler = filters.WithAuthentication(handler, authn)
handler = filters.WithRequestInfo(handler, requestInfoResolver)
s.Server.Handler = handler
}

42
pkg/apiserver/metric.go Normal file
View File

@@ -0,0 +1,42 @@
package apiserver
import (
compbasemetrics "k8s.io/component-base/metrics"
"kubesphere.io/kubesphere/pkg/utils/metrics"
)
var (
RequestCounter = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "ks_server_request_total",
Help: "Counter of ks_server requests broken out for each verb, group, version, resource and HTTP response code.",
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"verb", "group", "version", "resource", "code"},
)
RequestLatencies = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "ks_server_request_duration_seconds",
Help: "Response latency distribution in seconds for each verb, group, version, resource",
// This metric is used for verifying api call latencies SLO,
// as well as tracking regressions in this aspects.
// Thus we customize buckets significantly, to empower both usecases.
Buckets: []float64{0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60},
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"verb", "group", "version", "resource"},
)
metricsList = []compbasemetrics.Registerable{
RequestCounter,
RequestLatencies,
}
)
func registerMetrics() {
for _, m := range metricsList {
metrics.MustRegister(m)
}
}

24
pkg/controller/workspace/metrics.go Normal file
View File

@@ -0,0 +1,24 @@
package workspace
import (
compbasemetrics "k8s.io/component-base/metrics"
"kubesphere.io/kubesphere/pkg/utils/metrics"
)
var (
workspaceOperation = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "ks_controller_manager_workspace_operation",
Help: "Counter of ks controller manager workspace operation broken out for each operation, name",
// This metric is used for verifying api call latencies SLO,
// as well as tracking regressions in this aspects.
// Thus we customize buckets significantly, to empower both usecases.
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"operation", "name"},
)
)
func init() {
metrics.MustRegister(workspaceOperation)
}

2
pkg/controller/workspace/workspace_controller.go
View File

@@ -98,6 +98,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
if err := r.Update(rootCtx, workspace); err != nil {
return ctrl.Result{}, err
}
workspaceOperation.WithLabelValues("create", workspace.Name).Inc()
}
} else {
// The object is being deleted
@@ -111,6 +112,7 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
logger.Error(err, "update workspace failed")
return ctrl.Result{}, err
}
workspaceOperation.WithLabelValues("delete", workspace.Name).Inc()
}
// Our finalizer has finished, so the reconciler can do nothing.
return ctrl.Result{}, nil

3
pkg/models/openpitrix/apps.go
View File

@@ -154,7 +154,10 @@ func (c *appTemplateOperator) CreateApp(request *CreateAppRequest) (*CreateAppRe
resp, err := c.opClient.CreateApp(openpitrix.ContextWithUsername(request.Username), createAppRequest)
if err != nil {
klog.Error(err)
appTemplateCreationCounter.WithLabelValues(request.Isv, request.Name, "failed").Inc()
return nil, err
} else {
appTemplateCreationCounter.WithLabelValues(request.Isv, request.Name, "success").Inc()
}
return &CreateAppResponse{
AppID: resp.GetAppId().GetValue(),

21
pkg/models/openpitrix/metric.go Normal file
View File

@@ -0,0 +1,21 @@
package openpitrix
import (
compbasemetrics "k8s.io/component-base/metrics"
"kubesphere.io/kubesphere/pkg/utils/metrics"
)
var (
appTemplateCreationCounter = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "application_template_creation",
Help: "Counter of application template creation broken out for each workspace, name and create state",
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"workspace", "name", "state"},
)
)
func init() {
metrics.MustRegister(appTemplateCreationCounter)
}

63
pkg/utils/metrics/metrics.go Normal file
View File

@@ -0,0 +1,63 @@
package metrics
import (
"github.com/emicklei/go-restful"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
apimachineryversion "k8s.io/apimachinery/pkg/version"
compbasemetrics "k8s.io/component-base/metrics"
ksVersion "kubesphere.io/kubesphere/pkg/version"
"net/http"
)
var (
Defaults DefaultMetrics
defaultRegistry compbasemetrics.KubeRegistry
// MustRegister registers registerable metrics but uses the defaultRegistry, panic upon the first registration that causes an error
MustRegister func(...compbasemetrics.Registerable)
// Register registers a collectable metric but uses the defaultRegistry
Register func(compbasemetrics.Registerable) error
RawMustRegister func(...prometheus.Collector)
)
func init() {
defaultRegistry = compbasemetrics.NewKubeRegistry()
MustRegister = defaultRegistry.MustRegister
Register = defaultRegistry.Register
RawMustRegister = defaultRegistry.RawMustRegister
RawMustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
RawMustRegister(prometheus.NewGoCollector())
}
// DefaultMetrics installs the default prometheus metrics handler
type DefaultMetrics struct{}
// Install adds the DefaultMetrics handler
func (m DefaultMetrics) Install(c *restful.Container) {
c.Handle("/kapis/metrics", Handler())
}
//Overwrite version.Get
func versionGet() apimachineryversion.Info {
info := ksVersion.Get()
return apimachineryversion.Info{
Major: info.GitMajor,
Minor: info.GitMinor,
GitVersion: info.GitVersion,
GitCommit: info.GitCommit,
GitTreeState: info.GitTreeState,
BuildDate: info.BuildDate,
GoVersion: info.GoVersion,
Compiler: info.Compiler,
Platform: info.Platform,
}
}
// Handler returns an HTTP handler for the DefaultGatherer. It is
// already instrumented with InstrumentHandler (using "prometheus" as handler
// name).
func Handler() http.Handler {
return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, promhttp.HandlerFor(defaultRegistry, promhttp.HandlerOpts{}))
}