feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

vendor/k8s.io/component-base/metrics/buckets.go

@@ -33,6 +33,16 @@ func ExponentialBuckets(start, factor float64, count int) []float64 {
 	return prometheus.ExponentialBuckets(start, factor, count)
 }
 
+// ExponentialBucketsRange creates 'count' buckets, where the lowest bucket is
+// 'min' and the highest bucket is 'max'. The final +Inf bucket is not counted
+// and not included in the returned slice. The returned slice is meant to be
+// used for the Buckets field of HistogramOpts.
+//
+// The function panics if 'count' is 0 or negative, if 'min' is 0 or negative.
+func ExponentialBucketsRange(min, max float64, count int) []float64 {
+	return prometheus.ExponentialBucketsRange(min, max, count)
+}
+
 // MergeBuckets merges buckets together
 func MergeBuckets(buckets ...[]float64) []float64 {
 	result := make([]float64, 1)

vendor/k8s.io/component-base/metrics/features/kube_features.go

@@ -29,7 +29,7 @@ const (
 
 func featureGates() map[featuregate.Feature]featuregate.FeatureSpec {
 	return map[featuregate.Feature]featuregate.FeatureSpec{
-		ComponentSLIs: {Default: false, PreRelease: featuregate.Alpha},
+		ComponentSLIs: {Default: true, PreRelease: featuregate.Beta},
 	}
 }
 

vendor/k8s.io/component-base/metrics/http.go

@@ -19,19 +19,28 @@ package metrics
 import (
 	"io"
 	"net/http"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
+var (
+	processStartedAt time.Time
+)
+
+func init() {
+	processStartedAt = time.Now()
+}
+
 // These constants cause handlers serving metrics to behave as described if
 // errors are encountered.
 const (
-	// Serve an HTTP status code 500 upon the first error
+	// HTTPErrorOnError serve an HTTP status code 500 upon the first error
 	// encountered. Report the error message in the body.
 	HTTPErrorOnError promhttp.HandlerErrorHandling = iota
 
-	// Ignore errors and try to serve as many metrics as possible.  However,
-	// if no metrics can be served, serve an HTTP status code 500 and the
+	// ContinueOnError ignore errors and try to serve as many metrics as possible.
+	// However, if no metrics can be served, serve an HTTP status code 500 and the
 	// last error message in the body. Only use this in deliberate "best
 	// effort" metrics collection scenarios. In this case, it is highly
 	// recommended to provide other means of detecting errors: By setting an
@@ -41,7 +50,7 @@ const (
 	// alerts.
 	ContinueOnError
 
-	// Panic upon the first error encountered (useful for "crash only" apps).
+	// PanicOnError panics upon the first error encountered (useful for "crash only" apps).
 	PanicOnError
 )
 
@@ -50,6 +59,7 @@ const (
 type HandlerOpts promhttp.HandlerOpts
 
 func (ho *HandlerOpts) toPromhttpHandlerOpts() promhttp.HandlerOpts {
+	ho.ProcessStartTime = processStartedAt
 	return promhttp.HandlerOpts(*ho)
 }
 

vendor/k8s.io/component-base/metrics/legacyregistry/registry.go

@@ -18,6 +18,7 @@ package legacyregistry
 
 import (
 	"net/http"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
@@ -42,19 +43,25 @@ var (
 
 	// Register registers a collectable metric but uses the global registry
 	Register = defaultRegistry.Register
+
+	// Registerer exposes the global registerer
+	Registerer = defaultRegistry.Registerer
+
+	processStart time.Time
 )
 
 func init() {
 	RawMustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
 	RawMustRegister(collectors.NewGoCollector(collectors.WithGoCollectorRuntimeMetrics(collectors.MetricsAll)))
 	defaultRegistry.RegisterMetaMetrics()
+	processStart = time.Now()
 }
 
 // Handler returns an HTTP handler for the DefaultGatherer. It is
 // already instrumented with InstrumentHandler (using "prometheus" as handler
 // name).
 func Handler() http.Handler {
-	return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, promhttp.HandlerFor(defaultRegistry, promhttp.HandlerOpts{}))
+	return promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, promhttp.HandlerFor(defaultRegistry, promhttp.HandlerOpts{ProcessStartTime: processStart}))
 }
 
 // HandlerWithReset returns an HTTP handler for the DefaultGatherer but invokes
@@ -62,7 +69,7 @@ func Handler() http.Handler {
 func HandlerWithReset() http.Handler {
 	return promhttp.InstrumentMetricHandler(
 		prometheus.DefaultRegisterer,
-		metrics.HandlerWithReset(defaultRegistry, metrics.HandlerOpts{}))
+		metrics.HandlerWithReset(defaultRegistry, metrics.HandlerOpts{ProcessStartTime: processStart}))
 }
 
 // CustomRegister registers a custom collector but uses the global registry.

vendor/k8s.io/component-base/metrics/metric.go

@@ -97,9 +97,8 @@ func (r *lazyMetric) lazyInit(self kubeCollector, fqName string) {
 //  2. if the metric is manually disabled via a CLI flag.
 //
 // Disclaimer:  disabling a metric via a CLI flag has higher precedence than
-//
-//	  	deprecation and will override show-hidden-metrics for the explicitly
-//		disabled metric.
+// deprecation and will override show-hidden-metrics for the explicitly
+// disabled metric.
 func (r *lazyMetric) preprocessMetric(version semver.Version) {
 	disabledMetricsLock.RLock()
 	defer disabledMetricsLock.RUnlock()
@@ -167,7 +166,7 @@ func (r *lazyMetric) Create(version *semver.Version) bool {
 	if deprecatedV != nil {
 		dv = deprecatedV.String()
 	}
-	registeredMetrics.WithLabelValues(string(sl), dv).Inc()
+	registeredMetricsTotal.WithLabelValues(string(sl), dv).Inc()
 	return r.IsCreated()
 }
 
@@ -216,7 +215,6 @@ var noopCounterVec = &prometheus.CounterVec{}
 var noopHistogramVec = &prometheus.HistogramVec{}
 var noopTimingHistogramVec = &promext.TimingHistogramVec{}
 var noopGaugeVec = &prometheus.GaugeVec{}
-var noopObserverVec = &noopObserverVector{}
 
 // just use a convenience struct for all the no-ops
 var noop = &noopMetric{}
@@ -235,22 +233,3 @@ func (noopMetric) Desc() *prometheus.Desc            { return nil }
 func (noopMetric) Write(*dto.Metric) error           { return nil }
 func (noopMetric) Describe(chan<- *prometheus.Desc)  {}
 func (noopMetric) Collect(chan<- prometheus.Metric)  {}
-
-type noopObserverVector struct{}
-
-func (noopObserverVector) GetMetricWith(prometheus.Labels) (prometheus.Observer, error) {
-	return noop, nil
-}
-func (noopObserverVector) GetMetricWithLabelValues(...string) (prometheus.Observer, error) {
-	return noop, nil
-}
-func (noopObserverVector) With(prometheus.Labels) prometheus.Observer    { return noop }
-func (noopObserverVector) WithLabelValues(...string) prometheus.Observer { return noop }
-func (noopObserverVector) CurryWith(prometheus.Labels) (prometheus.ObserverVec, error) {
-	return noopObserverVec, nil
-}
-func (noopObserverVector) MustCurryWith(prometheus.Labels) prometheus.ObserverVec {
-	return noopObserverVec
-}
-func (noopObserverVector) Describe(chan<- *prometheus.Desc) {}
-func (noopObserverVector) Collect(chan<- prometheus.Metric) {}

vendor/k8s.io/component-base/metrics/options.go

@@ -31,6 +31,7 @@ type Options struct {
 	ShowHiddenMetricsForVersion string
 	DisabledMetrics             []string
 	AllowListMapping            map[string]string
+	AllowListMappingManifest    string
 }
 
 // NewOptions returns default metrics options
@@ -40,6 +41,10 @@ func NewOptions() *Options {
 
 // Validate validates metrics flags options.
 func (o *Options) Validate() []error {
+	if o == nil {
+		return nil
+	}
+
 	var errs []error
 	err := validateShowHiddenMetricsVersion(parseVersion(version.Get()), o.ShowHiddenMetricsForVersion)
 	if err != nil {
@@ -77,6 +82,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 		"The map from metric-label to value allow-list of this label. The key's format is <MetricName>,<LabelName>. "+
 			"The value's format is <allowed_value>,<allowed_value>..."+
 			"e.g. metric1,label1='v1,v2,v3', metric1,label2='v1,v2,v3' metric2,label1='v1,v2,v3'.")
+	fs.StringVar(&o.AllowListMappingManifest, "allow-metric-labels-manifest", o.AllowListMappingManifest,
+		"The path to the manifest file that contains the allow-list mapping. "+
+			"The format of the file is the same as the flag --allow-metric-labels. "+
+			"Note that the flag --allow-metric-labels will override the manifest file.")
 }
 
 // Apply applies parameters into global configuration of metrics.
@@ -93,6 +102,8 @@ func (o *Options) Apply() {
 	}
 	if o.AllowListMapping != nil {
 		SetLabelAllowListFromCLI(o.AllowListMapping)
+	} else if len(o.AllowListMappingManifest) > 0 {
+		SetLabelAllowListFromManifest(o.AllowListMappingManifest)
 	}
 }
 
@@ -118,7 +129,7 @@ func validateAllowMetricLabel(allowListMapping map[string]string) error {
 	for k := range allowListMapping {
 		reg := regexp.MustCompile(metricNameRegex + `,` + labelRegex)
 		if reg.FindString(k) != k {
-			return fmt.Errorf("--allow-metric-labels must has a list of kv pair with format `metricName:labelName=labelValue, labelValue,...`")
+			return fmt.Errorf("--allow-metric-labels must have a list of kv pair with format `metricName:labelName=labelValue, labelValue,...`")
 		}
 	}
 	return nil

vendor/k8s.io/component-base/metrics/opts.go

@@ -18,13 +18,18 @@ package metrics
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"gopkg.in/yaml.v2"
+
 	"k8s.io/apimachinery/pkg/util/sets"
 	promext "k8s.io/component-base/metrics/prometheusextension"
+	"k8s.io/klog/v2"
 )
 
 var (
@@ -319,6 +324,7 @@ func (allowList *MetricLabelAllowList) ConstrainToAllowedList(labelNameList, lab
 		if allowValues, ok := allowList.labelToAllowList[name]; ok {
 			if !allowValues.Has(value) {
 				labelValueList[index] = "unexpected"
+				cardinalityEnforcementUnexpectedCategorizationsTotal.Inc()
 			}
 		}
 	}
@@ -329,6 +335,7 @@ func (allowList *MetricLabelAllowList) ConstrainLabelMap(labels map[string]strin
 		if allowValues, ok := allowList.labelToAllowList[name]; ok {
 			if !allowValues.Has(value) {
 				labels[name] = "unexpected"
+				cardinalityEnforcementUnexpectedCategorizationsTotal.Inc()
 			}
 		}
 	}
@@ -354,3 +361,20 @@ func SetLabelAllowListFromCLI(allowListMapping map[string]string) {
 		}
 	}
 }
+
+func SetLabelAllowListFromManifest(manifest string) {
+	allowListLock.Lock()
+	defer allowListLock.Unlock()
+	allowListMapping := make(map[string]string)
+	data, err := os.ReadFile(filepath.Clean(manifest))
+	if err != nil {
+		klog.Errorf("Failed to read allow list manifest: %v", err)
+		return
+	}
+	err = yaml.Unmarshal(data, &allowListMapping)
+	if err != nil {
+		klog.Errorf("Failed to parse allow list manifest: %v", err)
+		return
+	}
+	SetLabelAllowListFromCLI(allowListMapping)
+}

vendor/k8s.io/component-base/metrics/prometheus/feature/metrics.go

@@ -30,7 +30,7 @@ var (
 			Namespace:      "kubernetes",
 			Name:           "feature_enabled",
 			Help:           "This metric records the data about the stage and enablement of a k8s feature.",
-			StabilityLevel: k8smetrics.ALPHA,
+			StabilityLevel: k8smetrics.BETA,
 		},
 		[]string{"name", "stage"},
 	)

vendor/k8s.io/component-base/metrics/prometheus/slis/metrics.go

@@ -37,7 +37,7 @@ var (
 			Namespace:      "kubernetes",
 			Name:           "healthcheck",
 			Help:           "This metric records the result of a single healthcheck.",
-			StabilityLevel: k8smetrics.ALPHA,
+			StabilityLevel: k8smetrics.STABLE,
 		},
 		[]string{"name", "type"},
 	)
@@ -48,7 +48,7 @@ var (
 			Namespace:      "kubernetes",
 			Name:           "healthchecks_total",
 			Help:           "This metric records the results of all healthcheck.",
-			StabilityLevel: k8smetrics.ALPHA,
+			StabilityLevel: k8smetrics.STABLE,
 		},
 		[]string{"name", "type", "status"},
 	)
@@ -57,6 +57,7 @@ var (
 func Register(registry k8smetrics.KubeRegistry) {
 	registry.Register(healthcheck)
 	registry.Register(healthchecksTotal)
+	_ = k8smetrics.RegisterProcessStartTime(registry.Register)
 }
 
 func ResetHealthMetrics() {

vendor/k8s.io/component-base/metrics/registry.go

@@ -37,27 +37,35 @@ var (
 	registriesLock      sync.RWMutex
 	disabledMetrics     = map[string]struct{}{}
 
-	registeredMetrics = NewCounterVec(
+	registeredMetricsTotal = NewCounterVec(
 		&CounterOpts{
-			Name:           "registered_metric_total",
+			Name:           "registered_metrics_total",
 			Help:           "The count of registered metrics broken by stability level and deprecation version.",
-			StabilityLevel: ALPHA,
+			StabilityLevel: BETA,
 		},
 		[]string{"stability_level", "deprecated_version"},
 	)
 
 	disabledMetricsTotal = NewCounter(
 		&CounterOpts{
-			Name:           "disabled_metric_total",
+			Name:           "disabled_metrics_total",
 			Help:           "The count of disabled metrics.",
-			StabilityLevel: ALPHA,
+			StabilityLevel: BETA,
 		},
 	)
 
 	hiddenMetricsTotal = NewCounter(
 		&CounterOpts{
-			Name:           "hidden_metric_total",
+			Name:           "hidden_metrics_total",
 			Help:           "The count of hidden metrics.",
+			StabilityLevel: BETA,
+		},
+	)
+
+	cardinalityEnforcementUnexpectedCategorizationsTotal = NewCounter(
+		&CounterOpts{
+			Name:           "cardinality_enforcement_unexpected_categorizations_total",
+			Help:           "The count of unexpected categorizations during cardinality enforcement.",
 			StabilityLevel: ALPHA,
 		},
 	)
@@ -379,7 +387,8 @@ func NewKubeRegistry() KubeRegistry {
 }
 
 func (r *kubeRegistry) RegisterMetaMetrics() {
-	r.MustRegister(registeredMetrics)
+	r.MustRegister(registeredMetricsTotal)
 	r.MustRegister(disabledMetricsTotal)
 	r.MustRegister(hiddenMetricsTotal)
+	r.MustRegister(cardinalityEnforcementUnexpectedCategorizationsTotal)
 }

vendor/k8s.io/component-base/metrics/testutil/testutil.go

@@ -19,11 +19,13 @@ package testutil
 import (
 	"fmt"
 	"io"
+	"testing"
 
 	"github.com/prometheus/client_golang/prometheus/testutil"
 
 	apimachineryversion "k8s.io/apimachinery/pkg/version"
 	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
 )
 
 // CollectAndCompare registers the provided Collector with a newly created
@@ -67,6 +69,13 @@ func CustomCollectAndCompare(c metrics.StableCollector, expected io.Reader, metr
 	return GatherAndCompare(registry, expected, metricNames...)
 }
 
+// ScrapeAndCompare calls a remote exporter's endpoint which is expected to return some metrics in
+// plain text format. Then it compares it with the results that the `expected` would return.
+// If the `metricNames` is not empty it would filter the comparison only to the given metric names.
+func ScrapeAndCompare(url string, expected io.Reader, metricNames ...string) error {
+	return testutil.ScrapeAndCompare(url, expected, metricNames...)
+}
+
 // NewFakeKubeRegistry creates a fake `KubeRegistry` that takes the input version as `build in version`.
 // It should only be used in testing scenario especially for the deprecated metrics.
 // The input version format should be `major.minor.patch`, e.g. '1.18.0'.
@@ -84,3 +93,62 @@ func NewFakeKubeRegistry(ver string) metrics.KubeRegistry {
 
 	return metrics.NewKubeRegistry()
 }
+
+func AssertVectorCount(t *testing.T, name string, labelFilter map[string]string, wantCount int) {
+	metrics, err := legacyregistry.DefaultGatherer.Gather()
+	if err != nil {
+		t.Fatalf("Failed to gather metrics: %s", err)
+	}
+
+	counterSum := 0
+	for _, mf := range metrics {
+		if mf.GetName() != name {
+			continue // Ignore other metrics.
+		}
+		for _, metric := range mf.GetMetric() {
+			if !LabelsMatch(metric, labelFilter) {
+				continue
+			}
+			counterSum += int(metric.GetCounter().GetValue())
+		}
+	}
+	if wantCount != counterSum {
+		t.Errorf("Wanted count %d, got %d for metric %s with labels %#+v", wantCount, counterSum, name, labelFilter)
+		for _, mf := range metrics {
+			if mf.GetName() == name {
+				for _, metric := range mf.GetMetric() {
+					t.Logf("\tnear match: %s", metric.String())
+				}
+			}
+		}
+	}
+}
+
+func AssertHistogramTotalCount(t *testing.T, name string, labelFilter map[string]string, wantCount int) {
+	metrics, err := legacyregistry.DefaultGatherer.Gather()
+	if err != nil {
+		t.Fatalf("Failed to gather metrics: %s", err)
+	}
+	counterSum := 0
+	for _, mf := range metrics {
+		if mf.GetName() != name {
+			continue // Ignore other metrics.
+		}
+		for _, metric := range mf.GetMetric() {
+			if !LabelsMatch(metric, labelFilter) {
+				continue
+			}
+			counterSum += int(metric.GetHistogram().GetSampleCount())
+		}
+	}
+	if wantCount != counterSum {
+		t.Errorf("Wanted count %d, got %d for metric %s with labels %#+v", wantCount, counterSum, name, labelFilter)
+		for _, mf := range metrics {
+			if mf.GetName() == name {
+				for _, metric := range mf.GetMetric() {
+					t.Logf("\tnear match: %s\n", metric.String())
+				}
+			}
+		}
+	}
+}

vendor/k8s.io/component-base/metrics/value.go

@@ -60,8 +60,7 @@ func NewConstMetric(desc *Desc, valueType ValueType, value float64, labelValues
 // NewLazyMetricWithTimestamp is a helper of NewMetricWithTimestamp.
 //
 // Warning: the Metric 'm' must be the one created by NewLazyConstMetric(),
-//
-//	otherwise, no stability guarantees would be offered.
+// otherwise, no stability guarantees would be offered.
 func NewLazyMetricWithTimestamp(t time.Time, m Metric) Metric {
 	if m == nil {
 		return nil