feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

vendor/k8s.io/component-base/logs/api/v1/options.go

@@ -17,12 +17,17 @@ limitations under the License.
 package v1
 
 import (
+	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"math"
+	"os"
 	"strings"
+	"sync/atomic"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/spf13/pflag"
 
 	"k8s.io/klog/v2"
@@ -31,6 +36,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/featuregate"
+	"k8s.io/component-base/logs/internal/setverbositylevel"
 	"k8s.io/component-base/logs/klogflags"
 )
 
@@ -54,6 +60,24 @@ func NewLoggingConfiguration() *LoggingConfiguration {
 	return &c
 }
 
+// Applying configurations multiple times is not safe unless it's guaranteed that there
+// are no goroutines which might call logging functions. The default for ValidateAndApply
+// and ValidateAndApplyWithOptions is to return an error when called more than once.
+// Binaries and unit tests can override that behavior.
+var ReapplyHandling = ReapplyHandlingError
+
+type ReapplyHandlingType int
+
+const (
+	// ReapplyHandlingError is the default: calling ValidateAndApply or
+	// ValidateAndApplyWithOptions again returns an error.
+	ReapplyHandlingError ReapplyHandlingType = iota
+	// ReapplyHandlingIgnoreUnchanged silently ignores any additional calls of
+	// ValidateAndApply or ValidateAndApplyWithOptions if the configuration
+	// is unchanged, otherwise they return an error.
+	ReapplyHandlingIgnoreUnchanged
+)
+
 // ValidateAndApply combines validation and application of the logging configuration.
 // This should be invoked as early as possible because then the rest of the program
 // startup (including validation of other options) will already run with the final
@@ -61,19 +85,50 @@ func NewLoggingConfiguration() *LoggingConfiguration {
 //
 // The optional FeatureGate controls logging features. If nil, the default for
 // these features is used.
+//
+// Logging options must be applied as early as possible during the program
+// startup. Some changes are global and cannot be done safely when there are
+// already goroutines running.
 func ValidateAndApply(c *LoggingConfiguration, featureGate featuregate.FeatureGate) error {
-	return ValidateAndApplyAsField(c, featureGate, nil)
+	return validateAndApply(c, nil, featureGate, nil)
+}
+
+// ValidateAndApplyWithOptions is a variant of ValidateAndApply which accepts
+// additional options beyond those that can be configured through the API. This
+// is meant for testing.
+//
+// Logging options must be applied as early as possible during the program
+// startup. Some changes are global and cannot be done safely when there are
+// already goroutines running.
+func ValidateAndApplyWithOptions(c *LoggingConfiguration, options *LoggingOptions, featureGate featuregate.FeatureGate) error {
+	return validateAndApply(c, options, featureGate, nil)
+}
+
+// +k8s:deepcopy-gen=false
+
+// LoggingOptions can be used with ValidateAndApplyWithOptions to override
+// certain global defaults.
+type LoggingOptions struct {
+	// ErrorStream can be used to override the os.Stderr default.
+	ErrorStream io.Writer
+
+	// InfoStream can be used to override the os.Stdout default.
+	InfoStream io.Writer
 }
 
 // ValidateAndApplyAsField is a variant of ValidateAndApply that should be used
 // when the LoggingConfiguration is embedded in some larger configuration
 // structure.
 func ValidateAndApplyAsField(c *LoggingConfiguration, featureGate featuregate.FeatureGate, fldPath *field.Path) error {
+	return validateAndApply(c, nil, featureGate, fldPath)
+}
+
+func validateAndApply(c *LoggingConfiguration, options *LoggingOptions, featureGate featuregate.FeatureGate, fldPath *field.Path) error {
 	errs := Validate(c, featureGate, fldPath)
 	if len(errs) > 0 {
 		return errs.ToAggregate()
 	}
-	return apply(c, featureGate)
+	return apply(c, options, featureGate)
 }
 
 // Validate can be used to check for invalid settings without applying them.
@@ -156,19 +211,50 @@ func featureEnabled(featureGate featuregate.FeatureGate, feature featuregate.Fea
 	return enabled
 }
 
-func apply(c *LoggingConfiguration, featureGate featuregate.FeatureGate) error {
-	contextualLoggingEnabled := contextualLoggingDefault
-	if featureGate != nil {
-		contextualLoggingEnabled = featureGate.Enabled(ContextualLogging)
+func apply(c *LoggingConfiguration, options *LoggingOptions, featureGate featuregate.FeatureGate) error {
+	p := &parameters{
+		C:                        c,
+		Options:                  options,
+		ContextualLoggingEnabled: contextualLoggingDefault,
 	}
+	if featureGate != nil {
+		p.ContextualLoggingEnabled = featureGate.Enabled(ContextualLogging)
+	}
+
+	oldP := applyParameters.Load()
+	if oldP != nil {
+		switch ReapplyHandling {
+		case ReapplyHandlingError:
+			return errors.New("logging configuration was already applied earlier, changing it is not allowed")
+		case ReapplyHandlingIgnoreUnchanged:
+			if diff := cmp.Diff(oldP, p); diff != "" {
+				return fmt.Errorf("the logging configuration should not be changed after setting it once (- old setting, + new setting):\n%s", diff)
+			}
+			return nil
+		default:
+			return fmt.Errorf("invalid value %d for ReapplyHandling", ReapplyHandling)
+		}
+	}
+	applyParameters.Store(p)
 
 	// if log format not exists, use nil loggr
 	format, _ := logRegistry.get(c.Format)
 	if format.factory == nil {
 		klog.ClearLogger()
 	} else {
-		log, flush := format.factory.Create(*c)
-		klog.SetLoggerWithOptions(log, klog.ContextualLogger(contextualLoggingEnabled), klog.FlushLogger(flush))
+		if options == nil {
+			options = &LoggingOptions{
+				ErrorStream: os.Stderr,
+				InfoStream:  os.Stdout,
+			}
+		}
+		log, control := format.factory.Create(*c, *options)
+		if control.SetVerbosityLevel != nil {
+			setverbositylevel.Mutex.Lock()
+			defer setverbositylevel.Mutex.Unlock()
+			setverbositylevel.Callbacks = append(setverbositylevel.Callbacks, control.SetVerbosityLevel)
+		}
+		klog.SetLoggerWithOptions(log, klog.ContextualLogger(p.ContextualLoggingEnabled), klog.FlushLogger(control.Flush))
 	}
 	if err := loggingFlags.Lookup("v").Value.Set(VerbosityLevelPflag(&c.Verbosity).String()); err != nil {
 		return fmt.Errorf("internal error while setting klog verbosity: %v", err)
@@ -176,19 +262,87 @@ func apply(c *LoggingConfiguration, featureGate featuregate.FeatureGate) error {
 	if err := loggingFlags.Lookup("vmodule").Value.Set(VModuleConfigurationPflag(&c.VModule).String()); err != nil {
 		return fmt.Errorf("internal error while setting klog vmodule: %v", err)
 	}
-	klog.StartFlushDaemon(c.FlushFrequency)
-	klog.EnableContextualLogging(contextualLoggingEnabled)
+	klog.StartFlushDaemon(c.FlushFrequency.Duration.Duration)
+	klog.EnableContextualLogging(p.ContextualLoggingEnabled)
+	return nil
+}
+
+type parameters struct {
+	C                        *LoggingConfiguration
+	Options                  *LoggingOptions
+	ContextualLoggingEnabled bool
+}
+
+var applyParameters atomic.Pointer[parameters]
+
+// ResetForTest restores the default settings. This is not thread-safe and should only
+// be used when there are no goroutines running. The intended users are unit
+// tests in other packages.
+func ResetForTest(featureGate featuregate.FeatureGate) error {
+	oldP := applyParameters.Load()
+	if oldP == nil {
+		// Nothing to do.
+		return nil
+	}
+
+	// This makes it possible to call apply again without triggering errors.
+	applyParameters.Store(nil)
+
+	// Restore defaults. Shouldn't fail, but check anyway.
+	config := NewLoggingConfiguration()
+	if err := ValidateAndApply(config, featureGate); err != nil {
+		return fmt.Errorf("apply default configuration: %v", err)
+	}
+
+	// And again...
+	applyParameters.Store(nil)
+
 	return nil
 }
 
 // AddFlags adds command line flags for the configuration.
 func AddFlags(c *LoggingConfiguration, fs *pflag.FlagSet) {
+	addFlags(c, fs)
+}
+
+// AddGoFlags is a variant of AddFlags for a standard FlagSet.
+func AddGoFlags(c *LoggingConfiguration, fs *flag.FlagSet) {
+	addFlags(c, goFlagSet{FlagSet: fs})
+}
+
+// flagSet is the interface implemented by pflag.FlagSet, with
+// just those methods defined which are needed by addFlags.
+type flagSet interface {
+	BoolVar(p *bool, name string, value bool, usage string)
+	DurationVar(p *time.Duration, name string, value time.Duration, usage string)
+	StringVar(p *string, name string, value string, usage string)
+	Var(value pflag.Value, name string, usage string)
+	VarP(value pflag.Value, name, shorthand, usage string)
+}
+
+// goFlagSet implements flagSet for a stdlib flag.FlagSet.
+type goFlagSet struct {
+	*flag.FlagSet
+}
+
+func (fs goFlagSet) Var(value pflag.Value, name string, usage string) {
+	fs.FlagSet.Var(value, name, usage)
+}
+
+func (fs goFlagSet) VarP(value pflag.Value, name, shorthand, usage string) {
+	// Ignore shorthand, it's not needed and not supported.
+	fs.FlagSet.Var(value, name, usage)
+}
+
+// addFlags can be used with both flag.FlagSet and pflag.FlagSet. The internal
+// interface definition avoids duplicating this code.
+func addFlags(c *LoggingConfiguration, fs flagSet) {
 	formats := logRegistry.list()
 	fs.StringVar(&c.Format, "logging-format", c.Format, fmt.Sprintf("Sets the log format. Permitted formats: %s.", formats))
 	// No new log formats should be added after generation is of flag options
 	logRegistry.freeze()
 
-	fs.DurationVar(&c.FlushFrequency, LogFlushFreqFlagName, c.FlushFrequency, "Maximum number of seconds between log flushes")
+	fs.DurationVar(&c.FlushFrequency.Duration.Duration, LogFlushFreqFlagName, c.FlushFrequency.Duration.Duration, "Maximum number of seconds between log flushes")
 	fs.VarP(VerbosityLevelPflag(&c.Verbosity), "v", "v", "number for the log level verbosity")
 	fs.Var(VModuleConfigurationPflag(&c.VModule), "vmodule", "comma-separated list of pattern=N settings for file-filtered logging (only works for text log format)")
 
@@ -210,8 +364,9 @@ func SetRecommendedLoggingConfiguration(c *LoggingConfiguration) {
 	if c.Format == "" {
 		c.Format = "text"
 	}
-	if c.FlushFrequency == 0 {
-		c.FlushFrequency = LogFlushFreqDefault
+	if c.FlushFrequency.Duration.Duration == 0 {
+		c.FlushFrequency.Duration.Duration = LogFlushFreqDefault
+		c.FlushFrequency.SerializeAsString = true
 	}
 	var empty resource.QuantityValue
 	if c.Options.JSON.InfoBufferSize == empty {

vendor/k8s.io/component-base/logs/api/v1/pflags.go

@@ -36,6 +36,9 @@ type vmoduleConfigurationPFlag struct {
 
 // String returns the -vmodule parameter (comma-separated list of pattern=N).
 func (wrapper vmoduleConfigurationPFlag) String() string {
+	if wrapper.value == nil {
+		return ""
+	}
 	var patterns []string
 	for _, item := range *wrapper.value {
 		patterns = append(patterns, fmt.Sprintf("%s=%d", item.FilePattern, item.Verbosity))
@@ -82,10 +85,16 @@ type verbosityLevelPflag struct {
 }
 
 func (wrapper verbosityLevelPflag) String() string {
+	if wrapper.value == nil {
+		return "0"
+	}
 	return strconv.FormatInt(int64(*wrapper.value), 10)
 }
 
 func (wrapper verbosityLevelPflag) Get() interface{} {
+	if wrapper.value == nil {
+		return VerbosityLevel(0)
+	}
 	return *wrapper.value
 }
 

vendor/k8s.io/component-base/logs/api/v1/registry.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/go-logr/logr"
 
@@ -30,6 +31,7 @@ var logRegistry = newLogFormatRegistry()
 
 // logFormatRegistry stores factories for all supported logging formats.
 type logFormatRegistry struct {
+	mutex    sync.Mutex
 	registry map[string]logFormat
 	frozen   bool
 }
@@ -39,14 +41,29 @@ type logFormat struct {
 	feature featuregate.Feature
 }
 
+// +k8s:deepcopy-gen=false
+
+// RuntimeControl provides operations that aren't available through the normal
+// Logger or LogSink API.
+type RuntimeControl struct {
+	// Flush ensures that all in-memory data is written.
+	// May be nil.
+	Flush func()
+
+	// SetVerbosityLevel changes the level for all Logger instances
+	// derived from the initial one. May be nil.
+	//
+	// The parameter is intentionally a plain uint32 instead of
+	// VerbosityLevel to enable implementations that don't need to import
+	// the API (helps avoid circular dependencies).
+	SetVerbosityLevel func(v uint32) error
+}
+
 // LogFormatFactory provides support for a certain additional,
 // non-default log format.
 type LogFormatFactory interface {
 	// Create returns a logger with the requested configuration.
-	// Returning a flush function for the logger is optional.
-	// If provided, the caller must ensure that it is called
-	// periodically (if desired) and at program exit.
-	Create(c LoggingConfiguration) (log logr.Logger, flush func())
+	Create(c LoggingConfiguration, o LoggingOptions) (logr.Logger, RuntimeControl)
 }
 
 // RegisterLogFormat registers support for a new logging format. This must be called
@@ -68,6 +85,8 @@ func newLogFormatRegistry() *logFormatRegistry {
 
 // register adds a new log format. It's an error to modify an existing one.
 func (lfr *logFormatRegistry) register(name string, format logFormat) error {
+	lfr.mutex.Lock()
+	defer lfr.mutex.Unlock()
 	if lfr.frozen {
 		return fmt.Errorf("log format registry is frozen, unable to register log format %s", name)
 	}
@@ -83,6 +102,8 @@ func (lfr *logFormatRegistry) register(name string, format logFormat) error {
 
 // get specified log format factory
 func (lfr *logFormatRegistry) get(name string) (*logFormat, error) {
+	lfr.mutex.Lock()
+	defer lfr.mutex.Unlock()
 	format, ok := lfr.registry[name]
 	if !ok {
 		return nil, fmt.Errorf("log format: %s does not exists", name)
@@ -92,6 +113,8 @@ func (lfr *logFormatRegistry) get(name string) (*logFormat, error) {
 
 // list names of registered log formats, including feature gates (sorted)
 func (lfr *logFormatRegistry) list() string {
+	lfr.mutex.Lock()
+	defer lfr.mutex.Unlock()
 	formats := make([]string, 0, len(lfr.registry))
 	for name, format := range lfr.registry {
 		item := fmt.Sprintf(`"%s"`, name)
@@ -106,5 +129,7 @@ func (lfr *logFormatRegistry) list() string {
 
 // freeze prevents further modifications of the registered log formats.
 func (lfr *logFormatRegistry) freeze() {
+	lfr.mutex.Lock()
+	defer lfr.mutex.Unlock()
 	lfr.frozen = true
 }

vendor/k8s.io/component-base/logs/api/v1/types.go

@@ -17,9 +17,11 @@ limitations under the License.
 package v1
 
 import (
-	"time"
+	"encoding/json"
+	"fmt"
 
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // Supported output formats.
@@ -39,10 +41,11 @@ type LoggingConfiguration struct {
 	// Format Flag specifies the structure of log messages.
 	// default value of format is `text`
 	Format string `json:"format,omitempty"`
-	// Maximum number of nanoseconds (i.e. 1s = 1000000000) between log
-	// flushes. Ignored if the selected logging backend writes log
-	// messages without buffering.
-	FlushFrequency time.Duration `json:"flushFrequency"`
+	// Maximum time between log flushes.
+	// If a string, parsed as a duration (i.e. "1s")
+	// If an int, the maximum number of nanoseconds (i.e. 1s = 1000000000).
+	// Ignored if the selected logging backend writes log messages without buffering.
+	FlushFrequency TimeOrMetaDuration `json:"flushFrequency"`
 	// Verbosity is the threshold that determines which log messages are
 	// logged. Default is zero which logs only the most important
 	// messages. Higher values enable additional messages. Error messages
@@ -58,6 +61,37 @@ type LoggingConfiguration struct {
 	Options FormatOptions `json:"options,omitempty"`
 }
 
+// TimeOrMetaDuration is present only for backwards compatibility for the
+// flushFrequency field, and new fields should use metav1.Duration.
+type TimeOrMetaDuration struct {
+	// Duration holds the duration
+	Duration metav1.Duration
+	// SerializeAsString controls whether the value is serialized as a string or an integer
+	SerializeAsString bool `json:"-"`
+}
+
+func (t TimeOrMetaDuration) MarshalJSON() ([]byte, error) {
+	if t.SerializeAsString {
+		return t.Duration.MarshalJSON()
+	} else {
+		// Marshal as integer for backwards compatibility
+		return json.Marshal(t.Duration.Duration)
+	}
+}
+
+func (t *TimeOrMetaDuration) UnmarshalJSON(b []byte) error {
+	if len(b) > 0 && b[0] == '"' {
+		// string values unmarshal as metav1.Duration
+		t.SerializeAsString = true
+		return json.Unmarshal(b, &t.Duration)
+	}
+	t.SerializeAsString = false
+	if err := json.Unmarshal(b, &t.Duration.Duration); err != nil {
+		return fmt.Errorf("invalid duration %q: %w", string(b), err)
+	}
+	return nil
+}
+
 // FormatOptions contains options for the different logging formats.
 type FormatOptions struct {
 	// [Alpha] JSON contains options for logging format "json".

vendor/k8s.io/component-base/logs/api/v1/zz_generated.deepcopy.go

@@ -58,6 +58,7 @@ func (in *JSONOptions) DeepCopy() *JSONOptions {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *LoggingConfiguration) DeepCopyInto(out *LoggingConfiguration) {
 	*out = *in
+	out.FlushFrequency = in.FlushFrequency
 	if in.VModule != nil {
 		in, out := &in.VModule, &out.VModule
 		*out = make(VModuleConfiguration, len(*in))
@@ -77,6 +78,23 @@ func (in *LoggingConfiguration) DeepCopy() *LoggingConfiguration {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TimeOrMetaDuration) DeepCopyInto(out *TimeOrMetaDuration) {
+	*out = *in
+	out.Duration = in.Duration
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TimeOrMetaDuration.
+func (in *TimeOrMetaDuration) DeepCopy() *TimeOrMetaDuration {
+	if in == nil {
+		return nil
+	}
+	out := new(TimeOrMetaDuration)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in VModuleConfiguration) DeepCopyInto(out *VModuleConfiguration) {
 	{

vendor/k8s.io/component-base/logs/internal/setverbositylevel/setverbositylevel.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package setverbositylevel stores callbacks that will be invoked by logs.GlogLevel.
+//
+// This is a separate package to avoid a dependency from
+// k8s.io/component-base/logs (uses the callbacks) to
+// k8s.io/component-base/logs/api/v1 (adds them). Not all users of the logs
+// package also use the API.
+package setverbositylevel
+
+import (
+	"sync"
+)
+
+var (
+	// Mutex controls access to the callbacks.
+	Mutex sync.Mutex
+
+	Callbacks []func(v uint32) error
+)

vendor/k8s.io/component-base/logs/logs.go

@@ -23,10 +23,12 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"strconv"
 	"time"
 
 	"github.com/spf13/pflag"
 	logsapi "k8s.io/component-base/logs/api/v1"
+	"k8s.io/component-base/logs/internal/setverbositylevel"
 	"k8s.io/component-base/logs/klogflags"
 	"k8s.io/klog/v2"
 )
@@ -182,11 +184,26 @@ func NewLogger(prefix string) *log.Logger {
 	return log.New(KlogWriter{}, prefix, 0)
 }
 
-// GlogSetter is a setter to set glog level.
+// GlogSetter modifies the verbosity threshold for the entire program.
+// Some components have HTTP-based APIs for invoking this at runtime.
 func GlogSetter(val string) (string, error) {
+	v, err := strconv.ParseUint(val, 10, 32)
+	if err != nil {
+		return "", err
+	}
+
 	var level klog.Level
 	if err := level.Set(val); err != nil {
 		return "", fmt.Errorf("failed set klog.logging.verbosity %s: %v", val, err)
 	}
+
+	setverbositylevel.Mutex.Lock()
+	defer setverbositylevel.Mutex.Unlock()
+	for _, cb := range setverbositylevel.Callbacks {
+		if err := cb(uint32(v)); err != nil {
+			return "", err
+		}
+	}
+
 	return fmt.Sprintf("successfully set klog.logging.verbosity to %s", val), nil
 }