feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions

View File

@@ -34,6 +34,7 @@ import (
jsonpatch "github.com/evanphx/json-patch"
"github.com/google/uuid"
"golang.org/x/crypto/cryptobyte"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -64,6 +65,7 @@ import (
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/routes"
serverstore "k8s.io/apiserver/pkg/server/storage"
storagevalue "k8s.io/apiserver/pkg/storage/value"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
@@ -76,6 +78,7 @@ import (
"k8s.io/component-base/tracing"
"k8s.io/klog/v2"
openapicommon "k8s.io/kube-openapi/pkg/common"
"k8s.io/kube-openapi/pkg/spec3"
"k8s.io/kube-openapi/pkg/validation/spec"
"k8s.io/utils/clock"
utilsnet "k8s.io/utils/net"
@@ -84,6 +87,13 @@ import (
_ "k8s.io/apiserver/pkg/apis/apiserver/install"
)
// hostnameFunc is a function to set the hostnameFunc of this apiserver.
// To be used for testing purpose only, to simulate scenarios where multiple apiservers
// exist. In such cases we want to ensure unique apiserver IDs which are a hash of hostnameFunc.
var (
hostnameFunc = os.Hostname
)
const (
// DefaultLegacyAPIPrefix is where the legacy APIs will be located.
DefaultLegacyAPIPrefix = "/api"
@@ -126,6 +136,7 @@ type Config struct {
EnableIndex bool
EnableProfiling bool
DebugSocketPath string
EnableDiscovery bool
// Requires generic profiling enabled
@@ -155,8 +166,14 @@ type Config struct {
// BuildHandlerChainFunc allows you to build custom handler chains by decorating the apiHandler.
BuildHandlerChainFunc func(apiHandler http.Handler, c *Config) (secure http.Handler)
// HandlerChainWaitGroup allows you to wait for all chain handlers exit after the server shutdown.
HandlerChainWaitGroup *utilwaitgroup.SafeWaitGroup
// NonLongRunningRequestWaitGroup allows you to wait for all chain
// handlers associated with non long-running requests
// to complete while the server is shuting down.
NonLongRunningRequestWaitGroup *utilwaitgroup.SafeWaitGroup
// WatchRequestWaitGroup allows us to wait for all chain
// handlers associated with active watch requests to
// complete while the server is shuting down.
WatchRequestWaitGroup *utilwaitgroup.RateLimitedSafeWaitGroup
// DiscoveryAddresses is used to build the IPs pass to discovery. If nil, the ExternalAddress is
// always reported
DiscoveryAddresses discovery.Addresses
@@ -178,10 +195,12 @@ type Config struct {
// OpenAPIConfig will be used in generating OpenAPI spec. This is nil by default. Use DefaultOpenAPIConfig for "working" defaults.
OpenAPIConfig *openapicommon.Config
// OpenAPIV3Config will be used in generating OpenAPI V3 spec. This is nil by default. Use DefaultOpenAPIV3Config for "working" defaults.
OpenAPIV3Config *openapicommon.Config
OpenAPIV3Config *openapicommon.OpenAPIV3Config
// SkipOpenAPIInstallation avoids installing the OpenAPI handler if set to true.
SkipOpenAPIInstallation bool
// ResourceTransformers are used to transform resources from and to etcd, e.g. encryption.
ResourceTransformers storagevalue.ResourceTransformers
// RESTOptionsGetter is used to construct RESTStorage types via the generic registry.
RESTOptionsGetter genericregistry.RESTOptionsGetter
@@ -268,6 +287,23 @@ type Config struct {
// AggregatedDiscoveryGroupManager serves /apis in an aggregated form.
AggregatedDiscoveryGroupManager discoveryendpoint.ResourceManager
// ShutdownWatchTerminationGracePeriod, if set to a positive value,
// is the maximum duration the apiserver will wait for all active
// watch request(s) to drain.
// Once this grace period elapses, the apiserver will no longer
// wait for any active watch request(s) in flight to drain, it will
// proceed to the next step in the graceful server shutdown process.
// If set to a positive value, the apiserver will keep track of the
// number of active watch request(s) in flight and during shutdown
// it will wait, at most, for the specified duration and allow these
// active watch requests to drain with some rate limiting in effect.
// The default is zero, which implies the apiserver will not keep
// track of active watch request(s) in flight and will not wait
// for them to drain, this maintains backward compatibility.
// This grace period is orthogonal to other grace periods, and
// it is not overridden by any other grace period.
ShutdownWatchTerminationGracePeriod time.Duration
}
type RecommendedConfig struct {
@@ -320,6 +356,8 @@ type AuthenticationInfo struct {
APIAudiences authenticator.Audiences
// Authenticator determines which subject is making the request
Authenticator authenticator.Request
RequestHeaderConfig *authenticatorfactory.RequestHeaderConfig
}
type AuthorizationInfo struct {
@@ -337,36 +375,54 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz}
var id string
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
hostname, err := os.Hostname()
hostname, err := hostnameFunc()
if err != nil {
klog.Fatalf("error getting hostname for apiserver identity: %v", err)
}
hash := sha256.Sum256([]byte(hostname))
id = "kube-apiserver-" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:16]))
// Since the hash needs to be unique across each kube-apiserver and aggregated apiservers,
// the hash used for the identity should include both the hostname and the identity value.
// TODO: receive the identity value as a parameter once the apiserver identity lease controller
// post start hook is moved to generic apiserver.
b := cryptobyte.NewBuilder(nil)
b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
b.AddBytes([]byte(hostname))
})
b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
b.AddBytes([]byte("kube-apiserver"))
})
hashData, err := b.Bytes()
if err != nil {
klog.Fatalf("error building hash data for apiserver identity: %v", err)
}
hash := sha256.Sum256(hashData)
id = "apiserver-" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:16]))
}
lifecycleSignals := newLifecycleSignals()
return &Config{
Serializer: codecs,
BuildHandlerChainFunc: DefaultBuildHandlerChain,
HandlerChainWaitGroup: new(utilwaitgroup.SafeWaitGroup),
LegacyAPIGroupPrefixes: sets.NewString(DefaultLegacyAPIPrefix),
DisabledPostStartHooks: sets.NewString(),
PostStartHooks: map[string]PostStartHookConfigEntry{},
HealthzChecks: append([]healthz.HealthChecker{}, defaultHealthChecks...),
ReadyzChecks: append([]healthz.HealthChecker{}, defaultHealthChecks...),
LivezChecks: append([]healthz.HealthChecker{}, defaultHealthChecks...),
EnableIndex: true,
EnableDiscovery: true,
EnableProfiling: true,
EnableMetrics: true,
MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(60) * time.Second,
MinRequestTimeout: 1800,
LivezGracePeriod: time.Duration(0),
ShutdownDelayDuration: time.Duration(0),
Serializer: codecs,
BuildHandlerChainFunc: DefaultBuildHandlerChain,
NonLongRunningRequestWaitGroup: new(utilwaitgroup.SafeWaitGroup),
WatchRequestWaitGroup: &utilwaitgroup.RateLimitedSafeWaitGroup{},
LegacyAPIGroupPrefixes: sets.NewString(DefaultLegacyAPIPrefix),
DisabledPostStartHooks: sets.NewString(),
PostStartHooks: map[string]PostStartHookConfigEntry{},
HealthzChecks: append([]healthz.HealthChecker{}, defaultHealthChecks...),
ReadyzChecks: append([]healthz.HealthChecker{}, defaultHealthChecks...),
LivezChecks: append([]healthz.HealthChecker{}, defaultHealthChecks...),
EnableIndex: true,
EnableDiscovery: true,
EnableProfiling: true,
DebugSocketPath: "",
EnableMetrics: true,
MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(60) * time.Second,
MinRequestTimeout: 1800,
LivezGracePeriod: time.Duration(0),
ShutdownDelayDuration: time.Duration(0),
// 1.5MB is the default client request size in bytes
// the etcd server should accept. See
// https://github.com/etcd-io/etcd/blob/release-3.4/embed/config.go#L56.
@@ -387,9 +443,10 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
// Default to treating watch as a long-running operation
// Generic API servers have no inherent long-running subresources
LongRunningFunc: genericfilters.BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString()),
lifecycleSignals: lifecycleSignals,
StorageObjectCountTracker: flowcontrolrequest.NewStorageObjectCountTracker(),
LongRunningFunc: genericfilters.BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString()),
lifecycleSignals: lifecycleSignals,
StorageObjectCountTracker: flowcontrolrequest.NewStorageObjectCountTracker(),
ShutdownWatchTerminationGracePeriod: time.Duration(0),
APIServerID: id,
StorageVersionManager: storageversion.NewDefaultManager(),
@@ -426,8 +483,23 @@ func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, de
}
// DefaultOpenAPIV3Config provides the default OpenAPIV3Config used to build the OpenAPI V3 spec
func DefaultOpenAPIV3Config(getDefinitions openapicommon.GetOpenAPIDefinitions, defNamer *apiopenapi.DefinitionNamer) *openapicommon.Config {
defaultConfig := DefaultOpenAPIConfig(getDefinitions, defNamer)
func DefaultOpenAPIV3Config(getDefinitions openapicommon.GetOpenAPIDefinitions, defNamer *apiopenapi.DefinitionNamer) *openapicommon.OpenAPIV3Config {
defaultConfig := &openapicommon.OpenAPIV3Config{
IgnorePrefixes: []string{},
Info: &spec.Info{
InfoProps: spec.InfoProps{
Title: "Generic API Server",
},
},
DefaultResponse: &spec3.Response{
ResponseProps: spec3.ResponseProps{
Description: "Default Response.",
},
},
GetOperationIDAndTags: apiopenapi.GetOperationIDAndTags,
GetDefinitionName: defNamer.GetDefinitionName,
GetDefinitions: getDefinitions,
}
defaultConfig.Definitions = getDefinitions(func(name string) spec.Ref {
defName, _ := defaultConfig.GetDefinitionName(name)
return spec.MustCreateRef("#/components/schemas/" + openapicommon.EscapeJsonPointer(defName))
@@ -552,6 +624,45 @@ func completeOpenAPI(config *openapicommon.Config, version *version.Info) {
}
}
func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Info) {
if config == nil {
return
}
if config.SecuritySchemes != nil {
// Setup OpenAPI security: all APIs will have the same authentication for now.
config.DefaultSecurity = []map[string][]string{}
keys := []string{}
for k := range config.SecuritySchemes {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
config.DefaultSecurity = append(config.DefaultSecurity, map[string][]string{k: {}})
}
if config.CommonResponses == nil {
config.CommonResponses = map[int]*spec3.Response{}
}
if _, exists := config.CommonResponses[http.StatusUnauthorized]; !exists {
config.CommonResponses[http.StatusUnauthorized] = &spec3.Response{
ResponseProps: spec3.ResponseProps{
Description: "Unauthorized",
},
}
}
}
// make sure we populate info, and info.version, if not manually set
if config.Info == nil {
config.Info = &spec.Info{}
}
if config.Info.Version == "" {
if version != nil {
config.Info.Version = strings.Split(version.String(), "-")[0]
} else {
config.Info.Version = "unversioned"
}
}
}
// DrainedNotify returns a lifecycle signal of genericapiserver already drained while shutting down.
func (c *Config) DrainedNotify() <-chan struct{} {
return c.lifecycleSignals.InFlightRequestsDrained.Signaled()
@@ -577,7 +688,7 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo
}
completeOpenAPI(c.OpenAPIConfig, c.Version)
completeOpenAPI(c.OpenAPIV3Config, c.Version)
completeOpenAPIV3(c.OpenAPIV3Config, c.Version)
if c.DiscoveryAddresses == nil {
c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress}
@@ -613,6 +724,12 @@ func (c *RecommendedConfig) Complete() CompletedConfig {
return c.Config.Complete(c.SharedInformerFactory)
}
var allowedMediaTypes = []string{
runtime.ContentTypeJSON,
runtime.ContentTypeYAML,
runtime.ContentTypeProtobuf,
}
// New creates a new server which logically combines the handling chain with the passed server.
// name is used to differentiate for logging. The handler chain in particular can be difficult as it starts delegating.
// delegationTarget may not be nil.
@@ -620,6 +737,18 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
if c.Serializer == nil {
return nil, fmt.Errorf("Genericapiserver.New() called with config.Serializer == nil")
}
for _, info := range c.Serializer.SupportedMediaTypes() {
var ok bool
for _, mt := range allowedMediaTypes {
if info.MediaType == mt {
ok = true
break
}
}
if !ok {
return nil, fmt.Errorf("refusing to create new apiserver %q with support for media type %q (allowed media types are: %s)", name, info.MediaType, strings.Join(allowedMediaTypes, ", "))
}
}
if c.LoopbackClientConfig == nil {
return nil, fmt.Errorf("Genericapiserver.New() called with config.LoopbackClientConfig == nil")
}
@@ -631,28 +760,36 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
return c.BuildHandlerChainFunc(handler, c.Config)
}
var debugSocket *routes.DebugSocket
if c.DebugSocketPath != "" {
debugSocket = routes.NewDebugSocket(c.DebugSocketPath)
}
apiServerHandler := NewAPIServerHandler(name, c.Serializer, handlerChainBuilder, delegationTarget.UnprotectedHandler())
s := &GenericAPIServer{
discoveryAddresses: c.DiscoveryAddresses,
LoopbackClientConfig: c.LoopbackClientConfig,
legacyAPIGroupPrefixes: c.LegacyAPIGroupPrefixes,
admissionControl: c.AdmissionControl,
Serializer: c.Serializer,
AuditBackend: c.AuditBackend,
Authorizer: c.Authorization.Authorizer,
delegationTarget: delegationTarget,
EquivalentResourceRegistry: c.EquivalentResourceRegistry,
HandlerChainWaitGroup: c.HandlerChainWaitGroup,
Handler: apiServerHandler,
discoveryAddresses: c.DiscoveryAddresses,
LoopbackClientConfig: c.LoopbackClientConfig,
legacyAPIGroupPrefixes: c.LegacyAPIGroupPrefixes,
admissionControl: c.AdmissionControl,
Serializer: c.Serializer,
AuditBackend: c.AuditBackend,
Authorizer: c.Authorization.Authorizer,
delegationTarget: delegationTarget,
EquivalentResourceRegistry: c.EquivalentResourceRegistry,
NonLongRunningRequestWaitGroup: c.NonLongRunningRequestWaitGroup,
WatchRequestWaitGroup: c.WatchRequestWaitGroup,
Handler: apiServerHandler,
UnprotectedDebugSocket: debugSocket,
listedPathProvider: apiServerHandler,
minRequestTimeout: time.Duration(c.MinRequestTimeout) * time.Second,
ShutdownTimeout: c.RequestTimeout,
ShutdownDelayDuration: c.ShutdownDelayDuration,
SecureServingInfo: c.SecureServing,
ExternalAddress: c.ExternalAddress,
minRequestTimeout: time.Duration(c.MinRequestTimeout) * time.Second,
ShutdownTimeout: c.RequestTimeout,
ShutdownDelayDuration: c.ShutdownDelayDuration,
ShutdownWatchTerminationGracePeriod: c.ShutdownWatchTerminationGracePeriod,
SecureServingInfo: c.SecureServing,
ExternalAddress: c.ExternalAddress,
openAPIConfig: c.OpenAPIConfig,
openAPIV3Config: c.OpenAPIV3Config,
@@ -686,10 +823,10 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
manager := c.AggregatedDiscoveryGroupManager
if manager == nil {
manager = discoveryendpoint.NewResourceManager()
manager = discoveryendpoint.NewResourceManager("apis")
}
s.AggregatedDiscoveryGroupManager = manager
s.AggregatedLegacyDiscoveryGroupManager = discoveryendpoint.NewResourceManager()
s.AggregatedLegacyDiscoveryGroupManager = discoveryendpoint.NewResourceManager("api")
}
for {
if c.JSONPatchMaxCopyBytes <= 0 {
@@ -840,16 +977,18 @@ func BuildHandlerChainWithStorageVersionPrecondition(apiHandler http.Handler, c
}
func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
handler := filterlatency.TrackCompleted(apiHandler)
handler := apiHandler
handler = filterlatency.TrackCompleted(handler)
handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer)
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authorization")
if c.FlowControl != nil {
workEstimatorCfg := flowcontrolrequest.DefaultWorkEstimatorConfig()
requestWorkEstimator := flowcontrolrequest.NewWorkEstimator(
c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg)
c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg, c.FlowControl.GetMaxSeats)
handler = filterlatency.TrackCompleted(handler)
handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, requestWorkEstimator)
handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, requestWorkEstimator, c.RequestTimeout/4)
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "priorityandfairness")
} else {
handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
@@ -868,7 +1007,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
failedHandler = filterlatency.TrackCompleted(failedHandler)
handler = filterlatency.TrackCompleted(handler)
handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences)
handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences, c.Authentication.RequestHeaderConfig)
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authentication")
handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")
@@ -879,7 +1018,10 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
handler = genericapifilters.WithRequestDeadline(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator,
c.LongRunningFunc, c.Serializer, c.RequestTimeout)
handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.HandlerChainWaitGroup)
handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.NonLongRunningRequestWaitGroup)
if c.ShutdownWatchTerminationGracePeriod > 0 {
handler = genericfilters.WithWatchTerminationDuringShutdown(handler, c.lifecycleSignals, c.WatchRequestWaitGroup)
}
if c.SecureServing != nil && !c.SecureServing.DisableHTTP2 && c.GoawayChance > 0 {
handler = genericfilters.WithProbabilisticGoaway(handler, c.GoawayChance)
}
@@ -914,18 +1056,21 @@ func installAPI(s *GenericAPIServer, c *Config) {
// so far, only logging related endpoints are considered valid to add for these debug flags.
routes.DebugFlags{}.Install(s.Handler.NonGoRestfulMux, "v", routes.StringFlagPutHandler(logs.GlogSetter))
}
if s.UnprotectedDebugSocket != nil {
s.UnprotectedDebugSocket.InstallProfiling()
s.UnprotectedDebugSocket.InstallDebugFlag("v", routes.StringFlagPutHandler(logs.GlogSetter))
if c.EnableContentionProfiling {
goruntime.SetBlockProfileRate(1)
}
}
if c.EnableMetrics {
if c.EnableProfiling {
routes.MetricsWithReset{}.Install(s.Handler.NonGoRestfulMux)
if utilfeature.DefaultFeatureGate.Enabled(features.ComponentSLIs) {
slis.SLIMetricsWithReset{}.Install(s.Handler.NonGoRestfulMux)
}
slis.SLIMetricsWithReset{}.Install(s.Handler.NonGoRestfulMux)
} else {
routes.DefaultMetrics{}.Install(s.Handler.NonGoRestfulMux)
if utilfeature.DefaultFeatureGate.Enabled(features.ComponentSLIs) {
slis.SLIMetrics{}.Install(s.Handler.NonGoRestfulMux)
}
slis.SLIMetrics{}.Install(s.Handler.NonGoRestfulMux)
}
}
@@ -939,7 +1084,7 @@ func installAPI(s *GenericAPIServer, c *Config) {
s.Handler.GoRestfulContainer.Add(s.DiscoveryGroupManager.WebService())
}
}
if c.FlowControl != nil && utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) {
if c.FlowControl != nil {
c.FlowControl.Install(s.Handler.NonGoRestfulMux)
}
}
@@ -1003,3 +1148,12 @@ func AuthorizeClientBearerToken(loopback *restclient.Config, authn *Authenticati
tokenAuthenticator := authenticatorfactory.NewFromTokens(tokens, authn.APIAudiences)
authn.Authenticator = authenticatorunion.New(tokenAuthenticator, authn.Authenticator)
}
// For testing purpose only
func SetHostnameFuncForTests(name string) {
hostnameFunc = func() (host string, err error) {
host = name
err = nil
return
}
}