feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions

View File

@@ -18,7 +18,7 @@ package manager
import (
"context"
"crypto/tls"
"errors"
"fmt"
"net"
"net/http"
@@ -26,25 +26,27 @@ import (
"time"
"github.com/go-logr/logr"
coordinationv1 "k8s.io/api/coordination/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"k8s.io/client-go/tools/record"
"k8s.io/utils/pointer"
"k8s.io/utils/ptr"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/cluster"
"sigs.k8s.io/controller-runtime/pkg/config" //nolint:staticcheck
"sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" //nolint:staticcheck
"sigs.k8s.io/controller-runtime/pkg/config"
"sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
"sigs.k8s.io/controller-runtime/pkg/healthz"
intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder"
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/metrics"
"sigs.k8s.io/controller-runtime/pkg/recorder"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/webhook"
)
@@ -55,8 +57,7 @@ type Manager interface {
cluster.Cluster
// Add will set requested dependencies on the component, and cause the component to be
// started when Start is called. Add will inject any dependencies for which the argument
// implements the inject interface - e.g. inject.Client.
// started when Start is called.
// Depending on if a Runnable implements LeaderElectionRunnable interface, a Runnable can be run in either
// non-leaderelection mode (always running) or leader election mode (managed by leader election if enabled).
Add(Runnable) error
@@ -66,13 +67,6 @@ type Manager interface {
// election was configured.
Elected() <-chan struct{}
// AddMetricsExtraHandler adds an extra handler served on path to the http server that serves metrics.
// Might be useful to register some diagnostic endpoints e.g. pprof. Note that these endpoints meant to be
// sensitive and shouldn't be exposed publicly.
// If the simple path -> handler mapping offered here is not enough, a new http server/listener should be added as
// Runnable to the manager via Add method.
AddMetricsExtraHandler(path string, handler http.Handler) error
// AddHealthzCheck allows you to add Healthz checker
AddHealthzCheck(name string, check healthz.Checker) error
@@ -88,17 +82,13 @@ type Manager interface {
Start(ctx context.Context) error
// GetWebhookServer returns a webhook.Server
GetWebhookServer() *webhook.Server
GetWebhookServer() webhook.Server
// GetLogger returns this manager's logger.
GetLogger() logr.Logger
// GetControllerOptions returns controller global configuration options.
//
// Deprecated: In a future version, the returned value is going to be replaced with a
// different type that doesn't rely on component configuration types.
// This is a temporary warning, and no action is needed as of today.
GetControllerOptions() v1alpha1.ControllerConfigurationSpec //nolint:staticcheck
GetControllerOptions() config.Controller
}
// Options are the arguments for creating a new Manager.
@@ -106,37 +96,44 @@ type Options struct {
// Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources.
// Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better
// to pass your own scheme in. See the documentation in pkg/scheme for more information.
//
// If set, the Scheme will be used to create the default Client and Cache.
Scheme *runtime.Scheme
// MapperProvider provides the rest mapper used to map go types to Kubernetes APIs
MapperProvider func(c *rest.Config) (meta.RESTMapper, error)
// MapperProvider provides the rest mapper used to map go types to Kubernetes APIs.
//
// If set, the RESTMapper returned by this function is used to create the RESTMapper
// used by the Client and Cache.
MapperProvider func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error)
// SyncPeriod determines the minimum frequency at which watched resources are
// reconciled. A lower period will correct entropy more quickly, but reduce
// responsiveness to change if there are many watched resources. Change this
// value only if you know what you are doing. Defaults to 10 hours if unset.
// there will a 10 percent jitter between the SyncPeriod of all controllers
// so that all controllers will not send list requests simultaneously.
// Cache is the cache.Options that will be used to create the default Cache.
// By default, the cache will watch and list requested objects in all namespaces.
Cache cache.Options
// NewCache is the function that will create the cache to be used
// by the manager. If not set this will use the default new cache function.
//
// This applies to all controllers.
// When using a custom NewCache, the Cache options will be passed to the
// NewCache function.
//
// A period sync happens for two reasons:
// 1. To insure against a bug in the controller that causes an object to not
// be requeued, when it otherwise should be requeued.
// 2. To insure against an unknown bug in controller-runtime, or its dependencies,
// that causes an object to not be requeued, when it otherwise should be
// requeued, or to be removed from the queue, when it otherwise should not
// be removed.
// NOTE: LOW LEVEL PRIMITIVE!
// Only use a custom NewCache if you know what you are doing.
NewCache cache.NewCacheFunc
// Client is the client.Options that will be used to create the default Client.
// By default, the client will use the cache for reads and direct calls for writes.
Client client.Options
// NewClient is the func that creates the client to be used by the manager.
// If not set this will create a Client backed by a Cache for read operations
// and a direct Client for write operations.
//
// If you want
// 1. to insure against missed watch events, or
// 2. to poll services that cannot be watched,
// then we recommend that, instead of changing the default period, the
// controller requeue, with a constant duration `t`, whenever the controller
// is "done" with an object, and would otherwise not requeue it, i.e., we
// recommend the `Reconcile` function return `reconcile.Result{RequeueAfter: t}`,
// instead of `reconcile.Result{}`.
SyncPeriod *time.Duration
// When using a custom NewClient, the Client options will be passed to the
// NewClient function.
//
// NOTE: LOW LEVEL PRIMITIVE!
// Only use a custom NewClient if you know what you are doing.
NewClient client.NewClientFunc
// Logger is the logger that should be used by this manager.
// If none is set, it defaults to log.Log global logger.
@@ -208,25 +205,17 @@ type Options struct {
// wait to force acquire leadership. This is measured against time of
// last observed ack. Default is 15 seconds.
LeaseDuration *time.Duration
// RenewDeadline is the duration that the acting controlplane will retry
// refreshing leadership before giving up. Default is 10 seconds.
RenewDeadline *time.Duration
// RetryPeriod is the duration the LeaderElector clients should wait
// between tries of actions. Default is 2 seconds.
RetryPeriod *time.Duration
// Namespace, if specified, restricts the manager's cache to watch objects in
// the desired namespace. Defaults to all namespaces.
//
// Note: If a namespace is specified, controllers can still Watch for a
// cluster-scoped resource (e.g Node). For namespaced resources, the cache
// will only hold objects from the desired namespace.
Namespace string
// MetricsBindAddress is the TCP address that the controller should bind to
// for serving prometheus metrics.
// It can be set to "0" to disable the metrics serving.
MetricsBindAddress string
// Metrics are the metricsserver.Options that will be used to create the metricsserver.Server.
Metrics metricsserver.Options
// HealthProbeBindAddress is the TCP address that the controller should bind to
// for serving health probes
@@ -239,52 +228,23 @@ type Options struct {
// Liveness probe endpoint name, defaults to "healthz"
LivenessEndpointName string
// Port is the port that the webhook server serves at.
// It is used to set webhook.Server.Port if WebhookServer is not set.
Port int
// Host is the hostname that the webhook server binds to.
// It is used to set webhook.Server.Host if WebhookServer is not set.
Host string
// CertDir is the directory that contains the server key and certificate.
// If not set, webhook server would look up the server key and certificate in
// {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate
// must be named tls.key and tls.crt, respectively.
// It is used to set webhook.Server.CertDir if WebhookServer is not set.
CertDir string
// TLSOpts is used to allow configuring the TLS config used for the webhook server.
TLSOpts []func(*tls.Config)
// PprofBindAddress is the TCP address that the controller should bind to
// for serving pprof.
// It can be set to "" or "0" to disable the pprof serving.
// Since pprof may contain sensitive information, make sure to protect it
// before exposing it to public.
PprofBindAddress string
// WebhookServer is an externally configured webhook.Server. By default,
// a Manager will create a default server using Port, Host, and CertDir;
// if this is set, the Manager will use this server instead.
WebhookServer *webhook.Server
// Functions to allow for a user to customize values that will be injected.
// NewCache is the function that will create the cache to be used
// by the manager. If not set this will use the default new cache function.
NewCache cache.NewCacheFunc
// NewClient is the func that creates the client to be used by the manager.
// If not set this will create the default DelegatingClient that will
// use the cache for reads and the client for writes.
NewClient cluster.NewClientFunc
// a Manager will create a server via webhook.NewServer with default settings.
// If this is set, the Manager will use this server instead.
WebhookServer webhook.Server
// BaseContext is the function that provides Context values to Runnables
// managed by the Manager. If a BaseContext function isn't provided, Runnables
// will receive a new Background Context instead.
BaseContext BaseContextFunc
// ClientDisableCacheFor tells the client that, if any cache is used, to bypass it
// for the given objects.
ClientDisableCacheFor []client.Object
// DryRunClient specifies whether the client should be configured to enforce
// dryRun mode.
DryRunClient bool
// EventBroadcaster records Events emitted by the manager and sends them to the Kubernetes API
// Use this to customize the event correlator and spam filter
//
@@ -301,11 +261,7 @@ type Options struct {
// Controller contains global configuration options for controllers
// registered within this manager.
// +optional
//
// Deprecated: In a future version, the type of this field is going to be replaced with a
// different struct that doesn't rely on component configuration types.
// This is a temporary warning, and no action is needed as of today.
Controller v1alpha1.ControllerConfigurationSpec //nolint:staticcheck
Controller config.Controller
// makeBroadcaster allows deferring the creation of the broadcaster to
// avoid leaking goroutines if we never call Start on this manager. It also
@@ -314,10 +270,11 @@ type Options struct {
makeBroadcaster intrec.EventBroadcasterProducer
// Dependency injection for testing
newRecorderProvider func(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error)
newRecorderProvider func(config *rest.Config, httpClient *http.Client, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error)
newResourceLock func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error)
newMetricsListener func(addr string) (net.Listener, error)
newMetricsServer func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error)
newHealthProbeListener func(addr string) (net.Listener, error)
newPprofListener func(addr string) (net.Listener, error)
}
// BaseContextFunc is a function used to provide a base Context to Runnables
@@ -352,7 +309,13 @@ type LeaderElectionRunnable interface {
}
// New returns a new Manager for creating Controllers.
// Note that if ContentType in the given config is not set, "application/vnd.kubernetes.protobuf"
// will be used for all built-in resources of Kubernetes, and "application/json" is for other types
// including all CRD resources.
func New(config *rest.Config, options Options) (Manager, error) {
if config == nil {
return nil, errors.New("must specify Config")
}
// Set default values for options fields
options = setOptionsDefaults(options)
@@ -360,22 +323,25 @@ func New(config *rest.Config, options Options) (Manager, error) {
clusterOptions.Scheme = options.Scheme
clusterOptions.MapperProvider = options.MapperProvider
clusterOptions.Logger = options.Logger
clusterOptions.SyncPeriod = options.SyncPeriod
clusterOptions.Namespace = options.Namespace
clusterOptions.NewCache = options.NewCache
clusterOptions.NewClient = options.NewClient
clusterOptions.ClientDisableCacheFor = options.ClientDisableCacheFor
clusterOptions.DryRunClient = options.DryRunClient
clusterOptions.Cache = options.Cache
clusterOptions.Client = options.Client
clusterOptions.EventBroadcaster = options.EventBroadcaster //nolint:staticcheck
})
if err != nil {
return nil, err
}
config = rest.CopyConfig(config)
if config.UserAgent == "" {
config.UserAgent = rest.DefaultKubernetesUserAgent()
}
// Create the recorder provider to inject event recorders for the components.
// TODO(directxman12): the log for the event provider should have a context (name, tags, etc) specific
// to the particular controller that it's being injected into, rather than a generic one like is here.
recorderProvider, err := options.newRecorderProvider(config, cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster)
recorderProvider, err := options.newRecorderProvider(config, cluster.GetHTTPClient(), cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster)
if err != nil {
return nil, err
}
@@ -389,7 +355,20 @@ func New(config *rest.Config, options Options) (Manager, error) {
leaderRecorderProvider = recorderProvider
} else {
leaderConfig = rest.CopyConfig(options.LeaderElectionConfig)
leaderRecorderProvider, err = options.newRecorderProvider(leaderConfig, cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster)
scheme := cluster.GetScheme()
err := corev1.AddToScheme(scheme)
if err != nil {
return nil, err
}
err = coordinationv1.AddToScheme(scheme)
if err != nil {
return nil, err
}
httpClient, err := rest.HTTPClientFor(options.LeaderElectionConfig)
if err != nil {
return nil, err
}
leaderRecorderProvider, err = options.newRecorderProvider(leaderConfig, httpClient, scheme, options.Logger.WithName("events"), options.makeBroadcaster)
if err != nil {
return nil, err
}
@@ -410,16 +389,12 @@ func New(config *rest.Config, options Options) (Manager, error) {
}
}
// Create the metrics listener. This will throw an error if the metrics bind
// address is invalid or already in use.
metricsListener, err := options.newMetricsListener(options.MetricsBindAddress)
// Create the metrics server.
metricsServer, err := options.newMetricsServer(options.Metrics, config, cluster.GetHTTPClient())
if err != nil {
return nil, err
}
// By default we have no extra endpoints to expose on metrics http server.
metricsExtraHandlers := make(map[string]http.Handler)
// Create health probes listener. This will throw an error if the bind
// address is invalid or already in use.
healthProbeListener, err := options.newHealthProbeListener(options.HealthProbeBindAddress)
@@ -427,25 +402,26 @@ func New(config *rest.Config, options Options) (Manager, error) {
return nil, err
}
errChan := make(chan error)
runnables := newRunnables(options.BaseContext, errChan)
// Create pprof listener. This will throw an error if the bind
// address is invalid or already in use.
pprofListener, err := options.newPprofListener(options.PprofBindAddress)
if err != nil {
return nil, fmt.Errorf("failed to new pprof listener: %w", err)
}
errChan := make(chan error, 1)
runnables := newRunnables(options.BaseContext, errChan)
return &controllerManager{
stopProcedureEngaged: pointer.Int64(0),
stopProcedureEngaged: ptr.To(int64(0)),
cluster: cluster,
runnables: runnables,
errChan: errChan,
recorderProvider: recorderProvider,
resourceLock: resourceLock,
metricsListener: metricsListener,
metricsExtraHandlers: metricsExtraHandlers,
controllerOptions: options.Controller,
metricsServer: metricsServer,
controllerConfig: options.Controller,
logger: options.Logger,
elected: make(chan struct{}),
port: options.Port,
host: options.Host,
certDir: options.CertDir,
tlsOpts: options.TLSOpts,
webhookServer: options.WebhookServer,
leaderElectionID: options.LeaderElectionID,
leaseDuration: *options.LeaseDuration,
@@ -454,6 +430,7 @@ func New(config *rest.Config, options Options) (Manager, error) {
healthProbeListener: healthProbeListener,
readinessEndpointName: options.ReadinessEndpointName,
livenessEndpointName: options.LivenessEndpointName,
pprofListener: pprofListener,
gracefulShutdownTimeout: *options.GracefulShutdownTimeout,
internalProceduresStop: make(chan struct{}),
leaderElectionStopped: make(chan struct{}),
@@ -465,15 +442,13 @@ func New(config *rest.Config, options Options) (Manager, error) {
// any options already set on Options will be ignored, this is used to allow
// cli flags to override anything specified in the config file.
//
// Deprecated: This method will be removed in a future release.
// Deprecated: This function has been deprecated and will be removed in a future release,
// The Component Configuration package has been unmaintained for over a year and is no longer
// actively developed. Users should migrate to their own configuration format
// and configure Manager.Options directly.
// See https://github.com/kubernetes-sigs/controller-runtime/issues/895
// for more information, feedback, and comments.
func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options, error) {
if inj, wantsScheme := loader.(inject.Scheme); wantsScheme {
err := inj.InjectScheme(o.Scheme)
if err != nil {
return o, err
}
}
newObj, err := loader.Complete()
if err != nil {
return o, err
@@ -481,16 +456,16 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options,
o = o.setLeaderElectionConfig(newObj)
if o.SyncPeriod == nil && newObj.SyncPeriod != nil {
o.SyncPeriod = &newObj.SyncPeriod.Duration
if o.Cache.SyncPeriod == nil && newObj.SyncPeriod != nil {
o.Cache.SyncPeriod = &newObj.SyncPeriod.Duration
}
if o.Namespace == "" && newObj.CacheNamespace != "" {
o.Namespace = newObj.CacheNamespace
if len(o.Cache.DefaultNamespaces) == 0 && newObj.CacheNamespace != "" {
o.Cache.DefaultNamespaces = map[string]cache.Config{newObj.CacheNamespace: {}}
}
if o.MetricsBindAddress == "" && newObj.Metrics.BindAddress != "" {
o.MetricsBindAddress = newObj.Metrics.BindAddress
if o.Metrics.BindAddress == "" && newObj.Metrics.BindAddress != "" {
o.Metrics.BindAddress = newObj.Metrics.BindAddress
}
if o.HealthProbeBindAddress == "" && newObj.Health.HealthProbeBindAddress != "" {
@@ -505,21 +480,21 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options,
o.LivenessEndpointName = newObj.Health.LivenessEndpointName
}
if o.Port == 0 && newObj.Webhook.Port != nil {
o.Port = *newObj.Webhook.Port
}
if o.Host == "" && newObj.Webhook.Host != "" {
o.Host = newObj.Webhook.Host
}
if o.CertDir == "" && newObj.Webhook.CertDir != "" {
o.CertDir = newObj.Webhook.CertDir
if o.WebhookServer == nil {
port := 0
if newObj.Webhook.Port != nil {
port = *newObj.Webhook.Port
}
o.WebhookServer = webhook.NewServer(webhook.Options{
Port: port,
Host: newObj.Webhook.Host,
CertDir: newObj.Webhook.CertDir,
})
}
if newObj.Controller != nil {
if o.Controller.CacheSyncTimeout == nil && newObj.Controller.CacheSyncTimeout != nil {
o.Controller.CacheSyncTimeout = newObj.Controller.CacheSyncTimeout
if o.Controller.CacheSyncTimeout == 0 && newObj.Controller.CacheSyncTimeout != nil {
o.Controller.CacheSyncTimeout = *newObj.Controller.CacheSyncTimeout
}
if len(o.Controller.GroupKindConcurrency) == 0 && len(newObj.Controller.GroupKindConcurrency) > 0 {
@@ -532,7 +507,12 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options,
// AndFromOrDie will use options.AndFrom() and will panic if there are errors.
//
// Deprecated: This method will be removed in a future release.
// Deprecated: This function has been deprecated and will be removed in a future release,
// The Component Configuration package has been unmaintained for over a year and is no longer
// actively developed. Users should migrate to their own configuration format
// and configure Manager.Options directly.
// See https://github.com/kubernetes-sigs/controller-runtime/issues/895
// for more information, feedback, and comments.
func (o Options) AndFromOrDie(loader config.ControllerManagerConfiguration) Options {
o, err := o.AndFrom(loader)
if err != nil {
@@ -541,7 +521,7 @@ func (o Options) AndFromOrDie(loader config.ControllerManagerConfiguration) Opti
return o
}
func (o Options) setLeaderElectionConfig(obj v1alpha1.ControllerManagerConfigurationSpec) Options { //nolint:staticcheck
func (o Options) setLeaderElectionConfig(obj v1alpha1.ControllerManagerConfigurationSpec) Options {
if obj.LeaderElection == nil {
// The source does not have any configuration; noop
return o
@@ -591,6 +571,19 @@ func defaultHealthProbeListener(addr string) (net.Listener, error) {
return ln, nil
}
// defaultPprofListener creates the default pprof listener bound to the given address.
func defaultPprofListener(addr string) (net.Listener, error) {
if addr == "" || addr == "0" {
return nil, nil
}
ln, err := net.Listen("tcp", addr)
if err != nil {
return nil, fmt.Errorf("error listening on %s: %w", addr, err)
}
return ln, nil
}
// defaultBaseContext is used as the BaseContext value in Options if one
// has not already been set.
func defaultBaseContext() context.Context {
@@ -623,8 +616,8 @@ func setOptionsDefaults(options Options) Options {
}
}
if options.newMetricsListener == nil {
options.newMetricsListener = metrics.NewListener
if options.newMetricsServer == nil {
options.newMetricsServer = metricsserver.NewServer
}
leaseDuration, renewDeadline, retryPeriod := defaultLeaseDuration, defaultRenewDeadline, defaultRetryPeriod
if options.LeaseDuration == nil {
@@ -651,6 +644,10 @@ func setOptionsDefaults(options Options) Options {
options.newHealthProbeListener = defaultHealthProbeListener
}
if options.newPprofListener == nil {
options.newPprofListener = defaultPprofListener
}
if options.GracefulShutdownTimeout == nil {
gracefulShutdownTimeout := defaultGracefulShutdownPeriod
options.GracefulShutdownTimeout = &gracefulShutdownTimeout
@@ -664,5 +661,9 @@ func setOptionsDefaults(options Options) Options {
options.BaseContext = defaultBaseContext
}
if options.WebhookServer == nil {
options.WebhookServer = webhook.NewServer(webhook.Options{})
}
return options
}