feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

vendor/sigs.k8s.io/controller-runtime/pkg/manager/internal.go

@@ -18,21 +18,19 @@ package manager
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"fmt"
 	"net"
 	"net/http"
+	"net/http/pprof"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/go-logr/logr"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/leaderelection"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
@@ -41,12 +39,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
-	"sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" //nolint:staticcheck
+	"sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/internal/httpserver"
 	intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder"
-	"sigs.k8s.io/controller-runtime/pkg/metrics"
-	"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
@@ -59,7 +56,6 @@ const (
 
 	defaultReadinessEndpoint = "/readyz"
 	defaultLivenessEndpoint  = "/healthz"
-	defaultMetricsEndpoint   = "/metrics"
 )
 
 var _ Runnable = &controllerManager{}
@@ -86,11 +82,8 @@ type controllerManager struct {
 	// on shutdown
 	leaderElectionReleaseOnCancel bool
 
-	// metricsListener is used to serve prometheus metrics
-	metricsListener net.Listener
-
-	// metricsExtraHandlers contains extra handlers to register on http server that serves metrics.
-	metricsExtraHandlers map[string]http.Handler
+	// metricsServer is used to serve prometheus metrics
+	metricsServer metricsserver.Server
 
 	// healthProbeListener is used to serve liveness probe
 	healthProbeListener net.Listener
@@ -107,8 +100,11 @@ type controllerManager struct {
 	// Healthz probe handler
 	healthzHandler *healthz.Handler
 
-	// controllerOptions are the global controller options.
-	controllerOptions v1alpha1.ControllerConfigurationSpec //nolint:staticcheck
+	// pprofListener is used to serve pprof
+	pprofListener net.Listener
+
+	// controllerConfig are the global controller options.
+	controllerConfig config.Controller
 
 	// Logger is the logger that should be used by this manager.
 	// If none is set, it defaults to log.Log global logger.
@@ -128,18 +124,7 @@ type controllerManager struct {
 	// election was configured.
 	elected chan struct{}
 
-	// port is the port that the webhook server serves at.
-	port int
-	// host is the hostname that the webhook server binds to.
-	host string
-	// CertDir is the directory that contains the server key and certificate.
-	// if not set, webhook server would look up the server key and certificate in
-	// {TempDir}/k8s-webhook-server/serving-certs
-	certDir string
-	// tlsOpts is used to allow configuring the TLS config used for the webhook server.
-	tlsOpts []func(*tls.Config)
-
-	webhookServer *webhook.Server
+	webhookServer webhook.Server
 	// webhookServerOnce will be called in GetWebhookServer() to optionally initialize
 	// webhookServer if unset, and Add() it to controllerManager.
 	webhookServerOnce sync.Once
@@ -191,53 +176,9 @@ func (cm *controllerManager) Add(r Runnable) error {
 }
 
 func (cm *controllerManager) add(r Runnable) error {
-	// Set dependencies on the object
-	if err := cm.SetFields(r); err != nil {
-		return err
-	}
 	return cm.runnables.Add(r)
 }
 
-// Deprecated: use the equivalent Options field to set a field. This method will be removed in v0.10.
-func (cm *controllerManager) SetFields(i interface{}) error {
-	if err := cm.cluster.SetFields(i); err != nil {
-		return err
-	}
-	if _, err := inject.InjectorInto(cm.SetFields, i); err != nil {
-		return err
-	}
-	if _, err := inject.StopChannelInto(cm.internalProceduresStop, i); err != nil {
-		return err
-	}
-	if _, err := inject.LoggerInto(cm.logger, i); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// AddMetricsExtraHandler adds extra handler served on path to the http server that serves metrics.
-func (cm *controllerManager) AddMetricsExtraHandler(path string, handler http.Handler) error {
-	cm.Lock()
-	defer cm.Unlock()
-
-	if cm.started {
-		return fmt.Errorf("unable to add new metrics handler because metrics endpoint has already been created")
-	}
-
-	if path == defaultMetricsEndpoint {
-		return fmt.Errorf("overriding builtin %s endpoint is not allowed", defaultMetricsEndpoint)
-	}
-
-	if _, found := cm.metricsExtraHandlers[path]; found {
-		return fmt.Errorf("can't register extra handler by duplicate path %q on metrics http server", path)
-	}
-
-	cm.metricsExtraHandlers[path] = handler
-	cm.logger.V(2).Info("Registering metrics http server extra handler", "path", path)
-	return nil
-}
-
 // AddHealthzCheck allows you to add Healthz checker.
 func (cm *controllerManager) AddHealthzCheck(name string, check healthz.Checker) error {
 	cm.Lock()
@@ -272,6 +213,10 @@ func (cm *controllerManager) AddReadyzCheck(name string, check healthz.Checker)
 	return nil
 }
 
+func (cm *controllerManager) GetHTTPClient() *http.Client {
+	return cm.cluster.GetHTTPClient()
+}
+
 func (cm *controllerManager) GetConfig() *rest.Config {
 	return cm.cluster.GetConfig()
 }
@@ -304,15 +249,10 @@ func (cm *controllerManager) GetAPIReader() client.Reader {
 	return cm.cluster.GetAPIReader()
 }
 
-func (cm *controllerManager) GetWebhookServer() *webhook.Server {
+func (cm *controllerManager) GetWebhookServer() webhook.Server {
 	cm.webhookServerOnce.Do(func() {
 		if cm.webhookServer == nil {
-			cm.webhookServer = &webhook.Server{
-				Port:    cm.port,
-				Host:    cm.host,
-				CertDir: cm.certDir,
-				TLSOpts: cm.tlsOpts,
-			}
+			panic("webhook should not be nil")
 		}
 		if err := cm.Add(cm.webhookServer); err != nil {
 			panic(fmt.Sprintf("unable to add webhook server to the controller manager: %s", err))
@@ -325,28 +265,13 @@ func (cm *controllerManager) GetLogger() logr.Logger {
 	return cm.logger
 }
 
-func (cm *controllerManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec { //nolint:staticcheck
-	return cm.controllerOptions
+func (cm *controllerManager) GetControllerOptions() config.Controller {
+	return cm.controllerConfig
 }
 
-func (cm *controllerManager) serveMetrics() {
-	handler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{
-		ErrorHandling: promhttp.HTTPErrorOnError,
-	})
-	// TODO(JoelSpeed): Use existing Kubernetes machinery for serving metrics
+func (cm *controllerManager) addHealthProbeServer() error {
 	mux := http.NewServeMux()
-	mux.Handle(defaultMetricsEndpoint, handler)
-	for path, extraHandler := range cm.metricsExtraHandlers {
-		mux.Handle(path, extraHandler)
-	}
-
-	server := httpserver.New(mux)
-	go cm.httpServe("metrics", cm.logger.WithValues("path", defaultMetricsEndpoint), server, cm.metricsListener)
-}
-
-func (cm *controllerManager) serveHealthProbes() {
-	mux := http.NewServeMux()
-	server := httpserver.New(mux)
+	srv := httpserver.New(mux)
 
 	if cm.readyzHandler != nil {
 		mux.Handle(cm.readinessEndpointName, http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler))
@@ -359,43 +284,30 @@ func (cm *controllerManager) serveHealthProbes() {
 		mux.Handle(cm.livenessEndpointName+"/", http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler))
 	}
 
-	go cm.httpServe("health probe", cm.logger, server, cm.healthProbeListener)
+	return cm.add(&server{
+		Kind:     "health probe",
+		Log:      cm.logger,
+		Server:   srv,
+		Listener: cm.healthProbeListener,
+	})
 }
 
-func (cm *controllerManager) httpServe(kind string, log logr.Logger, server *http.Server, ln net.Listener) {
-	log = log.WithValues("kind", kind, "addr", ln.Addr())
+func (cm *controllerManager) addPprofServer() error {
+	mux := http.NewServeMux()
+	srv := httpserver.New(mux)
 
-	go func() {
-		log.Info("Starting server")
-		if err := server.Serve(ln); err != nil {
-			if errors.Is(err, http.ErrServerClosed) {
-				return
-			}
-			if atomic.LoadInt64(cm.stopProcedureEngaged) > 0 {
-				// There might be cases where connections are still open and we try to shutdown
-				// but not having enough time to close the connection causes an error in Serve
-				//
-				// In that case we want to avoid returning an error to the main error channel.
-				log.Error(err, "error on Serve after stop has been engaged")
-				return
-			}
-			cm.errChan <- err
-		}
-	}()
+	mux.HandleFunc("/debug/pprof/", pprof.Index)
+	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+	mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
 
-	// Shutdown the server when stop is closed.
-	<-cm.internalProceduresStop
-	if err := server.Shutdown(cm.shutdownCtx); err != nil {
-		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
-			// Avoid logging context related errors.
-			return
-		}
-		if atomic.LoadInt64(cm.stopProcedureEngaged) > 0 {
-			cm.logger.Error(err, "error on Shutdown after stop has been engaged")
-			return
-		}
-		cm.errChan <- err
-	}
+	return cm.add(&server{
+		Kind:     "pprof",
+		Log:      cm.logger,
+		Server:   srv,
+		Listener: cm.pprofListener,
+	})
 }
 
 // Start starts the manager and waits indefinitely.
@@ -450,38 +362,61 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) {
 	// Metrics should be served whether the controller is leader or not.
 	// (If we don't serve metrics for non-leaders, prometheus will still scrape
 	// the pod but will get a connection refused).
-	if cm.metricsListener != nil {
-		cm.serveMetrics()
+	if cm.metricsServer != nil {
+		// Note: We are adding the metrics server directly to HTTPServers here as matching on the
+		// metricsserver.Server interface in cm.runnables.Add would be very brittle.
+		if err := cm.runnables.HTTPServers.Add(cm.metricsServer, nil); err != nil {
+			return fmt.Errorf("failed to add metrics server: %w", err)
+		}
 	}
 
 	// Serve health probes.
 	if cm.healthProbeListener != nil {
-		cm.serveHealthProbes()
+		if err := cm.addHealthProbeServer(); err != nil {
+			return fmt.Errorf("failed to add health probe server: %w", err)
+		}
 	}
 
-	// First start any webhook servers, which includes conversion, validation, and defaulting
+	// Add pprof server
+	if cm.pprofListener != nil {
+		if err := cm.addPprofServer(); err != nil {
+			return fmt.Errorf("failed to add pprof server: %w", err)
+		}
+	}
+
+	// First start any internal HTTP servers, which includes health probes, metrics and profiling if enabled.
+	//
+	// WARNING: Internal HTTP servers MUST start before any cache is populated, otherwise it would block
+	// conversion webhooks to be ready for serving which make the cache never get ready.
+	if err := cm.runnables.HTTPServers.Start(cm.internalCtx); err != nil {
+		if err != nil {
+			return fmt.Errorf("failed to start HTTP servers: %w", err)
+		}
+	}
+
+	// Start any webhook servers, which includes conversion, validation, and defaulting
 	// webhooks that are registered.
 	//
 	// WARNING: Webhooks MUST start before any cache is populated, otherwise there is a race condition
 	// between conversion webhooks and the cache sync (usually initial list) which causes the webhooks
 	// to never start because no cache can be populated.
 	if err := cm.runnables.Webhooks.Start(cm.internalCtx); err != nil {
-		if !errors.Is(err, wait.ErrWaitTimeout) {
-			return err
+		if err != nil {
+			return fmt.Errorf("failed to start webhooks: %w", err)
 		}
 	}
 
 	// Start and wait for caches.
 	if err := cm.runnables.Caches.Start(cm.internalCtx); err != nil {
-		if !errors.Is(err, wait.ErrWaitTimeout) {
-			return err
+		if err != nil {
+			return fmt.Errorf("failed to start caches: %w", err)
 		}
 	}
 
 	// Start the non-leaderelection Runnables after the cache has synced.
 	if err := cm.runnables.Others.Start(cm.internalCtx); err != nil {
-		if !errors.Is(err, wait.ErrWaitTimeout) {
-			return err
+		if err != nil {
+			return fmt.Errorf("failed to start other runnables: %w", err)
 		}
 	}
 
@@ -528,7 +463,12 @@ func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) e
 	//
 	// The shutdown context immediately expires if the gracefulShutdownTimeout is not set.
 	var shutdownCancel context.CancelFunc
-	cm.shutdownCtx, shutdownCancel = context.WithTimeout(context.Background(), cm.gracefulShutdownTimeout)
+	if cm.gracefulShutdownTimeout < 0 {
+		// We want to wait forever for the runnables to stop.
+		cm.shutdownCtx, shutdownCancel = context.WithCancel(context.Background())
+	} else {
+		cm.shutdownCtx, shutdownCancel = context.WithTimeout(context.Background(), cm.gracefulShutdownTimeout)
+	}
 	defer shutdownCancel()
 
 	// Start draining the errors before acquiring the lock to make sure we don't deadlock
@@ -586,10 +526,13 @@ func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) e
 		cm.logger.Info("Stopping and waiting for caches")
 		cm.runnables.Caches.StopAndWait(cm.shutdownCtx)
 
-		// Webhooks should come last, as they might be still serving some requests.
+		// Webhooks and internal HTTP servers should come last, as they might be still serving some requests.
 		cm.logger.Info("Stopping and waiting for webhooks")
 		cm.runnables.Webhooks.StopAndWait(cm.shutdownCtx)
 
+		cm.logger.Info("Stopping and waiting for HTTP servers")
+		cm.runnables.HTTPServers.StopAndWait(cm.shutdownCtx)
+
 		// Proceed to close the manager and overall shutdown context.
 		cm.logger.Info("Wait completed, proceeding to shutdown the manager")
 		shutdownCancel()

vendor/sigs.k8s.io/controller-runtime/pkg/manager/manager.go

@@ -18,7 +18,7 @@ package manager
 
 import (
 	"context"
-	"crypto/tls"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -26,25 +26,27 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
+	coordinationv1 "k8s.io/api/coordination/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/client-go/tools/record"
-	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
-	"sigs.k8s.io/controller-runtime/pkg/config"          //nolint:staticcheck
-	"sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" //nolint:staticcheck
+	"sigs.k8s.io/controller-runtime/pkg/config"
+	"sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder"
 	"sigs.k8s.io/controller-runtime/pkg/leaderelection"
 	"sigs.k8s.io/controller-runtime/pkg/log"
-	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/recorder"
-	"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
@@ -55,8 +57,7 @@ type Manager interface {
 	cluster.Cluster
 
 	// Add will set requested dependencies on the component, and cause the component to be
-	// started when Start is called.  Add will inject any dependencies for which the argument
-	// implements the inject interface - e.g. inject.Client.
+	// started when Start is called.
 	// Depending on if a Runnable implements LeaderElectionRunnable interface, a Runnable can be run in either
 	// non-leaderelection mode (always running) or leader election mode (managed by leader election if enabled).
 	Add(Runnable) error
@@ -66,13 +67,6 @@ type Manager interface {
 	// election was configured.
 	Elected() <-chan struct{}
 
-	// AddMetricsExtraHandler adds an extra handler served on path to the http server that serves metrics.
-	// Might be useful to register some diagnostic endpoints e.g. pprof. Note that these endpoints meant to be
-	// sensitive and shouldn't be exposed publicly.
-	// If the simple path -> handler mapping offered here is not enough, a new http server/listener should be added as
-	// Runnable to the manager via Add method.
-	AddMetricsExtraHandler(path string, handler http.Handler) error
-
 	// AddHealthzCheck allows you to add Healthz checker
 	AddHealthzCheck(name string, check healthz.Checker) error
 
@@ -88,17 +82,13 @@ type Manager interface {
 	Start(ctx context.Context) error
 
 	// GetWebhookServer returns a webhook.Server
-	GetWebhookServer() *webhook.Server
+	GetWebhookServer() webhook.Server
 
 	// GetLogger returns this manager's logger.
 	GetLogger() logr.Logger
 
 	// GetControllerOptions returns controller global configuration options.
-	//
-	// Deprecated: In a future version, the returned value is going to be replaced with a
-	// different type that doesn't rely on component configuration types.
-	// This is a temporary warning, and no action is needed as of today.
-	GetControllerOptions() v1alpha1.ControllerConfigurationSpec //nolint:staticcheck
+	GetControllerOptions() config.Controller
 }
 
 // Options are the arguments for creating a new Manager.
@@ -106,37 +96,44 @@ type Options struct {
 	// Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources.
 	// Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better
 	// to pass your own scheme in. See the documentation in pkg/scheme for more information.
+	//
+	// If set, the Scheme will be used to create the default Client and Cache.
 	Scheme *runtime.Scheme
 
-	// MapperProvider provides the rest mapper used to map go types to Kubernetes APIs
-	MapperProvider func(c *rest.Config) (meta.RESTMapper, error)
+	// MapperProvider provides the rest mapper used to map go types to Kubernetes APIs.
+	//
+	// If set, the RESTMapper returned by this function is used to create the RESTMapper
+	// used by the Client and Cache.
+	MapperProvider func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error)
 
-	// SyncPeriod determines the minimum frequency at which watched resources are
-	// reconciled. A lower period will correct entropy more quickly, but reduce
-	// responsiveness to change if there are many watched resources. Change this
-	// value only if you know what you are doing. Defaults to 10 hours if unset.
-	// there will a 10 percent jitter between the SyncPeriod of all controllers
-	// so that all controllers will not send list requests simultaneously.
+	// Cache is the cache.Options that will be used to create the default Cache.
+	// By default, the cache will watch and list requested objects in all namespaces.
+	Cache cache.Options
+
+	// NewCache is the function that will create the cache to be used
+	// by the manager. If not set this will use the default new cache function.
 	//
-	// This applies to all controllers.
+	// When using a custom NewCache, the Cache options will be passed to the
+	// NewCache function.
 	//
-	// A period sync happens for two reasons:
-	// 1. To insure against a bug in the controller that causes an object to not
-	// be requeued, when it otherwise should be requeued.
-	// 2. To insure against an unknown bug in controller-runtime, or its dependencies,
-	// that causes an object to not be requeued, when it otherwise should be
-	// requeued, or to be removed from the queue, when it otherwise should not
-	// be removed.
+	// NOTE: LOW LEVEL PRIMITIVE!
+	// Only use a custom NewCache if you know what you are doing.
+	NewCache cache.NewCacheFunc
+
+	// Client is the client.Options that will be used to create the default Client.
+	// By default, the client will use the cache for reads and direct calls for writes.
+	Client client.Options
+
+	// NewClient is the func that creates the client to be used by the manager.
+	// If not set this will create a Client backed by a Cache for read operations
+	// and a direct Client for write operations.
 	//
-	// If you want
-	// 1. to insure against missed watch events, or
-	// 2. to poll services that cannot be watched,
-	// then we recommend that, instead of changing the default period, the
-	// controller requeue, with a constant duration `t`, whenever the controller
-	// is "done" with an object, and would otherwise not requeue it, i.e., we
-	// recommend the `Reconcile` function return `reconcile.Result{RequeueAfter: t}`,
-	// instead of `reconcile.Result{}`.
-	SyncPeriod *time.Duration
+	// When using a custom NewClient, the Client options will be passed to the
+	// NewClient function.
+	//
+	// NOTE: LOW LEVEL PRIMITIVE!
+	// Only use a custom NewClient if you know what you are doing.
+	NewClient client.NewClientFunc
 
 	// Logger is the logger that should be used by this manager.
 	// If none is set, it defaults to log.Log global logger.
@@ -208,25 +205,17 @@ type Options struct {
 	// wait to force acquire leadership. This is measured against time of
 	// last observed ack. Default is 15 seconds.
 	LeaseDuration *time.Duration
+
 	// RenewDeadline is the duration that the acting controlplane will retry
 	// refreshing leadership before giving up. Default is 10 seconds.
 	RenewDeadline *time.Duration
+
 	// RetryPeriod is the duration the LeaderElector clients should wait
 	// between tries of actions. Default is 2 seconds.
 	RetryPeriod *time.Duration
 
-	// Namespace, if specified, restricts the manager's cache to watch objects in
-	// the desired namespace. Defaults to all namespaces.
-	//
-	// Note: If a namespace is specified, controllers can still Watch for a
-	// cluster-scoped resource (e.g Node). For namespaced resources, the cache
-	// will only hold objects from the desired namespace.
-	Namespace string
-
-	// MetricsBindAddress is the TCP address that the controller should bind to
-	// for serving prometheus metrics.
-	// It can be set to "0" to disable the metrics serving.
-	MetricsBindAddress string
+	// Metrics are the metricsserver.Options that will be used to create the metricsserver.Server.
+	Metrics metricsserver.Options
 
 	// HealthProbeBindAddress is the TCP address that the controller should bind to
 	// for serving health probes
@@ -239,52 +228,23 @@ type Options struct {
 	// Liveness probe endpoint name, defaults to "healthz"
 	LivenessEndpointName string
 
-	// Port is the port that the webhook server serves at.
-	// It is used to set webhook.Server.Port if WebhookServer is not set.
-	Port int
-	// Host is the hostname that the webhook server binds to.
-	// It is used to set webhook.Server.Host if WebhookServer is not set.
-	Host string
-
-	// CertDir is the directory that contains the server key and certificate.
-	// If not set, webhook server would look up the server key and certificate in
-	// {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate
-	// must be named tls.key and tls.crt, respectively.
-	// It is used to set webhook.Server.CertDir if WebhookServer is not set.
-	CertDir string
-
-	// TLSOpts is used to allow configuring the TLS config used for the webhook server.
-	TLSOpts []func(*tls.Config)
+	// PprofBindAddress is the TCP address that the controller should bind to
+	// for serving pprof.
+	// It can be set to "" or "0" to disable the pprof serving.
+	// Since pprof may contain sensitive information, make sure to protect it
+	// before exposing it to public.
+	PprofBindAddress string
 
 	// WebhookServer is an externally configured webhook.Server. By default,
-	// a Manager will create a default server using Port, Host, and CertDir;
-	// if this is set, the Manager will use this server instead.
-	WebhookServer *webhook.Server
-
-	// Functions to allow for a user to customize values that will be injected.
-
-	// NewCache is the function that will create the cache to be used
-	// by the manager. If not set this will use the default new cache function.
-	NewCache cache.NewCacheFunc
-
-	// NewClient is the func that creates the client to be used by the manager.
-	// If not set this will create the default DelegatingClient that will
-	// use the cache for reads and the client for writes.
-	NewClient cluster.NewClientFunc
+	// a Manager will create a server via webhook.NewServer with default settings.
+	// If this is set, the Manager will use this server instead.
+	WebhookServer webhook.Server
 
 	// BaseContext is the function that provides Context values to Runnables
 	// managed by the Manager. If a BaseContext function isn't provided, Runnables
 	// will receive a new Background Context instead.
 	BaseContext BaseContextFunc
 
-	// ClientDisableCacheFor tells the client that, if any cache is used, to bypass it
-	// for the given objects.
-	ClientDisableCacheFor []client.Object
-
-	// DryRunClient specifies whether the client should be configured to enforce
-	// dryRun mode.
-	DryRunClient bool
-
 	// EventBroadcaster records Events emitted by the manager and sends them to the Kubernetes API
 	// Use this to customize the event correlator and spam filter
 	//
@@ -301,11 +261,7 @@ type Options struct {
 	// Controller contains global configuration options for controllers
 	// registered within this manager.
 	// +optional
-	//
-	// Deprecated: In a future version, the type of this field is going to be replaced with a
-	// different struct that doesn't rely on component configuration types.
-	// This is a temporary warning, and no action is needed as of today.
-	Controller v1alpha1.ControllerConfigurationSpec //nolint:staticcheck
+	Controller config.Controller
 
 	// makeBroadcaster allows deferring the creation of the broadcaster to
 	// avoid leaking goroutines if we never call Start on this manager.  It also
@@ -314,10 +270,11 @@ type Options struct {
 	makeBroadcaster intrec.EventBroadcasterProducer
 
 	// Dependency injection for testing
-	newRecorderProvider    func(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error)
+	newRecorderProvider    func(config *rest.Config, httpClient *http.Client, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error)
 	newResourceLock        func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error)
-	newMetricsListener     func(addr string) (net.Listener, error)
+	newMetricsServer       func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error)
 	newHealthProbeListener func(addr string) (net.Listener, error)
+	newPprofListener       func(addr string) (net.Listener, error)
 }
 
 // BaseContextFunc is a function used to provide a base Context to Runnables
@@ -352,7 +309,13 @@ type LeaderElectionRunnable interface {
 }
 
 // New returns a new Manager for creating Controllers.
+// Note that if ContentType in the given config is not set, "application/vnd.kubernetes.protobuf"
+// will be used for all built-in resources of Kubernetes, and "application/json" is for other types
+// including all CRD resources.
 func New(config *rest.Config, options Options) (Manager, error) {
+	if config == nil {
+		return nil, errors.New("must specify Config")
+	}
 	// Set default values for options fields
 	options = setOptionsDefaults(options)
 
@@ -360,22 +323,25 @@ func New(config *rest.Config, options Options) (Manager, error) {
 		clusterOptions.Scheme = options.Scheme
 		clusterOptions.MapperProvider = options.MapperProvider
 		clusterOptions.Logger = options.Logger
-		clusterOptions.SyncPeriod = options.SyncPeriod
-		clusterOptions.Namespace = options.Namespace
 		clusterOptions.NewCache = options.NewCache
 		clusterOptions.NewClient = options.NewClient
-		clusterOptions.ClientDisableCacheFor = options.ClientDisableCacheFor
-		clusterOptions.DryRunClient = options.DryRunClient
+		clusterOptions.Cache = options.Cache
+		clusterOptions.Client = options.Client
 		clusterOptions.EventBroadcaster = options.EventBroadcaster //nolint:staticcheck
 	})
 	if err != nil {
 		return nil, err
 	}
 
+	config = rest.CopyConfig(config)
+	if config.UserAgent == "" {
+		config.UserAgent = rest.DefaultKubernetesUserAgent()
+	}
+
 	// Create the recorder provider to inject event recorders for the components.
 	// TODO(directxman12): the log for the event provider should have a context (name, tags, etc) specific
 	// to the particular controller that it's being injected into, rather than a generic one like is here.
-	recorderProvider, err := options.newRecorderProvider(config, cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster)
+	recorderProvider, err := options.newRecorderProvider(config, cluster.GetHTTPClient(), cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster)
 	if err != nil {
 		return nil, err
 	}
@@ -389,7 +355,20 @@ func New(config *rest.Config, options Options) (Manager, error) {
 		leaderRecorderProvider = recorderProvider
 	} else {
 		leaderConfig = rest.CopyConfig(options.LeaderElectionConfig)
-		leaderRecorderProvider, err = options.newRecorderProvider(leaderConfig, cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster)
+		scheme := cluster.GetScheme()
+		err := corev1.AddToScheme(scheme)
+		if err != nil {
+			return nil, err
+		}
+		err = coordinationv1.AddToScheme(scheme)
+		if err != nil {
+			return nil, err
+		}
+		httpClient, err := rest.HTTPClientFor(options.LeaderElectionConfig)
+		if err != nil {
+			return nil, err
+		}
+		leaderRecorderProvider, err = options.newRecorderProvider(leaderConfig, httpClient, scheme, options.Logger.WithName("events"), options.makeBroadcaster)
 		if err != nil {
 			return nil, err
 		}
@@ -410,16 +389,12 @@ func New(config *rest.Config, options Options) (Manager, error) {
 		}
 	}
 
-	// Create the metrics listener. This will throw an error if the metrics bind
-	// address is invalid or already in use.
-	metricsListener, err := options.newMetricsListener(options.MetricsBindAddress)
+	// Create the metrics server.
+	metricsServer, err := options.newMetricsServer(options.Metrics, config, cluster.GetHTTPClient())
 	if err != nil {
 		return nil, err
 	}
 
-	// By default we have no extra endpoints to expose on metrics http server.
-	metricsExtraHandlers := make(map[string]http.Handler)
-
 	// Create health probes listener. This will throw an error if the bind
 	// address is invalid or already in use.
 	healthProbeListener, err := options.newHealthProbeListener(options.HealthProbeBindAddress)
@@ -427,25 +402,26 @@ func New(config *rest.Config, options Options) (Manager, error) {
 		return nil, err
 	}
 
-	errChan := make(chan error)
-	runnables := newRunnables(options.BaseContext, errChan)
+	// Create pprof listener. This will throw an error if the bind
+	// address is invalid or already in use.
+	pprofListener, err := options.newPprofListener(options.PprofBindAddress)
+	if err != nil {
+		return nil, fmt.Errorf("failed to new pprof listener: %w", err)
+	}
 
+	errChan := make(chan error, 1)
+	runnables := newRunnables(options.BaseContext, errChan)
 	return &controllerManager{
-		stopProcedureEngaged:          pointer.Int64(0),
+		stopProcedureEngaged:          ptr.To(int64(0)),
 		cluster:                       cluster,
 		runnables:                     runnables,
 		errChan:                       errChan,
 		recorderProvider:              recorderProvider,
 		resourceLock:                  resourceLock,
-		metricsListener:               metricsListener,
-		metricsExtraHandlers:          metricsExtraHandlers,
-		controllerOptions:             options.Controller,
+		metricsServer:                 metricsServer,
+		controllerConfig:              options.Controller,
 		logger:                        options.Logger,
 		elected:                       make(chan struct{}),
-		port:                          options.Port,
-		host:                          options.Host,
-		certDir:                       options.CertDir,
-		tlsOpts:                       options.TLSOpts,
 		webhookServer:                 options.WebhookServer,
 		leaderElectionID:              options.LeaderElectionID,
 		leaseDuration:                 *options.LeaseDuration,
@@ -454,6 +430,7 @@ func New(config *rest.Config, options Options) (Manager, error) {
 		healthProbeListener:           healthProbeListener,
 		readinessEndpointName:         options.ReadinessEndpointName,
 		livenessEndpointName:          options.LivenessEndpointName,
+		pprofListener:                 pprofListener,
 		gracefulShutdownTimeout:       *options.GracefulShutdownTimeout,
 		internalProceduresStop:        make(chan struct{}),
 		leaderElectionStopped:         make(chan struct{}),
@@ -465,15 +442,13 @@ func New(config *rest.Config, options Options) (Manager, error) {
 // any options already set on Options will be ignored, this is used to allow
 // cli flags to override anything specified in the config file.
 //
-// Deprecated: This method will be removed in a future release.
+// Deprecated: This function has been deprecated and will be removed in a future release,
+// The Component Configuration package has been unmaintained for over a year and is no longer
+// actively developed. Users should migrate to their own configuration format
+// and configure Manager.Options directly.
+// See https://github.com/kubernetes-sigs/controller-runtime/issues/895
+// for more information, feedback, and comments.
 func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options, error) {
-	if inj, wantsScheme := loader.(inject.Scheme); wantsScheme {
-		err := inj.InjectScheme(o.Scheme)
-		if err != nil {
-			return o, err
-		}
-	}
-
 	newObj, err := loader.Complete()
 	if err != nil {
 		return o, err
@@ -481,16 +456,16 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options,
 
 	o = o.setLeaderElectionConfig(newObj)
 
-	if o.SyncPeriod == nil && newObj.SyncPeriod != nil {
-		o.SyncPeriod = &newObj.SyncPeriod.Duration
+	if o.Cache.SyncPeriod == nil && newObj.SyncPeriod != nil {
+		o.Cache.SyncPeriod = &newObj.SyncPeriod.Duration
 	}
 
-	if o.Namespace == "" && newObj.CacheNamespace != "" {
-		o.Namespace = newObj.CacheNamespace
+	if len(o.Cache.DefaultNamespaces) == 0 && newObj.CacheNamespace != "" {
+		o.Cache.DefaultNamespaces = map[string]cache.Config{newObj.CacheNamespace: {}}
 	}
 
-	if o.MetricsBindAddress == "" && newObj.Metrics.BindAddress != "" {
-		o.MetricsBindAddress = newObj.Metrics.BindAddress
+	if o.Metrics.BindAddress == "" && newObj.Metrics.BindAddress != "" {
+		o.Metrics.BindAddress = newObj.Metrics.BindAddress
 	}
 
 	if o.HealthProbeBindAddress == "" && newObj.Health.HealthProbeBindAddress != "" {
@@ -505,21 +480,21 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options,
 		o.LivenessEndpointName = newObj.Health.LivenessEndpointName
 	}
 
-	if o.Port == 0 && newObj.Webhook.Port != nil {
-		o.Port = *newObj.Webhook.Port
-	}
-
-	if o.Host == "" && newObj.Webhook.Host != "" {
-		o.Host = newObj.Webhook.Host
-	}
-
-	if o.CertDir == "" && newObj.Webhook.CertDir != "" {
-		o.CertDir = newObj.Webhook.CertDir
+	if o.WebhookServer == nil {
+		port := 0
+		if newObj.Webhook.Port != nil {
+			port = *newObj.Webhook.Port
+		}
+		o.WebhookServer = webhook.NewServer(webhook.Options{
+			Port:    port,
+			Host:    newObj.Webhook.Host,
+			CertDir: newObj.Webhook.CertDir,
+		})
 	}
 
 	if newObj.Controller != nil {
-		if o.Controller.CacheSyncTimeout == nil && newObj.Controller.CacheSyncTimeout != nil {
-			o.Controller.CacheSyncTimeout = newObj.Controller.CacheSyncTimeout
+		if o.Controller.CacheSyncTimeout == 0 && newObj.Controller.CacheSyncTimeout != nil {
+			o.Controller.CacheSyncTimeout = *newObj.Controller.CacheSyncTimeout
 		}
 
 		if len(o.Controller.GroupKindConcurrency) == 0 && len(newObj.Controller.GroupKindConcurrency) > 0 {
@@ -532,7 +507,12 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options,
 
 // AndFromOrDie will use options.AndFrom() and will panic if there are errors.
 //
-// Deprecated: This method will be removed in a future release.
+// Deprecated: This function has been deprecated and will be removed in a future release,
+// The Component Configuration package has been unmaintained for over a year and is no longer
+// actively developed. Users should migrate to their own configuration format
+// and configure Manager.Options directly.
+// See https://github.com/kubernetes-sigs/controller-runtime/issues/895
+// for more information, feedback, and comments.
 func (o Options) AndFromOrDie(loader config.ControllerManagerConfiguration) Options {
 	o, err := o.AndFrom(loader)
 	if err != nil {
@@ -541,7 +521,7 @@ func (o Options) AndFromOrDie(loader config.ControllerManagerConfiguration) Opti
 	return o
 }
 
-func (o Options) setLeaderElectionConfig(obj v1alpha1.ControllerManagerConfigurationSpec) Options { //nolint:staticcheck
+func (o Options) setLeaderElectionConfig(obj v1alpha1.ControllerManagerConfigurationSpec) Options {
 	if obj.LeaderElection == nil {
 		// The source does not have any configuration; noop
 		return o
@@ -591,6 +571,19 @@ func defaultHealthProbeListener(addr string) (net.Listener, error) {
 	return ln, nil
 }
 
+// defaultPprofListener creates the default pprof listener bound to the given address.
+func defaultPprofListener(addr string) (net.Listener, error) {
+	if addr == "" || addr == "0" {
+		return nil, nil
+	}
+
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		return nil, fmt.Errorf("error listening on %s: %w", addr, err)
+	}
+	return ln, nil
+}
+
 // defaultBaseContext is used as the BaseContext value in Options if one
 // has not already been set.
 func defaultBaseContext() context.Context {
@@ -623,8 +616,8 @@ func setOptionsDefaults(options Options) Options {
 		}
 	}
 
-	if options.newMetricsListener == nil {
-		options.newMetricsListener = metrics.NewListener
+	if options.newMetricsServer == nil {
+		options.newMetricsServer = metricsserver.NewServer
 	}
 	leaseDuration, renewDeadline, retryPeriod := defaultLeaseDuration, defaultRenewDeadline, defaultRetryPeriod
 	if options.LeaseDuration == nil {
@@ -651,6 +644,10 @@ func setOptionsDefaults(options Options) Options {
 		options.newHealthProbeListener = defaultHealthProbeListener
 	}
 
+	if options.newPprofListener == nil {
+		options.newPprofListener = defaultPprofListener
+	}
+
 	if options.GracefulShutdownTimeout == nil {
 		gracefulShutdownTimeout := defaultGracefulShutdownPeriod
 		options.GracefulShutdownTimeout = &gracefulShutdownTimeout
@@ -664,5 +661,9 @@ func setOptionsDefaults(options Options) Options {
 		options.BaseContext = defaultBaseContext
 	}
 
+	if options.WebhookServer == nil {
+		options.WebhookServer = webhook.NewServer(webhook.Options{})
+	}
+
 	return options
 }

vendor/sigs.k8s.io/controller-runtime/pkg/manager/runnable_group.go

@@ -28,6 +28,7 @@ type runnableCheck func(ctx context.Context) bool
 // runnables handles all the runnables for a manager by grouping them accordingly to their
 // type (webhooks, caches etc.).
 type runnables struct {
+	HTTPServers    *runnableGroup
 	Webhooks       *runnableGroup
 	Caches         *runnableGroup
 	LeaderElection *runnableGroup
@@ -37,6 +38,7 @@ type runnables struct {
 // newRunnables creates a new runnables object.
 func newRunnables(baseContext BaseContextFunc, errChan chan error) *runnables {
 	return &runnables{
+		HTTPServers:    newRunnableGroup(baseContext, errChan),
 		Webhooks:       newRunnableGroup(baseContext, errChan),
 		Caches:         newRunnableGroup(baseContext, errChan),
 		LeaderElection: newRunnableGroup(baseContext, errChan),
@@ -52,11 +54,13 @@ func newRunnables(baseContext BaseContextFunc, errChan chan error) *runnables {
 // The runnables added after Start are started directly.
 func (r *runnables) Add(fn Runnable) error {
 	switch runnable := fn.(type) {
+	case *server:
+		return r.HTTPServers.Add(fn, nil)
 	case hasCache:
 		return r.Caches.Add(fn, func(ctx context.Context) bool {
 			return runnable.GetCache().WaitForCacheSync(ctx)
 		})
-	case *webhook.Server:
+	case webhook.Server:
 		return r.Webhooks.Add(fn, nil)
 	case LeaderElectionRunnable:
 		if !runnable.NeedLeaderElection() {

vendor/sigs.k8s.io/controller-runtime/pkg/manager/server.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package manager
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+
+	"github.com/go-logr/logr"
+)
+
+// server is a general purpose HTTP server Runnable for a manager
+// to serve some internal handlers such as health probes, metrics and profiling.
+type server struct {
+	Kind     string
+	Log      logr.Logger
+	Server   *http.Server
+	Listener net.Listener
+}
+
+func (s *server) Start(ctx context.Context) error {
+	log := s.Log.WithValues("kind", s.Kind, "addr", s.Listener.Addr())
+
+	serverShutdown := make(chan struct{})
+	go func() {
+		<-ctx.Done()
+		log.Info("shutting down server")
+		if err := s.Server.Shutdown(context.Background()); err != nil {
+			log.Error(err, "error shutting down server")
+		}
+		close(serverShutdown)
+	}()
+
+	log.Info("starting server")
+	if err := s.Server.Serve(s.Listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		return err
+	}
+
+	<-serverShutdown
+	return nil
+}
+
+func (s *server) NeedLeaderElection() bool {
+	return false
+}