update dependencies (#6267)

Signed-off-by: hongming <coder.scala@gmail.com>
(cherry picked from commit cfebd96a1f)

vendor/k8s.io/apiserver/pkg/server/options/admission.go

@@ -19,26 +19,30 @@ package options
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/spf13/pflag"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/admission/initializer"
 	admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
 	"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
-	"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy"
+	validatingadmissionpolicy "k8s.io/apiserver/pkg/admission/plugin/policy/validating"
 	mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
 	validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
 	apiserverapi "k8s.io/apiserver/pkg/apis/apiserver"
 	apiserverapiv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
 	apiserverapiv1alpha1 "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
 	"k8s.io/apiserver/pkg/server"
+	cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/restmapper"
 	"k8s.io/component-base/featuregate"
 )
 
@@ -55,7 +59,7 @@ type AdmissionOptions struct {
 	// RecommendedPluginOrder holds an ordered list of plugin names we recommend to use by default
 	RecommendedPluginOrder []string
 	// DefaultOffPlugins is a set of plugin names that is disabled by default
-	DefaultOffPlugins sets.String
+	DefaultOffPlugins sets.Set[string]
 
 	// EnablePlugins indicates plugins to be enabled passed through `--enable-admission-plugins`.
 	EnablePlugins []string
@@ -87,7 +91,7 @@ func NewAdmissionOptions() *AdmissionOptions {
 		// after all the mutating ones, so their relative order in this list
 		// doesn't matter.
 		RecommendedPluginOrder: []string{lifecycle.PluginName, mutatingwebhook.PluginName, validatingadmissionpolicy.PluginName, validatingwebhook.PluginName},
-		DefaultOffPlugins:      sets.NewString(),
+		DefaultOffPlugins:      sets.Set[string]{},
 	}
 	server.RegisterAllAdmissionPlugins(options.Plugins)
 	return options
@@ -135,6 +139,9 @@ func (a *AdmissionOptions) ApplyTo(
 	if informers == nil {
 		return fmt.Errorf("admission depends on a Kubernetes core API shared informer, it cannot be nil")
 	}
+	if kubeClient == nil || dynamicClient == nil {
+		return fmt.Errorf("admission depends on a Kubernetes core API client, it cannot be nil")
+	}
 
 	pluginNames := a.enabledPluginNames()
 
@@ -143,11 +150,24 @@ func (a *AdmissionOptions) ApplyTo(
 		return fmt.Errorf("failed to read plugin config: %v", err)
 	}
 
+	discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery())
+	discoveryRESTMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
 	genericInitializer := initializer.New(kubeClient, dynamicClient, informers, c.Authorization.Authorizer, features,
-		c.DrainedNotify())
+		c.DrainedNotify(), discoveryRESTMapper)
 	initializersChain := admission.PluginInitializers{genericInitializer}
 	initializersChain = append(initializersChain, pluginInitializers...)
 
+	admissionPostStartHook := func(context server.PostStartHookContext) error {
+		discoveryRESTMapper.Reset()
+		go utilwait.Until(discoveryRESTMapper.Reset, 30*time.Second, context.StopCh)
+		return nil
+	}
+
+	err = c.AddPostStartHook("start-apiserver-admission-initializer", admissionPostStartHook)
+	if err != nil {
+		return fmt.Errorf("failed to add post start hook for policy admission: %w", err)
+	}
+
 	admissionChain, err := a.Plugins.NewFromPlugins(pluginNames, pluginsConfigProvider, initializersChain, a.Decorators)
 	if err != nil {
 		return err
@@ -206,7 +226,7 @@ func (a *AdmissionOptions) Validate() []error {
 // EnablePlugins, DisablePlugins fields
 // to prepare a list of ordered plugin names that are enabled.
 func (a *AdmissionOptions) enabledPluginNames() []string {
-	allOffPlugins := append(a.DefaultOffPlugins.List(), a.DisablePlugins...)
+	allOffPlugins := append(sets.List[string](a.DefaultOffPlugins), a.DisablePlugins...)
 	disabledPlugins := sets.NewString(allOffPlugins...)
 	enabledPlugins := sets.NewString(a.EnablePlugins...)
 	disabledPlugins = disabledPlugins.Difference(enabledPlugins)

vendor/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -26,6 +26,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/apis/apiserver"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
 	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
 	"k8s.io/apiserver/pkg/server"
@@ -222,8 +223,8 @@ type DelegatingAuthenticationOptions struct {
 	// CustomRoundTripperFn allows for specifying a middleware function for custom HTTP behaviour for the authentication webhook client.
 	CustomRoundTripperFn transport.WrapperFunc
 
-	// DisableAnonymous gives user an option to disable Anonymous authentication.
-	DisableAnonymous bool
+	// Anonymous gives user an option to enable/disable Anonymous authentication.
+	Anonymous *apiserver.AnonymousAuthConfig
 }
 
 func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
@@ -238,6 +239,7 @@ func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
 		},
 		WebhookRetryBackoff: DefaultAuthWebhookRetryBackoff(),
 		TokenRequestTimeout: 10 * time.Second,
+		Anonymous:           &apiserver.AnonymousAuthConfig{Enabled: true},
 	}
 }
 
@@ -305,7 +307,7 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.Aut
 	}
 
 	cfg := authenticatorfactory.DelegatingAuthenticatorConfig{
-		Anonymous:                !s.DisableAnonymous,
+		Anonymous:                &apiserver.AnonymousAuthConfig{Enabled: true},
 		CacheTTL:                 s.CacheTTL,
 		WebhookRetryBackoff:      s.WebhookRetryBackoff,
 		TokenAccessReviewTimeout: s.TokenRequestTimeout,

vendor/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go

@@ -1,126 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package options
-
-import (
-	"fmt"
-	"net"
-
-	"github.com/spf13/pflag"
-
-	"k8s.io/apiserver/pkg/server"
-)
-
-// DeprecatedInsecureServingOptions are for creating an unauthenticated, unauthorized, insecure port.
-// No one should be using these anymore.
-// DEPRECATED: all insecure serving options are removed in a future version
-type DeprecatedInsecureServingOptions struct {
-	BindAddress net.IP
-	BindPort    int
-	// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
-	// "tcp4", and "tcp6".
-	BindNetwork string
-
-	// Listener is the secure server network listener.
-	// either Listener or BindAddress/BindPort/BindNetwork is set,
-	// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork.
-	Listener net.Listener
-
-	// ListenFunc can be overridden to create a custom listener, e.g. for mocking in tests.
-	// It defaults to options.CreateListener.
-	ListenFunc func(network, addr string, config net.ListenConfig) (net.Listener, int, error)
-}
-
-// Validate ensures that the insecure port values within the range of the port.
-func (s *DeprecatedInsecureServingOptions) Validate() []error {
-	if s == nil {
-		return nil
-	}
-
-	errors := []error{}
-
-	if s.BindPort < 0 || s.BindPort > 65535 {
-		errors = append(errors, fmt.Errorf("insecure port %v must be between 0 and 65535, inclusive. 0 for turning off insecure (HTTP) port", s.BindPort))
-	}
-
-	return errors
-}
-
-// AddFlags adds flags related to insecure serving to the specified FlagSet.
-func (s *DeprecatedInsecureServingOptions) AddFlags(fs *pflag.FlagSet) {
-	if s == nil {
-		return
-	}
-
-	fs.IPVar(&s.BindAddress, "insecure-bind-address", s.BindAddress, ""+
-		"The IP address on which to serve the --insecure-port (set to 0.0.0.0 or :: for listening on all interfaces and IP address families).")
-	// Though this flag is deprecated, we discovered security concerns over how to do health checks without it e.g. #43784
-	fs.MarkDeprecated("insecure-bind-address", "This flag will be removed in a future version.")
-	fs.Lookup("insecure-bind-address").Hidden = false
-
-	fs.IntVar(&s.BindPort, "insecure-port", s.BindPort, ""+
-		"The port on which to serve unsecured, unauthenticated access.")
-	// Though this flag is deprecated, we discovered security concerns over how to do health checks without it e.g. #43784
-	fs.MarkDeprecated("insecure-port", "This flag will be removed in a future version.")
-	fs.Lookup("insecure-port").Hidden = false
-}
-
-// AddUnqualifiedFlags adds flags related to insecure serving without the --insecure prefix to the specified FlagSet.
-func (s *DeprecatedInsecureServingOptions) AddUnqualifiedFlags(fs *pflag.FlagSet) {
-	if s == nil {
-		return
-	}
-
-	fs.IPVar(&s.BindAddress, "address", s.BindAddress,
-		"The IP address on which to serve the insecure --port (set to '0.0.0.0' or '::' for listening on all interfaces and IP address families).")
-	fs.MarkDeprecated("address", "see --bind-address instead.")
-	fs.Lookup("address").Hidden = false
-
-	fs.IntVar(&s.BindPort, "port", s.BindPort, "The port on which to serve unsecured, unauthenticated access. Set to 0 to disable.")
-	fs.MarkDeprecated("port", "see --secure-port instead.")
-	fs.Lookup("port").Hidden = false
-}
-
-// ApplyTo adds DeprecatedInsecureServingOptions to the insecureserverinfo and kube-controller manager configuration.
-// Note: the double pointer allows to set the *DeprecatedInsecureServingInfo to nil without referencing the struct hosting this pointer.
-func (s *DeprecatedInsecureServingOptions) ApplyTo(c **server.DeprecatedInsecureServingInfo) error {
-	if s == nil {
-		return nil
-	}
-	if s.BindPort <= 0 {
-		return nil
-	}
-
-	if s.Listener == nil {
-		var err error
-		listen := CreateListener
-		if s.ListenFunc != nil {
-			listen = s.ListenFunc
-		}
-		addr := net.JoinHostPort(s.BindAddress.String(), fmt.Sprintf("%d", s.BindPort))
-		s.Listener, s.BindPort, err = listen(s.BindNetwork, addr, net.ListenConfig{})
-		if err != nil {
-			return fmt.Errorf("failed to create listener: %v", err)
-		}
-	}
-
-	*c = &server.DeprecatedInsecureServingInfo{
-		Listener: s.Listener,
-	}
-
-	return nil
-}

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -24,7 +24,6 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
 	"os"
 	"sync"
@@ -38,9 +37,9 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
-	apiserverconfig "k8s.io/apiserver/pkg/apis/config"
-	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
-	"k8s.io/apiserver/pkg/apis/config/validation"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
+	"k8s.io/apiserver/pkg/apis/apiserver/validation"
 	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
@@ -129,9 +128,9 @@ func GetKDF() bool {
 
 func init() {
 	configScheme := runtime.NewScheme()
-	utilruntime.Must(apiserverconfig.AddToScheme(configScheme))
-	utilruntime.Must(apiserverconfigv1.AddToScheme(configScheme))
-	codecs = serializer.NewCodecFactory(configScheme)
+	utilruntime.Must(apiserver.AddToScheme(configScheme))
+	utilruntime.Must(apiserverv1.AddToScheme(configScheme))
+	codecs = serializer.NewCodecFactory(configScheme, serializer.EnableStrict)
 	envelopemetrics.RegisterMetrics()
 	storagevalue.RegisterMetrics()
 	metrics.RegisterMetrics()
@@ -243,7 +242,7 @@ func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool, api
 // getTransformerOverridesAndKMSPluginHealthzCheckers creates the set of transformers and KMS healthz checks based on the given config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
+func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserver.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
 	var kmsHealthChecks []healthz.HealthChecker
 	transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, apiServerID)
 	if err != nil {
@@ -264,7 +263,7 @@ type healthChecker interface {
 // getTransformerOverridesAndKMSPluginProbes creates the set of transformers and KMS probes based on the given config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
+func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserver.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
 	resourceToPrefixTransformer := map[schema.GroupResource][]storagevalue.PrefixTransformer{}
 	var probes []healthChecker
 	var kmsUsed kmsState
@@ -503,7 +502,7 @@ func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.C
 }
 
 // loadConfig parses the encryption configuration file at filepath and returns the parsed config and hash of the file.
-func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfiguration, string, error) {
+func loadConfig(filepath string, reload bool) (*apiserver.EncryptionConfiguration, string, error) {
 	data, contentHash, err := loadDataAndHash(filepath)
 	if err != nil {
 		return nil, "", fmt.Errorf("error while loading file: %w", err)
@@ -513,7 +512,7 @@ func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfig
 	if err != nil {
 		return nil, "", fmt.Errorf("error decoding encryption provider configuration file %q: %w", filepath, err)
 	}
-	config, ok := configObj.(*apiserverconfig.EncryptionConfiguration)
+	config, ok := configObj.(*apiserver.EncryptionConfiguration)
 	if !ok {
 		return nil, "", fmt.Errorf("got unexpected config type: %v", gvk)
 	}
@@ -522,15 +521,9 @@ func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfig
 }
 
 func loadDataAndHash(filepath string) ([]byte, string, error) {
-	f, err := os.Open(filepath)
+	data, err := os.ReadFile(filepath)
 	if err != nil {
-		return nil, "", fmt.Errorf("error opening encryption provider configuration file %q: %w", filepath, err)
-	}
-	defer f.Close()
-
-	data, err := io.ReadAll(f)
-	if err != nil {
-		return nil, "", fmt.Errorf("could not read contents: %w", err)
+		return nil, "", fmt.Errorf("error reading encryption provider configuration file %q: %w", filepath, err)
 	}
 	if len(data) == 0 {
 		return nil, "", fmt.Errorf("encryption provider configuration file %q is empty", filepath)
@@ -549,7 +542,7 @@ func GetEncryptionConfigHash(filepath string) (string, error) {
 // prefixTransformersAndProbes creates the set of transformers and KMS probes based on the given resource config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration, apiServerID string) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
+func prefixTransformersAndProbes(ctx context.Context, config apiserver.ResourceConfiguration, apiServerID string) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
 	var transformers []storagevalue.PrefixTransformer
 	var probes []healthChecker
 	var kmsUsed kmsState
@@ -605,7 +598,7 @@ func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.Res
 
 type blockTransformerFunc func(cipher.Block) (storagevalue.Transformer, error)
 
-func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTransformerFunc, prefix string) (storagevalue.PrefixTransformer, error) {
+func aesPrefixTransformer(config *apiserver.AESConfiguration, fn blockTransformerFunc, prefix string) (storagevalue.PrefixTransformer, error) {
 	var result storagevalue.PrefixTransformer
 
 	if len(config.Keys) == 0 {
@@ -658,7 +651,7 @@ func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTran
 	return result, nil
 }
 
-func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (storagevalue.PrefixTransformer, error) {
+func secretboxPrefixTransformer(config *apiserver.SecretboxConfiguration) (storagevalue.PrefixTransformer, error) {
 	var result storagevalue.PrefixTransformer
 
 	if len(config.Keys) == 0 {
@@ -736,7 +729,7 @@ func (s *kmsState) accumulate(other *kmsState) {
 // kmsPrefixTransformer creates a KMS transformer and probe based on the given KMS config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration, apiServerID string) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
+func kmsPrefixTransformer(ctx context.Context, config *apiserver.KMSConfiguration, apiServerID string) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
 	kmsName := config.Name
 	switch config.APIVersion {
 	case kmsAPIVersionV1:
@@ -853,7 +846,7 @@ func primeAndProbeKMSv2(ctx context.Context, probe *kmsv2PluginProbe, kmsName st
 		})
 }
 
-func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) storagevalue.PrefixTransformer {
+func envelopePrefixTransformer(config *apiserver.KMSConfiguration, envelopeService envelope.Service, prefix string) storagevalue.PrefixTransformer {
 	baseTransformerFunc := func(block cipher.Block) (storagevalue.Transformer, error) {
 		gcm, err := aestransformer.NewGCMTransformer(block)
 		if err != nil {
@@ -902,7 +895,7 @@ func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte,
 // We use a hash instead of the raw file contents when tracking changes to avoid holding any encryption keys in memory outside of their associated transformers.
 // This hash must be used in-memory and not externalized to the process because it has no cross-release stability guarantees.
 func computeEncryptionConfigHash(data []byte) string {
-	return fmt.Sprintf("%x", sha256.Sum256(data))
+	return fmt.Sprintf("k8s:enc:unstable:1:%x", sha256.Sum256(data))
 }
 
 var _ storagevalue.ResourceTransformers = &DynamicTransformers{}

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go

@@ -49,7 +49,7 @@ type DynamicEncryptionConfigContent struct {
 	lastLoadedEncryptionConfigHash string
 
 	// queue for processing changes in encryption config file.
-	queue workqueue.RateLimitingInterface
+	queue workqueue.TypedRateLimitingInterface[string]
 
 	// dynamicTransformers updates the transformers when encryption config file changes.
 	dynamicTransformers *encryptionconfig.DynamicTransformers
@@ -78,8 +78,11 @@ func NewDynamicEncryptionConfiguration(
 		filePath:                       filePath,
 		lastLoadedEncryptionConfigHash: configContentHash,
 		dynamicTransformers:            dynamicTransformers,
-		queue:                          workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), name),
-		apiServerID:                    apiServerID,
+		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
+			workqueue.DefaultTypedControllerRateLimiter[string](),
+			workqueue.TypedRateLimitingQueueConfig[string]{Name: name},
+		),
+		apiServerID: apiServerID,
 		getEncryptionConfigHash: func(_ context.Context, filepath string) (string, error) {
 			return encryptionconfig.GetEncryptionConfigHash(filepath)
 		},
@@ -150,7 +153,7 @@ func (d *DynamicEncryptionConfigContent) processNextWorkItem(serverCtx context.C
 	return true
 }
 
-func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Context, workqueueKey interface{}) {
+func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Context, workqueueKey string) {
 	var (
 		updatedEffectiveConfig  bool
 		err                     error

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go

@@ -32,24 +32,41 @@ const (
 )
 
 var (
-	encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounterVec(
+	encryptionConfigAutomaticReloadsTotal = metrics.NewCounterVec(
 		&metrics.CounterOpts{
 			Namespace:      namespace,
 			Subsystem:      subsystem,
-			Name:           "automatic_reload_failures_total",
-			Help:           "Total number of failed automatic reloads of encryption configuration split by apiserver identity.",
+			Name:           "automatic_reloads_total",
+			Help:           "Total number of reload successes and failures of encryption configuration split by apiserver identity.",
 			StabilityLevel: metrics.ALPHA,
 		},
+		[]string{"status", "apiserver_id_hash"},
+	)
+
+	// deprecatedEncryptionConfigAutomaticReloadFailureTotal has been deprecated in 1.30.0
+	// use encryptionConfigAutomaticReloadsTotal instead
+	deprecatedEncryptionConfigAutomaticReloadFailureTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:         namespace,
+			Subsystem:         subsystem,
+			Name:              "automatic_reload_failures_total",
+			Help:              "Total number of failed automatic reloads of encryption configuration split by apiserver identity.",
+			StabilityLevel:    metrics.ALPHA,
+			DeprecatedVersion: "1.30.0",
+		},
 		[]string{"apiserver_id_hash"},
 	)
 
-	encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounterVec(
+	// deprecatedEncryptionConfigAutomaticReloadSuccessTotal has been deprecated in 1.30.0
+	// use encryptionConfigAutomaticReloadsTotal instead
+	deprecatedEncryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounterVec(
 		&metrics.CounterOpts{
-			Namespace:      namespace,
-			Subsystem:      subsystem,
-			Name:           "automatic_reload_success_total",
-			Help:           "Total number of successful automatic reloads of encryption configuration split by apiserver identity.",
-			StabilityLevel: metrics.ALPHA,
+			Namespace:         namespace,
+			Subsystem:         subsystem,
+			Name:              "automatic_reload_success_total",
+			Help:              "Total number of successful automatic reloads of encryption configuration split by apiserver identity.",
+			StabilityLevel:    metrics.ALPHA,
+			DeprecatedVersion: "1.30.0",
 		},
 		[]string{"apiserver_id_hash"},
 	)
@@ -76,21 +93,24 @@ func RegisterMetrics() {
 				return sha256.New()
 			},
 		}
-		legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal)
-		legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal)
+		legacyregistry.MustRegister(encryptionConfigAutomaticReloadsTotal)
+		legacyregistry.MustRegister(deprecatedEncryptionConfigAutomaticReloadFailureTotal)
+		legacyregistry.MustRegister(deprecatedEncryptionConfigAutomaticReloadSuccessTotal)
 		legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
 	})
 }
 
 func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) {
 	apiServerIDHash := getHash(apiServerID)
-	encryptionConfigAutomaticReloadFailureTotal.WithLabelValues(apiServerIDHash).Inc()
+	encryptionConfigAutomaticReloadsTotal.WithLabelValues("failure", apiServerIDHash).Inc()
+	deprecatedEncryptionConfigAutomaticReloadFailureTotal.WithLabelValues(apiServerIDHash).Inc()
 	recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash)
 }
 
 func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) {
 	apiServerIDHash := getHash(apiServerID)
-	encryptionConfigAutomaticReloadSuccessTotal.WithLabelValues(apiServerIDHash).Inc()
+	encryptionConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
+	deprecatedEncryptionConfigAutomaticReloadSuccessTotal.WithLabelValues(apiServerIDHash).Inc()
 	recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash)
 }
 

vendor/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -383,8 +383,8 @@ type StorageFactoryRestOptionsFactory struct {
 	StorageFactory serverstorage.StorageFactory
 }
 
-func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
-	storageConfig, err := f.StorageFactory.NewConfig(resource)
+func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) {
+	storageConfig, err := f.StorageFactory.NewConfig(resource, example)
 	if err != nil {
 		return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error())
 	}
@@ -399,6 +399,10 @@ func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupR
 		StorageObjectCountTracker: f.Options.StorageConfig.StorageObjectCountTracker,
 	}
 
+	if ret.StorageObjectCountTracker == nil {
+		ret.StorageObjectCountTracker = storageConfig.StorageObjectCountTracker
+	}
+
 	if f.Options.EnableWatchCache {
 		sizes, err := ParseWatchCacheSizes(f.Options.WatchCacheSizes)
 		if err != nil {
@@ -465,7 +469,7 @@ type SimpleStorageFactory struct {
 	StorageConfig storagebackend.Config
 }
 
-func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
+func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
 	return s.StorageConfig.ForResource(resource), nil
 }
 
@@ -489,8 +493,8 @@ type transformerStorageFactory struct {
 	resourceTransformers storagevalue.ResourceTransformers
 }
 
-func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
-	config, err := t.delegate.NewConfig(resource)
+func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
+	config, err := t.delegate.NewConfig(resource, example)
 	if err != nil {
 		return nil, err
 	}

vendor/k8s.io/apiserver/pkg/server/options/feature.go

@@ -71,6 +71,9 @@ func (o *FeatureOptions) ApplyTo(c *server.Config, clientset kubernetes.Interfac
 	c.EnableContentionProfiling = o.EnableContentionProfiling
 
 	if o.EnablePriorityAndFairness {
+		if clientset == nil {
+			return fmt.Errorf("invalid configuration: priority and fairness requires a core Kubernetes client")
+		}
 		if c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight <= 0 {
 			return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight)
 

vendor/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -120,9 +120,18 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if err := o.CoreAPI.ApplyTo(config); err != nil {
 		return err
 	}
-	kubeClient, err := kubernetes.NewForConfig(config.ClientConfig)
-	if err != nil {
-		return err
+	var kubeClient *kubernetes.Clientset
+	var dynamicClient *dynamic.DynamicClient
+	if config.ClientConfig != nil {
+		var err error
+		kubeClient, err = kubernetes.NewForConfig(config.ClientConfig)
+		if err != nil {
+			return err
+		}
+		dynamicClient, err = dynamic.NewForConfig(config.ClientConfig)
+		if err != nil {
+			return err
+		}
 	}
 	if err := o.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil {
 		return err
@@ -131,10 +140,6 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if err != nil {
 		return err
 	}
-	dynamicClient, err := dynamic.NewForConfig(config.ClientConfig)
-	if err != nil {
-		return err
-	}
 	if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, kubeClient, dynamicClient, o.FeatureGate,
 		initializers...); err != nil {
 		return err

vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -25,8 +25,10 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/util/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/server"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilversion "k8s.io/apiserver/pkg/util/version"
 
 	"github.com/spf13/pflag"
 )
@@ -45,16 +47,17 @@ const (
 type ServerRunOptions struct {
 	AdvertiseAddress net.IP
 
-	CorsAllowedOriginList       []string
-	HSTSDirectives              []string
-	ExternalHost                string
-	MaxRequestsInFlight         int
-	MaxMutatingRequestsInFlight int
-	RequestTimeout              time.Duration
-	GoawayChance                float64
-	LivezGracePeriod            time.Duration
-	MinRequestTimeout           int
-	ShutdownDelayDuration       time.Duration
+	CorsAllowedOriginList        []string
+	HSTSDirectives               []string
+	ExternalHost                 string
+	MaxRequestsInFlight          int
+	MaxMutatingRequestsInFlight  int
+	RequestTimeout               time.Duration
+	GoawayChance                 float64
+	LivezGracePeriod             time.Duration
+	MinRequestTimeout            int
+	StorageInitializationTimeout time.Duration
+	ShutdownDelayDuration        time.Duration
 	// We intentionally did not add a flag for this option. Users of the
 	// apiserver library can wire it to a flag.
 	JSONPatchMaxCopyBytes int64
@@ -89,9 +92,24 @@ type ServerRunOptions struct {
 	// This grace period is orthogonal to other grace periods, and
 	// it is not overridden by any other grace period.
 	ShutdownWatchTerminationGracePeriod time.Duration
+
+	// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
+	ComponentGlobalsRegistry utilversion.ComponentGlobalsRegistry
+	// ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry.
+	ComponentName string
 }
 
 func NewServerRunOptions() *ServerRunOptions {
+	if utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) == nil {
+		featureGate := utilfeature.DefaultMutableFeatureGate
+		effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
+		utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
+	}
+
+	return NewServerRunOptionsForComponent(utilversion.DefaultKubeComponent, utilversion.DefaultComponentGlobalsRegistry)
+}
+
+func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry utilversion.ComponentGlobalsRegistry) *ServerRunOptions {
 	defaults := server.NewConfig(serializer.CodecFactory{})
 	return &ServerRunOptions{
 		MaxRequestsInFlight:                 defaults.MaxRequestsInFlight,
@@ -99,16 +117,22 @@ func NewServerRunOptions() *ServerRunOptions {
 		RequestTimeout:                      defaults.RequestTimeout,
 		LivezGracePeriod:                    defaults.LivezGracePeriod,
 		MinRequestTimeout:                   defaults.MinRequestTimeout,
+		StorageInitializationTimeout:        defaults.StorageInitializationTimeout,
 		ShutdownDelayDuration:               defaults.ShutdownDelayDuration,
 		ShutdownWatchTerminationGracePeriod: defaults.ShutdownWatchTerminationGracePeriod,
 		JSONPatchMaxCopyBytes:               defaults.JSONPatchMaxCopyBytes,
 		MaxRequestBodyBytes:                 defaults.MaxRequestBodyBytes,
 		ShutdownSendRetryAfter:              false,
+		ComponentName:                       componentName,
+		ComponentGlobalsRegistry:            componentGlobalsRegistry,
 	}
 }
 
 // ApplyTo applies the run options to the method receiver and returns self
 func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
+	if err := s.ComponentGlobalsRegistry.SetFallback(); err != nil {
+		return err
+	}
 	c.CorsAllowedOriginList = s.CorsAllowedOriginList
 	c.HSTSDirectives = s.HSTSDirectives
 	c.ExternalAddress = s.ExternalHost
@@ -118,12 +142,15 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
 	c.RequestTimeout = s.RequestTimeout
 	c.GoawayChance = s.GoawayChance
 	c.MinRequestTimeout = s.MinRequestTimeout
+	c.StorageInitializationTimeout = s.StorageInitializationTimeout
 	c.ShutdownDelayDuration = s.ShutdownDelayDuration
 	c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
 	c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
 	c.PublicAddress = s.AdvertiseAddress
 	c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
 	c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod
+	c.EffectiveVersion = s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName)
+	c.FeatureGate = s.ComponentGlobalsRegistry.FeatureGateFor(s.ComponentName)
 
 	return nil
 }
@@ -173,6 +200,10 @@ func (s *ServerRunOptions) Validate() []error {
 		errors = append(errors, fmt.Errorf("--min-request-timeout can not be negative value"))
 	}
 
+	if s.StorageInitializationTimeout < 0 {
+		errors = append(errors, fmt.Errorf("--storage-initialization-timeout can not be negative value"))
+	}
+
 	if s.ShutdownDelayDuration < 0 {
 		errors = append(errors, fmt.Errorf("--shutdown-delay-duration can not be negative value"))
 	}
@@ -196,6 +227,9 @@ func (s *ServerRunOptions) Validate() []error {
 	if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil {
 		errors = append(errors, err)
 	}
+	if errs := s.ComponentGlobalsRegistry.Validate(); len(errs) != 0 {
+		errors = append(errors, errs...)
+	}
 	return errors
 }
 
@@ -323,6 +357,9 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"handler, which picks a randomized value above this number as the connection timeout, "+
 		"to spread out load.")
 
+	fs.DurationVar(&s.StorageInitializationTimeout, "storage-initialization-timeout", s.StorageInitializationTimeout,
+		"Maximum amount of time to wait for storage initialization before declaring apiserver ready. Defaults to 1m.")
+
 	fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
 		"Time to delay the termination. During that time the server keeps serving requests normally. The endpoints /healthz and /livez "+
 		"will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
@@ -337,5 +374,10 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"This option, if set, represents the maximum amount of grace period the apiserver will wait "+
 		"for active watch request(s) to drain during the graceful server shutdown window.")
 
-	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
+	s.ComponentGlobalsRegistry.AddFlags(fs)
+}
+
+// Complete fills missing fields with defaults.
+func (s *ServerRunOptions) Complete() error {
+	return s.ComponentGlobalsRegistry.SetFallback()
 }

vendor/k8s.io/apiserver/pkg/server/options/serving.go

@@ -44,6 +44,8 @@ type SecureServingOptions struct {
 	// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
 	// "tcp4", and "tcp6".
 	BindNetwork string
+	// DisableHTTP2Serving indicates that http2 serving should not be enabled.
+	DisableHTTP2Serving bool
 	// Required set to true means that BindPort cannot be zero.
 	Required bool
 	// ExternalAddress is the address advertised, even if BindAddress is a loopback. By default this
@@ -163,6 +165,9 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 	}
 	fs.IntVar(&s.BindPort, "secure-port", s.BindPort, desc)
 
+	fs.BoolVar(&s.DisableHTTP2Serving, "disable-http2-serving", s.DisableHTTP2Serving,
+		"If true, HTTP2 serving will be disabled [default=false]")
+
 	fs.StringVar(&s.ServerCert.CertDirectory, "cert-dir", s.ServerCert.CertDirectory, ""+
 		"The directory where the TLS certs are located. "+
 		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
@@ -256,6 +261,7 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	*config = &server.SecureServingInfo{
 		Listener:                     s.Listener,
 		HTTP2MaxStreamsPerConnection: s.HTTP2MaxStreamsPerConnection,
+		DisableHTTP2:                 s.DisableHTTP2Serving,
 	}
 	c := *config
 

vendor/k8s.io/apiserver/pkg/server/options/tracing.go

@@ -23,7 +23,9 @@ import (
 	"net"
 
 	"github.com/spf13/pflag"
+	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/metric/noop"
 	"go.opentelemetry.io/otel/sdk/resource"
 	"go.opentelemetry.io/otel/semconv/v1.12.0"
 	"google.golang.org/grpc"
@@ -48,6 +50,12 @@ var (
 	codecs    = serializer.NewCodecFactory(cfgScheme)
 )
 
+func init() {
+	// Prevent memory leak from OTel metrics, which we don't use:
+	// https://github.com/open-telemetry/opentelemetry-go-contrib/issues/5190
+	otel.SetMeterProvider(noop.NewMeterProvider())
+}
+
 func init() {
 	install.Install(cfgScheme)
 }