Upgrade k8s package verison (#5358)

* upgrade k8s package version

Signed-off-by: hongzhouzi <hongzhouzi@kubesphere.io>

* Script upgrade and code formatting.

Signed-off-by: hongzhouzi <hongzhouzi@kubesphere.io>

Signed-off-by: hongzhouzi <hongzhouzi@kubesphere.io>

vendor/k8s.io/apiserver/pkg/server/options/OWNERS

@@ -1,15 +1,13 @@
 # See the OWNERS docs at https://go.k8s.io/owners
 
 reviewers:
-- smarterclayton
-- wojtek-t
-- deads2k
-- liggitt
-- sttts
-- jlowdermilk
-- soltysh
-- dims
-- cjcullen
-- ping035627
-- xiangpengzhao
-- enj
+  - smarterclayton
+  - wojtek-t
+  - deads2k
+  - liggitt
+  - sttts
+  - soltysh
+  - dims
+  - cjcullen
+  - ping035627
+  - enj

vendor/k8s.io/apiserver/pkg/server/options/admission.go

@@ -70,12 +70,13 @@ type AdmissionOptions struct {
 
 // NewAdmissionOptions creates a new instance of AdmissionOptions
 // Note:
-//  In addition it calls RegisterAllAdmissionPlugins to register
-//  all generic admission plugins.
 //
-//  Provides the list of RecommendedPluginOrder that holds sane values
-//  that can be used by servers that don't care about admission chain.
-//  Servers that do care can overwrite/append that field after creation.
+//	In addition it calls RegisterAllAdmissionPlugins to register
+//	all generic admission plugins.
+//
+//	Provides the list of RecommendedPluginOrder that holds sane values
+//	that can be used by servers that don't care about admission chain.
+//	Servers that do care can overwrite/append that field after creation.
 func NewAdmissionOptions() *AdmissionOptions {
 	options := &AdmissionOptions{
 		Plugins:    admission.NewPlugins(),
@@ -115,7 +116,8 @@ func (a *AdmissionOptions) AddFlags(fs *pflag.FlagSet) {
 // In case admission plugin names were not provided by a cluster-admin they will be prepared from the recommended/default values.
 // In addition the method lazily initializes a generic plugin that is appended to the list of pluginInitializers
 // note this method uses:
-//  genericconfig.Authorizer
+//
+//	genericconfig.Authorizer
 func (a *AdmissionOptions) ApplyTo(
 	c *server.Config,
 	informers informers.SharedInformerFactory,
@@ -143,9 +145,8 @@ func (a *AdmissionOptions) ApplyTo(
 	if err != nil {
 		return err
 	}
-	genericInitializer := initializer.New(clientset, informers, c.Authorization.Authorizer, features)
-	initializersChain := admission.PluginInitializers{}
-	pluginInitializers = append(pluginInitializers, genericInitializer)
+	genericInitializer := initializer.New(clientset, informers, c.Authorization.Authorizer, features, c.DrainedNotify())
+	initializersChain := admission.PluginInitializers{genericInitializer}
 	initializersChain = append(initializersChain, pluginInitializers...)
 
 	admissionChain, err := a.Plugins.NewFromPlugins(pluginNames, pluginsConfigProvider, initializersChain, a.Decorators)
@@ -221,7 +222,7 @@ func (a *AdmissionOptions) enabledPluginNames() []string {
 	return orderedPlugins
 }
 
-//Return names of plugins which are enabled by default
+// Return names of plugins which are enabled by default
 func (a *AdmissionOptions) defaultEnabledPluginNames() []string {
 	defaultOnPluginNames := []string{}
 	for _, pluginName := range a.RecommendedPluginOrder {

vendor/k8s.io/apiserver/pkg/server/options/api_enablement.go

@@ -58,7 +58,7 @@ func (s *APIEnablementOptions) AddFlags(fs *pflag.FlagSet) {
 // But in the advanced (and usually not recommended) case of delegated apiservers there can be more.
 // Validate will filter out the known groups of each registry.
 // If anything is left over after that, an error is returned.
-func (s *APIEnablementOptions) Validate(registries ...GroupRegisty) []error {
+func (s *APIEnablementOptions) Validate(registries ...GroupRegistry) []error {
 	if s == nil {
 		return nil
 	}
@@ -98,7 +98,7 @@ func (s *APIEnablementOptions) ApplyTo(c *server.Config, defaultResourceConfig *
 	return err
 }
 
-func unknownGroups(groups []string, registry GroupRegisty) []string {
+func unknownGroups(groups []string, registry GroupRegistry) []string {
 	unknownGroups := []string{}
 	for _, group := range groups {
 		if !registry.IsGroupRegistered(group) {
@@ -108,8 +108,8 @@ func unknownGroups(groups []string, registry GroupRegisty) []string {
 	return unknownGroups
 }
 
-// GroupRegisty provides a method to check whether given group is registered.
-type GroupRegisty interface {
+// GroupRegistry provides a method to check whether given group is registered.
+type GroupRegistry interface {
 	// IsRegistered returns true if given group is registered.
 	IsGroupRegistered(group string) bool
 }

vendor/k8s.io/apiserver/pkg/server/options/audit.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -29,10 +30,9 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apimachinery/pkg/util/sets"
 	auditinternal "k8s.io/apiserver/pkg/apis/audit"
 	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
-	auditv1alpha1 "k8s.io/apiserver/pkg/apis/audit/v1alpha1"
-	auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1"
 	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/audit/policy"
 	"k8s.io/apiserver/pkg/server"
@@ -234,8 +234,6 @@ func validateBackendBatchOptions(pluginName string, options AuditBatchOptions) e
 }
 
 var knownGroupVersions = []schema.GroupVersion{
-	auditv1alpha1.SchemeGroupVersion,
-	auditv1beta1.SchemeGroupVersion,
 	auditv1.SchemeGroupVersion,
 }
 
@@ -288,16 +286,20 @@ func (o *AuditOptions) ApplyTo(
 		return fmt.Errorf("server config must be non-nil")
 	}
 
-	// 1. Build policy checker
-	checker, err := o.newPolicyChecker()
+	// 1. Build policy evaluator
+	evaluator, err := o.newPolicyRuleEvaluator()
 	if err != nil {
 		return err
 	}
 
 	// 2. Build log backend
 	var logBackend audit.Backend
-	if w := o.LogOptions.getWriter(); w != nil {
-		if checker == nil {
+	w, err := o.LogOptions.getWriter()
+	if err != nil {
+		return err
+	}
+	if w != nil {
+		if evaluator == nil {
 			klog.V(2).Info("No audit policy file provided, no events will be recorded for log backend")
 		} else {
 			logBackend = o.LogOptions.newBackend(w)
@@ -307,7 +309,7 @@ func (o *AuditOptions) ApplyTo(
 	// 3. Build webhook backend
 	var webhookBackend audit.Backend
 	if o.WebhookOptions.enabled() {
-		if checker == nil {
+		if evaluator == nil {
 			klog.V(2).Info("No audit policy file provided, no events will be recorded for webhook backend")
 		} else {
 			if c.EgressSelector != nil {
@@ -338,8 +340,8 @@ func (o *AuditOptions) ApplyTo(
 		dynamicBackend = o.WebhookOptions.TruncateOptions.wrapBackend(webhookBackend, groupVersion)
 	}
 
-	// 5. Set the policy checker
-	c.AuditPolicyChecker = checker
+	// 5. Set the policy rule evaluator
+	c.AuditPolicyRuleEvaluator = evaluator
 
 	// 6. Join the log backend with the webhooks
 	c.AuditBackend = appendBackend(logBackend, dynamicBackend)
@@ -350,7 +352,7 @@ func (o *AuditOptions) ApplyTo(
 	return nil
 }
 
-func (o *AuditOptions) newPolicyChecker() (policy.Checker, error) {
+func (o *AuditOptions) newPolicyRuleEvaluator() (audit.PolicyRuleEvaluator, error) {
 	if o.PolicyFile == "" {
 		return nil, nil
 	}
@@ -359,7 +361,7 @@ func (o *AuditOptions) newPolicyChecker() (policy.Checker, error) {
 	if err != nil {
 		return nil, fmt.Errorf("loading audit policy file: %v", err)
 	}
-	return policy.NewChecker(p), nil
+	return policy.NewPolicyRuleEvaluator(p), nil
 }
 
 func (o *AuditBatchOptions) AddFlags(pluginName string, fs *pflag.FlagSet) {
@@ -446,7 +448,7 @@ func (o *AuditLogOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&o.MaxAge, "audit-log-maxage", o.MaxAge,
 		"The maximum number of days to retain old audit log files based on the timestamp encoded in their filename.")
 	fs.IntVar(&o.MaxBackups, "audit-log-maxbackup", o.MaxBackups,
-		"The maximum number of old audit log files to retain.")
+		"The maximum number of old audit log files to retain. Setting a value of 0 will mean there's no restriction on the number of files.")
 	fs.IntVar(&o.MaxSize, "audit-log-maxsize", o.MaxSize,
 		"The maximum size in megabytes of the audit log file before it gets rotated.")
 	fs.StringVar(&o.Format, "audit-log-format", o.Format,
@@ -478,14 +480,7 @@ func (o *AuditLogOptions) Validate() []error {
 	}
 
 	// Check log format
-	validFormat := false
-	for _, f := range pluginlog.AllowedFormats {
-		if f == o.Format {
-			validFormat = true
-			break
-		}
-	}
-	if !validFormat {
+	if !sets.NewString(pluginlog.AllowedFormats...).Has(o.Format) {
 		allErrors = append(allErrors, fmt.Errorf("invalid audit log format %s, allowed formats are %q", o.Format, strings.Join(pluginlog.AllowedFormats, ",")))
 	}
 
@@ -508,22 +503,38 @@ func (o *AuditLogOptions) enabled() bool {
 	return o != nil && o.Path != ""
 }
 
-func (o *AuditLogOptions) getWriter() io.Writer {
+func (o *AuditLogOptions) getWriter() (io.Writer, error) {
 	if !o.enabled() {
-		return nil
+		return nil, nil
 	}
 
-	var w io.Writer = os.Stdout
-	if o.Path != "-" {
-		w = &lumberjack.Logger{
-			Filename:   o.Path,
-			MaxAge:     o.MaxAge,
-			MaxBackups: o.MaxBackups,
-			MaxSize:    o.MaxSize,
-			Compress:   o.Compress,
-		}
+	if o.Path == "-" {
+		return os.Stdout, nil
 	}
-	return w
+
+	if err := o.ensureLogFile(); err != nil {
+		return nil, fmt.Errorf("ensureLogFile: %w", err)
+	}
+
+	return &lumberjack.Logger{
+		Filename:   o.Path,
+		MaxAge:     o.MaxAge,
+		MaxBackups: o.MaxBackups,
+		MaxSize:    o.MaxSize,
+		Compress:   o.Compress,
+	}, nil
+}
+
+func (o *AuditLogOptions) ensureLogFile() error {
+	if err := os.MkdirAll(filepath.Dir(o.Path), 0700); err != nil {
+		return err
+	}
+	mode := os.FileMode(0600)
+	f, err := os.OpenFile(o.Path, os.O_CREATE|os.O_APPEND|os.O_RDWR, mode)
+	if err != nil {
+		return err
+	}
+	return f.Close()
 }
 
 func (o *AuditLogOptions) newBackend(w io.Writer) audit.Backend {

vendor/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -17,12 +17,11 @@ limitations under the License.
 package options
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"time"
 
-	"k8s.io/apiserver/pkg/server/dynamiccertificates"
-
 	"github.com/spf13/pflag"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -30,9 +29,11 @@ import (
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
 	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
 	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/transport"
 	"k8s.io/klog/v2"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
 )
@@ -198,6 +199,9 @@ type DelegatingAuthenticationOptions struct {
 	// TokenRequestTimeout specifies a time limit for requests made by the authorization webhook client.
 	// The default value is set to 10 seconds.
 	TokenRequestTimeout time.Duration
+
+	// CustomRoundTripperFn allows for specifying a middleware function for custom HTTP behaviour for the authentication webhook client.
+	CustomRoundTripperFn transport.WrapperFunc
 }
 
 func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
@@ -225,6 +229,11 @@ func (s *DelegatingAuthenticationOptions) WithRequestTimeout(timeout time.Durati
 	s.TokenRequestTimeout = timeout
 }
 
+// WithCustomRoundTripper allows for specifying a middleware function for custom HTTP behaviour for the authentication webhook client.
+func (s *DelegatingAuthenticationOptions) WithCustomRoundTripper(rt transport.WrapperFunc) {
+	s.CustomRoundTripperFn = rt
+}
+
 func (s *DelegatingAuthenticationOptions) Validate() []error {
 	if s == nil {
 		return nil
@@ -287,7 +296,7 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.Aut
 
 	// configure token review
 	if client != nil {
-		cfg.TokenAccessReviewClient = client.AuthenticationV1().TokenReviews()
+		cfg.TokenAccessReviewClient = client.AuthenticationV1()
 	}
 
 	// get the clientCA information
@@ -379,7 +388,10 @@ func (s *DelegatingAuthenticationOptions) createRequestHeaderConfig(client kuber
 	}
 
 	//  look up authentication configuration in the cluster and in case of an err defer to authentication-tolerate-lookup-failure flag
-	if err := dynamicRequestHeaderProvider.RunOnce(); err != nil {
+	//  We are passing the context to ProxyCerts.RunOnce as it needs to implement RunOnce(ctx) however the
+	//  context is not used at all. So passing a empty context shouldn't be a problem
+	ctx := context.TODO()
+	if err := dynamicRequestHeaderProvider.RunOnce(ctx); err != nil {
 		return nil, err
 	}
 
@@ -424,6 +436,9 @@ func (s *DelegatingAuthenticationOptions) getClient() (kubernetes.Interface, err
 	// if multiple timeouts were set, the request will pick the smaller timeout to be applied, leaving other useless.
 	//
 	// see https://github.com/golang/go/blob/a937729c2c2f6950a32bc5cd0f5b88700882f078/src/net/http/client.go#L364
+	if s.CustomRoundTripperFn != nil {
+		clientConfig.Wrap(s.CustomRoundTripperFn)
+	}
 
 	return kubernetes.NewForConfig(clientConfig)
 }

vendor/k8s.io/apiserver/pkg/server/options/authentication_dynamic_request_header.go

@@ -17,6 +17,7 @@ limitations under the License.
 package options
 
 import (
+	"context"
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/util/errors"
@@ -26,7 +27,6 @@ import (
 )
 
 var _ dynamiccertificates.ControllerRunner = &DynamicRequestHeaderController{}
-var _ dynamiccertificates.Notifier = &DynamicRequestHeaderController{}
 var _ dynamiccertificates.CAContentProvider = &DynamicRequestHeaderController{}
 
 var _ headerrequest.RequestHeaderAuthRequestProvider = &DynamicRequestHeaderController{}
@@ -65,15 +65,15 @@ func newDynamicRequestHeaderController(client kubernetes.Interface) (*DynamicReq
 	}, nil
 }
 
-func (c *DynamicRequestHeaderController) RunOnce() error {
+func (c *DynamicRequestHeaderController) RunOnce(ctx context.Context) error {
 	errs := []error{}
-	errs = append(errs, c.ConfigMapCAController.RunOnce())
-	errs = append(errs, c.RequestHeaderAuthRequestController.RunOnce())
+	errs = append(errs, c.ConfigMapCAController.RunOnce(ctx))
+	errs = append(errs, c.RequestHeaderAuthRequestController.RunOnce(ctx))
 	return errors.NewAggregate(errs)
 }
 
-func (c *DynamicRequestHeaderController) Run(workers int, stopCh <-chan struct{}) {
-	go c.ConfigMapCAController.Run(workers, stopCh)
-	go c.RequestHeaderAuthRequestController.Run(workers, stopCh)
-	<-stopCh
+func (c *DynamicRequestHeaderController) Run(ctx context.Context, workers int) {
+	go c.ConfigMapCAController.Run(ctx, workers)
+	go c.RequestHeaderAuthRequestController.Run(ctx, workers)
+	<-ctx.Done()
 }

vendor/k8s.io/apiserver/pkg/server/options/authorization.go

@@ -21,7 +21,6 @@ import (
 	"time"
 
 	"github.com/spf13/pflag"
-	"k8s.io/klog/v2"
 
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
@@ -32,13 +31,16 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/transport"
+	"k8s.io/klog/v2"
 )
 
 // DelegatingAuthorizationOptions provides an easy way for composing API servers to delegate their authorization to
 // the root kube API server.
 // WARNING: never assume that every authenticated incoming request already does authorization.
-//          The aggregator in the kube API server does this today, but this behaviour is not
-//          guaranteed in the future.
+//
+//	The aggregator in the kube API server does this today, but this behaviour is not
+//	guaranteed in the future.
 type DelegatingAuthorizationOptions struct {
 	// RemoteKubeConfigFile is the file to use to connect to a "normal" kube API server which hosts the
 	// SubjectAccessReview.authorization.k8s.io endpoint for checking tokens.
@@ -69,6 +71,9 @@ type DelegatingAuthorizationOptions struct {
 	// This allows us to configure the sleep time at each iteration and the maximum number of retries allowed
 	// before we fail the webhook call in order to limit the fan out that ensues when the system is degraded.
 	WebhookRetryBackoff *wait.Backoff
+
+	// CustomRoundTripperFn allows for specifying a middleware function for custom HTTP behaviour for the authorization webhook client.
+	CustomRoundTripperFn transport.WrapperFunc
 }
 
 func NewDelegatingAuthorizationOptions() *DelegatingAuthorizationOptions {
@@ -111,6 +116,11 @@ func (s *DelegatingAuthorizationOptions) WithCustomRetryBackoff(backoff wait.Bac
 	s.WebhookRetryBackoff = &backoff
 }
 
+// WithCustomRoundTripper allows for specifying a middleware function for custom HTTP behaviour for the authorization webhook client.
+func (s *DelegatingAuthorizationOptions) WithCustomRoundTripper(rt transport.WrapperFunc) {
+	s.CustomRoundTripperFn = rt
+}
+
 func (s *DelegatingAuthorizationOptions) Validate() []error {
 	if s == nil {
 		return nil
@@ -184,7 +194,7 @@ func (s *DelegatingAuthorizationOptions) toAuthorizer(client kubernetes.Interfac
 		klog.Warning("No authorization-kubeconfig provided, so SubjectAccessReview of authorization tokens won't work.")
 	} else {
 		cfg := authorizerfactory.DelegatingAuthorizerConfig{
-			SubjectAccessReviewClient: client.AuthorizationV1().SubjectAccessReviews(),
+			SubjectAccessReviewClient: client.AuthorizationV1(),
 			AllowCacheTTL:             s.AllowCacheTTL,
 			DenyCacheTTL:              s.DenyCacheTTL,
 			WebhookRetryBackoff:       s.WebhookRetryBackoff,
@@ -226,6 +236,9 @@ func (s *DelegatingAuthorizationOptions) getClient() (kubernetes.Interface, erro
 	clientConfig.QPS = 200
 	clientConfig.Burst = 400
 	clientConfig.Timeout = s.ClientTimeout
+	if s.CustomRoundTripperFn != nil {
+		clientConfig.Wrap(s.CustomRoundTripperFn)
+	}
 
 	return kubernetes.NewForConfig(clientConfig)
 }

vendor/k8s.io/apiserver/pkg/server/options/coreapi.go

@@ -21,11 +21,14 @@ import (
 	"time"
 
 	"github.com/spf13/pflag"
+	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/util/feature"
 	clientgoinformers "k8s.io/client-go/informers"
 	clientgoclientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	tracing "k8s.io/component-base/tracing"
 )
 
 // CoreAPIOptions contains options to configure the connection to a core API Kubernetes apiserver.
@@ -69,6 +72,9 @@ func (o *CoreAPIOptions) ApplyTo(config *server.RecommendedConfig) error {
 			return err
 		}
 	}
+	if feature.DefaultFeatureGate.Enabled(features.APIServerTracing) {
+		kubeconfig.Wrap(tracing.WrapperFor(config.TracerProvider))
+	}
 	clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeconfig)
 	if err != nil {
 		return fmt.Errorf("failed to create Kubernetes clientset: %v", err)

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/OWNERS

@@ -1,9 +1,8 @@
 # See the OWNERS docs at https://go.k8s.io/owners
 
 approvers:
-- sig-auth-encryption-at-rest-approvers
+  - sig-auth-encryption-at-rest-approvers
 reviewers:
-- sig-auth-encryption-at-rest-reviewers
+  - sig-auth-encryption-at-rest-reviewers
 labels:
-- sig/auth
-
+  - sig/auth

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -17,6 +17,7 @@ limitations under the License.
 package encryptionconfig
 
 import (
+	"context"
 	"crypto/aes"
 	"crypto/cipher"
 	"encoding/base64"
@@ -32,15 +33,19 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	apiserverconfig "k8s.io/apiserver/pkg/apis/config"
 	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
 	"k8s.io/apiserver/pkg/apis/config/validation"
+	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/storage/value"
 	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
 	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
+	envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
 	"k8s.io/apiserver/pkg/storage/value/encrypt/identity"
 	"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 )
 
 const (
@@ -48,8 +53,11 @@ const (
 	aesGCMTransformerPrefixV1    = "k8s:enc:aesgcm:v1:"
 	secretboxTransformerPrefixV1 = "k8s:enc:secretbox:v1:"
 	kmsTransformerPrefixV1       = "k8s:enc:kms:v1:"
+	kmsTransformerPrefixV2       = "k8s:enc:kms:v2:"
 	kmsPluginHealthzNegativeTTL  = 3 * time.Second
 	kmsPluginHealthzPositiveTTL  = 20 * time.Second
+	kmsAPIVersionV1              = "v1"
+	kmsAPIVersionV2              = "v2"
 )
 
 type kmsPluginHealthzResponse struct {
@@ -65,12 +73,26 @@ type kmsPluginProbe struct {
 	l            *sync.Mutex
 }
 
+type kmsv2PluginProbe struct {
+	name string
+	ttl  time.Duration
+	envelopekmsv2.Service
+	lastResponse *kmsPluginHealthzResponse
+	l            *sync.Mutex
+}
+
 func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
 	return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
 		return h.Check()
 	})
 }
 
+func (p *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
+	return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
+		return p.Check()
+	})
+}
+
 // GetKMSPluginHealthzCheckers extracts KMSPluginProbes from the EncryptionConfig.
 func GetKMSPluginHealthzCheckers(filepath string) ([]healthz.HealthChecker, error) {
 	f, err := os.Open(filepath)
@@ -78,47 +100,79 @@ func GetKMSPluginHealthzCheckers(filepath string) ([]healthz.HealthChecker, erro
 		return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err)
 	}
 	defer f.Close()
+
 	var result []healthz.HealthChecker
 	probes, err := getKMSPluginProbes(f)
 	if err != nil {
 		return nil, err
 	}
-
 	for i, p := range probes {
 		probe := p
-		result = append(result, probe.toHealthzCheck(i))
+		switch t := probe.(type) {
+		case *kmsPluginProbe:
+			result = append(result, t.toHealthzCheck(i))
+		case *kmsv2PluginProbe:
+			result = append(result, t.toHealthzCheck(i))
+		default:
+			return nil, fmt.Errorf("unsupported KMS plugin type: %T", t)
+		}
 	}
+
 	return result, nil
 }
 
-func getKMSPluginProbes(reader io.Reader) ([]*kmsPluginProbe, error) {
-	var result []*kmsPluginProbe
+func getKMSPluginProbes(reader io.Reader) ([]interface{}, error) {
+	var result []interface{}
 
 	configFileContents, err := ioutil.ReadAll(reader)
 	if err != nil {
-		return result, fmt.Errorf("could not read content of encryption provider configuration: %v", err)
+		return nil, fmt.Errorf("could not read content of encryption provider configuration: %v", err)
 	}
 
 	config, err := loadConfig(configFileContents)
 	if err != nil {
-		return result, fmt.Errorf("error while parsing encrypiton provider configuration: %v", err)
+		return nil, fmt.Errorf("error while parsing encryption provider configuration: %v", err)
 	}
 
 	for _, r := range config.Resources {
 		for _, p := range r.Providers {
 			if p.KMS != nil {
-				s, err := envelope.NewGRPCService(p.KMS.Endpoint, p.KMS.Timeout.Duration)
-				if err != nil {
-					return nil, fmt.Errorf("could not configure KMS-Plugin's probe %q, error: %v", p.KMS.Name, err)
-				}
+				switch p.KMS.APIVersion {
+				case kmsAPIVersionV1:
+					s, err := envelope.NewGRPCService(p.KMS.Endpoint, p.KMS.Timeout.Duration)
+					if err != nil {
+						return nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %v", p.KMS.Name, err)
+					}
 
-				result = append(result, &kmsPluginProbe{
-					name:         p.KMS.Name,
-					ttl:          kmsPluginHealthzNegativeTTL,
-					Service:      s,
-					l:            &sync.Mutex{},
-					lastResponse: &kmsPluginHealthzResponse{},
-				})
+					result = append(result, &kmsPluginProbe{
+						name:         p.KMS.Name,
+						ttl:          kmsPluginHealthzNegativeTTL,
+						Service:      s,
+						l:            &sync.Mutex{},
+						lastResponse: &kmsPluginHealthzResponse{},
+					})
+
+				case kmsAPIVersionV2:
+					if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
+						return nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, KMSv2 feature is not enabled", p.KMS.Name)
+					}
+
+					s, err := envelopekmsv2.NewGRPCService(p.KMS.Endpoint, p.KMS.Timeout.Duration)
+					if err != nil {
+						return nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %v", p.KMS.Name, err)
+					}
+
+					result = append(result, &kmsv2PluginProbe{
+						name:         p.KMS.Name,
+						ttl:          kmsPluginHealthzNegativeTTL,
+						Service:      s,
+						l:            &sync.Mutex{},
+						lastResponse: &kmsPluginHealthzResponse{},
+					})
+
+				default:
+					return nil, fmt.Errorf("could not configure KMS Plugin's probe %q, unsupported KMS API version %q", p.KMS.Name, p.KMS.APIVersion)
+				}
 			}
 		}
 	}
@@ -153,6 +207,53 @@ func (h *kmsPluginProbe) Check() error {
 	return nil
 }
 
+// Check gets the healthz status of the KMSv2-Plugin using the Status() method.
+func (h *kmsv2PluginProbe) Check() error {
+	h.l.Lock()
+	defer h.l.Unlock()
+
+	if (time.Since(h.lastResponse.received)) < h.ttl {
+		return h.lastResponse.err
+	}
+
+	ctx := context.Background()
+	p, err := h.Service.Status(ctx)
+	if err != nil {
+		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
+		h.ttl = kmsPluginHealthzNegativeTTL
+		return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %v", h.name, err)
+	}
+
+	if err := isKMSv2ProviderHealthy(h.name, p); err != nil {
+		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
+		h.ttl = kmsPluginHealthzNegativeTTL
+		return err
+	}
+
+	h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()}
+	h.ttl = kmsPluginHealthzPositiveTTL
+	return nil
+}
+
+// isKMSv2ProviderHealthy checks if the KMSv2-Plugin is healthy.
+func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse) error {
+	var errs []error
+	if response.Healthz != "ok" {
+		errs = append(errs, fmt.Errorf("got unexpected healthz status: %s", response.Healthz))
+	}
+	if response.Version != envelopekmsv2.KMSAPIVersion {
+		errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersion, response.Version))
+	}
+	if len(response.KeyID) == 0 {
+		errs = append(errs, fmt.Errorf("expected KMSv2 KeyID to be set, got %s", response.KeyID))
+	}
+
+	if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil {
+		return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %v", name, err)
+	}
+	return nil
+}
+
 // GetTransformerOverrides returns the transformer overrides by reading and parsing the encryption provider configuration file
 func GetTransformerOverrides(filepath string) (map[schema.GroupResource]value.Transformer, error) {
 	f, err := os.Open(filepath)
@@ -222,8 +323,13 @@ func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) {
 	return config, validation.ValidateEncryptionConfiguration(config).ToAggregate()
 }
 
-// The factory to create kms service. This is to make writing test easier.
-var envelopeServiceFactory = envelope.NewGRPCService
+var (
+	// The factory to create kms service. This is to make writing test easier.
+	envelopeServiceFactory = envelope.NewGRPCService
+
+	// The factory to create kmsv2 service.
+	envelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService
+)
 
 func prefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]value.PrefixTransformer, error) {
 	var result []value.PrefixTransformer
@@ -241,13 +347,26 @@ func prefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]value.
 		case provider.Secretbox != nil:
 			transformer, err = secretboxPrefixTransformer(provider.Secretbox)
 		case provider.KMS != nil:
-			var envelopeService envelope.Service
-			envelopeService, err = envelopeServiceFactory(provider.KMS.Endpoint, provider.KMS.Timeout.Duration)
-			if err != nil {
-				return nil, fmt.Errorf("could not configure KMS plugin %q, error: %v", provider.KMS.Name, err)
-			}
+			switch provider.KMS.APIVersion {
+			case kmsAPIVersionV1:
+				var envelopeService envelope.Service
+				if envelopeService, err = envelopeServiceFactory(provider.KMS.Endpoint, provider.KMS.Timeout.Duration); err != nil {
+					return nil, fmt.Errorf("could not configure KMS plugin %q, error: %v", provider.KMS.Name, err)
+				}
+				transformer, err = envelopePrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV1)
+			case kmsAPIVersionV2:
+				if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
+					return nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", provider.KMS.Name)
+				}
 
-			transformer, err = envelopePrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV1)
+				var envelopeService envelopekmsv2.Service
+				if envelopeService, err = envelopeKMSv2ServiceFactory(provider.KMS.Endpoint, provider.KMS.Timeout.Duration); err != nil {
+					return nil, fmt.Errorf("could not configure KMSv2 plugin %q, error: %v", provider.KMS.Name, err)
+				}
+				transformer, err = envelopekmsv2PrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV2)
+			default:
+				return nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", provider.KMS.Name, provider.KMS.APIVersion)
+			}
 		case provider.Identity != nil:
 			transformer = value.PrefixTransformer{
 				Transformer: identity.NewEncryptCheckTransformer(),
@@ -365,7 +484,15 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
 }
 
 func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) {
-	envelopeTransformer, err := envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), aestransformer.NewCBCTransformer)
+	baseTransformerFunc := func(block cipher.Block) value.Transformer {
+		// v1.24: write using AES-CBC only but support reads via AES-CBC and AES-GCM (so we can move to AES-GCM)
+		// v1.25: write using AES-GCM only but support reads via AES-GCM and fallback to AES-CBC for backwards compatibility
+		// TODO(aramase): Post v1.25: We cannot drop CBC read support until we automate storage migration.
+		// We could have a release note that hard requires users to perform storage migration.
+		return unionTransformers{aestransformer.NewGCMTransformer(block), aestransformer.NewCBCTransformer(block)}
+	}
+
+	envelopeTransformer, err := envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc)
 	if err != nil {
 		return value.PrefixTransformer{}, err
 	}
@@ -374,3 +501,39 @@ func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelop
 		Prefix:      []byte(prefix + config.Name + ":"),
 	}, nil
 }
+
+func envelopekmsv2PrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelopekmsv2.Service, prefix string) (value.PrefixTransformer, error) {
+	// using AES-GCM by default for encrypting data with KMSv2
+	envelopeTransformer, err := envelopekmsv2.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), aestransformer.NewGCMTransformer)
+	if err != nil {
+		return value.PrefixTransformer{}, err
+	}
+	return value.PrefixTransformer{
+		Transformer: envelopeTransformer,
+		Prefix:      []byte(prefix + config.Name + ":"),
+	}, nil
+}
+
+type unionTransformers []value.Transformer
+
+func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, stale bool, err error) {
+	var errs []error
+	for i, transformer := range u {
+		result, stale, err := transformer.TransformFromStorage(ctx, data, dataCtx)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		// when i != 0, we have transformed the data from storage using the new transformer,
+		// we want to issue a write to etcd even if the contents of the data haven't changed
+		return result, stale || i != 0, nil
+	}
+	if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil {
+		return nil, false, err
+	}
+	return nil, false, fmt.Errorf("unionTransformers: unable to transform from storage")
+}
+
+func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, err error) {
+	return u[0].TransformToStorage(ctx, data, dataCtx)
+}

vendor/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -137,21 +137,22 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&s.DefaultWatchCacheSize, "default-watch-cache-size", s.DefaultWatchCacheSize,
 		"Default watch cache size. If zero, watch cache will be disabled for resources that do not have a default watch size set.")
 
+	fs.MarkDeprecated("default-watch-cache-size",
+		"watch caches are sized automatically and this flag will be removed in a future version")
+
 	fs.StringSliceVar(&s.WatchCacheSizes, "watch-cache-sizes", s.WatchCacheSizes, ""+
 		"Watch cache size settings for some resources (pods, nodes, etc.), comma separated. "+
 		"The individual setting format: resource[.group]#size, where resource is lowercase plural (no version), "+
 		"group is omitted for resources of apiVersion v1 (the legacy core API) and included for others, "+
-		"and size is a number. It takes effect when watch-cache is enabled. "+
-		"Some resources (replicationcontrollers, endpoints, nodes, pods, services, apiservices.apiregistration.k8s.io) "+
-		"have system defaults set by heuristics, others default to default-watch-cache-size")
+		"and size is a number. This option is only meaningful for resources built into the apiserver, "+
+		"not ones defined by CRDs or aggregated from external servers, and is only consulted if the "+
+		"watch-cache is enabled. The only meaningful size setting to supply here is zero, which means to "+
+		"disable watch caching for the associated resource; all non-zero values are equivalent and mean "+
+		"to not disable watch caching for that resource")
 
 	fs.StringVar(&s.StorageConfig.Type, "storage-backend", s.StorageConfig.Type,
 		"The storage backend for persistence. Options: 'etcd3' (default).")
 
-	dummyCacheSize := 0
-	fs.IntVar(&dummyCacheSize, "deserialization-cache-size", 0, "Number of deserialized json objects to cache in memory.")
-	fs.MarkDeprecated("deserialization-cache-size", "the deserialization cache was dropped in 1.13 with support for etcd2")
-
 	fs.StringSliceVar(&s.StorageConfig.Transport.ServerList, "etcd-servers", s.StorageConfig.Transport.ServerList,
 		"List of etcd servers to connect with (scheme://ip:port), comma separated.")
 
@@ -167,10 +168,6 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.StorageConfig.Transport.TrustedCAFile, "etcd-cafile", s.StorageConfig.Transport.TrustedCAFile,
 		"SSL Certificate Authority file used to secure etcd communication.")
 
-	fs.StringVar(&s.EncryptionProviderConfigFilepath, "experimental-encryption-provider-config", s.EncryptionProviderConfigFilepath,
-		"The file containing configuration for encryption providers to be used for storing secrets in etcd")
-	fs.MarkDeprecated("experimental-encryption-provider-config", "use --encryption-provider-config.")
-
 	fs.StringVar(&s.EncryptionProviderConfigFilepath, "encryption-provider-config", s.EncryptionProviderConfigFilepath,
 		"The file containing configuration for encryption providers to be used for storing secrets in etcd")
 
@@ -186,6 +183,9 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&s.StorageConfig.HealthcheckTimeout, "etcd-healthcheck-timeout", s.StorageConfig.HealthcheckTimeout,
 		"The timeout to use when checking etcd health.")
 
+	fs.DurationVar(&s.StorageConfig.ReadycheckTimeout, "etcd-readycheck-timeout", s.StorageConfig.ReadycheckTimeout,
+		"The timeout to use when checking etcd readiness")
+
 	fs.Int64Var(&s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds, "lease-reuse-duration-seconds", s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds,
 		"The time in seconds that each lease is reused. A lower value could avoid large number of objects reusing the same lease. Notice that a too small value may cause performance problems at storage layer.")
 }
@@ -206,6 +206,9 @@ func (s *EtcdOptions) ApplyTo(c *server.Config) error {
 		}
 	}
 
+	// use the StorageObjectCountTracker interface instance from server.Config
+	s.StorageConfig.StorageObjectCountTracker = c.StorageObjectCountTracker
+
 	c.RESTOptionsGetter = &SimpleRestOptionsFactory{
 		Options:              *s,
 		TransformerOverrides: transformerOverrides,
@@ -217,12 +220,16 @@ func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFac
 	if err := s.addEtcdHealthEndpoint(c); err != nil {
 		return err
 	}
+
+	// use the StorageObjectCountTracker interface instance from server.Config
+	s.StorageConfig.StorageObjectCountTracker = c.StorageObjectCountTracker
+
 	c.RESTOptionsGetter = &StorageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory}
 	return nil
 }
 
 func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
-	healthCheck, err := storagefactory.CreateHealthCheck(s.StorageConfig)
+	healthCheck, err := storagefactory.CreateHealthCheck(s.StorageConfig, c.DrainedNotify())
 	if err != nil {
 		return err
 	}
@@ -230,6 +237,14 @@ func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
 		return healthCheck()
 	}))
 
+	readyCheck, err := storagefactory.CreateReadyCheck(s.StorageConfig, c.DrainedNotify())
+	if err != nil {
+		return err
+	}
+	c.AddReadyzChecks(healthz.NamedCheck("etcd-readiness", func(r *http.Request) error {
+		return readyCheck()
+	}))
+
 	if s.EncryptionProviderConfigFilepath != "" {
 		kmsPluginHealthzChecks, err := encryptionconfig.GetKMSPluginHealthzCheckers(s.EncryptionProviderConfigFilepath)
 		if err != nil {
@@ -248,12 +263,13 @@ type SimpleRestOptionsFactory struct {
 
 func (f *SimpleRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
 	ret := generic.RESTOptions{
-		StorageConfig:           &f.Options.StorageConfig,
-		Decorator:               generic.UndecoratedStorage,
-		EnableGarbageCollection: f.Options.EnableGarbageCollection,
-		DeleteCollectionWorkers: f.Options.DeleteCollectionWorkers,
-		ResourcePrefix:          resource.Group + "/" + resource.Resource,
-		CountMetricPollPeriod:   f.Options.StorageConfig.CountMetricPollPeriod,
+		StorageConfig:             f.Options.StorageConfig.ForResource(resource),
+		Decorator:                 generic.UndecoratedStorage,
+		EnableGarbageCollection:   f.Options.EnableGarbageCollection,
+		DeleteCollectionWorkers:   f.Options.DeleteCollectionWorkers,
+		ResourcePrefix:            resource.Group + "/" + resource.Resource,
+		CountMetricPollPeriod:     f.Options.StorageConfig.CountMetricPollPeriod,
+		StorageObjectCountTracker: f.Options.StorageConfig.StorageObjectCountTracker,
 	}
 	if f.TransformerOverrides != nil {
 		if transformer, ok := f.TransformerOverrides[resource]; ok {
@@ -270,8 +286,10 @@ func (f *SimpleRestOptionsFactory) GetRESTOptions(resource schema.GroupResource)
 			klog.Warningf("Dropping watch-cache-size for %v - watchCache size is now dynamic", resource)
 		}
 		if ok && size <= 0 {
+			klog.V(3).InfoS("Not using watch cache", "resource", resource)
 			ret.Decorator = generic.UndecoratedStorage
 		} else {
+			klog.V(3).InfoS("Using watch cache", "resource", resource)
 			ret.Decorator = genericregistry.StorageWithCacher()
 		}
 	}
@@ -290,12 +308,13 @@ func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupR
 	}
 
 	ret := generic.RESTOptions{
-		StorageConfig:           storageConfig,
-		Decorator:               generic.UndecoratedStorage,
-		DeleteCollectionWorkers: f.Options.DeleteCollectionWorkers,
-		EnableGarbageCollection: f.Options.EnableGarbageCollection,
-		ResourcePrefix:          f.StorageFactory.ResourcePrefix(resource),
-		CountMetricPollPeriod:   f.Options.StorageConfig.CountMetricPollPeriod,
+		StorageConfig:             storageConfig,
+		Decorator:                 generic.UndecoratedStorage,
+		DeleteCollectionWorkers:   f.Options.DeleteCollectionWorkers,
+		EnableGarbageCollection:   f.Options.EnableGarbageCollection,
+		ResourcePrefix:            f.StorageFactory.ResourcePrefix(resource),
+		CountMetricPollPeriod:     f.Options.StorageConfig.CountMetricPollPeriod,
+		StorageObjectCountTracker: f.Options.StorageConfig.StorageObjectCountTracker,
 	}
 	if f.Options.EnableWatchCache {
 		sizes, err := ParseWatchCacheSizes(f.Options.WatchCacheSizes)

vendor/k8s.io/apiserver/pkg/server/options/feature.go

@@ -46,9 +46,6 @@ func (o *FeatureOptions) AddFlags(fs *pflag.FlagSet) {
 		"Enable profiling via web interface host:port/debug/pprof/")
 	fs.BoolVar(&o.EnableContentionProfiling, "contention-profiling", o.EnableContentionProfiling,
 		"Enable lock contention profiling, if profiling is enabled")
-	dummy := false
-	fs.BoolVar(&dummy, "enable-swagger-ui", dummy, "Enables swagger ui on the apiserver at /swagger-ui")
-	fs.MarkDeprecated("enable-swagger-ui", "swagger 1.2 support has been removed")
 }
 
 func (o *FeatureOptions) ApplyTo(c *server.Config) error {

vendor/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -17,6 +17,8 @@ limitations under the License.
 package options
 
 import (
+	"fmt"
+
 	"github.com/spf13/pflag"
 
 	"k8s.io/apimachinery/pkg/runtime"
@@ -51,6 +53,8 @@ type RecommendedOptions struct {
 	Admission                  *AdmissionOptions
 	// API Server Egress Selector is used to control outbound traffic from the API Server
 	EgressSelector *EgressSelectorOptions
+	// Traces contains options to control distributed request tracing.
+	Traces *TracingOptions
 }
 
 func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptions {
@@ -77,6 +81,7 @@ func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptio
 		ExtraAdmissionInitializers: func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error) { return nil, nil },
 		Admission:                  NewAdmissionOptions(),
 		EgressSelector:             NewEgressSelectorOptions(),
+		Traces:                     NewTracingOptions(),
 	}
 }
 
@@ -90,6 +95,7 @@ func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) {
 	o.CoreAPI.AddFlags(fs)
 	o.Admission.AddFlags(fs)
 	o.EgressSelector.AddFlags(fs)
+	o.Traces.AddFlags(fs)
 }
 
 // ApplyTo adds RecommendedOptions to the server configuration.
@@ -98,6 +104,12 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if err := o.Etcd.ApplyTo(&config.Config); err != nil {
 		return err
 	}
+	if err := o.EgressSelector.ApplyTo(&config.Config); err != nil {
+		return err
+	}
+	if err := o.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil {
+		return err
+	}
 	if err := o.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil {
 		return err
 	}
@@ -121,14 +133,15 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	} else if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, config.ClientConfig, o.FeatureGate, initializers...); err != nil {
 		return err
 	}
-	if err := o.EgressSelector.ApplyTo(&config.Config); err != nil {
-		return err
-	}
 	if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
 		if config.ClientConfig != nil {
+			if config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight <= 0 {
+				return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", config.MaxRequestsInFlight, config.MaxMutatingRequestsInFlight)
+
+			}
 			config.FlowControl = utilflowcontrol.New(
 				config.SharedInformerFactory,
-				kubernetes.NewForConfigOrDie(config.ClientConfig).FlowcontrolV1beta1(),
+				kubernetes.NewForConfigOrDie(config.ClientConfig).FlowcontrolV1beta2(),
 				config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight,
 				config.RequestTimeout/4,
 			)
@@ -150,6 +163,7 @@ func (o *RecommendedOptions) Validate() []error {
 	errors = append(errors, o.CoreAPI.Validate()...)
 	errors = append(errors, o.Admission.Validate()...)
 	errors = append(errors, o.EgressSelector.Validate()...)
+	errors = append(errors, o.Traces.Validate()...)
 
 	return errors
 }

vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -54,6 +54,15 @@ type ServerRunOptions struct {
 	// apiserver library can wire it to a flag.
 	MaxRequestBodyBytes       int64
 	EnablePriorityAndFairness bool
+
+	// ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP
+	// Server during the graceful termination of the apiserver. If true, we wait
+	// for non longrunning requests in flight to be drained and then initiate a
+	// shutdown of the HTTP Server. If false, we initiate a shutdown of the HTTP
+	// Server as soon as ShutdownDelayDuration has elapsed.
+	// If enabled, after ShutdownDelayDuration elapses, any incoming request is
+	// rejected with a 429 status code and a 'Retry-After' response.
+	ShutdownSendRetryAfter bool
 }
 
 func NewServerRunOptions() *ServerRunOptions {
@@ -68,6 +77,7 @@ func NewServerRunOptions() *ServerRunOptions {
 		JSONPatchMaxCopyBytes:       defaults.JSONPatchMaxCopyBytes,
 		MaxRequestBodyBytes:         defaults.MaxRequestBodyBytes,
 		EnablePriorityAndFairness:   true,
+		ShutdownSendRetryAfter:      false,
 	}
 }
 
@@ -86,6 +96,7 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
 	c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
 	c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
 	c.PublicAddress = s.AdvertiseAddress
+	c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
 
 	return nil
 }
@@ -140,11 +151,11 @@ func (s *ServerRunOptions) Validate() []error {
 	}
 
 	if s.JSONPatchMaxCopyBytes < 0 {
-		errors = append(errors, fmt.Errorf("--json-patch-max-copy-bytes can not be negative value"))
+		errors = append(errors, fmt.Errorf("ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value"))
 	}
 
 	if s.MaxRequestBodyBytes < 0 {
-		errors = append(errors, fmt.Errorf("--max-resource-write-bytes can not be negative value"))
+		errors = append(errors, fmt.Errorf("ServerRunOptions.MaxRequestBodyBytes can not be negative value"))
 	}
 
 	if err := validateHSTSDirectives(s.HSTSDirectives); err != nil {
@@ -191,11 +202,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"List of directives for HSTS, comma separated. If this list is empty, then HSTS directives will not "+
 		"be added. Example: 'max-age=31536000,includeSubDomains,preload'")
 
-	deprecatedTargetRAMMB := 0
-	fs.IntVar(&deprecatedTargetRAMMB, "target-ram-mb", deprecatedTargetRAMMB,
-		"DEPRECATED: Memory limit for apiserver in MB (used to configure sizes of caches, etc.)")
-	fs.MarkDeprecated("target-ram-mb", "This flag will be removed in v1.23")
-
 	fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
 		"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs or OpenID Discovery).")
 
@@ -204,12 +210,16 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"DEPRECATED: the namespace from which the Kubernetes master services should be injected into pods.")
 
 	fs.IntVar(&s.MaxRequestsInFlight, "max-requests-inflight", s.MaxRequestsInFlight, ""+
-		"The maximum number of non-mutating requests in flight at a given time. When the server exceeds this, "+
-		"it rejects requests. Zero for no limit.")
+		"This and --max-mutating-requests-inflight are summed to determine the server's total concurrency limit "+
+		"(which must be positive) if --enable-priority-and-fairness is true. "+
+		"Otherwise, this flag limits the maximum number of non-mutating requests in flight, "+
+		"or a zero value disables the limit completely.")
 
 	fs.IntVar(&s.MaxMutatingRequestsInFlight, "max-mutating-requests-inflight", s.MaxMutatingRequestsInFlight, ""+
-		"The maximum number of mutating requests in flight at a given time. When the server exceeds this, "+
-		"it rejects requests. Zero for no limit.")
+		"This and --max-requests-inflight are summed to determine the server's total concurrency limit "+
+		"(which must be positive) if --enable-priority-and-fairness is true. "+
+		"Otherwise, this flag limits the maximum number of mutating requests in flight, "+
+		"or a zero value disables the limit completely.")
 
 	fs.DurationVar(&s.RequestTimeout, "request-timeout", s.RequestTimeout, ""+
 		"An optional field indicating the duration a handler must keep a request open before timing "+
@@ -241,5 +251,10 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
 		"has elapsed. This can be used to allow load balancer to stop sending traffic to this server.")
 
+	fs.BoolVar(&s.ShutdownSendRetryAfter, "shutdown-send-retry-after", s.ShutdownSendRetryAfter, ""+
+		"If true the HTTP Server will continue listening until all non long running request(s) in flight have been drained, "+
+		"during this window all incoming requests will be rejected with a status code 429 and a 'Retry-After' response header, "+
+		"in addition 'Connection: close' response header is set in order to tear down the TCP connection when idle.")
+
 	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
 }

vendor/k8s.io/apiserver/pkg/server/options/serving.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/spf13/pflag"
 	"k8s.io/klog/v2"
+	netutils "k8s.io/utils/net"
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/server"
@@ -108,7 +109,7 @@ type GeneratableKeyCert struct {
 
 func NewSecureServingOptions() *SecureServingOptions {
 	return &SecureServingOptions{
-		BindAddress: net.ParseIP("0.0.0.0"),
+		BindAddress: netutils.ParseIPSloppy("0.0.0.0"),
 		BindPort:    443,
 		ServerCert: GeneratableKeyCert{
 			PairName:      "apiserver",

vendor/k8s.io/apiserver/pkg/server/options/serving_unix.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 /*

vendor/k8s.io/apiserver/pkg/server/options/serving_windows.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 /*

vendor/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go

@@ -60,10 +60,14 @@ func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.Se
 		return fmt.Errorf("failed to generate self-signed certificate for loopback connection: %v", err)
 	}
 
+	// Write to the front of SNICerts so that this overrides any other certs with the same name
+	(*secureServingInfo).SNICerts = append([]dynamiccertificates.SNICertKeyContentProvider{certProvider}, (*secureServingInfo).SNICerts...)
+
 	secureLoopbackClientConfig, err := (*secureServingInfo).NewLoopbackClientConfig(uuid.New().String(), certPem)
 	switch {
 	// if we failed and there's no fallback loopback client config, we need to fail
 	case err != nil && *loopbackClientConfig == nil:
+		(*secureServingInfo).SNICerts = (*secureServingInfo).SNICerts[1:]
 		return err
 
 	// if we failed, but we already have a fallback loopback client config (usually insecure), allow it
@@ -71,8 +75,6 @@ func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.Se
 
 	default:
 		*loopbackClientConfig = secureLoopbackClientConfig
-		// Write to the front of SNICerts so that this overrides any other certs with the same name
-		(*secureServingInfo).SNICerts = append([]dynamiccertificates.SNICertKeyContentProvider{certProvider}, (*secureServingInfo).SNICerts...)
 	}
 
 	return nil

vendor/k8s.io/apiserver/pkg/server/options/tracing.go

@@ -0,0 +1,161 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net"
+
+	"github.com/spf13/pflag"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpgrpc"
+	"go.opentelemetry.io/otel/sdk/resource"
+	"go.opentelemetry.io/otel/semconv"
+	"google.golang.org/grpc"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/apiserver/pkg/apis/apiserver/install"
+	"k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/egressselector"
+	"k8s.io/apiserver/pkg/util/feature"
+	tracing "k8s.io/component-base/tracing"
+	tracingapi "k8s.io/component-base/tracing/api/v1"
+	"k8s.io/utils/path"
+)
+
+const apiserverService = "apiserver"
+
+var (
+	cfgScheme = runtime.NewScheme()
+	codecs    = serializer.NewCodecFactory(cfgScheme)
+)
+
+func init() {
+	install.Install(cfgScheme)
+}
+
+// TracingOptions contain configuration options for tracing
+// exporters
+type TracingOptions struct {
+	// ConfigFile is the file path with api-server tracing configuration.
+	ConfigFile string
+}
+
+// NewTracingOptions creates a new instance of TracingOptions
+func NewTracingOptions() *TracingOptions {
+	return &TracingOptions{}
+}
+
+// AddFlags adds flags related to tracing to the specified FlagSet
+func (o *TracingOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.ConfigFile, "tracing-config-file", o.ConfigFile,
+		"File with apiserver tracing configuration.")
+}
+
+// ApplyTo fills up Tracing config with options.
+func (o *TracingOptions) ApplyTo(es *egressselector.EgressSelector, c *server.Config) error {
+	if o == nil || o.ConfigFile == "" {
+		return nil
+	}
+	if !feature.DefaultFeatureGate.Enabled(features.APIServerTracing) {
+		return fmt.Errorf("APIServerTracing feature is not enabled, but tracing config file was provided")
+	}
+
+	traceConfig, err := ReadTracingConfiguration(o.ConfigFile)
+	if err != nil {
+		return fmt.Errorf("failed to read tracing config: %v", err)
+	}
+
+	errs := tracingapi.ValidateTracingConfiguration(traceConfig, feature.DefaultFeatureGate, nil)
+	if len(errs) > 0 {
+		return fmt.Errorf("failed to validate tracing configuration: %v", errs.ToAggregate())
+	}
+
+	opts := []otlpgrpc.Option{}
+	if es != nil {
+		// Only use the egressselector dialer if egressselector is enabled.
+		// Endpoint is on the "ControlPlane" network
+		egressDialer, err := es.Lookup(egressselector.ControlPlane.AsNetworkContext())
+		if err != nil {
+			return err
+		}
+
+		otelDialer := func(ctx context.Context, addr string) (net.Conn, error) {
+			return egressDialer(ctx, "tcp", addr)
+		}
+		opts = append(opts, otlpgrpc.WithDialOption(grpc.WithContextDialer(otelDialer)))
+	}
+
+	resourceOpts := []resource.Option{
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(apiserverService),
+			semconv.ServiceInstanceIDKey.String(c.APIServerID),
+		),
+	}
+	tp, err := tracing.NewProvider(context.Background(), traceConfig, opts, resourceOpts)
+	if err != nil {
+		return err
+	}
+	c.TracerProvider = tp
+	if c.LoopbackClientConfig != nil {
+		c.LoopbackClientConfig.Wrap(tracing.WrapperFor(c.TracerProvider))
+	}
+	return nil
+}
+
+// Validate verifies flags passed to TracingOptions.
+func (o *TracingOptions) Validate() (errs []error) {
+	if o == nil || o.ConfigFile == "" {
+		return
+	}
+
+	if exists, err := path.Exists(path.CheckFollowSymlink, o.ConfigFile); !exists {
+		errs = append(errs, fmt.Errorf("tracing-config-file %s does not exist", o.ConfigFile))
+	} else if err != nil {
+		errs = append(errs, fmt.Errorf("error checking if tracing-config-file %s exists: %v", o.ConfigFile, err))
+	}
+	return
+}
+
+// ReadTracingConfiguration reads the tracing configuration from a file
+func ReadTracingConfiguration(configFilePath string) (*tracingapi.TracingConfiguration, error) {
+	if configFilePath == "" {
+		return nil, fmt.Errorf("tracing config file was empty")
+	}
+	data, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read tracing configuration from %q: %v", configFilePath, err)
+	}
+	internalConfig := &apiserver.TracingConfiguration{}
+	// this handles json/yaml/whatever, and decodes all registered version to the internal version
+	if err := runtime.DecodeInto(codecs.UniversalDecoder(), data, internalConfig); err != nil {
+		return nil, fmt.Errorf("unable to decode tracing configuration data: %v", err)
+	}
+	tc := &tracingapi.TracingConfiguration{
+		Endpoint:               internalConfig.Endpoint,
+		SamplingRatePerMillion: internalConfig.SamplingRatePerMillion,
+	}
+	return tc, nil
+}