use istio client-go library instead of knative (#1661)

use istio client-go library instead of knative
bump kubernetes dependency version
change code coverage to codecov

vendor/k8s.io/apiserver/pkg/server/options/OWNERS

@@ -1,3 +1,5 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
 reviewers:
 - smarterclayton
 - wojtek-t

vendor/k8s.io/apiserver/pkg/server/options/admission.go

@@ -28,7 +28,6 @@ import (
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/admission/initializer"
 	admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
-	"k8s.io/apiserver/pkg/admission/plugin/initialization"
 	"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
 	mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
 	validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
@@ -62,6 +61,8 @@ type AdmissionOptions struct {
 	ConfigFile string
 	// Plugins contains all registered plugins.
 	Plugins *admission.Plugins
+	// Decorators is a list of admission decorator to wrap around the admission plugins
+	Decorators admission.Decorators
 }
 
 // NewAdmissionOptions creates a new instance of AdmissionOptions
@@ -74,13 +75,14 @@ type AdmissionOptions struct {
 //  Servers that do care can overwrite/append that field after creation.
 func NewAdmissionOptions() *AdmissionOptions {
 	options := &AdmissionOptions{
-		Plugins: admission.NewPlugins(),
+		Plugins:    admission.NewPlugins(),
+		Decorators: admission.Decorators{admission.DecoratorFunc(admissionmetrics.WithControllerMetrics)},
 		// This list is mix of mutating admission plugins and validating
 		// admission plugins. The apiserver always runs the validating ones
 		// after all the mutating ones, so their relative order in this list
 		// doesn't matter.
-		RecommendedPluginOrder: []string{lifecycle.PluginName, initialization.PluginName, mutatingwebhook.PluginName, validatingwebhook.PluginName},
-		DefaultOffPlugins:      sets.NewString(initialization.PluginName),
+		RecommendedPluginOrder: []string{lifecycle.PluginName, mutatingwebhook.PluginName, validatingwebhook.PluginName},
+		DefaultOffPlugins:      sets.NewString(),
 	}
 	server.RegisterAllAdmissionPlugins(options.Plugins)
 	return options
@@ -115,18 +117,12 @@ func (a *AdmissionOptions) ApplyTo(
 	c *server.Config,
 	informers informers.SharedInformerFactory,
 	kubeAPIServerClientConfig *rest.Config,
-	scheme *runtime.Scheme,
 	pluginInitializers ...admission.PluginInitializer,
 ) error {
 	if a == nil {
 		return nil
 	}
 
-	// Admission need scheme to construct admission initializer.
-	if scheme == nil {
-		return fmt.Errorf("admission depends on a scheme, it cannot be nil")
-	}
-
 	// Admission depends on CoreAPI to set SharedInformerFactory and ClientConfig.
 	if informers == nil {
 		return fmt.Errorf("admission depends on a Kubernetes core API shared informer, it cannot be nil")
@@ -143,12 +139,12 @@ func (a *AdmissionOptions) ApplyTo(
 	if err != nil {
 		return err
 	}
-	genericInitializer := initializer.New(clientset, informers, c.Authorization.Authorizer, scheme)
+	genericInitializer := initializer.New(clientset, informers, c.Authorization.Authorizer)
 	initializersChain := admission.PluginInitializers{}
 	pluginInitializers = append(pluginInitializers, genericInitializer)
 	initializersChain = append(initializersChain, pluginInitializers...)
 
-	admissionChain, err := a.Plugins.NewFromPlugins(pluginNames, pluginsConfigProvider, initializersChain, admission.DecoratorFunc(admissionmetrics.WithControllerMetrics))
+	admissionChain, err := a.Plugins.NewFromPlugins(pluginNames, pluginsConfigProvider, initializersChain, a.Decorators)
 	if err != nil {
 		return err
 	}

vendor/k8s.io/apiserver/pkg/server/options/api_enablement.go

@@ -25,18 +25,18 @@ import (
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/resourceconfig"
 	serverstore "k8s.io/apiserver/pkg/server/storage"
-	utilflag "k8s.io/apiserver/pkg/util/flag"
+	cliflag "k8s.io/component-base/cli/flag"
 )
 
 // APIEnablementOptions contains the options for which resources to turn on and off.
 // Given small aggregated API servers, this option isn't required for "normal" API servers
 type APIEnablementOptions struct {
-	RuntimeConfig utilflag.ConfigurationMap
+	RuntimeConfig cliflag.ConfigurationMap
 }
 
 func NewAPIEnablementOptions() *APIEnablementOptions {
 	return &APIEnablementOptions{
-		RuntimeConfig: make(utilflag.ConfigurationMap),
+		RuntimeConfig: make(cliflag.ConfigurationMap),
 	}
 }
 
@@ -84,6 +84,7 @@ func (s *APIEnablementOptions) Validate(registries ...GroupRegisty) []error {
 
 // ApplyTo override MergedResourceConfig with defaults and registry
 func (s *APIEnablementOptions) ApplyTo(c *server.Config, defaultResourceConfig *serverstore.ResourceConfig, registry resourceconfig.GroupVersionRegistry) error {
+
 	if s == nil {
 		return nil
 	}

vendor/k8s.io/apiserver/pkg/server/options/audit.go

@@ -148,9 +148,14 @@ type AuditWebhookOptions struct {
 	GroupVersionString string
 }
 
+// AuditDynamicOptions control the configuration of dynamic backends for audit events
 type AuditDynamicOptions struct {
 	// Enabled tells whether the dynamic audit capability is enabled.
 	Enabled bool
+
+	// Configuration for batching backend. This is currently only used as an override
+	// for integration tests
+	BatchConfig *pluginbuffered.BatchConfig
 }
 
 func NewAuditOptions() *AuditOptions {
@@ -174,7 +179,8 @@ func NewAuditOptions() *AuditOptions {
 			GroupVersionString: "audit.k8s.io/v1",
 		},
 		DynamicOptions: AuditDynamicOptions{
-			Enabled: false,
+			Enabled:     false,
+			BatchConfig: plugindynamic.NewDefaultWebhookBatchConfig(),
 		},
 	}
 }
@@ -634,7 +640,7 @@ func (o *AuditDynamicOptions) newBackend(
 
 	dc := &plugindynamic.Config{
 		Informer:       informer,
-		BufferedConfig: plugindynamic.NewDefaultWebhookBatchConfig(),
+		BufferedConfig: o.BatchConfig,
 		EventConfig: plugindynamic.EventConfig{
 			Sink: eventSink,
 			Source: corev1.EventSource{

vendor/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -260,7 +260,7 @@ func (s *DelegatingAuthenticationOptions) lookupMissingConfigInCluster(client ku
 		// ignore, authConfigMap is nil now
 	case errors.IsForbidden(err):
 		klog.Warningf("Unable to get configmap/%s in %s.  Usually fixed by "+
-			"'kubectl create rolebinding -n %s ROLE_NAME --role=%s --serviceaccount=YOUR_NS:YOUR_SA'",
+			"'kubectl create rolebinding -n %s ROLEBINDING_NAME --role=%s --serviceaccount=YOUR_NS:YOUR_SA'",
 			authenticationConfigMapName, authenticationConfigMapNamespace, authenticationConfigMapNamespace, authenticationRoleName)
 		return err
 	case err != nil:

vendor/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go

@@ -54,8 +54,8 @@ func (s *DeprecatedInsecureServingOptions) Validate() []error {
 
 	errors := []error{}
 
-	if s.BindPort < 0 || s.BindPort > 65335 {
-		errors = append(errors, fmt.Errorf("insecure port %v must be between 0 and 65335, inclusive. 0 for turning off insecure (HTTP) port", s.BindPort))
+	if s.BindPort < 0 || s.BindPort > 65535 {
+		errors = append(errors, fmt.Errorf("insecure port %v must be between 0 and 65535, inclusive. 0 for turning off insecure (HTTP) port", s.BindPort))
 	}
 
 	return errors
@@ -69,11 +69,13 @@ func (s *DeprecatedInsecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 
 	fs.IPVar(&s.BindAddress, "insecure-bind-address", s.BindAddress, ""+
 		"The IP address on which to serve the --insecure-port (set to 0.0.0.0 for all IPv4 interfaces and :: for all IPv6 interfaces).")
+	// Though this flag is deprecated, we discovered security concerns over how to do health checks without it e.g. #43784
 	fs.MarkDeprecated("insecure-bind-address", "This flag will be removed in a future version.")
 	fs.Lookup("insecure-bind-address").Hidden = false
 
 	fs.IntVar(&s.BindPort, "insecure-port", s.BindPort, ""+
 		"The port on which to serve unsecured, unauthenticated access.")
+	// Though this flag is deprecated, we discovered security concerns over how to do health checks without it e.g. #43784
 	fs.MarkDeprecated("insecure-port", "This flag will be removed in a future version.")
 	fs.Lookup("insecure-port").Hidden = false
 }
@@ -94,7 +96,7 @@ func (s *DeprecatedInsecureServingOptions) AddUnqualifiedFlags(fs *pflag.FlagSet
 	fs.Lookup("port").Hidden = false
 }
 
-// ApplyTo adds DeprecatedInsecureServingOptions to the insecureserverinfo amd kube-controller manager configuration.
+// ApplyTo adds DeprecatedInsecureServingOptions to the insecureserverinfo and kube-controller manager configuration.
 // Note: the double pointer allows to set the *DeprecatedInsecureServingInfo to nil without referencing the struct hosting this pointer.
 func (s *DeprecatedInsecureServingOptions) ApplyTo(c **server.DeprecatedInsecureServingInfo) error {
 	if s == nil {
@@ -130,7 +132,8 @@ func (o *DeprecatedInsecureServingOptions) WithLoopback() *DeprecatedInsecureSer
 }
 
 // DeprecatedInsecureServingOptionsWithLoopback adds loopback functionality to the DeprecatedInsecureServingOptions.
-// DEPRECATED: all insecure serving options are removed in a future version
+// DEPRECATED: all insecure serving options will be removed in a future version, however note that
+// there are security concerns over how health checks can work here - see e.g. #43784
 type DeprecatedInsecureServingOptionsWithLoopback struct {
 	*DeprecatedInsecureServingOptions
 }

vendor/k8s.io/apiserver/pkg/server/options/egress_selector.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"github.com/spf13/pflag"
+	"k8s.io/utils/path"
+
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/egressselector"
+)
+
+// EgressSelectorOptions holds the api server egress selector options.
+// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190226-network-proxy.md
+type EgressSelectorOptions struct {
+	// ConfigFile is the file path with api-server egress selector configuration.
+	ConfigFile string
+}
+
+// NewEgressSelectorOptions creates a new instance of EgressSelectorOptions
+//
+// The option is to point to a configuration file for egress/konnectivity.
+// This determines which types of requests use egress/konnectivity and how they use it.
+// If empty the API Server will attempt to connect directly using the network.
+func NewEgressSelectorOptions() *EgressSelectorOptions {
+	return &EgressSelectorOptions{}
+}
+
+// AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet
+func (o *EgressSelectorOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.ConfigFile, "egress-selector-config-file", o.ConfigFile,
+		"File with apiserver egress selector configuration.")
+}
+
+// ApplyTo adds the egress selector settings to the server configuration.
+// In case egress selector settings were not provided by a cluster-admin
+// they will be prepared from the recommended/default/no-op values.
+func (o *EgressSelectorOptions) ApplyTo(c *server.Config) error {
+	if o == nil {
+		return nil
+	}
+
+	npConfig, err := egressselector.ReadEgressSelectorConfiguration(o.ConfigFile)
+	if err != nil {
+		return fmt.Errorf("failed to read egress selector config: %v", err)
+	}
+	errs := egressselector.ValidateEgressSelectorConfiguration(npConfig)
+	if len(errs) > 0 {
+		return fmt.Errorf("failed to validate egress selector configuration: %v", errs.ToAggregate())
+	}
+
+	cs, err := egressselector.NewEgressSelector(npConfig)
+	if err != nil {
+		return fmt.Errorf("failed to setup egress selector with config %#v: %v", npConfig, err)
+	}
+	c.EgressSelector = cs
+	return nil
+}
+
+// Validate verifies flags passed to EgressSelectorOptions.
+func (o *EgressSelectorOptions) Validate() []error {
+	if o == nil || o.ConfigFile == "" {
+		return nil
+	}
+
+	errs := []error{}
+
+	if exists, err := path.Exists(path.CheckFollowSymlink, o.ConfigFile); exists == false || err != nil {
+		errs = append(errs, fmt.Errorf("egress-selector-config-file %s does not exist", o.ConfigFile))
+	}
+
+	return errs
+}

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/OWNERS

@@ -0,0 +1,9 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- sig-auth-encryption-at-rest-approvers
+reviewers:
+- sig-auth-encryption-at-rest-reviewers
+labels:
+- sig/auth
+

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -0,0 +1,428 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package encryptionconfig
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	apiserverconfig "k8s.io/apiserver/pkg/apis/config"
+	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/storage/value"
+	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/identity"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
+)
+
+const (
+	aesCBCTransformerPrefixV1    = "k8s:enc:aescbc:v1:"
+	aesGCMTransformerPrefixV1    = "k8s:enc:aesgcm:v1:"
+	secretboxTransformerPrefixV1 = "k8s:enc:secretbox:v1:"
+	kmsTransformerPrefixV1       = "k8s:enc:kms:v1:"
+	kmsPluginConnectionTimeout   = 3 * time.Second
+	kmsPluginHealthzTTL          = 3 * time.Second
+)
+
+type kmsPluginHealthzResponse struct {
+	err      error
+	received time.Time
+}
+
+type kmsPluginProbe struct {
+	name string
+	envelope.Service
+	lastResponse *kmsPluginHealthzResponse
+	l            *sync.Mutex
+}
+
+func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
+	return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
+		return h.Check()
+	})
+}
+
+// GetKMSPluginHealthzCheckers extracts KMSPluginProbes from the EncryptionConfig.
+func GetKMSPluginHealthzCheckers(filepath string) ([]healthz.HealthChecker, error) {
+	f, err := os.Open(filepath)
+	if err != nil {
+		return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err)
+	}
+	defer f.Close()
+	var result []healthz.HealthChecker
+	probes, err := getKMSPluginProbes(f)
+	if err != nil {
+		return nil, err
+	}
+
+	for i, p := range probes {
+		probe := p
+		result = append(result, probe.toHealthzCheck(i))
+	}
+	return result, nil
+}
+
+func getKMSPluginProbes(reader io.Reader) ([]*kmsPluginProbe, error) {
+	var result []*kmsPluginProbe
+
+	configFileContents, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return result, fmt.Errorf("could not read content of encryption provider configuration: %v", err)
+	}
+
+	config, err := loadConfig(configFileContents)
+	if err != nil {
+		return result, fmt.Errorf("error while parsing encrypiton provider configuration: %v", err)
+	}
+
+	for _, r := range config.Resources {
+		for _, p := range r.Providers {
+			if p.KMS != nil {
+				timeout := kmsPluginConnectionTimeout
+				if p.KMS.Timeout != nil {
+					if p.KMS.Timeout.Duration <= 0 {
+						return nil, fmt.Errorf("could not configure KMS-Plugin's probe %q, timeout should be a positive value", p.KMS.Name)
+					}
+					timeout = p.KMS.Timeout.Duration
+				}
+
+				s, err := envelope.NewGRPCService(p.KMS.Endpoint, timeout)
+				if err != nil {
+					return nil, fmt.Errorf("could not configure KMS-Plugin's probe %q, error: %v", p.KMS.Name, err)
+				}
+
+				result = append(result, &kmsPluginProbe{
+					name:         p.KMS.Name,
+					Service:      s,
+					l:            &sync.Mutex{},
+					lastResponse: &kmsPluginHealthzResponse{},
+				})
+			}
+		}
+	}
+
+	return result, nil
+}
+
+// Check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint.
+func (h *kmsPluginProbe) Check() error {
+	h.l.Lock()
+	defer h.l.Unlock()
+
+	if (time.Now().Sub(h.lastResponse.received)) < kmsPluginHealthzTTL {
+		return h.lastResponse.err
+	}
+
+	p, err := h.Service.Encrypt([]byte("ping"))
+	if err != nil {
+		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
+		return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
+	}
+
+	if _, err := h.Service.Decrypt(p); err != nil {
+		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
+		return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
+	}
+
+	h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()}
+	return nil
+}
+
+// GetTransformerOverrides returns the transformer overrides by reading and parsing the encryption provider configuration file
+func GetTransformerOverrides(filepath string) (map[schema.GroupResource]value.Transformer, error) {
+	f, err := os.Open(filepath)
+	if err != nil {
+		return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err)
+	}
+	defer f.Close()
+
+	result, err := ParseEncryptionConfiguration(f)
+	if err != nil {
+		return nil, fmt.Errorf("error while parsing encryption provider configuration file %q: %v", filepath, err)
+	}
+	return result, nil
+}
+
+// ParseEncryptionConfiguration parses configuration data and returns the transformer overrides
+func ParseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.Transformer, error) {
+	configFileContents, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, fmt.Errorf("could not read contents: %v", err)
+	}
+
+	config, err := loadConfig(configFileContents)
+	if err != nil {
+		return nil, fmt.Errorf("error while parsing file: %v", err)
+	}
+
+	resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{}
+
+	// For each entry in the configuration
+	for _, resourceConfig := range config.Resources {
+		transformers, err := GetPrefixTransformers(&resourceConfig)
+		if err != nil {
+			return nil, err
+		}
+
+		// For each resource, create a list of providers to use
+		for _, resource := range resourceConfig.Resources {
+			gr := schema.ParseGroupResource(resource)
+			resourceToPrefixTransformer[gr] = append(
+				resourceToPrefixTransformer[gr], transformers...)
+		}
+	}
+
+	result := map[schema.GroupResource]value.Transformer{}
+	for gr, transList := range resourceToPrefixTransformer {
+		result[gr] = value.NewMutableTransformer(value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...))
+	}
+	return result, nil
+
+}
+
+// loadConfig decodes data as a EncryptionConfiguration object.
+func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) {
+	scheme := runtime.NewScheme()
+	codecs := serializer.NewCodecFactory(scheme)
+	apiserverconfig.AddToScheme(scheme)
+	apiserverconfigv1.AddToScheme(scheme)
+
+	configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	config, ok := configObj.(*apiserverconfig.EncryptionConfiguration)
+	if !ok {
+		return nil, fmt.Errorf("got unexpected config type: %v", gvk)
+	}
+	return config, nil
+}
+
+// The factory to create kms service. This is to make writing test easier.
+var envelopeServiceFactory = envelope.NewGRPCService
+
+// GetPrefixTransformers constructs and returns the appropriate prefix transformers for the passed resource using its configuration.
+func GetPrefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]value.PrefixTransformer, error) {
+	var result []value.PrefixTransformer
+	for _, provider := range config.Providers {
+		found := false
+
+		var transformer value.PrefixTransformer
+		var err error
+
+		if provider.AESGCM != nil {
+			transformer, err = GetAESPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
+			if err != nil {
+				return result, err
+			}
+			found = true
+		}
+
+		if provider.AESCBC != nil {
+			if found == true {
+				return result, fmt.Errorf("more than one provider specified in a single element, should split into different list elements")
+			}
+			transformer, err = GetAESPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1)
+			found = true
+		}
+
+		if provider.Secretbox != nil {
+			if found == true {
+				return result, fmt.Errorf("more than one provider specified in a single element, should split into different list elements")
+			}
+			transformer, err = GetSecretboxPrefixTransformer(provider.Secretbox)
+			found = true
+		}
+
+		if provider.Identity != nil {
+			if found == true {
+				return result, fmt.Errorf("more than one provider specified in a single element, should split into different list elements")
+			}
+			transformer = value.PrefixTransformer{
+				Transformer: identity.NewEncryptCheckTransformer(),
+				Prefix:      []byte{},
+			}
+			found = true
+		}
+
+		if provider.KMS != nil {
+			if found == true {
+				return nil, fmt.Errorf("more than one provider specified in a single element, should split into different list elements")
+			}
+
+			// Ensure the endpoint is provided.
+			if len(provider.KMS.Endpoint) == 0 {
+				return nil, fmt.Errorf("remote KMS provider can't use empty string as endpoint")
+			}
+
+			timeout := kmsPluginConnectionTimeout
+			if provider.KMS.Timeout != nil {
+				if provider.KMS.Timeout.Duration <= 0 {
+					return nil, fmt.Errorf("could not configure KMS plugin %q, timeout should be a positive value", provider.KMS.Name)
+				}
+				timeout = provider.KMS.Timeout.Duration
+			}
+
+			// Get gRPC client service with endpoint.
+			envelopeService, err := envelopeServiceFactory(provider.KMS.Endpoint, timeout)
+			if err != nil {
+				return nil, fmt.Errorf("could not configure KMS plugin %q, error: %v", provider.KMS.Name, err)
+			}
+
+			transformer, err = getEnvelopePrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV1)
+			found = true
+		}
+
+		if err != nil {
+			return result, err
+		}
+		result = append(result, transformer)
+
+		if found == false {
+			return result, fmt.Errorf("invalid provider configuration: at least one provider must be specified")
+		}
+	}
+	return result, nil
+}
+
+// BlockTransformerFunc takes an AES cipher block and returns a value transformer.
+type BlockTransformerFunc func(cipher.Block) value.Transformer
+
+// GetAESPrefixTransformer returns a prefix transformer from the provided configuration.
+// Returns an AES transformer based on the provided prefix and block transformer.
+func GetAESPrefixTransformer(config *apiserverconfig.AESConfiguration, fn BlockTransformerFunc, prefix string) (value.PrefixTransformer, error) {
+	var result value.PrefixTransformer
+
+	if len(config.Keys) == 0 {
+		return result, fmt.Errorf("aes provider has no valid keys")
+	}
+	for _, key := range config.Keys {
+		if key.Name == "" {
+			return result, fmt.Errorf("key with invalid name provided")
+		}
+		if key.Secret == "" {
+			return result, fmt.Errorf("key %v has no provided secret", key.Name)
+		}
+	}
+
+	keyTransformers := []value.PrefixTransformer{}
+
+	for _, keyData := range config.Keys {
+		key, err := base64.StdEncoding.DecodeString(keyData.Secret)
+		if err != nil {
+			return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err)
+		}
+		block, err := aes.NewCipher(key)
+		if err != nil {
+			return result, fmt.Errorf("error while creating cipher for named key %s: %s", keyData.Name, err)
+		}
+
+		// Create a new PrefixTransformer for this key
+		keyTransformers = append(keyTransformers,
+			value.PrefixTransformer{
+				Transformer: fn(block),
+				Prefix:      []byte(keyData.Name + ":"),
+			})
+	}
+
+	// Create a prefixTransformer which can choose between these keys
+	keyTransformer := value.NewPrefixTransformers(
+		fmt.Errorf("no matching key was found for the provided AES transformer"), keyTransformers...)
+
+	// Create a PrefixTransformer which shall later be put in a list with other providers
+	result = value.PrefixTransformer{
+		Transformer: keyTransformer,
+		Prefix:      []byte(prefix),
+	}
+	return result, nil
+}
+
+// GetSecretboxPrefixTransformer returns a prefix transformer from the provided configuration
+func GetSecretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (value.PrefixTransformer, error) {
+	var result value.PrefixTransformer
+
+	if len(config.Keys) == 0 {
+		return result, fmt.Errorf("secretbox provider has no valid keys")
+	}
+	for _, key := range config.Keys {
+		if key.Name == "" {
+			return result, fmt.Errorf("key with invalid name provided")
+		}
+		if key.Secret == "" {
+			return result, fmt.Errorf("key %v has no provided secret", key.Name)
+		}
+	}
+
+	keyTransformers := []value.PrefixTransformer{}
+
+	for _, keyData := range config.Keys {
+		key, err := base64.StdEncoding.DecodeString(keyData.Secret)
+		if err != nil {
+			return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err)
+		}
+
+		if len(key) != 32 {
+			return result, fmt.Errorf("expected key size 32 for secretbox provider, got %v", len(key))
+		}
+
+		keyArray := [32]byte{}
+		copy(keyArray[:], key)
+
+		// Create a new PrefixTransformer for this key
+		keyTransformers = append(keyTransformers,
+			value.PrefixTransformer{
+				Transformer: secretbox.NewSecretboxTransformer(keyArray),
+				Prefix:      []byte(keyData.Name + ":"),
+			})
+	}
+
+	// Create a prefixTransformer which can choose between these keys
+	keyTransformer := value.NewPrefixTransformers(
+		fmt.Errorf("no matching key was found for the provided Secretbox transformer"), keyTransformers...)
+
+	// Create a PrefixTransformer which shall later be put in a list with other providers
+	result = value.PrefixTransformer{
+		Transformer: keyTransformer,
+		Prefix:      []byte(secretboxTransformerPrefixV1),
+	}
+	return result, nil
+}
+
+// getEnvelopePrefixTransformer returns a prefix transformer from the provided config.
+// envelopeService is used as the root of trust.
+func getEnvelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) {
+	envelopeTransformer, err := envelope.NewEnvelopeTransformer(envelopeService, int(config.CacheSize), aestransformer.NewCBCTransformer)
+	if err != nil {
+		return value.PrefixTransformer{}, err
+	}
+	return value.PrefixTransformer{
+		Transformer: envelopeTransformer,
+		Prefix:      []byte(prefix + config.Name + ":"),
+	}, nil
+}

vendor/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -31,6 +31,7 @@ import (
 	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
@@ -81,7 +82,7 @@ func (s *EtcdOptions) Validate() []error {
 	}
 
 	allErrors := []error{}
-	if len(s.StorageConfig.ServerList) == 0 {
+	if len(s.StorageConfig.Transport.ServerList) == 0 {
 		allErrors = append(allErrors, fmt.Errorf("--etcd-servers must be specified"))
 	}
 
@@ -134,9 +135,10 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
 		"Default watch cache size. If zero, watch cache will be disabled for resources that do not have a default watch size set.")
 
 	fs.StringSliceVar(&s.WatchCacheSizes, "watch-cache-sizes", s.WatchCacheSizes, ""+
-		"List of watch cache sizes for every resource (pods, nodes, etc.), comma separated. "+
-		"The individual override format: resource[.group]#size, where resource is lowercase plural (no version), "+
-		"group is optional, and size is a number. It takes effect when watch-cache is enabled. "+
+		"Watch cache size settings for some resources (pods, nodes, etc.), comma separated. "+
+		"The individual setting format: resource[.group]#size, where resource is lowercase plural (no version), "+
+		"group is omitted for resources of apiVersion v1 (the legacy core API) and included for others, "+
+		"and size is a number. It takes effect when watch-cache is enabled. "+
 		"Some resources (replicationcontrollers, endpoints, nodes, pods, services, apiservices.apiregistration.k8s.io) "+
 		"have system defaults set by heuristics, others default to default-watch-cache-size")
 
@@ -147,19 +149,19 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&dummyCacheSize, "deserialization-cache-size", 0, "Number of deserialized json objects to cache in memory.")
 	fs.MarkDeprecated("deserialization-cache-size", "the deserialization cache was dropped in 1.13 with support for etcd2")
 
-	fs.StringSliceVar(&s.StorageConfig.ServerList, "etcd-servers", s.StorageConfig.ServerList,
+	fs.StringSliceVar(&s.StorageConfig.Transport.ServerList, "etcd-servers", s.StorageConfig.Transport.ServerList,
 		"List of etcd servers to connect with (scheme://ip:port), comma separated.")
 
 	fs.StringVar(&s.StorageConfig.Prefix, "etcd-prefix", s.StorageConfig.Prefix,
 		"The prefix to prepend to all resource paths in etcd.")
 
-	fs.StringVar(&s.StorageConfig.KeyFile, "etcd-keyfile", s.StorageConfig.KeyFile,
+	fs.StringVar(&s.StorageConfig.Transport.KeyFile, "etcd-keyfile", s.StorageConfig.Transport.KeyFile,
 		"SSL key file used to secure etcd communication.")
 
-	fs.StringVar(&s.StorageConfig.CertFile, "etcd-certfile", s.StorageConfig.CertFile,
+	fs.StringVar(&s.StorageConfig.Transport.CertFile, "etcd-certfile", s.StorageConfig.Transport.CertFile,
 		"SSL certification file used to secure etcd communication.")
 
-	fs.StringVar(&s.StorageConfig.CAFile, "etcd-cafile", s.StorageConfig.CAFile,
+	fs.StringVar(&s.StorageConfig.Transport.CAFile, "etcd-cafile", s.StorageConfig.Transport.CAFile,
 		"SSL Certificate Authority file used to secure etcd communication.")
 
 	fs.StringVar(&s.EncryptionProviderConfigFilepath, "experimental-encryption-provider-config", s.EncryptionProviderConfigFilepath,
@@ -200,9 +202,18 @@ func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
 	if err != nil {
 		return err
 	}
-	c.HealthzChecks = append(c.HealthzChecks, healthz.NamedCheck("etcd", func(r *http.Request) error {
+	c.AddHealthChecks(healthz.NamedCheck("etcd", func(r *http.Request) error {
 		return healthCheck()
 	}))
+
+	if s.EncryptionProviderConfigFilepath != "" {
+		kmsPluginHealthzChecks, err := encryptionconfig.GetKMSPluginHealthzCheckers(s.EncryptionProviderConfigFilepath)
+		if err != nil {
+			return err
+		}
+		c.AddHealthChecks(kmsPluginHealthzChecks...)
+	}
+
 	return nil
 }
 
@@ -228,6 +239,7 @@ func (f *SimpleRestOptionsFactory) GetRESTOptions(resource schema.GroupResource)
 		if !ok {
 			cacheSize = f.Options.DefaultWatchCacheSize
 		}
+		// depending on cache size this might return an undecorated storage
 		ret.Decorator = genericregistry.StorageWithCacher(cacheSize)
 	}
 	return ret, nil
@@ -261,6 +273,7 @@ func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupR
 		if !ok {
 			cacheSize = f.Options.DefaultWatchCacheSize
 		}
+		// depending on cache size this might return an undecorated storage
 		ret.Decorator = genericregistry.StorageWithCacher(cacheSize)
 	}
 
@@ -284,7 +297,6 @@ func ParseWatchCacheSizes(cacheSizes []string) (map[schema.GroupResource]int, er
 		if size < 0 {
 			return nil, fmt.Errorf("watch cache size cannot be negative: %s", c)
 		}
-
 		watchCacheSizes[schema.ParseGroupResource(tokens[0])] = size
 	}
 	return watchCacheSizes, nil

vendor/k8s.io/apiserver/pkg/server/options/feature.go

@@ -26,7 +26,6 @@ import (
 type FeatureOptions struct {
 	EnableProfiling           bool
 	EnableContentionProfiling bool
-	EnableSwaggerUI           bool
 }
 
 func NewFeatureOptions() *FeatureOptions {
@@ -35,7 +34,6 @@ func NewFeatureOptions() *FeatureOptions {
 	return &FeatureOptions{
 		EnableProfiling:           defaults.EnableProfiling,
 		EnableContentionProfiling: defaults.EnableContentionProfiling,
-		EnableSwaggerUI:           defaults.EnableSwaggerUI,
 	}
 }
 
@@ -48,8 +46,9 @@ func (o *FeatureOptions) AddFlags(fs *pflag.FlagSet) {
 		"Enable profiling via web interface host:port/debug/pprof/")
 	fs.BoolVar(&o.EnableContentionProfiling, "contention-profiling", o.EnableContentionProfiling,
 		"Enable lock contention profiling, if profiling is enabled")
-	fs.BoolVar(&o.EnableSwaggerUI, "enable-swagger-ui", o.EnableSwaggerUI,
-		"Enables swagger ui on the apiserver at /swagger-ui")
+	dummy := false
+	fs.BoolVar(&dummy, "enable-swagger-ui", dummy, "Enables swagger ui on the apiserver at /swagger-ui")
+	fs.MarkDeprecated("enable-swagger-ui", "swagger 1.2 support has been removed")
 }
 
 func (o *FeatureOptions) ApplyTo(c *server.Config) error {
@@ -59,7 +58,6 @@ func (o *FeatureOptions) ApplyTo(c *server.Config) error {
 
 	c.EnableProfiling = o.EnableProfiling
 	c.EnableContentionProfiling = o.EnableContentionProfiling
-	c.EnableSwaggerUI = o.EnableSwaggerUI
 
 	return nil
 }

vendor/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -44,6 +44,8 @@ type RecommendedOptions struct {
 	// ProcessInfo is used to identify events created by the server.
 	ProcessInfo *ProcessInfo
 	Webhook     *WebhookOptions
+	// API Server Egress Selector is used to control outbound traffic from the API Server
+	EgressSelector *EgressSelectorOptions
 }
 
 func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *ProcessInfo) *RecommendedOptions {
@@ -67,6 +69,7 @@ func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *Proc
 		Admission:                  NewAdmissionOptions(),
 		ProcessInfo:                processInfo,
 		Webhook:                    NewWebhookOptions(),
+		EgressSelector:             NewEgressSelectorOptions(),
 	}
 }
 
@@ -79,12 +82,12 @@ func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) {
 	o.Features.AddFlags(fs)
 	o.CoreAPI.AddFlags(fs)
 	o.Admission.AddFlags(fs)
+	o.EgressSelector.AddFlags(fs)
 }
 
 // ApplyTo adds RecommendedOptions to the server configuration.
-// scheme is the scheme of the apiserver types that are sent to the admission chain.
 // pluginInitializers can be empty, it is only need for additional initializers.
-func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig, scheme *runtime.Scheme) error {
+func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if err := o.Etcd.ApplyTo(&config.Config); err != nil {
 		return err
 	}
@@ -108,7 +111,10 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig, scheme *r
 	}
 	if initializers, err := o.ExtraAdmissionInitializers(config); err != nil {
 		return err
-	} else if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, config.ClientConfig, scheme, initializers...); err != nil {
+	} else if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, config.ClientConfig, initializers...); err != nil {
+		return err
+	}
+	if err := o.EgressSelector.ApplyTo(&config.Config); err != nil {
 		return err
 	}
 
@@ -125,6 +131,7 @@ func (o *RecommendedOptions) Validate() []error {
 	errors = append(errors, o.Features.Validate()...)
 	errors = append(errors, o.CoreAPI.Validate()...)
 	errors = append(errors, o.Admission.Validate()...)
+	errors = append(errors, o.EgressSelector.Validate()...)
 
 	return errors
 }

vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -27,7 +27,7 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 
 	// add the generic feature gates
-	_ "k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/features"
 
 	"github.com/spf13/pflag"
 )
@@ -41,7 +41,9 @@ type ServerRunOptions struct {
 	MaxRequestsInFlight         int
 	MaxMutatingRequestsInFlight int
 	RequestTimeout              time.Duration
+	LivezGracePeriod            time.Duration
 	MinRequestTimeout           int
+	ShutdownDelayDuration       time.Duration
 	// We intentionally did not add a flag for this option. Users of the
 	// apiserver library can wire it to a flag.
 	JSONPatchMaxCopyBytes int64
@@ -49,8 +51,9 @@ type ServerRunOptions struct {
 	// decoded in a write request. 0 means no limit.
 	// We intentionally did not add a flag for this option. Users of the
 	// apiserver library can wire it to a flag.
-	MaxRequestBodyBytes int64
-	TargetRAMMB         int
+	MaxRequestBodyBytes       int64
+	TargetRAMMB               int
+	EnableInfightQuotaHandler bool
 }
 
 func NewServerRunOptions() *ServerRunOptions {
@@ -59,7 +62,9 @@ func NewServerRunOptions() *ServerRunOptions {
 		MaxRequestsInFlight:         defaults.MaxRequestsInFlight,
 		MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
 		RequestTimeout:              defaults.RequestTimeout,
+		LivezGracePeriod:            defaults.LivezGracePeriod,
 		MinRequestTimeout:           defaults.MinRequestTimeout,
+		ShutdownDelayDuration:       defaults.ShutdownDelayDuration,
 		JSONPatchMaxCopyBytes:       defaults.JSONPatchMaxCopyBytes,
 		MaxRequestBodyBytes:         defaults.MaxRequestBodyBytes,
 	}
@@ -71,8 +76,10 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
 	c.ExternalAddress = s.ExternalHost
 	c.MaxRequestsInFlight = s.MaxRequestsInFlight
 	c.MaxMutatingRequestsInFlight = s.MaxMutatingRequestsInFlight
+	c.LivezGracePeriod = s.LivezGracePeriod
 	c.RequestTimeout = s.RequestTimeout
 	c.MinRequestTimeout = s.MinRequestTimeout
+	c.ShutdownDelayDuration = s.ShutdownDelayDuration
 	c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
 	c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
 	c.PublicAddress = s.AdvertiseAddress
@@ -104,11 +111,31 @@ func (s *ServerRunOptions) Validate() []error {
 	if s.TargetRAMMB < 0 {
 		errors = append(errors, fmt.Errorf("--target-ram-mb can not be negative value"))
 	}
-	if s.MaxRequestsInFlight < 0 {
-		errors = append(errors, fmt.Errorf("--max-requests-inflight can not be negative value"))
+
+	if s.LivezGracePeriod < 0 {
+		errors = append(errors, fmt.Errorf("--livez-grace-period can not be a negative value"))
 	}
-	if s.MaxMutatingRequestsInFlight < 0 {
-		errors = append(errors, fmt.Errorf("--max-mutating-requests-inflight can not be negative value"))
+
+	if s.EnableInfightQuotaHandler {
+		if !utilfeature.DefaultFeatureGate.Enabled(features.RequestManagement) {
+			errors = append(errors, fmt.Errorf("--enable-inflight-quota-handler can not be set if feature "+
+				"gate RequestManagement is disabled"))
+		}
+		if s.MaxMutatingRequestsInFlight != 0 {
+			errors = append(errors, fmt.Errorf("--max-mutating-requests-inflight=%v "+
+				"can not be set if enabled inflight quota handler", s.MaxMutatingRequestsInFlight))
+		}
+		if s.MaxRequestsInFlight != 0 {
+			errors = append(errors, fmt.Errorf("--max-requests-inflight=%v "+
+				"can not be set if enabled inflight quota handler", s.MaxRequestsInFlight))
+		}
+	} else {
+		if s.MaxRequestsInFlight < 0 {
+			errors = append(errors, fmt.Errorf("--max-requests-inflight can not be negative value"))
+		}
+		if s.MaxMutatingRequestsInFlight < 0 {
+			errors = append(errors, fmt.Errorf("--max-mutating-requests-inflight can not be negative value"))
+		}
 	}
 
 	if s.RequestTimeout.Nanoseconds() < 0 {
@@ -119,6 +146,10 @@ func (s *ServerRunOptions) Validate() []error {
 		errors = append(errors, fmt.Errorf("--min-request-timeout can not be negative value"))
 	}
 
+	if s.ShutdownDelayDuration < 0 {
+		errors = append(errors, fmt.Errorf("--shutdown-delay-duration can not be negative value"))
+	}
+
 	if s.JSONPatchMaxCopyBytes < 0 {
 		errors = append(errors, fmt.Errorf("--json-patch-max-copy-bytes can not be negative value"))
 	}
@@ -168,11 +199,24 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"it out. This is the default request timeout for requests but may be overridden by flags such as "+
 		"--min-request-timeout for specific types of requests.")
 
+	fs.DurationVar(&s.LivezGracePeriod, "livez-grace-period", s.LivezGracePeriod, ""+
+		"This option represents the maximum amount of time it should take for apiserver to complete its startup sequence "+
+		"and become live. From apiserver's start time to when this amount of time has elapsed, /livez will assume "+
+		"that unfinished post-start hooks will complete successfully and therefore return true.")
+
 	fs.IntVar(&s.MinRequestTimeout, "min-request-timeout", s.MinRequestTimeout, ""+
 		"An optional field indicating the minimum number of seconds a handler must keep "+
 		"a request open before timing it out. Currently only honored by the watch request "+
 		"handler, which picks a randomized value above this number as the connection timeout, "+
 		"to spread out load.")
 
-	utilfeature.DefaultFeatureGate.AddFlag(fs)
+	fs.BoolVar(&s.EnableInfightQuotaHandler, "enable-inflight-quota-handler", s.EnableInfightQuotaHandler, ""+
+		"If true, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")
+
+	fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
+		"Time to delay the termination. During that time the server keeps serving requests normally and /healthz "+
+		"returns success, but /readzy immediately returns failure. Graceful termination starts after this delay "+
+		"has elapsed. This can be used to allow load balancer to stop sending traffic to this server.")
+
+	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
 }

vendor/k8s.io/apiserver/pkg/server/options/serving.go

@@ -29,8 +29,9 @@ import (
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/server"
-	utilflag "k8s.io/apiserver/pkg/util/flag"
 	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/keyutil"
+	cliflag "k8s.io/component-base/cli/flag"
 )
 
 type SecureServingOptions struct {
@@ -54,7 +55,7 @@ type SecureServingOptions struct {
 	// ServerCert is the TLS cert info for serving secure traffic
 	ServerCert GeneratableKeyCert
 	// SNICertKeys are named CertKeys for serving secure traffic with SNI support.
-	SNICertKeys []utilflag.NamedCertKey
+	SNICertKeys []cliflag.NamedCertKey
 	// CipherSuites is the list of allowed cipher suites for the server.
 	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
 	CipherSuites []string
@@ -165,18 +166,18 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile,
 		"File containing the default x509 private key matching --tls-cert-file.")
 
-	tlsCipherPossibleValues := utilflag.TLSCipherPossibleValues()
+	tlsCipherPossibleValues := cliflag.TLSCipherPossibleValues()
 	fs.StringSliceVar(&s.CipherSuites, "tls-cipher-suites", s.CipherSuites,
 		"Comma-separated list of cipher suites for the server. "+
 			"If omitted, the default Go cipher suites will be use.  "+
 			"Possible values: "+strings.Join(tlsCipherPossibleValues, ","))
 
-	tlsPossibleVersions := utilflag.TLSPossibleVersions()
+	tlsPossibleVersions := cliflag.TLSPossibleVersions()
 	fs.StringVar(&s.MinTLSVersion, "tls-min-version", s.MinTLSVersion,
 		"Minimum TLS version supported. "+
 			"Possible values: "+strings.Join(tlsPossibleVersions, ", "))
 
-	fs.Var(utilflag.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
+	fs.Var(cliflag.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
 		"A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+
 		"domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+
 		"segments. If no domain patterns are provided, the names of the certificate are "+
@@ -234,7 +235,7 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	}
 
 	if len(s.CipherSuites) != 0 {
-		cipherSuites, err := utilflag.TLSCipherSuites(s.CipherSuites)
+		cipherSuites, err := cliflag.TLSCipherSuites(s.CipherSuites)
 		if err != nil {
 			return err
 		}
@@ -242,7 +243,7 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	}
 
 	var err error
-	c.MinTLSVersion, err = utilflag.TLSVersion(s.MinTLSVersion)
+	c.MinTLSVersion, err = cliflag.TLSVersion(s.MinTLSVersion)
 	if err != nil {
 		return err
 	}
@@ -305,7 +306,7 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str
 			if err := certutil.WriteCert(keyCert.CertFile, cert); err != nil {
 				return err
 			}
-			if err := certutil.WriteKey(keyCert.KeyFile, key); err != nil {
+			if err := keyutil.WriteKey(keyCert.KeyFile, key); err != nil {
 				return err
 			}
 			klog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile)