improve IAM module

Signed-off-by: hongming <talonwan@yunify.com>

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/cert_key.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+)
+
+// CertKeyContentProvider provides a certificate and matching private key
+type CertKeyContentProvider interface {
+	// Name is just an identifier
+	Name() string
+	// CurrentCertKeyContent provides cert and key byte content
+	CurrentCertKeyContent() ([]byte, []byte)
+}
+
+// SNICertKeyContentProvider provides a certificate and matching private key as well as optional explicit names
+type SNICertKeyContentProvider interface {
+	CertKeyContentProvider
+	// SNINames provides names used for SNI. May return nil.
+	SNINames() []string
+}
+
+// certKeyContent holds the content for the cert and key
+type certKeyContent struct {
+	cert []byte
+	key  []byte
+}
+
+func (c *certKeyContent) Equal(rhs *certKeyContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	return bytes.Equal(c.key, rhs.key) && bytes.Equal(c.cert, rhs.cert)
+}
+
+// sniCertKeyContent holds the content for the cert and key as well as any explicit names
+type sniCertKeyContent struct {
+	certKeyContent
+	sniNames []string
+}
+
+func (c *sniCertKeyContent) Equal(rhs *sniCertKeyContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	if len(c.sniNames) != len(rhs.sniNames) {
+		return false
+	}
+
+	for i := range c.sniNames {
+		if c.sniNames[i] != rhs.sniNames[i] {
+			return false
+		}
+	}
+
+	return c.certKeyContent.Equal(&rhs.certKeyContent)
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"crypto/x509"
+)
+
+// CAContentProvider provides ca bundle byte content
+type CAContentProvider interface {
+	// Name is just an identifier
+	Name() string
+	// CurrentCABundleContent provides ca bundle byte content.  Errors can be contained to the controllers initializing
+	// the value.  By the time you get here, you should always be returning a value that won't fail.
+	CurrentCABundleContent() []byte
+	// VerifyOptions provides VerifyOptions for authenticators
+	VerifyOptions() (x509.VerifyOptions, bool)
+}
+
+// dynamicCertificateContent holds the content that overrides the baseTLSConfig
+type dynamicCertificateContent struct {
+	// clientCA holds the content for the clientCA bundle
+	clientCA    caBundleContent
+	servingCert certKeyContent
+	sniCerts    []sniCertKeyContent
+}
+
+// caBundleContent holds the content for the clientCA bundle.  Wrapping the bytes makes the Equals work nicely with the
+// method receiver.
+type caBundleContent struct {
+	caBundle []byte
+}
+
+func (c *dynamicCertificateContent) Equal(rhs *dynamicCertificateContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	if !c.clientCA.Equal(&rhs.clientCA) {
+		return false
+	}
+
+	if !c.servingCert.Equal(&rhs.servingCert) {
+		return false
+	}
+
+	if len(c.sniCerts) != len(rhs.sniCerts) {
+		return false
+	}
+
+	for i := range c.sniCerts {
+		if !c.sniCerts[i].Equal(&rhs.sniCerts[i]) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (c *caBundleContent) Equal(rhs *caBundleContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	return bytes.Equal(c.caBundle, rhs.caBundle)
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/configmap_cafile_content.go

@@ -0,0 +1,277 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"crypto/x509"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+)
+
+// ConfigMapCAController provies a CAContentProvider that can dynamically react to configmap changes
+// It also fulfills the authenticator interface to provide verifyoptions
+type ConfigMapCAController struct {
+	name string
+
+	configmapLister    corev1listers.ConfigMapLister
+	configmapNamespace string
+	configmapName      string
+	configmapKey       string
+	// configMapInformer is tracked so that we can start these on Run
+	configMapInformer cache.SharedIndexInformer
+
+	// caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file
+	caBundle atomic.Value
+
+	listeners []Listener
+
+	queue workqueue.RateLimitingInterface
+	// preRunCaches are the caches to sync before starting the work of this control loop
+	preRunCaches []cache.InformerSynced
+}
+
+var _ Notifier = &ConfigMapCAController{}
+var _ CAContentProvider = &ConfigMapCAController{}
+var _ ControllerRunner = &ConfigMapCAController{}
+
+// NewDynamicCAFromConfigMapController returns a CAContentProvider based on a configmap that automatically reloads content.
+// It is near-realtime via an informer.
+func NewDynamicCAFromConfigMapController(purpose, namespace, name, key string, kubeClient kubernetes.Interface) (*ConfigMapCAController, error) {
+	if len(purpose) == 0 {
+		return nil, fmt.Errorf("missing purpose for ca bundle")
+	}
+	if len(namespace) == 0 {
+		return nil, fmt.Errorf("missing namespace for ca bundle")
+	}
+	if len(name) == 0 {
+		return nil, fmt.Errorf("missing name for ca bundle")
+	}
+	if len(key) == 0 {
+		return nil, fmt.Errorf("missing key for ca bundle")
+	}
+	caContentName := fmt.Sprintf("%s::%s::%s::%s", purpose, namespace, name, key)
+
+	// we construct our own informer because we need such a small subset of the information available.  Just one namespace.
+	uncastConfigmapInformer := corev1informers.NewFilteredConfigMapInformer(kubeClient, namespace, 12*time.Hour, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, func(listOptions *v1.ListOptions) {
+		listOptions.FieldSelector = fields.OneTermEqualSelector("metadata.name", name).String()
+	})
+
+	configmapLister := corev1listers.NewConfigMapLister(uncastConfigmapInformer.GetIndexer())
+
+	c := &ConfigMapCAController{
+		name:               caContentName,
+		configmapNamespace: namespace,
+		configmapName:      name,
+		configmapKey:       key,
+		configmapLister:    configmapLister,
+		configMapInformer:  uncastConfigmapInformer,
+
+		queue:        workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicConfigMapCABundle-%s", purpose)),
+		preRunCaches: []cache.InformerSynced{uncastConfigmapInformer.HasSynced},
+	}
+	if err := c.loadCABundle(); err != nil {
+		// don't fail, but do print out a message
+		klog.Warningf("unable to load initial CA bundle for: %q due to: %s", c.name, err)
+	}
+	uncastConfigmapInformer.AddEventHandler(cache.FilteringResourceEventHandler{
+		FilterFunc: func(obj interface{}) bool {
+			if cast, ok := obj.(*corev1.ConfigMap); ok {
+				return cast.Name == c.configmapName && cast.Namespace == c.configmapNamespace
+			}
+			if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok {
+				if cast, ok := tombstone.Obj.(*corev1.ConfigMap); ok {
+					return cast.Name == c.configmapName && cast.Namespace == c.configmapNamespace
+				}
+			}
+			return true // always return true just in case.  The checks are fairly cheap
+		},
+		Handler: cache.ResourceEventHandlerFuncs{
+			// we have a filter, so any time we're called, we may as well queue. We only ever check one configmap
+			// so we don't have to be choosy about our key.
+			AddFunc: func(obj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+			DeleteFunc: func(obj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+		},
+	})
+
+	return c, nil
+}
+
+func (c *ConfigMapCAController) keyFn() string {
+	// this format matches DeletionHandlingMetaNamespaceKeyFunc for our single key
+	return c.configmapNamespace + "/" + c.configmapName
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c *ConfigMapCAController) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadCABundle determines the next set of content for the file.
+func (c *ConfigMapCAController) loadCABundle() error {
+	configMap, err := c.configmapLister.ConfigMaps(c.configmapNamespace).Get(c.configmapName)
+	if err != nil {
+		return err
+	}
+	caBundle := configMap.Data[c.configmapKey]
+	if len(caBundle) == 0 {
+		return fmt.Errorf("missing content for CA bundle %q", c.Name())
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	if !c.hasCAChanged([]byte(caBundle)) {
+		return nil
+	}
+
+	caBundleAndVerifier, err := newCABundleAndVerifier(c.Name(), []byte(caBundle))
+	if err != nil {
+		return err
+	}
+	c.caBundle.Store(caBundleAndVerifier)
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// hasCAChanged returns true if the caBundle is different than the current.
+func (c *ConfigMapCAController) hasCAChanged(caBundle []byte) bool {
+	uncastExisting := c.caBundle.Load()
+	if uncastExisting == nil {
+		return true
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := uncastExisting.(*caBundleAndVerifier)
+	if !ok {
+		return true
+	}
+	if !bytes.Equal(existing.caBundle, caBundle) {
+		return true
+	}
+
+	return false
+}
+
+// RunOnce runs a single sync loop
+func (c *ConfigMapCAController) RunOnce() error {
+	// Ignore the error when running once because when using a dynamically loaded ca file, because we think it's better to have nothing for
+	// a brief time than completely crash.  If crashing is necessary, higher order logic like a healthcheck and cause failures.
+	_ = c.loadCABundle()
+	return nil
+}
+
+// Run starts the kube-apiserver and blocks until stopCh is closed.
+func (c *ConfigMapCAController) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting %s", c.name)
+	defer klog.Infof("Shutting down %s", c.name)
+
+	// we have a personal informer that is narrowly scoped, start it.
+	go c.configMapInformer.Run(stopCh)
+
+	// wait for your secondary caches to fill before starting your work
+	if !cache.WaitForNamedCacheSync(c.name, stopCh, c.preRunCaches...) {
+		return
+	}
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	_ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) {
+		c.queue.Add(workItemKey)
+		return false, nil
+	}, stopCh)
+
+	<-stopCh
+}
+
+func (c *ConfigMapCAController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *ConfigMapCAController) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadCABundle()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *ConfigMapCAController) Name() string {
+	return c.name
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *ConfigMapCAController) CurrentCABundleContent() []byte {
+	uncastObj := c.caBundle.Load()
+	if uncastObj == nil {
+		return nil // this can happen if we've been unable load data from the apiserver for some reason
+	}
+
+	return c.caBundle.Load().(*caBundleAndVerifier).caBundle
+}
+
+// VerifyOptions provides verifyoptions compatible with authenticators
+func (c *ConfigMapCAController) VerifyOptions() (x509.VerifyOptions, bool) {
+	uncastObj := c.caBundle.Load()
+	if uncastObj == nil {
+		// This can happen if we've been unable load data from the apiserver for some reason.
+		// In this case, we should not accept any connections on the basis of this ca bundle.
+		return x509.VerifyOptions{}, false
+	}
+
+	return uncastObj.(*caBundleAndVerifier).verifyOptions, true
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go

@@ -0,0 +1,254 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"sync/atomic"
+	"time"
+
+	"k8s.io/client-go/util/cert"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+)
+
+// FileRefreshDuration is exposed so that integration tests can crank up the reload speed.
+var FileRefreshDuration = 1 * time.Minute
+
+// Listener is an interface to use to notify interested parties of a change.
+type Listener interface {
+	// Enqueue should be called when an input may have changed
+	Enqueue()
+}
+
+// Notifier is a way to add listeners
+type Notifier interface {
+	// AddListener is adds a listener to be notified of potential input changes
+	AddListener(listener Listener)
+}
+
+// ControllerRunner is a generic interface for starting a controller
+type ControllerRunner interface {
+	// RunOnce runs the sync loop a single time.  This useful for synchronous priming
+	RunOnce() error
+
+	// Run should be called a go .Run
+	Run(workers int, stopCh <-chan struct{})
+}
+
+// DynamicFileCAContent provies a CAContentProvider that can dynamically react to new file content
+// It also fulfills the authenticator interface to provide verifyoptions
+type DynamicFileCAContent struct {
+	name string
+
+	// filename is the name the file to read.
+	filename string
+
+	// caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file
+	caBundle atomic.Value
+
+	listeners []Listener
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue workqueue.RateLimitingInterface
+}
+
+var _ Notifier = &DynamicFileCAContent{}
+var _ CAContentProvider = &DynamicFileCAContent{}
+var _ ControllerRunner = &DynamicFileCAContent{}
+
+type caBundleAndVerifier struct {
+	caBundle      []byte
+	verifyOptions x509.VerifyOptions
+}
+
+// NewDynamicCAContentFromFile returns a CAContentProvider based on a filename that automatically reloads content
+func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAContent, error) {
+	if len(filename) == 0 {
+		return nil, fmt.Errorf("missing filename for ca bundle")
+	}
+	name := fmt.Sprintf("%s::%s", purpose, filename)
+
+	ret := &DynamicFileCAContent{
+		name:     name,
+		filename: filename,
+		queue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)),
+	}
+	if err := ret.loadCABundle(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c *DynamicFileCAContent) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadCABundle determines the next set of content for the file.
+func (c *DynamicFileCAContent) loadCABundle() error {
+	caBundle, err := ioutil.ReadFile(c.filename)
+	if err != nil {
+		return err
+	}
+	if len(caBundle) == 0 {
+		return fmt.Errorf("missing content for CA bundle %q", c.Name())
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	if !c.hasCAChanged(caBundle) {
+		return nil
+	}
+
+	caBundleAndVerifier, err := newCABundleAndVerifier(c.Name(), caBundle)
+	if err != nil {
+		return err
+	}
+	c.caBundle.Store(caBundleAndVerifier)
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// hasCAChanged returns true if the caBundle is different than the current.
+func (c *DynamicFileCAContent) hasCAChanged(caBundle []byte) bool {
+	uncastExisting := c.caBundle.Load()
+	if uncastExisting == nil {
+		return true
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := uncastExisting.(*caBundleAndVerifier)
+	if !ok {
+		return true
+	}
+	if !bytes.Equal(existing.caBundle, caBundle) {
+		return true
+	}
+
+	return false
+}
+
+// RunOnce runs a single sync loop
+func (c *DynamicFileCAContent) RunOnce() error {
+	return c.loadCABundle()
+}
+
+// Run starts the kube-apiserver and blocks until stopCh is closed.
+func (c *DynamicFileCAContent) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting %s", c.name)
+	defer klog.Infof("Shutting down %s", c.name)
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	_ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) {
+		c.queue.Add(workItemKey)
+		return false, nil
+	}, stopCh)
+
+	// TODO this can be wired to an fsnotifier as well.
+
+	<-stopCh
+}
+
+func (c *DynamicFileCAContent) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicFileCAContent) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadCABundle()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *DynamicFileCAContent) Name() string {
+	return c.name
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *DynamicFileCAContent) CurrentCABundleContent() (cabundle []byte) {
+	return c.caBundle.Load().(*caBundleAndVerifier).caBundle
+}
+
+// VerifyOptions provides verifyoptions compatible with authenticators
+func (c *DynamicFileCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
+	uncastObj := c.caBundle.Load()
+	if uncastObj == nil {
+		return x509.VerifyOptions{}, false
+	}
+
+	return uncastObj.(*caBundleAndVerifier).verifyOptions, true
+}
+
+// newVerifyOptions creates a new verification func from a file.  It reads the content and then fails.
+// It will return a nil function if you pass an empty CA file.
+func newCABundleAndVerifier(name string, caBundle []byte) (*caBundleAndVerifier, error) {
+	if len(caBundle) == 0 {
+		return nil, fmt.Errorf("missing content for CA bundle %q", name)
+	}
+
+	// Wrap with an x509 verifier
+	var err error
+	verifyOptions := defaultVerifyOptions()
+	verifyOptions.Roots, err = cert.NewPoolFromBytes(caBundle)
+	if err != nil {
+		return nil, fmt.Errorf("error loading CA bundle for %q: %v", name, err)
+	}
+
+	return &caBundleAndVerifier{
+		caBundle:      caBundle,
+		verifyOptions: verifyOptions,
+	}, nil
+}
+
+// defaultVerifyOptions returns VerifyOptions that use the system root certificates, current time,
+// and requires certificates to be valid for client auth (x509.ExtKeyUsageClientAuth)
+func defaultVerifyOptions() x509.VerifyOptions {
+	return x509.VerifyOptions{
+		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	}
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go

@@ -0,0 +1,179 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"sync/atomic"
+	"time"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+)
+
+// DynamicFileServingContent provides a CertKeyContentProvider that can dynamically react to new file content
+type DynamicFileServingContent struct {
+	name string
+
+	// certFile is the name of the certificate file to read.
+	certFile string
+	// keyFile is the name of the key file to read.
+	keyFile string
+
+	// servingCert is a certKeyContent that contains the last read, non-zero length content of the key and cert
+	servingCert atomic.Value
+
+	listeners []Listener
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue workqueue.RateLimitingInterface
+}
+
+var _ Notifier = &DynamicFileServingContent{}
+var _ CertKeyContentProvider = &DynamicFileServingContent{}
+var _ ControllerRunner = &DynamicFileServingContent{}
+
+// NewDynamicServingContentFromFiles returns a dynamic CertKeyContentProvider based on a cert and key filename
+func NewDynamicServingContentFromFiles(purpose, certFile, keyFile string) (*DynamicFileServingContent, error) {
+	if len(certFile) == 0 || len(keyFile) == 0 {
+		return nil, fmt.Errorf("missing filename for serving cert")
+	}
+	name := fmt.Sprintf("%s::%s::%s", purpose, certFile, keyFile)
+
+	ret := &DynamicFileServingContent{
+		name:     name,
+		certFile: certFile,
+		keyFile:  keyFile,
+		queue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)),
+	}
+	if err := ret.loadServingCert(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// AddListener adds a listener to be notified when the serving cert content changes.
+func (c *DynamicFileServingContent) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadServingCert determines the next set of content for the file.
+func (c *DynamicFileServingContent) loadServingCert() error {
+	cert, err := ioutil.ReadFile(c.certFile)
+	if err != nil {
+		return err
+	}
+	key, err := ioutil.ReadFile(c.keyFile)
+	if err != nil {
+		return err
+	}
+	if len(cert) == 0 || len(key) == 0 {
+		return fmt.Errorf("missing content for serving cert %q", c.Name())
+	}
+
+	// Ensure that the key matches the cert and both are valid
+	_, err = tls.X509KeyPair(cert, key)
+	if err != nil {
+		return err
+	}
+
+	newCertKey := &certKeyContent{
+		cert: cert,
+		key:  key,
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := c.servingCert.Load().(*certKeyContent)
+	if ok && existing != nil && existing.Equal(newCertKey) {
+		return nil
+	}
+
+	c.servingCert.Store(newCertKey)
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// RunOnce runs a single sync loop
+func (c *DynamicFileServingContent) RunOnce() error {
+	return c.loadServingCert()
+}
+
+// Run starts the controller and blocks until stopCh is closed.
+func (c *DynamicFileServingContent) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting %s", c.name)
+	defer klog.Infof("Shutting down %s", c.name)
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	_ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) {
+		c.queue.Add(workItemKey)
+		return false, nil
+	}, stopCh)
+
+	// TODO this can be wired to an fsnotifier as well.
+
+	<-stopCh
+}
+
+func (c *DynamicFileServingContent) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicFileServingContent) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadServingCert()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *DynamicFileServingContent) Name() string {
+	return c.name
+}
+
+// CurrentCertKeyContent provides serving cert byte content
+func (c *DynamicFileServingContent) CurrentCertKeyContent() ([]byte, []byte) {
+	certKeyContent := c.servingCert.Load().(*certKeyContent)
+	return certKeyContent.cert, certKeyContent.key
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_sni_content.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+// DynamicFileSNIContent provides a SNICertKeyContentProvider that can dynamically react to new file content
+type DynamicFileSNIContent struct {
+	*DynamicFileServingContent
+	sniNames []string
+}
+
+var _ Notifier = &DynamicFileSNIContent{}
+var _ SNICertKeyContentProvider = &DynamicFileSNIContent{}
+var _ ControllerRunner = &DynamicFileSNIContent{}
+
+// NewDynamicSNIContentFromFiles returns a dynamic SNICertKeyContentProvider based on a cert and key filename and explicit names
+func NewDynamicSNIContentFromFiles(purpose, certFile, keyFile string, sniNames ...string) (*DynamicFileSNIContent, error) {
+	servingContent, err := NewDynamicServingContentFromFiles(purpose, certFile, keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	ret := &DynamicFileSNIContent{
+		DynamicFileServingContent: servingContent,
+		sniNames:                  sniNames,
+	}
+	if err := ret.loadServingCert(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// SNINames returns explicitly set SNI names for the certificate. These are not dynamic.
+func (c *DynamicFileSNIContent) SNINames() []string {
+	return c.sniNames
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/named_certificates.go

@@ -0,0 +1,89 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"strings"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/klog"
+)
+
+// BuildNamedCertificates returns a map of *tls.Certificate by name. It's
+// suitable for use in tls.Config#NamedCertificates. Returns an error if any of the certs
+// is invalid. Returns nil if len(certs) == 0
+func (c *DynamicServingCertificateController) BuildNamedCertificates(sniCerts []sniCertKeyContent) (map[string]*tls.Certificate, error) {
+	nameToCertificate := map[string]*tls.Certificate{}
+	byNameExplicit := map[string]*tls.Certificate{}
+
+	// Iterate backwards so that earlier certs take precedence in the names map
+	for i := len(sniCerts) - 1; i >= 0; i-- {
+		cert, err := tls.X509KeyPair(sniCerts[i].cert, sniCerts[i].key)
+		if err != nil {
+			return nil, fmt.Errorf("invalid SNI cert keypair [%d/%q]: %v", i, c.sniCerts[i].Name(), err)
+		}
+
+		// error is not possible given above call to X509KeyPair
+		x509Cert, _ := x509.ParseCertificate(cert.Certificate[0])
+
+		names := sniCerts[i].sniNames
+		for _, name := range names {
+			byNameExplicit[name] = &cert
+		}
+
+		klog.V(2).Infof("loaded SNI cert [%d/%q]: %s", i, c.sniCerts[i].Name(), GetHumanCertDetail(x509Cert))
+		if c.eventRecorder != nil {
+			c.eventRecorder.Eventf(nil, nil, v1.EventTypeWarning, "TLSConfigChanged", "SNICertificateReload", "loaded SNI cert [%d/%q]: %s with explicit names %v", i, c.sniCerts[i].Name(), GetHumanCertDetail(x509Cert), names)
+		}
+
+		if len(names) == 0 {
+			names = getCertificateNames(x509Cert)
+			for _, name := range names {
+				nameToCertificate[name] = &cert
+			}
+		}
+	}
+
+	// Explicitly set names must override
+	for k, v := range byNameExplicit {
+		nameToCertificate[k] = v
+	}
+
+	return nameToCertificate, nil
+}
+
+// getCertificateNames returns names for an x509.Certificate. The names are
+// suitable for use in tls.Config#NamedCertificates.
+func getCertificateNames(cert *x509.Certificate) []string {
+	var names []string
+
+	cn := cert.Subject.CommonName
+	if cn == "*" || len(validation.IsDNS1123Subdomain(strings.TrimPrefix(cn, "*."))) == 0 {
+		names = append(names, cn)
+	}
+	for _, san := range cert.DNSNames {
+		names = append(names, san)
+	}
+	// intentionally all IPs in the cert are ignored as SNI forbids passing IPs
+	// to select a cert. Before go 1.6 the tls happily passed IPs as SNI values.
+
+	return names
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go

@@ -0,0 +1,171 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+)
+
+type staticCAContent struct {
+	name     string
+	caBundle *caBundleAndVerifier
+}
+
+var _ CAContentProvider = &staticCAContent{}
+
+// NewStaticCAContentFromFile returns a CAContentProvider based on a filename
+func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) {
+	if len(filename) == 0 {
+		return nil, fmt.Errorf("missing filename for ca bundle")
+	}
+
+	caBundle, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return NewStaticCAContent(filename, caBundle)
+}
+
+// NewStaticCAContent returns a CAContentProvider that always returns the same value
+func NewStaticCAContent(name string, caBundle []byte) (CAContentProvider, error) {
+	caBundleAndVerifier, err := newCABundleAndVerifier(name, caBundle)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticCAContent{
+		name:     name,
+		caBundle: caBundleAndVerifier,
+	}, nil
+}
+
+// Name is just an identifier
+func (c *staticCAContent) Name() string {
+	return c.name
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) {
+	return c.caBundle.caBundle
+}
+
+func (c *staticCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
+	return c.caBundle.verifyOptions, true
+}
+
+type staticCertKeyContent struct {
+	name string
+	cert []byte
+	key  []byte
+}
+
+type staticSNICertKeyContent struct {
+	staticCertKeyContent
+	sniNames []string
+}
+
+// NewStaticCertKeyContentFromFiles returns a CertKeyContentProvider based on a filename
+func NewStaticCertKeyContentFromFiles(certFile, keyFile string) (CertKeyContentProvider, error) {
+	if len(certFile) == 0 {
+		return nil, fmt.Errorf("missing filename for certificate")
+	}
+	if len(keyFile) == 0 {
+		return nil, fmt.Errorf("missing filename for key")
+	}
+
+	certPEMBlock, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return nil, err
+	}
+	keyPEMBlock, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewStaticCertKeyContent(fmt.Sprintf("cert: %s, key: %s", certFile, keyFile), certPEMBlock, keyPEMBlock)
+}
+
+// NewStaticSNICertKeyContentFromFiles returns a SNICertKeyContentProvider based on a filename
+func NewStaticSNICertKeyContentFromFiles(certFile, keyFile string, sniNames ...string) (SNICertKeyContentProvider, error) {
+	if len(certFile) == 0 {
+		return nil, fmt.Errorf("missing filename for certificate")
+	}
+	if len(keyFile) == 0 {
+		return nil, fmt.Errorf("missing filename for key")
+	}
+
+	certPEMBlock, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return nil, err
+	}
+	keyPEMBlock, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewStaticSNICertKeyContent(fmt.Sprintf("cert: %s, key: %s", certFile, keyFile), certPEMBlock, keyPEMBlock, sniNames...)
+}
+
+// NewStaticCertKeyContent returns a CertKeyContentProvider that always returns the same value
+func NewStaticCertKeyContent(name string, cert, key []byte) (CertKeyContentProvider, error) {
+	// Ensure that the key matches the cert and both are valid
+	_, err := tls.X509KeyPair(cert, key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticCertKeyContent{
+		name: name,
+		cert: cert,
+		key:  key,
+	}, nil
+}
+
+// NewStaticSNICertKeyContent returns a SNICertKeyContentProvider that always returns the same value
+func NewStaticSNICertKeyContent(name string, cert, key []byte, sniNames ...string) (SNICertKeyContentProvider, error) {
+	// Ensure that the key matches the cert and both are valid
+	_, err := tls.X509KeyPair(cert, key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticSNICertKeyContent{
+		staticCertKeyContent: staticCertKeyContent{
+			name: name,
+			cert: cert,
+			key:  key,
+		},
+		sniNames: sniNames,
+	}, nil
+}
+
+// Name is just an identifier
+func (c *staticCertKeyContent) Name() string {
+	return c.name
+}
+
+// CurrentCertKeyContent provides cert and key content
+func (c *staticCertKeyContent) CurrentCertKeyContent() ([]byte, []byte) {
+	return c.cert, c.key
+}
+
+func (c *staticSNICertKeyContent) SNINames() []string {
+	return c.sniNames
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go

@@ -0,0 +1,263 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/events"
+	"k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+)
+
+const workItemKey = "key"
+
+// DynamicServingCertificateController dynamically loads certificates and provides a golang tls compatible dynamic GetCertificate func.
+type DynamicServingCertificateController struct {
+	// baseTLSConfig is the static portion of the tlsConfig for serving to clients.  It is copied and the copy is mutated
+	// based on the dynamic cert state.
+	baseTLSConfig tls.Config
+
+	// clientCA provides the very latest content of the ca bundle
+	clientCA CAContentProvider
+	// servingCert provides the very latest content of the default serving certificate
+	servingCert CertKeyContentProvider
+	// sniCerts are a list of CertKeyContentProvider with associated names used for SNI
+	sniCerts []SNICertKeyContentProvider
+
+	// currentlyServedContent holds the original bytes that we are serving. This is used to decide if we need to set a
+	// new atomic value. The types used for efficient TLSConfig preclude using the processed value.
+	currentlyServedContent *dynamicCertificateContent
+	// currentServingTLSConfig holds a *tls.Config that will be used to serve requests
+	currentServingTLSConfig atomic.Value
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue         workqueue.RateLimitingInterface
+	eventRecorder events.EventRecorder
+}
+
+var _ Listener = &DynamicServingCertificateController{}
+
+// NewDynamicServingCertificateController returns a controller that can be used to keep a TLSConfig up to date.
+func NewDynamicServingCertificateController(
+	baseTLSConfig tls.Config,
+	clientCA CAContentProvider,
+	servingCert CertKeyContentProvider,
+	sniCerts []SNICertKeyContentProvider,
+	eventRecorder events.EventRecorder,
+) *DynamicServingCertificateController {
+	c := &DynamicServingCertificateController{
+		baseTLSConfig: baseTLSConfig,
+		clientCA:      clientCA,
+		servingCert:   servingCert,
+		sniCerts:      sniCerts,
+
+		queue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DynamicServingCertificateController"),
+		eventRecorder: eventRecorder,
+	}
+
+	return c
+}
+
+// GetConfigForClient is an implementation of tls.Config.GetConfigForClient
+func (c *DynamicServingCertificateController) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls.Config, error) {
+	uncastObj := c.currentServingTLSConfig.Load()
+	if uncastObj == nil {
+		return nil, errors.New("dynamiccertificates: configuration not ready")
+	}
+	tlsConfig, ok := uncastObj.(*tls.Config)
+	if !ok {
+		return nil, errors.New("dynamiccertificates: unexpected config type")
+	}
+
+	return tlsConfig.Clone(), nil
+}
+
+// newTLSContent determines the next set of content for overriding the baseTLSConfig.
+func (c *DynamicServingCertificateController) newTLSContent() (*dynamicCertificateContent, error) {
+	newContent := &dynamicCertificateContent{}
+
+	if c.clientCA != nil {
+		currClientCABundle := c.clientCA.CurrentCABundleContent()
+		// we allow removing all client ca bundles because the server is still secure when this happens. it just means
+		// that there isn't a hint to clients about which client-cert to used.  this happens when there is no client-ca
+		// yet known for authentication, which can happen in aggregated apiservers and some kube-apiserver deployment modes.
+		newContent.clientCA = caBundleContent{caBundle: currClientCABundle}
+	}
+
+	if c.servingCert != nil {
+		currServingCert, currServingKey := c.servingCert.CurrentCertKeyContent()
+		if len(currServingCert) == 0 || len(currServingKey) == 0 {
+			return nil, fmt.Errorf("not loading an empty serving certificate from %q", c.servingCert.Name())
+		}
+
+		newContent.servingCert = certKeyContent{cert: currServingCert, key: currServingKey}
+	}
+
+	for i, sniCert := range c.sniCerts {
+		currCert, currKey := sniCert.CurrentCertKeyContent()
+		if len(currCert) == 0 || len(currKey) == 0 {
+			return nil, fmt.Errorf("not loading an empty SNI certificate from %d/%q", i, sniCert.Name())
+		}
+
+		newContent.sniCerts = append(newContent.sniCerts, sniCertKeyContent{certKeyContent: certKeyContent{cert: currCert, key: currKey}, sniNames: sniCert.SNINames()})
+	}
+
+	return newContent, nil
+}
+
+// syncCerts gets newTLSContent, if it has changed from the existing, the content is parsed and stored for usage in
+// GetConfigForClient.
+func (c *DynamicServingCertificateController) syncCerts() error {
+	newContent, err := c.newTLSContent()
+	if err != nil {
+		return err
+	}
+	// if the content is the same as what we currently have, we can simply skip it.  This works because we are single
+	// threaded.  If you ever make this multi-threaded, add a lock.
+	if newContent.Equal(c.currentlyServedContent) {
+		return nil
+	}
+
+	// make a shallow copy and override the dynamic pieces which have changed.
+	newTLSConfigCopy := c.baseTLSConfig.Clone()
+
+	// parse new content to add to TLSConfig
+	if len(newContent.clientCA.caBundle) > 0 {
+		newClientCAPool := x509.NewCertPool()
+		newClientCAs, err := cert.ParseCertsPEM(newContent.clientCA.caBundle)
+		if err != nil {
+			return fmt.Errorf("unable to load client CA file %q: %v", string(newContent.clientCA.caBundle), err)
+		}
+		for i, cert := range newClientCAs {
+			klog.V(2).Infof("loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert))
+			if c.eventRecorder != nil {
+				c.eventRecorder.Eventf(nil, nil, v1.EventTypeWarning, "TLSConfigChanged", "CACertificateReload", "loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert))
+			}
+
+			newClientCAPool.AddCert(cert)
+		}
+
+		newTLSConfigCopy.ClientCAs = newClientCAPool
+	}
+
+	if len(newContent.servingCert.cert) > 0 && len(newContent.servingCert.key) > 0 {
+		cert, err := tls.X509KeyPair(newContent.servingCert.cert, newContent.servingCert.key)
+		if err != nil {
+			return fmt.Errorf("invalid serving cert keypair: %v", err)
+		}
+
+		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return fmt.Errorf("invalid serving cert: %v", err)
+		}
+
+		klog.V(2).Infof("loaded serving cert [%q]: %s", c.servingCert.Name(), GetHumanCertDetail(x509Cert))
+		if c.eventRecorder != nil {
+			c.eventRecorder.Eventf(nil, nil, v1.EventTypeWarning, "TLSConfigChanged", "ServingCertificateReload", "loaded serving cert [%q]: %s", c.clientCA.Name(), GetHumanCertDetail(x509Cert))
+		}
+
+		newTLSConfigCopy.Certificates = []tls.Certificate{cert}
+	}
+
+	if len(newContent.sniCerts) > 0 {
+		newTLSConfigCopy.NameToCertificate, err = c.BuildNamedCertificates(newContent.sniCerts)
+		if err != nil {
+			return fmt.Errorf("unable to build named certificate map: %v", err)
+		}
+
+		// append all named certs. Otherwise, the go tls stack will think no SNI processing
+		// is necessary because there is only one cert anyway.
+		// Moreover, if servingCert is not set, the first SNI
+		// cert will become the default cert. That's what we expect anyway.
+		for _, sniCert := range newTLSConfigCopy.NameToCertificate {
+			newTLSConfigCopy.Certificates = append(newTLSConfigCopy.Certificates, *sniCert)
+		}
+	}
+
+	// store new values of content for serving.
+	c.currentServingTLSConfig.Store(newTLSConfigCopy)
+	c.currentlyServedContent = newContent // this is single threaded, so we have no locking issue
+
+	return nil
+}
+
+// RunOnce runs a single sync step to ensure that we have a valid starting configuration.
+func (c *DynamicServingCertificateController) RunOnce() error {
+	return c.syncCerts()
+}
+
+// Run starts the kube-apiserver and blocks until stopCh is closed.
+func (c *DynamicServingCertificateController) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting DynamicServingCertificateController")
+	defer klog.Infof("Shutting down DynamicServingCertificateController")
+
+	// synchronously load once.  We will trigger again, so ignoring any error is fine
+	_ = c.RunOnce()
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	go wait.Until(func() {
+		c.Enqueue()
+	}, 1*time.Minute, stopCh)
+
+	<-stopCh
+}
+
+func (c *DynamicServingCertificateController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicServingCertificateController) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.syncCerts()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Enqueue a method to allow separate control loops to cause the certificate controller to trigger and read content.
+func (c *DynamicServingCertificateController) Enqueue() {
+	c.queue.Add(workItemKey)
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go

@@ -0,0 +1,107 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"crypto/x509"
+	"strings"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+)
+
+type unionCAContent []CAContentProvider
+
+var _ Notifier = &unionCAContent{}
+var _ CAContentProvider = &unionCAContent{}
+var _ ControllerRunner = &unionCAContent{}
+
+// NewUnionCAContentProvider returns a CAContentProvider that is a union of other CAContentProviders
+func NewUnionCAContentProvider(caContentProviders ...CAContentProvider) CAContentProvider {
+	return unionCAContent(caContentProviders)
+}
+
+// Name is just an identifier
+func (c unionCAContent) Name() string {
+	names := []string{}
+	for _, curr := range c {
+		names = append(names, curr.Name())
+	}
+	return strings.Join(names, ",")
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c unionCAContent) CurrentCABundleContent() []byte {
+	caBundles := [][]byte{}
+	for _, curr := range c {
+		if currCABytes := curr.CurrentCABundleContent(); len(currCABytes) > 0 {
+			caBundles = append(caBundles, []byte(strings.TrimSpace(string(currCABytes))))
+		}
+	}
+
+	return bytes.Join(caBundles, []byte("\n"))
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c unionCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
+	currCABundle := c.CurrentCABundleContent()
+	if len(currCABundle) == 0 {
+		return x509.VerifyOptions{}, false
+	}
+
+	// TODO make more efficient.  This isn't actually used in any of our mainline paths.  It's called to build the TLSConfig
+	// TODO on file changes, but the actual authentication runs against the individual items, not the union.
+	ret, err := newCABundleAndVerifier(c.Name(), c.CurrentCABundleContent())
+	if err != nil {
+		// because we're made up of already vetted values, this indicates some kind of coding error
+		panic(err)
+	}
+
+	return ret.verifyOptions, true
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c unionCAContent) AddListener(listener Listener) {
+	for _, curr := range c {
+		if notifier, ok := curr.(Notifier); ok {
+			notifier.AddListener(listener)
+		}
+	}
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c unionCAContent) RunOnce() error {
+	errors := []error{}
+	for _, curr := range c {
+		if controller, ok := curr.(ControllerRunner); ok {
+			if err := controller.RunOnce(); err != nil {
+				errors = append(errors, err)
+			}
+		}
+	}
+
+	return utilerrors.NewAggregate(errors)
+}
+
+// Run runs the controller
+func (c unionCAContent) Run(workers int, stopCh <-chan struct{}) {
+	for _, curr := range c {
+		if controller, ok := curr.(ControllerRunner); ok {
+			go controller.Run(workers, stopCh)
+		}
+	}
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/util.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/x509"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// GetHumanCertDetail is a convenient method for printing compact details of certificate that helps when debugging
+// kube-apiserver usage of certs.
+func GetHumanCertDetail(certificate *x509.Certificate) string {
+	humanName := certificate.Subject.CommonName
+	signerHumanName := certificate.Issuer.CommonName
+	if certificate.Subject.CommonName == certificate.Issuer.CommonName {
+		signerHumanName = "<self>"
+	}
+
+	usages := []string{}
+	for _, curr := range certificate.ExtKeyUsage {
+		if curr == x509.ExtKeyUsageClientAuth {
+			usages = append(usages, "client")
+			continue
+		}
+		if curr == x509.ExtKeyUsageServerAuth {
+			usages = append(usages, "serving")
+			continue
+		}
+
+		usages = append(usages, fmt.Sprintf("%d", curr))
+	}
+
+	validServingNames := []string{}
+	for _, ip := range certificate.IPAddresses {
+		validServingNames = append(validServingNames, ip.String())
+	}
+	for _, dnsName := range certificate.DNSNames {
+		validServingNames = append(validServingNames, dnsName)
+	}
+	servingString := ""
+	if len(validServingNames) > 0 {
+		servingString = fmt.Sprintf(" validServingFor=[%s]", strings.Join(validServingNames, ","))
+	}
+
+	groupString := ""
+	if len(certificate.Subject.Organization) > 0 {
+		groupString = fmt.Sprintf(" groups=[%s]", strings.Join(certificate.Subject.Organization, ","))
+	}
+
+	return fmt.Sprintf("%q [%s]%s%s issuer=%q (%v to %v (now=%v))", humanName, strings.Join(usages, ","), groupString, servingString, signerHumanName, certificate.NotBefore.UTC(), certificate.NotAfter.UTC(),
+		time.Now().UTC())
+}