use istio client-go library instead of knative (#1661)

use istio client-go library instead of knative
bump kubernetes dependency version
change code coverage to codecov

vendor/k8s.io/apiserver/pkg/storage/OWNERS

@@ -1,3 +1,5 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
 approvers:
 - lavalamp
 - liggitt

vendor/k8s.io/apiserver/pkg/storage/cacher/watch_cache.go

@@ -29,8 +29,9 @@ import (
 	"k8s.io/apimachinery/pkg/util/clock"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/storage"
-	utiltrace "k8s.io/apiserver/pkg/util/trace"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/klog"
+	utiltrace "k8s.io/utils/trace"
 )
 
 const (
@@ -46,30 +47,27 @@ const (
 // the previous value of the object to enable proper filtering in the
 // upper layers.
 type watchCacheEvent struct {
-	Type                 watch.EventType
-	Object               runtime.Object
-	ObjLabels            labels.Set
-	ObjFields            fields.Set
-	ObjUninitialized     bool
-	PrevObject           runtime.Object
-	PrevObjLabels        labels.Set
-	PrevObjFields        fields.Set
-	PrevObjUninitialized bool
-	Key                  string
-	ResourceVersion      uint64
+	Type            watch.EventType
+	Object          runtime.Object
+	ObjLabels       labels.Set
+	ObjFields       fields.Set
+	PrevObject      runtime.Object
+	PrevObjLabels   labels.Set
+	PrevObjFields   fields.Set
+	Key             string
+	ResourceVersion uint64
 }
 
 // Computing a key of an object is generally non-trivial (it performs
 // e.g. validation underneath). Similarly computing object fields and
 // labels. To avoid computing them multiple times (to serve the event
 // in different List/Watch requests), in the underlying store we are
-// keeping structs (key, object, labels, fields, uninitialized).
+// keeping structs (key, object, labels, fields).
 type storeElement struct {
-	Key           string
-	Object        runtime.Object
-	Labels        labels.Set
-	Fields        fields.Set
-	Uninitialized bool
+	Key    string
+	Object runtime.Object
+	Labels labels.Set
+	Fields fields.Set
 }
 
 func storeElementKey(obj interface{}) (string, error) {
@@ -80,14 +78,6 @@ func storeElementKey(obj interface{}) (string, error) {
 	return elem.Key, nil
 }
 
-// watchCacheElement is a single "watch event" stored in a cache.
-// It contains the resource version of the object and the object
-// itself.
-type watchCacheElement struct {
-	resourceVersion uint64
-	watchCacheEvent *watchCacheEvent
-}
-
 // watchCache implements a Store interface.
 // However, it depends on the elements implementing runtime.Object interface.
 //
@@ -107,14 +97,14 @@ type watchCache struct {
 	keyFunc func(runtime.Object) (string, error)
 
 	// getAttrsFunc is used to get labels and fields of an object.
-	getAttrsFunc func(runtime.Object) (labels.Set, fields.Set, bool, error)
+	getAttrsFunc func(runtime.Object) (labels.Set, fields.Set, error)
 
 	// cache is used a cyclic buffer - its first element (with the smallest
 	// resourceVersion) is defined by startIndex, its last element is defined
 	// by endIndex (if cache is full it will be startIndex + capacity).
 	// Both startIndex and endIndex can be greater than buffer capacity -
 	// you should always apply modulo capacity to get an index in cache array.
-	cache      []watchCacheElement
+	cache      []*watchCacheEvent
 	startIndex int
 	endIndex   int
 
@@ -135,7 +125,7 @@ type watchCache struct {
 
 	// This handler is run at the end of every Add/Update/Delete method
 	// and additionally gets the previous value of the object.
-	onEvent func(*watchCacheEvent)
+	eventHandler func(*watchCacheEvent)
 
 	// for testing timeouts.
 	clock clock.Clock
@@ -147,18 +137,20 @@ type watchCache struct {
 func newWatchCache(
 	capacity int,
 	keyFunc func(runtime.Object) (string, error),
-	getAttrsFunc func(runtime.Object) (labels.Set, fields.Set, bool, error),
+	eventHandler func(*watchCacheEvent),
+	getAttrsFunc func(runtime.Object) (labels.Set, fields.Set, error),
 	versioner storage.Versioner) *watchCache {
 	wc := &watchCache{
 		capacity:            capacity,
 		keyFunc:             keyFunc,
 		getAttrsFunc:        getAttrsFunc,
-		cache:               make([]watchCacheElement, capacity),
+		cache:               make([]*watchCacheEvent, capacity),
 		startIndex:          0,
 		endIndex:            0,
 		store:               cache.NewStore(storeElementKey),
 		resourceVersion:     0,
 		listResourceVersion: 0,
+		eventHandler:        eventHandler,
 		clock:               clock.RealClock{},
 		versioner:           versioner,
 	}
@@ -214,61 +206,72 @@ func (w *watchCache) objectToVersionedRuntimeObject(obj interface{}) (runtime.Ob
 	return object, resourceVersion, nil
 }
 
+// processEvent is safe as long as there is at most one call to it in flight
+// at any point in time.
 func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, updateFunc func(*storeElement) error) error {
 	key, err := w.keyFunc(event.Object)
 	if err != nil {
 		return fmt.Errorf("couldn't compute key: %v", err)
 	}
 	elem := &storeElement{Key: key, Object: event.Object}
-	elem.Labels, elem.Fields, elem.Uninitialized, err = w.getAttrsFunc(event.Object)
+	elem.Labels, elem.Fields, err = w.getAttrsFunc(event.Object)
 	if err != nil {
 		return err
 	}
 
 	watchCacheEvent := &watchCacheEvent{
-		Type:             event.Type,
-		Object:           elem.Object,
-		ObjLabels:        elem.Labels,
-		ObjFields:        elem.Fields,
-		ObjUninitialized: elem.Uninitialized,
-		Key:              key,
-		ResourceVersion:  resourceVersion,
+		Type:            event.Type,
+		Object:          elem.Object,
+		ObjLabels:       elem.Labels,
+		ObjFields:       elem.Fields,
+		Key:             key,
+		ResourceVersion: resourceVersion,
 	}
 
-	// TODO: We should consider moving this lock below after the watchCacheEvent
-	// is created. In such situation, the only problematic scenario is Replace(
-	// happening after getting object from store and before acquiring a lock.
-	// Maybe introduce another lock for this purpose.
-	w.Lock()
-	defer w.Unlock()
-	previous, exists, err := w.store.Get(elem)
-	if err != nil {
+	if err := func() error {
+		// TODO: We should consider moving this lock below after the watchCacheEvent
+		// is created. In such situation, the only problematic scenario is Replace(
+		// happening after getting object from store and before acquiring a lock.
+		// Maybe introduce another lock for this purpose.
+		w.Lock()
+		defer w.Unlock()
+
+		previous, exists, err := w.store.Get(elem)
+		if err != nil {
+			return err
+		}
+		if exists {
+			previousElem := previous.(*storeElement)
+			watchCacheEvent.PrevObject = previousElem.Object
+			watchCacheEvent.PrevObjLabels = previousElem.Labels
+			watchCacheEvent.PrevObjFields = previousElem.Fields
+		}
+
+		w.updateCache(watchCacheEvent)
+		w.resourceVersion = resourceVersion
+		defer w.cond.Broadcast()
+
+		return updateFunc(elem)
+	}(); err != nil {
 		return err
 	}
-	if exists {
-		previousElem := previous.(*storeElement)
-		watchCacheEvent.PrevObject = previousElem.Object
-		watchCacheEvent.PrevObjLabels = previousElem.Labels
-		watchCacheEvent.PrevObjFields = previousElem.Fields
-		watchCacheEvent.PrevObjUninitialized = previousElem.Uninitialized
-	}
 
-	if w.onEvent != nil {
-		w.onEvent(watchCacheEvent)
+	// Avoid calling event handler under lock.
+	// This is safe as long as there is at most one call to processEvent in flight
+	// at any point in time.
+	if w.eventHandler != nil {
+		w.eventHandler(watchCacheEvent)
 	}
-	w.updateCache(resourceVersion, watchCacheEvent)
-	w.resourceVersion = resourceVersion
-	w.cond.Broadcast()
-	return updateFunc(elem)
+	return nil
 }
 
 // Assumes that lock is already held for write.
-func (w *watchCache) updateCache(resourceVersion uint64, event *watchCacheEvent) {
+func (w *watchCache) updateCache(event *watchCacheEvent) {
 	if w.endIndex == w.startIndex+w.capacity {
 		// Cache is full - remove the oldest element.
 		w.startIndex++
 	}
-	w.cache[w.endIndex%w.capacity] = watchCacheElement{resourceVersion, event}
+	w.cache[w.endIndex%w.capacity] = event
 	w.endIndex++
 }
 
@@ -373,16 +376,15 @@ func (w *watchCache) Replace(objs []interface{}, resourceVersion string) error {
 		if err != nil {
 			return fmt.Errorf("couldn't compute key: %v", err)
 		}
-		objLabels, objFields, objUninitialized, err := w.getAttrsFunc(object)
+		objLabels, objFields, err := w.getAttrsFunc(object)
 		if err != nil {
 			return err
 		}
 		toReplace = append(toReplace, &storeElement{
-			Key:           key,
-			Object:        object,
-			Labels:        objLabels,
-			Fields:        objFields,
-			Uninitialized: objUninitialized,
+			Key:    key,
+			Object: object,
+			Labels: objLabels,
+			Fields: objFields,
 		})
 	}
 
@@ -400,6 +402,7 @@ func (w *watchCache) Replace(objs []interface{}, resourceVersion string) error {
 		w.onReplace()
 	}
 	w.cond.Broadcast()
+	klog.V(3).Infof("Replace watchCache (rev: %v) ", resourceVersion)
 	return nil
 }
 
@@ -409,12 +412,6 @@ func (w *watchCache) SetOnReplace(onReplace func()) {
 	w.onReplace = onReplace
 }
 
-func (w *watchCache) SetOnEvent(onEvent func(*watchCacheEvent)) {
-	w.Lock()
-	defer w.Unlock()
-	w.onEvent = onEvent
-}
-
 func (w *watchCache) GetAllEventsSinceThreadUnsafe(resourceVersion uint64) ([]*watchCacheEvent, error) {
 	size := w.endIndex - w.startIndex
 	var oldest uint64
@@ -422,7 +419,7 @@ func (w *watchCache) GetAllEventsSinceThreadUnsafe(resourceVersion uint64) ([]*w
 	case size >= w.capacity:
 		// Once the watch event buffer is full, the oldest watch event we can deliver
 		// is the first one in the buffer.
-		oldest = w.cache[w.startIndex%w.capacity].resourceVersion
+		oldest = w.cache[w.startIndex%w.capacity].ResourceVersion
 	case w.listResourceVersion > 0:
 		// If the watch event buffer isn't full, the oldest watch event we can deliver
 		// is one greater than the resource version of the last full list.
@@ -432,7 +429,7 @@ func (w *watchCache) GetAllEventsSinceThreadUnsafe(resourceVersion uint64) ([]*w
 		// in the buffer.
 		// This should only happen in unit tests that populate the buffer without
 		// performing list/replace operations.
-		oldest = w.cache[w.startIndex%w.capacity].resourceVersion
+		oldest = w.cache[w.startIndex%w.capacity].ResourceVersion
 	default:
 		return nil, fmt.Errorf("watch cache isn't correctly initialized")
 	}
@@ -451,18 +448,17 @@ func (w *watchCache) GetAllEventsSinceThreadUnsafe(resourceVersion uint64) ([]*w
 			if !ok {
 				return nil, fmt.Errorf("not a storeElement: %v", elem)
 			}
-			objLabels, objFields, objUninitialized, err := w.getAttrsFunc(elem.Object)
+			objLabels, objFields, err := w.getAttrsFunc(elem.Object)
 			if err != nil {
 				return nil, err
 			}
 			result[i] = &watchCacheEvent{
-				Type:             watch.Added,
-				Object:           elem.Object,
-				ObjLabels:        objLabels,
-				ObjFields:        objFields,
-				ObjUninitialized: objUninitialized,
-				Key:              elem.Key,
-				ResourceVersion:  w.resourceVersion,
+				Type:            watch.Added,
+				Object:          elem.Object,
+				ObjLabels:       objLabels,
+				ObjFields:       objFields,
+				Key:             elem.Key,
+				ResourceVersion: w.resourceVersion,
 			}
 		}
 		return result, nil
@@ -473,12 +469,12 @@ func (w *watchCache) GetAllEventsSinceThreadUnsafe(resourceVersion uint64) ([]*w
 
 	// Binary search the smallest index at which resourceVersion is greater than the given one.
 	f := func(i int) bool {
-		return w.cache[(w.startIndex+i)%w.capacity].resourceVersion > resourceVersion
+		return w.cache[(w.startIndex+i)%w.capacity].ResourceVersion > resourceVersion
 	}
 	first := sort.Search(size, f)
 	result := make([]*watchCacheEvent, size-first)
 	for i := 0; i < size-first; i++ {
-		result[i] = w.cache[(w.startIndex+first+i)%w.capacity].watchCacheEvent
+		result[i] = w.cache[(w.startIndex+first+i)%w.capacity]
 	}
 	return result, nil
 }

vendor/k8s.io/apiserver/pkg/storage/errors/doc.go

@@ -14,5 +14,5 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package etcd provides conversion of etcd errors to API errors.
+// Package storage provides conversion of storage errors to API errors.
 package storage // import "k8s.io/apiserver/pkg/storage/errors"

vendor/k8s.io/apiserver/pkg/storage/etcd/OWNERS

@@ -1,25 +0,0 @@
-reviewers:
-- lavalamp
-- smarterclayton
-- wojtek-t
-- deads2k
-- derekwaynecarr
-- caesarxuchao
-- mikedanese
-- liggitt
-- davidopp
-- pmorie
-- luxas
-- janetkuo
-- roberthbailey
-- tallclair
-- timothysc
-- dims
-- hongchaodeng
-- krousey
-- fgrzadkowski
-- resouer
-- pweil-
-- mqliang
-- feihujiang
-- enj

vendor/k8s.io/apiserver/pkg/storage/etcd/metrics/metrics.go

@@ -1,122 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package metrics
-
-import (
-	"sync"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-)
-
-var (
-	cacheHitCounterOpts = prometheus.CounterOpts{
-		Name: "etcd_helper_cache_hit_count",
-		Help: "Counter of etcd helper cache hits.",
-	}
-	cacheHitCounter      = prometheus.NewCounter(cacheHitCounterOpts)
-	cacheMissCounterOpts = prometheus.CounterOpts{
-		Name: "etcd_helper_cache_miss_count",
-		Help: "Counter of etcd helper cache miss.",
-	}
-	cacheMissCounter      = prometheus.NewCounter(cacheMissCounterOpts)
-	cacheEntryCounterOpts = prometheus.CounterOpts{
-		Name: "etcd_helper_cache_entry_count",
-		Help: "Counter of etcd helper cache entries. This can be different from etcd_helper_cache_miss_count " +
-			"because two concurrent threads can miss the cache and generate the same entry twice.",
-	}
-	cacheEntryCounter = prometheus.NewCounter(cacheEntryCounterOpts)
-	cacheGetLatency   = prometheus.NewSummary(
-		prometheus.SummaryOpts{
-			Name: "etcd_request_cache_get_latencies_summary",
-			Help: "Latency in microseconds of getting an object from etcd cache",
-		},
-	)
-	cacheAddLatency = prometheus.NewSummary(
-		prometheus.SummaryOpts{
-			Name: "etcd_request_cache_add_latencies_summary",
-			Help: "Latency in microseconds of adding an object to etcd cache",
-		},
-	)
-	etcdRequestLatenciesSummary = prometheus.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "etcd_request_latencies_summary",
-			Help: "Etcd request latency summary in microseconds for each operation and object type.",
-		},
-		[]string{"operation", "type"},
-	)
-	objectCounts = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "etcd_object_counts",
-			Help: "Number of stored objects at the time of last check split by kind.",
-		},
-		[]string{"resource"},
-	)
-)
-
-var registerMetrics sync.Once
-
-// Register all metrics.
-func Register() {
-	// Register the metrics.
-	registerMetrics.Do(func() {
-		prometheus.MustRegister(cacheHitCounter)
-		prometheus.MustRegister(cacheMissCounter)
-		prometheus.MustRegister(cacheEntryCounter)
-		prometheus.MustRegister(cacheAddLatency)
-		prometheus.MustRegister(cacheGetLatency)
-		prometheus.MustRegister(etcdRequestLatenciesSummary)
-		prometheus.MustRegister(objectCounts)
-	})
-}
-
-func UpdateObjectCount(resourcePrefix string, count int64) {
-	objectCounts.WithLabelValues(resourcePrefix).Set(float64(count))
-}
-
-func RecordEtcdRequestLatency(verb, resource string, startTime time.Time) {
-	etcdRequestLatenciesSummary.WithLabelValues(verb, resource).Observe(float64(time.Since(startTime) / time.Microsecond))
-}
-
-func ObserveGetCache(startTime time.Time) {
-	cacheGetLatency.Observe(float64(time.Since(startTime) / time.Microsecond))
-}
-
-func ObserveAddCache(startTime time.Time) {
-	cacheAddLatency.Observe(float64(time.Since(startTime) / time.Microsecond))
-}
-
-func ObserveCacheHit() {
-	cacheHitCounter.Inc()
-}
-
-func ObserveCacheMiss() {
-	cacheMissCounter.Inc()
-}
-
-func ObserveNewEntry() {
-	cacheEntryCounter.Inc()
-}
-
-func Reset() {
-	cacheHitCounter = prometheus.NewCounter(cacheHitCounterOpts)
-	cacheMissCounter = prometheus.NewCounter(cacheMissCounterOpts)
-	cacheEntryCounter = prometheus.NewCounter(cacheEntryCounterOpts)
-	// TODO: Reset cacheAddLatency.
-	// TODO: Reset cacheGetLatency.
-	etcdRequestLatenciesSummary.Reset()
-}

vendor/k8s.io/apiserver/pkg/storage/etcd3/OWNERS

@@ -1,3 +1,5 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
 reviewers:
 - wojtek-t
 - timothysc

vendor/k8s.io/apiserver/pkg/storage/etcd3/api_object_versioner.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package etcd
+package etcd3
 
 import (
 	"strconv"
@@ -44,7 +44,7 @@ func (a APIObjectVersioner) UpdateObject(obj runtime.Object, resourceVersion uin
 }
 
 // UpdateList implements Versioner
-func (a APIObjectVersioner) UpdateList(obj runtime.Object, resourceVersion uint64, nextKey string) error {
+func (a APIObjectVersioner) UpdateList(obj runtime.Object, resourceVersion uint64, nextKey string, count *int64) error {
 	listAccessor, err := meta.ListAccessor(obj)
 	if err != nil || listAccessor == nil {
 		return err
@@ -55,6 +55,7 @@ func (a APIObjectVersioner) UpdateList(obj runtime.Object, resourceVersion uint6
 	}
 	listAccessor.SetResourceVersion(versionString)
 	listAccessor.SetContinue(nextKey)
+	listAccessor.SetRemainingItemCount(count)
 	return nil
 }
 
@@ -101,7 +102,7 @@ func (a APIObjectVersioner) ParseResourceVersion(resourceVersion string) (uint64
 	return version, nil
 }
 
-// APIObjectVersioner implements Versioner
+// Versioner implements Versioner
 var Versioner storage.Versioner = APIObjectVersioner{}
 
 // CompareResourceVersion compares etcd resource versions.  Outside this API they are all strings,

vendor/k8s.io/apiserver/pkg/storage/etcd3/healthcheck.go

@@ -14,4 +14,27 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package etcd // import "k8s.io/apiserver/pkg/storage/etcd"
+package etcd3
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// etcdHealth encodes data returned from etcd /healthz handler.
+type etcdHealth struct {
+	// Note this has to be public so the json library can modify it.
+	Health string `json:"health"`
+}
+
+// EtcdHealthCheck decodes data returned from etcd /healthz handler.
+func EtcdHealthCheck(data []byte) error {
+	obj := etcdHealth{}
+	if err := json.Unmarshal(data, &obj); err != nil {
+		return err
+	}
+	if obj.Health != "true" {
+		return fmt.Errorf("Unhealthy status: %s", obj.Health)
+	}
+	return nil
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/logger.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd3
+
+import (
+	"fmt"
+
+	"github.com/coreos/etcd/clientv3"
+	"k8s.io/klog"
+)
+
+func init() {
+	clientv3.SetLogger(klogWrapper{})
+}
+
+type klogWrapper struct{}
+
+const klogWrapperDepth = 4
+
+func (klogWrapper) Info(args ...interface{}) {
+	klog.InfoDepth(klogWrapperDepth, args...)
+}
+
+func (klogWrapper) Infoln(args ...interface{}) {
+	klog.InfoDepth(klogWrapperDepth, fmt.Sprintln(args...))
+}
+
+func (klogWrapper) Infof(format string, args ...interface{}) {
+	klog.InfoDepth(klogWrapperDepth, fmt.Sprintf(format, args...))
+}
+
+func (klogWrapper) Warning(args ...interface{}) {
+	klog.WarningDepth(klogWrapperDepth, args...)
+}
+
+func (klogWrapper) Warningln(args ...interface{}) {
+	klog.WarningDepth(klogWrapperDepth, fmt.Sprintln(args...))
+}
+
+func (klogWrapper) Warningf(format string, args ...interface{}) {
+	klog.WarningDepth(klogWrapperDepth, fmt.Sprintf(format, args...))
+}
+
+func (klogWrapper) Error(args ...interface{}) {
+	klog.ErrorDepth(klogWrapperDepth, args...)
+}
+
+func (klogWrapper) Errorln(args ...interface{}) {
+	klog.ErrorDepth(klogWrapperDepth, fmt.Sprintln(args...))
+}
+
+func (klogWrapper) Errorf(format string, args ...interface{}) {
+	klog.ErrorDepth(klogWrapperDepth, fmt.Sprintf(format, args...))
+}
+
+func (klogWrapper) Fatal(args ...interface{}) {
+	klog.FatalDepth(klogWrapperDepth, args...)
+}
+
+func (klogWrapper) Fatalln(args ...interface{}) {
+	klog.FatalDepth(klogWrapperDepth, fmt.Sprintln(args...))
+}
+
+func (klogWrapper) Fatalf(format string, args ...interface{}) {
+	klog.FatalDepth(klogWrapperDepth, fmt.Sprintf(format, args...))
+}
+
+func (klogWrapper) V(l int) bool {
+	return bool(klog.V(klog.Level(l)))
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go

@@ -0,0 +1,103 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"sync"
+	"time"
+
+	compbasemetrics "k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+/*
+ * By default, all the following metrics are defined as falling under
+ * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
+ *
+ * Promoting the stability level of the metric is a responsibility of the component owner, since it
+ * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
+ * the metric stability policy.
+ */
+var (
+	etcdRequestLatency = compbasemetrics.NewHistogramVec(
+		&compbasemetrics.HistogramOpts{
+			Name:           "etcd_request_duration_seconds",
+			Help:           "Etcd request latency in seconds for each operation and object type.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"operation", "type"},
+	)
+	objectCounts = compbasemetrics.NewGaugeVec(
+		&compbasemetrics.GaugeOpts{
+			Name:           "etcd_object_counts",
+			Help:           "Number of stored objects at the time of last check split by kind.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
+
+	deprecatedEtcdRequestLatenciesSummary = compbasemetrics.NewSummaryVec(
+		&compbasemetrics.SummaryOpts{
+			Name:           "etcd_request_latencies_summary",
+			Help:           "(Deprecated) Etcd request latency summary in microseconds for each operation and object type.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"operation", "type"},
+	)
+)
+
+var registerMetrics sync.Once
+
+// Register all metrics.
+func Register() {
+	// Register the metrics.
+	registerMetrics.Do(func() {
+		legacyregistry.MustRegister(etcdRequestLatency)
+		legacyregistry.MustRegister(objectCounts)
+
+		// TODO(danielqsj): Remove the following metrics, they are deprecated
+		legacyregistry.MustRegister(deprecatedEtcdRequestLatenciesSummary)
+	})
+}
+
+// UpdateObjectCount sets the etcd_object_counts metric.
+func UpdateObjectCount(resourcePrefix string, count int64) {
+	objectCounts.WithLabelValues(resourcePrefix).Set(float64(count))
+}
+
+// RecordEtcdRequestLatency sets the etcd_request_duration_seconds metrics.
+func RecordEtcdRequestLatency(verb, resource string, startTime time.Time) {
+	etcdRequestLatency.WithLabelValues(verb, resource).Observe(sinceInSeconds(startTime))
+	deprecatedEtcdRequestLatenciesSummary.WithLabelValues(verb, resource).Observe(sinceInMicroseconds(startTime))
+}
+
+// Reset resets the etcd_request_duration_seconds metric.
+func Reset() {
+	etcdRequestLatency.Reset()
+
+	deprecatedEtcdRequestLatenciesSummary.Reset()
+}
+
+// sinceInMicroseconds gets the time since the specified start in microseconds.
+func sinceInMicroseconds(start time.Time) float64 {
+	return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())
+}
+
+// sinceInSeconds gets the time since the specified start in seconds.
+func sinceInSeconds(start time.Time) float64 {
+	return time.Since(start).Seconds()
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/store.go

@@ -36,10 +36,12 @@ import (
 	"k8s.io/apimachinery/pkg/conversion"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/storage"
-	"k8s.io/apiserver/pkg/storage/etcd"
+	"k8s.io/apiserver/pkg/storage/etcd3/metrics"
 	"k8s.io/apiserver/pkg/storage/value"
-	utiltrace "k8s.io/apiserver/pkg/util/trace"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utiltrace "k8s.io/utils/trace"
 )
 
 // authenticatedDataString satisfies the value.Context interface. It uses the key to
@@ -86,7 +88,7 @@ func New(c *clientv3.Client, codec runtime.Codec, prefix string, transformer val
 }
 
 func newStore(c *clientv3.Client, pagingEnabled bool, codec runtime.Codec, prefix string, transformer value.Transformer) *store {
-	versioner := etcd.APIObjectVersioner{}
+	versioner := APIObjectVersioner{}
 	result := &store{
 		client:        c,
 		codec:         codec,
@@ -111,7 +113,9 @@ func (s *store) Versioner() storage.Versioner {
 // Get implements storage.Interface.Get.
 func (s *store) Get(ctx context.Context, key string, resourceVersion string, out runtime.Object, ignoreNotFound bool) error {
 	key = path.Join(s.pathPrefix, key)
+	startTime := time.Now()
 	getResp, err := s.client.KV.Get(ctx, key, s.getOps...)
+	metrics.RecordEtcdRequestLatency("get", getTypeName(out), startTime)
 	if err != nil {
 		return err
 	}
@@ -156,11 +160,13 @@ func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object,
 		return storage.NewInternalError(err.Error())
 	}
 
+	startTime := time.Now()
 	txnResp, err := s.client.KV.Txn(ctx).If(
 		notFound(key),
 	).Then(
 		clientv3.OpPut(key, string(newData), opts...),
 	).Commit()
+	metrics.RecordEtcdRequestLatency("create", getTypeName(obj), startTime)
 	if err != nil {
 		return err
 	}
@@ -176,43 +182,19 @@ func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object,
 }
 
 // Delete implements storage.Interface.Delete.
-func (s *store) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions) error {
+func (s *store) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions, validateDeletion storage.ValidateObjectFunc) error {
 	v, err := conversion.EnforcePtr(out)
 	if err != nil {
-		panic("unable to convert output object to pointer")
+		return fmt.Errorf("unable to convert output object to pointer: %v", err)
 	}
 	key = path.Join(s.pathPrefix, key)
-	if preconditions == nil {
-		return s.unconditionalDelete(ctx, key, out)
-	}
-	return s.conditionalDelete(ctx, key, out, v, preconditions)
+	return s.conditionalDelete(ctx, key, out, v, preconditions, validateDeletion)
 }
 
-func (s *store) unconditionalDelete(ctx context.Context, key string, out runtime.Object) error {
-	// We need to do get and delete in single transaction in order to
-	// know the value and revision before deleting it.
-	txnResp, err := s.client.KV.Txn(ctx).If().Then(
-		clientv3.OpGet(key),
-		clientv3.OpDelete(key),
-	).Commit()
-	if err != nil {
-		return err
-	}
-	getResp := txnResp.Responses[0].GetResponseRange()
-	if len(getResp.Kvs) == 0 {
-		return storage.NewKeyNotFoundError(key, 0)
-	}
-
-	kv := getResp.Kvs[0]
-	data, _, err := s.transformer.TransformFromStorage(kv.Value, authenticatedDataString(key))
-	if err != nil {
-		return storage.NewInternalError(err.Error())
-	}
-	return decode(s.codec, s.versioner, data, out, kv.ModRevision)
-}
-
-func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.Object, v reflect.Value, preconditions *storage.Preconditions) error {
+func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.Object, v reflect.Value, preconditions *storage.Preconditions, validateDeletion storage.ValidateObjectFunc) error {
+	startTime := time.Now()
 	getResp, err := s.client.KV.Get(ctx, key)
+	metrics.RecordEtcdRequestLatency("get", getTypeName(out), startTime)
 	if err != nil {
 		return err
 	}
@@ -221,9 +203,15 @@ func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.O
 		if err != nil {
 			return err
 		}
-		if err := preconditions.Check(key, origState.obj); err != nil {
+		if preconditions != nil {
+			if err := preconditions.Check(key, origState.obj); err != nil {
+				return err
+			}
+		}
+		if err := validateDeletion(ctx, origState.obj); err != nil {
 			return err
 		}
+		startTime := time.Now()
 		txnResp, err := s.client.KV.Txn(ctx).If(
 			clientv3.Compare(clientv3.ModRevision(key), "=", origState.rev),
 		).Then(
@@ -231,6 +219,7 @@ func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.O
 		).Else(
 			clientv3.OpGet(key),
 		).Commit()
+		metrics.RecordEtcdRequestLatency("delete", getTypeName(out), startTime)
 		if err != nil {
 			return err
 		}
@@ -247,17 +236,19 @@ func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.O
 func (s *store) GuaranteedUpdate(
 	ctx context.Context, key string, out runtime.Object, ignoreNotFound bool,
 	preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, suggestion ...runtime.Object) error {
-	trace := utiltrace.New(fmt.Sprintf("GuaranteedUpdate etcd3: %s", reflect.TypeOf(out).String()))
+	trace := utiltrace.New("GuaranteedUpdate etcd3", utiltrace.Field{"type", getTypeName(out)})
 	defer trace.LogIfLong(500 * time.Millisecond)
 
 	v, err := conversion.EnforcePtr(out)
 	if err != nil {
-		panic("unable to convert output object to pointer")
+		return fmt.Errorf("unable to convert output object to pointer: %v", err)
 	}
 	key = path.Join(s.pathPrefix, key)
 
 	getCurrentState := func() (*objState, error) {
+		startTime := time.Now()
 		getResp, err := s.client.KV.Get(ctx, key, s.getOps...)
+		metrics.RecordEtcdRequestLatency("get", getTypeName(out), startTime)
 		if err != nil {
 			return nil, err
 		}
@@ -283,24 +274,38 @@ func (s *store) GuaranteedUpdate(
 	transformContext := authenticatedDataString(key)
 	for {
 		if err := preconditions.Check(key, origState.obj); err != nil {
-			return err
+			// If our data is already up to date, return the error
+			if !mustCheckData {
+				return err
+			}
+
+			// It's possible we were working with stale data
+			// Actually fetch
+			origState, err = getCurrentState()
+			if err != nil {
+				return err
+			}
+			mustCheckData = false
+			// Retry
+			continue
 		}
 
 		ret, ttl, err := s.updateState(origState, tryUpdate)
 		if err != nil {
-			// It's possible we were working with stale data
-			if mustCheckData && apierrors.IsConflict(err) {
-				// Actually fetch
-				origState, err = getCurrentState()
-				if err != nil {
-					return err
-				}
-				mustCheckData = false
-				// Retry
-				continue
+			// If our data is already up to date, return the error
+			if !mustCheckData {
+				return err
 			}
 
-			return err
+			// It's possible we were working with stale data
+			// Actually fetch
+			origState, err = getCurrentState()
+			if err != nil {
+				return err
+			}
+			mustCheckData = false
+			// Retry
+			continue
 		}
 
 		data, err := runtime.Encode(s.codec, ret)
@@ -339,6 +344,7 @@ func (s *store) GuaranteedUpdate(
 		}
 		trace.Step("Transaction prepared")
 
+		startTime := time.Now()
 		txnResp, err := s.client.KV.Txn(ctx).If(
 			clientv3.Compare(clientv3.ModRevision(key), "=", origState.rev),
 		).Then(
@@ -346,6 +352,7 @@ func (s *store) GuaranteedUpdate(
 		).Else(
 			clientv3.OpGet(key),
 		).Commit()
+		metrics.RecordEtcdRequestLatency("update", getTypeName(out), startTime)
 		if err != nil {
 			return err
 		}
@@ -369,17 +376,25 @@ func (s *store) GuaranteedUpdate(
 
 // GetToList implements storage.Interface.GetToList.
 func (s *store) GetToList(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error {
+	trace := utiltrace.New("GetToList etcd3",
+		utiltrace.Field{"key", key},
+		utiltrace.Field{"resourceVersion", resourceVersion},
+		utiltrace.Field{"limit", pred.Limit},
+		utiltrace.Field{"continue", pred.Continue})
+	defer trace.LogIfLong(500 * time.Millisecond)
 	listPtr, err := meta.GetItemsPtr(listObj)
 	if err != nil {
 		return err
 	}
 	v, err := conversion.EnforcePtr(listPtr)
 	if err != nil || v.Kind() != reflect.Slice {
-		panic("need ptr to slice")
+		return fmt.Errorf("need ptr to slice: %v", err)
 	}
 
 	key = path.Join(s.pathPrefix, key)
+	startTime := time.Now()
 	getResp, err := s.client.KV.Get(ctx, key, s.getOps...)
+	metrics.RecordEtcdRequestLatency("get", getTypeName(listPtr), startTime)
 	if err != nil {
 		return err
 	}
@@ -394,12 +409,14 @@ func (s *store) GetToList(ctx context.Context, key string, resourceVersion strin
 		}
 	}
 	// update version with cluster level revision
-	return s.versioner.UpdateList(listObj, uint64(getResp.Header.Revision), "")
+	return s.versioner.UpdateList(listObj, uint64(getResp.Header.Revision), "", nil)
 }
 
 func (s *store) Count(key string) (int64, error) {
 	key = path.Join(s.pathPrefix, key)
+	startTime := time.Now()
 	getResp, err := s.client.KV.Get(context.Background(), key, clientv3.WithRange(clientv3.GetPrefixRangeEnd(key)), clientv3.WithCountOnly())
+	metrics.RecordEtcdRequestLatency("listWithCount", key, startTime)
 	if err != nil {
 		return 0, err
 	}
@@ -468,13 +485,19 @@ func encodeContinue(key, keyPrefix string, resourceVersion int64) (string, error
 
 // List implements storage.Interface.List.
 func (s *store) List(ctx context.Context, key, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error {
+	trace := utiltrace.New("List etcd3",
+		utiltrace.Field{"key", key},
+		utiltrace.Field{"resourceVersion", resourceVersion},
+		utiltrace.Field{"limit", pred.Limit},
+		utiltrace.Field{"continue", pred.Continue})
+	defer trace.LogIfLong(500 * time.Millisecond)
 	listPtr, err := meta.GetItemsPtr(listObj)
 	if err != nil {
 		return err
 	}
 	v, err := conversion.EnforcePtr(listPtr)
 	if err != nil || v.Kind() != reflect.Slice {
-		panic("need ptr to slice")
+		return fmt.Errorf("need ptr to slice: %v", err)
 	}
 
 	if s.pathPrefix != "" {
@@ -553,8 +576,11 @@ func (s *store) List(ctx context.Context, key, resourceVersion string, pred stor
 	// loop until we have filled the requested limit from etcd or there are no more results
 	var lastKey []byte
 	var hasMore bool
+	var getResp *clientv3.GetResponse
 	for {
-		getResp, err := s.client.KV.Get(ctx, key, options...)
+		startTime := time.Now()
+		getResp, err = s.client.KV.Get(ctx, key, options...)
+		metrics.RecordEtcdRequestLatency("list", getTypeName(listPtr), startTime)
 		if err != nil {
 			return interpretListError(err, len(pred.Continue) > 0, continueKey, keyPrefix)
 		}
@@ -614,11 +640,21 @@ func (s *store) List(ctx context.Context, key, resourceVersion string, pred stor
 		if err != nil {
 			return err
 		}
-		return s.versioner.UpdateList(listObj, uint64(returnedRV), next)
+		var remainingItemCount *int64
+		// getResp.Count counts in objects that do not match the pred.
+		// Instead of returning inaccurate count for non-empty selectors, we return nil.
+		// Only set remainingItemCount if the predicate is empty.
+		if utilfeature.DefaultFeatureGate.Enabled(features.RemainingItemCount) {
+			if pred.Empty() {
+				c := int64(getResp.Count - pred.Limit)
+				remainingItemCount = &c
+			}
+		}
+		return s.versioner.UpdateList(listObj, uint64(returnedRV), next, remainingItemCount)
 	}
 
 	// no continuation
-	return s.versioner.UpdateList(listObj, uint64(returnedRV), "")
+	return s.versioner.UpdateList(listObj, uint64(returnedRV), "", nil)
 }
 
 // growSlice takes a slice value and grows its capacity up
@@ -673,9 +709,15 @@ func (s *store) watch(ctx context.Context, key string, rv string, pred storage.S
 
 func (s *store) getState(getResp *clientv3.GetResponse, key string, v reflect.Value, ignoreNotFound bool) (*objState, error) {
 	state := &objState{
-		obj:  reflect.New(v.Type()).Interface().(runtime.Object),
 		meta: &storage.ResponseMeta{},
 	}
+
+	if u, ok := v.Addr().Interface().(runtime.Unstructured); ok {
+		state.obj = u.NewEmptyInstance()
+	} else {
+		state.obj = reflect.New(v.Type()).Interface().(runtime.Object)
+	}
+
 	if len(getResp.Kvs) == 0 {
 		if !ignoreNotFound {
 			return nil, storage.NewKeyNotFoundError(key, 0)
@@ -721,7 +763,9 @@ func (s *store) getStateFromObject(obj runtime.Object) (*objState, error) {
 	if err != nil {
 		return nil, err
 	}
-	s.versioner.UpdateObject(state.obj, uint64(rv))
+	if err := s.versioner.UpdateObject(state.obj, uint64(rv)); err != nil {
+		klog.Errorf("failed to update object version: %v", err)
+	}
 	return state, nil
 }
 
@@ -758,14 +802,16 @@ func (s *store) ttlOpts(ctx context.Context, ttl int64) ([]clientv3.OpOption, er
 // On success, objPtr would be set to the object.
 func decode(codec runtime.Codec, versioner storage.Versioner, value []byte, objPtr runtime.Object, rev int64) error {
 	if _, err := conversion.EnforcePtr(objPtr); err != nil {
-		panic("unable to convert output object to pointer")
+		return fmt.Errorf("unable to convert output object to pointer: %v", err)
 	}
 	_, _, err := codec.Decode(value, nil, objPtr)
 	if err != nil {
 		return err
 	}
 	// being unable to set the version does not prevent the object from being extracted
-	versioner.UpdateObject(objPtr, uint64(rev))
+	if err := versioner.UpdateObject(objPtr, uint64(rev)); err != nil {
+		klog.Errorf("failed to update object version: %v", err)
+	}
 	return nil
 }
 
@@ -776,7 +822,9 @@ func appendListItem(v reflect.Value, data []byte, rev uint64, pred storage.Selec
 		return err
 	}
 	// being unable to set the version does not prevent the object from being extracted
-	versioner.UpdateObject(obj, rev)
+	if err := versioner.UpdateObject(obj, rev); err != nil {
+		klog.Errorf("failed to update object version: %v", err)
+	}
 	if matched, err := pred.Matches(obj); err == nil && matched {
 		v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem()))
 	}
@@ -786,3 +834,8 @@ func appendListItem(v reflect.Value, data []byte, rev uint64, pred storage.Selec
 func notFound(key string) clientv3.Cmp {
 	return clientv3.Compare(clientv3.ModRevision(key), "=", 0)
 }
+
+// getTypeName returns type name of an object for reporting purposes.
+func getTypeName(obj interface{}) string {
+	return reflect.TypeOf(obj).String()
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/watcher.go

@@ -56,9 +56,15 @@ func testingDeferOnDecodeError() {
 
 func init() {
 	// check to see if we are running in a test environment
+	TestOnlySetFatalOnDecodeError(true)
 	fatalOnDecodeError, _ = strconv.ParseBool(os.Getenv("KUBE_PANIC_WATCH_DECODE_ERROR"))
 }
 
+// TestOnlySetFatalOnDecodeError should only be used for cases where decode errors are expected and need to be tested. e.g. conversion webhooks.
+func TestOnlySetFatalOnDecodeError(b bool) {
+	fatalOnDecodeError = b
+}
+
 type watcher struct {
 	client      *clientv3.Client
 	codec       runtime.Codec

vendor/k8s.io/apiserver/pkg/storage/interfaces.go

@@ -40,10 +40,12 @@ type Versioner interface {
 	// from database.
 	UpdateObject(obj runtime.Object, resourceVersion uint64) error
 	// UpdateList sets the resource version into an API list object. Returns an error if the object
-	// cannot be updated correctly. May return nil if the requested object does not need metadata
-	// from database. continueValue is optional and indicates that more results are available if
-	// the client passes that value to the server in a subsequent call.
-	UpdateList(obj runtime.Object, resourceVersion uint64, continueValue string) error
+	// cannot be updated correctly. May return nil if the requested object does not need metadata from
+	// database. continueValue is optional and indicates that more results are available if the client
+	// passes that value to the server in a subsequent call. remainingItemCount indicates the number
+	// of remaining objects if the list is partial. The remainingItemCount field is omitted during
+	// serialization if it is set to nil.
+	UpdateList(obj runtime.Object, resourceVersion uint64, continueValue string, remainingItemCount *int64) error
 	// PrepareObjectForStorage should set SelfLink and ResourceVersion to the empty value. Should
 	// return an error if the specified object cannot be updated.
 	PrepareObjectForStorage(obj runtime.Object) error
@@ -71,23 +73,18 @@ type ResponseMeta struct {
 	ResourceVersion uint64
 }
 
-// MatchValue defines a pair (<index name>, <value for that index>).
-type MatchValue struct {
-	IndexName string
-	Value     string
-}
+// IndexerFunc is a function that for a given object computes
+// <value of an index> for a particular <index>.
+type IndexerFunc func(obj runtime.Object) string
 
-// TriggerPublisherFunc is a function that takes an object, and returns a list of pairs
-// (<index name>, <index value for the given object>) for all indexes known
-// to that function.
-type TriggerPublisherFunc func(obj runtime.Object) []MatchValue
+// IndexerFuncs is a mapping from <index name> to function that
+// for a given object computes <value for that index>.
+type IndexerFuncs map[string]IndexerFunc
 
 // Everything accepts all objects.
 var Everything = SelectionPredicate{
 	Label: labels.Everything(),
 	Field: fields.Everything(),
-	// TODO: split this into a new top level constant?
-	IncludeUninitialized: true,
 }
 
 // Pass an UpdateFunc to Interface.GuaranteedUpdate to make an update
@@ -95,11 +92,24 @@ var Everything = SelectionPredicate{
 // See the comment for GuaranteedUpdate for more details.
 type UpdateFunc func(input runtime.Object, res ResponseMeta) (output runtime.Object, ttl *uint64, err error)
 
+// ValidateObjectFunc is a function to act on a given object. An error may be returned
+// if the hook cannot be completed. The function may NOT transform the provided
+// object.
+type ValidateObjectFunc func(ctx context.Context, obj runtime.Object) error
+
+// ValidateAllObjectFunc is a "admit everything" instance of ValidateObjectFunc.
+func ValidateAllObjectFunc(ctx context.Context, obj runtime.Object) error {
+	return nil
+}
+
 // Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.
 type Preconditions struct {
 	// Specifies the target UID.
 	// +optional
 	UID *types.UID `json:"uid,omitempty"`
+	// Specifies the target ResourceVersion
+	// +optional
+	ResourceVersion *string `json:"resourceVersion,omitempty"`
 }
 
 // NewUIDPreconditions returns a Preconditions with UID set.
@@ -127,8 +137,14 @@ func (p *Preconditions) Check(key string, obj runtime.Object) error {
 			objMeta.GetUID())
 		return NewInvalidObjError(key, err)
 	}
+	if p.ResourceVersion != nil && *p.ResourceVersion != objMeta.GetResourceVersion() {
+		err := fmt.Sprintf(
+			"Precondition failed: ResourceVersion in precondition: %v, ResourceVersion in object meta: %v",
+			*p.ResourceVersion,
+			objMeta.GetResourceVersion())
+		return NewInvalidObjError(key, err)
+	}
 	return nil
-
 }
 
 // Interface offers a common interface for object marshaling/unmarshaling operations and
@@ -144,7 +160,7 @@ type Interface interface {
 
 	// Delete removes the specified key and returns the value that existed at that spot.
 	// If key didn't exist, it will return NotFound storage error.
-	Delete(ctx context.Context, key string, out runtime.Object, preconditions *Preconditions) error
+	Delete(ctx context.Context, key string, out runtime.Object, preconditions *Preconditions, validateDeletion ValidateObjectFunc) error
 
 	// Watch begins watching the specified key. Events are decoded into API objects,
 	// and any items selected by 'p' are sent down to returned watch.Interface.
@@ -212,8 +228,8 @@ type Interface interface {
 	//       // Return the modified object - return an error to stop iterating. Return
 	//       // a uint64 to alter the TTL on the object, or nil to keep it the same value.
 	//       return cur, nil, nil
-	//    }
-	// })
+	//    },
+	// )
 	GuaranteedUpdate(
 		ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool,
 		precondtions *Preconditions, tryUpdate UpdateFunc, suggestion ...runtime.Object) error

vendor/k8s.io/apiserver/pkg/storage/selection_predicate.go

@@ -25,59 +25,59 @@ import (
 
 // AttrFunc returns label and field sets and the uninitialized flag for List or Watch to match.
 // In any failure to parse given object, it returns error.
-type AttrFunc func(obj runtime.Object) (labels.Set, fields.Set, bool, error)
+type AttrFunc func(obj runtime.Object) (labels.Set, fields.Set, error)
 
 // FieldMutationFunc allows the mutation of the field selection fields.  It is mutating to
 // avoid the extra allocation on this common path
 type FieldMutationFunc func(obj runtime.Object, fieldSet fields.Set) error
 
-func DefaultClusterScopedAttr(obj runtime.Object) (labels.Set, fields.Set, bool, error) {
+func DefaultClusterScopedAttr(obj runtime.Object) (labels.Set, fields.Set, error) {
 	metadata, err := meta.Accessor(obj)
 	if err != nil {
-		return nil, nil, false, err
+		return nil, nil, err
 	}
 	fieldSet := fields.Set{
 		"metadata.name": metadata.GetName(),
 	}
 
-	return labels.Set(metadata.GetLabels()), fieldSet, metadata.GetInitializers() != nil, nil
+	return labels.Set(metadata.GetLabels()), fieldSet, nil
 }
 
-func DefaultNamespaceScopedAttr(obj runtime.Object) (labels.Set, fields.Set, bool, error) {
+func DefaultNamespaceScopedAttr(obj runtime.Object) (labels.Set, fields.Set, error) {
 	metadata, err := meta.Accessor(obj)
 	if err != nil {
-		return nil, nil, false, err
+		return nil, nil, err
 	}
 	fieldSet := fields.Set{
 		"metadata.name":      metadata.GetName(),
 		"metadata.namespace": metadata.GetNamespace(),
 	}
 
-	return labels.Set(metadata.GetLabels()), fieldSet, metadata.GetInitializers() != nil, nil
+	return labels.Set(metadata.GetLabels()), fieldSet, nil
 }
 
 func (f AttrFunc) WithFieldMutation(fieldMutator FieldMutationFunc) AttrFunc {
-	return func(obj runtime.Object) (labels.Set, fields.Set, bool, error) {
-		labelSet, fieldSet, initialized, err := f(obj)
+	return func(obj runtime.Object) (labels.Set, fields.Set, error) {
+		labelSet, fieldSet, err := f(obj)
 		if err != nil {
-			return nil, nil, false, err
+			return nil, nil, err
 		}
 		if err := fieldMutator(obj, fieldSet); err != nil {
-			return nil, nil, false, err
+			return nil, nil, err
 		}
-		return labelSet, fieldSet, initialized, nil
+		return labelSet, fieldSet, nil
 	}
 }
 
 // SelectionPredicate is used to represent the way to select objects from api storage.
 type SelectionPredicate struct {
-	Label                labels.Selector
-	Field                fields.Selector
-	IncludeUninitialized bool
-	GetAttrs             AttrFunc
-	IndexFields          []string
-	Limit                int64
-	Continue             string
+	Label               labels.Selector
+	Field               fields.Selector
+	GetAttrs            AttrFunc
+	IndexFields         []string
+	Limit               int64
+	Continue            string
+	AllowWatchBookmarks bool
 }
 
 // Matches returns true if the given object's labels and fields (as
@@ -87,13 +87,10 @@ func (s *SelectionPredicate) Matches(obj runtime.Object) (bool, error) {
 	if s.Empty() {
 		return true, nil
 	}
-	labels, fields, uninitialized, err := s.GetAttrs(obj)
+	labels, fields, err := s.GetAttrs(obj)
 	if err != nil {
 		return false, err
 	}
-	if !s.IncludeUninitialized && uninitialized {
-		return false, nil
-	}
 	matched := s.Label.Matches(labels)
 	if matched && s.Field != nil {
 		matched = matched && s.Field.Matches(fields)
@@ -103,10 +100,7 @@ func (s *SelectionPredicate) Matches(obj runtime.Object) (bool, error) {
 
 // MatchesObjectAttributes returns true if the given labels and fields
 // match s.Label and s.Field.
-func (s *SelectionPredicate) MatchesObjectAttributes(l labels.Set, f fields.Set, uninitialized bool) bool {
-	if !s.IncludeUninitialized && uninitialized {
-		return false
-	}
+func (s *SelectionPredicate) MatchesObjectAttributes(l labels.Set, f fields.Set) bool {
 	if s.Label.Empty() && s.Field.Empty() {
 		return true
 	}
@@ -130,21 +124,7 @@ func (s *SelectionPredicate) MatchesSingle() (string, bool) {
 	return "", false
 }
 
-// For any index defined by IndexFields, if a matcher can match only (a subset)
-// of objects that return <value> for a given index, a pair (<index name>, <value>)
-// wil be returned.
-// TODO: Consider supporting also labels.
-func (s *SelectionPredicate) MatcherIndex() []MatchValue {
-	var result []MatchValue
-	for _, field := range s.IndexFields {
-		if value, ok := s.Field.RequiresExactMatch(field); ok {
-			result = append(result, MatchValue{IndexName: field, Value: value})
-		}
-	}
-	return result
-}
-
 // Empty returns true if the predicate performs no filtering.
 func (s *SelectionPredicate) Empty() bool {
-	return s.Label.Empty() && s.Field.Empty() && s.IncludeUninitialized
+	return s.Label.Empty() && s.Field.Empty()
 }

vendor/k8s.io/apiserver/pkg/storage/storagebackend/OWNERS

@@ -1,3 +1,5 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
 reviewers:
 - lavalamp
 - smarterclayton

vendor/k8s.io/apiserver/pkg/storage/storagebackend/config.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/server/egressselector"
 	"k8s.io/apiserver/pkg/storage/value"
 )
 
@@ -30,18 +31,26 @@ const (
 	DefaultCompactInterval = 5 * time.Minute
 )
 
-// Config is configuration for creating a storage backend.
-type Config struct {
-	// Type defines the type of storage backend. Default ("") is "etcd3".
-	Type string
-	// Prefix is the prefix to all keys passed to storage.Interface methods.
-	Prefix string
+// TransportConfig holds all connection related info,  i.e. equal TransportConfig means equal servers we talk to.
+type TransportConfig struct {
 	// ServerList is the list of storage servers to connect with.
 	ServerList []string
 	// TLS credentials
 	KeyFile  string
 	CertFile string
 	CAFile   string
+	// function to determine the egress dialer. (i.e. konnectivity server dialer)
+	EgressLookup egressselector.Lookup
+}
+
+// Config is configuration for creating a storage backend.
+type Config struct {
+	// Type defines the type of storage backend. Default ("") is "etcd3".
+	Type string
+	// Prefix is the prefix to all keys passed to storage.Interface methods.
+	Prefix string
+	// Transport holds all connection related info, i.e. equal TransportConfig means equal servers we talk to.
+	Transport TransportConfig
 	// Paging indicates whether the server implementation should allow paging (if it is
 	// supported). This is generally configured by feature gating, or by a specific
 	// resource type not wishing to allow paging, and is not intended for end users to
@@ -49,19 +58,24 @@ type Config struct {
 	Paging bool
 
 	Codec runtime.Codec
+	// EncodeVersioner is the same groupVersioner used to build the
+	// storage encoder. Given a list of kinds the input object might belong
+	// to, the EncodeVersioner outputs the gvk the object will be
+	// converted to before persisted in etcd.
+	EncodeVersioner runtime.GroupVersioner
 	// Transformer allows the value to be transformed prior to persisting into etcd.
 	Transformer value.Transformer
 
 	// CompactionInterval is an interval of requesting compaction from apiserver.
 	// If the value is 0, no compaction will be issued.
 	CompactionInterval time.Duration
-
 	// CountMetricPollPeriod specifies how often should count metric be updated
 	CountMetricPollPeriod time.Duration
 }
 
 func NewDefaultConfig(prefix string, codec runtime.Codec) *Config {
 	return &Config{
+		Paging:             true,
 		Prefix:             prefix,
 		Codec:              codec,
 		CompactionInterval: DefaultCompactInterval,

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go

@@ -19,6 +19,10 @@ package factory
 import (
 	"context"
 	"fmt"
+	"net"
+	"net/url"
+	"path"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -27,11 +31,14 @@ import (
 	grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"google.golang.org/grpc"
 
+	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/egressselector"
 	"k8s.io/apiserver/pkg/storage"
 	"k8s.io/apiserver/pkg/storage/etcd3"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/apiserver/pkg/storage/value"
+	"k8s.io/component-base/metrics/legacyregistry"
 )
 
 // The short keepalive timeout and interval have been chosen to aggressively
@@ -44,6 +51,14 @@ const keepaliveTimeout = 10 * time.Second
 // on heavily loaded arm64 CPUs (issue #64649)
 const dialTimeout = 20 * time.Second
 
+func init() {
+	// grpcprom auto-registers (via an init function) their client metrics, since we are opting out of
+	// using the global prometheus registry and using our own wrapped global registry,
+	// we need to explicitly register these metrics to our global registry here.
+	// For reference: https://github.com/kubernetes/kubernetes/pull/81387
+	legacyregistry.RawMustRegister(grpcprom.DefaultClientMetrics)
+}
+
 func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
 	// constructing the etcd v3 client blocks and times out if etcd is not available.
 	// retry in a loop in the background until we successfully create the client, storing the client or error encountered
@@ -54,7 +69,7 @@ func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
 	clientErrMsg.Store("etcd client connection not yet established")
 
 	go wait.PollUntil(time.Second, func() (bool, error) {
-		client, err := newETCD3Client(c)
+		client, err := newETCD3Client(c.Transport)
 		if err != nil {
 			clientErrMsg.Store(err.Error())
 			return false, nil
@@ -71,14 +86,16 @@ func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
 		client := clientValue.Load().(*clientv3.Client)
 		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
 		defer cancel()
-		if _, err := client.Cluster.MemberList(ctx); err != nil {
-			return fmt.Errorf("error listing etcd members: %v", err)
+		// See https://github.com/etcd-io/etcd/blob/master/etcdctl/ctlv3/command/ep_command.go#L118
+		_, err := client.Get(ctx, path.Join(c.Prefix, "health"))
+		if err == nil {
+			return nil
 		}
-		return nil
+		return fmt.Errorf("error getting data from etcd: %v", err)
 	}, nil
 }
 
-func newETCD3Client(c storagebackend.Config) (*clientv3.Client, error) {
+func newETCD3Client(c storagebackend.TransportConfig) (*clientv3.Client, error) {
 	tlsInfo := transport.TLSInfo{
 		CertFile: c.CertFile,
 		KeyFile:  c.KeyFile,
@@ -93,31 +110,123 @@ func newETCD3Client(c storagebackend.Config) (*clientv3.Client, error) {
 	if len(c.CertFile) == 0 && len(c.KeyFile) == 0 && len(c.CAFile) == 0 {
 		tlsConfig = nil
 	}
+	networkContext := egressselector.Etcd.AsNetworkContext()
+	var egressDialer utilnet.DialFunc
+	if c.EgressLookup != nil {
+		egressDialer, err = c.EgressLookup(networkContext)
+		if err != nil {
+			return nil, err
+		}
+	}
+	dialOptions := []grpc.DialOption{
+		grpc.WithBlock(), // block until the underlying connection is up
+		grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor),
+		grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor),
+	}
+	if egressDialer != nil {
+		dialer := func(ctx context.Context, addr string) (net.Conn, error) {
+			u, err := url.Parse(addr)
+			if err != nil {
+				return nil, err
+			}
+			return egressDialer(ctx, "tcp", u.Host)
+		}
+		dialOptions = append(dialOptions, grpc.WithContextDialer(dialer))
+	}
 	cfg := clientv3.Config{
 		DialTimeout:          dialTimeout,
 		DialKeepAliveTime:    keepaliveTime,
 		DialKeepAliveTimeout: keepaliveTimeout,
-		DialOptions: []grpc.DialOption{
-			grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor),
-			grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor),
-		},
-		Endpoints: c.ServerList,
-		TLS:       tlsConfig,
+		DialOptions:          dialOptions,
+		Endpoints:            c.ServerList,
+		TLS:                  tlsConfig,
 	}
 
 	return clientv3.New(cfg)
 }
 
+type runningCompactor struct {
+	interval time.Duration
+	cancel   context.CancelFunc
+	client   *clientv3.Client
+	refs     int
+}
+
+var (
+	lock       sync.Mutex
+	compactors = map[string]*runningCompactor{}
+)
+
+// startCompactorOnce start one compactor per transport. If the interval get smaller on repeated calls, the
+// compactor is replaced. A destroy func is returned. If all destroy funcs with the same transport are called,
+// the compactor is stopped.
+func startCompactorOnce(c storagebackend.TransportConfig, interval time.Duration) (func(), error) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	key := fmt.Sprintf("%v", c) // gives: {[server1 server2] keyFile certFile caFile}
+	if compactor, foundBefore := compactors[key]; !foundBefore || compactor.interval > interval {
+		compactorClient, err := newETCD3Client(c)
+		if err != nil {
+			return nil, err
+		}
+
+		if foundBefore {
+			// replace compactor
+			compactor.cancel()
+			compactor.client.Close()
+		} else {
+			// start new compactor
+			compactor = &runningCompactor{}
+			compactors[key] = compactor
+		}
+
+		ctx, cancel := context.WithCancel(context.Background())
+
+		compactor.interval = interval
+		compactor.cancel = cancel
+		compactor.client = compactorClient
+
+		etcd3.StartCompactor(ctx, compactorClient, interval)
+	}
+
+	compactors[key].refs++
+
+	return func() {
+		lock.Lock()
+		defer lock.Unlock()
+
+		compactor := compactors[key]
+		compactor.refs--
+		if compactor.refs == 0 {
+			compactor.cancel()
+			compactor.client.Close()
+			delete(compactors, key)
+		}
+	}, nil
+}
+
 func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
-	client, err := newETCD3Client(c)
+	stopCompactor, err := startCompactorOnce(c.Transport, c.CompactionInterval)
 	if err != nil {
 		return nil, nil, err
 	}
-	ctx, cancel := context.WithCancel(context.Background())
-	etcd3.StartCompactor(ctx, client, c.CompactionInterval)
+
+	client, err := newETCD3Client(c.Transport)
+	if err != nil {
+		stopCompactor()
+		return nil, nil, err
+	}
+
+	var once sync.Once
 	destroyFunc := func() {
-		cancel()
-		client.Close()
+		// we know that storage destroy funcs are called multiple times (due to reuse in subresources).
+		// Hence, we only destroy once.
+		// TODO: fix duplicated storage destroy calls higher level
+		once.Do(func() {
+			stopCompactor()
+			client.Close()
+		})
 	}
 	transformer := c.Transformer
 	if transformer == nil {

vendor/k8s.io/apiserver/pkg/storage/util.go

@@ -39,14 +39,6 @@ func EverythingFunc(runtime.Object) bool {
 	return true
 }
 
-func NoTriggerFunc() []MatchValue {
-	return nil
-}
-
-func NoTriggerPublisher(runtime.Object) []MatchValue {
-	return nil
-}
-
 func NamespaceKeyFunc(prefix string, obj runtime.Object) (string, error) {
 	meta, err := meta.Accessor(obj)
 	if err != nil {

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes.go

@@ -0,0 +1,152 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package aes transforms values for storage at rest using AES-GCM.
+package aes
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"io"
+
+	"k8s.io/apiserver/pkg/storage/value"
+)
+
+// gcm implements AEAD encryption of the provided values given a cipher.Block algorithm.
+// The authenticated data provided as part of the value.Context method must match when the same
+// value is set to and loaded from storage. In order to ensure that values cannot be copied by
+// an attacker from a location under their control, use characteristics of the storage location
+// (such as the etcd key) as part of the authenticated data.
+//
+// Because this mode requires a generated IV and IV reuse is a known weakness of AES-GCM, keys
+// must be rotated before a birthday attack becomes feasible. NIST SP 800-38D
+// (http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf) recommends using the same
+// key with random 96-bit nonces (the default nonce length) no more than 2^32 times, and
+// therefore transformers using this implementation *must* ensure they allow for frequent key
+// rotation. Future work should include investigation of AES-GCM-SIV as an alternative to
+// random nonces.
+type gcm struct {
+	block cipher.Block
+}
+
+// NewGCMTransformer takes the given block cipher and performs encryption and decryption on the given
+// data.
+func NewGCMTransformer(block cipher.Block) value.Transformer {
+	return &gcm{block: block}
+}
+
+func (t *gcm) TransformFromStorage(data []byte, context value.Context) ([]byte, bool, error) {
+	aead, err := cipher.NewGCM(t.block)
+	if err != nil {
+		return nil, false, err
+	}
+	nonceSize := aead.NonceSize()
+	if len(data) < nonceSize {
+		return nil, false, fmt.Errorf("the stored data was shorter than the required size")
+	}
+	result, err := aead.Open(nil, data[:nonceSize], data[nonceSize:], context.AuthenticatedData())
+	return result, false, err
+}
+
+func (t *gcm) TransformToStorage(data []byte, context value.Context) ([]byte, error) {
+	aead, err := cipher.NewGCM(t.block)
+	if err != nil {
+		return nil, err
+	}
+	nonceSize := aead.NonceSize()
+	result := make([]byte, nonceSize+aead.Overhead()+len(data))
+	n, err := rand.Read(result[:nonceSize])
+	if err != nil {
+		return nil, err
+	}
+	if n != nonceSize {
+		return nil, fmt.Errorf("unable to read sufficient random bytes")
+	}
+	cipherText := aead.Seal(result[nonceSize:nonceSize], result[:nonceSize], data, context.AuthenticatedData())
+	return result[:nonceSize+len(cipherText)], nil
+}
+
+// cbc implements encryption at rest of the provided values given a cipher.Block algorithm.
+type cbc struct {
+	block cipher.Block
+}
+
+// NewCBCTransformer takes the given block cipher and performs encryption and decryption on the given
+// data.
+func NewCBCTransformer(block cipher.Block) value.Transformer {
+	return &cbc{block: block}
+}
+
+var (
+	errInvalidBlockSize    = fmt.Errorf("the stored data is not a multiple of the block size")
+	errInvalidPKCS7Data    = errors.New("invalid PKCS7 data (empty or not padded)")
+	errInvalidPKCS7Padding = errors.New("invalid padding on input")
+)
+
+func (t *cbc) TransformFromStorage(data []byte, context value.Context) ([]byte, bool, error) {
+	blockSize := aes.BlockSize
+	if len(data) < blockSize {
+		return nil, false, fmt.Errorf("the stored data was shorter than the required size")
+	}
+	iv := data[:blockSize]
+	data = data[blockSize:]
+
+	if len(data)%blockSize != 0 {
+		return nil, false, errInvalidBlockSize
+	}
+
+	result := make([]byte, len(data))
+	copy(result, data)
+	mode := cipher.NewCBCDecrypter(t.block, iv)
+	mode.CryptBlocks(result, result)
+
+	// remove and verify PKCS#7 padding for CBC
+	c := result[len(result)-1]
+	paddingSize := int(c)
+	size := len(result) - paddingSize
+	if paddingSize == 0 || paddingSize > len(result) {
+		return nil, false, errInvalidPKCS7Data
+	}
+	for i := 0; i < paddingSize; i++ {
+		if result[size+i] != c {
+			return nil, false, errInvalidPKCS7Padding
+		}
+	}
+
+	return result[:size], false, nil
+}
+
+func (t *cbc) TransformToStorage(data []byte, context value.Context) ([]byte, error) {
+	blockSize := aes.BlockSize
+	paddingSize := blockSize - (len(data) % blockSize)
+	result := make([]byte, blockSize+len(data)+paddingSize)
+	iv := result[:blockSize]
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		return nil, fmt.Errorf("unable to read sufficient random bytes")
+	}
+	copy(result[blockSize:], data)
+
+	// add PKCS#7 padding for CBC
+	copy(result[blockSize+len(data):], bytes.Repeat([]byte{byte(paddingSize)}, paddingSize))
+
+	mode := cipher.NewCBCEncrypter(t.block, iv)
+	mode.CryptBlocks(result[blockSize:], result[blockSize:])
+	return result, nil
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go

@@ -0,0 +1,173 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package envelope transforms values for storage at rest using a Envelope provider
+package envelope
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"time"
+
+	"k8s.io/apiserver/pkg/storage/value"
+
+	lru "github.com/hashicorp/golang-lru"
+	"golang.org/x/crypto/cryptobyte"
+)
+
+// defaultCacheSize is the number of decrypted DEKs which would be cached by the transformer.
+const defaultCacheSize = 1000
+
+func init() {
+	value.RegisterMetrics()
+}
+
+// Service allows encrypting and decrypting data using an external Key Management Service.
+type Service interface {
+	// Decrypt a given bytearray to obtain the original data as bytes.
+	Decrypt(data []byte) ([]byte, error)
+	// Encrypt bytes to a ciphertext.
+	Encrypt(data []byte) ([]byte, error)
+}
+
+type envelopeTransformer struct {
+	envelopeService Service
+
+	// transformers is a thread-safe LRU cache which caches decrypted DEKs indexed by their encrypted form.
+	transformers *lru.Cache
+
+	// baseTransformerFunc creates a new transformer for encrypting the data with the DEK.
+	baseTransformerFunc func(cipher.Block) value.Transformer
+}
+
+// NewEnvelopeTransformer returns a transformer which implements a KEK-DEK based envelope encryption scheme.
+// It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to
+// the data items they encrypt. A cache (of size cacheSize) is maintained to store the most recently
+// used decrypted DEKs in memory.
+func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) value.Transformer) (value.Transformer, error) {
+	if cacheSize == 0 {
+		cacheSize = defaultCacheSize
+	}
+	cache, err := lru.New(cacheSize)
+	if err != nil {
+		return nil, err
+	}
+	return &envelopeTransformer{
+		envelopeService:     envelopeService,
+		transformers:        cache,
+		baseTransformerFunc: baseTransformerFunc,
+	}, nil
+}
+
+// TransformFromStorage decrypts data encrypted by this transformer using envelope encryption.
+func (t *envelopeTransformer) TransformFromStorage(data []byte, context value.Context) ([]byte, bool, error) {
+	// Read the 16 bit length-of-DEK encoded at the start of the encrypted DEK. 16 bits can
+	// represent a maximum key length of 65536 bytes. We are using a 256 bit key, whose
+	// length cannot fit in 8 bits (1 byte). Thus, we use 16 bits (2 bytes) to store the length.
+	var encKey cryptobyte.String
+	s := cryptobyte.String(data)
+	if ok := s.ReadUint16LengthPrefixed(&encKey); !ok {
+		return nil, false, fmt.Errorf("invalid data encountered by envelope transformer: failed to read uint16 length prefixed data")
+	}
+
+	encData := []byte(s)
+
+	// Look up the decrypted DEK from cache or Envelope.
+	transformer := t.getTransformer(encKey)
+	if transformer == nil {
+		value.RecordCacheMiss()
+		key, err := t.envelopeService.Decrypt(encKey)
+		if err != nil {
+			return nil, false, fmt.Errorf("error while decrypting key: %q", err)
+		}
+		transformer, err = t.addTransformer(encKey, key)
+		if err != nil {
+			return nil, false, err
+		}
+	}
+	return transformer.TransformFromStorage(encData, context)
+}
+
+// TransformToStorage encrypts data to be written to disk using envelope encryption.
+func (t *envelopeTransformer) TransformToStorage(data []byte, context value.Context) ([]byte, error) {
+	newKey, err := generateKey(32)
+	if err != nil {
+		return nil, err
+	}
+
+	encKey, err := t.envelopeService.Encrypt(newKey)
+	if err != nil {
+		return nil, err
+	}
+
+	transformer, err := t.addTransformer(encKey, newKey)
+	if err != nil {
+		return nil, err
+	}
+
+	result, err := transformer.TransformToStorage(data, context)
+	if err != nil {
+		return nil, err
+	}
+	// Append the length of the encrypted DEK as the first 2 bytes.
+	b := cryptobyte.NewBuilder(nil)
+	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes([]byte(encKey))
+	})
+	b.AddBytes(result)
+
+	return b.Bytes()
+}
+
+var _ value.Transformer = &envelopeTransformer{}
+
+// addTransformer inserts a new transformer to the Envelope cache of DEKs for future reads.
+func (t *envelopeTransformer) addTransformer(encKey []byte, key []byte) (value.Transformer, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	transformer := t.baseTransformerFunc(block)
+	// Use base64 of encKey as the key into the cache because hashicorp/golang-lru
+	// cannot hash []uint8.
+	t.transformers.Add(base64.StdEncoding.EncodeToString(encKey), transformer)
+	return transformer, nil
+}
+
+// getTransformer fetches the transformer corresponding to encKey from cache, if it exists.
+func (t *envelopeTransformer) getTransformer(encKey []byte) value.Transformer {
+	_transformer, found := t.transformers.Get(base64.StdEncoding.EncodeToString(encKey))
+	if found {
+		return _transformer.(value.Transformer)
+	}
+	return nil
+}
+
+// generateKey generates a random key using system randomness.
+func generateKey(length int) (key []byte, err error) {
+	defer func(start time.Time) {
+		value.RecordDataKeyGeneration(start, err)
+	}(time.Now())
+	key = make([]byte, length)
+	if _, err = rand.Read(key); err != nil {
+		return nil, err
+	}
+
+	return key, nil
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go

@@ -0,0 +1,167 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package envelope transforms values for storage at rest using a Envelope provider
+package envelope
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"k8s.io/klog"
+
+	"google.golang.org/grpc"
+
+	kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
+)
+
+const (
+	// Now only supported unix domain socket.
+	unixProtocol = "unix"
+
+	// Current version for the protocol interface definition.
+	kmsapiVersion = "v1beta1"
+
+	versionErrorf = "KMS provider api version %s is not supported, only %s is supported now"
+)
+
+// The gRPC implementation for envelope.Service.
+type gRPCService struct {
+	kmsClient      kmsapi.KeyManagementServiceClient
+	connection     *grpc.ClientConn
+	callTimeout    time.Duration
+	mux            sync.RWMutex
+	versionChecked bool
+}
+
+// NewGRPCService returns an envelope.Service which use gRPC to communicate the remote KMS provider.
+func NewGRPCService(endpoint string, callTimeout time.Duration) (Service, error) {
+	klog.V(4).Infof("Configure KMS provider with endpoint: %s", endpoint)
+
+	addr, err := parseEndpoint(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	connection, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithDefaultCallOptions(grpc.FailFast(false)), grpc.WithDialer(
+		func(string, time.Duration) (net.Conn, error) {
+			// Ignoring addr and timeout arguments:
+			// addr - comes from the closure
+			// timeout - is ignored since we are connecting in a non-blocking configuration
+			c, err := net.DialTimeout(unixProtocol, addr, 0)
+			if err != nil {
+				klog.Errorf("failed to create connection to unix socket: %s, error: %v", addr, err)
+			}
+			return c, err
+		}))
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to create connection to %s, error: %v", endpoint, err)
+	}
+
+	kmsClient := kmsapi.NewKeyManagementServiceClient(connection)
+	return &gRPCService{
+		kmsClient:   kmsClient,
+		connection:  connection,
+		callTimeout: callTimeout,
+	}, nil
+}
+
+// Parse the endpoint to extract schema, host or path.
+func parseEndpoint(endpoint string) (string, error) {
+	if len(endpoint) == 0 {
+		return "", fmt.Errorf("remote KMS provider can't use empty string as endpoint")
+	}
+
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return "", fmt.Errorf("invalid endpoint %q for remote KMS provider, error: %v", endpoint, err)
+	}
+
+	if u.Scheme != unixProtocol {
+		return "", fmt.Errorf("unsupported scheme %q for remote KMS provider", u.Scheme)
+	}
+
+	// Linux abstract namespace socket - no physical file required
+	// Warning: Linux Abstract sockets have not concept of ACL (unlike traditional file based sockets).
+	// However, Linux Abstract sockets are subject to Linux networking namespace, so will only be accessible to
+	// containers within the same pod (unless host networking is used).
+	if strings.HasPrefix(u.Path, "/@") {
+		return strings.TrimPrefix(u.Path, "/"), nil
+	}
+
+	return u.Path, nil
+}
+
+func (g *gRPCService) checkAPIVersion(ctx context.Context) error {
+	g.mux.Lock()
+	defer g.mux.Unlock()
+
+	if g.versionChecked {
+		return nil
+	}
+
+	request := &kmsapi.VersionRequest{Version: kmsapiVersion}
+	response, err := g.kmsClient.Version(ctx, request)
+	if err != nil {
+		return fmt.Errorf("failed get version from remote KMS provider: %v", err)
+	}
+	if response.Version != kmsapiVersion {
+		return fmt.Errorf(versionErrorf, response.Version, kmsapiVersion)
+	}
+	g.versionChecked = true
+
+	klog.V(4).Infof("Version of KMS provider is %s", response.Version)
+	return nil
+}
+
+// Decrypt a given data string to obtain the original byte data.
+func (g *gRPCService) Decrypt(cipher []byte) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), g.callTimeout)
+	defer cancel()
+
+	if err := g.checkAPIVersion(ctx); err != nil {
+		return nil, err
+	}
+
+	request := &kmsapi.DecryptRequest{Cipher: cipher, Version: kmsapiVersion}
+	response, err := g.kmsClient.Decrypt(ctx, request)
+	if err != nil {
+		return nil, err
+	}
+	return response.Plain, nil
+}
+
+// Encrypt bytes to a string ciphertext.
+func (g *gRPCService) Encrypt(plain []byte) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), g.callTimeout)
+	defer cancel()
+	if err := g.checkAPIVersion(ctx); err != nil {
+		return nil, err
+	}
+
+	request := &kmsapi.EncryptRequest{Plain: plain, Version: kmsapiVersion}
+	response, err := g.kmsClient.Encrypt(ctx, request)
+	if err != nil {
+		return nil, err
+	}
+	return response.Cipher, nil
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1/service.pb.go

@@ -0,0 +1,502 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: service.proto
+
+package v1beta1
+
+import (
+	context "context"
+	fmt "fmt"
+	proto "github.com/gogo/protobuf/proto"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+
+type VersionRequest struct {
+	// Version of the KMS plugin API.
+	Version              string   `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *VersionRequest) Reset()         { *m = VersionRequest{} }
+func (m *VersionRequest) String() string { return proto.CompactTextString(m) }
+func (*VersionRequest) ProtoMessage()    {}
+func (*VersionRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a0b84a42fa06f626, []int{0}
+}
+func (m *VersionRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_VersionRequest.Unmarshal(m, b)
+}
+func (m *VersionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_VersionRequest.Marshal(b, m, deterministic)
+}
+func (m *VersionRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_VersionRequest.Merge(m, src)
+}
+func (m *VersionRequest) XXX_Size() int {
+	return xxx_messageInfo_VersionRequest.Size(m)
+}
+func (m *VersionRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_VersionRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_VersionRequest proto.InternalMessageInfo
+
+func (m *VersionRequest) GetVersion() string {
+	if m != nil {
+		return m.Version
+	}
+	return ""
+}
+
+type VersionResponse struct {
+	// Version of the KMS plugin API.
+	Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"`
+	// Name of the KMS provider.
+	RuntimeName string `protobuf:"bytes,2,opt,name=runtime_name,json=runtimeName,proto3" json:"runtime_name,omitempty"`
+	// Version of the KMS provider. The string must be semver-compatible.
+	RuntimeVersion       string   `protobuf:"bytes,3,opt,name=runtime_version,json=runtimeVersion,proto3" json:"runtime_version,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *VersionResponse) Reset()         { *m = VersionResponse{} }
+func (m *VersionResponse) String() string { return proto.CompactTextString(m) }
+func (*VersionResponse) ProtoMessage()    {}
+func (*VersionResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a0b84a42fa06f626, []int{1}
+}
+func (m *VersionResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_VersionResponse.Unmarshal(m, b)
+}
+func (m *VersionResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_VersionResponse.Marshal(b, m, deterministic)
+}
+func (m *VersionResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_VersionResponse.Merge(m, src)
+}
+func (m *VersionResponse) XXX_Size() int {
+	return xxx_messageInfo_VersionResponse.Size(m)
+}
+func (m *VersionResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_VersionResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_VersionResponse proto.InternalMessageInfo
+
+func (m *VersionResponse) GetVersion() string {
+	if m != nil {
+		return m.Version
+	}
+	return ""
+}
+
+func (m *VersionResponse) GetRuntimeName() string {
+	if m != nil {
+		return m.RuntimeName
+	}
+	return ""
+}
+
+func (m *VersionResponse) GetRuntimeVersion() string {
+	if m != nil {
+		return m.RuntimeVersion
+	}
+	return ""
+}
+
+type DecryptRequest struct {
+	// Version of the KMS plugin API.
+	Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"`
+	// The data to be decrypted.
+	Cipher               []byte   `protobuf:"bytes,2,opt,name=cipher,proto3" json:"cipher,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *DecryptRequest) Reset()         { *m = DecryptRequest{} }
+func (m *DecryptRequest) String() string { return proto.CompactTextString(m) }
+func (*DecryptRequest) ProtoMessage()    {}
+func (*DecryptRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a0b84a42fa06f626, []int{2}
+}
+func (m *DecryptRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_DecryptRequest.Unmarshal(m, b)
+}
+func (m *DecryptRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_DecryptRequest.Marshal(b, m, deterministic)
+}
+func (m *DecryptRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DecryptRequest.Merge(m, src)
+}
+func (m *DecryptRequest) XXX_Size() int {
+	return xxx_messageInfo_DecryptRequest.Size(m)
+}
+func (m *DecryptRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_DecryptRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DecryptRequest proto.InternalMessageInfo
+
+func (m *DecryptRequest) GetVersion() string {
+	if m != nil {
+		return m.Version
+	}
+	return ""
+}
+
+func (m *DecryptRequest) GetCipher() []byte {
+	if m != nil {
+		return m.Cipher
+	}
+	return nil
+}
+
+type DecryptResponse struct {
+	// The decrypted data.
+	Plain                []byte   `protobuf:"bytes,1,opt,name=plain,proto3" json:"plain,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *DecryptResponse) Reset()         { *m = DecryptResponse{} }
+func (m *DecryptResponse) String() string { return proto.CompactTextString(m) }
+func (*DecryptResponse) ProtoMessage()    {}
+func (*DecryptResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a0b84a42fa06f626, []int{3}
+}
+func (m *DecryptResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_DecryptResponse.Unmarshal(m, b)
+}
+func (m *DecryptResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_DecryptResponse.Marshal(b, m, deterministic)
+}
+func (m *DecryptResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DecryptResponse.Merge(m, src)
+}
+func (m *DecryptResponse) XXX_Size() int {
+	return xxx_messageInfo_DecryptResponse.Size(m)
+}
+func (m *DecryptResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_DecryptResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DecryptResponse proto.InternalMessageInfo
+
+func (m *DecryptResponse) GetPlain() []byte {
+	if m != nil {
+		return m.Plain
+	}
+	return nil
+}
+
+type EncryptRequest struct {
+	// Version of the KMS plugin API.
+	Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"`
+	// The data to be encrypted.
+	Plain                []byte   `protobuf:"bytes,2,opt,name=plain,proto3" json:"plain,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *EncryptRequest) Reset()         { *m = EncryptRequest{} }
+func (m *EncryptRequest) String() string { return proto.CompactTextString(m) }
+func (*EncryptRequest) ProtoMessage()    {}
+func (*EncryptRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a0b84a42fa06f626, []int{4}
+}
+func (m *EncryptRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_EncryptRequest.Unmarshal(m, b)
+}
+func (m *EncryptRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_EncryptRequest.Marshal(b, m, deterministic)
+}
+func (m *EncryptRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_EncryptRequest.Merge(m, src)
+}
+func (m *EncryptRequest) XXX_Size() int {
+	return xxx_messageInfo_EncryptRequest.Size(m)
+}
+func (m *EncryptRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_EncryptRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_EncryptRequest proto.InternalMessageInfo
+
+func (m *EncryptRequest) GetVersion() string {
+	if m != nil {
+		return m.Version
+	}
+	return ""
+}
+
+func (m *EncryptRequest) GetPlain() []byte {
+	if m != nil {
+		return m.Plain
+	}
+	return nil
+}
+
+type EncryptResponse struct {
+	// The encrypted data.
+	Cipher               []byte   `protobuf:"bytes,1,opt,name=cipher,proto3" json:"cipher,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *EncryptResponse) Reset()         { *m = EncryptResponse{} }
+func (m *EncryptResponse) String() string { return proto.CompactTextString(m) }
+func (*EncryptResponse) ProtoMessage()    {}
+func (*EncryptResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a0b84a42fa06f626, []int{5}
+}
+func (m *EncryptResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_EncryptResponse.Unmarshal(m, b)
+}
+func (m *EncryptResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_EncryptResponse.Marshal(b, m, deterministic)
+}
+func (m *EncryptResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_EncryptResponse.Merge(m, src)
+}
+func (m *EncryptResponse) XXX_Size() int {
+	return xxx_messageInfo_EncryptResponse.Size(m)
+}
+func (m *EncryptResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_EncryptResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_EncryptResponse proto.InternalMessageInfo
+
+func (m *EncryptResponse) GetCipher() []byte {
+	if m != nil {
+		return m.Cipher
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*VersionRequest)(nil), "v1beta1.VersionRequest")
+	proto.RegisterType((*VersionResponse)(nil), "v1beta1.VersionResponse")
+	proto.RegisterType((*DecryptRequest)(nil), "v1beta1.DecryptRequest")
+	proto.RegisterType((*DecryptResponse)(nil), "v1beta1.DecryptResponse")
+	proto.RegisterType((*EncryptRequest)(nil), "v1beta1.EncryptRequest")
+	proto.RegisterType((*EncryptResponse)(nil), "v1beta1.EncryptResponse")
+}
+
+func init() { proto.RegisterFile("service.proto", fileDescriptor_a0b84a42fa06f626) }
+
+var fileDescriptor_a0b84a42fa06f626 = []byte{
+	// 287 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0xcd, 0x4a, 0xc4, 0x30,
+	0x10, 0xde, 0xae, 0xb8, 0xc5, 0xb1, 0xb6, 0x10, 0x16, 0x2d, 0x9e, 0x34, 0x97, 0x55, 0x0f, 0x85,
+	0xd5, 0xbb, 0x88, 0xe8, 0x49, 0xf4, 0x50, 0xc1, 0xab, 0x64, 0xcb, 0xa0, 0x05, 0x9b, 0xc6, 0x24,
+	0x5b, 0xd9, 0x17, 0xf5, 0x79, 0xc4, 0x66, 0x5a, 0xd3, 0x15, 0x71, 0x8f, 0x33, 0x99, 0xef, 0x6f,
+	0x26, 0xb0, 0x67, 0x50, 0x37, 0x65, 0x81, 0x99, 0xd2, 0xb5, 0xad, 0x59, 0xd8, 0xcc, 0x17, 0x68,
+	0xc5, 0x9c, 0x9f, 0x41, 0xfc, 0x84, 0xda, 0x94, 0xb5, 0xcc, 0xf1, 0x7d, 0x89, 0xc6, 0xb2, 0x14,
+	0xc2, 0xc6, 0x75, 0xd2, 0xe0, 0x28, 0x38, 0xd9, 0xc9, 0xbb, 0x92, 0x7f, 0x40, 0xd2, 0xcf, 0x1a,
+	0x55, 0x4b, 0x83, 0x7f, 0x0f, 0xb3, 0x63, 0x88, 0xf4, 0x52, 0xda, 0xb2, 0xc2, 0x67, 0x29, 0x2a,
+	0x4c, 0xc7, 0xed, 0xf3, 0x2e, 0xf5, 0x1e, 0x44, 0x85, 0x6c, 0x06, 0x49, 0x37, 0xd2, 0x91, 0x6c,
+	0xb5, 0x53, 0x31, 0xb5, 0x49, 0x8d, 0x5f, 0x43, 0x7c, 0x83, 0x85, 0x5e, 0x29, 0xfb, 0xaf, 0x49,
+	0xb6, 0x0f, 0x93, 0xa2, 0x54, 0xaf, 0xa8, 0x5b, 0xc5, 0x28, 0xa7, 0x8a, 0xcf, 0x20, 0xe9, 0x39,
+	0xc8, 0xfc, 0x14, 0xb6, 0xd5, 0x9b, 0x28, 0x1d, 0x45, 0x94, 0xbb, 0x82, 0x5f, 0x41, 0x7c, 0x2b,
+	0x37, 0x14, 0xeb, 0x19, 0xc6, 0x3e, 0xc3, 0x29, 0x24, 0x3d, 0x03, 0x49, 0xfd, 0xb8, 0x0a, 0x7c,
+	0x57, 0xe7, 0x9f, 0x01, 0x4c, 0xef, 0x70, 0x75, 0x2f, 0xa4, 0x78, 0xc1, 0x0a, 0xa5, 0x7d, 0x74,
+	0x67, 0x62, 0x97, 0x10, 0x52, 0x7a, 0x76, 0x90, 0xd1, 0xb1, 0xb2, 0xe1, 0xa5, 0x0e, 0xd3, 0xdf,
+	0x0f, 0x4e, 0x8e, 0x8f, 0xbe, 0xf1, 0x14, 0xd7, 0xc3, 0x0f, 0x97, 0xe8, 0xe1, 0xd7, 0x36, 0xe3,
+	0xf0, 0x94, 0xc1, 0xc3, 0x0f, 0xf7, 0xe2, 0xe1, 0xd7, 0xe2, 0xf2, 0xd1, 0x62, 0xd2, 0xfe, 0xb3,
+	0x8b, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x33, 0x8d, 0x09, 0xe1, 0x78, 0x02, 0x00, 0x00,
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// KeyManagementServiceClient is the client API for KeyManagementService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type KeyManagementServiceClient interface {
+	// Version returns the runtime name and runtime version of the KMS provider.
+	Version(ctx context.Context, in *VersionRequest, opts ...grpc.CallOption) (*VersionResponse, error)
+	// Execute decryption operation in KMS provider.
+	Decrypt(ctx context.Context, in *DecryptRequest, opts ...grpc.CallOption) (*DecryptResponse, error)
+	// Execute encryption operation in KMS provider.
+	Encrypt(ctx context.Context, in *EncryptRequest, opts ...grpc.CallOption) (*EncryptResponse, error)
+}
+
+type keyManagementServiceClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewKeyManagementServiceClient(cc *grpc.ClientConn) KeyManagementServiceClient {
+	return &keyManagementServiceClient{cc}
+}
+
+func (c *keyManagementServiceClient) Version(ctx context.Context, in *VersionRequest, opts ...grpc.CallOption) (*VersionResponse, error) {
+	out := new(VersionResponse)
+	err := c.cc.Invoke(ctx, "/v1beta1.KeyManagementService/Version", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *keyManagementServiceClient) Decrypt(ctx context.Context, in *DecryptRequest, opts ...grpc.CallOption) (*DecryptResponse, error) {
+	out := new(DecryptResponse)
+	err := c.cc.Invoke(ctx, "/v1beta1.KeyManagementService/Decrypt", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *keyManagementServiceClient) Encrypt(ctx context.Context, in *EncryptRequest, opts ...grpc.CallOption) (*EncryptResponse, error) {
+	out := new(EncryptResponse)
+	err := c.cc.Invoke(ctx, "/v1beta1.KeyManagementService/Encrypt", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// KeyManagementServiceServer is the server API for KeyManagementService service.
+type KeyManagementServiceServer interface {
+	// Version returns the runtime name and runtime version of the KMS provider.
+	Version(context.Context, *VersionRequest) (*VersionResponse, error)
+	// Execute decryption operation in KMS provider.
+	Decrypt(context.Context, *DecryptRequest) (*DecryptResponse, error)
+	// Execute encryption operation in KMS provider.
+	Encrypt(context.Context, *EncryptRequest) (*EncryptResponse, error)
+}
+
+// UnimplementedKeyManagementServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedKeyManagementServiceServer struct {
+}
+
+func (*UnimplementedKeyManagementServiceServer) Version(ctx context.Context, req *VersionRequest) (*VersionResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Version not implemented")
+}
+func (*UnimplementedKeyManagementServiceServer) Decrypt(ctx context.Context, req *DecryptRequest) (*DecryptResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Decrypt not implemented")
+}
+func (*UnimplementedKeyManagementServiceServer) Encrypt(ctx context.Context, req *EncryptRequest) (*EncryptResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Encrypt not implemented")
+}
+
+func RegisterKeyManagementServiceServer(s *grpc.Server, srv KeyManagementServiceServer) {
+	s.RegisterService(&_KeyManagementService_serviceDesc, srv)
+}
+
+func _KeyManagementService_Version_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(VersionRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(KeyManagementServiceServer).Version(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/v1beta1.KeyManagementService/Version",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(KeyManagementServiceServer).Version(ctx, req.(*VersionRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _KeyManagementService_Decrypt_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DecryptRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(KeyManagementServiceServer).Decrypt(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/v1beta1.KeyManagementService/Decrypt",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(KeyManagementServiceServer).Decrypt(ctx, req.(*DecryptRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _KeyManagementService_Encrypt_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(EncryptRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(KeyManagementServiceServer).Encrypt(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/v1beta1.KeyManagementService/Encrypt",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(KeyManagementServiceServer).Encrypt(ctx, req.(*EncryptRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _KeyManagementService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "v1beta1.KeyManagementService",
+	HandlerType: (*KeyManagementServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Version",
+			Handler:    _KeyManagementService_Version_Handler,
+		},
+		{
+			MethodName: "Decrypt",
+			Handler:    _KeyManagementService_Decrypt_Handler,
+		},
+		{
+			MethodName: "Encrypt",
+			Handler:    _KeyManagementService_Encrypt_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "service.proto",
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1/service.proto

@@ -0,0 +1,70 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// To regenerate service.pb.go run hack/update-generated-kms.sh
+syntax = "proto3";
+
+package v1beta1;
+
+// This service defines the public APIs for remote KMS provider.
+service KeyManagementService {
+    // Version returns the runtime name and runtime version of the KMS provider.
+    rpc Version(VersionRequest) returns (VersionResponse) {}
+
+    // Execute decryption operation in KMS provider.
+    rpc Decrypt(DecryptRequest) returns (DecryptResponse) {}
+    // Execute encryption operation in KMS provider.
+    rpc Encrypt(EncryptRequest) returns (EncryptResponse) {}
+}
+
+message VersionRequest {
+    // Version of the KMS plugin API.
+    string version = 1;
+}
+
+message VersionResponse {
+    // Version of the KMS plugin API.
+    string version = 1;
+    // Name of the KMS provider.
+    string runtime_name = 2;
+    // Version of the KMS provider. The string must be semver-compatible.
+    string runtime_version = 3;
+}
+
+message DecryptRequest {
+    // Version of the KMS plugin API.
+    string version = 1;
+    // The data to be decrypted.
+    bytes cipher = 2;
+}
+
+message DecryptResponse {
+    // The decrypted data.
+    bytes plain = 1;
+}
+
+message EncryptRequest {
+    // Version of the KMS plugin API.
+    string version = 1;
+    // The data to be encrypted.
+    bytes plain = 2;
+}
+
+message EncryptResponse {
+    // The encrypted data.
+    bytes cipher = 1;
+}
+

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/identity/identity.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package identity
+
+import (
+	"bytes"
+	"fmt"
+
+	"k8s.io/apiserver/pkg/storage/value"
+)
+
+// identityTransformer performs no transformation on provided data, but validates
+// that the data is not encrypted data during TransformFromStorage
+type identityTransformer struct{}
+
+// NewEncryptCheckTransformer returns an identityTransformer which returns an error
+// on attempts to read encrypted data
+func NewEncryptCheckTransformer() value.Transformer {
+	return identityTransformer{}
+}
+
+// TransformFromStorage returns the input bytes if the data is not encrypted
+func (identityTransformer) TransformFromStorage(b []byte, context value.Context) ([]byte, bool, error) {
+	// identityTransformer has to return an error if the data is encoded using another transformer.
+	// JSON data starts with '{'. Protobuf data has a prefix 'k8s[\x00-\xFF]'.
+	// Prefix 'k8s:enc:' is reserved for encrypted data on disk.
+	if bytes.HasPrefix(b, []byte("k8s:enc:")) {
+		return []byte{}, false, fmt.Errorf("identity transformer tried to read encrypted data")
+	}
+	return b, false, nil
+}
+
+// TransformToStorage implements the Transformer interface for identityTransformer
+func (identityTransformer) TransformToStorage(b []byte, context value.Context) ([]byte, error) {
+	return b, nil
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/secretbox/secretbox.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package secretbox transforms values for storage at rest using XSalsa20 and Poly1305.
+package secretbox
+
+import (
+	"crypto/rand"
+	"fmt"
+
+	"golang.org/x/crypto/nacl/secretbox"
+
+	"k8s.io/apiserver/pkg/storage/value"
+)
+
+// secretbox implements at rest encryption of the provided values given a 32 byte secret key.
+// Uses a standard 24 byte nonce (placed at the beginning of the cipher text) generated
+// from crypto/rand. Does not perform authentication of the data at rest.
+type secretboxTransformer struct {
+	key [32]byte
+}
+
+const nonceSize = 24
+
+// NewSecretboxTransformer takes the given key and performs encryption and decryption on the given
+// data.
+func NewSecretboxTransformer(key [32]byte) value.Transformer {
+	return &secretboxTransformer{key: key}
+}
+
+func (t *secretboxTransformer) TransformFromStorage(data []byte, context value.Context) ([]byte, bool, error) {
+	if len(data) < (secretbox.Overhead + nonceSize) {
+		return nil, false, fmt.Errorf("the stored data was shorter than the required size")
+	}
+	var nonce [nonceSize]byte
+	copy(nonce[:], data[:nonceSize])
+	data = data[nonceSize:]
+	out := make([]byte, 0, len(data)-secretbox.Overhead)
+	result, ok := secretbox.Open(out, data, &nonce, &t.key)
+	if !ok {
+		return nil, false, fmt.Errorf("output array was not large enough for encryption")
+	}
+	return result, false, nil
+}
+
+func (t *secretboxTransformer) TransformToStorage(data []byte, context value.Context) ([]byte, error) {
+	var nonce [nonceSize]byte
+	n, err := rand.Read(nonce[:])
+	if err != nil {
+		return nil, err
+	}
+	if n != nonceSize {
+		return nil, fmt.Errorf("unable to read sufficient random bytes")
+	}
+	return secretbox.Seal(nonce[:], data, &nonce, &t.key), nil
+}

vendor/k8s.io/apiserver/pkg/storage/value/metrics.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2017 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,6 +21,10 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"google.golang.org/grpc/status"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
 )
 
 const (
@@ -28,53 +32,101 @@ const (
 	subsystem = "storage"
 )
 
+/*
+ * By default, all the following metrics are defined as falling under
+ * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
+ *
+ * Promoting the stability level of the metric is a responsibility of the component owner, since it
+ * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
+ * the metric stability policy.
+ */
 var (
-	transformerLatencies = prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
+	transformerLatencies = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "transformation_duration_seconds",
+			Help:      "Latencies in seconds of value transformation operations.",
+			// In-process transformations (ex. AES CBC) complete on the order of 20 microseconds. However, when
+			// external KMS is involved latencies may climb into milliseconds.
+			Buckets:        prometheus.ExponentialBuckets(5e-6, 2, 14),
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"transformation_type"},
+	)
+	deprecatedTransformerLatencies = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
 			Name:      "transformation_latencies_microseconds",
-			Help:      "Latencies in microseconds of value transformation operations.",
+			Help:      "(Deprecated) Latencies in microseconds of value transformation operations.",
 			// In-process transformations (ex. AES CBC) complete on the order of 20 microseconds. However, when
 			// external KMS is involved latencies may climb into milliseconds.
-			Buckets: prometheus.ExponentialBuckets(5, 2, 14),
-		},
-		[]string{"transformation_type"},
-	)
-	transformerFailuresTotal = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "transformation_failures_total",
-			Help:      "Total number of failed transformation operations.",
+			Buckets:        prometheus.ExponentialBuckets(5, 2, 14),
+			StabilityLevel: metrics.ALPHA,
 		},
 		[]string{"transformation_type"},
 	)
 
-	envelopeTransformationCacheMissTotal = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "envelope_transformation_cache_misses_total",
-			Help:      "Total number of cache misses while accessing key decryption key(KEK).",
+	transformerOperationsTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "transformation_operations_total",
+			Help:           "Total number of transformations.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"transformation_type", "transformer_prefix", "status"},
+	)
+
+	deprecatedTransformerFailuresTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "transformation_failures_total",
+			Help:           "(Deprecated) Total number of failed transformation operations.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"transformation_type"},
+	)
+
+	envelopeTransformationCacheMissTotal = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "envelope_transformation_cache_misses_total",
+			Help:           "Total number of cache misses while accessing key decryption key(KEK).",
+			StabilityLevel: metrics.ALPHA,
 		},
 	)
 
-	dataKeyGenerationLatencies = prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "data_key_generation_latencies_microseconds",
-			Help:      "Latencies in microseconds of data encryption key(DEK) generation operations.",
-			Buckets:   prometheus.ExponentialBuckets(5, 2, 14),
+	dataKeyGenerationLatencies = metrics.NewHistogram(
+		&metrics.HistogramOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "data_key_generation_duration_seconds",
+			Help:           "Latencies in seconds of data encryption key(DEK) generation operations.",
+			Buckets:        prometheus.ExponentialBuckets(5e-6, 2, 14),
+			StabilityLevel: metrics.ALPHA,
 		},
 	)
-	dataKeyGenerationFailuresTotal = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "data_key_generation_failures_total",
-			Help:      "Total number of failed data encryption key(DEK) generation operations.",
+	deprecatedDataKeyGenerationLatencies = metrics.NewHistogram(
+		&metrics.HistogramOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "data_key_generation_latencies_microseconds",
+			Help:           "(Deprecated) Latencies in microseconds of data encryption key(DEK) generation operations.",
+			Buckets:        prometheus.ExponentialBuckets(5, 2, 14),
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+	dataKeyGenerationFailuresTotal = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "data_key_generation_failures_total",
+			Help:           "Total number of failed data encryption key(DEK) generation operations.",
+			StabilityLevel: metrics.ALPHA,
 		},
 	)
 )
@@ -83,23 +135,29 @@ var registerMetrics sync.Once
 
 func RegisterMetrics() {
 	registerMetrics.Do(func() {
-		prometheus.MustRegister(transformerLatencies)
-		prometheus.MustRegister(transformerFailuresTotal)
-		prometheus.MustRegister(envelopeTransformationCacheMissTotal)
-		prometheus.MustRegister(dataKeyGenerationLatencies)
-		prometheus.MustRegister(dataKeyGenerationFailuresTotal)
+		legacyregistry.MustRegister(transformerLatencies)
+		legacyregistry.MustRegister(deprecatedTransformerLatencies)
+		legacyregistry.MustRegister(transformerOperationsTotal)
+		legacyregistry.MustRegister(deprecatedTransformerFailuresTotal)
+		legacyregistry.MustRegister(envelopeTransformationCacheMissTotal)
+		legacyregistry.MustRegister(dataKeyGenerationLatencies)
+		legacyregistry.MustRegister(deprecatedDataKeyGenerationLatencies)
+		legacyregistry.MustRegister(dataKeyGenerationFailuresTotal)
 	})
 }
 
 // RecordTransformation records latencies and count of TransformFromStorage and TransformToStorage operations.
-func RecordTransformation(transformationType string, start time.Time, err error) {
-	if err != nil {
-		transformerFailuresTotal.WithLabelValues(transformationType).Inc()
-		return
-	}
+// Note that transformation_failures_total metric is deprecated, use transformation_operations_total instead.
+func RecordTransformation(transformationType, transformerPrefix string, start time.Time, err error) {
+	transformerOperationsTotal.WithLabelValues(transformationType, transformerPrefix, status.Code(err).String()).Inc()
 
-	since := sinceInMicroseconds(start)
-	transformerLatencies.WithLabelValues(transformationType).Observe(float64(since))
+	switch {
+	case err == nil:
+		transformerLatencies.WithLabelValues(transformationType).Observe(sinceInSeconds(start))
+		deprecatedTransformerLatencies.WithLabelValues(transformationType).Observe(sinceInMicroseconds(start))
+	default:
+		deprecatedTransformerFailuresTotal.WithLabelValues(transformationType).Inc()
+	}
 }
 
 // RecordCacheMiss records a miss on Key Encryption Key(KEK) - call to KMS was required to decrypt KEK.
@@ -114,11 +172,16 @@ func RecordDataKeyGeneration(start time.Time, err error) {
 		return
 	}
 
-	since := sinceInMicroseconds(start)
-	dataKeyGenerationLatencies.Observe(float64(since))
+	dataKeyGenerationLatencies.Observe(sinceInSeconds(start))
+	deprecatedDataKeyGenerationLatencies.Observe(sinceInMicroseconds(start))
 }
 
-func sinceInMicroseconds(start time.Time) int64 {
-	elapsedNanoseconds := time.Since(start).Nanoseconds()
-	return elapsedNanoseconds / int64(time.Microsecond)
+// sinceInMicroseconds gets the time since the specified start in microseconds.
+func sinceInMicroseconds(start time.Time) float64 {
+	return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())
+}
+
+// sinceInSeconds gets the time since the specified start in seconds.
+func sinceInSeconds(start time.Time) float64 {
+	return time.Since(start).Seconds()
 }

vendor/k8s.io/apiserver/pkg/storage/value/transformer.go

@@ -22,6 +22,8 @@ import (
 	"fmt"
 	"sync"
 	"time"
+
+	"k8s.io/apimachinery/pkg/util/errors"
 )
 
 func init() {
@@ -85,18 +87,12 @@ func (t *MutableTransformer) Set(transformer Transformer) {
 }
 
 func (t *MutableTransformer) TransformFromStorage(data []byte, context Context) (out []byte, stale bool, err error) {
-	defer func(start time.Time) {
-		RecordTransformation("from_storage", start, err)
-	}(time.Now())
 	t.lock.RLock()
 	transformer := t.transformer
 	t.lock.RUnlock()
 	return transformer.TransformFromStorage(data, context)
 }
 func (t *MutableTransformer) TransformToStorage(data []byte, context Context) (out []byte, err error) {
-	defer func(start time.Time) {
-		RecordTransformation("to_storage", start, err)
-	}(time.Now())
 	t.lock.RLock()
 	transformer := t.transformer
 	t.lock.RUnlock()
@@ -134,28 +130,77 @@ func NewPrefixTransformers(err error, transformers ...PrefixTransformer) Transfo
 // the result of transforming the value. It will always mark any transformation as stale that is not using
 // the first transformer.
 func (t *prefixTransformers) TransformFromStorage(data []byte, context Context) ([]byte, bool, error) {
+	start := time.Now()
+	var errs []error
 	for i, transformer := range t.transformers {
 		if bytes.HasPrefix(data, transformer.Prefix) {
 			result, stale, err := transformer.Transformer.TransformFromStorage(data[len(transformer.Prefix):], context)
 			// To migrate away from encryption, user can specify an identity transformer higher up
 			// (in the config file) than the encryption transformer. In that scenario, the identity transformer needs to
 			// identify (during reads from disk) whether the data being read is encrypted or not. If the data is encrypted,
-			// it shall throw an error, but that error should not prevent subsequent transformers from being tried.
+			// it shall throw an error, but that error should not prevent the next subsequent transformer from being tried.
 			if len(transformer.Prefix) == 0 && err != nil {
 				continue
 			}
+			if len(transformer.Prefix) == 0 {
+				RecordTransformation("from_storage", "identity", start, err)
+			} else {
+				RecordTransformation("from_storage", string(transformer.Prefix), start, err)
+			}
+
+			// It is valid to have overlapping prefixes when the same encryption provider
+			// is specified multiple times but with different keys (the first provider is
+			// being rotated to and some later provider is being rotated away from).
+			//
+			// Example:
+			//
+			//  {
+			//    "aescbc": {
+			//      "keys": [
+			//        {
+			//          "name": "2",
+			//          "secret": "some key 2"
+			//        }
+			//      ]
+			//    }
+			//  },
+			//  {
+			//    "aescbc": {
+			//      "keys": [
+			//        {
+			//          "name": "1",
+			//          "secret": "some key 1"
+			//        }
+			//      ]
+			//    }
+			//  },
+			//
+			// The transformers for both aescbc configs share the prefix k8s:enc:aescbc:v1:
+			// but a failure in the first one should not prevent a later match from being attempted.
+			// Thus we never short-circuit on a prefix match that results in an error.
+			if err != nil {
+				errs = append(errs, err)
+				continue
+			}
+
 			return result, stale || i != 0, err
 		}
 	}
+	if err := errors.Reduce(errors.NewAggregate(errs)); err != nil {
+		return nil, false, err
+	}
+	RecordTransformation("from_storage", "unknown", start, t.err)
 	return nil, false, t.err
 }
 
 // TransformToStorage uses the first transformer and adds its prefix to the data.
 func (t *prefixTransformers) TransformToStorage(data []byte, context Context) ([]byte, error) {
+	start := time.Now()
 	transformer := t.transformers[0]
 	prefixedData := make([]byte, len(transformer.Prefix), len(data)+len(transformer.Prefix))
 	copy(prefixedData, transformer.Prefix)
 	result, err := transformer.Transformer.TransformToStorage(data, context)
+	RecordTransformation("to_storage", string(transformer.Prefix), start, err)
 	if err != nil {
 		return nil, err
 	}