update dependencies

Signed-off-by: hongming <talonwan@yunify.com>
This commit is contained in:
hongming
2020-12-22 16:48:26 +08:00
parent 4a11a50544
commit fe6c5de00f
2857 changed files with 252134 additions and 115656 deletions

View File

@@ -20,7 +20,10 @@ import (
"fmt"
"strings"
"github.com/go-logr/logr"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -35,17 +38,29 @@ import (
var newController = controller.New
var getGvk = apiutil.GVKForObject
// project represents other forms that the we can use to
// send/receive a given resource (metadata-only, unstructured, etc)
type objectProjection int
const (
// projectAsNormal doesn't change the object from the form given
projectAsNormal objectProjection = iota
// projectAsMetadata turns this into an metadata-only watch
projectAsMetadata
)
// Builder builds a Controller.
type Builder struct {
apiType runtime.Object
mgr manager.Manager
predicates []predicate.Predicate
managedObjects []runtime.Object
watchRequest []watchRequest
config *rest.Config
ctrl controller.Controller
ctrlOptions controller.Options
name string
forInput ForInput
ownsInput []OwnsInput
watchesInput []WatchesInput
mgr manager.Manager
globalPredicates []predicate.Predicate
config *rest.Config
ctrl controller.Controller
ctrlOptions controller.Options
log logr.Logger
name string
}
// ControllerManagedBy returns a new controller builder that will be started by the provided Manager
@@ -63,32 +78,65 @@ func (blder *Builder) ForType(apiType runtime.Object) *Builder {
return blder.For(apiType)
}
// ForInput represents the information set by For method.
type ForInput struct {
object runtime.Object
predicates []predicate.Predicate
objectProjection objectProjection
}
// For defines the type of Object being *reconciled*, and configures the ControllerManagedBy to respond to create / delete /
// update events by *reconciling the object*.
// This is the equivalent of calling
// Watches(&source.Kind{Type: apiType}, &handler.EnqueueRequestForObject{})
func (blder *Builder) For(apiType runtime.Object) *Builder {
blder.apiType = apiType
func (blder *Builder) For(object runtime.Object, opts ...ForOption) *Builder {
input := ForInput{object: object}
for _, opt := range opts {
opt.ApplyToFor(&input)
}
blder.forInput = input
return blder
}
// OwnsInput represents the information set by Owns method.
type OwnsInput struct {
object runtime.Object
predicates []predicate.Predicate
objectProjection objectProjection
}
// Owns defines types of Objects being *generated* by the ControllerManagedBy, and configures the ControllerManagedBy to respond to
// create / delete / update events by *reconciling the owner object*. This is the equivalent of calling
// Watches(&handler.EnqueueRequestForOwner{&source.Kind{Type: <ForType-apiType>}, &handler.EnqueueRequestForOwner{OwnerType: apiType, IsController: true})
func (blder *Builder) Owns(apiType runtime.Object) *Builder {
blder.managedObjects = append(blder.managedObjects, apiType)
// Watches(&source.Kind{Type: <ForType-forInput>}, &handler.EnqueueRequestForOwner{OwnerType: apiType, IsController: true})
func (blder *Builder) Owns(object runtime.Object, opts ...OwnsOption) *Builder {
input := OwnsInput{object: object}
for _, opt := range opts {
opt.ApplyToOwns(&input)
}
blder.ownsInput = append(blder.ownsInput, input)
return blder
}
type watchRequest struct {
src source.Source
eventhandler handler.EventHandler
// WatchesInput represents the information set by Watches method.
type WatchesInput struct {
src source.Source
eventhandler handler.EventHandler
predicates []predicate.Predicate
objectProjection objectProjection
}
// Watches exposes the lower-level ControllerManagedBy Watches functions through the builder. Consider using
// Owns or For instead of Watches directly.
func (blder *Builder) Watches(src source.Source, eventhandler handler.EventHandler) *Builder {
blder.watchRequest = append(blder.watchRequest, watchRequest{src: src, eventhandler: eventhandler})
// Specified predicates are registered only for given source.
func (blder *Builder) Watches(src source.Source, eventhandler handler.EventHandler, opts ...WatchesOption) *Builder {
input := WatchesInput{src: src, eventhandler: eventhandler}
for _, opt := range opts {
opt.ApplyToWatches(&input)
}
blder.watchesInput = append(blder.watchesInput, input)
return blder
}
@@ -102,9 +150,10 @@ func (blder *Builder) WithConfig(config *rest.Config) *Builder {
// WithEventFilter sets the event filters, to filter which create/update/delete/generic events eventually
// trigger reconciliations. For example, filtering on whether the resource version has changed.
// Given predicate is added for all watched objects.
// Defaults to the empty list.
func (blder *Builder) WithEventFilter(p predicate.Predicate) *Builder {
blder.predicates = append(blder.predicates, p)
blder.globalPredicates = append(blder.globalPredicates, p)
return blder
}
@@ -124,6 +173,12 @@ func (blder *Builder) Named(name string) *Builder {
return blder
}
// WithLogger overrides the controller options's logger used.
func (blder *Builder) WithLogger(log logr.Logger) *Builder {
blder.log = log
return blder
}
// Complete builds the Application ControllerManagedBy.
func (blder *Builder) Complete(r reconcile.Reconciler) error {
_, err := blder.Build(r)
@@ -155,33 +210,71 @@ func (blder *Builder) Build(r reconcile.Reconciler) (controller.Controller, erro
return blder.ctrl, nil
}
func (blder *Builder) project(obj runtime.Object, proj objectProjection) (runtime.Object, error) {
switch proj {
case projectAsNormal:
return obj, nil
case projectAsMetadata:
metaObj := &metav1.PartialObjectMetadata{}
gvk, err := getGvk(obj, blder.mgr.GetScheme())
if err != nil {
return nil, fmt.Errorf("unable to determine GVK of %T for a metadata-only watch: %w", obj, err)
}
metaObj.SetGroupVersionKind(gvk)
return metaObj, nil
default:
panic(fmt.Sprintf("unexpected projection type %v on type %T, should not be possible since this is an internal field", proj, obj))
}
}
func (blder *Builder) doWatch() error {
// Reconcile type
src := &source.Kind{Type: blder.apiType}
hdler := &handler.EnqueueRequestForObject{}
err := blder.ctrl.Watch(src, hdler, blder.predicates...)
typeForSrc, err := blder.project(blder.forInput.object, blder.forInput.objectProjection)
if err != nil {
return err
}
src := &source.Kind{Type: typeForSrc}
hdler := &handler.EnqueueRequestForObject{}
allPredicates := append(blder.globalPredicates, blder.forInput.predicates...)
if err := blder.ctrl.Watch(src, hdler, allPredicates...); err != nil {
return err
}
// Watches the managed types
for _, obj := range blder.managedObjects {
src := &source.Kind{Type: obj}
for _, own := range blder.ownsInput {
typeForSrc, err := blder.project(own.object, own.objectProjection)
if err != nil {
return err
}
src := &source.Kind{Type: typeForSrc}
hdler := &handler.EnqueueRequestForOwner{
OwnerType: blder.apiType,
OwnerType: blder.forInput.object,
IsController: true,
}
if err := blder.ctrl.Watch(src, hdler, blder.predicates...); err != nil {
allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...)
allPredicates = append(allPredicates, own.predicates...)
if err := blder.ctrl.Watch(src, hdler, allPredicates...); err != nil {
return err
}
}
// Do the watch requests
for _, w := range blder.watchRequest {
if err := blder.ctrl.Watch(w.src, w.eventhandler, blder.predicates...); err != nil {
return err
for _, w := range blder.watchesInput {
allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...)
allPredicates = append(allPredicates, w.predicates...)
// If the source of this watch is of type *source.Kind, project it.
if srckind, ok := w.src.(*source.Kind); ok {
typeForSrc, err := blder.project(srckind.Type, w.objectProjection)
if err != nil {
return err
}
srckind.Type = typeForSrc
}
if err := blder.ctrl.Watch(w.src, w.eventhandler, allPredicates...); err != nil {
return err
}
}
return nil
}
@@ -192,24 +285,33 @@ func (blder *Builder) loadRestConfig() {
}
}
func (blder *Builder) getControllerName() (string, error) {
func (blder *Builder) getControllerName(gvk schema.GroupVersionKind) string {
if blder.name != "" {
return blder.name, nil
return blder.name
}
gvk, err := getGvk(blder.apiType, blder.mgr.GetScheme())
if err != nil {
return "", err
}
return strings.ToLower(gvk.Kind), nil
return strings.ToLower(gvk.Kind)
}
func (blder *Builder) doController(r reconcile.Reconciler) error {
name, err := blder.getControllerName()
ctrlOptions := blder.ctrlOptions
if ctrlOptions.Reconciler == nil {
ctrlOptions.Reconciler = r
}
// Retrieve the GVK from the object we're reconciling
// to prepopulate logger information, and to optionally generate a default name.
gvk, err := getGvk(blder.forInput.object, blder.mgr.GetScheme())
if err != nil {
return err
}
ctrlOptions := blder.ctrlOptions
ctrlOptions.Reconciler = r
blder.ctrl, err = newController(name, blder.mgr, ctrlOptions)
// Setup the logger.
if ctrlOptions.Log == nil {
ctrlOptions.Log = blder.mgr.GetLogger()
}
ctrlOptions.Log = ctrlOptions.Log.WithValues("reconcilerGroup", gvk.Group, "reconcilerKind", gvk.Kind)
// Build the controller and return.
blder.ctrl, err = newController(blder.getControllerName(gvk), blder.mgr, ctrlOptions)
return err
}

View File

@@ -0,0 +1,117 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package builder
import (
"sigs.k8s.io/controller-runtime/pkg/predicate"
)
// {{{ "Functional" Option Interfaces
// ForOption is some configuration that modifies options for a For request.
type ForOption interface {
// ApplyToFor applies this configuration to the given for input.
ApplyToFor(*ForInput)
}
// OwnsOption is some configuration that modifies options for a owns request.
type OwnsOption interface {
// ApplyToOwns applies this configuration to the given owns input.
ApplyToOwns(*OwnsInput)
}
// WatchesOption is some configuration that modifies options for a watches request.
type WatchesOption interface {
// ApplyToWatches applies this configuration to the given watches options.
ApplyToWatches(*WatchesInput)
}
// }}}
// {{{ Multi-Type Options
// WithPredicates sets the given predicates list.
func WithPredicates(predicates ...predicate.Predicate) Predicates {
return Predicates{
predicates: predicates,
}
}
// Predicates filters events before enqueuing the keys.
type Predicates struct {
predicates []predicate.Predicate
}
// ApplyToFor applies this configuration to the given ForInput options.
func (w Predicates) ApplyToFor(opts *ForInput) {
opts.predicates = w.predicates
}
// ApplyToOwns applies this configuration to the given OwnsInput options.
func (w Predicates) ApplyToOwns(opts *OwnsInput) {
opts.predicates = w.predicates
}
// ApplyToWatches applies this configuration to the given WatchesInput options.
func (w Predicates) ApplyToWatches(opts *WatchesInput) {
opts.predicates = w.predicates
}
var _ ForOption = &Predicates{}
var _ OwnsOption = &Predicates{}
var _ WatchesOption = &Predicates{}
// }}}
// {{{ For & Owns Dual-Type options
// asProjection configures the projection (currently only metadata) on the input.
// Currently only metadata is supported. We might want to expand
// this to arbitrary non-special local projections in the future.
type projectAs objectProjection
// ApplyToFor applies this configuration to the given ForInput options.
func (p projectAs) ApplyToFor(opts *ForInput) {
opts.objectProjection = objectProjection(p)
}
// ApplyToOwns applies this configuration to the given OwnsInput options.
func (p projectAs) ApplyToOwns(opts *OwnsInput) {
opts.objectProjection = objectProjection(p)
}
// ApplyToWatches applies this configuration to the given WatchesInput options.
func (p projectAs) ApplyToWatches(opts *WatchesInput) {
opts.objectProjection = objectProjection(p)
}
var (
// OnlyMetadata tells the controller to *only* cache metadata, and to watch
// the the API server in metadata-only form. This is useful when watching
// lots of objects, really big objects, or objects for which you only know
// the the GVK, but not the structure. You'll need to pass
// metav1.PartialObjectMetadata to the client when fetching objects in your
// reconciler, otherwise you'll end up with a duplicate structured or
// unstructured cache.
OnlyMetadata = projectAs(projectAsMetadata)
_ ForOption = OnlyMetadata
_ OwnsOption = OnlyMetadata
_ WatchesOption = OnlyMetadata
)
// }}}

View File

@@ -38,6 +38,7 @@ type WebhookBuilder struct {
config *rest.Config
}
// WebhookManagedBy allows inform its manager.Manager
func WebhookManagedBy(m manager.Manager) *WebhookBuilder {
return &WebhookBuilder{mgr: m}
}

View File

@@ -17,6 +17,7 @@ limitations under the License.
package cache
import (
"context"
"fmt"
"time"
@@ -51,11 +52,11 @@ type Cache interface {
type Informers interface {
// GetInformer fetches or constructs an informer for the given object that corresponds to a single
// API kind and resource.
GetInformer(obj runtime.Object) (Informer, error)
GetInformer(ctx context.Context, obj runtime.Object) (Informer, error)
// GetInformerForKind is similar to GetInformer, except that it takes a group-version-kind, instead
// of the underlying object.
GetInformerForKind(gvk schema.GroupVersionKind) (Informer, error)
GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error)
// Start runs all the informers known to this cache until the given channel is closed.
// It blocks.
@@ -93,7 +94,10 @@ type Options struct {
// Mapper is the RESTMapper to use for mapping GroupVersionKinds to Resources
Mapper meta.RESTMapper
// Resync is the resync period. Defaults to defaultResyncTime.
// Resync is the base frequency the informers are resynced.
// Defaults to defaultResyncTime.
// A 10 percent jitter will be added to the Resync period between informers
// So that all informers will not send list requests simultaneously.
Resync *time.Duration
// Namespace restricts the cache's ListWatch to the desired namespace

View File

@@ -57,7 +57,7 @@ func (ip *informerCache) Get(ctx context.Context, key client.ObjectKey, out runt
return err
}
started, cache, err := ip.InformersMap.Get(gvk, out)
started, cache, err := ip.InformersMap.Get(ctx, gvk, out)
if err != nil {
return err
}
@@ -70,38 +70,13 @@ func (ip *informerCache) Get(ctx context.Context, key client.ObjectKey, out runt
// List implements Reader
func (ip *informerCache) List(ctx context.Context, out runtime.Object, opts ...client.ListOption) error {
gvk, err := apiutil.GVKForObject(out, ip.Scheme)
gvk, cacheTypeObj, err := ip.objectTypeForListObject(out)
if err != nil {
return err
}
if !strings.HasSuffix(gvk.Kind, "List") {
return fmt.Errorf("non-list type %T (kind %q) passed as output", out, gvk)
}
// we need the non-list GVK, so chop off the "List" from the end of the kind
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
_, isUnstructured := out.(*unstructured.UnstructuredList)
var cacheTypeObj runtime.Object
if isUnstructured {
u := &unstructured.Unstructured{}
u.SetGroupVersionKind(gvk)
cacheTypeObj = u
} else {
itemsPtr, err := apimeta.GetItemsPtr(out)
if err != nil {
return nil
}
// http://knowyourmeme.com/memes/this-is-fine
elemType := reflect.Indirect(reflect.ValueOf(itemsPtr)).Type().Elem()
cacheTypeValue := reflect.Zero(reflect.PtrTo(elemType))
var ok bool
cacheTypeObj, ok = cacheTypeValue.Interface().(runtime.Object)
if !ok {
return fmt.Errorf("cannot get cache for %T, its element %T is not a runtime.Object", out, cacheTypeValue.Interface())
}
}
started, cache, err := ip.InformersMap.Get(gvk, cacheTypeObj)
started, cache, err := ip.InformersMap.Get(ctx, *gvk, cacheTypeObj)
if err != nil {
return err
}
@@ -113,14 +88,57 @@ func (ip *informerCache) List(ctx context.Context, out runtime.Object, opts ...c
return cache.Reader.List(ctx, out, opts...)
}
// objectTypeForListObject tries to find the runtime.Object and associated GVK
// for a single object corresponding to the passed-in list type. We need them
// because they are used as cache map key.
func (ip *informerCache) objectTypeForListObject(list runtime.Object) (*schema.GroupVersionKind, runtime.Object, error) {
gvk, err := apiutil.GVKForObject(list, ip.Scheme)
if err != nil {
return nil, nil, err
}
if !strings.HasSuffix(gvk.Kind, "List") {
return nil, nil, fmt.Errorf("non-list type %T (kind %q) passed as output", list, gvk)
}
// we need the non-list GVK, so chop off the "List" from the end of the kind
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
_, isUnstructured := list.(*unstructured.UnstructuredList)
var cacheTypeObj runtime.Object
if isUnstructured {
u := &unstructured.Unstructured{}
u.SetGroupVersionKind(gvk)
cacheTypeObj = u
} else {
itemsPtr, err := apimeta.GetItemsPtr(list)
if err != nil {
return nil, nil, err
}
// http://knowyourmeme.com/memes/this-is-fine
elemType := reflect.Indirect(reflect.ValueOf(itemsPtr)).Type().Elem()
if elemType.Kind() != reflect.Ptr {
elemType = reflect.PtrTo(elemType)
}
cacheTypeValue := reflect.Zero(elemType)
var ok bool
cacheTypeObj, ok = cacheTypeValue.Interface().(runtime.Object)
if !ok {
return nil, nil, fmt.Errorf("cannot get cache for %T, its element %T is not a runtime.Object", list, cacheTypeValue.Interface())
}
}
return &gvk, cacheTypeObj, nil
}
// GetInformerForKind returns the informer for the GroupVersionKind
func (ip *informerCache) GetInformerForKind(gvk schema.GroupVersionKind) (Informer, error) {
func (ip *informerCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) {
// Map the gvk to an object
obj, err := ip.Scheme.New(gvk)
if err != nil {
return nil, err
}
_, i, err := ip.InformersMap.Get(gvk, obj)
_, i, err := ip.InformersMap.Get(ctx, gvk, obj)
if err != nil {
return nil, err
}
@@ -128,12 +146,13 @@ func (ip *informerCache) GetInformerForKind(gvk schema.GroupVersionKind) (Inform
}
// GetInformer returns the informer for the obj
func (ip *informerCache) GetInformer(obj runtime.Object) (Informer, error) {
func (ip *informerCache) GetInformer(ctx context.Context, obj runtime.Object) (Informer, error) {
gvk, err := apiutil.GVKForObject(obj, ip.Scheme)
if err != nil {
return nil, err
}
_, i, err := ip.InformersMap.Get(gvk, obj)
_, i, err := ip.InformersMap.Get(ctx, gvk, obj)
if err != nil {
return nil, err
}
@@ -151,8 +170,8 @@ func (ip *informerCache) NeedLeaderElection() bool {
// to List. For one-to-one compatibility with "normal" field selectors, only return one value.
// The values may be anything. They will automatically be prefixed with the namespace of the
// given object, if present. The objects passed are guaranteed to be objects of the correct type.
func (ip *informerCache) IndexField(obj runtime.Object, field string, extractValue client.IndexerFunc) error {
informer, err := ip.GetInformer(obj)
func (ip *informerCache) IndexField(ctx context.Context, obj runtime.Object, field string, extractValue client.IndexerFunc) error {
informer, err := ip.GetInformer(ctx, obj)
if err != nil {
return err
}

View File

@@ -17,9 +17,11 @@ limitations under the License.
package internal
import (
"context"
"time"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -30,10 +32,12 @@ import (
// InformersMap create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs.
// It uses a standard parameter codec constructed based on the given generated Scheme.
type InformersMap struct {
// we abstract over the details of structured vs unstructured with the specificInformerMaps
// we abstract over the details of structured/unstructured/metadata with the specificInformerMaps
// TODO(directxman12): genericize this over different projections now that we have 3 different maps
structured *specificInformersMap
unstructured *specificInformersMap
metadata *specificInformersMap
// Scheme maps runtime.Objects to GroupVersionKinds
Scheme *runtime.Scheme
@@ -50,6 +54,7 @@ func NewInformersMap(config *rest.Config,
return &InformersMap{
structured: newStructuredInformersMap(config, scheme, mapper, resync, namespace),
unstructured: newUnstructuredInformersMap(config, scheme, mapper, resync, namespace),
metadata: newMetadataInformersMap(config, scheme, mapper, resync, namespace),
Scheme: scheme,
}
@@ -59,30 +64,43 @@ func NewInformersMap(config *rest.Config,
func (m *InformersMap) Start(stop <-chan struct{}) error {
go m.structured.Start(stop)
go m.unstructured.Start(stop)
go m.metadata.Start(stop)
<-stop
return nil
}
// WaitForCacheSync waits until all the caches have been synced.
// WaitForCacheSync waits until all the caches have been started and synced.
func (m *InformersMap) WaitForCacheSync(stop <-chan struct{}) bool {
syncedFuncs := append([]cache.InformerSynced(nil), m.structured.HasSyncedFuncs()...)
syncedFuncs = append(syncedFuncs, m.unstructured.HasSyncedFuncs()...)
if !m.structured.waitForStarted(stop) {
return false
}
if !m.unstructured.waitForStarted(stop) {
return false
}
if !m.metadata.waitForStarted(stop) {
return false
}
return cache.WaitForCacheSync(stop, syncedFuncs...)
}
// Get will create a new Informer and add it to the map of InformersMap if none exists. Returns
// the Informer from the map.
func (m *InformersMap) Get(gvk schema.GroupVersionKind, obj runtime.Object) (bool, *MapEntry, error) {
_, isUnstructured := obj.(*unstructured.Unstructured)
_, isUnstructuredList := obj.(*unstructured.UnstructuredList)
isUnstructured = isUnstructured || isUnstructuredList
if isUnstructured {
return m.unstructured.Get(gvk, obj)
func (m *InformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *MapEntry, error) {
switch obj.(type) {
case *unstructured.Unstructured:
return m.unstructured.Get(ctx, gvk, obj)
case *unstructured.UnstructuredList:
return m.unstructured.Get(ctx, gvk, obj)
case *metav1.PartialObjectMetadata:
return m.metadata.Get(ctx, gvk, obj)
case *metav1.PartialObjectMetadataList:
return m.metadata.Get(ctx, gvk, obj)
default:
return m.structured.Get(ctx, gvk, obj)
}
return m.structured.Get(gvk, obj)
}
// newStructuredInformersMap creates a new InformersMap for structured objects.
@@ -94,3 +112,8 @@ func newStructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapp
func newUnstructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, namespace string) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, createUnstructuredListWatch)
}
// newMetadataInformersMap creates a new InformersMap for metadata-only objects.
func newMetadataInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, namespace string) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, createMetadataListWatch)
}

View File

@@ -17,10 +17,13 @@ limitations under the License.
package internal
import (
"context"
"fmt"
"math/rand"
"sync"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -28,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/metadata"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
@@ -53,6 +57,7 @@ func newSpecificInformersMap(config *rest.Config,
codecs: serializer.NewCodecFactory(scheme),
paramCodec: runtime.NewParameterCodec(scheme),
resync: resync,
startWait: make(chan struct{}),
createListWatcher: createListWatcher,
namespace: namespace,
}
@@ -92,7 +97,9 @@ type specificInformersMap struct {
// stop is the stop channel to stop informers
stop <-chan struct{}
// resync is the frequency the informers are resynced
// resync is the base frequency the informers are resynced
// a 10 percent jitter will be added to the resync period between informers
// so that all informers will not send list requests simultaneously.
resync time.Duration
// mu guards access to the map
@@ -101,6 +108,10 @@ type specificInformersMap struct {
// start is true if the informers have been started
started bool
// startWait is a channel that is closed after the
// informer has been started.
startWait chan struct{}
// createClient knows how to create a client and a list object,
// and allows for abstracting over the particulars of structured vs
// unstructured objects.
@@ -128,10 +139,20 @@ func (ip *specificInformersMap) Start(stop <-chan struct{}) {
// Set started to true so we immediately start any informers added later.
ip.started = true
close(ip.startWait)
}()
<-stop
}
func (ip *specificInformersMap) waitForStarted(stop <-chan struct{}) bool {
select {
case <-ip.startWait:
return true
case <-stop:
return false
}
}
// HasSyncedFuncs returns all the HasSynced functions for the informers in this map.
func (ip *specificInformersMap) HasSyncedFuncs() []cache.InformerSynced {
ip.mu.RLock()
@@ -145,7 +166,7 @@ func (ip *specificInformersMap) HasSyncedFuncs() []cache.InformerSynced {
// Get will create a new Informer and add it to the map of specificInformersMap if none exists. Returns
// the Informer from the map.
func (ip *specificInformersMap) Get(gvk schema.GroupVersionKind, obj runtime.Object) (bool, *MapEntry, error) {
func (ip *specificInformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *MapEntry, error) {
// Return the informer if it is found
i, started, ok := func() (*MapEntry, bool, bool) {
ip.mu.RLock()
@@ -163,8 +184,8 @@ func (ip *specificInformersMap) Get(gvk schema.GroupVersionKind, obj runtime.Obj
if started && !i.Informer.HasSynced() {
// Wait for it to sync before returning the Informer so that folks don't read from a stale cache.
if !cache.WaitForCacheSync(ip.stop, i.Informer.HasSynced) {
return started, nil, fmt.Errorf("failed waiting for %T Informer to sync", obj)
if !cache.WaitForCacheSync(ctx.Done(), i.Informer.HasSynced) {
return started, nil, apierrors.NewTimeoutError(fmt.Sprintf("failed waiting for %T Informer to sync", obj), 0)
}
}
@@ -188,7 +209,7 @@ func (ip *specificInformersMap) addInformerToMap(gvk schema.GroupVersionKind, ob
if err != nil {
return nil, false, err
}
ni := cache.NewSharedIndexInformer(lw, obj, ip.resync, cache.Indexers{
ni := cache.NewSharedIndexInformer(lw, obj, resyncPeriod(ip.resync)(), cache.Indexers{
cache.NamespaceIndex: cache.MetaNamespaceIndexFunc,
})
i := &MapEntry{
@@ -225,12 +246,15 @@ func createStructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformer
return nil, err
}
// TODO: the functions that make use of this ListWatch should be adapted to
// pass in their own contexts instead of relying on this fixed one here.
ctx := context.TODO()
// Create a new ListWatch for the obj
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
res := listObj.DeepCopyObject()
isNamespaceScoped := ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot
err := client.Get().NamespaceIfScoped(ip.namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Do().Into(res)
err := client.Get().NamespaceIfScoped(ip.namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Do(ctx).Into(res)
return res, err
},
// Setup the watch function
@@ -238,7 +262,7 @@ func createStructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformer
// Watch needs to be set to true separately
opts.Watch = true
isNamespaceScoped := ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot
return client.Get().NamespaceIfScoped(ip.namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Watch()
return client.Get().NamespaceIfScoped(ip.namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Watch(ctx)
},
}, nil
}
@@ -255,22 +279,74 @@ func createUnstructuredListWatch(gvk schema.GroupVersionKind, ip *specificInform
return nil, err
}
// TODO: the functions that make use of this ListWatch should be adapted to
// pass in their own contexts instead of relying on this fixed one here.
ctx := context.TODO()
// Create a new ListWatch for the obj
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
if ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
return dynamicClient.Resource(mapping.Resource).Namespace(ip.namespace).List(opts)
return dynamicClient.Resource(mapping.Resource).Namespace(ip.namespace).List(ctx, opts)
}
return dynamicClient.Resource(mapping.Resource).List(opts)
return dynamicClient.Resource(mapping.Resource).List(ctx, opts)
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
// Watch needs to be set to true separately
opts.Watch = true
if ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
return dynamicClient.Resource(mapping.Resource).Namespace(ip.namespace).Watch(opts)
return dynamicClient.Resource(mapping.Resource).Namespace(ip.namespace).Watch(ctx, opts)
}
return dynamicClient.Resource(mapping.Resource).Watch(opts)
return dynamicClient.Resource(mapping.Resource).Watch(ctx, opts)
},
}, nil
}
func createMetadataListWatch(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) {
// Kubernetes APIs work against Resources, not GroupVersionKinds. Map the
// groupVersionKind to the Resource API we will use.
mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
return nil, err
}
// grab the metadata client
client, err := metadata.NewForConfig(ip.config)
if err != nil {
return nil, err
}
// TODO: the functions that make use of this ListWatch should be adapted to
// pass in their own contexts instead of relying on this fixed one here.
ctx := context.TODO()
// create the relevant listwaatch
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
if ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
return client.Resource(mapping.Resource).Namespace(ip.namespace).List(ctx, opts)
}
return client.Resource(mapping.Resource).List(ctx, opts)
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
// Watch needs to be set to true separately
opts.Watch = true
if ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
return client.Resource(mapping.Resource).Namespace(ip.namespace).Watch(ctx, opts)
}
return client.Resource(mapping.Resource).Watch(ctx, opts)
},
}, nil
}
// resyncPeriod returns a function which generates a duration each time it is
// invoked; this is so that multiple controllers don't get into lock-step and all
// hammer the apiserver with list requests simultaneously.
func resyncPeriod(resync time.Duration) func() time.Duration {
return func() time.Duration {
// the factor will fall into [0.9, 1.1)
factor := rand.Float64()/5.0 + 0.9
return time.Duration(float64(resync.Nanoseconds()) * factor)
}
}

View File

@@ -36,7 +36,9 @@ type NewCacheFunc func(config *rest.Config, opts Options) (Cache, error)
// MultiNamespacedCacheBuilder - Builder function to create a new multi-namespaced cache.
// This will scope the cache to a list of namespaces. Listing for all namespaces
// will list for all the namespaces that this knows about.
// will list for all the namespaces that this knows about. Note that this is not intended
// to be used for excluding namespaces, this is better done via a Predicate. Also note that
// you may face performance issues when using this with a high number of namespaces.
func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
return func(config *rest.Config, opts Options) (Cache, error) {
opts, err := defaultOpts(config, opts)
@@ -68,10 +70,10 @@ type multiNamespaceCache struct {
var _ Cache = &multiNamespaceCache{}
// Methods for multiNamespaceCache to conform to the Informers interface
func (c *multiNamespaceCache) GetInformer(obj runtime.Object) (Informer, error) {
func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj runtime.Object) (Informer, error) {
informers := map[string]Informer{}
for ns, cache := range c.namespaceToCache {
informer, err := cache.GetInformer(obj)
informer, err := cache.GetInformer(ctx, obj)
if err != nil {
return nil, err
}
@@ -80,10 +82,10 @@ func (c *multiNamespaceCache) GetInformer(obj runtime.Object) (Informer, error)
return &multiNamespaceInformer{namespaceToInformer: informers}, nil
}
func (c *multiNamespaceCache) GetInformerForKind(gvk schema.GroupVersionKind) (Informer, error) {
func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) {
informers := map[string]Informer{}
for ns, cache := range c.namespaceToCache {
informer, err := cache.GetInformerForKind(gvk)
informer, err := cache.GetInformerForKind(ctx, gvk)
if err != nil {
return nil, err
}
@@ -115,9 +117,9 @@ func (c *multiNamespaceCache) WaitForCacheSync(stop <-chan struct{}) bool {
return synced
}
func (c *multiNamespaceCache) IndexField(obj runtime.Object, field string, extractValue client.IndexerFunc) error {
func (c *multiNamespaceCache) IndexField(ctx context.Context, obj runtime.Object, field string, extractValue client.IndexerFunc) error {
for _, cache := range c.namespaceToCache {
if err := cache.IndexField(obj, field, extractValue); err != nil {
if err := cache.IndexField(ctx, obj, field, extractValue); err != nil {
return err
}
}

View File

@@ -23,6 +23,7 @@ import (
"fmt"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -48,12 +49,33 @@ func NewDiscoveryRESTMapper(c *rest.Config) (meta.RESTMapper, error) {
// GVKForObject finds the GroupVersionKind associated with the given object, if there is only a single such GVK.
func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersionKind, error) {
// TODO(directxman12): do we want to generalize this to arbitrary container types?
// I think we'd need a generalized form of scheme or something. It's a
// shame there's not a reliable "GetGVK" interface that works by default
// for unpopulated static types and populated "dynamic" types
// (unstructured, partial, etc)
// check for PartialObjectMetadata, which is analogous to unstructured, but isn't handled by ObjectKinds
_, isPartial := obj.(*metav1.PartialObjectMetadata)
_, isPartialList := obj.(*metav1.PartialObjectMetadataList)
if isPartial || isPartialList {
// we require that the GVK be populated in order to recognize the object
gvk := obj.GetObjectKind().GroupVersionKind()
if len(gvk.Kind) == 0 {
return schema.GroupVersionKind{}, runtime.NewMissingKindErr("unstructured object has no kind")
}
if len(gvk.Version) == 0 {
return schema.GroupVersionKind{}, runtime.NewMissingVersionErr("unstructured object has no version")
}
return gvk, nil
}
gvks, isUnversioned, err := scheme.ObjectKinds(obj)
if err != nil {
return schema.GroupVersionKind{}, err
}
if isUnversioned {
return schema.GroupVersionKind{}, fmt.Errorf("cannot create a new informer for the unversioned type %T", obj)
return schema.GroupVersionKind{}, fmt.Errorf("cannot create group-version-kind for unversioned type %T", obj)
}
if len(gvks) < 1 {

View File

@@ -17,11 +17,11 @@ limitations under the License.
package apiutil
import (
"errors"
"sync"
"time"
"golang.org/x/time/rate"
"golang.org/x/xerrors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/discovery"
@@ -45,7 +45,7 @@ func (e ErrRateLimited) Error() string {
// time.Duration value and false are returned if err is not a ErrRateLimited.
func DelayIfRateLimited(err error) (time.Duration, bool) {
var rlerr ErrRateLimited
if xerrors.As(err, &rlerr) {
if errors.As(err, &rlerr) {
return rlerr.Delay, true
}
return 0, false
@@ -182,7 +182,7 @@ func (drm *dynamicRESTMapper) checkAndReload(needsReloadErr error, checkNeedsRel
// NB(directxman12): `Is` and `As` have a confusing relationship --
// `Is` is like `== or does this implement .Is`, whereas `As` says
// `can I type-assert into`
needsReload := xerrors.As(err, &needsReloadErr)
needsReload := errors.As(err, &needsReloadErr)
if !needsReload {
return err
}
@@ -193,7 +193,7 @@ func (drm *dynamicRESTMapper) checkAndReload(needsReloadErr error, checkNeedsRel
// ... and double-check that we didn't reload in the meantime
err = checkNeedsReload()
needsReload = xerrors.As(err, &needsReloadErr)
needsReload = errors.As(err, &needsReloadErr)
if !needsReload {
return err
}
@@ -318,5 +318,6 @@ func (b *dynamicLimiter) checkRate() error {
if res.Delay() == 0 {
return nil
}
res.Cancel()
return ErrRateLimited{res.Delay()}
}

View File

@@ -19,15 +19,15 @@ package client
import (
"context"
"fmt"
"reflect"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/metadata"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
@@ -64,30 +64,36 @@ func New(config *rest.Config, options Options) (Client, error) {
// Init a Mapper if none provided
if options.Mapper == nil {
var err error
options.Mapper, err = apiutil.NewDiscoveryRESTMapper(config)
options.Mapper, err = apiutil.NewDynamicRESTMapper(config)
if err != nil {
return nil, err
}
}
dynamicClient, err := dynamic.NewForConfig(config)
clientcache := &clientCache{
config: config,
scheme: options.Scheme,
mapper: options.Mapper,
codecs: serializer.NewCodecFactory(options.Scheme),
resourceByType: make(map[schema.GroupVersionKind]*resourceMeta),
}
rawMetaClient, err := metadata.NewForConfig(config)
if err != nil {
return nil, err
return nil, fmt.Errorf("unable to construct metadata-only client for use as part of client: %w", err)
}
c := &client{
typedClient: typedClient{
cache: clientCache{
config: config,
scheme: options.Scheme,
mapper: options.Mapper,
codecs: serializer.NewCodecFactory(options.Scheme),
resourceByType: make(map[reflect.Type]*resourceMeta),
},
cache: clientcache,
paramCodec: runtime.NewParameterCodec(options.Scheme),
},
unstructuredClient: unstructuredClient{
client: dynamicClient,
cache: clientcache,
paramCodec: noConversionParamCodec{},
},
metadataClient: metadataClient{
client: rawMetaClient,
restMapper: options.Mapper,
},
}
@@ -102,6 +108,7 @@ var _ Client = &client{}
type client struct {
typedClient typedClient
unstructuredClient unstructuredClient
metadataClient metadataClient
}
// resetGroupVersionKind is a helper function to restore and preserve GroupVersionKind on an object.
@@ -116,67 +123,88 @@ func (c *client) resetGroupVersionKind(obj runtime.Object, gvk schema.GroupVersi
// Create implements client.Client
func (c *client) Create(ctx context.Context, obj runtime.Object, opts ...CreateOption) error {
_, ok := obj.(*unstructured.Unstructured)
if ok {
switch obj.(type) {
case *unstructured.Unstructured:
return c.unstructuredClient.Create(ctx, obj, opts...)
case *metav1.PartialObjectMetadata:
return fmt.Errorf("cannot create using only metadata")
default:
return c.typedClient.Create(ctx, obj, opts...)
}
return c.typedClient.Create(ctx, obj, opts...)
}
// Update implements client.Client
func (c *client) Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
_, ok := obj.(*unstructured.Unstructured)
if ok {
switch obj.(type) {
case *unstructured.Unstructured:
return c.unstructuredClient.Update(ctx, obj, opts...)
case *metav1.PartialObjectMetadata:
return fmt.Errorf("cannot update using only metadata -- did you mean to patch?")
default:
return c.typedClient.Update(ctx, obj, opts...)
}
return c.typedClient.Update(ctx, obj, opts...)
}
// Delete implements client.Client
func (c *client) Delete(ctx context.Context, obj runtime.Object, opts ...DeleteOption) error {
_, ok := obj.(*unstructured.Unstructured)
if ok {
switch obj.(type) {
case *unstructured.Unstructured:
return c.unstructuredClient.Delete(ctx, obj, opts...)
case *metav1.PartialObjectMetadata:
return c.metadataClient.Delete(ctx, obj, opts...)
default:
return c.typedClient.Delete(ctx, obj, opts...)
}
return c.typedClient.Delete(ctx, obj, opts...)
}
// DeleteAllOf implements client.Client
func (c *client) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...DeleteAllOfOption) error {
_, ok := obj.(*unstructured.Unstructured)
if ok {
switch obj.(type) {
case *unstructured.Unstructured:
return c.unstructuredClient.DeleteAllOf(ctx, obj, opts...)
case *metav1.PartialObjectMetadata:
return c.metadataClient.DeleteAllOf(ctx, obj, opts...)
default:
return c.typedClient.DeleteAllOf(ctx, obj, opts...)
}
return c.typedClient.DeleteAllOf(ctx, obj, opts...)
}
// Patch implements client.Client
func (c *client) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
_, ok := obj.(*unstructured.Unstructured)
if ok {
switch obj.(type) {
case *unstructured.Unstructured:
return c.unstructuredClient.Patch(ctx, obj, patch, opts...)
case *metav1.PartialObjectMetadata:
return c.metadataClient.Patch(ctx, obj, patch, opts...)
default:
return c.typedClient.Patch(ctx, obj, patch, opts...)
}
return c.typedClient.Patch(ctx, obj, patch, opts...)
}
// Get implements client.Client
func (c *client) Get(ctx context.Context, key ObjectKey, obj runtime.Object) error {
_, ok := obj.(*unstructured.Unstructured)
if ok {
switch obj.(type) {
case *unstructured.Unstructured:
return c.unstructuredClient.Get(ctx, key, obj)
case *metav1.PartialObjectMetadata:
return c.metadataClient.Get(ctx, key, obj)
default:
return c.typedClient.Get(ctx, key, obj)
}
return c.typedClient.Get(ctx, key, obj)
}
// List implements client.Client
func (c *client) List(ctx context.Context, obj runtime.Object, opts ...ListOption) error {
_, ok := obj.(*unstructured.UnstructuredList)
if ok {
switch obj.(type) {
case *unstructured.Unstructured:
return c.unstructuredClient.List(ctx, obj, opts...)
case *metav1.PartialObjectMetadataList:
return c.metadataClient.List(ctx, obj, opts...)
default:
return c.typedClient.List(ctx, obj, opts...)
}
return c.typedClient.List(ctx, obj, opts...)
}
// Status implements client.StatusClient
@@ -195,19 +223,25 @@ var _ StatusWriter = &statusWriter{}
// Update implements client.StatusWriter
func (sw *statusWriter) Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
defer sw.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
_, ok := obj.(*unstructured.Unstructured)
if ok {
switch obj.(type) {
case *unstructured.Unstructured:
return sw.client.unstructuredClient.UpdateStatus(ctx, obj, opts...)
case *metav1.PartialObjectMetadata:
return fmt.Errorf("cannot update status using only metadata -- did you mean to patch?")
default:
return sw.client.typedClient.UpdateStatus(ctx, obj, opts...)
}
return sw.client.typedClient.UpdateStatus(ctx, obj, opts...)
}
// Patch implements client.Client
func (sw *statusWriter) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
defer sw.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
_, ok := obj.(*unstructured.Unstructured)
if ok {
switch obj.(type) {
case *unstructured.Unstructured:
return sw.client.unstructuredClient.PatchStatus(ctx, obj, patch, opts...)
case *metav1.PartialObjectMetadata:
return sw.client.metadataClient.PatchStatus(ctx, obj, patch, opts...)
default:
return sw.client.typedClient.PatchStatus(ctx, obj, patch, opts...)
}
return sw.client.typedClient.PatchStatus(ctx, obj, patch, opts...)
}

View File

@@ -17,7 +17,6 @@ limitations under the License.
package client
import (
"reflect"
"strings"
"sync"
@@ -45,19 +44,14 @@ type clientCache struct {
codecs serializer.CodecFactory
// resourceByType caches type metadata
resourceByType map[reflect.Type]*resourceMeta
resourceByType map[schema.GroupVersionKind]*resourceMeta
mu sync.RWMutex
}
// newResource maps obj to a Kubernetes Resource and constructs a client for that Resource.
// If the object is a list, the resource represents the item's type instead.
func (c *clientCache) newResource(obj runtime.Object) (*resourceMeta, error) {
gvk, err := apiutil.GVKForObject(obj, c.scheme)
if err != nil {
return nil, err
}
if strings.HasSuffix(gvk.Kind, "List") && meta.IsListType(obj) {
func (c *clientCache) newResource(gvk schema.GroupVersionKind, isList bool) (*resourceMeta, error) {
if strings.HasSuffix(gvk.Kind, "List") && isList {
// if this was a list, treat it as a request for the item's resource
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
@@ -76,12 +70,15 @@ func (c *clientCache) newResource(obj runtime.Object) (*resourceMeta, error) {
// getResource returns the resource meta information for the given type of object.
// If the object is a list, the resource represents the item's type instead.
func (c *clientCache) getResource(obj runtime.Object) (*resourceMeta, error) {
typ := reflect.TypeOf(obj)
gvk, err := apiutil.GVKForObject(obj, c.scheme)
if err != nil {
return nil, err
}
// It's better to do creation work twice than to not let multiple
// people make requests at once
c.mu.RLock()
r, known := c.resourceByType[typ]
r, known := c.resourceByType[gvk]
c.mu.RUnlock()
if known {
@@ -91,11 +88,11 @@ func (c *clientCache) getResource(obj runtime.Object) (*resourceMeta, error) {
// Initialize a new Client
c.mu.Lock()
defer c.mu.Unlock()
r, err := c.newResource(obj)
r, err = c.newResource(gvk, meta.IsListType(obj))
if err != nil {
return nil, err
}
c.resourceByType[typ] = r
c.resourceByType[gvk] = r
return r, err
}
@@ -124,10 +121,8 @@ type resourceMeta struct {
// isNamespaced returns true if the type is namespaced
func (r *resourceMeta) isNamespaced() bool {
if r.mapping.Scope.Name() == meta.RESTScopeNameRoot {
return false
}
return true
return r.mapping.Scope.Name() != meta.RESTScopeNameRoot
}
// resource returns the resource name of the type

View File

@@ -0,0 +1,24 @@
package client
import (
"errors"
"net/url"
"k8s.io/apimachinery/pkg/conversion/queryparams"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
)
var _ runtime.ParameterCodec = noConversionParamCodec{}
// noConversionParamCodec is a no-conversion codec for serializing parameters into URL query strings.
// it's useful in scenarios with the unstructured client and arbitrary resouces.
type noConversionParamCodec struct{}
func (noConversionParamCodec) EncodeParameters(obj runtime.Object, to schema.GroupVersion) (url.Values, error) {
return queryparams.Convert(obj)
}
func (noConversionParamCodec) DecodeParameters(parameters url.Values, from schema.GroupVersion, into runtime.Object) error {
return errors.New("DecodeParameters not implemented on noConversionParamCodec")
}

View File

@@ -20,6 +20,8 @@ import (
"flag"
"fmt"
"os"
"os/user"
"path"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
@@ -117,11 +119,22 @@ func loadConfig(context string) (*rest.Config, error) {
// If the recommended kubeconfig env variable is set, or there
// is no in-cluster config, try the default recommended locations.
if c, err := loadConfigWithContext(apiServerURL, clientcmd.NewDefaultClientConfigLoadingRules(), context); err == nil {
return c, nil
//
// NOTE: For default config file locations, upstream only checks
// $HOME for the user's home directory, but we can also try
// os/user.HomeDir when $HOME is unset.
//
// TODO(jlanford): could this be done upstream?
loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
if _, ok := os.LookupEnv("HOME"); !ok {
u, err := user.Current()
if err != nil {
return nil, fmt.Errorf("could not get current user: %v", err)
}
loadingRules.Precedence = append(loadingRules.Precedence, path.Join(u.HomeDir, clientcmd.RecommendedHomeDir, clientcmd.RecommendedFileName))
}
return nil, fmt.Errorf("could not locate a kubeconfig")
return loadConfigWithContext(apiServerURL, loadingRules, context)
}
func loadConfigWithContext(apiServerURL string, loader clientcmd.ClientConfigLoader, context string) (*rest.Config, error) {

View File

@@ -0,0 +1,95 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"context"
"k8s.io/apimachinery/pkg/runtime"
)
// NewDryRunClient wraps an existing client and enforces DryRun mode
// on all mutating api calls.
func NewDryRunClient(c Client) Client {
return &dryRunClient{client: c}
}
var _ Client = &dryRunClient{}
// dryRunClient is a Client that wraps another Client in order to enforce DryRun mode.
type dryRunClient struct {
client Client
}
// Create implements client.Client
func (c *dryRunClient) Create(ctx context.Context, obj runtime.Object, opts ...CreateOption) error {
return c.client.Create(ctx, obj, append(opts, DryRunAll)...)
}
// Update implements client.Client
func (c *dryRunClient) Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
return c.client.Update(ctx, obj, append(opts, DryRunAll)...)
}
// Delete implements client.Client
func (c *dryRunClient) Delete(ctx context.Context, obj runtime.Object, opts ...DeleteOption) error {
return c.client.Delete(ctx, obj, append(opts, DryRunAll)...)
}
// DeleteAllOf implements client.Client
func (c *dryRunClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...DeleteAllOfOption) error {
return c.client.DeleteAllOf(ctx, obj, append(opts, DryRunAll)...)
}
// Patch implements client.Client
func (c *dryRunClient) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
return c.client.Patch(ctx, obj, patch, append(opts, DryRunAll)...)
}
// Get implements client.Client
func (c *dryRunClient) Get(ctx context.Context, key ObjectKey, obj runtime.Object) error {
return c.client.Get(ctx, key, obj)
}
// List implements client.Client
func (c *dryRunClient) List(ctx context.Context, obj runtime.Object, opts ...ListOption) error {
return c.client.List(ctx, obj, opts...)
}
// Status implements client.StatusClient
func (c *dryRunClient) Status() StatusWriter {
return &dryRunStatusWriter{client: c.client.Status()}
}
// ensure dryRunStatusWriter implements client.StatusWriter
var _ StatusWriter = &dryRunStatusWriter{}
// dryRunStatusWriter is client.StatusWriter that writes status subresource with dryRun mode
// enforced.
type dryRunStatusWriter struct {
client StatusWriter
}
// Update implements client.StatusWriter
func (sw *dryRunStatusWriter) Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
return sw.client.Update(ctx, obj, append(opts, DryRunAll)...)
}
// Patch implements client.StatusWriter
func (sw *dryRunStatusWriter) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
return sw.client.Patch(ctx, obj, patch, append(opts, DryRunAll)...)
}

View File

@@ -122,7 +122,7 @@ type FieldIndexer interface {
// and "equality" in the field selector means that at least one key matches the value.
// The FieldIndexer will automatically take care of indexing over namespace
// and supporting efficient all-namespace queries.
IndexField(obj runtime.Object, field string, extractValue IndexerFunc) error
IndexField(ctx context.Context, obj runtime.Object, field string, extractValue IndexerFunc) error
}
// IgnoreNotFound returns nil on NotFound errors.

View File

@@ -0,0 +1,194 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"context"
"fmt"
"strings"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/metadata"
)
// TODO(directxman12): we could rewrite this on top of the low-level REST
// client to avoid the extra shallow copy at the end, but I'm not sure it's
// worth it -- the metadata client deals with falling back to loading the whole
// object on older API servers, etc, and we'd have to reproduce that.
// metadataClient is a client that reads & writes metadata-only requests to/from the API server.
type metadataClient struct {
client metadata.Interface
restMapper meta.RESTMapper
}
func (mc *metadataClient) getResourceInterface(gvk schema.GroupVersionKind, ns string) (metadata.ResourceInterface, error) {
mapping, err := mc.restMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
return nil, err
}
if mapping.Scope.Name() == meta.RESTScopeNameRoot {
return mc.client.Resource(mapping.Resource), nil
}
return mc.client.Resource(mapping.Resource).Namespace(ns), nil
}
// Delete implements client.Client
func (mc *metadataClient) Delete(ctx context.Context, obj runtime.Object, opts ...DeleteOption) error {
metadata, ok := obj.(*metav1.PartialObjectMetadata)
if !ok {
return fmt.Errorf("metadata client did not understand object: %T", obj)
}
resInt, err := mc.getResourceInterface(metadata.GroupVersionKind(), metadata.Namespace)
if err != nil {
return err
}
deleteOpts := DeleteOptions{}
deleteOpts.ApplyOptions(opts)
return resInt.Delete(ctx, metadata.Name, *deleteOpts.AsDeleteOptions())
}
// DeleteAllOf implements client.Client
func (mc *metadataClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...DeleteAllOfOption) error {
metadata, ok := obj.(*metav1.PartialObjectMetadata)
if !ok {
return fmt.Errorf("metadata client did not understand object: %T", obj)
}
deleteAllOfOpts := DeleteAllOfOptions{}
deleteAllOfOpts.ApplyOptions(opts)
resInt, err := mc.getResourceInterface(metadata.GroupVersionKind(), deleteAllOfOpts.ListOptions.Namespace)
if err != nil {
return err
}
return resInt.DeleteCollection(ctx, *deleteAllOfOpts.AsDeleteOptions(), *deleteAllOfOpts.AsListOptions())
}
// Patch implements client.Client
func (mc *metadataClient) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
metadata, ok := obj.(*metav1.PartialObjectMetadata)
if !ok {
return fmt.Errorf("metadata client did not understand object: %T", obj)
}
gvk := metadata.GroupVersionKind()
resInt, err := mc.getResourceInterface(gvk, metadata.Namespace)
if err != nil {
return err
}
data, err := patch.Data(obj)
if err != nil {
return err
}
patchOpts := &PatchOptions{}
res, err := resInt.Patch(ctx, metadata.Name, patch.Type(), data, *patchOpts.AsPatchOptions())
if err != nil {
return err
}
*metadata = *res
metadata.SetGroupVersionKind(gvk) // restore the GVK, which isn't set on metadata
return nil
}
// Get implements client.Client
func (mc *metadataClient) Get(ctx context.Context, key ObjectKey, obj runtime.Object) error {
metadata, ok := obj.(*metav1.PartialObjectMetadata)
if !ok {
return fmt.Errorf("metadata client did not understand object: %T", obj)
}
gvk := metadata.GroupVersionKind()
resInt, err := mc.getResourceInterface(gvk, key.Namespace)
if err != nil {
return err
}
res, err := resInt.Get(ctx, key.Name, metav1.GetOptions{})
if err != nil {
return err
}
*metadata = *res
metadata.SetGroupVersionKind(gvk) // restore the GVK, which isn't set on metadata
return nil
}
// List implements client.Client
func (mc *metadataClient) List(ctx context.Context, obj runtime.Object, opts ...ListOption) error {
metadata, ok := obj.(*metav1.PartialObjectMetadataList)
if !ok {
return fmt.Errorf("metadata client did not understand object: %T", obj)
}
gvk := metadata.GroupVersionKind()
if strings.HasSuffix(gvk.Kind, "List") {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
listOpts := ListOptions{}
listOpts.ApplyOptions(opts)
resInt, err := mc.getResourceInterface(gvk, listOpts.Namespace)
if err != nil {
return err
}
res, err := resInt.List(ctx, *listOpts.AsListOptions())
if err != nil {
return err
}
*metadata = *res
metadata.SetGroupVersionKind(gvk) // restore the GVK, which isn't set on metadata
return nil
}
func (mc *metadataClient) PatchStatus(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
metadata, ok := obj.(*metav1.PartialObjectMetadata)
if !ok {
return fmt.Errorf("metadata client did not understand object: %T", obj)
}
gvk := metadata.GroupVersionKind()
resInt, err := mc.getResourceInterface(gvk, metadata.Namespace)
if err != nil {
return err
}
data, err := patch.Data(obj)
if err != nil {
return err
}
patchOpts := &PatchOptions{}
res, err := resInt.Patch(ctx, metadata.Name, patch.Type(), data, *patchOpts.AsPatchOptions(), "status")
if err != nil {
return err
}
*metadata = *res
metadata.SetGroupVersionKind(gvk) // restore the GVK, which isn't set on metadata
return nil
}

View File

@@ -20,6 +20,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/selection"
)
// {{{ "Functional" Option Interfaces
@@ -70,28 +71,43 @@ var DryRunAll = dryRunAll{}
type dryRunAll struct{}
// ApplyToCreate applies this configuration to the given create options.
func (dryRunAll) ApplyToCreate(opts *CreateOptions) {
opts.DryRun = []string{metav1.DryRunAll}
}
// ApplyToUpdate applies this configuration to the given update options.
func (dryRunAll) ApplyToUpdate(opts *UpdateOptions) {
opts.DryRun = []string{metav1.DryRunAll}
}
// ApplyToPatch applies this configuration to the given patch options.
func (dryRunAll) ApplyToPatch(opts *PatchOptions) {
opts.DryRun = []string{metav1.DryRunAll}
}
// ApplyToPatch applies this configuration to the given delete options.
func (dryRunAll) ApplyToDelete(opts *DeleteOptions) {
opts.DryRun = []string{metav1.DryRunAll}
}
func (dryRunAll) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
opts.DryRun = []string{metav1.DryRunAll}
}
// FieldOwner set the field manager name for the given server-side apply patch.
type FieldOwner string
// ApplyToPatch applies this configuration to the given patch options.
func (f FieldOwner) ApplyToPatch(opts *PatchOptions) {
opts.FieldManager = string(f)
}
// ApplyToCreate applies this configuration to the given create options.
func (f FieldOwner) ApplyToCreate(opts *CreateOptions) {
opts.FieldManager = string(f)
}
// ApplyToUpdate applies this configuration to the given update options.
func (f FieldOwner) ApplyToUpdate(opts *UpdateOptions) {
opts.FieldManager = string(f)
}
@@ -251,33 +267,49 @@ func (o *DeleteOptions) ApplyToDelete(do *DeleteOptions) {
// to the given number of seconds.
type GracePeriodSeconds int64
// ApplyToDelete applies this configuration to the given delete options.
func (s GracePeriodSeconds) ApplyToDelete(opts *DeleteOptions) {
secs := int64(s)
opts.GracePeriodSeconds = &secs
}
// ApplyToDeleteAllOf applies this configuration to the given an List options.
func (s GracePeriodSeconds) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
s.ApplyToDelete(&opts.DeleteOptions)
}
// Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.
type Preconditions metav1.Preconditions
// ApplyToDelete applies this configuration to the given delete options.
func (p Preconditions) ApplyToDelete(opts *DeleteOptions) {
preconds := metav1.Preconditions(p)
opts.Preconditions = &preconds
}
// ApplyToDeleteAllOf applies this configuration to the given an List options.
func (p Preconditions) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
p.ApplyToDelete(&opts.DeleteOptions)
}
// PropagationPolicy determined whether and how garbage collection will be
// performed. Either this field or OrphanDependents may be set, but not both.
// The default policy is decided by the existing finalizer set in the
// metadata.finalizers and the resource-specific default policy.
// Acceptable values are: 'Orphan' - orphan the dependents; 'Background' -
// allow the garbage collector to delete the dependents in the background;
// 'Foreground' - a cascading policy that deletes all dependents in the
// foreground.
type PropagationPolicy metav1.DeletionPropagation
// ApplyToDelete applies the given delete options on these options.
// It will propagate to the dependents of the object to let the garbage collector handle it.
func (p PropagationPolicy) ApplyToDelete(opts *DeleteOptions) {
policy := metav1.DeletionPropagation(p)
opts.PropagationPolicy = &policy
}
// ApplyToDeleteAllOf applies this configuration to the given an List options.
func (p PropagationPolicy) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
p.ApplyToDelete(&opts.DeleteOptions)
}
@@ -378,16 +410,39 @@ func (o *ListOptions) ApplyOptions(opts []ListOption) *ListOptions {
// MatchingLabels filters the list/delete operation on the given set of labels.
type MatchingLabels map[string]string
// ApplyToList applies this configuration to the given list options.
func (m MatchingLabels) ApplyToList(opts *ListOptions) {
// TODO(directxman12): can we avoid reserializing this over and over?
sel := labels.SelectorFromSet(map[string]string(m))
sel := labels.SelectorFromValidatedSet(map[string]string(m))
opts.LabelSelector = sel
}
// ApplyToDeleteAllOf applies this configuration to the given an List options.
func (m MatchingLabels) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
m.ApplyToList(&opts.ListOptions)
}
// HasLabels filters the list/delete operation checking if the set of labels exists
// without checking their values.
type HasLabels []string
// ApplyToList applies this configuration to the given list options.
func (m HasLabels) ApplyToList(opts *ListOptions) {
sel := labels.NewSelector()
for _, label := range m {
r, err := labels.NewRequirement(label, selection.Exists, nil)
if err == nil {
sel = sel.Add(*r)
}
}
opts.LabelSelector = sel
}
// ApplyToDeleteAllOf applies this configuration to the given an List options.
func (m HasLabels) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
m.ApplyToList(&opts.ListOptions)
}
// MatchingLabelsSelector filters the list/delete operation on the given label
// selector (or index in the case of cached lists). A struct is used because
// labels.Selector is an interface, which cannot be aliased.
@@ -395,10 +450,12 @@ type MatchingLabelsSelector struct {
labels.Selector
}
// ApplyToList applies this configuration to the given list options.
func (m MatchingLabelsSelector) ApplyToList(opts *ListOptions) {
opts.LabelSelector = m
}
// ApplyToDeleteAllOf applies this configuration to the given an List options.
func (m MatchingLabelsSelector) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
m.ApplyToList(&opts.ListOptions)
}
@@ -411,16 +468,18 @@ func MatchingField(name, val string) MatchingFields {
return MatchingFields{name: val}
}
// MatchingField filters the list/delete operation on the given field Set
// MatchingFields filters the list/delete operation on the given field Set
// (or index in the case of cached lists).
type MatchingFields fields.Set
// ApplyToList applies this configuration to the given list options.
func (m MatchingFields) ApplyToList(opts *ListOptions) {
// TODO(directxman12): can we avoid re-serializing this?
sel := fields.SelectorFromSet(fields.Set(m))
sel := fields.Set(m).AsSelector()
opts.FieldSelector = sel
}
// ApplyToDeleteAllOf applies this configuration to the given an List options.
func (m MatchingFields) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
m.ApplyToList(&opts.ListOptions)
}
@@ -432,10 +491,12 @@ type MatchingFieldsSelector struct {
fields.Selector
}
// ApplyToList applies this configuration to the given list options.
func (m MatchingFieldsSelector) ApplyToList(opts *ListOptions) {
opts.FieldSelector = m
}
// ApplyToDeleteAllOf applies this configuration to the given an List options.
func (m MatchingFieldsSelector) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
m.ApplyToList(&opts.ListOptions)
}
@@ -443,10 +504,12 @@ func (m MatchingFieldsSelector) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
// InNamespace restricts the list/delete operation to the given namespace.
type InNamespace string
// ApplyToList applies this configuration to the given list options.
func (n InNamespace) ApplyToList(opts *ListOptions) {
opts.Namespace = string(n)
}
// ApplyToDeleteAllOf applies this configuration to the given an List options.
func (n InNamespace) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
n.ApplyToList(&opts.ListOptions)
}
@@ -456,6 +519,7 @@ func (n InNamespace) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
// does not support setting it for deletecollection operations.
type Limit int64
// ApplyToList applies this configuration to the given an list options.
func (l Limit) ApplyToList(opts *ListOptions) {
opts.Limit = int64(l)
}
@@ -465,6 +529,7 @@ func (l Limit) ApplyToList(opts *ListOptions) {
// does not support setting it for deletecollection operations.
type Continue string
// ApplyToList applies this configuration to the given an List options.
func (c Continue) ApplyToList(opts *ListOptions) {
opts.Continue = string(c)
}

View File

@@ -17,7 +17,11 @@ limitations under the License.
package client
import (
"fmt"
jsonpatch "github.com/evanphx/json-patch"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/json"
@@ -26,6 +30,10 @@ import (
var (
// Apply uses server-side apply to patch the given object.
Apply = applyPatch{}
// Merge uses the raw object as a merge patch, without modifications.
// Use MergeFrom if you wish to compute a diff instead.
Merge = mergePatch{}
)
type patch struct {
@@ -43,13 +51,51 @@ func (s *patch) Data(obj runtime.Object) ([]byte, error) {
return s.data, nil
}
// ConstantPatch constructs a new Patch with the given PatchType and data.
func ConstantPatch(patchType types.PatchType, data []byte) Patch {
// RawPatch constructs a new Patch with the given PatchType and data.
func RawPatch(patchType types.PatchType, data []byte) Patch {
return &patch{patchType, data}
}
// ConstantPatch constructs a new Patch with the given PatchType and data.
//
// Deprecated: use RawPatch instead
func ConstantPatch(patchType types.PatchType, data []byte) Patch {
return RawPatch(patchType, data)
}
// MergeFromWithOptimisticLock can be used if clients want to make sure a patch
// is being applied to the latest resource version of an object.
//
// The behavior is similar to what an Update would do, without the need to send the
// whole object. Usually this method is useful if you might have multiple clients
// acting on the same object and the same API version, but with different versions of the Go structs.
//
// For example, an "older" copy of a Widget that has fields A and B, and a "newer" copy with A, B, and C.
// Sending an update using the older struct definition results in C being dropped, whereas using a patch does not.
type MergeFromWithOptimisticLock struct{}
// ApplyToMergeFrom applies this configuration to the given patch options.
func (m MergeFromWithOptimisticLock) ApplyToMergeFrom(in *MergeFromOptions) {
in.OptimisticLock = true
}
// MergeFromOption is some configuration that modifies options for a merge-from patch data.
type MergeFromOption interface {
// ApplyToMergeFrom applies this configuration to the given patch options.
ApplyToMergeFrom(*MergeFromOptions)
}
// MergeFromOptions contains options to generate a merge-from patch data.
type MergeFromOptions struct {
// OptimisticLock, when true, includes `metadata.resourceVersion` into the final
// patch data. If the `resourceVersion` field doesn't match what's stored,
// the operation results in a conflict and clients will need to try again.
OptimisticLock bool
}
type mergeFromPatch struct {
from runtime.Object
opts MergeFromOptions
}
// Type implements patch.
@@ -69,12 +115,64 @@ func (s *mergeFromPatch) Data(obj runtime.Object) ([]byte, error) {
return nil, err
}
return jsonpatch.CreateMergePatch(originalJSON, modifiedJSON)
data, err := jsonpatch.CreateMergePatch(originalJSON, modifiedJSON)
if err != nil {
return nil, err
}
if s.opts.OptimisticLock {
dataMap := map[string]interface{}{}
if err := json.Unmarshal(data, &dataMap); err != nil {
return nil, err
}
fromMeta, ok := s.from.(metav1.Object)
if !ok {
return nil, fmt.Errorf("cannot use OptimisticLock, from object %q is not a valid metav1.Object", s.from)
}
resourceVersion := fromMeta.GetResourceVersion()
if len(resourceVersion) == 0 {
return nil, fmt.Errorf("cannot use OptimisticLock, from object %q does not have any resource version we can use", s.from)
}
u := &unstructured.Unstructured{Object: dataMap}
u.SetResourceVersion(resourceVersion)
data, err = json.Marshal(u)
if err != nil {
return nil, err
}
}
return data, nil
}
// MergeFrom creates a Patch that patches using the merge-patch strategy with the given object as base.
func MergeFrom(obj runtime.Object) Patch {
return &mergeFromPatch{obj}
return &mergeFromPatch{from: obj}
}
// MergeFromWithOptions creates a Patch that patches using the merge-patch strategy with the given object as base.
func MergeFromWithOptions(obj runtime.Object, opts ...MergeFromOption) Patch {
options := &MergeFromOptions{}
for _, opt := range opts {
opt.ApplyToMergeFrom(options)
}
return &mergeFromPatch{from: obj, opts: *options}
}
// mergePatch uses a raw merge strategy to patch the object.
type mergePatch struct{}
// Type implements Patch.
func (p mergePatch) Type() types.PatchType {
return types.MergePatchType
}
// Data implements Patch.
func (p mergePatch) Data(obj runtime.Object) ([]byte, error) {
// NB(directxman12): we might technically want to be using an actual encoder
// here (in case some more performant encoder is introduced) but this is
// correct and sufficient for our uses (it's what the JSON serializer in
// client-go does, more-or-less).
return json.Marshal(obj)
}
// applyPatch uses server-side apply to patch the object.

View File

@@ -25,7 +25,7 @@ import (
// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes
// new clients at the time they are used, and caches the client.
type typedClient struct {
cache clientCache
cache *clientCache
paramCodec runtime.ParameterCodec
}
@@ -43,8 +43,7 @@ func (c *typedClient) Create(ctx context.Context, obj runtime.Object, opts ...Cr
Resource(o.resource()).
Body(obj).
VersionedParams(createOpts.AsCreateOptions(), c.paramCodec).
Context(ctx).
Do().
Do(ctx).
Into(obj)
}
@@ -63,8 +62,7 @@ func (c *typedClient) Update(ctx context.Context, obj runtime.Object, opts ...Up
Name(o.GetName()).
Body(obj).
VersionedParams(updateOpts.AsUpdateOptions(), c.paramCodec).
Context(ctx).
Do().
Do(ctx).
Into(obj)
}
@@ -83,8 +81,7 @@ func (c *typedClient) Delete(ctx context.Context, obj runtime.Object, opts ...De
Resource(o.resource()).
Name(o.GetName()).
Body(deleteOpts.AsDeleteOptions()).
Context(ctx).
Do().
Do(ctx).
Error()
}
@@ -103,8 +100,7 @@ func (c *typedClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts
Resource(o.resource()).
VersionedParams(deleteAllOfOpts.AsListOptions(), c.paramCodec).
Body(deleteAllOfOpts.AsDeleteOptions()).
Context(ctx).
Do().
Do(ctx).
Error()
}
@@ -127,8 +123,7 @@ func (c *typedClient) Patch(ctx context.Context, obj runtime.Object, patch Patch
Name(o.GetName()).
VersionedParams(patchOpts.ApplyOptions(opts).AsPatchOptions(), c.paramCodec).
Body(data).
Context(ctx).
Do().
Do(ctx).
Into(obj)
}
@@ -141,8 +136,7 @@ func (c *typedClient) Get(ctx context.Context, key ObjectKey, obj runtime.Object
return r.Get().
NamespaceIfScoped(key.Namespace, r.isNamespaced()).
Resource(r.resource()).
Context(ctx).
Name(key.Name).Do().Into(obj)
Name(key.Name).Do(ctx).Into(obj)
}
// List implements client.Client
@@ -157,8 +151,7 @@ func (c *typedClient) List(ctx context.Context, obj runtime.Object, opts ...List
NamespaceIfScoped(listOpts.Namespace, r.isNamespaced()).
Resource(r.resource()).
VersionedParams(listOpts.AsListOptions(), c.paramCodec).
Context(ctx).
Do().
Do(ctx).
Into(obj)
}
@@ -179,8 +172,7 @@ func (c *typedClient) UpdateStatus(ctx context.Context, obj runtime.Object, opts
SubResource("status").
Body(obj).
VersionedParams((&UpdateOptions{}).ApplyOptions(opts).AsUpdateOptions(), c.paramCodec).
Context(ctx).
Do().
Do(ctx).
Into(obj)
}
@@ -204,7 +196,6 @@ func (c *typedClient) PatchStatus(ctx context.Context, obj runtime.Object, patch
SubResource("status").
Body(data).
VersionedParams(patchOpts.ApplyOptions(opts).AsPatchOptions(), c.paramCodec).
Context(ctx).
Do().
Do(ctx).
Into(obj)
}

View File

@@ -21,101 +21,128 @@ import (
"fmt"
"strings"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
)
// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes
// new clients at the time they are used, and caches the client.
type unstructuredClient struct {
client dynamic.Interface
restMapper meta.RESTMapper
cache *clientCache
paramCodec runtime.ParameterCodec
}
// Create implements client.Client
func (uc *unstructuredClient) Create(_ context.Context, obj runtime.Object, opts ...CreateOption) error {
func (uc *unstructuredClient) Create(ctx context.Context, obj runtime.Object, opts ...CreateOption) error {
u, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
createOpts := CreateOptions{}
gvk := u.GroupVersionKind()
o, err := uc.cache.getObjMeta(obj)
if err != nil {
return err
}
createOpts := &CreateOptions{}
createOpts.ApplyOptions(opts)
r, err := uc.getResourceInterface(u.GroupVersionKind(), u.GetNamespace())
if err != nil {
return err
}
i, err := r.Create(u, *createOpts.AsCreateOptions())
if err != nil {
return err
}
u.Object = i.Object
return nil
result := o.Post().
NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()).
Resource(o.resource()).
Body(obj).
VersionedParams(createOpts.AsCreateOptions(), uc.paramCodec).
Do(ctx).
Into(obj)
u.SetGroupVersionKind(gvk)
return result
}
// Update implements client.Client
func (uc *unstructuredClient) Update(_ context.Context, obj runtime.Object, opts ...UpdateOption) error {
func (uc *unstructuredClient) Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
u, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
gvk := u.GroupVersionKind()
o, err := uc.cache.getObjMeta(obj)
if err != nil {
return err
}
updateOpts := UpdateOptions{}
updateOpts.ApplyOptions(opts)
r, err := uc.getResourceInterface(u.GroupVersionKind(), u.GetNamespace())
if err != nil {
return err
}
i, err := r.Update(u, *updateOpts.AsUpdateOptions())
if err != nil {
return err
}
u.Object = i.Object
return nil
result := o.Put().
NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()).
Resource(o.resource()).
Name(o.GetName()).
Body(obj).
VersionedParams(updateOpts.AsUpdateOptions(), uc.paramCodec).
Do(ctx).
Into(obj)
u.SetGroupVersionKind(gvk)
return result
}
// Delete implements client.Client
func (uc *unstructuredClient) Delete(_ context.Context, obj runtime.Object, opts ...DeleteOption) error {
u, ok := obj.(*unstructured.Unstructured)
func (uc *unstructuredClient) Delete(ctx context.Context, obj runtime.Object, opts ...DeleteOption) error {
_, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
r, err := uc.getResourceInterface(u.GroupVersionKind(), u.GetNamespace())
o, err := uc.cache.getObjMeta(obj)
if err != nil {
return err
}
deleteOpts := DeleteOptions{}
deleteOpts.ApplyOptions(opts)
err = r.Delete(u.GetName(), deleteOpts.AsDeleteOptions())
return err
return o.Delete().
NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()).
Resource(o.resource()).
Name(o.GetName()).
Body(deleteOpts.AsDeleteOptions()).
Do(ctx).
Error()
}
// DeleteAllOf implements client.Client
func (uc *unstructuredClient) DeleteAllOf(_ context.Context, obj runtime.Object, opts ...DeleteAllOfOption) error {
u, ok := obj.(*unstructured.Unstructured)
func (uc *unstructuredClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...DeleteAllOfOption) error {
_, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
r, err := uc.getResourceInterface(u.GroupVersionKind(), u.GetNamespace())
o, err := uc.cache.getObjMeta(obj)
if err != nil {
return err
}
deleteAllOfOpts := DeleteAllOfOptions{}
deleteAllOfOpts.ApplyOptions(opts)
err = r.DeleteCollection(deleteAllOfOpts.AsDeleteOptions(), *deleteAllOfOpts.AsListOptions())
return err
return o.Delete().
NamespaceIfScoped(deleteAllOfOpts.ListOptions.Namespace, o.isNamespaced()).
Resource(o.resource()).
VersionedParams(deleteAllOfOpts.AsListOptions(), uc.paramCodec).
Body(deleteAllOfOpts.AsDeleteOptions()).
Do(ctx).
Error()
}
// Patch implements client.Client
func (uc *unstructuredClient) Patch(_ context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
u, ok := obj.(*unstructured.Unstructured)
func (uc *unstructuredClient) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
_, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
r, err := uc.getResourceInterface(u.GroupVersionKind(), u.GetNamespace())
o, err := uc.cache.getObjMeta(obj)
if err != nil {
return err
}
@@ -126,81 +153,101 @@ func (uc *unstructuredClient) Patch(_ context.Context, obj runtime.Object, patch
}
patchOpts := &PatchOptions{}
i, err := r.Patch(u.GetName(), patch.Type(), data, *patchOpts.ApplyOptions(opts).AsPatchOptions())
if err != nil {
return err
}
u.Object = i.Object
return nil
return o.Patch(patch.Type()).
NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()).
Resource(o.resource()).
Name(o.GetName()).
VersionedParams(patchOpts.ApplyOptions(opts).AsPatchOptions(), uc.paramCodec).
Body(data).
Do(ctx).
Into(obj)
}
// Get implements client.Client
func (uc *unstructuredClient) Get(_ context.Context, key ObjectKey, obj runtime.Object) error {
func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj runtime.Object) error {
u, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
r, err := uc.getResourceInterface(u.GroupVersionKind(), key.Namespace)
gvk := u.GroupVersionKind()
r, err := uc.cache.getResource(obj)
if err != nil {
return err
}
i, err := r.Get(key.Name, metav1.GetOptions{})
if err != nil {
return err
}
u.Object = i.Object
return nil
result := r.Get().
NamespaceIfScoped(key.Namespace, r.isNamespaced()).
Resource(r.resource()).
Name(key.Name).
Do(ctx).
Into(obj)
u.SetGroupVersionKind(gvk)
return result
}
// List implements client.Client
func (uc *unstructuredClient) List(_ context.Context, obj runtime.Object, opts ...ListOption) error {
func (uc *unstructuredClient) List(ctx context.Context, obj runtime.Object, opts ...ListOption) error {
u, ok := obj.(*unstructured.UnstructuredList)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
gvk := u.GroupVersionKind()
if strings.HasSuffix(gvk.Kind, "List") {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
listOpts := ListOptions{}
listOpts.ApplyOptions(opts)
r, err := uc.getResourceInterface(gvk, listOpts.Namespace)
r, err := uc.cache.getResource(obj)
if err != nil {
return err
}
i, err := r.List(*listOpts.AsListOptions())
if err != nil {
return err
}
u.Items = i.Items
u.Object = i.Object
return nil
return r.Get().
NamespaceIfScoped(listOpts.Namespace, r.isNamespaced()).
Resource(r.resource()).
VersionedParams(listOpts.AsListOptions(), uc.paramCodec).
Do(ctx).
Into(obj)
}
func (uc *unstructuredClient) UpdateStatus(_ context.Context, obj runtime.Object, opts ...UpdateOption) error {
func (uc *unstructuredClient) UpdateStatus(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
_, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
o, err := uc.cache.getObjMeta(obj)
if err != nil {
return err
}
return o.Put().
NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()).
Resource(o.resource()).
Name(o.GetName()).
SubResource("status").
Body(obj).
VersionedParams((&UpdateOptions{}).ApplyOptions(opts).AsUpdateOptions(), uc.paramCodec).
Do(ctx).
Into(obj)
}
func (uc *unstructuredClient) PatchStatus(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
u, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
r, err := uc.getResourceInterface(u.GroupVersionKind(), u.GetNamespace())
if err != nil {
return err
}
i, err := r.UpdateStatus(u, *(&UpdateOptions{}).ApplyOptions(opts).AsUpdateOptions())
if err != nil {
return err
}
u.Object = i.Object
return nil
}
func (uc *unstructuredClient) PatchStatus(_ context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
u, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
r, err := uc.getResourceInterface(u.GroupVersionKind(), u.GetNamespace())
gvk := u.GroupVersionKind()
o, err := uc.cache.getObjMeta(obj)
if err != nil {
return err
}
@@ -210,21 +257,17 @@ func (uc *unstructuredClient) PatchStatus(_ context.Context, obj runtime.Object,
return err
}
i, err := r.Patch(u.GetName(), patch.Type(), data, *(&PatchOptions{}).ApplyOptions(opts).AsPatchOptions(), "status")
if err != nil {
return err
}
u.Object = i.Object
return nil
}
patchOpts := &PatchOptions{}
result := o.Patch(patch.Type()).
NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()).
Resource(o.resource()).
Name(o.GetName()).
SubResource("status").
Body(data).
VersionedParams(patchOpts.ApplyOptions(opts).AsPatchOptions(), uc.paramCodec).
Do(ctx).
Into(u)
func (uc *unstructuredClient) getResourceInterface(gvk schema.GroupVersionKind, ns string) (dynamic.ResourceInterface, error) {
mapping, err := uc.restMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
return nil, err
}
if mapping.Scope.Name() == meta.RESTScopeNameRoot {
return uc.client.Resource(mapping.Resource), nil
}
return uc.client.Resource(mapping.Resource).Namespace(ns), nil
u.SetGroupVersionKind(gvk)
return result
}

View File

@@ -19,11 +19,13 @@ package controller
import (
"fmt"
"github.com/go-logr/logr"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/internal/controller"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/ratelimiter"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
)
@@ -35,6 +37,14 @@ type Options struct {
// Reconciler reconciles an object
Reconciler reconcile.Reconciler
// RateLimiter is used to limit how frequently requests may be queued.
// Defaults to MaxOfRateLimiter which has both overall and per-item rate limiting.
// The overall is a token bucket and the per-item is exponential.
RateLimiter ratelimiter.RateLimiter
// Log is the logger used for this controller.
Log logr.Logger
}
// Controller implements a Kubernetes API. A Controller manages a work queue fed reconcile.Requests
@@ -61,6 +71,18 @@ type Controller interface {
// New returns a new Controller registered with the Manager. The Manager will ensure that shared Caches have
// been synced before the Controller is Started.
func New(name string, mgr manager.Manager, options Options) (Controller, error) {
c, err := NewUnmanaged(name, mgr, options)
if err != nil {
return nil, err
}
// Add the controller as a Manager components
return c, mgr.Add(c)
}
// NewUnmanaged returns a new controller without adding it to the manager. The
// caller is responsible for starting the returned controller.
func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller, error) {
if options.Reconciler == nil {
return nil, fmt.Errorf("must specify Reconciler")
}
@@ -73,26 +95,28 @@ func New(name string, mgr manager.Manager, options Options) (Controller, error)
options.MaxConcurrentReconciles = 1
}
if options.RateLimiter == nil {
options.RateLimiter = workqueue.DefaultControllerRateLimiter()
}
if options.Log == nil {
options.Log = mgr.GetLogger()
}
// Inject dependencies into Reconciler
if err := mgr.SetFields(options.Reconciler); err != nil {
return nil, err
}
// Create controller with dependencies set
c := &controller.Controller{
Do: options.Reconciler,
Cache: mgr.GetCache(),
Config: mgr.GetConfig(),
Scheme: mgr.GetScheme(),
Client: mgr.GetClient(),
Recorder: mgr.GetEventRecorderFor(name),
return &controller.Controller{
Do: options.Reconciler,
MakeQueue: func() workqueue.RateLimitingInterface {
return workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), name)
return workqueue.NewNamedRateLimitingQueue(options.RateLimiter, name)
},
MaxConcurrentReconciles: options.MaxConcurrentReconciles,
SetFields: mgr.SetFields,
Name: name,
}
// Add the controller as a Manager components
return c, mgr.Add(c)
Log: options.Log.WithName("controller").WithValues("controller", name),
}, nil
}

View File

@@ -19,13 +19,14 @@ package controllerutil
import (
"context"
"fmt"
"reflect"
"k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
@@ -49,17 +50,98 @@ func newAlreadyOwnedError(Object metav1.Object, Owner metav1.OwnerReference) *Al
}
}
// SetControllerReference sets owner as a Controller OwnerReference on owned.
// This is used for garbage collection of the owned object and for
// reconciling the owner object on changes to owned (with a Watch + EnqueueRequestForOwner).
// SetControllerReference sets owner as a Controller OwnerReference on controlled.
// This is used for garbage collection of the controlled object and for
// reconciling the owner object on changes to controlled (with a Watch + EnqueueRequestForOwner).
// Since only one OwnerReference can be a controller, it returns an error if
// there is another OwnerReference with Controller flag set.
func SetControllerReference(owner, object metav1.Object, scheme *runtime.Scheme) error {
func SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Scheme) error {
// Validate the owner.
ro, ok := owner.(runtime.Object)
if !ok {
return fmt.Errorf("%T is not a runtime.Object, cannot call SetControllerReference", owner)
}
if err := validateOwner(owner, controlled); err != nil {
return err
}
// Create a new controller ref.
gvk, err := apiutil.GVKForObject(ro, scheme)
if err != nil {
return err
}
ref := metav1.OwnerReference{
APIVersion: gvk.GroupVersion().String(),
Kind: gvk.Kind,
Name: owner.GetName(),
UID: owner.GetUID(),
BlockOwnerDeletion: pointer.BoolPtr(true),
Controller: pointer.BoolPtr(true),
}
// Return early with an error if the object is already controlled.
if existing := metav1.GetControllerOf(controlled); existing != nil && !referSameObject(*existing, ref) {
return newAlreadyOwnedError(controlled, *existing)
}
// Update owner references and return.
upsertOwnerRef(ref, controlled)
return nil
}
// SetOwnerReference is a helper method to make sure the given object contains an object reference to the object provided.
// This allows you to declare that owner has a dependency on the object without specifying it as a controller.
// If a reference to the same object already exists, it'll be overwritten with the newly provided version.
func SetOwnerReference(owner, object metav1.Object, scheme *runtime.Scheme) error {
// Validate the owner.
ro, ok := owner.(runtime.Object)
if !ok {
return fmt.Errorf("%T is not a runtime.Object, cannot call SetOwnerReference", owner)
}
if err := validateOwner(owner, object); err != nil {
return err
}
// Create a new owner ref.
gvk, err := apiutil.GVKForObject(ro, scheme)
if err != nil {
return err
}
ref := metav1.OwnerReference{
APIVersion: gvk.GroupVersion().String(),
Kind: gvk.Kind,
UID: owner.GetUID(),
Name: owner.GetName(),
}
// Update owner references and return.
upsertOwnerRef(ref, object)
return nil
}
func upsertOwnerRef(ref metav1.OwnerReference, object metav1.Object) {
owners := object.GetOwnerReferences()
idx := indexOwnerRef(owners, ref)
if idx == -1 {
owners = append(owners, ref)
} else {
owners[idx] = ref
}
object.SetOwnerReferences(owners)
}
// indexOwnerRef returns the index of the owner reference in the slice if found, or -1.
func indexOwnerRef(ownerReferences []metav1.OwnerReference, ref metav1.OwnerReference) int {
for index, r := range ownerReferences {
if referSameObject(r, ref) {
return index
}
}
return -1
}
func validateOwner(owner, object metav1.Object) error {
ownerNs := owner.GetNamespace()
if ownerNs != "" {
objNs := object.GetNamespace()
@@ -70,32 +152,6 @@ func SetControllerReference(owner, object metav1.Object, scheme *runtime.Scheme)
return fmt.Errorf("cross-namespace owner references are disallowed, owner's namespace %s, obj's namespace %s", owner.GetNamespace(), object.GetNamespace())
}
}
gvk, err := apiutil.GVKForObject(ro, scheme)
if err != nil {
return err
}
// Create a new ref
ref := *metav1.NewControllerRef(owner, schema.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind})
existingRefs := object.GetOwnerReferences()
fi := -1
for i, r := range existingRefs {
if referSameObject(ref, r) {
fi = i
} else if r.Controller != nil && *r.Controller {
return newAlreadyOwnedError(object, r)
}
}
if fi == -1 {
existingRefs = append(existingRefs, ref)
} else {
existingRefs[fi] = ref
}
// Update owner references
object.SetOwnerReferences(existingRefs)
return nil
}
@@ -111,7 +167,7 @@ func referSameObject(a, b metav1.OwnerReference) bool {
return false
}
return aGV == bGV && a.Kind == b.Kind && a.Name == b.Name
return aGV.Group == bGV.Group && a.Kind == b.Kind && a.Name == b.Name
}
// OperationResult is the action result of a CreateOrUpdate call
@@ -157,7 +213,7 @@ func CreateOrUpdate(ctx context.Context, c client.Client, obj runtime.Object, f
return OperationResultNone, err
}
if reflect.DeepEqual(existing, obj) {
if equality.Semantic.DeepEqual(existing, obj) {
return OperationResultNone, nil
}
@@ -181,8 +237,8 @@ func mutate(f MutateFn, key client.ObjectKey, obj runtime.Object) error {
// MutateFn is a function which mutates the existing object into it's desired state.
type MutateFn func() error
// AddFinalizer accepts a metav1 object and adds the provided finalizer if not present.
func AddFinalizer(o metav1.Object, finalizer string) {
// AddFinalizer accepts an Object and adds the provided finalizer if not present.
func AddFinalizer(o Object, finalizer string) {
f := o.GetFinalizers()
for _, e := range f {
if e == finalizer {
@@ -194,21 +250,24 @@ func AddFinalizer(o metav1.Object, finalizer string) {
// AddFinalizerWithError tries to convert a runtime object to a metav1 object and add the provided finalizer.
// It returns an error if the provided object cannot provide an accessor.
//
// Deprecated: Use AddFinalizer instead. Check is performing on compile time.
func AddFinalizerWithError(o runtime.Object, finalizer string) error {
m, err := meta.Accessor(o)
if err != nil {
return err
}
AddFinalizer(m, finalizer)
AddFinalizer(m.(Object), finalizer)
return nil
}
// RemoveFinalizer accepts a metav1 object and removes the provided finalizer if present.
func RemoveFinalizer(o metav1.Object, finalizer string) {
// RemoveFinalizer accepts an Object and removes the provided finalizer if present.
func RemoveFinalizer(o Object, finalizer string) {
f := o.GetFinalizers()
for i, e := range f {
if e == finalizer {
for i := 0; i < len(f); i++ {
if f[i] == finalizer {
f = append(f[:i], f[i+1:]...)
i--
}
}
o.SetFinalizers(f)
@@ -216,11 +275,31 @@ func RemoveFinalizer(o metav1.Object, finalizer string) {
// RemoveFinalizerWithError tries to convert a runtime object to a metav1 object and remove the provided finalizer.
// It returns an error if the provided object cannot provide an accessor.
//
// Deprecated: Use RemoveFinalizer instead. Check is performing on compile time.
func RemoveFinalizerWithError(o runtime.Object, finalizer string) error {
m, err := meta.Accessor(o)
if err != nil {
return err
}
RemoveFinalizer(m, finalizer)
RemoveFinalizer(m.(Object), finalizer)
return nil
}
// ContainsFinalizer checks an Object that the provided finalizer is present.
func ContainsFinalizer(o Object, finalizer string) bool {
f := o.GetFinalizers()
for _, e := range f {
if e == finalizer {
return true
}
}
return false
}
// Object allows functions to work indistinctly with any resource that
// implements both Object interfaces.
type Object interface {
metav1.Object
runtime.Object
}

View File

@@ -19,29 +19,33 @@ package envtest
import (
"bufio"
"bytes"
"context"
"io"
"io/ioutil"
"os"
"path/filepath"
"time"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
k8syaml "k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/yaml"
)
// CRDInstallOptions are the options for installing CRDs
type CRDInstallOptions struct {
// Paths is a list of paths to the directories containing CRDs
// Paths is a list of paths to the directories or files containing CRDs
Paths []string
// CRDs is a list of CRDs to install
CRDs []*apiextensionsv1beta1.CustomResourceDefinition
CRDs []runtime.Object
// ErrorIfPathMissing will cause an error if a Path does not exist
ErrorIfPathMissing bool
@@ -51,13 +55,18 @@ type CRDInstallOptions struct {
// PollInterval is the interval to check
PollInterval time.Duration
// CleanUpAfterUse will cause the CRDs listed for installation to be
// uninstalled when terminating the test environment.
// Defaults to false.
CleanUpAfterUse bool
}
const defaultPollInterval = 100 * time.Millisecond
const defaultMaxWait = 10 * time.Second
// InstallCRDs installs a collection of CRDs into a cluster by reading the crd yaml files from a directory
func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]*apiextensionsv1beta1.CustomResourceDefinition, error) {
func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]runtime.Object, error) {
defaultCRDOptions(&options)
// Read the CRD yamls into options.CRDs
@@ -81,16 +90,12 @@ func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]*apiextensio
// readCRDFiles reads the directories of CRDs in options.Paths and adds the CRD structs to options.CRDs
func readCRDFiles(options *CRDInstallOptions) error {
if len(options.Paths) > 0 {
for _, path := range options.Paths {
if _, err := os.Stat(path); !options.ErrorIfPathMissing && os.IsNotExist(err) {
continue
}
new, err := readCRDs(path)
if err != nil {
return err
}
options.CRDs = append(options.CRDs, new...)
crdList, err := renderCRDs(options)
if err != nil {
return err
}
options.CRDs = append(options.CRDs, crdList...)
}
return nil
}
@@ -106,19 +111,49 @@ func defaultCRDOptions(o *CRDInstallOptions) {
}
// WaitForCRDs waits for the CRDs to appear in discovery
func WaitForCRDs(config *rest.Config, crds []*apiextensionsv1beta1.CustomResourceDefinition, options CRDInstallOptions) error {
func WaitForCRDs(config *rest.Config, crds []runtime.Object, options CRDInstallOptions) error {
// Add each CRD to a map of GroupVersion to Resource
waitingFor := map[schema.GroupVersion]*sets.String{}
for _, crd := range crds {
for _, crd := range runtimeCRDListToUnstructured(crds) {
gvs := []schema.GroupVersion{}
if crd.Spec.Version != "" {
gvs = append(gvs, schema.GroupVersion{Group: crd.Spec.Group, Version: crd.Spec.Version})
crdGroup, _, err := unstructured.NestedString(crd.Object, "spec", "group")
if err != nil {
return err
}
for _, ver := range crd.Spec.Versions {
if ver.Served {
gvs = append(gvs, schema.GroupVersion{Group: crd.Spec.Group, Version: ver.Name})
crdPlural, _, err := unstructured.NestedString(crd.Object, "spec", "names", "plural")
if err != nil {
return err
}
crdVersion, _, err := unstructured.NestedString(crd.Object, "spec", "version")
if err != nil {
return err
}
if crdVersion != "" {
gvs = append(gvs, schema.GroupVersion{Group: crdGroup, Version: crdVersion})
}
versions, _, err := unstructured.NestedSlice(crd.Object, "spec", "versions")
if err != nil {
return err
}
for _, version := range versions {
versionMap, ok := version.(map[string]interface{})
if !ok {
continue
}
served, _, err := unstructured.NestedBool(versionMap, "served")
if err != nil {
return err
}
if served {
versionName, _, err := unstructured.NestedString(versionMap, "name")
if err != nil {
return err
}
gvs = append(gvs, schema.GroupVersion{Group: crdGroup, Version: versionName})
}
}
for _, gv := range gvs {
log.V(1).Info("adding API in waitlist", "GV", gv)
if _, found := waitingFor[gv]; !found {
@@ -126,7 +161,7 @@ func WaitForCRDs(config *rest.Config, crds []*apiextensionsv1beta1.CustomResourc
waitingFor[gv] = &sets.String{}
}
// Add the Resource
waitingFor[gv].Insert(crd.Spec.Names.Plural)
waitingFor[gv].Insert(crdPlural)
}
}
@@ -180,57 +215,159 @@ func (p *poller) poll() (done bool, err error) {
return allFound, nil
}
// UninstallCRDs uninstalls a collection of CRDs by reading the crd yaml files from a directory
func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error {
// Read the CRD yamls into options.CRDs
if err := readCRDFiles(&options); err != nil {
return err
}
// Delete the CRDs from the apiserver
cs, err := client.New(config, client.Options{})
if err != nil {
return err
}
// Uninstall each CRD
for _, crd := range runtimeCRDListToUnstructured(options.CRDs) {
log.V(1).Info("uninstalling CRD", "crd", crd.GetName())
if err := cs.Delete(context.TODO(), crd); err != nil {
// If CRD is not found, we can consider success
if !apierrors.IsNotFound(err) {
return err
}
}
}
return nil
}
// CreateCRDs creates the CRDs
func CreateCRDs(config *rest.Config, crds []*apiextensionsv1beta1.CustomResourceDefinition) error {
cs, err := clientset.NewForConfig(config)
func CreateCRDs(config *rest.Config, crds []runtime.Object) error {
cs, err := client.New(config, client.Options{})
if err != nil {
return err
}
// Create each CRD
for _, crd := range crds {
log.V(1).Info("installing CRD", "crd", crd.Name)
if _, err := cs.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
for _, crd := range runtimeCRDListToUnstructured(crds) {
log.V(1).Info("installing CRD", "crd", crd.GetName())
existingCrd := crd.DeepCopy()
err := cs.Get(context.TODO(), client.ObjectKey{Name: crd.GetName()}, existingCrd)
switch {
case apierrors.IsNotFound(err):
if err := cs.Create(context.TODO(), crd); err != nil {
return err
}
case err != nil:
return err
default:
log.V(1).Info("CRD already exists, updating", "crd", crd.GetName())
crd.SetResourceVersion(existingCrd.GetResourceVersion())
if err := cs.Update(context.TODO(), crd); err != nil {
return err
}
}
}
return nil
}
// readCRDs reads the CRDs from files and Unmarshals them into structs
func readCRDs(path string) ([]*apiextensionsv1beta1.CustomResourceDefinition, error) {
// Get the CRD files
var files []os.FileInfo
var err error
log.V(1).Info("reading CRDs from path", "path", path)
if files, err = ioutil.ReadDir(path); err != nil {
return nil, err
// renderCRDs iterate through options.Paths and extract all CRD files.
func renderCRDs(options *CRDInstallOptions) ([]runtime.Object, error) {
var (
err error
info os.FileInfo
files []os.FileInfo
)
type GVKN struct {
GVK schema.GroupVersionKind
Name string
}
crds := map[GVKN]*unstructured.Unstructured{}
for _, path := range options.Paths {
var filePath = path
// Return the error if ErrorIfPathMissing exists
if info, err = os.Stat(path); os.IsNotExist(err) {
if options.ErrorIfPathMissing {
return nil, err
}
continue
}
if !info.IsDir() {
filePath, files = filepath.Dir(path), []os.FileInfo{info}
} else {
if files, err = ioutil.ReadDir(path); err != nil {
return nil, err
}
}
log.V(1).Info("reading CRDs from path", "path", path)
crdList, err := readCRDs(filePath, files)
if err != nil {
return nil, err
}
for i, crd := range crdList {
gvkn := GVKN{GVK: crd.GroupVersionKind(), Name: crd.GetName()}
if _, found := crds[gvkn]; found {
// Currently, we only print a log when there are duplicates. We may want to error out if that makes more sense.
log.Info("there are more than one CRD definitions with the same <Group, Version, Kind, Name>", "GVKN", gvkn)
}
// We always use the CRD definition that we found last.
crds[gvkn] = crdList[i]
}
}
// Converting map to a list to return
var res []runtime.Object
for _, obj := range crds {
res = append(res, obj)
}
return res, nil
}
// readCRDs reads the CRDs from files and Unmarshals them into structs
func readCRDs(basePath string, files []os.FileInfo) ([]*unstructured.Unstructured, error) {
var crds []*unstructured.Unstructured
// White list the file extensions that may contain CRDs
crdExts := sets.NewString(".json", ".yaml", ".yml")
var crds []*apiextensionsv1beta1.CustomResourceDefinition
for _, file := range files {
// Only parse whitelisted file types
// Only parse allowlisted file types
if !crdExts.Has(filepath.Ext(file.Name())) {
continue
}
// Unmarshal CRDs from file into structs
docs, err := readDocuments(filepath.Join(path, file.Name()))
docs, err := readDocuments(filepath.Join(basePath, file.Name()))
if err != nil {
return nil, err
}
for _, doc := range docs {
crd := &apiextensionsv1beta1.CustomResourceDefinition{}
crd := &unstructured.Unstructured{}
if err = yaml.Unmarshal(doc, crd); err != nil {
return nil, err
}
// Check that it is actually a CRD
if crd.Spec.Names.Kind == "" || crd.Spec.Group == "" {
crdKind, _, err := unstructured.NestedString(crd.Object, "spec", "names", "kind")
if err != nil {
return nil, err
}
crdGroup, _, err := unstructured.NestedString(crd.Object, "spec", "group")
if err != nil {
return nil, err
}
if crd.GetKind() != "CustomResourceDefinition" || crdKind == "" || crdGroup == "" {
continue
}
crds = append(crds, crd)

View File

@@ -1,11 +0,0 @@
package envtest
import (
"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
)
// NewlineReporter is Reporter that Prints a newline after the default Reporter output so that the results
// are correctly parsed by test automation.
// See issue https://github.com/jstemmer/go-junit-report/issues/31
// It's re-exported here to avoid compatibility breakage/mass rewrites.
type NewlineReporter = printer.NewlineReporter

View File

@@ -1,6 +1,21 @@
package envtest
import apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
import (
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
)
var (
crdScheme = runtime.NewScheme()
)
// init is required to correctly initialize the crdScheme package variable.
func init() {
_ = apiextensionsv1.AddToScheme(crdScheme)
_ = apiextensionsv1beta1.AddToScheme(crdScheme)
}
// mergePaths merges two string slices containing paths.
// This function makes no guarantees about order of the merged slice.
@@ -23,19 +38,32 @@ func mergePaths(s1, s2 []string) []string {
// mergeCRDs merges two CRD slices using their names.
// This function makes no guarantees about order of the merged slice.
func mergeCRDs(s1, s2 []*apiextensionsv1beta1.CustomResourceDefinition) []*apiextensionsv1beta1.CustomResourceDefinition {
m := make(map[string]*apiextensionsv1beta1.CustomResourceDefinition)
for _, crd := range s1 {
m[crd.Name] = crd
func mergeCRDs(s1, s2 []runtime.Object) []runtime.Object {
m := make(map[string]*unstructured.Unstructured)
for _, obj := range runtimeCRDListToUnstructured(s1) {
m[obj.GetName()] = obj
}
for _, crd := range s2 {
m[crd.Name] = crd
for _, obj := range runtimeCRDListToUnstructured(s2) {
m[obj.GetName()] = obj
}
merged := make([]*apiextensionsv1beta1.CustomResourceDefinition, len(m))
merged := make([]runtime.Object, len(m))
i := 0
for _, crd := range m {
merged[i] = crd
for _, obj := range m {
merged[i] = obj
i++
}
return merged
}
func runtimeCRDListToUnstructured(l []runtime.Object) []*unstructured.Unstructured {
res := []*unstructured.Unstructured{}
for _, obj := range l {
u := &unstructured.Unstructured{}
if err := crdScheme.Convert(obj, u, nil); err != nil {
log.Error(err, "error converting to unstructured object", "object-kind", obj.GetObjectKind())
continue
}
res = append(res, u)
}
return res
}

View File

@@ -1,53 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package printer contains setup for a friendlier Ginkgo printer that's easier
// to parse by test automation.
package printer
import (
"fmt"
"github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/config"
"github.com/onsi/ginkgo/types"
)
var _ ginkgo.Reporter = NewlineReporter{}
// NewlineReporter is Reporter that Prints a newline after the default Reporter output so that the results
// are correctly parsed by test automation.
// See issue https://github.com/jstemmer/go-junit-report/issues/31
type NewlineReporter struct{}
// SpecSuiteWillBegin implements ginkgo.Reporter
func (NewlineReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) {
}
// BeforeSuiteDidRun implements ginkgo.Reporter
func (NewlineReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {}
// AfterSuiteDidRun implements ginkgo.Reporter
func (NewlineReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {}
// SpecWillRun implements ginkgo.Reporter
func (NewlineReporter) SpecWillRun(specSummary *types.SpecSummary) {}
// SpecDidComplete implements ginkgo.Reporter
func (NewlineReporter) SpecDidComplete(specSummary *types.SpecSummary) {}
// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:"
func (NewlineReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { fmt.Printf("\n") }

View File

@@ -23,10 +23,10 @@ import (
"strings"
"time"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/testing_frameworks/integration"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
@@ -72,19 +72,14 @@ func defaultAssetPath(binary string) string {
}
// DefaultKubeAPIServerFlags are default flags necessary to bring up apiserver.
var DefaultKubeAPIServerFlags = []string{
// Allow tests to run offline, by preventing API server from attempting to
// use default route to determine its --advertise-address
"--advertise-address=127.0.0.1",
"--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}",
"--cert-dir={{ .CertDir }}",
"--insecure-port={{ if .URL }}{{ .URL.Port }}{{ end }}",
"--insecure-bind-address={{ if .URL }}{{ .URL.Hostname }}{{ end }}",
"--secure-port={{ if .SecurePort }}{{ .SecurePort }}{{ end }}",
"--admission-control=AlwaysAdmit",
"--service-cluster-ip-range=10.0.0.0/24",
}
// ControlPlane is the re-exported ControlPlane type from the internal integration package
type ControlPlane = integration.ControlPlane
// APIServer is the re-exported APIServer type from the internal integration package
type APIServer = integration.APIServer
// Etcd is the re-exported Etcd type from the internal integration package
type Etcd = integration.Etcd
// Environment creates a Kubernetes test environment that will start / stop the Kubernetes control plane and
// install extension APIs
@@ -100,10 +95,18 @@ type Environment struct {
// CRDInstallOptions are the options for installing CRDs.
CRDInstallOptions CRDInstallOptions
// WebhookInstallOptions are the options for installing webhooks.
WebhookInstallOptions WebhookInstallOptions
// ErrorIfCRDPathMissing provides an interface for the underlying
// CRDInstallOptions.ErrorIfPathMissing. It prevents silent failures
// for missing CRD paths.
ErrorIfCRDPathMissing bool
// CRDs is a list of CRDs to install.
// If both this field and CRDs field in CRDInstallOptions are specified, the
// values are merged.
CRDs []*apiextensionsv1beta1.CustomResourceDefinition
CRDs []runtime.Object
// CRDDirectoryPaths is a list of paths containing CRD yaml or json configs.
// If both this field and Paths field in CRDInstallOptions are specified, the
@@ -135,19 +138,30 @@ type Environment struct {
}
// Stop stops a running server.
// If USE_EXISTING_CLUSTER is set to true, this method is a no-op.
// Previously installed CRDs, as listed in CRDInstallOptions.CRDs, will be uninstalled
// if CRDInstallOptions.CleanUpAfterUse are set to true.
func (te *Environment) Stop() error {
if te.CRDInstallOptions.CleanUpAfterUse {
if err := UninstallCRDs(te.Config, te.CRDInstallOptions); err != nil {
return err
}
}
if te.useExistingCluster() {
return nil
}
err := te.WebhookInstallOptions.Cleanup()
if err != nil {
return err
}
return te.ControlPlane.Stop()
}
// getAPIServerFlags returns flags to be used with the Kubernetes API server.
// it returns empty slice for api server defined defaults to be applied if no args specified
func (te Environment) getAPIServerFlags() []string {
// Set default API server flags if not set.
if len(te.KubeAPIServerFlags) == 0 {
return DefaultKubeAPIServerFlags
return []string{}
}
// Check KubeAPIServerFlags contains service-cluster-ip-range, if not, set default value to service-cluster-ip-range
containServiceClusterIPRange := false
@@ -216,7 +230,7 @@ func (te *Environment) Start() (*rest.Config, error) {
}
if err := te.defaultTimeouts(); err != nil {
return nil, fmt.Errorf("failed to default controlplane timeouts: %v", err)
return nil, fmt.Errorf("failed to default controlplane timeouts: %w", err)
}
te.ControlPlane.Etcd.StartTimeout = te.ControlPlaneStartTimeout
te.ControlPlane.Etcd.StopTimeout = te.ControlPlaneStopTimeout
@@ -240,7 +254,16 @@ func (te *Environment) Start() (*rest.Config, error) {
log.V(1).Info("installing CRDs")
te.CRDInstallOptions.CRDs = mergeCRDs(te.CRDInstallOptions.CRDs, te.CRDs)
te.CRDInstallOptions.Paths = mergePaths(te.CRDInstallOptions.Paths, te.CRDDirectoryPaths)
_, err := InstallCRDs(te.Config, te.CRDInstallOptions)
te.CRDInstallOptions.ErrorIfPathMissing = te.ErrorIfCRDPathMissing
crds, err := InstallCRDs(te.Config, te.CRDInstallOptions)
if err != nil {
return te.Config, err
}
te.CRDs = crds
log.V(1).Info("installing webhooks")
err = te.WebhookInstallOptions.Install(te.Config)
return te.Config, err
}
@@ -256,7 +279,7 @@ func (te *Environment) startControlPlane() error {
log.Error(err, "unable to start the controlplane", "tries", numTries)
}
if numTries == maxRetries {
return fmt.Errorf("failed to start the controlplane. retried %d times: %v", numTries, err)
return fmt.Errorf("failed to start the controlplane. retried %d times: %w", numTries, err)
}
return nil
}
@@ -293,3 +316,7 @@ func (te *Environment) useExistingCluster() bool {
}
return *te.UseExistingCluster
}
// DefaultKubeAPIServerFlags exposes the default args for the APIServer so that
// you can use those to append your own additional arguments.
var DefaultKubeAPIServerFlags = integration.APIServerDefaultArgs

View File

@@ -0,0 +1,440 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package envtest
import (
"context"
"encoding/base64"
"fmt"
"io/ioutil"
"net"
"os"
"path/filepath"
"time"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/addr"
"sigs.k8s.io/yaml"
)
// WebhookInstallOptions are the options for installing mutating or validating webhooks
type WebhookInstallOptions struct {
// Paths is a list of paths to the directories containing the mutating or validating webhooks yaml or json configs.
DirectoryPaths []string
// MutatingWebhooks is a list of MutatingWebhookConfigurations to install
MutatingWebhooks []runtime.Object
// ValidatingWebhooks is a list of ValidatingWebhookConfigurations to install
ValidatingWebhooks []runtime.Object
// IgnoreErrorIfPathMissing will ignore an error if a DirectoryPath does not exist when set to true
IgnoreErrorIfPathMissing bool
// LocalServingHost is the host for serving webhooks on.
// it will be automatically populated
LocalServingHost string
// LocalServingPort is the allocated port for serving webhooks on.
// it will be automatically populated by a random available local port
LocalServingPort int
// LocalServingCertDir is the allocated directory for serving certificates.
// it will be automatically populated by the local temp dir
LocalServingCertDir string
// CAData is the CA that can be used to trust the serving certificates in LocalServingCertDir.
LocalServingCAData []byte
// MaxTime is the max time to wait
MaxTime time.Duration
// PollInterval is the interval to check
PollInterval time.Duration
}
// ModifyWebhookDefinitions modifies webhook definitions by:
// - applying CABundle based on the provided tinyca
// - if webhook client config uses service spec, it's removed and replaced with direct url
func (o *WebhookInstallOptions) ModifyWebhookDefinitions(caData []byte) error {
hostPort, err := o.generateHostPort()
if err != nil {
return err
}
for i, unstructuredHook := range runtimeListToUnstructured(o.MutatingWebhooks) {
webhooks, found, err := unstructured.NestedSlice(unstructuredHook.Object, "webhooks")
if !found || err != nil {
return fmt.Errorf("unexpected object, %v", err)
}
for j := range webhooks {
webhook, err := modifyWebhook(webhooks[j].(map[string]interface{}), caData, hostPort)
if err != nil {
return err
}
webhooks[j] = webhook
unstructuredHook.Object["webhooks"] = webhooks
o.MutatingWebhooks[i] = unstructuredHook
}
}
for i, unstructuredHook := range runtimeListToUnstructured(o.ValidatingWebhooks) {
webhooks, found, err := unstructured.NestedSlice(unstructuredHook.Object, "webhooks")
if !found || err != nil {
return fmt.Errorf("unexpected object, %v", err)
}
for j := range webhooks {
webhook, err := modifyWebhook(webhooks[j].(map[string]interface{}), caData, hostPort)
if err != nil {
return err
}
webhooks[j] = webhook
unstructuredHook.Object["webhooks"] = webhooks
o.ValidatingWebhooks[i] = unstructuredHook
}
}
return nil
}
func modifyWebhook(webhook map[string]interface{}, caData []byte, hostPort string) (map[string]interface{}, error) {
clientConfig, found, err := unstructured.NestedMap(webhook, "clientConfig")
if !found || err != nil {
return nil, fmt.Errorf("cannot find clientconfig: %v", err)
}
clientConfig["caBundle"] = base64.StdEncoding.EncodeToString(caData)
servicePath, found, err := unstructured.NestedString(clientConfig, "service", "path")
if found && err == nil {
// we cannot use service in integration tests since we're running controller outside cluster
// the intent here is that we swap out service for raw address because we don't have an actually standard kube service network.
// We want to users to be able to use your standard config though
url := fmt.Sprintf("https://%s/%s", hostPort, servicePath)
clientConfig["url"] = url
clientConfig["service"] = nil
}
webhook["clientConfig"] = clientConfig
return webhook, nil
}
func (o *WebhookInstallOptions) generateHostPort() (string, error) {
port, host, err := addr.Suggest()
if err != nil {
return "", fmt.Errorf("unable to grab random port for serving webhooks on: %v", err)
}
o.LocalServingPort = port
o.LocalServingHost = host
return net.JoinHostPort(host, fmt.Sprintf("%d", port)), nil
}
// PrepWithoutInstalling does the setup parts of Install (populating host-port,
// setting up CAs, etc), without actually truing to do anything with webhook
// definitions. This is largely useful for internal testing of
// controller-runtime, where we need a random host-port & caData for webhook
// tests, but may be useful in similar scenarios.
func (o *WebhookInstallOptions) PrepWithoutInstalling() error {
hookCA, err := o.setupCA()
if err != nil {
return err
}
if err := parseWebhookDirs(o); err != nil {
return err
}
err = o.ModifyWebhookDefinitions(hookCA)
if err != nil {
return err
}
return nil
}
// Install installs specified webhooks to the API server
func (o *WebhookInstallOptions) Install(config *rest.Config) error {
if err := o.PrepWithoutInstalling(); err != nil {
return err
}
if err := createWebhooks(config, o.MutatingWebhooks, o.ValidatingWebhooks); err != nil {
return err
}
if err := WaitForWebhooks(config, o.MutatingWebhooks, o.ValidatingWebhooks, *o); err != nil {
return err
}
return nil
}
// Cleanup cleans up cert directories
func (o *WebhookInstallOptions) Cleanup() error {
if o.LocalServingCertDir != "" {
return os.RemoveAll(o.LocalServingCertDir)
}
return nil
}
// WaitForWebhooks waits for the Webhooks to be available through API server
func WaitForWebhooks(config *rest.Config,
mutatingWebhooks []runtime.Object,
validatingWebhooks []runtime.Object,
options WebhookInstallOptions) error {
waitingFor := map[schema.GroupVersionKind]*sets.String{}
for _, hook := range runtimeListToUnstructured(append(validatingWebhooks, mutatingWebhooks...)) {
if _, ok := waitingFor[hook.GroupVersionKind()]; !ok {
waitingFor[hook.GroupVersionKind()] = &sets.String{}
}
waitingFor[hook.GroupVersionKind()].Insert(hook.GetName())
}
// Poll until all resources are found in discovery
p := &webhookPoller{config: config, waitingFor: waitingFor}
return wait.PollImmediate(options.PollInterval, options.MaxTime, p.poll)
}
// poller checks if all the resources have been found in discovery, and returns false if not
type webhookPoller struct {
// config is used to get discovery
config *rest.Config
// waitingFor is the map of resources keyed by group version that have not yet been found in discovery
waitingFor map[schema.GroupVersionKind]*sets.String
}
// poll checks if all the resources have been found in discovery, and returns false if not
func (p *webhookPoller) poll() (done bool, err error) {
// Create a new clientset to avoid any client caching of discovery
c, err := client.New(p.config, client.Options{})
if err != nil {
return false, err
}
allFound := true
for gvk, names := range p.waitingFor {
if names.Len() == 0 {
delete(p.waitingFor, gvk)
continue
}
for _, name := range names.List() {
var obj = &unstructured.Unstructured{}
obj.SetGroupVersionKind(gvk)
err := c.Get(context.Background(), client.ObjectKey{
Namespace: "",
Name: name,
}, obj)
if err == nil {
names.Delete(name)
}
if errors.IsNotFound(err) {
allFound = false
}
if err != nil {
return false, err
}
}
}
return allFound, nil
}
// setupCA creates CA for testing and writes them to disk
func (o *WebhookInstallOptions) setupCA() ([]byte, error) {
hookCA, err := integration.NewTinyCA()
if err != nil {
return nil, fmt.Errorf("unable to set up webhook CA: %v", err)
}
hookCert, err := hookCA.NewServingCert()
if err != nil {
return nil, fmt.Errorf("unable to set up webhook serving certs: %v", err)
}
localServingCertsDir, err := ioutil.TempDir("", "envtest-serving-certs-")
o.LocalServingCertDir = localServingCertsDir
if err != nil {
return nil, fmt.Errorf("unable to create directory for webhook serving certs: %v", err)
}
certData, keyData, err := hookCert.AsBytes()
if err != nil {
return nil, fmt.Errorf("unable to marshal webhook serving certs: %v", err)
}
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil {
return nil, fmt.Errorf("unable to write webhook serving cert to disk: %v", err)
}
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil {
return nil, fmt.Errorf("unable to write webhook serving key to disk: %v", err)
}
o.LocalServingCAData = certData
return certData, nil
}
func createWebhooks(config *rest.Config, mutHooks []runtime.Object, valHooks []runtime.Object) error {
cs, err := client.New(config, client.Options{})
if err != nil {
return err
}
// Create each webhook
for _, hook := range runtimeListToUnstructured(mutHooks) {
log.V(1).Info("installing mutating webhook", "webhook", hook.GetName())
if err := ensureCreated(cs, hook); err != nil {
return err
}
}
for _, hook := range runtimeListToUnstructured(valHooks) {
log.V(1).Info("installing validating webhook", "webhook", hook.GetName())
if err := ensureCreated(cs, hook); err != nil {
return err
}
}
return nil
}
// ensureCreated creates or update object if already exists in the cluster
func ensureCreated(cs client.Client, obj *unstructured.Unstructured) error {
existing := obj.DeepCopy()
err := cs.Get(context.Background(), client.ObjectKey{Name: obj.GetName()}, existing)
switch {
case apierrors.IsNotFound(err):
if err := cs.Create(context.Background(), obj); err != nil {
return err
}
case err != nil:
return err
default:
log.V(1).Info("Webhook configuration already exists, updating", "webhook", obj.GetName())
obj.SetResourceVersion(existing.GetResourceVersion())
if err := cs.Update(context.Background(), obj); err != nil {
return err
}
}
return nil
}
// parseWebhookDirs reads the directories of Webhooks in options.DirectoryPaths and adds the Webhook structs to options
func parseWebhookDirs(options *WebhookInstallOptions) error {
if len(options.DirectoryPaths) > 0 {
for _, path := range options.DirectoryPaths {
_, err := os.Stat(path)
if options.IgnoreErrorIfPathMissing && os.IsNotExist(err) {
continue // skip this path
}
if !options.IgnoreErrorIfPathMissing && os.IsNotExist(err) {
return err // treat missing path as error
}
mutHooks, valHooks, err := readWebhooks(path)
if err != nil {
return err
}
options.MutatingWebhooks = append(options.MutatingWebhooks, mutHooks...)
options.ValidatingWebhooks = append(options.ValidatingWebhooks, valHooks...)
}
}
return nil
}
// readWebhooks reads the Webhooks from files and Unmarshals them into structs
// returns slice of mutating and validating webhook configurations
func readWebhooks(path string) ([]runtime.Object, []runtime.Object, error) {
// Get the webhook files
var files []os.FileInfo
var err error
log.V(1).Info("reading Webhooks from path", "path", path)
if files, err = ioutil.ReadDir(path); err != nil {
return nil, nil, err
}
// file extensions that may contain Webhooks
resourceExtensions := sets.NewString(".json", ".yaml", ".yml")
var mutHooks []runtime.Object
var valHooks []runtime.Object
for _, file := range files {
// Only parse allowlisted file types
if !resourceExtensions.Has(filepath.Ext(file.Name())) {
continue
}
// Unmarshal Webhooks from file into structs
docs, err := readDocuments(filepath.Join(path, file.Name()))
if err != nil {
return nil, nil, err
}
for _, doc := range docs {
var generic metav1.PartialObjectMetadata
if err = yaml.Unmarshal(doc, &generic); err != nil {
return nil, nil, err
}
const (
admissionregv1 = "admissionregistration.k8s.io/v1"
admissionregv1beta1 = "admissionregistration.k8s.io/v1beta1"
)
switch {
case generic.Kind == "MutatingWebhookConfiguration":
if generic.APIVersion != admissionregv1beta1 && generic.APIVersion != admissionregv1 {
return nil, nil, fmt.Errorf("only v1beta1 and v1 are supported right now for MutatingWebhookConfiguration (name: %s)", generic.Name)
}
hook := &unstructured.Unstructured{}
if err := yaml.Unmarshal(doc, &hook); err != nil {
return nil, nil, err
}
mutHooks = append(mutHooks, hook)
case generic.Kind == "ValidatingWebhookConfiguration":
if generic.APIVersion != admissionregv1beta1 && generic.APIVersion != admissionregv1 {
return nil, nil, fmt.Errorf("only v1beta1 and v1 are supported right now for ValidatingWebhookConfiguration (name: %s)", generic.Name)
}
hook := &unstructured.Unstructured{}
if err := yaml.Unmarshal(doc, &hook); err != nil {
return nil, nil, err
}
valHooks = append(valHooks, hook)
default:
continue
}
}
log.V(1).Info("read webhooks from file", "file", file.Name())
}
return mutHooks, valHooks, nil
}
func runtimeListToUnstructured(l []runtime.Object) []*unstructured.Unstructured {
res := []*unstructured.Unstructured{}
for _, obj := range l {
m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj.DeepCopyObject())
if err != nil {
continue
}
res = append(res, &unstructured.Unstructured{
Object: m,
})
}
return res
}

View File

@@ -21,25 +21,18 @@ import (
"sync"
"time"
"k8s.io/apimachinery/pkg/runtime"
"github.com/go-logr/logr"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/source"
)
var log = logf.RuntimeLog.WithName("controller")
var _ inject.Injector = &Controller{}
// Controller implements controller.Controller
@@ -55,19 +48,6 @@ type Controller struct {
// Defaults to the DefaultReconcileFunc.
Do reconcile.Reconciler
// Client is a lazily initialized Client. The controllerManager will initialize this when Start is called.
Client client.Client
// Scheme is injected by the controllerManager when controllerManager.Start is called
Scheme *runtime.Scheme
// informers are injected by the controllerManager when controllerManager.Start is called
Cache cache.Cache
// Config is the rest.Config used to talk to the apiserver. Defaults to one of in-cluster, environment variable
// specified, or the ~/.kube/Config.
Config *rest.Config
// MakeQueue constructs the queue for this controller once the controller is ready to start.
// This exists because the standard Kubernetes workqueues start themselves immediately, which
// leads to goroutine leaks if something calls controller.New repeatedly.
@@ -86,21 +66,16 @@ type Controller struct {
// JitterPeriod allows tests to reduce the JitterPeriod so they complete faster
JitterPeriod time.Duration
// WaitForCacheSync allows tests to mock out the WaitForCacheSync function to return an error
// defaults to Cache.WaitForCacheSync
WaitForCacheSync func(stopCh <-chan struct{}) bool
// Started is true if the Controller has been Started
Started bool
// Recorder is an event recorder for recording Event resources to the
// Kubernetes API.
Recorder record.EventRecorder
// TODO(community): Consider initializing a logger with the Controller Name as the tag
// watches maintains a list of sources, handlers, and predicates to start when the controller is started.
watches []watchDescription
// startWatches maintains a list of sources, handlers, and predicates to start when the controller is started.
startWatches []watchDescription
// Log is used to log messages to users during reconciliation, or for example when a watch is started.
Log logr.Logger
}
// watchDescription contains all the information necessary to start a watch.
@@ -133,13 +108,16 @@ func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prc
}
}
c.watches = append(c.watches, watchDescription{src: src, handler: evthdler, predicates: prct})
if c.Started {
log.Info("Starting EventSource", "controller", c.Name, "source", src)
return src.Start(evthdler, c.Queue, prct...)
// Controller hasn't started yet, store the watches locally and return.
//
// These watches are going to be held on the controller struct until the manager or user calls Start(...).
if !c.Started {
c.startWatches = append(c.startWatches, watchDescription{src: src, handler: evthdler, predicates: prct})
return nil
}
return nil
c.Log.Info("Starting EventSource", "source", src)
return src.Start(evthdler, c.Queue, prct...)
}
// Start implements controller.Controller
@@ -160,34 +138,42 @@ func (c *Controller) Start(stop <-chan struct{}) error {
// NB(directxman12): launch the sources *before* trying to wait for the
// caches to sync so that they have a chance to register their intendeded
// caches.
for _, watch := range c.watches {
log.Info("Starting EventSource", "controller", c.Name, "source", watch.src)
for _, watch := range c.startWatches {
c.Log.Info("Starting EventSource", "source", watch.src)
if err := watch.src.Start(watch.handler, c.Queue, watch.predicates...); err != nil {
return err
}
}
// Start the SharedIndexInformer factories to begin populating the SharedIndexInformer caches
log.Info("Starting Controller", "controller", c.Name)
c.Log.Info("Starting Controller")
// Wait for the caches to be synced before starting workers
if c.WaitForCacheSync == nil {
c.WaitForCacheSync = c.Cache.WaitForCacheSync
}
if ok := c.WaitForCacheSync(stop); !ok {
// This code is unreachable right now since WaitForCacheSync will never return an error
// Leaving it here because that could happen in the future
err := fmt.Errorf("failed to wait for %s caches to sync", c.Name)
log.Error(err, "Could not wait for Cache to sync", "controller", c.Name)
return err
for _, watch := range c.startWatches {
syncingSource, ok := watch.src.(source.SyncingSource)
if !ok {
continue
}
if err := syncingSource.WaitForSync(stop); err != nil {
// This code is unreachable in case of kube watches since WaitForCacheSync will never return an error
// Leaving it here because that could happen in the future
err := fmt.Errorf("failed to wait for %s caches to sync: %w", c.Name, err)
c.Log.Error(err, "Could not wait for Cache to sync")
return err
}
}
// All the watches have been started, we can reset the local slice.
//
// We should never hold watches more than necessary, each watch source can hold a backing cache,
// which won't be garbage collected if we hold a reference to it.
c.startWatches = nil
if c.JitterPeriod == 0 {
c.JitterPeriod = 1 * time.Second
}
// Launch workers to process resources
log.Info("Starting workers", "controller", c.Name, "worker count", c.MaxConcurrentReconciles)
c.Log.Info("Starting workers", "worker count", c.MaxConcurrentReconciles)
for i := 0; i < c.MaxConcurrentReconciles; i++ {
// Process work items
go wait.Until(c.worker, c.JitterPeriod, stop)
@@ -201,7 +187,7 @@ func (c *Controller) Start(stop <-chan struct{}) error {
}
<-stop
log.Info("Stopping workers", "controller", c.Name)
c.Log.Info("Stopping workers")
return nil
}
@@ -236,26 +222,28 @@ func (c *Controller) reconcileHandler(obj interface{}) bool {
// Update metrics after processing each item
reconcileStartTS := time.Now()
defer func() {
c.updateMetrics(time.Now().Sub(reconcileStartTS))
c.updateMetrics(time.Since(reconcileStartTS))
}()
var req reconcile.Request
var ok bool
if req, ok = obj.(reconcile.Request); !ok {
// Make sure that the the object is a valid request.
req, ok := obj.(reconcile.Request)
if !ok {
// As the item in the workqueue is actually invalid, we call
// Forget here else we'd go into a loop of attempting to
// process a work item that is invalid.
c.Queue.Forget(obj)
log.Error(nil, "Queue item was not a Request",
"controller", c.Name, "type", fmt.Sprintf("%T", obj), "value", obj)
c.Log.Error(nil, "Queue item was not a Request", "type", fmt.Sprintf("%T", obj), "value", obj)
// Return true, don't take a break
return true
}
log := c.Log.WithValues("name", req.Name, "namespace", req.Namespace)
// RunInformersAndControllers the syncHandler, passing it the namespace/Name string of the
// resource to be synced.
if result, err := c.Do.Reconcile(req); err != nil {
c.Queue.AddRateLimited(req)
log.Error(err, "Reconciler error", "controller", c.Name, "request", req)
log.Error(err, "Reconciler error")
ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Inc()
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "error").Inc()
return false
@@ -279,7 +267,7 @@ func (c *Controller) reconcileHandler(obj interface{}) bool {
c.Queue.Forget(obj)
// TODO(directxman12): What does 1 mean? Do we want level constants? Do we want levels at all?
log.V(1).Info("Successfully Reconciled", "controller", c.Name, "request", req)
log.V(1).Info("Successfully Reconciled")
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "success").Inc()
// Return true, don't take a break

View File

@@ -42,7 +42,7 @@ type provider struct {
func NewProvider(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, broadcaster record.EventBroadcaster) (recorder.Provider, error) {
clientSet, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, fmt.Errorf("failed to init clientSet: %v", err)
return nil, fmt.Errorf("failed to init clientSet: %w", err)
}
p := &provider{scheme: scheme, logger: logger, eventBroadcaster: broadcaster}

View File

@@ -0,0 +1 @@
assets/bin

View File

@@ -0,0 +1,10 @@
# Integration Testing Framework
This package has been moved from [https://github.com/kubernetes-sigs/testing_frameworks/tree/master/integration](https://github.com/kubernetes-sigs/testing_frameworks/tree/master/integration).
A framework for integration testing components of kubernetes. This framework is
intended to work properly both in CI, and on a local dev machine. It therefore
explicitly supports both Linux and Darwin.
For detailed documentation see the
[![GoDoc](https://godoc.org/github.com/kubernetes-sigs/controller-runtime/pkg/internal/testing/integration?status.svg)](https://godoc.org/github.com/kubernetes-sigs/controller-runtime/pkg/internal/testing/integration).

View File

@@ -0,0 +1,74 @@
package addr
import (
"fmt"
"net"
"sync"
"time"
)
const (
portReserveTime = 1 * time.Minute
portConflictRetry = 100
)
type portCache struct {
lock sync.Mutex
ports map[int]time.Time
}
func (c *portCache) add(port int) bool {
c.lock.Lock()
defer c.lock.Unlock()
// remove outdated port
for p, t := range c.ports {
if time.Since(t) > portReserveTime {
delete(c.ports, p)
}
}
// try allocating new port
if _, ok := c.ports[port]; ok {
return false
}
c.ports[port] = time.Now()
return true
}
var cache = &portCache{
ports: make(map[int]time.Time),
}
func suggest() (port int, resolvedHost string, err error) {
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
if err != nil {
return
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return
}
port = l.Addr().(*net.TCPAddr).Port
defer func() {
err = l.Close()
}()
resolvedHost = addr.IP.String()
return
}
// Suggest suggests an address a process can listen on. It returns
// a tuple consisting of a free port and the hostname resolved to its IP.
// It makes sure that new port allocated does not conflict with old ports
// allocated within 1 minute.
func Suggest() (port int, resolvedHost string, err error) {
for i := 0; i < portConflictRetry; i++ {
port, resolvedHost, err = suggest()
if err != nil {
return
}
if cache.add(port) {
return
}
}
err = fmt.Errorf("no free ports found after %d retries", portConflictRetry)
return
}

View File

@@ -0,0 +1,177 @@
package integration
import (
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"path/filepath"
"time"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/addr"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/internal"
)
// APIServer knows how to run a kubernetes apiserver.
type APIServer struct {
// URL is the address the ApiServer should listen on for client connections.
//
// If this is not specified, we default to a random free port on localhost.
URL *url.URL
// SecurePort is the additional secure port that the APIServer should listen on.
SecurePort int
// Path is the path to the apiserver binary.
//
// If this is left as the empty string, we will attempt to locate a binary,
// by checking for the TEST_ASSET_KUBE_APISERVER environment variable, and
// the default test assets directory. See the "Binaries" section above (in
// doc.go) for details.
Path string
// Args is a list of arguments which will passed to the APIServer binary.
// Before they are passed on, they will be evaluated as go-template strings.
// This means you can use fields which are defined and exported on this
// APIServer struct (e.g. "--cert-dir={{ .Dir }}").
// Those templates will be evaluated after the defaulting of the APIServer's
// fields has already happened and just before the binary actually gets
// started. Thus you have access to calculated fields like `URL` and others.
//
// If not specified, the minimal set of arguments to run the APIServer will
// be used.
Args []string
// CertDir is a path to a directory containing whatever certificates the
// APIServer will need.
//
// If left unspecified, then the Start() method will create a fresh temporary
// directory, and the Stop() method will clean it up.
CertDir string
// EtcdURL is the URL of the Etcd the APIServer should use.
//
// If this is not specified, the Start() method will return an error.
EtcdURL *url.URL
// StartTimeout, StopTimeout specify the time the APIServer is allowed to
// take when starting and stoppping before an error is emitted.
//
// If not specified, these default to 20 seconds.
StartTimeout time.Duration
StopTimeout time.Duration
// Out, Err specify where APIServer should write its StdOut, StdErr to.
//
// If not specified, the output will be discarded.
Out io.Writer
Err io.Writer
processState *internal.ProcessState
}
// Start starts the apiserver, waits for it to come up, and returns an error,
// if occurred.
func (s *APIServer) Start() error {
if s.processState == nil {
if err := s.setProcessState(); err != nil {
return err
}
}
return s.processState.Start(s.Out, s.Err)
}
func (s *APIServer) setProcessState() error {
if s.EtcdURL == nil {
return fmt.Errorf("expected EtcdURL to be configured")
}
var err error
s.processState = &internal.ProcessState{}
s.processState.DefaultedProcessInput, err = internal.DoDefaulting(
"kube-apiserver",
s.URL,
s.CertDir,
s.Path,
s.StartTimeout,
s.StopTimeout,
)
if err != nil {
return err
}
// Defaulting the secure port
if s.SecurePort == 0 {
s.SecurePort, _, err = addr.Suggest()
if err != nil {
return err
}
}
s.processState.HealthCheckEndpoint = "/healthz"
s.URL = &s.processState.URL
s.CertDir = s.processState.Dir
s.Path = s.processState.Path
s.StartTimeout = s.processState.StartTimeout
s.StopTimeout = s.processState.StopTimeout
if err := s.populateAPIServerCerts(); err != nil {
return err
}
s.processState.Args, err = internal.RenderTemplates(
internal.DoAPIServerArgDefaulting(s.Args), s,
)
return err
}
func (s *APIServer) populateAPIServerCerts() error {
_, statErr := os.Stat(filepath.Join(s.CertDir, "apiserver.crt"))
if !os.IsNotExist(statErr) {
return statErr
}
ca, err := internal.NewTinyCA()
if err != nil {
return err
}
certs, err := ca.NewServingCert()
if err != nil {
return err
}
certData, keyData, err := certs.AsBytes()
if err != nil {
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0640); err != nil {
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0640); err != nil {
return err
}
return nil
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the CertDir if necessary.
func (s *APIServer) Stop() error {
if s.processState != nil {
return s.processState.Stop()
}
return nil
}
// APIServerDefaultArgs exposes the default args for the APIServer so that you
// can use those to append your own additional arguments.
//
// The internal default arguments are explicitly copied here, we don't want to
// allow users to change the internal ones.
var APIServerDefaultArgs = append([]string{}, internal.APIServerDefaultArgs...)

View File

@@ -0,0 +1,86 @@
package integration
import (
"fmt"
"net/url"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/internal"
)
// NewTinyCA creates a new a tiny CA utility for provisioning serving certs and client certs FOR TESTING ONLY.
// Don't use this for anything else!
var NewTinyCA = internal.NewTinyCA
// ControlPlane is a struct that knows how to start your test control plane.
//
// Right now, that means Etcd and your APIServer. This is likely to increase in
// future.
type ControlPlane struct {
APIServer *APIServer
Etcd *Etcd
}
// Start will start your control plane processes. To stop them, call Stop().
func (f *ControlPlane) Start() error {
if f.Etcd == nil {
f.Etcd = &Etcd{}
}
if err := f.Etcd.Start(); err != nil {
return err
}
if f.APIServer == nil {
f.APIServer = &APIServer{}
}
f.APIServer.EtcdURL = f.Etcd.URL
return f.APIServer.Start()
}
// Stop will stop your control plane processes, and clean up their data.
func (f *ControlPlane) Stop() error {
var errList []error
if f.APIServer != nil {
if err := f.APIServer.Stop(); err != nil {
errList = append(errList, err)
}
}
if f.Etcd != nil {
if err := f.Etcd.Stop(); err != nil {
errList = append(errList, err)
}
}
return utilerrors.NewAggregate(errList)
}
// APIURL returns the URL you should connect to to talk to your API.
func (f *ControlPlane) APIURL() *url.URL {
return f.APIServer.URL
}
// KubeCtl returns a pre-configured KubeCtl, ready to connect to this
// ControlPlane.
func (f *ControlPlane) KubeCtl() *KubeCtl {
k := &KubeCtl{}
k.Opts = append(k.Opts, fmt.Sprintf("--server=%s", f.APIURL()))
return k
}
// RESTClientConfig returns a pre-configured restconfig, ready to connect to
// this ControlPlane.
func (f *ControlPlane) RESTClientConfig() (*rest.Config, error) {
c := &rest.Config{
Host: f.APIURL().String(),
ContentConfig: rest.ContentConfig{
NegotiatedSerializer: serializer.WithoutConversionCodecFactory{CodecFactory: scheme.Codecs},
},
}
err := rest.SetKubernetesDefaults(c)
return c, err
}

View File

@@ -0,0 +1,112 @@
/*
Package integration implements an integration testing framework for kubernetes.
It provides components for standing up a kubernetes API, against which you can test a
kubernetes client, or other kubernetes components. The lifecycle of the components
needed to provide this API is managed by this framework.
Quickstart
Add something like the following to
your tests:
cp := &integration.ControlPlane{}
cp.Start()
kubeCtl := cp.KubeCtl()
stdout, stderr, err := kubeCtl.Run("get", "pods")
// You can check on err, stdout & stderr and build up
// your tests
cp.Stop()
Components
Currently the framework provides the following components:
ControlPlane: The ControlPlane wraps Etcd & APIServer (see below) and wires
them together correctly. A ControlPlane can be stopped & started and can
provide the URL to connect to the API. The ControlPlane can also be asked for a
KubeCtl which is already correctly configured for this ControlPlane. The
ControlPlane is a good entry point for default setups.
Etcd: Manages an Etcd binary, which can be started, stopped and connected to.
By default Etcd will listen on a random port for http connections and will
create a temporary directory for its data. To configure it differently, see the
Etcd type documentation below.
APIServer: Manages an Kube-APIServer binary, which can be started, stopped and
connected to. By default APIServer will listen on a random port for http
connections and will create a temporary directory to store the (auto-generated)
certificates. To configure it differently, see the APIServer type
documentation below.
KubeCtl: Wraps around a `kubectl` binary and can `Run(...)` arbitrary commands
against a kubernetes control plane.
Binaries
Etcd, APIServer & KubeCtl use the same mechanism to determine which binaries to
use when they get started.
1. If the component is configured with a `Path` the framework tries to run that
binary.
For example:
myEtcd := &Etcd{
Path: "/some/other/etcd",
}
cp := &integration.ControlPlane{
Etcd: myEtcd,
}
cp.Start()
2. If the Path field on APIServer, Etcd or KubeCtl is left unset and an
environment variable named `TEST_ASSET_KUBE_APISERVER`, `TEST_ASSET_ETCD` or
`TEST_ASSET_KUBECTL` is set, its value is used as a path to the binary for the
APIServer, Etcd or KubeCtl.
3. If neither the `Path` field, nor the environment variable is set, the
framework tries to use the binaries `kube-apiserver`, `etcd` or `kubectl` in
the directory `${FRAMEWORK_DIR}/assets/bin/`.
Arguments for Etcd and APIServer
Those components will start without any configuration. However, if you want or
need to, you can override certain configuration -- one of which are the
arguments used when calling the binary.
When you choose to specify your own set of arguments, those won't be appended
to the default set of arguments, it is your responsibility to provide all the
arguments needed for the binary to start successfully.
However, the default arguments for APIServer and Etcd are exported as
`APIServerDefaultArgs` and `EtcdDefaultArgs` from this package. Treat those
variables as read-only constants. Internally we have a set of default
arguments for defaulting, the `APIServerDefaultArgs` and `EtcdDefaultArgs` are
just copies of those. So when you override them you loose access to the actual
internal default arguments, but your override won't affect the defaulting.
All arguments are interpreted as go templates. Those templates have access to
all exported fields of the `APIServer`/`Etcd` struct. It does not matter if
those fields where explicitly set up or if they were defaulted by calling the
`Start()` method, the template evaluation runs just before the binary is
executed and right after the defaulting of all the struct's fields has
happened.
// When you want to append additional arguments ...
etcd := &Etcd{
// Additional custom arguments will appended to the set of default
// arguments
Args: append(EtcdDefaultArgs, "--additional=arg"),
DataDir: "/my/special/data/dir",
}
// When you want to use a custom set of arguments ...
etcd := &Etcd{
// Only custom arguments will be passed to the binary
Args: []string{"--one=1", "--two=2", "--three=3"},
DataDir: "/my/special/data/dir",
}
*/
package integration

View File

@@ -0,0 +1,114 @@
package integration
import (
"io"
"time"
"net/url"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/internal"
)
// Etcd knows how to run an etcd server.
type Etcd struct {
// URL is the address the Etcd should listen on for client connections.
//
// If this is not specified, we default to a random free port on localhost.
URL *url.URL
// Path is the path to the etcd binary.
//
// If this is left as the empty string, we will attempt to locate a binary,
// by checking for the TEST_ASSET_ETCD environment variable, and the default
// test assets directory. See the "Binaries" section above (in doc.go) for
// details.
Path string
// Args is a list of arguments which will passed to the Etcd binary. Before
// they are passed on, the`y will be evaluated as go-template strings. This
// means you can use fields which are defined and exported on this Etcd
// struct (e.g. "--data-dir={{ .Dir }}").
// Those templates will be evaluated after the defaulting of the Etcd's
// fields has already happened and just before the binary actually gets
// started. Thus you have access to calculated fields like `URL` and others.
//
// If not specified, the minimal set of arguments to run the Etcd will be
// used.
Args []string
// DataDir is a path to a directory in which etcd can store its state.
//
// If left unspecified, then the Start() method will create a fresh temporary
// directory, and the Stop() method will clean it up.
DataDir string
// StartTimeout, StopTimeout specify the time the Etcd is allowed to
// take when starting and stopping before an error is emitted.
//
// If not specified, these default to 20 seconds.
StartTimeout time.Duration
StopTimeout time.Duration
// Out, Err specify where Etcd should write its StdOut, StdErr to.
//
// If not specified, the output will be discarded.
Out io.Writer
Err io.Writer
processState *internal.ProcessState
}
// Start starts the etcd, waits for it to come up, and returns an error, if one
// occoured.
func (e *Etcd) Start() error {
if e.processState == nil {
if err := e.setProcessState(); err != nil {
return err
}
}
return e.processState.Start(e.Out, e.Err)
}
func (e *Etcd) setProcessState() error {
var err error
e.processState = &internal.ProcessState{}
e.processState.DefaultedProcessInput, err = internal.DoDefaulting(
"etcd",
e.URL,
e.DataDir,
e.Path,
e.StartTimeout,
e.StopTimeout,
)
if err != nil {
return err
}
e.processState.StartMessage = internal.GetEtcdStartMessage(e.processState.URL)
e.URL = &e.processState.URL
e.DataDir = e.processState.Dir
e.Path = e.processState.Path
e.StartTimeout = e.processState.StartTimeout
e.StopTimeout = e.processState.StopTimeout
e.processState.Args, err = internal.RenderTemplates(
internal.DoEtcdArgDefaulting(e.Args), e,
)
return err
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the DataDir if necessary.
func (e *Etcd) Stop() error {
return e.processState.Stop()
}
// EtcdDefaultArgs exposes the default args for Etcd so that you
// can use those to append your own additional arguments.
//
// The internal default arguments are explicitly copied here, we don't want to
// allow users to change the internal ones.
var EtcdDefaultArgs = append([]string{}, internal.EtcdDefaultArgs...)

View File

@@ -0,0 +1,27 @@
package internal
// APIServerDefaultArgs allow tests to run offline, by preventing API server from attempting to
// use default route to determine its --advertise-address.
var APIServerDefaultArgs = []string{
"--advertise-address=127.0.0.1",
"--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}",
"--cert-dir={{ .CertDir }}",
"--insecure-port={{ if .URL }}{{ .URL.Port }}{{ end }}",
"--insecure-bind-address={{ if .URL }}{{ .URL.Hostname }}{{ end }}",
"--secure-port={{ if .SecurePort }}{{ .SecurePort }}{{ end }}",
// we're keeping this disabled because if enabled, default SA is missing which would force all tests to create one
// in normal apiserver operation this SA is created by controller, but that is not run in integration environment
"--disable-admission-plugins=ServiceAccount",
"--service-cluster-ip-range=10.0.0.0/24",
"--allow-privileged=true",
}
// DoAPIServerArgDefaulting will set default values to allow tests to run offline when the args are not informed. Otherwise,
// it will return the same []string arg passed as param.
func DoAPIServerArgDefaulting(args []string) []string {
if len(args) != 0 {
return args
}
return APIServerDefaultArgs
}

View File

@@ -0,0 +1,29 @@
package internal
import (
"bytes"
"html/template"
)
// RenderTemplates returns an []string to render the templates
func RenderTemplates(argTemplates []string, data interface{}) (args []string, err error) {
var t *template.Template
for _, arg := range argTemplates {
t, err = template.New(arg).Parse(arg)
if err != nil {
args = nil
return
}
buf := &bytes.Buffer{}
err = t.Execute(buf, data)
if err != nil {
args = nil
return
}
args = append(args, buf.String())
}
return
}

View File

@@ -0,0 +1,35 @@
package internal
import (
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
)
var assetsPath string
func init() {
_, thisFile, _, ok := runtime.Caller(0)
if !ok {
panic("Could not determine the path of the BinPathFinder")
}
assetsPath = filepath.Join(filepath.Dir(thisFile), "..", "assets", "bin")
}
// BinPathFinder checks the an environment variable, derived from the symbolic name,
// and falls back to a default assets location when this variable is not set
func BinPathFinder(symbolicName string) (binPath string) {
punctuationPattern := regexp.MustCompile("[^A-Z0-9]+")
sanitizedName := punctuationPattern.ReplaceAllString(strings.ToUpper(symbolicName), "_")
leadingNumberPattern := regexp.MustCompile("^[0-9]+")
sanitizedName = leadingNumberPattern.ReplaceAllString(sanitizedName, "")
envVar := "TEST_ASSET_" + sanitizedName
if val, ok := os.LookupEnv(envVar); ok {
return val
}
return filepath.Join(assetsPath, symbolicName)
}

View File

@@ -0,0 +1,45 @@
package internal
import (
"net/url"
)
// EtcdDefaultArgs allow tests to run offline, by preventing API server from attempting to
// use default route to determine its urls.
var EtcdDefaultArgs = []string{
"--listen-peer-urls=http://localhost:0",
"--advertise-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}",
"--listen-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}",
"--data-dir={{ .DataDir }}",
}
// DoEtcdArgDefaulting will set default values to allow tests to run offline when the args are not informed. Otherwise,
// it will return the same []string arg passed as param.
func DoEtcdArgDefaulting(args []string) []string {
if len(args) != 0 {
return args
}
return EtcdDefaultArgs
}
// isSecureScheme returns false when the schema is insecure.
func isSecureScheme(scheme string) bool {
// https://github.com/coreos/etcd/blob/d9deeff49a080a88c982d328ad9d33f26d1ad7b6/pkg/transport/listener.go#L53
if scheme == "https" || scheme == "unixs" {
return true
}
return false
}
// GetEtcdStartMessage returns an start message to inform if the client is or not insecure.
// It will return true when the URL informed has the scheme == "https" || scheme == "unixs"
func GetEtcdStartMessage(listenURL url.URL) string {
if isSecureScheme(listenURL.Scheme) {
// https://github.com/coreos/etcd/blob/a7f1fbe00ec216fcb3a1919397a103b41dca8413/embed/serve.go#L167
return "serving client requests on "
}
// https://github.com/coreos/etcd/blob/a7f1fbe00ec216fcb3a1919397a103b41dca8413/embed/serve.go#L124
return "serving insecure client requests on "
}

View File

@@ -0,0 +1,225 @@
package internal
import (
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"strconv"
"time"
"github.com/onsi/gomega/gbytes"
"github.com/onsi/gomega/gexec"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/addr"
)
// ProcessState define the state of the process.
type ProcessState struct {
DefaultedProcessInput
Session *gexec.Session
// Healthcheck Endpoint. If we get http.StatusOK from this endpoint, we
// assume the process is ready to operate. E.g. "/healthz". If this is set,
// we ignore StartMessage.
HealthCheckEndpoint string
// HealthCheckPollInterval is the interval which will be used for polling the
// HealthCheckEndpoint.
// If left empty it will default to 100 Milliseconds.
HealthCheckPollInterval time.Duration
// StartMessage is the message to wait for on stderr. If we receive this
// message, we assume the process is ready to operate. Ignored if
// HealthCheckEndpoint is specified.
//
// The usage of StartMessage is discouraged, favour HealthCheckEndpoint
// instead!
//
// Deprecated: Use HealthCheckEndpoint in favour of StartMessage
StartMessage string
Args []string
// ready holds wether the process is currently in ready state (hit the ready condition) or not.
// It will be set to true on a successful `Start()` and set to false on a successful `Stop()`
ready bool
}
// DefaultedProcessInput defines the default process input required to perform the test.
type DefaultedProcessInput struct {
URL url.URL
Dir string
DirNeedsCleaning bool
Path string
StopTimeout time.Duration
StartTimeout time.Duration
}
// DoDefaulting sets the default configuration according to the data informed and return an DefaultedProcessInput
// and an error if some requirement was not informed.
func DoDefaulting(
name string,
listenURL *url.URL,
dir string,
path string,
startTimeout time.Duration,
stopTimeout time.Duration,
) (DefaultedProcessInput, error) {
defaults := DefaultedProcessInput{
Dir: dir,
Path: path,
StartTimeout: startTimeout,
StopTimeout: stopTimeout,
}
if listenURL == nil {
port, host, err := addr.Suggest()
if err != nil {
return DefaultedProcessInput{}, err
}
defaults.URL = url.URL{
Scheme: "http",
Host: net.JoinHostPort(host, strconv.Itoa(port)),
}
} else {
defaults.URL = *listenURL
}
if dir == "" {
newDir, err := ioutil.TempDir("", "k8s_test_framework_")
if err != nil {
return DefaultedProcessInput{}, err
}
defaults.Dir = newDir
defaults.DirNeedsCleaning = true
}
if path == "" {
if name == "" {
return DefaultedProcessInput{}, fmt.Errorf("must have at least one of name or path")
}
defaults.Path = BinPathFinder(name)
}
if startTimeout == 0 {
defaults.StartTimeout = 20 * time.Second
}
if stopTimeout == 0 {
defaults.StopTimeout = 20 * time.Second
}
return defaults, nil
}
type stopChannel chan struct{}
// Start starts the apiserver, waits for it to come up, and returns an error,
// if occurred.
func (ps *ProcessState) Start(stdout, stderr io.Writer) (err error) {
if ps.ready {
return nil
}
command := exec.Command(ps.Path, ps.Args...)
ready := make(chan bool)
timedOut := time.After(ps.StartTimeout)
var pollerStopCh stopChannel
if ps.HealthCheckEndpoint != "" {
healthCheckURL := ps.URL
healthCheckURL.Path = ps.HealthCheckEndpoint
pollerStopCh = make(stopChannel)
go pollURLUntilOK(healthCheckURL, ps.HealthCheckPollInterval, ready, pollerStopCh)
} else {
startDetectStream := gbytes.NewBuffer()
ready = startDetectStream.Detect(ps.StartMessage)
stderr = safeMultiWriter(stderr, startDetectStream)
}
ps.Session, err = gexec.Start(command, stdout, stderr)
if err != nil {
return err
}
select {
case <-ready:
ps.ready = true
return nil
case <-timedOut:
if pollerStopCh != nil {
close(pollerStopCh)
}
if ps.Session != nil {
ps.Session.Terminate()
}
return fmt.Errorf("timeout waiting for process %s to start", path.Base(ps.Path))
}
}
func safeMultiWriter(writers ...io.Writer) io.Writer {
safeWriters := []io.Writer{}
for _, w := range writers {
if w != nil {
safeWriters = append(safeWriters, w)
}
}
return io.MultiWriter(safeWriters...)
}
func pollURLUntilOK(url url.URL, interval time.Duration, ready chan bool, stopCh stopChannel) {
if interval <= 0 {
interval = 100 * time.Millisecond
}
for {
res, err := http.Get(url.String())
if err == nil {
res.Body.Close()
if res.StatusCode == http.StatusOK {
ready <- true
return
}
}
select {
case <-stopCh:
return
default:
time.Sleep(interval)
}
}
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the CertDir if necessary.
func (ps *ProcessState) Stop() error {
if ps.Session == nil {
return nil
}
// gexec's Session methods (Signal, Kill, ...) do not check if the Process is
// nil, so we are doing this here for now.
// This should probably be fixed in gexec.
if ps.Session.Command.Process == nil {
return nil
}
detectedStop := ps.Session.Terminate().Exited
timedOut := time.After(ps.StopTimeout)
select {
case <-detectedStop:
break
case <-timedOut:
return fmt.Errorf("timeout waiting for process %s to stop", path.Base(ps.Path))
}
ps.ready = false
if ps.DirNeedsCleaning {
return os.RemoveAll(ps.Dir)
}
return nil
}

View File

@@ -0,0 +1,151 @@
package internal
// NB(directxman12): nothing has verified that this has good settings. In fact,
// the setting generated here are probably terrible, but they're fine for integration
// tests. These ABSOLUTELY SHOULD NOT ever be exposed in the public API. They're
// ONLY for use with envtest's ability to configure webhook testing.
// If I didn't otherwise not want to add a dependency on cfssl, I'd just use that.
import (
"crypto"
crand "crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"math/big"
"net"
"time"
certutil "k8s.io/client-go/util/cert"
)
var (
rsaKeySize = 2048 // a decent number, as of 2019
bigOne = big.NewInt(1)
)
// CertPair is a private key and certificate for use for client auth, as a CA, or serving.
type CertPair struct {
Key crypto.Signer
Cert *x509.Certificate
}
// CertBytes returns the PEM-encoded version of the certificate for this pair.
func (k CertPair) CertBytes() []byte {
return pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: k.Cert.Raw,
})
}
// AsBytes encodes keypair in the appropriate formats for on-disk storage (PEM and
// PKCS8, respectively).
func (k CertPair) AsBytes() (cert []byte, key []byte, err error) {
cert = k.CertBytes()
rawKeyData, err := x509.MarshalPKCS8PrivateKey(k.Key)
if err != nil {
return nil, nil, fmt.Errorf("unable to encode private key: %v", err)
}
key = pem.EncodeToMemory(&pem.Block{
Type: "PRIVATE KEY",
Bytes: rawKeyData,
})
return cert, key, nil
}
// TinyCA supports signing serving certs and client-certs,
// and can be used as an auth mechanism with envtest.
type TinyCA struct {
CA CertPair
orgName string
nextSerial *big.Int
}
// newPrivateKey generates a new private key of a relatively sane size (see
// rsaKeySize).
func newPrivateKey() (crypto.Signer, error) {
return rsa.GenerateKey(crand.Reader, rsaKeySize)
}
// NewTinyCA creates a new a tiny CA utility for provisioning serving certs and client certs FOR TESTING ONLY.
// Don't use this for anything else!
func NewTinyCA() (*TinyCA, error) {
caPrivateKey, err := newPrivateKey()
if err != nil {
return nil, fmt.Errorf("unable to generate private key for CA: %v", err)
}
caCfg := certutil.Config{CommonName: "envtest-environment", Organization: []string{"envtest"}}
caCert, err := certutil.NewSelfSignedCACert(caCfg, caPrivateKey)
if err != nil {
return nil, fmt.Errorf("unable to generate certificate for CA: %v", err)
}
return &TinyCA{
CA: CertPair{Key: caPrivateKey, Cert: caCert},
orgName: "envtest",
nextSerial: big.NewInt(1),
}, nil
}
func (c *TinyCA) makeCert(cfg certutil.Config) (CertPair, error) {
now := time.Now()
key, err := newPrivateKey()
if err != nil {
return CertPair{}, fmt.Errorf("unable to create private key: %v", err)
}
serial := new(big.Int).Set(c.nextSerial)
c.nextSerial.Add(c.nextSerial, bigOne)
template := x509.Certificate{
Subject: pkix.Name{CommonName: cfg.CommonName, Organization: cfg.Organization},
DNSNames: cfg.AltNames.DNSNames,
IPAddresses: cfg.AltNames.IPs,
SerialNumber: serial,
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: cfg.Usages,
// technically not necessary for testing, but let's set anyway just in case.
NotBefore: now.UTC(),
// 1 week -- the default for cfssl, and just long enough for a
// long-term test, but not too long that anyone would try to use this
// seriously.
NotAfter: now.Add(168 * time.Hour).UTC(),
}
certRaw, err := x509.CreateCertificate(crand.Reader, &template, c.CA.Cert, key.Public(), c.CA.Key)
if err != nil {
return CertPair{}, fmt.Errorf("unable to create certificate: %v", err)
}
cert, err := x509.ParseCertificate(certRaw)
if err != nil {
return CertPair{}, fmt.Errorf("generated invalid certificate, could not parse: %v", err)
}
return CertPair{
Key: key,
Cert: cert,
}, nil
}
// NewServingCert returns a new CertPair for a serving HTTPS on localhost.
func (c *TinyCA) NewServingCert() (CertPair, error) {
return c.makeCert(certutil.Config{
CommonName: "localhost",
Organization: []string{c.orgName},
AltNames: certutil.AltNames{
DNSNames: []string{"localhost"},
IPs: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
})
}

View File

@@ -0,0 +1,47 @@
package integration
import (
"bytes"
"io"
"os/exec"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/internal"
)
// KubeCtl is a wrapper around the kubectl binary.
type KubeCtl struct {
// Path where the kubectl binary can be found.
//
// If this is left empty, we will attempt to locate a binary, by checking for
// the TEST_ASSET_KUBECTL environment variable, and the default test assets
// directory. See the "Binaries" section above (in doc.go) for details.
Path string
// Opts can be used to configure additional flags which will be used each
// time the wrapped binary is called.
//
// For example, you might want to use this to set the URL of the APIServer to
// connect to.
Opts []string
}
// Run executes the wrapped binary with some preconfigured options and the
// arguments given to this method. It returns Readers for the stdout and
// stderr.
func (k *KubeCtl) Run(args ...string) (stdout, stderr io.Reader, err error) {
if k.Path == "" {
k.Path = internal.BinPathFinder("kubectl")
}
stdoutBuffer := &bytes.Buffer{}
stderrBuffer := &bytes.Buffer{}
allArgs := append(k.Opts, args...)
cmd := exec.Command(k.Path, allArgs...)
cmd.Stdout = stdoutBuffer
cmd.Stderr = stderrBuffer
err = cmd.Run()
return stdoutBuffer, stderrBuffer, err
}

View File

@@ -17,6 +17,7 @@ limitations under the License.
package leaderelection
import (
"errors"
"fmt"
"io/ioutil"
"os"
@@ -52,9 +53,9 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
return nil, nil
}
// Default the LeaderElectionID
// LeaderElectionID must be provided to prevent clashes
if options.LeaderElectionID == "" {
options.LeaderElectionID = "controller-leader-election-helper"
return nil, errors.New("LeaderElectionID must be configured")
}
// Default the namespace (if running in cluster)
@@ -62,7 +63,7 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
var err error
options.LeaderElectionNamespace, err = getInClusterNamespace()
if err != nil {
return nil, fmt.Errorf("unable to find leader election namespace: %v", err)
return nil, fmt.Errorf("unable to find leader election namespace: %w", err)
}
}
@@ -74,7 +75,7 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
id = id + "_" + string(uuid.NewUUID())
// Construct client for leader election
client, err := kubernetes.NewForConfig(config)
client, err := kubernetes.NewForConfig(rest.AddUserAgent(config, "leader-election"))
if err != nil {
return nil, err
}
@@ -98,13 +99,13 @@ func getInClusterNamespace() (string, error) {
if os.IsNotExist(err) {
return "", fmt.Errorf("not running in-cluster, please specify LeaderElectionNamespace")
} else if err != nil {
return "", fmt.Errorf("error checking namespace file: %v", err)
return "", fmt.Errorf("error checking namespace file: %w", err)
}
// Load the namespace file and return its content
namespace, err := ioutil.ReadFile(inClusterNamespacePath)
if err != nil {
return "", fmt.Errorf("error reading namespace file: %v", err)
return "", fmt.Errorf("error reading namespace file: %w", err)
}
return string(namespace), nil
}

View File

@@ -34,9 +34,15 @@ limitations under the License.
package log
import (
"context"
"github.com/go-logr/logr"
)
var (
contextKey = &struct{}{}
)
// SetLogger sets a concrete logging implementation for all deferred Loggers.
func SetLogger(l logr.Logger) {
Log.Fulfill(l)
@@ -46,3 +52,22 @@ func SetLogger(l logr.Logger) {
// to another logr.Logger. You *must* call SetLogger to
// get any actual logging.
var Log = NewDelegatingLogger(NullLogger{})
// FromContext returns a logger with predefined values from a context.Context.
func FromContext(ctx context.Context, keysAndValues ...interface{}) logr.Logger {
var log logr.Logger
if ctx == nil {
log = Log
} else {
lv := ctx.Value(contextKey)
log = lv.(logr.Logger)
}
log.WithValues(keysAndValues...)
return log
}
// IntoContext takes a context and sets the logger as one of its keys.
// Use FromContext function to retrieve the logger.
func IntoContext(ctx context.Context, log logr.Logger) context.Context {
return context.WithValue(ctx, contextKey, log)
}

View File

@@ -18,15 +18,18 @@ package manager
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"sync"
"time"
"github.com/go-logr/logr"
"github.com/prometheus/client_golang/prometheus/promhttp"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
@@ -44,12 +47,14 @@ import (
const (
// Values taken from: https://github.com/kubernetes/apiserver/blob/master/pkg/apis/config/v1alpha1/defaults.go
defaultLeaseDuration = 15 * time.Second
defaultRenewDeadline = 10 * time.Second
defaultRetryPeriod = 2 * time.Second
defaultLeaseDuration = 15 * time.Second
defaultRenewDeadline = 10 * time.Second
defaultRetryPeriod = 2 * time.Second
defaultGracefulShutdownPeriod = 30 * time.Second
defaultReadinessEndpoint = "/readyz"
defaultLivenessEndpoint = "/healthz"
defaultReadinessEndpoint = "/readyz/"
defaultLivenessEndpoint = "/healthz/"
defaultMetricsEndpoint = "/metrics"
)
var log = logf.RuntimeLog.WithName("manager")
@@ -95,6 +100,9 @@ type controllerManager struct {
// metricsListener is used to serve prometheus metrics
metricsListener net.Listener
// metricsExtraHandlers contains extra handlers to register on http server that serves metrics.
metricsExtraHandlers map[string]http.Handler
// healthProbeListener is used to serve liveness probe
healthProbeListener net.Listener
@@ -114,11 +122,7 @@ type controllerManager struct {
started bool
startedLeader bool
healthzStarted bool
// NB(directxman12): we don't just use an error channel here to avoid the situation where the
// error channel is too small and we end up blocking some goroutines waiting to report their errors.
// errSignal lets us track when we should stop because an error occurred
errSignal *errSignaler
errChan chan error
// internalStop is the stop channel *actually* used by everything involved
// with the manager as a stop channel, so that we can pass a stop channel
@@ -130,6 +134,23 @@ type controllerManager struct {
// It and `internalStop` should point to the same channel.
internalStopper chan<- struct{}
// Logger is the logger that should be used by this manager.
// If none is set, it defaults to log.Log global logger.
logger logr.Logger
// leaderElectionCancel is used to cancel the leader election. It is distinct from internalStopper,
// because for safety reasons we need to os.Exit() when we lose the leader election, meaning that
// it must be deferred until after gracefulShutdown is done.
leaderElectionCancel context.CancelFunc
// stop procedure engaged. In other words, we should not add anything else to the manager
stopProcedureEngaged bool
// elected is closed when this manager becomes the leader of a group of
// managers, either because it won a leader election or because no leader
// election was configured.
elected chan struct{}
startCache func(stop <-chan struct{}) error
// port is the port that the webhook server serves at.
@@ -146,63 +167,38 @@ type controllerManager struct {
// leaseDuration is the duration that non-leader candidates will
// wait to force acquire leadership.
leaseDuration time.Duration
// renewDeadline is the duration that the acting master will retry
// renewDeadline is the duration that the acting controlplane will retry
// refreshing leadership before giving up.
renewDeadline time.Duration
// retryPeriod is the duration the LeaderElector clients should wait
// between tries of actions.
retryPeriod time.Duration
}
type errSignaler struct {
// errSignal indicates that an error occurred, when closed. It shouldn't
// be written to.
errSignal chan struct{}
// waitForRunnable is holding the number of runnables currently running so that
// we can wait for them to exit before quitting the manager
waitForRunnable sync.WaitGroup
// err is the received error
err error
// gracefulShutdownTimeout is the duration given to runnable to stop
// before the manager actually returns on stop.
gracefulShutdownTimeout time.Duration
mu sync.Mutex
}
// onStoppedLeading is callled when the leader election lease is lost.
// It can be overridden for tests.
onStoppedLeading func()
func (r *errSignaler) SignalError(err error) {
r.mu.Lock()
defer r.mu.Unlock()
if err == nil {
// non-error, ignore
log.Error(nil, "SignalError called without an (with a nil) error, which should never happen, ignoring")
return
}
if r.err != nil {
// we already have an error, don't try again
return
}
// save the error and report it
r.err = err
close(r.errSignal)
}
func (r *errSignaler) Error() error {
r.mu.Lock()
defer r.mu.Unlock()
return r.err
}
func (r *errSignaler) GotError() chan struct{} {
r.mu.Lock()
defer r.mu.Unlock()
return r.errSignal
// shutdownCtx is the context that can be used during shutdown. It will be cancelled
// after the gracefulShutdownTimeout ended. It must not be accessed before internalStop
// is closed because it will be nil.
shutdownCtx context.Context
}
// Add sets dependencies on i, and adds it to the list of Runnables to start.
func (cm *controllerManager) Add(r Runnable) error {
cm.mu.Lock()
defer cm.mu.Unlock()
if cm.stopProcedureEngaged {
return errors.New("can't accept new runnable as stop procedure is already engaged")
}
// Set dependencies on the object
if err := cm.SetFields(r); err != nil {
@@ -222,11 +218,7 @@ func (cm *controllerManager) Add(r Runnable) error {
if shouldStart {
// If already started, start the controller
go func() {
if err := r.Start(cm.internalStop); err != nil {
cm.errSignal.SignalError(err)
}
}()
cm.startRunnable(r)
}
return nil
@@ -257,6 +249,28 @@ func (cm *controllerManager) SetFields(i interface{}) error {
if _, err := inject.MapperInto(cm.mapper, i); err != nil {
return err
}
if _, err := inject.LoggerInto(log, i); err != nil {
return err
}
return nil
}
// AddMetricsExtraHandler adds extra handler served on path to the http server that serves metrics.
func (cm *controllerManager) AddMetricsExtraHandler(path string, handler http.Handler) error {
if path == defaultMetricsEndpoint {
return fmt.Errorf("overriding builtin %s endpoint is not allowed", defaultMetricsEndpoint)
}
cm.mu.Lock()
defer cm.mu.Unlock()
_, found := cm.metricsExtraHandlers[path]
if found {
return fmt.Errorf("can't register extra handler by duplicate path %q on metrics http server", path)
}
cm.metricsExtraHandlers[path] = handler
log.V(2).Info("Registering metrics http server extra handler", "path", path)
return nil
}
@@ -265,6 +279,10 @@ func (cm *controllerManager) AddHealthzCheck(name string, check healthz.Checker)
cm.mu.Lock()
defer cm.mu.Unlock()
if cm.stopProcedureEngaged {
return errors.New("can't accept new healthCheck as stop procedure is already engaged")
}
if cm.healthzStarted {
return fmt.Errorf("unable to add new checker because healthz endpoint has already been created")
}
@@ -282,6 +300,10 @@ func (cm *controllerManager) AddReadyzCheck(name string, check healthz.Checker)
cm.mu.Lock()
defer cm.mu.Unlock()
if cm.stopProcedureEngaged {
return errors.New("can't accept new ready check as stop procedure is already engaged")
}
if cm.healthzStarted {
return fmt.Errorf("unable to add new checker because readyz endpoint has already been created")
}
@@ -327,48 +349,77 @@ func (cm *controllerManager) GetAPIReader() client.Reader {
}
func (cm *controllerManager) GetWebhookServer() *webhook.Server {
if cm.webhookServer == nil {
server, wasNew := func() (*webhook.Server, bool) {
cm.mu.Lock()
defer cm.mu.Unlock()
if cm.webhookServer != nil {
return cm.webhookServer, false
}
cm.webhookServer = &webhook.Server{
Port: cm.port,
Host: cm.host,
CertDir: cm.certDir,
}
if err := cm.Add(cm.webhookServer); err != nil {
panic("unable to add webhookServer to the controller manager")
return cm.webhookServer, true
}()
// only add the server if *we ourselves* just registered it.
// Add has its own lock, so just do this separately -- there shouldn't
// be a "race" in this lock gap because the condition is the population
// of cm.webhookServer, not anything to do with Add.
if wasNew {
if err := cm.Add(server); err != nil {
panic("unable to add webhook server to the controller manager")
}
}
return cm.webhookServer
return server
}
func (cm *controllerManager) GetLogger() logr.Logger {
return cm.logger
}
func (cm *controllerManager) serveMetrics(stop <-chan struct{}) {
var metricsPath = "/metrics"
handler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{
ErrorHandling: promhttp.HTTPErrorOnError,
})
// TODO(JoelSpeed): Use existing Kubernetes machinery for serving metrics
mux := http.NewServeMux()
mux.Handle(metricsPath, handler)
mux.Handle(defaultMetricsEndpoint, handler)
func() {
cm.mu.Lock()
defer cm.mu.Unlock()
for path, extraHandler := range cm.metricsExtraHandlers {
mux.Handle(path, extraHandler)
}
}()
server := http.Server{
Handler: mux,
}
// Run the server
go func() {
log.Info("starting metrics server", "path", metricsPath)
cm.startRunnable(RunnableFunc(func(stop <-chan struct{}) error {
log.Info("starting metrics server", "path", defaultMetricsEndpoint)
if err := server.Serve(cm.metricsListener); err != nil && err != http.ErrServerClosed {
cm.errSignal.SignalError(err)
return err
}
}()
return nil
}))
// Shutdown the server when stop is closed
select {
case <-stop:
if err := server.Shutdown(context.Background()); err != nil {
cm.errSignal.SignalError(err)
}
<-stop
if err := server.Shutdown(cm.shutdownCtx); err != nil {
cm.errChan <- err
}
}
func (cm *controllerManager) serveHealthProbes(stop <-chan struct{}) {
// TODO(hypnoglow): refactor locking to use anonymous func in the similar way
// it's done in serveMetrics.
cm.mu.Lock()
mux := http.NewServeMux()
@@ -383,29 +434,48 @@ func (cm *controllerManager) serveHealthProbes(stop <-chan struct{}) {
Handler: mux,
}
// Run server
go func() {
cm.startRunnable(RunnableFunc(func(stop <-chan struct{}) error {
if err := server.Serve(cm.healthProbeListener); err != nil && err != http.ErrServerClosed {
cm.errSignal.SignalError(err)
return err
}
}()
return nil
}))
cm.healthzStarted = true
cm.mu.Unlock()
// Shutdown the server when stop is closed
select {
case <-stop:
if err := server.Shutdown(context.Background()); err != nil {
cm.errSignal.SignalError(err)
}
<-stop
if err := server.Shutdown(cm.shutdownCtx); err != nil {
cm.errChan <- err
}
}
func (cm *controllerManager) Start(stop <-chan struct{}) error {
// join the passed-in stop channel as an upstream feeding into cm.internalStopper
defer close(cm.internalStopper)
func (cm *controllerManager) Start(stop <-chan struct{}) (err error) {
// This chan indicates that stop is complete, in other words all runnables have returned or timeout on stop request
stopComplete := make(chan struct{})
defer close(stopComplete)
// This must be deferred after closing stopComplete, otherwise we deadlock
defer func() {
// https://hips.hearstapps.com/hmg-prod.s3.amazonaws.com/images/gettyimages-459889618-1533579787.jpg
stopErr := cm.engageStopProcedure(stopComplete)
if stopErr != nil {
if err != nil {
// Utilerrors.Aggregate allows to use errors.Is for all contained errors
// whereas fmt.Errorf allows wrapping at most one error which means the
// other one can not be found anymore.
err = utilerrors.NewAggregate([]error{err, stopErr})
} else {
err = stopErr
}
}
}()
// initialize this here so that we reset the signal channel state on every start
cm.errSignal = &errSignaler{errSignal: make(chan struct{})}
// Everything that might write into this channel must be started in a new goroutine,
// because otherwise we might block this routine trying to write into the full channel
// and will not be able to enter the deferred cm.engageStopProcedure() which drains
// it.
cm.errChan = make(chan error)
// Metrics should be served whether the controller is leader or not.
// (If we don't serve metrics for non-leaders, prometheus will still scrape
@@ -421,25 +491,88 @@ func (cm *controllerManager) Start(stop <-chan struct{}) error {
go cm.startNonLeaderElectionRunnables()
if cm.resourceLock != nil {
err := cm.startLeaderElection()
if err != nil {
return err
go func() {
if cm.resourceLock != nil {
err := cm.startLeaderElection()
if err != nil {
cm.errChan <- err
}
} else {
// Treat not having leader election enabled the same as being elected.
close(cm.elected)
go cm.startLeaderElectionRunnables()
}
} else {
go cm.startLeaderElectionRunnables()
}
}()
select {
case <-stop:
// We are done
return nil
case <-cm.errSignal.GotError():
// Error starting a controller
return cm.errSignal.Error()
case err := <-cm.errChan:
// Error starting or running a runnable
return err
}
}
// engageStopProcedure signals all runnables to stop, reads potential errors
// from the errChan and waits for them to end. It must not be called more than once.
func (cm *controllerManager) engageStopProcedure(stopComplete chan struct{}) error {
var cancel context.CancelFunc
if cm.gracefulShutdownTimeout > 0 {
cm.shutdownCtx, cancel = context.WithTimeout(context.Background(), cm.gracefulShutdownTimeout)
} else {
cm.shutdownCtx, cancel = context.WithCancel(context.Background())
}
defer cancel()
close(cm.internalStopper)
// Start draining the errors before acquiring the lock to make sure we don't deadlock
// if something that has the lock is blocked on trying to write into the unbuffered
// channel after something else already wrote into it.
go func() {
for {
select {
case err, ok := <-cm.errChan:
if ok {
log.Error(err, "error received after stop sequence was engaged")
}
case <-stopComplete:
return
}
}
}()
if cm.gracefulShutdownTimeout == 0 {
return nil
}
cm.mu.Lock()
defer cm.mu.Unlock()
cm.stopProcedureEngaged = true
return cm.waitForRunnableToEnd(cm.shutdownCtx, cancel)
}
// waitForRunnableToEnd blocks until all runnables ended or the
// tearDownTimeout was reached. In the latter case, an error is returned.
func (cm *controllerManager) waitForRunnableToEnd(ctx context.Context, cancel context.CancelFunc) error {
defer cancel()
// Cancel leader election only after we waited. It will os.Exit() the app for safety.
defer func() {
if cm.leaderElectionCancel != nil {
cm.leaderElectionCancel()
}
}()
go func() {
cm.waitForRunnable.Wait()
cancel()
}()
<-ctx.Done()
if err := ctx.Err(); err != nil && err != context.Canceled {
return fmt.Errorf("failed waiting for all runnables to end within grace period of %s: %w", cm.gracefulShutdownTimeout, err)
}
return nil
}
func (cm *controllerManager) startNonLeaderElectionRunnables() {
cm.mu.Lock()
defer cm.mu.Unlock()
@@ -450,15 +583,7 @@ func (cm *controllerManager) startNonLeaderElectionRunnables() {
for _, c := range cm.nonLeaderElectionRunnables {
// Controllers block, but we want to return an error if any have an error starting.
// Write any Start errors to a channel so we can return them
ctrl := c
go func() {
if err := ctrl.Start(cm.internalStop); err != nil {
cm.errSignal.SignalError(err)
}
// we use %T here because we don't have a good stand-in for "name",
// and the full runnable might not serialize (mutexes, etc)
log.V(1).Info("non-leader-election runnable finished", "runnable type", fmt.Sprintf("%T", ctrl))
}()
cm.startRunnable(c)
}
}
@@ -472,15 +597,7 @@ func (cm *controllerManager) startLeaderElectionRunnables() {
for _, c := range cm.leaderElectionRunnables {
// Controllers block, but we want to return an error if any have an error starting.
// Write any Start errors to a channel so we can return them
ctrl := c
go func() {
if err := ctrl.Start(cm.internalStop); err != nil {
cm.errSignal.SignalError(err)
}
// we use %T here because we don't have a good stand-in for "name",
// and the full runnable might not serialize (mutexes, etc)
log.V(1).Info("leader-election runnable finished", "runnable type", fmt.Sprintf("%T", ctrl))
}()
cm.startRunnable(c)
}
cm.startedLeader = true
@@ -495,19 +612,37 @@ func (cm *controllerManager) waitForCache() {
if cm.startCache == nil {
cm.startCache = cm.cache.Start
}
go func() {
if err := cm.startCache(cm.internalStop); err != nil {
cm.errSignal.SignalError(err)
}
}()
cm.startRunnable(RunnableFunc(func(stop <-chan struct{}) error {
return cm.startCache(stop)
}))
// Wait for the caches to sync.
// TODO(community): Check the return value and write a test
cm.cache.WaitForCacheSync(cm.internalStop)
// TODO: This should be the return value of cm.cache.WaitForCacheSync but we abuse
// cm.started as check if we already started the cache so it must always become true.
// Making sure that the cache doesn't get started twice is needed to not get a "close
// of closed channel" panic
cm.started = true
}
func (cm *controllerManager) startLeaderElection() (err error) {
ctx, cancel := context.WithCancel(context.Background())
cm.mu.Lock()
cm.leaderElectionCancel = cancel
cm.mu.Unlock()
if cm.onStoppedLeading == nil {
cm.onStoppedLeading = func() {
// Make sure graceful shutdown is skipped if we lost the leader lock without
// intending to.
cm.gracefulShutdownTimeout = time.Duration(0)
// Most implementations of leader election log.Fatal() here.
// Since Start is wrapped in log.Fatal when called, we can just return
// an error here which will cause the program to exit.
cm.errChan <- errors.New("leader election lost")
}
}
l, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
Lock: cm.resourceLock,
LeaseDuration: cm.leaseDuration,
@@ -515,30 +650,31 @@ func (cm *controllerManager) startLeaderElection() (err error) {
RetryPeriod: cm.retryPeriod,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(_ context.Context) {
close(cm.elected)
cm.startLeaderElectionRunnables()
},
OnStoppedLeading: func() {
// Most implementations of leader election log.Fatal() here.
// Since Start is wrapped in log.Fatal when called, we can just return
// an error here which will cause the program to exit.
cm.errSignal.SignalError(fmt.Errorf("leader election lost"))
},
OnStoppedLeading: cm.onStoppedLeading,
},
})
if err != nil {
return err
}
ctx, cancel := context.WithCancel(context.Background())
go func() {
select {
case <-cm.internalStop:
cancel()
case <-ctx.Done():
}
}()
// Start the leader elector process
go l.Run(ctx)
return nil
}
func (cm *controllerManager) Elected() <-chan struct{} {
return cm.elected
}
func (cm *controllerManager) startRunnable(r Runnable) {
cm.waitForRunnable.Add(1)
go func() {
defer cm.waitForRunnable.Done()
if err := r.Start(cm.internalStop); err != nil {
cm.errChan <- err
}
}()
}

View File

@@ -19,6 +19,7 @@ package manager
import (
"fmt"
"net"
"net/http"
"time"
"github.com/go-logr/logr"
@@ -35,6 +36,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/healthz"
internalrecorder "sigs.k8s.io/controller-runtime/pkg/internal/recorder"
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/metrics"
"sigs.k8s.io/controller-runtime/pkg/recorder"
"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -50,10 +52,22 @@ type Manager interface {
// non-leaderelection mode (always running) or leader election mode (managed by leader election if enabled).
Add(Runnable) error
// Elected is closed when this manager is elected leader of a group of
// managers, either because it won a leader election or because no leader
// election was configured.
Elected() <-chan struct{}
// SetFields will set any dependencies on an object for which the object has implemented the inject
// interface - e.g. inject.Client.
SetFields(interface{}) error
// AddMetricsExtraHandler adds an extra handler served on path to the http server that serves metrics.
// Might be useful to register some diagnostic endpoints e.g. pprof. Note that these endpoints meant to be
// sensitive and shouldn't be exposed publicly.
// If the simple path -> handler mapping offered here is not enough, a new http server/listener should be added as
// Runnable to the manager via Add method.
AddMetricsExtraHandler(path string, handler http.Handler) error
// AddHealthzCheck allows you to add Healthz checker
AddHealthzCheck(name string, check healthz.Checker) error
@@ -62,6 +76,9 @@ type Manager interface {
// Start starts all registered Controllers and blocks until the Stop channel is closed.
// Returns an error if there is an error starting any controller.
// If LeaderElection is used, the binary must be exited immediately after this returns,
// otherwise components that need leader election might continue to run after the leader
// lock was lost.
Start(<-chan struct{}) error
// GetConfig returns an initialized Config
@@ -95,6 +112,9 @@ type Manager interface {
// GetWebhookServer returns a webhook.Server
GetWebhookServer() *webhook.Server
// GetLogger returns this manager's logger.
GetLogger() logr.Logger
}
// Options are the arguments for creating a new Manager
@@ -111,8 +131,14 @@ type Options struct {
// reconciled. A lower period will correct entropy more quickly, but reduce
// responsiveness to change if there are many watched resources. Change this
// value only if you know what you are doing. Defaults to 10 hours if unset.
// there will a 10 percent jitter between the SyncPeriod of all controllers
// so that all controllers will not send list requests simultaneously.
SyncPeriod *time.Duration
// Logger is the logger that should be used by this manager.
// If none is set, it defaults to log.Log global logger.
Logger logr.Logger
// LeaderElection determines whether or not to use leader election when
// starting the manager.
LeaderElection bool
@@ -125,11 +151,15 @@ type Options struct {
// will use for holding the leader lock.
LeaderElectionID string
// LeaderElectionConfig can be specified to override the default configuration
// that is used to build the leader election client.
LeaderElectionConfig *rest.Config
// LeaseDuration is the duration that non-leader candidates will
// wait to force acquire leadership. This is measured against time of
// last observed ack. Default is 15 seconds.
LeaseDuration *time.Duration
// RenewDeadline is the duration that the acting master will retry
// RenewDeadline is the duration that the acting controlplane will retry
// refreshing leadership before giving up. Default is 10 seconds.
RenewDeadline *time.Duration
// RetryPeriod is the duration the LeaderElector clients should wait
@@ -168,7 +198,8 @@ type Options struct {
// CertDir is the directory that contains the server key and certificate.
// if not set, webhook server would look up the server key and certificate in
// {TempDir}/k8s-webhook-server/serving-certs
// {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate
// must be named tls.key and tls.crt, respectively.
CertDir string
// Functions to all for a user to customize the values that will be injected.
@@ -181,10 +212,20 @@ type Options struct {
// use the cache for reads and the client for writes.
NewClient NewClientFunc
// DryRunClient specifies whether the client should be configured to enforce
// dryRun mode.
DryRunClient bool
// EventBroadcaster records Events emitted by the manager and sends them to the Kubernetes API
// Use this to customize the event correlator and spam filter
EventBroadcaster record.EventBroadcaster
// GracefulShutdownTimeout is the duration given to runnable to stop before the manager actually returns on stop.
// To disable graceful shutdown, set to time.Duration(0)
// To use graceful shutdown without timeout, set to a negative duration, e.G. time.Duration(-1)
// The graceful shutdown is skipped for safety reasons in case the leadere election lease is lost.
GracefulShutdownTimeout *time.Duration
// Dependency injection for testing
newRecorderProvider func(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, broadcaster record.EventBroadcaster) (recorder.Provider, error)
newResourceLock func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error)
@@ -254,6 +295,11 @@ func New(config *rest.Config, options Options) (Manager, error) {
if err != nil {
return nil, err
}
if options.DryRunClient {
writeObj = client.NewDryRunClient(writeObj)
}
// Create the recorder provider to inject event recorders for the components.
// TODO(directxman12): the log for the event provider should have a context (name, tags, etc) specific
// to the particular controller that it's being injected into, rather than a generic one like is here.
@@ -263,7 +309,11 @@ func New(config *rest.Config, options Options) (Manager, error) {
}
// Create the resource lock to enable leader election)
resourceLock, err := options.newResourceLock(config, recorderProvider, leaderelection.Options{
leaderConfig := config
if options.LeaderElectionConfig != nil {
leaderConfig = options.LeaderElectionConfig
}
resourceLock, err := options.newResourceLock(leaderConfig, recorderProvider, leaderelection.Options{
LeaderElection: options.LeaderElection,
LeaderElectionID: options.LeaderElectionID,
LeaderElectionNamespace: options.LeaderElectionNamespace,
@@ -279,6 +329,9 @@ func New(config *rest.Config, options Options) (Manager, error) {
return nil, err
}
// By default we have no extra endpoints to expose on metrics http server.
metricsExtraHandlers := make(map[string]http.Handler)
// Create health probes listener. This will throw an error if the bind
// address is invalid or already in use.
healthProbeListener, err := options.newHealthProbeListener(options.HealthProbeBindAddress)
@@ -289,32 +342,36 @@ func New(config *rest.Config, options Options) (Manager, error) {
stop := make(chan struct{})
return &controllerManager{
config: config,
scheme: options.Scheme,
cache: cache,
fieldIndexes: cache,
client: writeObj,
apiReader: apiReader,
recorderProvider: recorderProvider,
resourceLock: resourceLock,
mapper: mapper,
metricsListener: metricsListener,
internalStop: stop,
internalStopper: stop,
port: options.Port,
host: options.Host,
certDir: options.CertDir,
leaseDuration: *options.LeaseDuration,
renewDeadline: *options.RenewDeadline,
retryPeriod: *options.RetryPeriod,
healthProbeListener: healthProbeListener,
readinessEndpointName: options.ReadinessEndpointName,
livenessEndpointName: options.LivenessEndpointName,
config: config,
scheme: options.Scheme,
cache: cache,
fieldIndexes: cache,
client: writeObj,
apiReader: apiReader,
recorderProvider: recorderProvider,
resourceLock: resourceLock,
mapper: mapper,
metricsListener: metricsListener,
metricsExtraHandlers: metricsExtraHandlers,
logger: options.Logger,
internalStop: stop,
internalStopper: stop,
elected: make(chan struct{}),
port: options.Port,
host: options.Host,
certDir: options.CertDir,
leaseDuration: *options.LeaseDuration,
renewDeadline: *options.RenewDeadline,
retryPeriod: *options.RetryPeriod,
healthProbeListener: healthProbeListener,
readinessEndpointName: options.ReadinessEndpointName,
livenessEndpointName: options.LivenessEndpointName,
gracefulShutdownTimeout: *options.GracefulShutdownTimeout,
}, nil
}
// defaultNewClient creates the default caching client
func defaultNewClient(cache cache.Cache, config *rest.Config, options client.Options) (client.Client, error) {
// DefaultNewClient creates the default caching client
func DefaultNewClient(cache cache.Cache, config *rest.Config, options client.Options) (client.Client, error) {
// Create the Client for Write operations.
c, err := client.New(config, options)
if err != nil {
@@ -359,7 +416,7 @@ func setOptionsDefaults(options Options) Options {
// Allow newClient to be mocked
if options.NewClient == nil {
options.NewClient = defaultNewClient
options.NewClient = DefaultNewClient
}
// Allow newCache to be mocked
@@ -409,5 +466,14 @@ func setOptionsDefaults(options Options) Options {
options.newHealthProbeListener = defaultHealthProbeListener
}
if options.GracefulShutdownTimeout == nil {
gracefulShutdownTimeout := defaultGracefulShutdownPeriod
options.GracefulShutdownTimeout = &gracefulShutdownTimeout
}
if options.Logger == nil {
options.Logger = logf.Log
}
return options
}

View File

@@ -29,78 +29,91 @@ import (
// that client-go registers metrics. We copy the names and formats
// from Kubernetes so that we match the core controllers.
// Metrics subsystem and all of the keys used by the rest client.
const (
RestClientSubsystem = "rest_client"
LatencyKey = "request_latency_seconds"
ResultKey = "requests_total"
)
// Metrics subsystem and all keys used by the reflectors.
const (
ReflectorSubsystem = "reflector"
ListsTotalKey = "lists_total"
ListsDurationKey = "list_duration_seconds"
ItemsPerListKey = "items_per_list"
WatchesTotalKey = "watches_total"
ShortWatchesTotalKey = "short_watches_total"
WatchDurationKey = "watch_duration_seconds"
ItemsPerWatchKey = "items_per_watch"
LastResourceVersionKey = "last_resource_version"
)
var (
// client metrics
requestLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Subsystem: RestClientSubsystem,
Name: LatencyKey,
Help: "Request latency in seconds. Broken down by verb and URL.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 10),
}, []string{"verb", "url"})
requestLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "rest_client_request_latency_seconds",
Help: "Request latency in seconds. Broken down by verb and URL.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 10),
},
[]string{"verb", "url"},
)
requestResult = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "rest_client_requests_total",
Help: "Number of HTTP requests, partitioned by status code, method, and host.",
},
[]string{"code", "method", "host"},
)
requestResult = prometheus.NewCounterVec(prometheus.CounterOpts{
Subsystem: RestClientSubsystem,
Name: ResultKey,
Help: "Number of HTTP requests, partitioned by status code, method, and host.",
}, []string{"code", "method", "host"})
// reflector metrics
// TODO(directxman12): update these to be histograms once the metrics overhaul KEP
// PRs start landing.
reflectorSubsystem = "reflector"
listsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Subsystem: reflectorSubsystem,
Name: "lists_total",
Subsystem: ReflectorSubsystem,
Name: ListsTotalKey,
Help: "Total number of API lists done by the reflectors",
}, []string{"name"})
listsDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
Subsystem: reflectorSubsystem,
Name: "list_duration_seconds",
Subsystem: ReflectorSubsystem,
Name: ListsDurationKey,
Help: "How long an API list takes to return and decode for the reflectors",
}, []string{"name"})
itemsPerList = prometheus.NewSummaryVec(prometheus.SummaryOpts{
Subsystem: reflectorSubsystem,
Name: "items_per_list",
Subsystem: ReflectorSubsystem,
Name: ItemsPerListKey,
Help: "How many items an API list returns to the reflectors",
}, []string{"name"})
watchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Subsystem: reflectorSubsystem,
Name: "watches_total",
Subsystem: ReflectorSubsystem,
Name: WatchesTotalKey,
Help: "Total number of API watches done by the reflectors",
}, []string{"name"})
shortWatchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Subsystem: reflectorSubsystem,
Name: "short_watches_total",
Subsystem: ReflectorSubsystem,
Name: ShortWatchesTotalKey,
Help: "Total number of short API watches done by the reflectors",
}, []string{"name"})
watchDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
Subsystem: reflectorSubsystem,
Name: "watch_duration_seconds",
Subsystem: ReflectorSubsystem,
Name: WatchDurationKey,
Help: "How long an API watch takes to return and decode for the reflectors",
}, []string{"name"})
itemsPerWatch = prometheus.NewSummaryVec(prometheus.SummaryOpts{
Subsystem: reflectorSubsystem,
Name: "items_per_watch",
Subsystem: ReflectorSubsystem,
Name: ItemsPerWatchKey,
Help: "How many items an API watch returns to the reflectors",
}, []string{"name"})
lastResourceVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Subsystem: reflectorSubsystem,
Name: "last_resource_version",
Subsystem: ReflectorSubsystem,
Name: LastResourceVersionKey,
Help: "Last resource version seen for the reflectors",
}, []string{"name"})
)
@@ -117,7 +130,10 @@ func registerClientMetrics() {
Registry.MustRegister(requestResult)
// register the metrics with client-go
clientmetrics.Register(&latencyAdapter{metric: requestLatency}, &resultAdapter{metric: requestResult})
clientmetrics.Register(clientmetrics.RegisterOpts{
RequestLatency: &latencyAdapter{metric: requestLatency},
RequestResult: &resultAdapter{metric: requestResult},
})
}
// registerReflectorMetrics sets up reflector (reconcile) loop metrics

View File

@@ -19,8 +19,12 @@ package metrics
import (
"fmt"
"net"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
var log = logf.RuntimeLog.WithName("metrics")
// DefaultBindAddress sets the default bind address for the metrics listener
// The metrics is on by default.
var DefaultBindAddress = ":8080"
@@ -40,7 +44,7 @@ func NewListener(addr string) (net.Listener, error) {
log.Info("metrics server is starting to listen", "addr", addr)
ln, err := net.Listen("tcp", addr)
if err != nil {
er := fmt.Errorf("error listening on %s: %v", addr, err)
er := fmt.Errorf("error listening on %s: %w", addr, err)
log.Error(er, "metrics server failed to listen. You may want to disable the metrics server or use another port if it is due to conflicts")
return nil, er
}

View File

@@ -18,6 +18,13 @@ package metrics
import "github.com/prometheus/client_golang/prometheus"
// RegistererGatherer combines both parts of the API of a Prometheus
// registry, both the Registerer and the Gatherer interfaces.
type RegistererGatherer interface {
prometheus.Registerer
prometheus.Gatherer
}
// Registry is a prometheus registry for storing metrics within the
// controller-runtime
var Registry = prometheus.NewRegistry()
var Registry RegistererGatherer = prometheus.NewRegistry()

View File

@@ -19,106 +19,112 @@ package metrics
import (
"github.com/prometheus/client_golang/prometheus"
"k8s.io/client-go/util/workqueue"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
var log = logf.RuntimeLog.WithName("metrics")
// This file is copied and adapted from k8s.io/kubernetes/pkg/util/workqueue/prometheus
// which registers metrics to the default prometheus Registry. We require very
// similar functionality, but must register metrics to a different Registry.
func init() {
workqueue.SetProvider(workqueueMetricsProvider{})
}
// Metrics subsystem and all keys used by the workqueue.
const (
WorkQueueSubsystem = "workqueue"
DepthKey = "depth"
AddsKey = "adds_total"
QueueLatencyKey = "queue_duration_seconds"
WorkDurationKey = "work_duration_seconds"
UnfinishedWorkKey = "unfinished_work_seconds"
LongestRunningProcessorKey = "longest_running_processor_seconds"
RetriesKey = "retries_total"
)
func registerWorkqueueMetric(c prometheus.Collector, name, queue string) {
if err := Registry.Register(c); err != nil {
log.Error(err, "failed to register metric", "name", name, "queue", queue)
}
var (
depth = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Subsystem: WorkQueueSubsystem,
Name: DepthKey,
Help: "Current depth of workqueue",
}, []string{"name"})
adds = prometheus.NewCounterVec(prometheus.CounterOpts{
Subsystem: WorkQueueSubsystem,
Name: AddsKey,
Help: "Total number of adds handled by workqueue",
}, []string{"name"})
latency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Subsystem: WorkQueueSubsystem,
Name: QueueLatencyKey,
Help: "How long in seconds an item stays in workqueue before being requested",
Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10),
}, []string{"name"})
workDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Subsystem: WorkQueueSubsystem,
Name: WorkDurationKey,
Help: "How long in seconds processing an item from workqueue takes.",
Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10),
}, []string{"name"})
unfinished = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Subsystem: WorkQueueSubsystem,
Name: UnfinishedWorkKey,
Help: "How many seconds of work has been done that " +
"is in progress and hasn't been observed by work_duration. Large " +
"values indicate stuck threads. One can deduce the number of stuck " +
"threads by observing the rate at which this increases.",
}, []string{"name"})
longestRunningProcessor = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Subsystem: WorkQueueSubsystem,
Name: LongestRunningProcessorKey,
Help: "How many seconds has the longest running " +
"processor for workqueue been running.",
}, []string{"name"})
retries = prometheus.NewCounterVec(prometheus.CounterOpts{
Subsystem: WorkQueueSubsystem,
Name: RetriesKey,
Help: "Total number of retries handled by workqueue",
}, []string{"name"})
)
func init() {
Registry.MustRegister(depth)
Registry.MustRegister(adds)
Registry.MustRegister(latency)
Registry.MustRegister(workDuration)
Registry.MustRegister(unfinished)
Registry.MustRegister(longestRunningProcessor)
Registry.MustRegister(retries)
workqueue.SetProvider(workqueueMetricsProvider{})
}
type workqueueMetricsProvider struct{}
func (workqueueMetricsProvider) NewDepthMetric(queue string) workqueue.GaugeMetric {
const name = "workqueue_depth"
m := prometheus.NewGauge(prometheus.GaugeOpts{
Name: name,
Help: "Current depth of workqueue",
ConstLabels: prometheus.Labels{"name": queue},
})
registerWorkqueueMetric(m, name, queue)
return m
func (workqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {
return depth.WithLabelValues(name)
}
func (workqueueMetricsProvider) NewAddsMetric(queue string) workqueue.CounterMetric {
const name = "workqueue_adds_total"
m := prometheus.NewCounter(prometheus.CounterOpts{
Name: name,
Help: "Total number of adds handled by workqueue",
ConstLabels: prometheus.Labels{"name": queue},
})
registerWorkqueueMetric(m, name, queue)
return m
func (workqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric {
return adds.WithLabelValues(name)
}
func (workqueueMetricsProvider) NewLatencyMetric(queue string) workqueue.HistogramMetric {
const name = "workqueue_queue_duration_seconds"
m := prometheus.NewHistogram(prometheus.HistogramOpts{
Name: name,
Help: "How long in seconds an item stays in workqueue before being requested.",
ConstLabels: prometheus.Labels{"name": queue},
Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10),
})
registerWorkqueueMetric(m, name, queue)
return m
func (workqueueMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric {
return latency.WithLabelValues(name)
}
func (workqueueMetricsProvider) NewWorkDurationMetric(queue string) workqueue.HistogramMetric {
const name = "workqueue_work_duration_seconds"
m := prometheus.NewHistogram(prometheus.HistogramOpts{
Name: name,
Help: "How long in seconds processing an item from workqueue takes.",
ConstLabels: prometheus.Labels{"name": queue},
Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10),
})
registerWorkqueueMetric(m, name, queue)
return m
func (workqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric {
return workDuration.WithLabelValues(name)
}
func (workqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(queue string) workqueue.SettableGaugeMetric {
const name = "workqueue_unfinished_work_seconds"
m := prometheus.NewGauge(prometheus.GaugeOpts{
Name: name,
Help: "How many seconds of work has done that " +
"is in progress and hasn't been observed by work_duration. Large " +
"values indicate stuck threads. One can deduce the number of stuck " +
"threads by observing the rate at which this increases.",
ConstLabels: prometheus.Labels{"name": queue},
})
registerWorkqueueMetric(m, name, queue)
return m
func (workqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric {
return unfinished.WithLabelValues(name)
}
func (workqueueMetricsProvider) NewLongestRunningProcessorSecondsMetric(queue string) workqueue.SettableGaugeMetric {
const name = "workqueue_longest_running_processor_seconds"
m := prometheus.NewGauge(prometheus.GaugeOpts{
Name: name,
Help: "How many seconds has the longest running " +
"processor for workqueue been running.",
ConstLabels: prometheus.Labels{"name": queue},
})
registerWorkqueueMetric(m, name, queue)
return m
func (workqueueMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric {
return longestRunningProcessor.WithLabelValues(name)
}
func (workqueueMetricsProvider) NewRetriesMetric(queue string) workqueue.CounterMetric {
const name = "workqueue_retries_total"
m := prometheus.NewCounter(prometheus.CounterOpts{
Name: name,
Help: "Total number of retries handled by workqueue",
ConstLabels: prometheus.Labels{"name": queue},
})
registerWorkqueueMetric(m, name, queue)
return m
func (workqueueMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric {
return retries.WithLabelValues(name)
}

View File

@@ -17,6 +17,9 @@ limitations under the License.
package predicate
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/event"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
@@ -41,6 +44,8 @@ type Predicate interface {
var _ Predicate = Funcs{}
var _ Predicate = ResourceVersionChangedPredicate{}
var _ Predicate = GenerationChangedPredicate{}
var _ Predicate = or{}
var _ Predicate = and{}
// Funcs is a function that implements Predicate.
type Funcs struct {
@@ -89,6 +94,26 @@ func (p Funcs) Generic(e event.GenericEvent) bool {
return true
}
// NewPredicateFuncs returns a predicate funcs that applies the given filter function
// on CREATE, UPDATE, DELETE and GENERIC events. For UPDATE events, the filter is applied
// to the new object.
func NewPredicateFuncs(filter func(meta metav1.Object, object runtime.Object) bool) Funcs {
return Funcs{
CreateFunc: func(e event.CreateEvent) bool {
return filter(e.Meta, e.Object)
},
UpdateFunc: func(e event.UpdateEvent) bool {
return filter(e.MetaNew, e.ObjectNew)
},
DeleteFunc: func(e event.DeleteEvent) bool {
return filter(e.Meta, e.Object)
},
GenericFunc: func(e event.GenericEvent) bool {
return filter(e.Meta, e.Object)
},
}
}
// ResourceVersionChangedPredicate implements a default update predicate function on resource version change
type ResourceVersionChangedPredicate struct {
Funcs
@@ -112,10 +137,7 @@ func (ResourceVersionChangedPredicate) Update(e event.UpdateEvent) bool {
log.Error(nil, "UpdateEvent has no new metadata", "event", e)
return false
}
if e.MetaNew.GetResourceVersion() == e.MetaOld.GetResourceVersion() {
return false
}
return true
return e.MetaNew.GetResourceVersion() != e.MetaOld.GetResourceVersion()
}
// GenerationChangedPredicate implements a default update predicate function on Generation change.
@@ -156,8 +178,95 @@ func (GenerationChangedPredicate) Update(e event.UpdateEvent) bool {
log.Error(nil, "Update event has no new metadata", "event", e)
return false
}
if e.MetaNew.GetGeneration() == e.MetaOld.GetGeneration() {
return false
return e.MetaNew.GetGeneration() != e.MetaOld.GetGeneration()
}
// And returns a composite predicate that implements a logical AND of the predicates passed to it.
func And(predicates ...Predicate) Predicate {
return and{predicates}
}
type and struct {
predicates []Predicate
}
func (a and) Create(e event.CreateEvent) bool {
for _, p := range a.predicates {
if !p.Create(e) {
return false
}
}
return true
}
func (a and) Update(e event.UpdateEvent) bool {
for _, p := range a.predicates {
if !p.Update(e) {
return false
}
}
return true
}
func (a and) Delete(e event.DeleteEvent) bool {
for _, p := range a.predicates {
if !p.Delete(e) {
return false
}
}
return true
}
func (a and) Generic(e event.GenericEvent) bool {
for _, p := range a.predicates {
if !p.Generic(e) {
return false
}
}
return true
}
// Or returns a composite predicate that implements a logical OR of the predicates passed to it.
func Or(predicates ...Predicate) Predicate {
return or{predicates}
}
type or struct {
predicates []Predicate
}
func (o or) Create(e event.CreateEvent) bool {
for _, p := range o.predicates {
if p.Create(e) {
return true
}
}
return false
}
func (o or) Update(e event.UpdateEvent) bool {
for _, p := range o.predicates {
if p.Update(e) {
return true
}
}
return false
}
func (o or) Delete(e event.DeleteEvent) bool {
for _, p := range o.predicates {
if p.Delete(e) {
return true
}
}
return false
}
func (o or) Generic(e event.GenericEvent) bool {
for _, p := range o.predicates {
if p.Generic(e) {
return true
}
}
return false
}

View File

@@ -1,5 +1,5 @@
/*
Copyright 2018 The Kubernetes Authors.
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -14,12 +14,9 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package manager
/*
Package ratelimiter defines rate limiters used by Controllers to limit how frequently requests may be queued.
// func SetCacheForTest(options *Options, c func(config *rest.Config, opts cache.Options) (cache.Cache, error)) {
// options.newCache = c
// }
// func SetClientForTest(options *Options, c func(config *rest.Config, options client.Options) (client.Client, error)) {
// options.newClient = c
// }
Typical rate limiters that can be used are implemented in client-go's workqueue package.
*/
package ratelimiter

View File

@@ -0,0 +1,30 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ratelimiter
import "time"
// RateLimiter is an identical interface of client-go workqueue RateLimiter.
type RateLimiter interface {
// When gets an item and gets to decide how long that item should wait
When(item interface{}) time.Duration
// Forget indicates that an item is finished being retried. Doesn't matter whether its for perm failing
// or for success, we'll stop tracking it
Forget(item interface{})
// NumRequeues returns back how many failures the item has had
NumRequeues(item interface{}) int
}

View File

@@ -32,6 +32,14 @@ type Result struct {
RequeueAfter time.Duration
}
// IsZero returns true if this result is empty.
func (r *Result) IsZero() bool {
if r == nil {
return true
}
return *r == Result{}
}
// Request contains the information necessary to reconcile a Kubernetes object. This includes the
// information to uniquely identify the object - its Name and Namespace. It does NOT contain information about
// any specific Event or the object contents itself.
@@ -52,7 +60,7 @@ Deleting Kubernetes objects) or external Events (GitHub Webhooks, polling extern
Example reconcile Logic:
* Reader an object and all the Pods it owns.
* Read an object and all the Pods it owns.
* Observe that the object spec specifies 5 replicas but actual cluster contains only 1 Pod replica.
* Create 4 Pods and set their OwnerReferences to the object.

View File

@@ -33,7 +33,7 @@ type Cache interface {
}
// CacheInto will set informers on i and return the result if it implements Cache. Returns
//// false if i does not implement Cache.
// false if i does not implement Cache.
func CacheInto(c cache.Cache, i interface{}) (bool, error) {
if s, ok := i.(Cache); ok {
return true, s.InjectCache(c)
@@ -62,7 +62,7 @@ type Config interface {
}
// ConfigInto will set config on i and return the result if it implements Config. Returns
//// false if i does not implement Config.
// false if i does not implement Config.
func ConfigInto(config *rest.Config, i interface{}) (bool, error) {
if s, ok := i.(Config); ok {
return true, s.InjectConfig(config)

View File

@@ -36,13 +36,13 @@ limitations under the License.
// )
//
// This also true of the built-in Kubernetes types. Then, in the entrypoint for
// your manager, assemble the scheme containing exactly the types you need.
// For instance, if our controller needs types from the core/v1 API group (e.g. Pod),
// plus types from my.api.group/v1:
// your manager, assemble the scheme containing exactly the types you need,
// panicing if scheme registration failed. For instance, if our controller needs
// types from the core/v1 API group (e.g. Pod), plus types from my.api.group/v1:
//
// func init() {
// myapigroupv1.AddToScheme(scheme)
// kubernetesscheme.AddToScheme(scheme)
// utilruntime.Must(myapigroupv1.AddToScheme(scheme))
// utilruntime.Must(kubernetesscheme.AddToScheme(scheme))
// }
//
// func main() {

View File

@@ -35,7 +35,7 @@ var log = logf.RuntimeLog.WithName("source").WithName("EventHandler")
var _ cache.ResourceEventHandler = EventHandler{}
// EventHandler adapts a eventhandler.EventHandler interface to a cache.ResourceEventHandler interface
// EventHandler adapts a handler.EventHandler interface to a cache.ResourceEventHandler interface
type EventHandler struct {
EventHandler handler.EventHandler
Queue workqueue.RateLimitingInterface

View File

@@ -17,6 +17,8 @@ limitations under the License.
package source
import (
"context"
"errors"
"fmt"
"sync"
@@ -55,6 +57,33 @@ type Source interface {
Start(handler.EventHandler, workqueue.RateLimitingInterface, ...predicate.Predicate) error
}
// SyncingSource is a source that needs syncing prior to being usable. The controller
// will call its WaitForSync prior to starting workers.
type SyncingSource interface {
Source
WaitForSync(stop <-chan struct{}) error
}
// NewKindWithCache creates a Source without InjectCache, so that it is assured that the given cache is used
// and not overwritten. It can be used to watch objects in a different cluster by passing the cache
// from that other cluster
func NewKindWithCache(object runtime.Object, cache cache.Cache) SyncingSource {
return &kindWithCache{kind: Kind{Type: object, cache: cache}}
}
type kindWithCache struct {
kind Kind
}
func (ks *kindWithCache) Start(handler handler.EventHandler, queue workqueue.RateLimitingInterface,
prct ...predicate.Predicate) error {
return ks.kind.Start(handler, queue, prct...)
}
func (ks *kindWithCache) WaitForSync(stop <-chan struct{}) error {
return ks.kind.WaitForSync(stop)
}
// Kind is used to provide a source of events originating inside the cluster from Watches (e.g. Pod Create)
type Kind struct {
// Type is the type of object to watch. e.g. &v1.Pod{}
@@ -64,7 +93,7 @@ type Kind struct {
cache cache.Cache
}
var _ Source = &Kind{}
var _ SyncingSource = &Kind{}
// Start is internal and should be called only by the Controller to register an EventHandler with the Informer
// to enqueue reconcile.Requests.
@@ -82,7 +111,7 @@ func (ks *Kind) Start(handler handler.EventHandler, queue workqueue.RateLimiting
}
// Lookup the Informer from the Cache and add an EventHandler which populates the Queue
i, err := ks.cache.GetInformer(ks.Type)
i, err := ks.cache.GetInformer(context.TODO(), ks.Type)
if err != nil {
if kindMatchErr, ok := err.(*meta.NoKindMatchError); ok {
log.Error(err, "if kind is a CRD, it should be installed before calling Start",
@@ -101,6 +130,16 @@ func (ks *Kind) String() string {
return fmt.Sprintf("kind source: unknown GVK")
}
// WaitForSync implements SyncingSource to allow controllers to wait with starting
// workers until the cache is synced.
func (ks *Kind) WaitForSync(stop <-chan struct{}) error {
if !ks.cache.WaitForCacheSync(stop) {
// Would be great to return something more informative here
return errors.New("cache did not sync")
}
return nil
}
var _ inject.Cache = &Kind{}
// InjectCache is internal should be called only by the Controller. InjectCache is used to inject
@@ -266,6 +305,8 @@ func (is *Informer) String() string {
return fmt.Sprintf("informer source: %p", is.Informer)
}
var _ Source = Func(nil)
// Func is a function that implements Source
type Func func(handler.EventHandler, workqueue.RateLimitingInterface, ...predicate.Predicate) error

View File

@@ -99,6 +99,11 @@ func (wh *Webhook) writeResponse(w io.Writer, response Response) {
wh.writeResponse(w, Errored(http.StatusInternalServerError, err))
} else {
res := responseAdmissionReview.Response
wh.log.V(1).Info("wrote response", "UID", res.UID, "allowed", res.Allowed, "result", res.Result)
if log := wh.log; log.V(1).Enabled() {
if res.Result != nil {
log = log.WithValues("code", res.Result.Code, "reason", res.Result.Reason)
}
log.V(1).Info("wrote response", "UID", res.UID, "allowed", res.Allowed)
}
}
}

View File

@@ -47,7 +47,7 @@ func (hs multiMutating) Handle(ctx context.Context, req Request) Response {
var err error
marshaledPatch, err := json.Marshal(patches)
if err != nil {
return Errored(http.StatusBadRequest, fmt.Errorf("error when marshaling the patch: %v", err))
return Errored(http.StatusBadRequest, fmt.Errorf("error when marshaling the patch: %w", err))
}
return Response{
AdmissionResponse: admissionv1beta1.AdmissionResponse{

View File

@@ -161,12 +161,12 @@ func (wh *Webhook) convertViaHub(src, dst conversion.Convertible) error {
err = src.ConvertTo(hub)
if err != nil {
return fmt.Errorf("%T failed to convert to hub version %T : %v", src, hub, err)
return fmt.Errorf("%T failed to convert to hub version %T : %w", src, hub, err)
}
err = dst.ConvertFrom(hub)
if err != nil {
return fmt.Errorf("%T failed to convert from hub version %T : %v", dst, hub, err)
return fmt.Errorf("%T failed to convert from hub version %T : %w", dst, hub, err)
}
return nil
@@ -187,7 +187,7 @@ func (wh *Webhook) getHub(obj runtime.Object) (conversion.Hub, error) {
for _, gvk := range gvks {
instance, err := wh.scheme.New(gvk)
if err != nil {
return nil, fmt.Errorf("failed to allocate an instance for gvk %v %v", gvk, err)
return nil, fmt.Errorf("failed to allocate an instance for gvk %v: %w", gvk, err)
}
if val, isHub := instance.(conversion.Hub); isHub {
if hubFoundAlready {
@@ -237,7 +237,7 @@ func IsConvertible(scheme *runtime.Scheme, obj runtime.Object) (bool, error) {
for _, gvk := range gvks {
instance, err := scheme.New(gvk)
if err != nil {
return false, fmt.Errorf("failed to allocate an instance for gvk %v %v", gvk, err)
return false, fmt.Errorf("failed to allocate an instance for gvk %v: %w", gvk, err)
}
if isHub(instance) {
@@ -264,10 +264,6 @@ func IsConvertible(scheme *runtime.Scheme, obj runtime.Object) (bool, error) {
}
if len(hubs) == 1 && len(nonSpokes) == 0 { // convertible
spokeVersions := []string{}
for _, sp := range spokes {
spokeVersions = append(spokeVersions, sp.GetObjectKind().GroupVersionKind().String())
}
return true, nil
}

View File

@@ -20,7 +20,7 @@ import (
"crypto/tls"
"sync"
"gopkg.in/fsnotify.v1"
"github.com/fsnotify/fsnotify"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)

View File

@@ -23,16 +23,6 @@ import (
)
var (
// TotalRequests is a prometheus metric which counts the total number of requests that
// the webhook server has received.
TotalRequests = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "controller_runtime_webhook_requests_total",
Help: "Total number of admission requests",
},
[]string{"webhook", "succeeded"},
)
// RequestLatency is a prometheus metric which is a histogram of the latency
// of processing admission requests.
RequestLatency = prometheus.NewHistogramVec(
@@ -42,10 +32,30 @@ var (
},
[]string{"webhook"},
)
// RequestTotal is a prometheus metric which is a counter of the total processed admission requests.
RequestTotal = func() *prometheus.CounterVec {
return prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "controller_runtime_webhook_requests_total",
Help: "Total number of admission requests by HTTP status code.",
},
[]string{"webhook", "code"},
)
}()
// RequestInFlight is a prometheus metric which is a gauge of the in-flight admission requests.
RequestInFlight = func() *prometheus.GaugeVec {
return prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "controller_runtime_webhook_requests_in_flight",
Help: "Current number of admission requests being served.",
},
[]string{"webhook"},
)
}()
)
func init() {
metrics.Registry.MustRegister(
TotalRequests,
RequestLatency)
metrics.Registry.MustRegister(RequestLatency, RequestTotal, RequestInFlight)
}

View File

@@ -19,25 +19,23 @@ package webhook
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
"path/filepath"
"strconv"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics"
)
const (
certName = "tls.crt"
keyName = "tls.key"
)
// DefaultPort is the default port that the webhook server serves.
var DefaultPort = 443
@@ -52,13 +50,20 @@ type Server struct {
// It will be defaulted to 443 if unspecified.
Port int
// CertDir is the directory that contains the server key and certificate.
// If using FSCertWriter in Provisioner, the server itself will provision the certificate and
// store it in this directory.
// If using SecretCertWriter in Provisioner, the server will provision the certificate in a secret,
// the user is responsible to mount the secret to the this location for the server to consume.
// CertDir is the directory that contains the server key and certificate. The
// server key and certificate.
CertDir string
// CertName is the server certificate name. Defaults to tls.crt.
CertName string
// KeyName is the server key name. Defaults to tls.key.
KeyName string
// ClientCAName is the CA certificate name which server used to verify remote(client)'s certificate.
// Defaults to "", which means server does not verify client's certificate.
ClientCAName string
// WebhookMux is the multiplexer that handles different webhooks.
WebhookMux *http.ServeMux
@@ -71,6 +76,9 @@ type Server struct {
// defaultingOnce ensures that the default fields are only ever set once.
defaultingOnce sync.Once
// mu protects access to the webhook map & setFields for Start, Register, etc
mu sync.Mutex
}
// setDefaults does defaulting for the Server.
@@ -87,6 +95,14 @@ func (s *Server) setDefaults() {
if len(s.CertDir) == 0 {
s.CertDir = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs")
}
if len(s.CertName) == 0 {
s.CertName = "tls.crt"
}
if len(s.KeyName) == 0 {
s.KeyName = "tls.key"
}
}
// NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates
@@ -98,6 +114,9 @@ func (*Server) NeedLeaderElection() bool {
// Register marks the given webhook as being served at the given path.
// It panics if two hooks are registered on the same path.
func (s *Server) Register(path string, hook http.Handler) {
s.mu.Lock()
defer s.mu.Unlock()
s.defaultingOnce.Do(s.setDefaults)
_, found := s.webhooks[path]
if found {
@@ -106,18 +125,49 @@ func (s *Server) Register(path string, hook http.Handler) {
// TODO(directxman12): call setfields if we've already started the server
s.webhooks[path] = hook
s.WebhookMux.Handle(path, instrumentedHook(path, hook))
log.Info("registering webhook", "path", path)
regLog := log.WithValues("path", path)
regLog.Info("registering webhook")
// we've already been "started", inject dependencies here.
// Otherwise, InjectFunc will do this for us later.
if s.setFields != nil {
if err := s.setFields(hook); err != nil {
// TODO(directxman12): swallowing this error isn't great, but we'd have to
// change the signature to fix that
regLog.Error(err, "unable to inject fields into webhook during registration")
}
baseHookLog := log.WithName("webhooks")
// NB(directxman12): we don't propagate this further by wrapping setFields because it's
// unclear if this is how we want to deal with log propagation. In this specific instance,
// we want to be able to pass a logger to webhooks because they don't know their own path.
if _, err := inject.LoggerInto(baseHookLog.WithValues("webhook", path), hook); err != nil {
regLog.Error(err, "unable to logger into webhook during registration")
}
}
}
// instrumentedHook adds some instrumentation on top of the given webhook.
func instrumentedHook(path string, hookRaw http.Handler) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
startTS := time.Now()
defer func() { metrics.RequestLatency.WithLabelValues(path).Observe(time.Now().Sub(startTS).Seconds()) }()
hookRaw.ServeHTTP(resp, req)
lbl := prometheus.Labels{"webhook": path}
// TODO(directxman12): add back in metric about total requests broken down by result?
})
lat := metrics.RequestLatency.MustCurryWith(lbl)
cnt := metrics.RequestTotal.MustCurryWith(lbl)
gge := metrics.RequestInFlight.With(lbl)
// Initialize the most likely HTTP status codes.
cnt.WithLabelValues("200")
cnt.WithLabelValues("500")
return promhttp.InstrumentHandlerDuration(
lat,
promhttp.InstrumentHandlerCounter(
cnt,
promhttp.InstrumentHandlerInFlight(gge, hookRaw),
),
)
}
// Start runs the server.
@@ -128,23 +178,8 @@ func (s *Server) Start(stop <-chan struct{}) error {
baseHookLog := log.WithName("webhooks")
baseHookLog.Info("starting webhook server")
// inject fields here as opposed to in Register so that we're certain to have our setFields
// function available.
for hookPath, webhook := range s.webhooks {
if err := s.setFields(webhook); err != nil {
return err
}
// NB(directxman12): we don't propagate this further by wrapping setFields because it's
// unclear if this is how we want to deal with log propagation. In this specific instance,
// we want to be able to pass a logger to webhooks because they don't know their own path.
if _, err := inject.LoggerInto(baseHookLog.WithValues("webhook", hookPath), webhook); err != nil {
return err
}
}
certPath := filepath.Join(s.CertDir, certName)
keyPath := filepath.Join(s.CertDir, keyName)
certPath := filepath.Join(s.CertDir, s.CertName)
keyPath := filepath.Join(s.CertDir, s.KeyName)
certWatcher, err := certwatcher.New(certPath, keyPath)
if err != nil {
@@ -162,6 +197,23 @@ func (s *Server) Start(stop <-chan struct{}) error {
GetCertificate: certWatcher.GetCertificate,
}
// load CA to verify client certificate
if s.ClientCAName != "" {
certPool := x509.NewCertPool()
clientCABytes, err := ioutil.ReadFile(filepath.Join(s.CertDir, s.ClientCAName))
if err != nil {
return fmt.Errorf("failed to read client CA cert: %v", err)
}
ok := certPool.AppendCertsFromPEM(clientCABytes)
if !ok {
return fmt.Errorf("failed to append client CA cert to CA pool")
}
cfg.ClientCAs = certPool
cfg.ClientAuth = tls.RequireAndVerifyClientCert
}
listener, err := tls.Listen("tcp", net.JoinHostPort(s.Host, strconv.Itoa(int(s.Port))), cfg)
if err != nil {
return err
@@ -198,5 +250,20 @@ func (s *Server) Start(stop <-chan struct{}) error {
// InjectFunc injects the field setter into the server.
func (s *Server) InjectFunc(f inject.Func) error {
s.setFields = f
// inject fields here that weren't injected in Register because we didn't have setFields yet.
baseHookLog := log.WithName("webhooks")
for hookPath, webhook := range s.webhooks {
if err := s.setFields(webhook); err != nil {
return err
}
// NB(directxman12): we don't propagate this further by wrapping setFields because it's
// unclear if this is how we want to deal with log propagation. In this specific instance,
// we want to be able to pass a logger to webhooks because they don't know their own path.
if _, err := inject.LoggerInto(baseHookLog.WithValues("webhook", hookPath), webhook); err != nil {
return err
}
}
return nil
}