Upgrade k8s package verison (#5358)

* upgrade k8s package version

Signed-off-by: hongzhouzi <hongzhouzi@kubesphere.io>

* Script upgrade and code formatting.

Signed-off-by: hongzhouzi <hongzhouzi@kubesphere.io>

Signed-off-by: hongzhouzi <hongzhouzi@kubesphere.io>
This commit is contained in:
hongzhouzi
2022-11-15 14:56:38 +08:00
committed by GitHub
parent 5f91c1663a
commit 44167aa47a
3106 changed files with 321340 additions and 172080 deletions

View File

@@ -23,6 +23,7 @@ import (
"github.com/go-logr/logr"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
@@ -148,9 +149,9 @@ func (blder *Builder) WithOptions(options controller.Options) *Builder {
return blder
}
// WithLogger overrides the controller options's logger used.
func (blder *Builder) WithLogger(log logr.Logger) *Builder {
blder.ctrlOptions.Log = log
// WithLogConstructor overrides the controller options's LogConstructor.
func (blder *Builder) WithLogConstructor(logConstructor func(*reconcile.Request) logr.Logger) *Builder {
blder.ctrlOptions.LogConstructor = logConstructor
return blder
}
@@ -304,13 +305,31 @@ func (blder *Builder) doController(r reconcile.Reconciler) error {
ctrlOptions.CacheSyncTimeout = *globalOpts.CacheSyncTimeout
}
controllerName := blder.getControllerName(gvk)
// Setup the logger.
if ctrlOptions.Log == nil {
ctrlOptions.Log = blder.mgr.GetLogger()
if ctrlOptions.LogConstructor == nil {
log = blder.mgr.GetLogger().WithValues(
"controller", controllerName,
"controllerGroup", gvk.Group,
"controllerKind", gvk.Kind,
)
lowerCamelCaseKind := strings.ToLower(gvk.Kind[:1]) + gvk.Kind[1:]
ctrlOptions.LogConstructor = func(req *reconcile.Request) logr.Logger {
log := log
if req != nil {
log = log.WithValues(
lowerCamelCaseKind, klog.KRef(req.Namespace, req.Name),
"namespace", req.Namespace, "name", req.Name,
)
}
return log
}
}
ctrlOptions.Log = ctrlOptions.Log.WithValues("reconciler group", gvk.Group, "reconciler kind", gvk.Kind)
// Build the controller and return.
blder.ctrl, err = newController(blder.getControllerName(gvk), blder.mgr, ctrlOptions)
blder.ctrl, err = newController(controllerName, blder.mgr, ctrlOptions)
return err
}

View File

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// Package builder provides wraps other controller-runtime libraries and exposes simple
// Package builder wraps other controller-runtime libraries and exposes simple
// patterns for building common Controllers.
//
// Projects built with the builder package can trivially be rebased on top of the underlying

View File

@@ -107,6 +107,29 @@ var (
// metav1.PartialObjectMetadata to the client when fetching objects in your
// reconciler, otherwise you'll end up with a duplicate structured or
// unstructured cache.
//
// When watching a resource with OnlyMetadata, for example the v1.Pod, you
// should not Get and List using the v1.Pod type. Instead, you should use
// the special metav1.PartialObjectMetadata type.
//
// ❌ Incorrect:
//
// pod := &v1.Pod{}
// mgr.GetClient().Get(ctx, nsAndName, pod)
//
// ✅ Correct:
//
// pod := &metav1.PartialObjectMetadata{}
// pod.SetGroupVersionKind(schema.GroupVersionKind{
// Group: "",
// Version: "v1",
// Kind: "Pod",
// })
// mgr.GetClient().Get(ctx, nsAndName, pod)
//
// In the first case, controller-runtime will create another cache for the
// concrete type on top of the metadata cache; this increases memory
// consumption and leads to race conditions as caches are not in sync.
OnlyMetadata = projectAs(projectAsMetadata)
_ ForOption = OnlyMetadata

View File

@@ -17,6 +17,7 @@ limitations under the License.
package builder
import (
"errors"
"net/http"
"net/url"
"strings"
@@ -32,10 +33,12 @@ import (
// WebhookBuilder builds a Webhook.
type WebhookBuilder struct {
apiType runtime.Object
gvk schema.GroupVersionKind
mgr manager.Manager
config *rest.Config
apiType runtime.Object
withDefaulter admission.CustomDefaulter
withValidator admission.CustomValidator
gvk schema.GroupVersionKind
mgr manager.Manager
config *rest.Config
}
// WebhookManagedBy allows inform its manager.Manager.
@@ -53,6 +56,18 @@ func (blder *WebhookBuilder) For(apiType runtime.Object) *WebhookBuilder {
return blder
}
// WithDefaulter takes a admission.WithDefaulter interface, a MutatingWebhook will be wired for this type.
func (blder *WebhookBuilder) WithDefaulter(defaulter admission.CustomDefaulter) *WebhookBuilder {
blder.withDefaulter = defaulter
return blder
}
// WithValidator takes a admission.WithValidator interface, a ValidatingWebhook will be wired for this type.
func (blder *WebhookBuilder) WithValidator(validator admission.CustomValidator) *WebhookBuilder {
blder.withValidator = validator
return blder
}
// Complete builds the webhook.
func (blder *WebhookBuilder) Complete() error {
// Set the Config
@@ -69,9 +84,13 @@ func (blder *WebhookBuilder) loadRestConfig() {
}
func (blder *WebhookBuilder) registerWebhooks() error {
typ, err := blder.getType()
if err != nil {
return err
}
// Create webhook(s) for each type
var err error
blder.gvk, err = apiutil.GVKForObject(blder.apiType, blder.mgr.GetScheme())
blder.gvk, err = apiutil.GVKForObject(typ, blder.mgr.GetScheme())
if err != nil {
return err
}
@@ -88,12 +107,7 @@ func (blder *WebhookBuilder) registerWebhooks() error {
// registerDefaultingWebhook registers a defaulting webhook if th.
func (blder *WebhookBuilder) registerDefaultingWebhook() {
defaulter, isDefaulter := blder.apiType.(admission.Defaulter)
if !isDefaulter {
log.Info("skip registering a mutating webhook, admission.Defaulter interface is not implemented", "GVK", blder.gvk)
return
}
mwh := admission.DefaultingWebhookFor(defaulter)
mwh := blder.getDefaultingWebhook()
if mwh != nil {
path := generateMutatePath(blder.gvk)
@@ -108,13 +122,21 @@ func (blder *WebhookBuilder) registerDefaultingWebhook() {
}
}
func (blder *WebhookBuilder) registerValidatingWebhook() {
validator, isValidator := blder.apiType.(admission.Validator)
if !isValidator {
log.Info("skip registering a validating webhook, admission.Validator interface is not implemented", "GVK", blder.gvk)
return
func (blder *WebhookBuilder) getDefaultingWebhook() *admission.Webhook {
if defaulter := blder.withDefaulter; defaulter != nil {
return admission.WithCustomDefaulter(blder.apiType, defaulter)
}
vwh := admission.ValidatingWebhookFor(validator)
if defaulter, ok := blder.apiType.(admission.Defaulter); ok {
return admission.DefaultingWebhookFor(defaulter)
}
log.Info(
"skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called",
"GVK", blder.gvk)
return nil
}
func (blder *WebhookBuilder) registerValidatingWebhook() {
vwh := blder.getValidatingWebhook()
if vwh != nil {
path := generateValidatePath(blder.gvk)
@@ -129,22 +151,42 @@ func (blder *WebhookBuilder) registerValidatingWebhook() {
}
}
func (blder *WebhookBuilder) getValidatingWebhook() *admission.Webhook {
if validator := blder.withValidator; validator != nil {
return admission.WithCustomValidator(blder.apiType, validator)
}
if validator, ok := blder.apiType.(admission.Validator); ok {
return admission.ValidatingWebhookFor(validator)
}
log.Info(
"skip registering a validating webhook, object does not implement admission.Validator or WithValidator wasn't called",
"GVK", blder.gvk)
return nil
}
func (blder *WebhookBuilder) registerConversionWebhook() error {
ok, err := conversion.IsConvertible(blder.mgr.GetScheme(), blder.apiType)
if err != nil {
log.Error(err, "conversion check failed", "object", blder.apiType)
log.Error(err, "conversion check failed", "GVK", blder.gvk)
return err
}
if ok {
if !blder.isAlreadyHandled("/convert") {
blder.mgr.GetWebhookServer().Register("/convert", &conversion.Webhook{})
}
log.Info("conversion webhook enabled", "object", blder.apiType)
log.Info("Conversion webhook enabled", "GVK", blder.gvk)
}
return nil
}
func (blder *WebhookBuilder) getType() (runtime.Object, error) {
if blder.apiType != nil {
return blder.apiType, nil
}
return nil, errors.New("For() must be called with a valid object")
}
func (blder *WebhookBuilder) isAlreadyHandled(path string) bool {
if blder.mgr.GetWebhookServer().WebhookMux == nil {
return false

View File

@@ -86,8 +86,13 @@ type Informer interface {
HasSynced() bool
}
// ObjectSelector is an alias name of internal.Selector.
type ObjectSelector internal.Selector
// SelectorsByObject associate a client.Object's GVK to a field/label selector.
type SelectorsByObject map[client.Object]internal.Selector
// There is also `DefaultSelector` to set a global default (which will be overridden by
// a more specific setting here, if any).
type SelectorsByObject map[client.Object]ObjectSelector
// Options are the optional arguments for creating a new InformersMap object.
type Options struct {
@@ -113,6 +118,28 @@ type Options struct {
// [1] https://pkg.go.dev/k8s.io/apimachinery/pkg/fields#Selector
// [2] https://pkg.go.dev/k8s.io/apimachinery/pkg/fields#Set
SelectorsByObject SelectorsByObject
// DefaultSelector will be used as selectors for all object types
// that do not have a selector in SelectorsByObject defined.
DefaultSelector ObjectSelector
// UnsafeDisableDeepCopyByObject indicates not to deep copy objects during get or
// list objects per GVK at the specified object.
// Be very careful with this, when enabled you must DeepCopy any object before mutating it,
// otherwise you will mutate the object in the cache.
UnsafeDisableDeepCopyByObject DisableDeepCopyByObject
// TransformByObject is a map from GVKs to transformer functions which
// get applied when objects of the transformation are about to be committed
// to cache.
//
// This function is called both for new objects to enter the cache,
// and for updated objects.
TransformByObject TransformByObject
// DefaultTransform is the transform used for all GVKs which do
// not have an explicit transform func set in TransformByObject
DefaultTransform toolscache.TransformFunc
}
var defaultResyncTime = 10 * time.Hour
@@ -123,11 +150,20 @@ func New(config *rest.Config, opts Options) (Cache, error) {
if err != nil {
return nil, err
}
selectorsByGVK, err := convertToSelectorsByGVK(opts.SelectorsByObject, opts.Scheme)
selectorsByGVK, err := convertToSelectorsByGVK(opts.SelectorsByObject, opts.DefaultSelector, opts.Scheme)
if err != nil {
return nil, err
}
im := internal.NewInformersMap(config, opts.Scheme, opts.Mapper, *opts.Resync, opts.Namespace, selectorsByGVK)
disableDeepCopyByGVK, err := convertToDisableDeepCopyByGVK(opts.UnsafeDisableDeepCopyByObject, opts.Scheme)
if err != nil {
return nil, err
}
transformByGVK, err := convertToTransformByKindAndGVK(opts.TransformByObject, opts.DefaultTransform, opts.Scheme)
if err != nil {
return nil, err
}
im := internal.NewInformersMap(config, opts.Scheme, opts.Mapper, *opts.Resync, opts.Namespace, selectorsByGVK, disableDeepCopyByGVK, transformByGVK)
return &informerCache{InformersMap: im}, nil
}
@@ -136,22 +172,27 @@ func New(config *rest.Config, opts Options) (Cache, error) {
// SelectorsByObject
// WARNING: if SelectorsByObject is specified. filtered out resources are not
// returned.
// WARNING: if UnsafeDisableDeepCopy is enabled, you must DeepCopy any object
// returned from cache get/list before mutating it.
func BuilderWithOptions(options Options) NewCacheFunc {
return func(config *rest.Config, opts Options) (Cache, error) {
if opts.Scheme == nil {
opts.Scheme = options.Scheme
if options.Scheme == nil {
options.Scheme = opts.Scheme
}
if opts.Mapper == nil {
opts.Mapper = options.Mapper
if options.Mapper == nil {
options.Mapper = opts.Mapper
}
if options.Resync == nil {
options.Resync = opts.Resync
}
if options.Namespace == "" {
options.Namespace = opts.Namespace
}
if opts.Resync == nil {
opts.Resync = options.Resync
}
if opts.Namespace == "" {
opts.Namespace = options.Namespace
}
opts.SelectorsByObject = options.SelectorsByObject
return New(config, opts)
return New(config, options)
}
}
@@ -178,14 +219,57 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) {
return opts, nil
}
func convertToSelectorsByGVK(selectorsByObject SelectorsByObject, scheme *runtime.Scheme) (internal.SelectorsByGVK, error) {
func convertToSelectorsByGVK(selectorsByObject SelectorsByObject, defaultSelector ObjectSelector, scheme *runtime.Scheme) (internal.SelectorsByGVK, error) {
selectorsByGVK := internal.SelectorsByGVK{}
for object, selector := range selectorsByObject {
gvk, err := apiutil.GVKForObject(object, scheme)
if err != nil {
return nil, err
}
selectorsByGVK[gvk] = selector
selectorsByGVK[gvk] = internal.Selector(selector)
}
selectorsByGVK[schema.GroupVersionKind{}] = internal.Selector(defaultSelector)
return selectorsByGVK, nil
}
// DisableDeepCopyByObject associate a client.Object's GVK to disable DeepCopy during get or list from cache.
type DisableDeepCopyByObject map[client.Object]bool
var _ client.Object = &ObjectAll{}
// ObjectAll is the argument to represent all objects' types.
type ObjectAll struct {
client.Object
}
func convertToDisableDeepCopyByGVK(disableDeepCopyByObject DisableDeepCopyByObject, scheme *runtime.Scheme) (internal.DisableDeepCopyByGVK, error) {
disableDeepCopyByGVK := internal.DisableDeepCopyByGVK{}
for obj, disable := range disableDeepCopyByObject {
switch obj.(type) {
case ObjectAll, *ObjectAll:
disableDeepCopyByGVK[internal.GroupVersionKindAll] = disable
default:
gvk, err := apiutil.GVKForObject(obj, scheme)
if err != nil {
return nil, err
}
disableDeepCopyByGVK[gvk] = disable
}
}
return disableDeepCopyByGVK, nil
}
// TransformByObject associate a client.Object's GVK to a transformer function
// to be applied when storing the object into the cache.
type TransformByObject map[client.Object]toolscache.TransformFunc
func convertToTransformByKindAndGVK(t TransformByObject, defaultTransform toolscache.TransformFunc, scheme *runtime.Scheme) (internal.TransformFuncByObject, error) {
result := internal.NewTransformFuncByObject()
for obj, transformation := range t {
if err := result.Set(obj, scheme, transformation); err != nil {
return nil, err
}
}
result.SetDefault(defaultTransform)
return result, nil
}

View File

@@ -96,11 +96,11 @@ func (ip *informerCache) objectTypeForListObject(list client.ObjectList) (*schem
return nil, nil, err
}
if !strings.HasSuffix(gvk.Kind, "List") {
return nil, nil, fmt.Errorf("non-list type %T (kind %q) passed as output", list, gvk)
}
// we need the non-list GVK, so chop off the "List" from the end of the kind
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
if strings.HasSuffix(gvk.Kind, "List") && apimeta.IsListType(list) {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
_, isUnstructured := list.(*unstructured.UnstructuredList)
var cacheTypeObj runtime.Object
if isUnstructured {
@@ -193,8 +193,8 @@ func indexByField(indexer Informer, field string, extractor client.IndexerFunc)
rawVals := extractor(obj)
var vals []string
if ns == "" {
// if we're not doubling the keys for the namespaced case, just re-use what was returned to us
vals = rawVals
// if we're not doubling the keys for the namespaced case, just create a new slice with same length
vals = make([]string, len(rawVals))
} else {
// if we need to add non-namespaced versions too, double the length
vals = make([]string, len(rawVals)*2)

View File

@@ -46,6 +46,11 @@ type CacheReader struct {
// scopeName is the scope of the resource (namespaced or cluster-scoped).
scopeName apimeta.RESTScopeName
// disableDeepCopy indicates not to deep copy objects during get or list objects.
// Be very careful with this, when enabled you must DeepCopy any object before mutating it,
// otherwise you will mutate the object in the cache.
disableDeepCopy bool
}
// Get checks the indexer for the object and writes a copy of it if found.
@@ -76,9 +81,13 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Ob
return fmt.Errorf("cache contained %T, which is not an Object", obj)
}
// deep copy to avoid mutating cache
// TODO(directxman12): revisit the decision to always deepcopy
obj = obj.(runtime.Object).DeepCopyObject()
if c.disableDeepCopy {
// skip deep copy which might be unsafe
// you must DeepCopy any object before mutating it outside
} else {
// deep copy to avoid mutating cache
obj = obj.(runtime.Object).DeepCopyObject()
}
// Copy the value of the item in the cache to the returned value
// TODO(directxman12): this is a terrible hack, pls fix (we should have deepcopyinto)
@@ -88,7 +97,9 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Ob
return fmt.Errorf("cache had type %s, but %s was asked for", objVal.Type(), outVal.Type())
}
reflect.Indirect(outVal).Set(reflect.Indirect(objVal))
out.GetObjectKind().SetGroupVersionKind(c.groupVersionKind)
if !c.disableDeepCopy {
out.GetObjectKind().SetGroupVersionKind(c.groupVersionKind)
}
return nil
}
@@ -129,10 +140,10 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli
limitSet := listOpts.Limit > 0
runtimeObjs := make([]runtime.Object, 0, len(objs))
for i, item := range objs {
for _, item := range objs {
// if the Limit option is set and the number of items
// listed exceeds this limit, then stop reading.
if limitSet && int64(i) >= listOpts.Limit {
if limitSet && int64(len(runtimeObjs)) >= listOpts.Limit {
break
}
obj, isObj := item.(runtime.Object)
@@ -150,8 +161,15 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli
}
}
outObj := obj.DeepCopyObject()
outObj.GetObjectKind().SetGroupVersionKind(c.groupVersionKind)
var outObj runtime.Object
if c.disableDeepCopy {
// skip deep copy which might be unsafe
// you must DeepCopy any object before mutating it outside
outObj = obj
} else {
outObj = obj.DeepCopyObject()
outObj.GetObjectKind().SetGroupVersionKind(c.groupVersionKind)
}
runtimeObjs = append(runtimeObjs, outObj)
}
return apimeta.SetList(out, runtimeObjs)

View File

@@ -51,11 +51,13 @@ func NewInformersMap(config *rest.Config,
resync time.Duration,
namespace string,
selectors SelectorsByGVK,
disableDeepCopy DisableDeepCopyByGVK,
transformers TransformFuncByObject,
) *InformersMap {
return &InformersMap{
structured: newStructuredInformersMap(config, scheme, mapper, resync, namespace, selectors),
unstructured: newUnstructuredInformersMap(config, scheme, mapper, resync, namespace, selectors),
metadata: newMetadataInformersMap(config, scheme, mapper, resync, namespace, selectors),
structured: newStructuredInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, transformers),
unstructured: newUnstructuredInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, transformers),
metadata: newMetadataInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, transformers),
Scheme: scheme,
}
@@ -107,18 +109,18 @@ func (m *InformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj
// newStructuredInformersMap creates a new InformersMap for structured objects.
func newStructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration,
namespace string, selectors SelectorsByGVK) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, createStructuredListWatch)
namespace string, selectors SelectorsByGVK, disableDeepCopy DisableDeepCopyByGVK, transformers TransformFuncByObject) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, transformers, createStructuredListWatch)
}
// newUnstructuredInformersMap creates a new InformersMap for unstructured objects.
func newUnstructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration,
namespace string, selectors SelectorsByGVK) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, createUnstructuredListWatch)
namespace string, selectors SelectorsByGVK, disableDeepCopy DisableDeepCopyByGVK, transformers TransformFuncByObject) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, transformers, createUnstructuredListWatch)
}
// newMetadataInformersMap creates a new InformersMap for metadata-only objects.
func newMetadataInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration,
namespace string, selectors SelectorsByGVK) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, createMetadataListWatch)
namespace string, selectors SelectorsByGVK, disableDeepCopy DisableDeepCopyByGVK, transformers TransformFuncByObject) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, transformers, createMetadataListWatch)
}

View File

@@ -0,0 +1,35 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
import "k8s.io/apimachinery/pkg/runtime/schema"
// GroupVersionKindAll is the argument to represent all GroupVersionKind types.
var GroupVersionKindAll = schema.GroupVersionKind{}
// DisableDeepCopyByGVK associate a GroupVersionKind to disable DeepCopy during get or list from cache.
type DisableDeepCopyByGVK map[schema.GroupVersionKind]bool
// IsDisabled returns whether a GroupVersionKind is disabled DeepCopy.
func (disableByGVK DisableDeepCopyByGVK) IsDisabled(gvk schema.GroupVersionKind) bool {
if d, ok := disableByGVK[gvk]; ok {
return d
} else if d, ok = disableByGVK[GroupVersionKindAll]; ok {
return d
}
return false
}

View File

@@ -34,6 +34,7 @@ import (
"k8s.io/client-go/metadata"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
@@ -52,7 +53,10 @@ func newSpecificInformersMap(config *rest.Config,
resync time.Duration,
namespace string,
selectors SelectorsByGVK,
createListWatcher createListWatcherFunc) *specificInformersMap {
disableDeepCopy DisableDeepCopyByGVK,
transformers TransformFuncByObject,
createListWatcher createListWatcherFunc,
) *specificInformersMap {
ip := &specificInformersMap{
config: config,
Scheme: scheme,
@@ -64,7 +68,9 @@ func newSpecificInformersMap(config *rest.Config,
startWait: make(chan struct{}),
createListWatcher: createListWatcher,
namespace: namespace,
selectors: selectors,
selectors: selectors.forGVK,
disableDeepCopy: disableDeepCopy,
transformers: transformers,
}
return ip
}
@@ -128,7 +134,13 @@ type specificInformersMap struct {
// selectors are the label or field selectors that will be added to the
// ListWatch ListOptions.
selectors SelectorsByGVK
selectors func(gvk schema.GroupVersionKind) Selector
// disableDeepCopy indicates not to deep copy objects during get or list objects.
disableDeepCopy DisableDeepCopyByGVK
// transform funcs are applied to objects before they are committed to the cache
transformers TransformFuncByObject
}
// Start calls Run on each of the informers and sets started to true. Blocks on the context.
@@ -221,20 +233,25 @@ func (ip *specificInformersMap) addInformerToMap(gvk schema.GroupVersionKind, ob
ni := cache.NewSharedIndexInformer(lw, obj, resyncPeriod(ip.resync)(), cache.Indexers{
cache.NamespaceIndex: cache.MetaNamespaceIndexFunc,
})
// Check to see if there is a transformer for this gvk
if err := ni.SetTransform(ip.transformers.Get(gvk)); err != nil {
return nil, false, err
}
rm, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
return nil, false, err
}
switch obj.(type) {
case *metav1.PartialObjectMetadata, *metav1.PartialObjectMetadataList:
ni = metadataSharedIndexInformerPreserveGVK(gvk, ni)
default:
}
i := &MapEntry{
Informer: ni,
Reader: CacheReader{indexer: ni.GetIndexer(), groupVersionKind: gvk, scopeName: rm.Scope.Name()},
Reader: CacheReader{
indexer: ni.GetIndexer(),
groupVersionKind: gvk,
scopeName: rm.Scope.Name(),
disableDeepCopy: ip.disableDeepCopy.IsDisabled(gvk),
},
}
ip.informersByGVK[gvk] = i
@@ -272,19 +289,19 @@ func createStructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformer
// Create a new ListWatch for the obj
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
ip.selectors[gvk].ApplyToList(&opts)
ip.selectors(gvk).ApplyToList(&opts)
res := listObj.DeepCopyObject()
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors[gvk])
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors(gvk))
isNamespaceScoped := namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot
err := client.Get().NamespaceIfScoped(namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Do(ctx).Into(res)
return res, err
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
ip.selectors[gvk].ApplyToList(&opts)
ip.selectors(gvk).ApplyToList(&opts)
// Watch needs to be set to true separately
opts.Watch = true
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors[gvk])
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors(gvk))
isNamespaceScoped := namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot
return client.Get().NamespaceIfScoped(namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Watch(ctx)
},
@@ -314,8 +331,8 @@ func createUnstructuredListWatch(gvk schema.GroupVersionKind, ip *specificInform
// Create a new ListWatch for the obj
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
ip.selectors[gvk].ApplyToList(&opts)
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors[gvk])
ip.selectors(gvk).ApplyToList(&opts)
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors(gvk))
if namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
return dynamicClient.Resource(mapping.Resource).Namespace(namespace).List(ctx, opts)
}
@@ -323,10 +340,10 @@ func createUnstructuredListWatch(gvk schema.GroupVersionKind, ip *specificInform
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
ip.selectors[gvk].ApplyToList(&opts)
ip.selectors(gvk).ApplyToList(&opts)
// Watch needs to be set to true separately
opts.Watch = true
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors[gvk])
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors(gvk))
if namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
return dynamicClient.Resource(mapping.Resource).Namespace(namespace).Watch(ctx, opts)
}
@@ -361,27 +378,76 @@ func createMetadataListWatch(gvk schema.GroupVersionKind, ip *specificInformersM
// create the relevant listwatch
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
ip.selectors[gvk].ApplyToList(&opts)
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors[gvk])
ip.selectors(gvk).ApplyToList(&opts)
var (
list *metav1.PartialObjectMetadataList
err error
)
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors(gvk))
if namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
return client.Resource(mapping.Resource).Namespace(namespace).List(ctx, opts)
list, err = client.Resource(mapping.Resource).Namespace(namespace).List(ctx, opts)
} else {
list, err = client.Resource(mapping.Resource).List(ctx, opts)
}
return client.Resource(mapping.Resource).List(ctx, opts)
if list != nil {
for i := range list.Items {
list.Items[i].SetGroupVersionKind(gvk)
}
}
return list, err
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
ip.selectors[gvk].ApplyToList(&opts)
ip.selectors(gvk).ApplyToList(&opts)
// Watch needs to be set to true separately
opts.Watch = true
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors[gvk])
var (
watcher watch.Interface
err error
)
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors(gvk))
if namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
return client.Resource(mapping.Resource).Namespace(namespace).Watch(ctx, opts)
watcher, err = client.Resource(mapping.Resource).Namespace(namespace).Watch(ctx, opts)
} else {
watcher, err = client.Resource(mapping.Resource).Watch(ctx, opts)
}
return client.Resource(mapping.Resource).Watch(ctx, opts)
if watcher != nil {
watcher = newGVKFixupWatcher(gvk, watcher)
}
return watcher, err
},
}, nil
}
// newGVKFixupWatcher adds a wrapper that preserves the GVK information when
// events come in.
//
// This works around a bug where GVK information is not passed into mapping
// functions when using the OnlyMetadata option in the builder.
// This issue is most likely caused by kubernetes/kubernetes#80609.
// See kubernetes-sigs/controller-runtime#1484.
//
// This was originally implemented as a cache.ResourceEventHandler wrapper but
// that contained a data race which was resolved by setting the GVK in a watch
// wrapper, before the objects are written to the cache.
// See kubernetes-sigs/controller-runtime#1650.
//
// The original watch wrapper was found to be incompatible with
// k8s.io/client-go/tools/cache.Reflector so it has been re-implemented as a
// watch.Filter which is compatible.
// See kubernetes-sigs/controller-runtime#1789.
func newGVKFixupWatcher(gvk schema.GroupVersionKind, watcher watch.Interface) watch.Interface {
return watch.Filter(
watcher,
func(in watch.Event) (watch.Event, bool) {
in.Object.GetObjectKind().SetGroupVersionKind(gvk)
return in, true
},
)
}
// resyncPeriod returns a function which generates a duration each time it is
// invoked; this is so that multiple controllers don't get into lock-step and all
// hammer the apiserver with list requests simultaneously.

View File

@@ -1,71 +0,0 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
import (
"time"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/tools/cache"
)
func metadataSharedIndexInformerPreserveGVK(gvk schema.GroupVersionKind, si cache.SharedIndexInformer) cache.SharedIndexInformer {
return &sharedInformerWrapper{
gvk: gvk,
SharedIndexInformer: si,
}
}
type sharedInformerWrapper struct {
gvk schema.GroupVersionKind
cache.SharedIndexInformer
}
func (s *sharedInformerWrapper) AddEventHandler(handler cache.ResourceEventHandler) {
s.SharedIndexInformer.AddEventHandler(&handlerPreserveGVK{s.gvk, handler})
}
func (s *sharedInformerWrapper) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, resyncPeriod time.Duration) {
s.SharedIndexInformer.AddEventHandlerWithResyncPeriod(&handlerPreserveGVK{s.gvk, handler}, resyncPeriod)
}
type handlerPreserveGVK struct {
gvk schema.GroupVersionKind
cache.ResourceEventHandler
}
func (h *handlerPreserveGVK) resetGroupVersionKind(obj interface{}) {
if v, ok := obj.(schema.ObjectKind); ok {
v.SetGroupVersionKind(h.gvk)
}
}
func (h *handlerPreserveGVK) OnAdd(obj interface{}) {
h.resetGroupVersionKind(obj)
h.ResourceEventHandler.OnAdd(obj)
}
func (h *handlerPreserveGVK) OnUpdate(oldObj, newObj interface{}) {
h.resetGroupVersionKind(oldObj)
h.resetGroupVersionKind(newObj)
h.ResourceEventHandler.OnUpdate(oldObj, newObj)
}
func (h *handlerPreserveGVK) OnDelete(obj interface{}) {
h.resetGroupVersionKind(obj)
h.ResourceEventHandler.OnDelete(obj)
}

View File

@@ -26,6 +26,17 @@ import (
// SelectorsByGVK associate a GroupVersionKind to a field/label selector.
type SelectorsByGVK map[schema.GroupVersionKind]Selector
func (s SelectorsByGVK) forGVK(gvk schema.GroupVersionKind) Selector {
if specific, found := s[gvk]; found {
return specific
}
if defaultSelector, found := s[schema.GroupVersionKind{}]; found {
return defaultSelector
}
return Selector{}
}
// Selector specify the label/field selector to fill in ListOptions.
type Selector struct {
Label labels.Selector

View File

@@ -0,0 +1,50 @@
package internal
import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// TransformFuncByObject provides access to the correct transform function for
// any given GVK.
type TransformFuncByObject interface {
Set(runtime.Object, *runtime.Scheme, cache.TransformFunc) error
Get(schema.GroupVersionKind) cache.TransformFunc
SetDefault(transformer cache.TransformFunc)
}
type transformFuncByGVK struct {
defaultTransform cache.TransformFunc
transformers map[schema.GroupVersionKind]cache.TransformFunc
}
// NewTransformFuncByObject creates a new TransformFuncByObject instance.
func NewTransformFuncByObject() TransformFuncByObject {
return &transformFuncByGVK{
transformers: make(map[schema.GroupVersionKind]cache.TransformFunc),
defaultTransform: nil,
}
}
func (t *transformFuncByGVK) SetDefault(transformer cache.TransformFunc) {
t.defaultTransform = transformer
}
func (t *transformFuncByGVK) Set(obj runtime.Object, scheme *runtime.Scheme, transformer cache.TransformFunc) error {
gvk, err := apiutil.GVKForObject(obj, scheme)
if err != nil {
return err
}
t.transformers[gvk] = transformer
return nil
}
func (t transformFuncByGVK) Get(gvk schema.GroupVersionKind) cache.TransformFunc {
if val, ok := t.transformers[gvk]; ok {
return val
}
return t.defaultTransform
}

View File

@@ -55,7 +55,7 @@ func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
// create a cache for cluster scoped resources
gCache, err := New(config, opts)
if err != nil {
return nil, fmt.Errorf("error creating global cache %v", err)
return nil, fmt.Errorf("error creating global cache: %w", err)
}
for _, ns := range namespaces {

View File

@@ -22,6 +22,7 @@ import (
"sync"
"github.com/fsnotify/fsnotify"
"sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
@@ -116,8 +117,10 @@ func (cw *CertWatcher) Watch() {
// and updates the current certificate on the watcher. If a callback is set, it
// is invoked with the new certificate.
func (cw *CertWatcher) ReadCertificate() error {
metrics.ReadCertificateTotal.Inc()
cert, err := tls.LoadX509KeyPair(cw.certPath, cw.keyPath)
if err != nil {
metrics.ReadCertificateErrors.Inc()
return err
}

View File

@@ -0,0 +1,45 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package metrics
import (
"github.com/prometheus/client_golang/prometheus"
"sigs.k8s.io/controller-runtime/pkg/metrics"
)
var (
// ReadCertificateTotal is a prometheus counter metrics which holds the total
// number of certificate reads.
ReadCertificateTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "certwatcher_read_certificate_total",
Help: "Total number of certificate reads",
})
// ReadCertificateErrors is a prometheus counter metrics which holds the total
// number of errors from certificate read.
ReadCertificateErrors = prometheus.NewCounter(prometheus.CounterOpts{
Name: "certwatcher_read_certificate_errors_total",
Help: "Total number of certificate read errors",
})
)
func init() {
metrics.Registry.MustRegister(
ReadCertificateTotal,
ReadCertificateErrors,
)
}

View File

@@ -21,6 +21,7 @@ package apiutil
import (
"fmt"
"reflect"
"sync"
"k8s.io/apimachinery/pkg/api/meta"
@@ -162,8 +163,34 @@ func createRestConfig(gvk schema.GroupVersionKind, isUnstructured bool, baseConf
// Use our own custom serializer.
cfg.NegotiatedSerializer = serializerWithDecodedGVK{serializer.WithoutConversionCodecFactory{CodecFactory: codecs}}
} else {
cfg.NegotiatedSerializer = serializer.WithoutConversionCodecFactory{CodecFactory: codecs}
cfg.NegotiatedSerializer = serializerWithTargetZeroingDecode{NegotiatedSerializer: serializer.WithoutConversionCodecFactory{CodecFactory: codecs}}
}
return cfg
}
type serializerWithTargetZeroingDecode struct {
runtime.NegotiatedSerializer
}
func (s serializerWithTargetZeroingDecode) DecoderToVersion(serializer runtime.Decoder, r runtime.GroupVersioner) runtime.Decoder {
return targetZeroingDecoder{upstream: s.NegotiatedSerializer.DecoderToVersion(serializer, r)}
}
type targetZeroingDecoder struct {
upstream runtime.Decoder
}
func (t targetZeroingDecoder) Decode(data []byte, defaults *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) {
zero(into)
return t.upstream.Decode(data, defaults, into)
}
// zero zeros the value of a pointer.
func zero(x interface{}) {
if x == nil {
return
}
res := reflect.ValueOf(x).Elem()
res.Set(reflect.Zero(res.Type()))
}

View File

@@ -19,6 +19,7 @@ package apiutil
import (
"errors"
"sync"
"sync/atomic"
"golang.org/x/time/rate"
"k8s.io/apimachinery/pkg/api/meta"
@@ -38,7 +39,8 @@ type dynamicRESTMapper struct {
lazy bool
// Used for lazy init.
initOnce sync.Once
inited uint32
initMtx sync.Mutex
}
// DynamicRESTMapperOption is a functional option on the dynamicRESTMapper.
@@ -125,11 +127,18 @@ func (drm *dynamicRESTMapper) setStaticMapper() error {
// init initializes drm only once if drm is lazy.
func (drm *dynamicRESTMapper) init() (err error) {
drm.initOnce.Do(func() {
if drm.lazy {
err = drm.setStaticMapper()
// skip init if drm is not lazy or has initialized
if !drm.lazy || atomic.LoadUint32(&drm.inited) != 0 {
return nil
}
drm.initMtx.Lock()
defer drm.initMtx.Unlock()
if drm.inited == 0 {
if err = drm.setStaticMapper(); err == nil {
atomic.StoreUint32(&drm.inited, 1)
}
})
}
return err
}

View File

@@ -21,7 +21,7 @@ import (
"fmt"
"os"
"os/user"
"path"
"path/filepath"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
@@ -123,9 +123,9 @@ func loadConfig(context string) (*rest.Config, error) {
if _, ok := os.LookupEnv("HOME"); !ok {
u, err := user.Current()
if err != nil {
return nil, fmt.Errorf("could not get current user: %v", err)
return nil, fmt.Errorf("could not get current user: %w", err)
}
loadingRules.Precedence = append(loadingRules.Precedence, path.Join(u.HomeDir, clientcmd.RecommendedHomeDir, clientcmd.RecommendedFileName))
loadingRules.Precedence = append(loadingRules.Precedence, filepath.Join(u.HomeDir, clientcmd.RecommendedHomeDir, clientcmd.RecommendedFileName))
}
return loadConfigWithContext("", loadingRules, context)

View File

@@ -21,6 +21,7 @@ import (
"encoding/json"
"errors"
"fmt"
"reflect"
"strconv"
"strings"
"sync"
@@ -50,6 +51,7 @@ type versionedTracker struct {
type fakeClient struct {
tracker versionedTracker
scheme *runtime.Scheme
restMapper meta.RESTMapper
schemeWriteLock sync.Mutex
}
@@ -86,9 +88,11 @@ func NewClientBuilder() *ClientBuilder {
// ClientBuilder builds a fake client.
type ClientBuilder struct {
scheme *runtime.Scheme
restMapper meta.RESTMapper
initObject []client.Object
initLists []client.ObjectList
initRuntimeObjects []runtime.Object
objectTracker testing.ObjectTracker
}
// WithScheme sets this builder's internal scheme.
@@ -98,6 +102,15 @@ func (f *ClientBuilder) WithScheme(scheme *runtime.Scheme) *ClientBuilder {
return f
}
// WithRESTMapper sets this builder's restMapper.
// The restMapper is directly set as mapper in the Client. This can be used for example
// with a meta.DefaultRESTMapper to provide a static rest mapping.
// If not set, defaults to an empty meta.DefaultRESTMapper.
func (f *ClientBuilder) WithRESTMapper(restMapper meta.RESTMapper) *ClientBuilder {
f.restMapper = restMapper
return f
}
// WithObjects can be optionally used to initialize this fake client with client.Object(s).
func (f *ClientBuilder) WithObjects(initObjs ...client.Object) *ClientBuilder {
f.initObject = append(f.initObject, initObjs...)
@@ -116,13 +129,29 @@ func (f *ClientBuilder) WithRuntimeObjects(initRuntimeObjs ...runtime.Object) *C
return f
}
// WithObjectTracker can be optionally used to initialize this fake client with testing.ObjectTracker.
func (f *ClientBuilder) WithObjectTracker(ot testing.ObjectTracker) *ClientBuilder {
f.objectTracker = ot
return f
}
// Build builds and returns a new fake client.
func (f *ClientBuilder) Build() client.WithWatch {
if f.scheme == nil {
f.scheme = scheme.Scheme
}
if f.restMapper == nil {
f.restMapper = meta.NewDefaultRESTMapper([]schema.GroupVersion{})
}
var tracker versionedTracker
if f.objectTracker == nil {
tracker = versionedTracker{ObjectTracker: testing.NewObjectTracker(f.scheme, scheme.Codecs.UniversalDecoder()), scheme: f.scheme}
} else {
tracker = versionedTracker{ObjectTracker: f.objectTracker, scheme: f.scheme}
}
tracker := versionedTracker{ObjectTracker: testing.NewObjectTracker(f.scheme, scheme.Codecs.UniversalDecoder()), scheme: f.scheme}
for _, obj := range f.initObject {
if err := tracker.Add(obj); err != nil {
panic(fmt.Errorf("failed to add object %v to fake client: %w", obj, err))
@@ -139,8 +168,9 @@ func (f *ClientBuilder) Build() client.WithWatch {
}
}
return &fakeClient{
tracker: tracker,
scheme: f.scheme,
tracker: tracker,
scheme: f.scheme,
restMapper: f.restMapper,
}
}
@@ -169,6 +199,11 @@ func (t versionedTracker) Add(obj runtime.Object) error {
// be recognized
accessor.SetResourceVersion(trackerAddResourceVersion)
}
obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj)
if err != nil {
return err
}
if err := t.ObjectTracker.Add(obj); err != nil {
return err
}
@@ -180,7 +215,7 @@ func (t versionedTracker) Add(obj runtime.Object) error {
func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error {
accessor, err := meta.Accessor(obj)
if err != nil {
return fmt.Errorf("failed to get accessor for object: %v", err)
return fmt.Errorf("failed to get accessor for object: %w", err)
}
if accessor.GetName() == "" {
return apierrors.NewInvalid(
@@ -192,17 +227,49 @@ func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Ob
return apierrors.NewBadRequest("resourceVersion can not be set for Create requests")
}
accessor.SetResourceVersion("1")
obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj)
if err != nil {
return err
}
if err := t.ObjectTracker.Create(gvr, obj, ns); err != nil {
accessor.SetResourceVersion("")
return err
}
return nil
}
// convertFromUnstructuredIfNecessary will convert *unstructured.Unstructured for a GVK that is recocnized
// by the schema into the whatever the schema produces with New() for said GVK.
// This is required because the tracker unconditionally saves on manipulations, but its List() implementation
// tries to assign whatever it finds into a ListType it gets from schema.New() - Thus we have to ensure
// we save as the very same type, otherwise subsequent List requests will fail.
func convertFromUnstructuredIfNecessary(s *runtime.Scheme, o runtime.Object) (runtime.Object, error) {
u, isUnstructured := o.(*unstructured.Unstructured)
if !isUnstructured || !s.Recognizes(u.GroupVersionKind()) {
return o, nil
}
typed, err := s.New(u.GroupVersionKind())
if err != nil {
return nil, fmt.Errorf("scheme recognizes %s but failed to produce an object for it: %w", u.GroupVersionKind().String(), err)
}
unstructuredSerialized, err := json.Marshal(u)
if err != nil {
return nil, fmt.Errorf("failed to serialize %T: %w", unstructuredSerialized, err)
}
if err := json.Unmarshal(unstructuredSerialized, typed); err != nil {
return nil, fmt.Errorf("failed to unmarshal the content of %T into %T: %w", u, typed, err)
}
return typed, nil
}
func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error {
accessor, err := meta.Accessor(obj)
if err != nil {
return fmt.Errorf("failed to get accessor for object: %v", err)
return fmt.Errorf("failed to get accessor for object: %w", err)
}
if accessor.GetName() == "" {
@@ -248,13 +315,17 @@ func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Ob
}
intResourceVersion, err := strconv.ParseUint(oldAccessor.GetResourceVersion(), 10, 64)
if err != nil {
return fmt.Errorf("can not convert resourceVersion %q to int: %v", oldAccessor.GetResourceVersion(), err)
return fmt.Errorf("can not convert resourceVersion %q to int: %w", oldAccessor.GetResourceVersion(), err)
}
intResourceVersion++
accessor.SetResourceVersion(strconv.FormatUint(intResourceVersion, 10))
if !accessor.GetDeletionTimestamp().IsZero() && len(accessor.GetFinalizers()) == 0 {
return t.ObjectTracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName())
}
obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj)
if err != nil {
return err
}
return t.ObjectTracker.Update(gvr, obj, ns)
}
@@ -284,6 +355,7 @@ func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.O
return err
}
decoder := scheme.Codecs.UniversalDecoder()
zero(obj)
_, _, err = decoder.Decode(j, nil, obj)
return err
}
@@ -294,9 +366,7 @@ func (c *fakeClient) Watch(ctx context.Context, list client.ObjectList, opts ...
return nil, err
}
if strings.HasSuffix(gvk.Kind, "List") {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
listOpts := client.ListOptions{}
listOpts.ApplyOptions(opts)
@@ -313,12 +383,10 @@ func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...cl
originalKind := gvk.Kind
if strings.HasSuffix(gvk.Kind, "List") {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
if _, isUnstructuredList := obj.(*unstructured.UnstructuredList); isUnstructuredList && !c.scheme.Recognizes(gvk) {
// We need tor register the ListKind with UnstructuredList:
// We need to register the ListKind with UnstructuredList:
// https://github.com/kubernetes/kubernetes/blob/7b2776b89fb1be28d4e9203bdeec079be903c103/staging/src/k8s.io/client-go/dynamic/fake/simple.go#L44-L51
c.schemeWriteLock.Lock()
c.scheme.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{})
@@ -346,6 +414,7 @@ func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...cl
return err
}
decoder := scheme.Codecs.UniversalDecoder()
zero(obj)
_, _, err = decoder.Decode(j, nil, obj)
if err != nil {
return err
@@ -373,8 +442,7 @@ func (c *fakeClient) Scheme() *runtime.Scheme {
}
func (c *fakeClient) RESTMapper() meta.RESTMapper {
// TODO: Implement a fake RESTMapper.
return nil
return c.restMapper
}
func (c *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
@@ -419,6 +487,34 @@ func (c *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...clie
delOptions := client.DeleteOptions{}
delOptions.ApplyOptions(opts)
for _, dryRunOpt := range delOptions.DryRun {
if dryRunOpt == metav1.DryRunAll {
return nil
}
}
// Check the ResourceVersion if that Precondition was specified.
if delOptions.Preconditions != nil && delOptions.Preconditions.ResourceVersion != nil {
name := accessor.GetName()
dbObj, err := c.tracker.Get(gvr, accessor.GetNamespace(), name)
if err != nil {
return err
}
oldAccessor, err := meta.Accessor(dbObj)
if err != nil {
return err
}
actualRV := oldAccessor.GetResourceVersion()
expectRV := *delOptions.Preconditions.ResourceVersion
if actualRV != expectRV {
msg := fmt.Sprintf(
"the ResourceVersion in the precondition (%s) does not match the ResourceVersion in record (%s). "+
"The object might have been modified",
expectRV, actualRV)
return apierrors.NewConflict(gvr.GroupResource(), name, errors.New(msg))
}
}
return c.deleteObject(gvr, accessor)
}
@@ -431,6 +527,12 @@ func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ..
dcOptions := client.DeleteAllOfOptions{}
dcOptions.ApplyOptions(opts)
for _, dryRunOpt := range dcOptions.DryRun {
if dryRunOpt == metav1.DryRunAll {
return nil
}
}
gvr, _ := meta.UnsafeGuessKindToResource(gvk)
o, err := c.tracker.List(gvr, gvk, dcOptions.Namespace)
if err != nil {
@@ -527,6 +629,7 @@ func (c *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.
return err
}
decoder := scheme.Codecs.UniversalDecoder()
zero(obj)
_, _, err = decoder.Decode(j, nil, obj)
return err
}
@@ -673,3 +776,12 @@ func allowsCreateOnUpdate(gvk schema.GroupVersionKind) bool {
return false
}
// zero zeros the value of a pointer.
func zero(x interface{}) {
if x == nil {
return
}
res := reflect.ValueOf(x).Elem()
res.Set(reflect.Zero(res.Type()))
}

View File

@@ -146,9 +146,7 @@ func (mc *metadataClient) List(ctx context.Context, obj ObjectList, opts ...List
}
gvk := metadata.GroupVersionKind()
if strings.HasSuffix(gvk.Kind, "List") {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
listOpts := ListOptions{}
listOpts.ApplyOptions(opts)

View File

@@ -18,14 +18,11 @@ package client
import (
"context"
"errors"
"fmt"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
"sigs.k8s.io/controller-runtime/pkg/internal/objectutil"
)
// NewNamespacedClient wraps an existing client enforcing the namespace value.
@@ -55,51 +52,11 @@ func (n *namespacedClient) RESTMapper() meta.RESTMapper {
return n.client.RESTMapper()
}
// isNamespaced returns true if the object is namespace scoped.
// For unstructured objects the gvk is found from the object itself.
// TODO: this is repetitive code. Remove this and use ojectutil.IsNamespaced.
func isNamespaced(c Client, obj runtime.Object) (bool, error) {
var gvk schema.GroupVersionKind
var err error
_, isUnstructured := obj.(*unstructured.Unstructured)
_, isUnstructuredList := obj.(*unstructured.UnstructuredList)
isUnstructured = isUnstructured || isUnstructuredList
if isUnstructured {
gvk = obj.GetObjectKind().GroupVersionKind()
} else {
gvk, err = apiutil.GVKForObject(obj, c.Scheme())
if err != nil {
return false, err
}
}
gk := schema.GroupKind{
Group: gvk.Group,
Kind: gvk.Kind,
}
restmapping, err := c.RESTMapper().RESTMapping(gk)
if err != nil {
return false, fmt.Errorf("failed to get restmapping: %w", err)
}
scope := restmapping.Scope.Name()
if scope == "" {
return false, errors.New("scope cannot be identified, empty scope returned")
}
if scope != meta.RESTScopeNameRoot {
return true, nil
}
return false, nil
}
// Create implements clinet.Client.
func (n *namespacedClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error {
isNamespaceScoped, err := isNamespaced(n.client, obj)
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper())
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
return fmt.Errorf("error finding the scope of the object: %w", err)
}
objectNamespace := obj.GetNamespace()
@@ -115,9 +72,9 @@ func (n *namespacedClient) Create(ctx context.Context, obj Object, opts ...Creat
// Update implements client.Client.
func (n *namespacedClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
isNamespaceScoped, err := isNamespaced(n.client, obj)
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper())
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
return fmt.Errorf("error finding the scope of the object: %w", err)
}
objectNamespace := obj.GetNamespace()
@@ -133,9 +90,9 @@ func (n *namespacedClient) Update(ctx context.Context, obj Object, opts ...Updat
// Delete implements client.Client.
func (n *namespacedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error {
isNamespaceScoped, err := isNamespaced(n.client, obj)
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper())
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
return fmt.Errorf("error finding the scope of the object: %w", err)
}
objectNamespace := obj.GetNamespace()
@@ -151,9 +108,9 @@ func (n *namespacedClient) Delete(ctx context.Context, obj Object, opts ...Delet
// DeleteAllOf implements client.Client.
func (n *namespacedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error {
isNamespaceScoped, err := isNamespaced(n.client, obj)
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper())
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
return fmt.Errorf("error finding the scope of the object: %w", err)
}
if isNamespaceScoped {
@@ -164,9 +121,9 @@ func (n *namespacedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...
// Patch implements client.Client.
func (n *namespacedClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
isNamespaceScoped, err := isNamespaced(n.client, obj)
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper())
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
return fmt.Errorf("error finding the scope of the object: %w", err)
}
objectNamespace := obj.GetNamespace()
@@ -182,9 +139,9 @@ func (n *namespacedClient) Patch(ctx context.Context, obj Object, patch Patch, o
// Get implements client.Client.
func (n *namespacedClient) Get(ctx context.Context, key ObjectKey, obj Object) error {
isNamespaceScoped, err := isNamespaced(n.client, obj)
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper())
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
return fmt.Errorf("error finding the scope of the object: %w", err)
}
if isNamespaceScoped {
if key.Namespace != "" && key.Namespace != n.namespace {
@@ -219,9 +176,10 @@ type namespacedClientStatusWriter struct {
// Update implements client.StatusWriter.
func (nsw *namespacedClientStatusWriter) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
isNamespaceScoped, err := isNamespaced(nsw.namespacedclient, obj)
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, nsw.namespacedclient.Scheme(), nsw.namespacedclient.RESTMapper())
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
return fmt.Errorf("error finding the scope of the object: %w", err)
}
objectNamespace := obj.GetNamespace()
@@ -237,9 +195,10 @@ func (nsw *namespacedClientStatusWriter) Update(ctx context.Context, obj Object,
// Patch implements client.StatusWriter.
func (nsw *namespacedClientStatusWriter) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
isNamespaceScoped, err := isNamespaced(nsw.namespacedclient, obj)
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, nsw.namespacedclient.Scheme(), nsw.namespacedclient.RESTMapper())
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
return fmt.Errorf("error finding the scope of the object: %w", err)
}
objectNamespace := obj.GetNamespace()

View File

@@ -318,7 +318,7 @@ func (p PropagationPolicy) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
// pre-parsed selectors (since generally, selectors will be executed
// against the cache).
type ListOptions struct {
// LabelSelector filters results by label. Use SetLabelSelector to
// LabelSelector filters results by label. Use labels.Parse() to
// set from raw string form.
LabelSelector labels.Selector
// FieldSelector filters results by a particular field. In order

View File

@@ -95,8 +95,7 @@ func (uc *unstructuredClient) Update(ctx context.Context, obj Object, opts ...Up
// Delete implements client.Client.
func (uc *unstructuredClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error {
_, ok := obj.(*unstructured.Unstructured)
if !ok {
if _, ok := obj.(*unstructured.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
@@ -118,8 +117,7 @@ func (uc *unstructuredClient) Delete(ctx context.Context, obj Object, opts ...De
// DeleteAllOf implements client.Client.
func (uc *unstructuredClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error {
_, ok := obj.(*unstructured.Unstructured)
if !ok {
if _, ok := obj.(*unstructured.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
@@ -141,8 +139,7 @@ func (uc *unstructuredClient) DeleteAllOf(ctx context.Context, obj Object, opts
// Patch implements client.Client.
func (uc *unstructuredClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
_, ok := obj.(*unstructured.Unstructured)
if !ok {
if _, ok := obj.(*unstructured.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
@@ -201,9 +198,7 @@ func (uc *unstructuredClient) List(ctx context.Context, obj ObjectList, opts ...
}
gvk := u.GroupVersionKind()
if strings.HasSuffix(gvk.Kind, "List") {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
listOpts := ListOptions{}
listOpts.ApplyOptions(opts)
@@ -222,8 +217,7 @@ func (uc *unstructuredClient) List(ctx context.Context, obj ObjectList, opts ...
}
func (uc *unstructuredClient) UpdateStatus(ctx context.Context, obj Object, opts ...UpdateOption) error {
_, ok := obj.(*unstructured.Unstructured)
if !ok {
if _, ok := obj.(*unstructured.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}

View File

@@ -69,9 +69,7 @@ func (w *watchingClient) listOpts(opts ...ListOption) ListOptions {
func (w *watchingClient) metadataWatch(ctx context.Context, obj *metav1.PartialObjectMetadataList, opts ...ListOption) (watch.Interface, error) {
gvk := obj.GroupVersionKind()
if strings.HasSuffix(gvk.Kind, "List") {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
listOpts := w.listOpts(opts...)
@@ -85,9 +83,7 @@ func (w *watchingClient) metadataWatch(ctx context.Context, obj *metav1.PartialO
func (w *watchingClient) unstructuredWatch(ctx context.Context, obj *unstructured.UnstructuredList, opts ...ListOption) (watch.Interface, error) {
gvk := obj.GroupVersionKind()
if strings.HasSuffix(gvk.Kind, "List") {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
r, err := w.client.unstructuredClient.cache.getResource(obj)
if err != nil {

View File

@@ -245,7 +245,7 @@ func setOptionsDefaults(options Options) Options {
}
}
if options.Logger == nil {
if options.Logger.GetSink() == nil {
options.Logger = logf.RuntimeLog.WithName("cluster")
}

View File

@@ -18,7 +18,7 @@ package config
import (
"fmt"
ioutil "io/ioutil"
"os"
"sync"
"k8s.io/apimachinery/pkg/runtime"
@@ -96,7 +96,7 @@ func (d *DeferredFileLoader) loadFile() {
return
}
content, err := ioutil.ReadFile(d.path)
content, err := os.ReadFile(d.path)
if err != nil {
d.err = fmt.Errorf("could not read file at %s", d.path)
return

View File

@@ -1,3 +1,4 @@
//go:build !ignore_autogenerated
// +build !ignore_autogenerated
// Code generated by controller-gen. DO NOT EDIT.

View File

@@ -23,6 +23,8 @@ import (
"github.com/go-logr/logr"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/internal/controller"
"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -45,13 +47,16 @@ type Options struct {
// The overall is a token bucket and the per-item is exponential.
RateLimiter ratelimiter.RateLimiter
// Log is the logger used for this controller and passed to each reconciliation
// request via the context field.
Log logr.Logger
// LogConstructor is used to construct a logger used for this controller and passed
// to each reconciliation via the context field.
LogConstructor func(request *reconcile.Request) logr.Logger
// CacheSyncTimeout refers to the time limit set to wait for syncing caches.
// Defaults to 2 minutes if not set.
CacheSyncTimeout time.Duration
// RecoverPanic indicates whether the panic caused by reconcile should be recovered.
RecoverPanic bool
}
// Controller implements a Kubernetes API. A Controller manages a work queue fed reconcile.Requests
@@ -101,8 +106,20 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller
return nil, fmt.Errorf("must specify Name for Controller")
}
if options.Log == nil {
options.Log = mgr.GetLogger()
if options.LogConstructor == nil {
log := mgr.GetLogger().WithValues(
"controller", name,
)
options.LogConstructor = func(req *reconcile.Request) logr.Logger {
log := log
if req != nil {
log = log.WithValues(
"object", klog.KRef(req.Namespace, req.Name),
"namespace", req.Namespace, "name", req.Name,
)
}
return log
}
}
if options.MaxConcurrentReconciles <= 0 {
@@ -132,6 +149,7 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller
CacheSyncTimeout: options.CacheSyncTimeout,
SetFields: mgr.SetFields,
Name: name,
Log: options.Log.WithName("controller").WithName(name),
LogConstructor: options.LogConstructor,
RecoverPanic: options.RecoverPanic,
}, nil
}

View File

@@ -345,30 +345,35 @@ func mutate(f MutateFn, key client.ObjectKey, obj client.Object) error {
return nil
}
// MutateFn is a function which mutates the existing object into it's desired state.
// MutateFn is a function which mutates the existing object into its desired state.
type MutateFn func() error
// AddFinalizer accepts an Object and adds the provided finalizer if not present.
func AddFinalizer(o client.Object, finalizer string) {
// It returns an indication of whether it updated the object's list of finalizers.
func AddFinalizer(o client.Object, finalizer string) (finalizersUpdated bool) {
f := o.GetFinalizers()
for _, e := range f {
if e == finalizer {
return
return false
}
}
o.SetFinalizers(append(f, finalizer))
return true
}
// RemoveFinalizer accepts an Object and removes the provided finalizer if present.
func RemoveFinalizer(o client.Object, finalizer string) {
// It returns an indication of whether it updated the object's list of finalizers.
func RemoveFinalizer(o client.Object, finalizer string) (finalizersUpdated bool) {
f := o.GetFinalizers()
for i := 0; i < len(f); i++ {
if f[i] == finalizer {
f = append(f[:i], f[i+1:]...)
i--
finalizersUpdated = true
}
}
o.SetFinalizers(f)
return
}
// ContainsFinalizer checks an Object that the provided finalizer is present.

View File

@@ -20,19 +20,16 @@ import (
"bufio"
"bytes"
"context"
"encoding/base64"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"time"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
@@ -42,9 +39,10 @@ import (
"k8s.io/client-go/rest"
"k8s.io/client-go/util/retry"
"k8s.io/utils/pointer"
"sigs.k8s.io/yaml"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/conversion"
"sigs.k8s.io/yaml"
)
// CRDInstallOptions are the options for installing CRDs.
@@ -62,7 +60,7 @@ type CRDInstallOptions struct {
Paths []string
// CRDs is a list of CRDs to install
CRDs []client.Object
CRDs []*apiextensionsv1.CustomResourceDefinition
// ErrorIfPathMissing will cause an error if a Path does not exist
ErrorIfPathMissing bool
@@ -90,7 +88,7 @@ const defaultPollInterval = 100 * time.Millisecond
const defaultMaxWait = 10 * time.Second
// InstallCRDs installs a collection of CRDs into a cluster by reading the crd yaml files from a directory.
func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]client.Object, error) {
func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]*apiextensionsv1.CustomResourceDefinition, error) {
defaultCRDOptions(&options)
// Read the CRD yamls into options.CRDs
@@ -142,49 +140,14 @@ func defaultCRDOptions(o *CRDInstallOptions) {
}
// WaitForCRDs waits for the CRDs to appear in discovery.
func WaitForCRDs(config *rest.Config, crds []client.Object, options CRDInstallOptions) error {
func WaitForCRDs(config *rest.Config, crds []*apiextensionsv1.CustomResourceDefinition, options CRDInstallOptions) error {
// Add each CRD to a map of GroupVersion to Resource
waitingFor := map[schema.GroupVersion]*sets.String{}
for _, crd := range runtimeCRDListToUnstructured(crds) {
for _, crd := range crds {
gvs := []schema.GroupVersion{}
crdGroup, _, err := unstructured.NestedString(crd.Object, "spec", "group")
if err != nil {
return err
}
crdPlural, _, err := unstructured.NestedString(crd.Object, "spec", "names", "plural")
if err != nil {
return err
}
crdVersion, _, err := unstructured.NestedString(crd.Object, "spec", "version")
if err != nil {
return err
}
versions, found, err := unstructured.NestedSlice(crd.Object, "spec", "versions")
if err != nil {
return err
}
// gvs should be added here only if single version is found. If multiple version is found we will add those version
// based on the version is served or not.
if crdVersion != "" && !found {
gvs = append(gvs, schema.GroupVersion{Group: crdGroup, Version: crdVersion})
}
for _, version := range versions {
versionMap, ok := version.(map[string]interface{})
if !ok {
continue
}
served, _, err := unstructured.NestedBool(versionMap, "served")
if err != nil {
return err
}
if served {
versionName, _, err := unstructured.NestedString(versionMap, "name")
if err != nil {
return err
}
gvs = append(gvs, schema.GroupVersion{Group: crdGroup, Version: versionName})
for _, version := range crd.Spec.Versions {
if version.Served {
gvs = append(gvs, schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name})
}
}
@@ -195,7 +158,7 @@ func WaitForCRDs(config *rest.Config, crds []client.Object, options CRDInstallOp
waitingFor[gv] = &sets.String{}
}
// Add the Resource
waitingFor[gv].Insert(crdPlural)
waitingFor[gv].Insert(crd.Spec.Names.Plural)
}
}
@@ -263,7 +226,8 @@ func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error {
}
// Uninstall each CRD
for _, crd := range runtimeCRDListToUnstructured(options.CRDs) {
for _, crd := range options.CRDs {
crd := crd
log.V(1).Info("uninstalling CRD", "crd", crd.GetName())
if err := cs.Delete(context.TODO(), crd); err != nil {
// If CRD is not found, we can consider success
@@ -277,14 +241,15 @@ func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error {
}
// CreateCRDs creates the CRDs.
func CreateCRDs(config *rest.Config, crds []client.Object) error {
func CreateCRDs(config *rest.Config, crds []*apiextensionsv1.CustomResourceDefinition) error {
cs, err := client.New(config, client.Options{})
if err != nil {
return fmt.Errorf("unable to create client: %w", err)
}
// Create each CRD
for _, crd := range runtimeCRDListToUnstructured(crds) {
for _, crd := range crds {
crd := crd
log.V(1).Info("installing CRD", "crd", crd.GetName())
existingCrd := crd.DeepCopy()
err := cs.Get(context.TODO(), client.ObjectKey{Name: crd.GetName()}, existingCrd)
@@ -312,22 +277,21 @@ func CreateCRDs(config *rest.Config, crds []client.Object) error {
}
// renderCRDs iterate through options.Paths and extract all CRD files.
func renderCRDs(options *CRDInstallOptions) ([]client.Object, error) {
var (
err error
info os.FileInfo
files []os.FileInfo
)
func renderCRDs(options *CRDInstallOptions) ([]*apiextensionsv1.CustomResourceDefinition, error) {
type GVKN struct {
GVK schema.GroupVersionKind
Name string
}
crds := map[GVKN]*unstructured.Unstructured{}
crds := map[GVKN]*apiextensionsv1.CustomResourceDefinition{}
for _, path := range options.Paths {
var filePath = path
var (
err error
info os.FileInfo
files []string
filePath = path
)
// Return the error if ErrorIfPathMissing exists
if info, err = os.Stat(path); os.IsNotExist(err) {
@@ -338,9 +302,15 @@ func renderCRDs(options *CRDInstallOptions) ([]client.Object, error) {
}
if !info.IsDir() {
filePath, files = filepath.Dir(path), []os.FileInfo{info}
} else if files, err = ioutil.ReadDir(path); err != nil {
return nil, err
filePath, files = filepath.Dir(path), []string{info.Name()}
} else {
entries, err := os.ReadDir(path)
if err != nil {
return nil, err
}
for _, e := range entries {
files = append(files, e.Name())
}
}
log.V(1).Info("reading CRDs from path", "path", path)
@@ -361,7 +331,7 @@ func renderCRDs(options *CRDInstallOptions) ([]client.Object, error) {
}
// Converting map to a list to return
res := []client.Object{}
res := []*apiextensionsv1.CustomResourceDefinition{}
for _, obj := range crds {
res = append(res, obj)
}
@@ -370,12 +340,7 @@ func renderCRDs(options *CRDInstallOptions) ([]client.Object, error) {
// modifyConversionWebhooks takes all the registered CustomResourceDefinitions and applies modifications
// to conditionally enable webhooks if the type is registered within the scheme.
//
// The complexity of this function is high mostly due to all the edge cases that we need to handle:
// CRDv1beta1, CRDv1, and their unstructured counterpart.
//
// We should be able to simplify this code once we drop support for v1beta1 and standardize around the typed CRDv1 object.
func modifyConversionWebhooks(crds []client.Object, scheme *runtime.Scheme, webhookOptions WebhookInstallOptions) error { //nolint:gocyclo
func modifyConversionWebhooks(crds []*apiextensionsv1.CustomResourceDefinition, scheme *runtime.Scheme, webhookOptions WebhookInstallOptions) error {
if len(webhookOptions.LocalServingCAData) == 0 {
return nil
}
@@ -399,210 +364,74 @@ func modifyConversionWebhooks(crds []client.Object, scheme *runtime.Scheme, webh
}
url := pointer.StringPtr(fmt.Sprintf("https://%s/convert", hostPort))
for _, crd := range crds {
switch c := crd.(type) {
case *apiextensionsv1beta1.CustomResourceDefinition:
// Continue if we're preserving unknown fields.
//
// preserveUnknownFields defaults to true if `nil` in v1beta1.
if c.Spec.PreserveUnknownFields == nil || *c.Spec.PreserveUnknownFields {
continue
}
// Continue if the GroupKind isn't registered as being convertible.
if _, ok := convertibles[schema.GroupKind{
Group: c.Spec.Group,
Kind: c.Spec.Names.Kind,
}]; !ok {
continue
}
c.Spec.Conversion.Strategy = apiextensionsv1beta1.WebhookConverter
c.Spec.Conversion.WebhookClientConfig.Service = nil
c.Spec.Conversion.WebhookClientConfig = &apiextensionsv1beta1.WebhookClientConfig{
Service: nil,
URL: url,
CABundle: webhookOptions.LocalServingCAData,
}
case *apiextensionsv1.CustomResourceDefinition:
// Continue if we're preserving unknown fields.
if c.Spec.PreserveUnknownFields {
continue
}
// Continue if the GroupKind isn't registered as being convertible.
if _, ok := convertibles[schema.GroupKind{
Group: c.Spec.Group,
Kind: c.Spec.Names.Kind,
}]; !ok {
continue
}
c.Spec.Conversion.Strategy = apiextensionsv1.WebhookConverter
c.Spec.Conversion.Webhook.ClientConfig.Service = nil
c.Spec.Conversion.Webhook.ClientConfig = &apiextensionsv1.WebhookClientConfig{
Service: nil,
URL: url,
CABundle: webhookOptions.LocalServingCAData,
}
case *unstructured.Unstructured:
webhookClientConfig := map[string]interface{}{
"url": *url,
"caBundle": base64.StdEncoding.EncodeToString(webhookOptions.LocalServingCAData),
}
switch c.GroupVersionKind().Version {
case "v1beta1":
// Continue if we're preserving unknown fields.
//
// preserveUnknownFields defaults to true if `nil` in v1beta1.
if preserve, found, err := unstructured.NestedBool(c.Object, "spec", "preserveUnknownFields"); preserve || !found {
continue
} else if err != nil {
return err
}
// Continue if the GroupKind isn't registered as being convertible.
group, found, err := unstructured.NestedString(c.Object, "spec", "group")
if !found {
continue
} else if err != nil {
return err
}
kind, found, err := unstructured.NestedString(c.Object, "spec", "names", "kind")
if !found {
continue
} else if err != nil {
return err
}
if _, ok := convertibles[schema.GroupKind{
Group: group,
Kind: kind,
}]; !ok {
continue
}
// Set the strategy.
if err := unstructured.SetNestedField(
c.Object,
string(apiextensionsv1beta1.WebhookConverter),
"spec", "conversion", "strategy"); err != nil {
return err
}
// Set the conversion review versions.
if err := unstructured.SetNestedStringSlice(
c.Object,
[]string{"v1beta1"},
"spec", "conversion", "webhook", "clientConfig"); err != nil {
return err
}
// Set the client configuration.
if err := unstructured.SetNestedMap(
c.Object,
webhookClientConfig,
"spec", "conversion", "webhookClientConfig"); err != nil {
return err
}
case "v1":
if preserve, _, err := unstructured.NestedBool(c.Object, "spec", "preserveUnknownFields"); preserve {
continue
} else if err != nil {
return err
}
// Continue if the GroupKind isn't registered as being convertible.
group, found, err := unstructured.NestedString(c.Object, "spec", "group")
if !found {
continue
} else if err != nil {
return err
}
kind, found, err := unstructured.NestedString(c.Object, "spec", "names", "kind")
if !found {
continue
} else if err != nil {
return err
}
if _, ok := convertibles[schema.GroupKind{
Group: group,
Kind: kind,
}]; !ok {
continue
}
// Set the strategy.
if err := unstructured.SetNestedField(
c.Object,
string(apiextensionsv1.WebhookConverter),
"spec", "conversion", "strategy"); err != nil {
return err
}
// Set the conversion review versions.
if err := unstructured.SetNestedStringSlice(
c.Object,
[]string{"v1", "v1beta1"},
"spec", "conversion", "webhook", "conversionReviewVersions"); err != nil {
return err
}
// Set the client configuration.
if err := unstructured.SetNestedMap(
c.Object,
webhookClientConfig,
"spec", "conversion", "webhook", "clientConfig"); err != nil {
return err
}
for i := range crds {
// Continue if we're preserving unknown fields.
if crds[i].Spec.PreserveUnknownFields {
continue
}
// Continue if the GroupKind isn't registered as being convertible.
if _, ok := convertibles[schema.GroupKind{
Group: crds[i].Spec.Group,
Kind: crds[i].Spec.Names.Kind,
}]; !ok {
continue
}
if crds[i].Spec.Conversion == nil {
crds[i].Spec.Conversion = &apiextensionsv1.CustomResourceConversion{
Webhook: &apiextensionsv1.WebhookConversion{},
}
}
crds[i].Spec.Conversion.Strategy = apiextensionsv1.WebhookConverter
crds[i].Spec.Conversion.Webhook.ConversionReviewVersions = []string{"v1", "v1beta1"}
crds[i].Spec.Conversion.Webhook.ClientConfig = &apiextensionsv1.WebhookClientConfig{
Service: nil,
URL: url,
CABundle: webhookOptions.LocalServingCAData,
}
}
return nil
}
// readCRDs reads the CRDs from files and Unmarshals them into structs.
func readCRDs(basePath string, files []os.FileInfo) ([]*unstructured.Unstructured, error) {
var crds []*unstructured.Unstructured
func readCRDs(basePath string, files []string) ([]*apiextensionsv1.CustomResourceDefinition, error) {
var crds []*apiextensionsv1.CustomResourceDefinition
// White list the file extensions that may contain CRDs
crdExts := sets.NewString(".json", ".yaml", ".yml")
for _, file := range files {
// Only parse allowlisted file types
if !crdExts.Has(filepath.Ext(file.Name())) {
if !crdExts.Has(filepath.Ext(file)) {
continue
}
// Unmarshal CRDs from file into structs
docs, err := readDocuments(filepath.Join(basePath, file.Name()))
docs, err := readDocuments(filepath.Join(basePath, file))
if err != nil {
return nil, err
}
for _, doc := range docs {
crd := &unstructured.Unstructured{}
crd := &apiextensionsv1.CustomResourceDefinition{}
if err = yaml.Unmarshal(doc, crd); err != nil {
return nil, err
}
// Check that it is actually a CRD
crdKind, _, err := unstructured.NestedString(crd.Object, "spec", "names", "kind")
if err != nil {
return nil, err
}
crdGroup, _, err := unstructured.NestedString(crd.Object, "spec", "group")
if err != nil {
return nil, err
}
if crd.GetKind() != "CustomResourceDefinition" || crdKind == "" || crdGroup == "" {
if crd.Kind != "CustomResourceDefinition" || crd.Spec.Names.Kind == "" || crd.Spec.Group == "" {
continue
}
crds = append(crds, crd)
}
log.V(1).Info("read CRDs from file", "file", file.Name())
log.V(1).Info("read CRDs from file", "file", file)
}
return crds, nil
}
// readDocuments reads documents from file.
func readDocuments(fp string) ([][]byte, error) {
b, err := ioutil.ReadFile(fp) //nolint:gosec
b, err := os.ReadFile(fp)
if err != nil {
return nil, err
}
@@ -613,7 +442,7 @@ func readDocuments(fp string) ([][]byte, error) {
// Read document
doc, err := reader.Read()
if err != nil {
if err == io.EOF {
if errors.Is(err, io.EOF) {
break
}

View File

@@ -18,20 +18,16 @@ package envtest
import (
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"k8s.io/client-go/kubernetes/scheme"
)
var (
crdScheme = runtime.NewScheme()
crdScheme = scheme.Scheme
)
// init is required to correctly initialize the crdScheme package variable.
func init() {
_ = apiextensionsv1.AddToScheme(crdScheme)
_ = apiextensionsv1beta1.AddToScheme(crdScheme)
}
// mergePaths merges two string slices containing paths.
@@ -55,32 +51,19 @@ func mergePaths(s1, s2 []string) []string {
// mergeCRDs merges two CRD slices using their names.
// This function makes no guarantees about order of the merged slice.
func mergeCRDs(s1, s2 []client.Object) []client.Object {
m := make(map[string]*unstructured.Unstructured)
for _, obj := range runtimeCRDListToUnstructured(s1) {
func mergeCRDs(s1, s2 []*apiextensionsv1.CustomResourceDefinition) []*apiextensionsv1.CustomResourceDefinition {
m := make(map[string]*apiextensionsv1.CustomResourceDefinition)
for _, obj := range s1 {
m[obj.GetName()] = obj
}
for _, obj := range runtimeCRDListToUnstructured(s2) {
for _, obj := range s2 {
m[obj.GetName()] = obj
}
merged := make([]client.Object, len(m))
merged := make([]*apiextensionsv1.CustomResourceDefinition, len(m))
i := 0
for _, obj := range m {
merged[i] = obj
merged[i] = obj.DeepCopy()
i++
}
return merged
}
func runtimeCRDListToUnstructured(l []client.Object) []*unstructured.Unstructured {
res := []*unstructured.Unstructured{}
for _, obj := range l {
u := &unstructured.Unstructured{}
if err := crdScheme.Convert(obj, u, nil); err != nil {
log.Error(err, "error converting to unstructured object", "object-kind", obj.GetObjectKind())
continue
}
res = append(res, u)
}
return res
}

View File

@@ -22,15 +22,15 @@ import (
"strings"
"time"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/controlplane"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/process"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
var log = logf.RuntimeLog.WithName("test-env")
@@ -136,7 +136,7 @@ type Environment struct {
// CRDs is a list of CRDs to install.
// If both this field and CRDs field in CRDInstallOptions are specified, the
// values are merged.
CRDs []client.Object
CRDs []*apiextensionsv1.CustomResourceDefinition
// CRDDirectoryPaths is a list of paths containing CRD yaml or json configs.
// If both this field and Paths field in CRDInstallOptions are specified, the

View File

@@ -15,26 +15,27 @@ package envtest
import (
"context"
"encoding/base64"
"fmt"
"io/ioutil"
"net"
"os"
"path/filepath"
"time"
admissionv1 "k8s.io/api/admissionregistration/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
"sigs.k8s.io/yaml"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/addr"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/certs"
"sigs.k8s.io/yaml"
)
// WebhookInstallOptions are the options for installing mutating or validating webhooks.
@@ -43,10 +44,10 @@ type WebhookInstallOptions struct {
Paths []string
// MutatingWebhooks is a list of MutatingWebhookConfigurations to install
MutatingWebhooks []client.Object
MutatingWebhooks []*admissionv1.MutatingWebhookConfiguration
// ValidatingWebhooks is a list of ValidatingWebhookConfigurations to install
ValidatingWebhooks []client.Object
ValidatingWebhooks []*admissionv1.ValidatingWebhookConfiguration
// IgnoreErrorIfPathMissing will ignore an error if a DirectoryPath does not exist when set to true
IgnoreErrorIfPathMissing bool
@@ -88,64 +89,34 @@ func (o *WebhookInstallOptions) ModifyWebhookDefinitions() error {
return err
}
for i, unstructuredHook := range runtimeListToUnstructured(o.MutatingWebhooks) {
webhooks, found, err := unstructured.NestedSlice(unstructuredHook.Object, "webhooks")
if !found || err != nil {
return fmt.Errorf("unexpected object, %v", err)
}
for j := range webhooks {
webhook, err := modifyWebhook(webhooks[j].(map[string]interface{}), caData, hostPort)
if err != nil {
return err
}
webhooks[j] = webhook
unstructuredHook.Object["webhooks"] = webhooks
o.MutatingWebhooks[i] = unstructuredHook
for i := range o.MutatingWebhooks {
for j := range o.MutatingWebhooks[i].Webhooks {
updateClientConfig(&o.MutatingWebhooks[i].Webhooks[j].ClientConfig, hostPort, caData)
}
}
for i, unstructuredHook := range runtimeListToUnstructured(o.ValidatingWebhooks) {
webhooks, found, err := unstructured.NestedSlice(unstructuredHook.Object, "webhooks")
if !found || err != nil {
return fmt.Errorf("unexpected object, %v", err)
}
for j := range webhooks {
webhook, err := modifyWebhook(webhooks[j].(map[string]interface{}), caData, hostPort)
if err != nil {
return err
}
webhooks[j] = webhook
unstructuredHook.Object["webhooks"] = webhooks
o.ValidatingWebhooks[i] = unstructuredHook
for i := range o.ValidatingWebhooks {
for j := range o.ValidatingWebhooks[i].Webhooks {
updateClientConfig(&o.ValidatingWebhooks[i].Webhooks[j].ClientConfig, hostPort, caData)
}
}
return nil
}
func modifyWebhook(webhook map[string]interface{}, caData []byte, hostPort string) (map[string]interface{}, error) {
clientConfig, found, err := unstructured.NestedMap(webhook, "clientConfig")
if !found || err != nil {
return nil, fmt.Errorf("cannot find clientconfig: %v", err)
func updateClientConfig(cc *admissionv1.WebhookClientConfig, hostPort string, caData []byte) {
cc.CABundle = caData
if cc.Service != nil && cc.Service.Path != nil {
url := fmt.Sprintf("https://%s/%s", hostPort, *cc.Service.Path)
cc.URL = &url
cc.Service = nil
}
clientConfig["caBundle"] = base64.StdEncoding.EncodeToString(caData)
servicePath, found, err := unstructured.NestedString(clientConfig, "service", "path")
if found && err == nil {
// we cannot use service in integration tests since we're running controller outside cluster
// the intent here is that we swap out service for raw address because we don't have an actually standard kube service network.
// We want to users to be able to use your standard config though
url := fmt.Sprintf("https://%s/%s", hostPort, servicePath)
clientConfig["url"] = url
clientConfig["service"] = nil
}
webhook["clientConfig"] = clientConfig
return webhook, nil
}
func (o *WebhookInstallOptions) generateHostPort() (string, error) {
if o.LocalServingPort == 0 {
port, host, err := addr.Suggest(o.LocalServingHost)
if err != nil {
return "", fmt.Errorf("unable to grab random port for serving webhooks on: %v", err)
return "", fmt.Errorf("unable to grab random port for serving webhooks on: %w", err)
}
o.LocalServingPort = port
o.LocalServingHost = host
@@ -199,16 +170,35 @@ func (o *WebhookInstallOptions) Cleanup() error {
// WaitForWebhooks waits for the Webhooks to be available through API server.
func WaitForWebhooks(config *rest.Config,
mutatingWebhooks []client.Object,
validatingWebhooks []client.Object,
mutatingWebhooks []*admissionv1.MutatingWebhookConfiguration,
validatingWebhooks []*admissionv1.ValidatingWebhookConfiguration,
options WebhookInstallOptions) error {
waitingFor := map[schema.GroupVersionKind]*sets.String{}
for _, hook := range runtimeListToUnstructured(append(validatingWebhooks, mutatingWebhooks...)) {
if _, ok := waitingFor[hook.GroupVersionKind()]; !ok {
waitingFor[hook.GroupVersionKind()] = &sets.String{}
for _, hook := range mutatingWebhooks {
h := hook
gvk, err := apiutil.GVKForObject(h, scheme.Scheme)
if err != nil {
return fmt.Errorf("unable to get gvk for MutatingWebhookConfiguration %s: %w", hook.GetName(), err)
}
waitingFor[hook.GroupVersionKind()].Insert(hook.GetName())
if _, ok := waitingFor[gvk]; !ok {
waitingFor[gvk] = &sets.String{}
}
waitingFor[gvk].Insert(h.GetName())
}
for _, hook := range validatingWebhooks {
h := hook
gvk, err := apiutil.GVKForObject(h, scheme.Scheme)
if err != nil {
return fmt.Errorf("unable to get gvk for ValidatingWebhookConfiguration %s: %w", hook.GetName(), err)
}
if _, ok := waitingFor[gvk]; !ok {
waitingFor[gvk] = &sets.String{}
}
waitingFor[gvk].Insert(hook.GetName())
}
// Poll until all resources are found in discovery
@@ -266,51 +256,53 @@ func (p *webhookPoller) poll() (done bool, err error) {
func (o *WebhookInstallOptions) setupCA() error {
hookCA, err := certs.NewTinyCA()
if err != nil {
return fmt.Errorf("unable to set up webhook CA: %v", err)
return fmt.Errorf("unable to set up webhook CA: %w", err)
}
names := []string{"localhost", o.LocalServingHost, o.LocalServingHostExternalName}
hookCert, err := hookCA.NewServingCert(names...)
if err != nil {
return fmt.Errorf("unable to set up webhook serving certs: %v", err)
return fmt.Errorf("unable to set up webhook serving certs: %w", err)
}
localServingCertsDir, err := ioutil.TempDir("", "envtest-serving-certs-")
localServingCertsDir, err := os.MkdirTemp("", "envtest-serving-certs-")
o.LocalServingCertDir = localServingCertsDir
if err != nil {
return fmt.Errorf("unable to create directory for webhook serving certs: %v", err)
return fmt.Errorf("unable to create directory for webhook serving certs: %w", err)
}
certData, keyData, err := hookCert.AsBytes()
if err != nil {
return fmt.Errorf("unable to marshal webhook serving certs: %v", err)
return fmt.Errorf("unable to marshal webhook serving certs: %w", err)
}
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil { //nolint:gosec
return fmt.Errorf("unable to write webhook serving cert to disk: %v", err)
if err := os.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil { //nolint:gosec
return fmt.Errorf("unable to write webhook serving cert to disk: %w", err)
}
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil { //nolint:gosec
return fmt.Errorf("unable to write webhook serving key to disk: %v", err)
if err := os.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil { //nolint:gosec
return fmt.Errorf("unable to write webhook serving key to disk: %w", err)
}
o.LocalServingCAData = certData
return err
}
func createWebhooks(config *rest.Config, mutHooks []client.Object, valHooks []client.Object) error {
func createWebhooks(config *rest.Config, mutHooks []*admissionv1.MutatingWebhookConfiguration, valHooks []*admissionv1.ValidatingWebhookConfiguration) error {
cs, err := client.New(config, client.Options{})
if err != nil {
return err
}
// Create each webhook
for _, hook := range runtimeListToUnstructured(mutHooks) {
for _, hook := range mutHooks {
hook := hook
log.V(1).Info("installing mutating webhook", "webhook", hook.GetName())
if err := ensureCreated(cs, hook); err != nil {
return err
}
}
for _, hook := range runtimeListToUnstructured(valHooks) {
for _, hook := range valHooks {
hook := hook
log.V(1).Info("installing validating webhook", "webhook", hook.GetName())
if err := ensureCreated(cs, hook); err != nil {
return err
@@ -320,8 +312,8 @@ func createWebhooks(config *rest.Config, mutHooks []client.Object, valHooks []cl
}
// ensureCreated creates or update object if already exists in the cluster.
func ensureCreated(cs client.Client, obj *unstructured.Unstructured) error {
existing := obj.DeepCopy()
func ensureCreated(cs client.Client, obj client.Object) error {
existing := obj.DeepCopyObject().(client.Object)
err := cs.Get(context.Background(), client.ObjectKey{Name: obj.GetName()}, existing)
switch {
case apierrors.IsNotFound(err):
@@ -364,9 +356,9 @@ func parseWebhook(options *WebhookInstallOptions) error {
// readWebhooks reads the Webhooks from files and Unmarshals them into structs
// returns slice of mutating and validating webhook configurations.
func readWebhooks(path string) ([]client.Object, []client.Object, error) {
func readWebhooks(path string) ([]*admissionv1.MutatingWebhookConfiguration, []*admissionv1.ValidatingWebhookConfiguration, error) {
// Get the webhook files
var files []os.FileInfo
var files []string
var err error
log.V(1).Info("reading Webhooks from path", "path", path)
info, err := os.Stat(path)
@@ -374,24 +366,30 @@ func readWebhooks(path string) ([]client.Object, []client.Object, error) {
return nil, nil, err
}
if !info.IsDir() {
path, files = filepath.Dir(path), []os.FileInfo{info}
} else if files, err = ioutil.ReadDir(path); err != nil {
return nil, nil, err
path, files = filepath.Dir(path), []string{info.Name()}
} else {
entries, err := os.ReadDir(path)
if err != nil {
return nil, nil, err
}
for _, e := range entries {
files = append(files, e.Name())
}
}
// file extensions that may contain Webhooks
resourceExtensions := sets.NewString(".json", ".yaml", ".yml")
var mutHooks []client.Object
var valHooks []client.Object
var mutHooks []*admissionv1.MutatingWebhookConfiguration
var valHooks []*admissionv1.ValidatingWebhookConfiguration
for _, file := range files {
// Only parse allowlisted file types
if !resourceExtensions.Has(filepath.Ext(file.Name())) {
if !resourceExtensions.Has(filepath.Ext(file)) {
continue
}
// Unmarshal Webhooks from file into structs
docs, err := readDocuments(filepath.Join(path, file.Name()))
docs, err := readDocuments(filepath.Join(path, file))
if err != nil {
return nil, nil, err
}
@@ -403,25 +401,24 @@ func readWebhooks(path string) ([]client.Object, []client.Object, error) {
}
const (
admissionregv1 = "admissionregistration.k8s.io/v1"
admissionregv1beta1 = "admissionregistration.k8s.io/v1beta1"
admissionregv1 = "admissionregistration.k8s.io/v1"
)
switch {
case generic.Kind == "MutatingWebhookConfiguration":
if generic.APIVersion != admissionregv1beta1 && generic.APIVersion != admissionregv1 {
return nil, nil, fmt.Errorf("only v1beta1 and v1 are supported right now for MutatingWebhookConfiguration (name: %s)", generic.Name)
if generic.APIVersion != admissionregv1 {
return nil, nil, fmt.Errorf("only v1 is supported right now for MutatingWebhookConfiguration (name: %s)", generic.Name)
}
hook := &unstructured.Unstructured{}
if err := yaml.Unmarshal(doc, &hook); err != nil {
hook := &admissionv1.MutatingWebhookConfiguration{}
if err := yaml.Unmarshal(doc, hook); err != nil {
return nil, nil, err
}
mutHooks = append(mutHooks, hook)
case generic.Kind == "ValidatingWebhookConfiguration":
if generic.APIVersion != admissionregv1beta1 && generic.APIVersion != admissionregv1 {
return nil, nil, fmt.Errorf("only v1beta1 and v1 are supported right now for ValidatingWebhookConfiguration (name: %s)", generic.Name)
if generic.APIVersion != admissionregv1 {
return nil, nil, fmt.Errorf("only v1 is supported right now for ValidatingWebhookConfiguration (name: %s)", generic.Name)
}
hook := &unstructured.Unstructured{}
if err := yaml.Unmarshal(doc, &hook); err != nil {
hook := &admissionv1.ValidatingWebhookConfiguration{}
if err := yaml.Unmarshal(doc, hook); err != nil {
return nil, nil, err
}
valHooks = append(valHooks, hook)
@@ -430,21 +427,7 @@ func readWebhooks(path string) ([]client.Object, []client.Object, error) {
}
}
log.V(1).Info("read webhooks from file", "file", file.Name())
log.V(1).Info("read webhooks from file", "file", file)
}
return mutHooks, valHooks, nil
}
func runtimeListToUnstructured(l []client.Object) []*unstructured.Unstructured {
res := []*unstructured.Unstructured{}
for _, obj := range l {
m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj.DeepCopyObject())
if err != nil {
continue
}
res = append(res, &unstructured.Unstructured{
Object: m,
})
}
return res
}

View File

@@ -25,6 +25,7 @@ import (
"github.com/go-logr/logr"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/handler"
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics"
@@ -83,8 +84,14 @@ type Controller struct {
// startWatches maintains a list of sources, handlers, and predicates to start when the controller is started.
startWatches []watchDescription
// Log is used to log messages to users during reconciliation, or for example when a watch is started.
Log logr.Logger
// LogConstructor is used to construct a logger to then log messages to users during reconciliation,
// or for example when a watch is started.
// Note: LogConstructor has to be able to handle nil requests as we are also using it
// outside the context of a reconciliation.
LogConstructor func(request *reconcile.Request) logr.Logger
// RecoverPanic indicates whether the panic caused by reconcile should be recovered.
RecoverPanic bool
}
// watchDescription contains all the information necessary to start a watch.
@@ -95,9 +102,22 @@ type watchDescription struct {
}
// Reconcile implements reconcile.Reconciler.
func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
log := c.Log.WithValues("name", req.Name, "namespace", req.Namespace)
ctx = logf.IntoContext(ctx, log)
func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) {
defer func() {
if r := recover(); r != nil {
if c.RecoverPanic {
for _, fn := range utilruntime.PanicHandlers {
fn(r)
}
err = fmt.Errorf("panic: %v [recovered]", r)
return
}
log := logf.FromContext(ctx)
log.Info(fmt.Sprintf("Observed a panic in reconciler: %v", r))
panic(r)
}
}()
return c.Do.Reconcile(ctx, req)
}
@@ -127,7 +147,7 @@ func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prc
return nil
}
c.Log.Info("Starting EventSource", "source", src)
c.LogConstructor(nil).Info("Starting EventSource", "source", src)
return src.Start(c.ctx, evthdler, c.Queue, prct...)
}
@@ -162,7 +182,7 @@ func (c *Controller) Start(ctx context.Context) error {
// caches to sync so that they have a chance to register their intendeded
// caches.
for _, watch := range c.startWatches {
c.Log.Info("Starting EventSource", "source", watch.src)
c.LogConstructor(nil).Info("Starting EventSource", "source", fmt.Sprintf("%s", watch.src))
if err := watch.src.Start(ctx, watch.handler, c.Queue, watch.predicates...); err != nil {
return err
@@ -170,7 +190,7 @@ func (c *Controller) Start(ctx context.Context) error {
}
// Start the SharedIndexInformer factories to begin populating the SharedIndexInformer caches
c.Log.Info("Starting Controller")
c.LogConstructor(nil).Info("Starting Controller")
for _, watch := range c.startWatches {
syncingSource, ok := watch.src.(source.SyncingSource)
@@ -187,7 +207,7 @@ func (c *Controller) Start(ctx context.Context) error {
// is an error or a timeout
if err := syncingSource.WaitForSync(sourceStartCtx); err != nil {
err := fmt.Errorf("failed to wait for %s caches to sync: %w", c.Name, err)
c.Log.Error(err, "Could not wait for Cache to sync")
c.LogConstructor(nil).Error(err, "Could not wait for Cache to sync")
return err
}
@@ -204,7 +224,7 @@ func (c *Controller) Start(ctx context.Context) error {
c.startWatches = nil
// Launch workers to process resources
c.Log.Info("Starting workers", "worker count", c.MaxConcurrentReconciles)
c.LogConstructor(nil).Info("Starting workers", "worker count", c.MaxConcurrentReconciles)
wg.Add(c.MaxConcurrentReconciles)
for i := 0; i < c.MaxConcurrentReconciles; i++ {
go func() {
@@ -224,9 +244,9 @@ func (c *Controller) Start(ctx context.Context) error {
}
<-ctx.Done()
c.Log.Info("Shutdown signal received, waiting for all workers to finish")
c.LogConstructor(nil).Info("Shutdown signal received, waiting for all workers to finish")
wg.Wait()
c.Log.Info("All workers finished")
c.LogConstructor(nil).Info("All workers finished")
return nil
}
@@ -278,24 +298,26 @@ func (c *Controller) reconcileHandler(ctx context.Context, obj interface{}) {
c.updateMetrics(time.Since(reconcileStartTS))
}()
// Make sure that the the object is a valid request.
// Make sure that the object is a valid request.
req, ok := obj.(reconcile.Request)
if !ok {
// As the item in the workqueue is actually invalid, we call
// Forget here else we'd go into a loop of attempting to
// process a work item that is invalid.
c.Queue.Forget(obj)
c.Log.Error(nil, "Queue item was not a Request", "type", fmt.Sprintf("%T", obj), "value", obj)
c.LogConstructor(nil).Error(nil, "Queue item was not a Request", "type", fmt.Sprintf("%T", obj), "value", obj)
// Return true, don't take a break
return
}
log := c.Log.WithValues("name", req.Name, "namespace", req.Namespace)
log := c.LogConstructor(&req)
log = log.WithValues("reconcileID", uuid.NewUUID())
ctx = logf.IntoContext(ctx, log)
// RunInformersAndControllers the syncHandler, passing it the Namespace/Name string of the
// resource to be synced.
result, err := c.Do.Reconcile(ctx, req)
result, err := c.Reconcile(ctx, req)
switch {
case err != nil:
c.Queue.AddRateLimited(req)
@@ -323,7 +345,7 @@ func (c *Controller) reconcileHandler(ctx context.Context, obj interface{}) {
// GetLogger returns this controller's logger.
func (c *Controller) GetLogger() logr.Logger {
return c.Log
return c.LogConstructor(nil)
}
// InjectFunc implement SetFields.Injector.

View File

@@ -1,3 +1,4 @@
//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly
// +build linux darwin freebsd openbsd netbsd dragonfly
/*

View File

@@ -0,0 +1,16 @@
package httpserver
import (
"net/http"
"time"
)
// New returns a new server with sane defaults.
func New(handler http.Handler) *http.Server {
return &http.Server{
Handler: handler,
MaxHeaderBytes: 1 << 20,
IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout
ReadHeaderTimeout: 32 * time.Second,
}
}

View File

@@ -24,8 +24,7 @@ import (
"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
)
@@ -45,7 +44,7 @@ type Provider struct {
scheme *runtime.Scheme
// logger is the logger to use when logging diagnostic event info
logger logr.Logger
evtClient typedcorev1.EventInterface
evtClient corev1client.EventInterface
makeBroadcaster EventBroadcasterProducer
broadcasterOnce sync.Once
@@ -98,7 +97,7 @@ func (p *Provider) getBroadcaster() record.EventBroadcaster {
p.broadcasterOnce.Do(func() {
broadcaster, stop := p.makeBroadcaster()
broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: p.evtClient})
broadcaster.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: p.evtClient})
broadcaster.StartEventWatcher(
func(e *corev1.Event) {
p.logger.V(1).Info(e.Type, "object", e.InvolvedObject, "reason", e.Reason, "message", e.Message)
@@ -112,12 +111,12 @@ func (p *Provider) getBroadcaster() record.EventBroadcaster {
// NewProvider create a new Provider instance.
func NewProvider(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster EventBroadcasterProducer) (*Provider, error) {
clientSet, err := kubernetes.NewForConfig(config)
corev1Client, err := corev1client.NewForConfig(config)
if err != nil {
return nil, fmt.Errorf("failed to init clientSet: %w", err)
return nil, fmt.Errorf("failed to init client: %w", err)
}
p := &Provider{scheme: scheme, logger: logger, makeBroadcaster: makeBroadcaster, evtClient: clientSet.CoreV1().Events("")}
p := &Provider{scheme: scheme, logger: logger, makeBroadcaster: makeBroadcaster, evtClient: corev1Client.Events("")}
return p, nil
}

View File

@@ -43,11 +43,17 @@ var (
func init() {
baseDir, err := os.UserCacheDir()
if err != nil {
baseDir = os.TempDir()
if err == nil {
cacheDir = filepath.Join(baseDir, "kubebuilder-envtest")
err = os.MkdirAll(cacheDir, 0o750)
}
cacheDir = filepath.Join(baseDir, "kubebuilder-envtest")
if err := os.MkdirAll(cacheDir, 0750); err != nil {
if err != nil {
// Either we didn't get a cache directory, or we can't use it
baseDir = os.TempDir()
cacheDir = filepath.Join(baseDir, "kubebuilder-envtest")
err = os.MkdirAll(cacheDir, 0o750)
}
if err != nil {
panic(err)
}
}

View File

@@ -24,8 +24,9 @@ package certs
import (
"crypto"
"crypto/ecdsa"
"crypto/elliptic"
crand "crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
@@ -38,8 +39,8 @@ import (
)
var (
rsaKeySize = 2048 // a decent number, as of 2019
bigOne = big.NewInt(1)
ellipticCurve = elliptic.P256()
bigOne = big.NewInt(1)
)
// CertPair is a private key and certificate for use for client auth, as a CA, or serving.
@@ -63,7 +64,7 @@ func (k CertPair) AsBytes() (cert []byte, key []byte, err error) {
rawKeyData, err := x509.MarshalPKCS8PrivateKey(k.Key)
if err != nil {
return nil, nil, fmt.Errorf("unable to encode private key: %v", err)
return nil, nil, fmt.Errorf("unable to encode private key: %w", err)
}
key = pem.EncodeToMemory(&pem.Block{
@@ -86,7 +87,7 @@ type TinyCA struct {
// newPrivateKey generates a new private key of a relatively sane size (see
// rsaKeySize).
func newPrivateKey() (crypto.Signer, error) {
return rsa.GenerateKey(crand.Reader, rsaKeySize)
return ecdsa.GenerateKey(ellipticCurve, crand.Reader)
}
// NewTinyCA creates a new a tiny CA utility for provisioning serving certs and client certs FOR TESTING ONLY.
@@ -94,12 +95,12 @@ func newPrivateKey() (crypto.Signer, error) {
func NewTinyCA() (*TinyCA, error) {
caPrivateKey, err := newPrivateKey()
if err != nil {
return nil, fmt.Errorf("unable to generate private key for CA: %v", err)
return nil, fmt.Errorf("unable to generate private key for CA: %w", err)
}
caCfg := certutil.Config{CommonName: "envtest-environment", Organization: []string{"envtest"}}
caCert, err := certutil.NewSelfSignedCACert(caCfg, caPrivateKey)
if err != nil {
return nil, fmt.Errorf("unable to generate certificate for CA: %v", err)
return nil, fmt.Errorf("unable to generate certificate for CA: %w", err)
}
return &TinyCA{
@@ -114,7 +115,7 @@ func (c *TinyCA) makeCert(cfg certutil.Config) (CertPair, error) {
key, err := newPrivateKey()
if err != nil {
return CertPair{}, fmt.Errorf("unable to create private key: %v", err)
return CertPair{}, fmt.Errorf("unable to create private key: %w", err)
}
serial := new(big.Int).Set(c.nextSerial)
@@ -139,12 +140,12 @@ func (c *TinyCA) makeCert(cfg certutil.Config) (CertPair, error) {
certRaw, err := x509.CreateCertificate(crand.Reader, &template, c.CA.Cert, key.Public(), c.CA.Key)
if err != nil {
return CertPair{}, fmt.Errorf("unable to create certificate: %v", err)
return CertPair{}, fmt.Errorf("unable to create certificate: %w", err)
}
cert, err := x509.ParseCertificate(certRaw)
if err != nil {
return CertPair{}, fmt.Errorf("generated invalid certificate, could not parse: %v", err)
return CertPair{}, fmt.Errorf("generated invalid certificate, could not parse: %w", err)
}
return CertPair{

View File

@@ -19,7 +19,6 @@ package controlplane
import (
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"path/filepath"
@@ -385,10 +384,10 @@ func (s *APIServer) populateAPIServerCerts() error {
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0640); err != nil { //nolint:gosec
if err := os.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0640); err != nil { //nolint:gosec
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0640); err != nil { //nolint:gosec
if err := os.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0640); err != nil { //nolint:gosec
return err
}
@@ -405,19 +404,19 @@ func (s *APIServer) populateAPIServerCerts() error {
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, saCertFile), saCert, 0640); err != nil { //nolint:gosec
if err := os.WriteFile(filepath.Join(s.CertDir, saCertFile), saCert, 0640); err != nil { //nolint:gosec
return err
}
return ioutil.WriteFile(filepath.Join(s.CertDir, saKeyFile), saKey, 0640) //nolint:gosec
return os.WriteFile(filepath.Join(s.CertDir, saKeyFile), saKey, 0640) //nolint:gosec
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the CertDir if necessary.
func (s *APIServer) Stop() error {
if s.processState.DirNeedsCleaning {
s.CertDir = "" // reset the directory if it was randomly allocated, so that we can safely restart
}
if s.processState != nil {
if s.processState.DirNeedsCleaning {
s.CertDir = "" // reset the directory if it was randomly allocated, so that we can safely restart
}
if err := s.processState.Stop(); err != nil {
return err
}

View File

@@ -18,7 +18,7 @@ package controlplane
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"k8s.io/client-go/rest"
@@ -128,7 +128,7 @@ func (c *CertAuthn) Start() error {
return fmt.Errorf("start called before configure")
}
caCrt := c.ca.CA.CertBytes()
if err := ioutil.WriteFile(c.caCrtPath(), caCrt, 0640); err != nil { //nolint:gosec
if err := os.WriteFile(c.caCrtPath(), caCrt, 0640); err != nil { //nolint:gosec
return fmt.Errorf("unable to save the client certificate CA to %s: %w", c.caCrtPath(), err)
}

View File

@@ -84,10 +84,14 @@ type Etcd struct {
// args contains the structured arguments to use for running etcd.
// Lazily initialized by .Configure(), Defaulted eventually with .defaultArgs()
args *process.Arguments
// listenPeerURL is the address the Etcd should listen on for peer connections.
// It's automatically generated and a random port is picked during execution.
listenPeerURL *url.URL
}
// Start starts the etcd, waits for it to come up, and returns an error, if one
// occoured.
// occurred.
func (e *Etcd) Start() error {
if err := e.setProcessState(); err != nil {
return err
@@ -111,6 +115,7 @@ func (e *Etcd) setProcessState() error {
return err
}
// Set the listen url.
if e.URL == nil {
port, host, err := addr.Suggest("")
if err != nil {
@@ -122,6 +127,18 @@ func (e *Etcd) setProcessState() error {
}
}
// Set the listen peer URL.
{
port, host, err := addr.Suggest("")
if err != nil {
return err
}
e.listenPeerURL = &url.URL{
Scheme: "http",
Host: net.JoinHostPort(host, strconv.Itoa(port)),
}
}
// can use /health as of etcd 3.3.0
e.processState.HealthCheck.URL = *e.URL
e.processState.HealthCheck.Path = "/health"
@@ -150,7 +167,7 @@ func (e *Etcd) Stop() error {
func (e *Etcd) defaultArgs() map[string][]string {
args := map[string][]string{
"listen-peer-urls": {"http://localhost:0"},
"listen-peer-urls": {e.listenPeerURL.String()},
"data-dir": {e.DataDir},
}
if e.URL != nil {

View File

@@ -47,13 +47,18 @@ type ControlPlane struct {
}
// Start will start your control plane processes. To stop them, call Stop().
func (f *ControlPlane) Start() error {
func (f *ControlPlane) Start() (retErr error) {
if f.Etcd == nil {
f.Etcd = &Etcd{}
}
if err := f.Etcd.Start(); err != nil {
return err
}
defer func() {
if retErr != nil {
_ = f.Etcd.Stop()
}
}()
if f.APIServer == nil {
f.APIServer = &APIServer{}
@@ -62,6 +67,11 @@ func (f *ControlPlane) Start() error {
if err := f.APIServer.Start(); err != nil {
return err
}
defer func() {
if retErr != nil {
_ = f.APIServer.Stop()
}
}()
// provision the default user -- can be removed when the related
// methods are removed. The default user has admin permissions to
@@ -88,6 +98,7 @@ func (f *ControlPlane) Stop() error {
errList = append(errList, err)
}
}
if f.Etcd != nil {
if err := f.Etcd.Stop(); err != nil {
errList = append(errList, err)

View File

@@ -20,7 +20,6 @@ import (
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
@@ -84,7 +83,7 @@ type State struct {
DirNeedsCleaning bool
Path string
// ready holds wether the process is currently in ready state (hit the ready condition) or not.
// ready holds whether the process is currently in ready state (hit the ready condition) or not.
// It will be set to true on a successful `Start()` and set to false on a successful `Stop()`
ready bool
@@ -109,7 +108,7 @@ func (ps *State) Init(name string) error {
}
if ps.Dir == "" {
newDir, err := ioutil.TempDir("", "k8s_test_framework_")
newDir, err := os.MkdirTemp("", "k8s_test_framework_")
if err != nil {
return err
}

View File

@@ -19,13 +19,14 @@ package leaderelection
import (
"errors"
"fmt"
"io/ioutil"
"os"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/client-go/kubernetes"
coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"sigs.k8s.io/controller-runtime/pkg/recorder"
)
@@ -38,7 +39,7 @@ type Options struct {
LeaderElection bool
// LeaderElectionResourceLock determines which resource lock to use for leader election,
// defaults to "configmapsleases".
// defaults to "leases".
LeaderElectionResourceLock string
// LeaderElectionNamespace determines the namespace in which the leader
@@ -56,11 +57,12 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
return nil, nil
}
// Default resource lock to "configmapsleases". We must keep this default until we are sure all controller-runtime
// users have upgraded from the original default ConfigMap lock to a controller-runtime version that has this new
// default. Many users of controller-runtime skip versions, so we should be extremely conservative here.
// Default resource lock to "leases". The previous default (from v0.7.0 to v0.11.x) was configmapsleases, which was
// used to migrate from configmaps to leases. Since the default was "configmapsleases" for over a year, spanning
// five minor releases, any actively maintained operators are very likely to have a released version that uses
// "configmapsleases". Therefore defaulting to "leases" should be safe.
if options.LeaderElectionResourceLock == "" {
options.LeaderElectionResourceLock = resourcelock.ConfigMapsLeasesResourceLock
options.LeaderElectionResourceLock = resourcelock.LeasesResourceLock
}
// LeaderElectionID must be provided to prevent clashes
@@ -84,8 +86,14 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
}
id = id + "_" + string(uuid.NewUUID())
// Construct client for leader election
client, err := kubernetes.NewForConfig(rest.AddUserAgent(config, "leader-election"))
// Construct clients for leader election
rest.AddUserAgent(config, "leader-election")
corev1Client, err := corev1client.NewForConfig(config)
if err != nil {
return nil, err
}
coordinationClient, err := coordinationv1client.NewForConfig(config)
if err != nil {
return nil, err
}
@@ -93,8 +101,8 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
return resourcelock.New(options.LeaderElectionResourceLock,
options.LeaderElectionNamespace,
options.LeaderElectionID,
client.CoreV1(),
client.CoordinationV1(),
corev1Client,
coordinationClient,
resourcelock.ResourceLockConfig{
Identity: id,
EventRecorder: recorderProvider.GetEventRecorderFor(id),
@@ -111,7 +119,7 @@ func getInClusterNamespace() (string, error) {
}
// Load the namespace file and return its content
namespace, err := ioutil.ReadFile(inClusterNamespacePath)
namespace, err := os.ReadFile(inClusterNamespacePath)
if err != nil {
return "", fmt.Errorf("error reading namespace file: %w", err)
}

View File

@@ -25,16 +25,15 @@ import (
// loggerPromise knows how to populate a concrete logr.Logger
// with options, given an actual base logger later on down the line.
type loggerPromise struct {
logger *DelegatingLogger
logger *DelegatingLogSink
childPromises []*loggerPromise
promisesLock sync.Mutex
name *string
tags []interface{}
level int
name *string
tags []interface{}
}
func (p *loggerPromise) WithName(l *DelegatingLogger, name string) *loggerPromise {
func (p *loggerPromise) WithName(l *DelegatingLogSink, name string) *loggerPromise {
res := &loggerPromise{
logger: l,
name: &name,
@@ -48,7 +47,7 @@ func (p *loggerPromise) WithName(l *DelegatingLogger, name string) *loggerPromis
}
// WithValues provides a new Logger with the tags appended.
func (p *loggerPromise) WithValues(l *DelegatingLogger, tags ...interface{}) *loggerPromise {
func (p *loggerPromise) WithValues(l *DelegatingLogSink, tags ...interface{}) *loggerPromise {
res := &loggerPromise{
logger: l,
tags: tags,
@@ -61,61 +60,53 @@ func (p *loggerPromise) WithValues(l *DelegatingLogger, tags ...interface{}) *lo
return res
}
func (p *loggerPromise) V(l *DelegatingLogger, level int) *loggerPromise {
res := &loggerPromise{
logger: l,
level: level,
promisesLock: sync.Mutex{},
}
p.promisesLock.Lock()
defer p.promisesLock.Unlock()
p.childPromises = append(p.childPromises, res)
return res
}
// Fulfill instantiates the Logger with the provided logger.
func (p *loggerPromise) Fulfill(parentLogger logr.Logger) {
logger := logr.WithCallDepth(parentLogger, 1)
func (p *loggerPromise) Fulfill(parentLogSink logr.LogSink) {
sink := parentLogSink
if p.name != nil {
logger = logger.WithName(*p.name)
sink = sink.WithName(*p.name)
}
if p.tags != nil {
logger = logger.WithValues(p.tags...)
}
if p.level != 0 {
logger = logger.V(p.level)
sink = sink.WithValues(p.tags...)
}
p.logger.lock.Lock()
p.logger.logger = logger
p.logger.logger = sink
p.logger.promise = nil
p.logger.lock.Unlock()
for _, childPromise := range p.childPromises {
childPromise.Fulfill(logger)
childPromise.Fulfill(sink)
}
}
// DelegatingLogger is a logr.Logger that delegates to another logr.Logger.
// DelegatingLogSink is a logsink that delegates to another logr.LogSink.
// If the underlying promise is not nil, it registers calls to sub-loggers with
// the logging factory to be populated later, and returns a new delegating
// logger. It expects to have *some* logr.Logger set at all times (generally
// a no-op logger before the promises are fulfilled).
type DelegatingLogger struct {
type DelegatingLogSink struct {
lock sync.RWMutex
logger logr.Logger
logger logr.LogSink
promise *loggerPromise
info logr.RuntimeInfo
}
// Init implements logr.LogSink.
func (l *DelegatingLogSink) Init(info logr.RuntimeInfo) {
l.lock.Lock()
defer l.lock.Unlock()
l.info = info
}
// Enabled tests whether this Logger is enabled. For example, commandline
// flags might be used to set the logging verbosity and disable some info
// logs.
func (l *DelegatingLogger) Enabled() bool {
func (l *DelegatingLogSink) Enabled(level int) bool {
l.lock.RLock()
defer l.lock.RUnlock()
return l.logger.Enabled()
return l.logger.Enabled(level)
}
// Info logs a non-error message with the given key/value pairs as context.
@@ -124,10 +115,10 @@ func (l *DelegatingLogger) Enabled() bool {
// the log line. The key/value pairs can then be used to add additional
// variable information. The key/value pairs should alternate string
// keys and arbitrary values.
func (l *DelegatingLogger) Info(msg string, keysAndValues ...interface{}) {
func (l *DelegatingLogSink) Info(level int, msg string, keysAndValues ...interface{}) {
l.lock.RLock()
defer l.lock.RUnlock()
l.logger.Info(msg, keysAndValues...)
l.logger.Info(level, msg, keysAndValues...)
}
// Error logs an error, with the given message and key/value pairs as context.
@@ -138,33 +129,14 @@ func (l *DelegatingLogger) Info(msg string, keysAndValues ...interface{}) {
// The msg field should be used to add context to any underlying error,
// while the err field should be used to attach the actual error that
// triggered this log line, if present.
func (l *DelegatingLogger) Error(err error, msg string, keysAndValues ...interface{}) {
func (l *DelegatingLogSink) Error(err error, msg string, keysAndValues ...interface{}) {
l.lock.RLock()
defer l.lock.RUnlock()
l.logger.Error(err, msg, keysAndValues...)
}
// V returns an Logger value for a specific verbosity level, relative to
// this Logger. In other words, V values are additive. V higher verbosity
// level means a log message is less important. It's illegal to pass a log
// level less than zero.
func (l *DelegatingLogger) V(level int) logr.Logger {
l.lock.RLock()
defer l.lock.RUnlock()
if l.promise == nil {
return l.logger.V(level)
}
res := &DelegatingLogger{logger: l.logger}
promise := l.promise.V(res, level)
res.promise = promise
return res
}
// WithName provides a new Logger with the name appended.
func (l *DelegatingLogger) WithName(name string) logr.Logger {
func (l *DelegatingLogSink) WithName(name string) logr.LogSink {
l.lock.RLock()
defer l.lock.RUnlock()
@@ -172,7 +144,7 @@ func (l *DelegatingLogger) WithName(name string) logr.Logger {
return l.logger.WithName(name)
}
res := &DelegatingLogger{logger: l.logger}
res := &DelegatingLogSink{logger: l.logger}
promise := l.promise.WithName(res, name)
res.promise = promise
@@ -180,7 +152,7 @@ func (l *DelegatingLogger) WithName(name string) logr.Logger {
}
// WithValues provides a new Logger with the tags appended.
func (l *DelegatingLogger) WithValues(tags ...interface{}) logr.Logger {
func (l *DelegatingLogSink) WithValues(tags ...interface{}) logr.LogSink {
l.lock.RLock()
defer l.lock.RUnlock()
@@ -188,7 +160,7 @@ func (l *DelegatingLogger) WithValues(tags ...interface{}) logr.Logger {
return l.logger.WithValues(tags...)
}
res := &DelegatingLogger{logger: l.logger}
res := &DelegatingLogSink{logger: l.logger}
promise := l.promise.WithValues(res, tags...)
res.promise = promise
@@ -198,16 +170,16 @@ func (l *DelegatingLogger) WithValues(tags ...interface{}) logr.Logger {
// Fulfill switches the logger over to use the actual logger
// provided, instead of the temporary initial one, if this method
// has not been previously called.
func (l *DelegatingLogger) Fulfill(actual logr.Logger) {
func (l *DelegatingLogSink) Fulfill(actual logr.LogSink) {
if l.promise != nil {
l.promise.Fulfill(actual)
}
}
// NewDelegatingLogger constructs a new DelegatingLogger which uses
// the given logger before it's promise is fulfilled.
func NewDelegatingLogger(initial logr.Logger) *DelegatingLogger {
l := &DelegatingLogger{
// NewDelegatingLogSink constructs a new DelegatingLogSink which uses
// the given logger before its promise is fulfilled.
func NewDelegatingLogSink(initial logr.LogSink) *DelegatingLogSink {
l := &DelegatingLogSink{
logger: initial,
promise: &loggerPromise{promisesLock: sync.Mutex{}},
}

View File

@@ -29,7 +29,7 @@ limitations under the License.
//
// All logging in controller-runtime is structured, using a set of interfaces
// defined by a package called logr
// (https://godoc.org/github.com/go-logr/logr). The sub-package zap provides
// (https://pkg.go.dev/github.com/go-logr/logr). The sub-package zap provides
// helpers for setting up logr backed by Zap (go.uber.org/zap).
package log
@@ -47,14 +47,14 @@ func SetLogger(l logr.Logger) {
defer loggerWasSetLock.Unlock()
loggerWasSet = true
Log.Fulfill(l)
dlog.Fulfill(l.GetSink())
}
// It is safe to assume that if this wasn't set within the first 30 seconds of a binaries
// lifetime, it will never get set. The DelegatingLogger causes a high number of memory
// allocations when not given an actual Logger, so we set a NullLogger to avoid that.
// lifetime, it will never get set. The DelegatingLogSink causes a high number of memory
// allocations when not given an actual Logger, so we set a NullLogSink to avoid that.
//
// We need to keep the DelegatingLogger because we have various inits() that get a logger from
// We need to keep the DelegatingLogSink because we have various inits() that get a logger from
// here. They will always get executed before any code that imports controller-runtime
// has a chance to run and hence to set an actual logger.
func init() {
@@ -64,7 +64,7 @@ func init() {
loggerWasSetLock.Lock()
defer loggerWasSetLock.Unlock()
if !loggerWasSet {
Log.Fulfill(NullLogger{})
dlog.Fulfill(NullLogSink{})
}
}()
}
@@ -78,14 +78,17 @@ var (
// to another logr.Logger. You *must* call SetLogger to
// get any actual logging. If SetLogger is not called within
// the first 30 seconds of a binaries lifetime, it will get
// set to a NullLogger.
var Log = NewDelegatingLogger(NullLogger{})
// set to a NullLogSink.
var (
dlog = NewDelegatingLogSink(NullLogSink{})
Log = logr.New(dlog)
)
// FromContext returns a logger with predefined values from a context.Context.
func FromContext(ctx context.Context, keysAndValues ...interface{}) logr.Logger {
var log logr.Logger = Log
log := Log
if ctx != nil {
if logger := logr.FromContext(ctx); logger != nil {
if logger, err := logr.FromContext(ctx); err == nil {
log = logger
}
}

View File

@@ -24,37 +24,36 @@ import (
// but avoids accidentally adding the testing flags to
// all binaries.
// NullLogger is a logr.Logger that does nothing.
type NullLogger struct{}
// NullLogSink is a logr.Logger that does nothing.
type NullLogSink struct{}
var _ logr.Logger = NullLogger{}
var _ logr.LogSink = NullLogSink{}
// Init implements logr.LogSink.
func (log NullLogSink) Init(logr.RuntimeInfo) {
}
// Info implements logr.InfoLogger.
func (NullLogger) Info(_ string, _ ...interface{}) {
func (NullLogSink) Info(_ int, _ string, _ ...interface{}) {
// Do nothing.
}
// Enabled implements logr.InfoLogger.
func (NullLogger) Enabled() bool {
func (NullLogSink) Enabled(level int) bool {
return false
}
// Error implements logr.Logger.
func (NullLogger) Error(_ error, _ string, _ ...interface{}) {
func (NullLogSink) Error(_ error, _ string, _ ...interface{}) {
// Do nothing.
}
// V implements logr.Logger.
func (log NullLogger) V(_ int) logr.Logger {
return log
}
// WithName implements logr.Logger.
func (log NullLogger) WithName(_ string) logr.Logger {
func (log NullLogSink) WithName(_ string) logr.LogSink {
return log
}
// WithValues implements logr.Logger.
func (log NullLogger) WithValues(_ ...interface{}) logr.Logger {
func (log NullLogSink) WithValues(_ ...interface{}) logr.LogSink {
return log
}

View File

@@ -47,7 +47,7 @@ type KubeAPIWarningLogger struct {
}
// HandleWarningHeader handles logging for responses from API server that are
// warnings with code being 299 and uses a logr.Logger for it's logging purposes.
// warnings with code being 299 and uses a logr.Logger for its logging purposes.
func (l *KubeAPIWarningLogger) HandleWarningHeader(code int, agent string, message string) {
if code != 299 || len(message) == 0 {
return

View File

@@ -23,6 +23,7 @@ import (
"net"
"net/http"
"sync"
"sync/atomic"
"time"
"github.com/go-logr/logr"
@@ -30,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
@@ -40,6 +42,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/cluster"
"sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/internal/httpserver"
intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder"
"sigs.k8s.io/controller-runtime/pkg/metrics"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
@@ -47,7 +50,7 @@ import (
)
const (
// Values taken from: https://github.com/kubernetes/apiserver/blob/master/pkg/apis/config/v1alpha1/defaults.go
// Values taken from: https://github.com/kubernetes/component-base/blob/master/config/v1alpha1/defaults.go
defaultLeaseDuration = 15 * time.Second
defaultRenewDeadline = 10 * time.Second
defaultRetryPeriod = 2 * time.Second
@@ -61,17 +64,16 @@ const (
var _ Runnable = &controllerManager{}
type controllerManager struct {
sync.Mutex
started bool
stopProcedureEngaged *int64
errChan chan error
runnables *runnables
// cluster holds a variety of methods to interact with a cluster. Required.
cluster cluster.Cluster
// leaderElectionRunnables is the set of Controllers that the controllerManager injects deps into and Starts.
// These Runnables are managed by lead election.
leaderElectionRunnables []Runnable
// nonLeaderElectionRunnables is the set of webhook servers that the controllerManager injects deps into and Starts.
// These Runnables will not be blocked by lead election.
nonLeaderElectionRunnables []Runnable
// recorderProvider is used to generate event recorders that will be injected into Controllers
// (and EventHandlers, Sources and Predicates).
recorderProvider *intrec.Provider
@@ -104,12 +106,6 @@ type controllerManager struct {
// Healthz probe handler
healthzHandler *healthz.Handler
mu sync.Mutex
started bool
startedLeader bool
healthzStarted bool
errChan chan error
// controllerOptions are the global controller options.
controllerOptions v1alpha1.ControllerConfigurationSpec
@@ -117,25 +113,20 @@ type controllerManager struct {
// If none is set, it defaults to log.Log global logger.
logger logr.Logger
// leaderElectionStopped is an internal channel used to signal the stopping procedure that the
// LeaderElection.Run(...) function has returned and the shutdown can proceed.
leaderElectionStopped chan struct{}
// leaderElectionCancel is used to cancel the leader election. It is distinct from internalStopper,
// because for safety reasons we need to os.Exit() when we lose the leader election, meaning that
// it must be deferred until after gracefulShutdown is done.
leaderElectionCancel context.CancelFunc
// leaderElectionStopped is an internal channel used to signal the stopping procedure that the
// LeaderElection.Run(...) function has returned and the shutdown can proceed.
leaderElectionStopped chan struct{}
// stop procedure engaged. In other words, we should not add anything else to the manager
stopProcedureEngaged bool
// elected is closed when this manager becomes the leader of a group of
// managers, either because it won a leader election or because no leader
// election was configured.
elected chan struct{}
caches []hasCache
// port is the port that the webhook server serves at.
port int
// host is the hostname that the webhook server binds to.
@@ -160,10 +151,6 @@ type controllerManager struct {
// between tries of actions.
retryPeriod time.Duration
// waitForRunnable is holding the number of runnables currently running so that
// we can wait for them to exit before quitting the manager
waitForRunnable sync.WaitGroup
// gracefulShutdownTimeout is the duration given to runnable to stop
// before the manager actually returns on stop.
gracefulShutdownTimeout time.Duration
@@ -192,36 +179,17 @@ type hasCache interface {
// Add sets dependencies on i, and adds it to the list of Runnables to start.
func (cm *controllerManager) Add(r Runnable) error {
cm.mu.Lock()
defer cm.mu.Unlock()
if cm.stopProcedureEngaged {
return errors.New("can't accept new runnable as stop procedure is already engaged")
}
cm.Lock()
defer cm.Unlock()
return cm.add(r)
}
func (cm *controllerManager) add(r Runnable) error {
// Set dependencies on the object
if err := cm.SetFields(r); err != nil {
return err
}
var shouldStart bool
// Add the runnable to the leader election or the non-leaderelection list
if leRunnable, ok := r.(LeaderElectionRunnable); ok && !leRunnable.NeedLeaderElection() {
shouldStart = cm.started
cm.nonLeaderElectionRunnables = append(cm.nonLeaderElectionRunnables, r)
} else if hasCache, ok := r.(hasCache); ok {
cm.caches = append(cm.caches, hasCache)
} else {
shouldStart = cm.startedLeader
cm.leaderElectionRunnables = append(cm.leaderElectionRunnables, r)
}
if shouldStart {
// If already started, start the controller
cm.startRunnable(r)
}
return nil
return cm.runnables.Add(r)
}
// Deprecated: use the equivalent Options field to set a field. This method will be removed in v0.10.
@@ -244,13 +212,17 @@ func (cm *controllerManager) SetFields(i interface{}) error {
// AddMetricsExtraHandler adds extra handler served on path to the http server that serves metrics.
func (cm *controllerManager) AddMetricsExtraHandler(path string, handler http.Handler) error {
cm.Lock()
defer cm.Unlock()
if cm.started {
return fmt.Errorf("unable to add new metrics handler because metrics endpoint has already been created")
}
if path == defaultMetricsEndpoint {
return fmt.Errorf("overriding builtin %s endpoint is not allowed", defaultMetricsEndpoint)
}
cm.mu.Lock()
defer cm.mu.Unlock()
if _, found := cm.metricsExtraHandlers[path]; found {
return fmt.Errorf("can't register extra handler by duplicate path %q on metrics http server", path)
}
@@ -262,14 +234,10 @@ func (cm *controllerManager) AddMetricsExtraHandler(path string, handler http.Ha
// AddHealthzCheck allows you to add Healthz checker.
func (cm *controllerManager) AddHealthzCheck(name string, check healthz.Checker) error {
cm.mu.Lock()
defer cm.mu.Unlock()
cm.Lock()
defer cm.Unlock()
if cm.stopProcedureEngaged {
return errors.New("can't accept new healthCheck as stop procedure is already engaged")
}
if cm.healthzStarted {
if cm.started {
return fmt.Errorf("unable to add new checker because healthz endpoint has already been created")
}
@@ -283,15 +251,11 @@ func (cm *controllerManager) AddHealthzCheck(name string, check healthz.Checker)
// AddReadyzCheck allows you to add Readyz checker.
func (cm *controllerManager) AddReadyzCheck(name string, check healthz.Checker) error {
cm.mu.Lock()
defer cm.mu.Unlock()
cm.Lock()
defer cm.Unlock()
if cm.stopProcedureEngaged {
return errors.New("can't accept new ready check as stop procedure is already engaged")
}
if cm.healthzStarted {
return fmt.Errorf("unable to add new checker because readyz endpoint has already been created")
if cm.started {
return fmt.Errorf("unable to add new checker because healthz endpoint has already been created")
}
if cm.readyzHandler == nil {
@@ -344,7 +308,7 @@ func (cm *controllerManager) GetWebhookServer() *webhook.Server {
}
}
if err := cm.Add(cm.webhookServer); err != nil {
panic("unable to add webhook server to the controller manager")
panic(fmt.Sprintf("unable to add webhook server to the controller manager: %s", err))
}
})
return cm.webhookServer
@@ -365,77 +329,89 @@ func (cm *controllerManager) serveMetrics() {
// TODO(JoelSpeed): Use existing Kubernetes machinery for serving metrics
mux := http.NewServeMux()
mux.Handle(defaultMetricsEndpoint, handler)
func() {
cm.mu.Lock()
defer cm.mu.Unlock()
for path, extraHandler := range cm.metricsExtraHandlers {
mux.Handle(path, extraHandler)
}
}()
server := http.Server{
Handler: mux,
for path, extraHandler := range cm.metricsExtraHandlers {
mux.Handle(path, extraHandler)
}
// Run the server
cm.startRunnable(RunnableFunc(func(_ context.Context) error {
cm.logger.Info("starting metrics server", "path", defaultMetricsEndpoint)
if err := server.Serve(cm.metricsListener); err != nil && err != http.ErrServerClosed {
return err
}
return nil
}))
// Shutdown the server when stop is closed
<-cm.internalProceduresStop
if err := server.Shutdown(cm.shutdownCtx); err != nil {
cm.errChan <- err
}
server := httpserver.New(mux)
go cm.httpServe("metrics", cm.logger.WithValues("path", defaultMetricsEndpoint), server, cm.metricsListener)
}
func (cm *controllerManager) serveHealthProbes() {
mux := http.NewServeMux()
server := http.Server{
Handler: mux,
server := httpserver.New(mux)
if cm.readyzHandler != nil {
mux.Handle(cm.readinessEndpointName, http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler))
// Append '/' suffix to handle subpaths
mux.Handle(cm.readinessEndpointName+"/", http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler))
}
if cm.healthzHandler != nil {
mux.Handle(cm.livenessEndpointName, http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler))
// Append '/' suffix to handle subpaths
mux.Handle(cm.livenessEndpointName+"/", http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler))
}
func() {
cm.mu.Lock()
defer cm.mu.Unlock()
go cm.httpServe("health probe", cm.logger, server, cm.healthProbeListener)
}
if cm.readyzHandler != nil {
mux.Handle(cm.readinessEndpointName, http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler))
// Append '/' suffix to handle subpaths
mux.Handle(cm.readinessEndpointName+"/", http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler))
}
if cm.healthzHandler != nil {
mux.Handle(cm.livenessEndpointName, http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler))
// Append '/' suffix to handle subpaths
mux.Handle(cm.livenessEndpointName+"/", http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler))
}
func (cm *controllerManager) httpServe(kind string, log logr.Logger, server *http.Server, ln net.Listener) {
log = log.WithValues("kind", kind, "addr", ln.Addr())
// Run server
cm.startRunnable(RunnableFunc(func(_ context.Context) error {
if err := server.Serve(cm.healthProbeListener); err != nil && err != http.ErrServerClosed {
return err
go func() {
log.Info("Starting server")
if err := server.Serve(ln); err != nil {
if errors.Is(err, http.ErrServerClosed) {
return
}
return nil
}))
cm.healthzStarted = true
if atomic.LoadInt64(cm.stopProcedureEngaged) > 0 {
// There might be cases where connections are still open and we try to shutdown
// but not having enough time to close the connection causes an error in Serve
//
// In that case we want to avoid returning an error to the main error channel.
log.Error(err, "error on Serve after stop has been engaged")
return
}
cm.errChan <- err
}
}()
// Shutdown the server when stop is closed
// Shutdown the server when stop is closed.
<-cm.internalProceduresStop
if err := server.Shutdown(cm.shutdownCtx); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
// Avoid logging context related errors.
return
}
if atomic.LoadInt64(cm.stopProcedureEngaged) > 0 {
cm.logger.Error(err, "error on Shutdown after stop has been engaged")
return
}
cm.errChan <- err
}
}
// Start starts the manager and waits indefinitely.
// There is only two ways to have start return:
// An error has occurred during in one of the internal operations,
// such as leader election, cache start, webhooks, and so on.
// Or, the context is cancelled.
func (cm *controllerManager) Start(ctx context.Context) (err error) {
if err := cm.Add(cm.cluster); err != nil {
return fmt.Errorf("failed to add cluster to runnables: %w", err)
cm.Lock()
if cm.started {
cm.Unlock()
return errors.New("manager already started")
}
var ready bool
defer func() {
// Only unlock the manager if we haven't reached
// the internal readiness condition.
if !ready {
cm.Unlock()
}
}()
// Initialize the internal context.
cm.internalCtx, cm.internalCancel = context.WithCancel(ctx)
// This chan indicates that stop is complete, in other words all runnables have returned or timeout on stop request
@@ -457,40 +433,70 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) {
}
}()
// initialize this here so that we reset the signal channel state on every start
// Everything that might write into this channel must be started in a new goroutine,
// because otherwise we might block this routine trying to write into the full channel
// and will not be able to enter the deferred cm.engageStopProcedure() which drains
// it.
cm.errChan = make(chan error)
// Add the cluster runnable.
if err := cm.add(cm.cluster); err != nil {
return fmt.Errorf("failed to add cluster to runnables: %w", err)
}
// Metrics should be served whether the controller is leader or not.
// (If we don't serve metrics for non-leaders, prometheus will still scrape
// the pod but will get a connection refused)
// the pod but will get a connection refused).
if cm.metricsListener != nil {
go cm.serveMetrics()
cm.serveMetrics()
}
// Serve health probes
// Serve health probes.
if cm.healthProbeListener != nil {
go cm.serveHealthProbes()
cm.serveHealthProbes()
}
go cm.startNonLeaderElectionRunnables()
go func() {
if cm.resourceLock != nil {
err := cm.startLeaderElection()
if err != nil {
cm.errChan <- err
}
} else {
// Treat not having leader election enabled the same as being elected.
cm.startLeaderElectionRunnables()
close(cm.elected)
// First start any webhook servers, which includes conversion, validation, and defaulting
// webhooks that are registered.
//
// WARNING: Webhooks MUST start before any cache is populated, otherwise there is a race condition
// between conversion webhooks and the cache sync (usually initial list) which causes the webhooks
// to never start because no cache can be populated.
if err := cm.runnables.Webhooks.Start(cm.internalCtx); err != nil {
if !errors.Is(err, wait.ErrWaitTimeout) {
return err
}
}()
}
// Start and wait for caches.
if err := cm.runnables.Caches.Start(cm.internalCtx); err != nil {
if !errors.Is(err, wait.ErrWaitTimeout) {
return err
}
}
// Start the non-leaderelection Runnables after the cache has synced.
if err := cm.runnables.Others.Start(cm.internalCtx); err != nil {
if !errors.Is(err, wait.ErrWaitTimeout) {
return err
}
}
// Start the leader election and all required runnables.
{
ctx, cancel := context.WithCancel(context.Background())
cm.leaderElectionCancel = cancel
go func() {
if cm.resourceLock != nil {
if err := cm.startLeaderElection(ctx); err != nil {
cm.errChan <- err
}
} else {
// Treat not having leader election enabled the same as being elected.
if err := cm.startLeaderElectionRunnables(); err != nil {
cm.errChan <- err
}
close(cm.elected)
}
}()
}
ready = true
cm.Unlock()
select {
case <-ctx.Done():
// We are done
@@ -504,24 +510,31 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) {
// engageStopProcedure signals all runnables to stop, reads potential errors
// from the errChan and waits for them to end. It must not be called more than once.
func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) error {
// Populate the shutdown context.
var shutdownCancel context.CancelFunc
if cm.gracefulShutdownTimeout > 0 {
cm.shutdownCtx, shutdownCancel = context.WithTimeout(context.Background(), cm.gracefulShutdownTimeout)
} else {
cm.shutdownCtx, shutdownCancel = context.WithCancel(context.Background())
if !atomic.CompareAndSwapInt64(cm.stopProcedureEngaged, 0, 1) {
return errors.New("stop procedure already engaged")
}
defer shutdownCancel()
// Cancel the internal stop channel and wait for the procedures to stop and complete.
close(cm.internalProceduresStop)
cm.internalCancel()
// Populate the shutdown context, this operation MUST be done before
// closing the internalProceduresStop channel.
//
// The shutdown context immediately expires if the gracefulShutdownTimeout is not set.
var shutdownCancel context.CancelFunc
cm.shutdownCtx, shutdownCancel = context.WithTimeout(context.Background(), cm.gracefulShutdownTimeout)
defer shutdownCancel()
// Start draining the errors before acquiring the lock to make sure we don't deadlock
// if something that has the lock is blocked on trying to write into the unbuffered
// channel after something else already wrote into it.
var closeOnce sync.Once
go func() {
for {
// Closing in the for loop is required to avoid race conditions between
// the closure of all internal procedures and making sure to have a reader off the error channel.
closeOnce.Do(func() {
// Cancel the internal stop channel and wait for the procedures to stop and complete.
close(cm.internalProceduresStop)
cm.internalCancel()
})
select {
case err, ok := <-cm.errChan:
if ok {
@@ -532,26 +545,14 @@ func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) e
}
}
}()
if cm.gracefulShutdownTimeout == 0 {
return nil
}
cm.mu.Lock()
defer cm.mu.Unlock()
cm.stopProcedureEngaged = true
// we want to close this after the other runnables stop, because we don't
// We want to close this after the other runnables stop, because we don't
// want things like leader election to try and emit events on a closed
// channel
defer cm.recorderProvider.Stop(cm.shutdownCtx)
return cm.waitForRunnableToEnd(shutdownCancel)
}
// waitForRunnableToEnd blocks until all runnables ended or the
// tearDownTimeout was reached. In the latter case, an error is returned.
func (cm *controllerManager) waitForRunnableToEnd(shutdownCancel context.CancelFunc) (retErr error) {
// Cancel leader election only after we waited. It will os.Exit() the app for safety.
defer func() {
if retErr == nil && cm.leaderElectionCancel != nil {
// Cancel leader election only after we waited. It will os.Exit() the app for safety.
if cm.resourceLock != nil {
// After asking the context to be cancelled, make sure
// we wait for the leader stopped channel to be closed, otherwise
// we might encounter race conditions between this code
@@ -562,102 +563,49 @@ func (cm *controllerManager) waitForRunnableToEnd(shutdownCancel context.CancelF
}()
go func() {
cm.waitForRunnable.Wait()
// First stop the non-leader election runnables.
cm.logger.Info("Stopping and waiting for non leader election runnables")
cm.runnables.Others.StopAndWait(cm.shutdownCtx)
// Stop all the leader election runnables, which includes reconcilers.
cm.logger.Info("Stopping and waiting for leader election runnables")
cm.runnables.LeaderElection.StopAndWait(cm.shutdownCtx)
// Stop the caches before the leader election runnables, this is an important
// step to make sure that we don't race with the reconcilers by receiving more events
// from the API servers and enqueueing them.
cm.logger.Info("Stopping and waiting for caches")
cm.runnables.Caches.StopAndWait(cm.shutdownCtx)
// Webhooks should come last, as they might be still serving some requests.
cm.logger.Info("Stopping and waiting for webhooks")
cm.runnables.Webhooks.StopAndWait(cm.shutdownCtx)
// Proceed to close the manager and overall shutdown context.
cm.logger.Info("Wait completed, proceeding to shutdown the manager")
shutdownCancel()
}()
<-cm.shutdownCtx.Done()
if err := cm.shutdownCtx.Err(); err != nil && err != context.Canceled {
return fmt.Errorf("failed waiting for all runnables to end within grace period of %s: %w", cm.gracefulShutdownTimeout, err)
if err := cm.shutdownCtx.Err(); err != nil && !errors.Is(err, context.Canceled) {
if errors.Is(err, context.DeadlineExceeded) {
if cm.gracefulShutdownTimeout > 0 {
return fmt.Errorf("failed waiting for all runnables to end within grace period of %s: %w", cm.gracefulShutdownTimeout, err)
}
return nil
}
// For any other error, return the error.
return err
}
return nil
}
func (cm *controllerManager) startNonLeaderElectionRunnables() {
cm.mu.Lock()
defer cm.mu.Unlock()
// First start any webhook servers, which includes conversion, validation, and defaulting
// webhooks that are registered.
//
// WARNING: Webhooks MUST start before any cache is populated, otherwise there is a race condition
// between conversion webhooks and the cache sync (usually initial list) which causes the webhooks
// to never start because no cache can be populated.
for _, c := range cm.nonLeaderElectionRunnables {
if _, ok := c.(*webhook.Server); ok {
cm.startRunnable(c)
}
}
// Start and wait for caches.
cm.waitForCache(cm.internalCtx)
// Start the non-leaderelection Runnables after the cache has synced
for _, c := range cm.nonLeaderElectionRunnables {
if _, ok := c.(*webhook.Server); ok {
continue
}
// Controllers block, but we want to return an error if any have an error starting.
// Write any Start errors to a channel so we can return them
cm.startRunnable(c)
}
func (cm *controllerManager) startLeaderElectionRunnables() error {
return cm.runnables.LeaderElection.Start(cm.internalCtx)
}
func (cm *controllerManager) startLeaderElectionRunnables() {
cm.mu.Lock()
defer cm.mu.Unlock()
cm.waitForCache(cm.internalCtx)
// Start the leader election Runnables after the cache has synced
for _, c := range cm.leaderElectionRunnables {
// Controllers block, but we want to return an error if any have an error starting.
// Write any Start errors to a channel so we can return them
cm.startRunnable(c)
}
cm.startedLeader = true
}
func (cm *controllerManager) waitForCache(ctx context.Context) {
if cm.started {
return
}
for _, cache := range cm.caches {
cm.startRunnable(cache)
}
// Wait for the caches to sync.
// TODO(community): Check the return value and write a test
for _, cache := range cm.caches {
cache.GetCache().WaitForCacheSync(ctx)
}
// TODO: This should be the return value of cm.cache.WaitForCacheSync but we abuse
// cm.started as check if we already started the cache so it must always become true.
// Making sure that the cache doesn't get started twice is needed to not get a "close
// of closed channel" panic
cm.started = true
}
func (cm *controllerManager) startLeaderElection() (err error) {
ctx, cancel := context.WithCancel(context.Background())
cm.mu.Lock()
cm.leaderElectionCancel = cancel
cm.mu.Unlock()
if cm.onStoppedLeading == nil {
cm.onStoppedLeading = func() {
// Make sure graceful shutdown is skipped if we lost the leader lock without
// intending to.
cm.gracefulShutdownTimeout = time.Duration(0)
// Most implementations of leader election log.Fatal() here.
// Since Start is wrapped in log.Fatal when called, we can just return
// an error here which will cause the program to exit.
cm.errChan <- errors.New("leader election lost")
}
}
func (cm *controllerManager) startLeaderElection(ctx context.Context) (err error) {
l, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
Lock: cm.resourceLock,
LeaseDuration: cm.leaseDuration,
@@ -665,10 +613,24 @@ func (cm *controllerManager) startLeaderElection() (err error) {
RetryPeriod: cm.retryPeriod,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(_ context.Context) {
cm.startLeaderElectionRunnables()
if err := cm.startLeaderElectionRunnables(); err != nil {
cm.errChan <- err
return
}
close(cm.elected)
},
OnStoppedLeading: cm.onStoppedLeading,
OnStoppedLeading: func() {
if cm.onStoppedLeading != nil {
cm.onStoppedLeading()
}
// Make sure graceful shutdown is skipped if we lost the leader lock without
// intending to.
cm.gracefulShutdownTimeout = time.Duration(0)
// Most implementations of leader election log.Fatal() here.
// Since Start is wrapped in log.Fatal when called, we can just return
// an error here which will cause the program to exit.
cm.errChan <- errors.New("leader election lost")
},
},
ReleaseOnCancel: cm.leaderElectionReleaseOnCancel,
})
@@ -688,13 +650,3 @@ func (cm *controllerManager) startLeaderElection() (err error) {
func (cm *controllerManager) Elected() <-chan struct{} {
return cm.elected
}
func (cm *controllerManager) startRunnable(r Runnable) {
cm.waitForRunnable.Add(1)
go func() {
defer cm.waitForRunnable.Done()
if err := r.Start(cm.internalCtx); err != nil {
cm.errChan <- err
}
}()
}

View File

@@ -31,6 +31,7 @@ import (
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"k8s.io/client-go/tools/record"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/cluster"
@@ -97,9 +98,9 @@ type Manager interface {
// Options are the arguments for creating a new Manager.
type Options struct {
// Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources
// Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources.
// Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better
// idea to pass your own scheme in. See the documentation in pkg/scheme for more information.
// to pass your own scheme in. See the documentation in pkg/scheme for more information.
Scheme *runtime.Scheme
// MapperProvider provides the rest mapper used to map go types to Kubernetes APIs
@@ -185,11 +186,11 @@ type Options struct {
// between tries of actions. Default is 2 seconds.
RetryPeriod *time.Duration
// Namespace if specified restricts the manager's cache to watch objects in
// the desired namespace Defaults to all namespaces
// Namespace, if specified, restricts the manager's cache to watch objects in
// the desired namespace. Defaults to all namespaces.
//
// Note: If a namespace is specified, controllers can still Watch for a
// cluster-scoped resource (e.g Node). For namespaced resources the cache
// cluster-scoped resource (e.g Node). For namespaced resources, the cache
// will only hold objects from the desired namespace.
Namespace string
@@ -227,7 +228,7 @@ type Options struct {
// if this is set, the Manager will use this server instead.
WebhookServer *webhook.Server
// Functions to all for a user to customize the values that will be injected.
// Functions to allow for a user to customize values that will be injected.
// NewCache is the function that will create the cache to be used
// by the manager. If not set this will use the default new cache function.
@@ -238,6 +239,11 @@ type Options struct {
// use the cache for reads and the client for writes.
NewClient cluster.NewClientFunc
// BaseContext is the function that provides Context values to Runnables
// managed by the Manager. If a BaseContext function isn't provided, Runnables
// will receive a new Background Context instead.
BaseContext BaseContextFunc
// ClientDisableCacheFor tells the client that, if any cache is used, to bypass it
// for the given objects.
ClientDisableCacheFor []client.Object
@@ -277,6 +283,10 @@ type Options struct {
newHealthProbeListener func(addr string) (net.Listener, error)
}
// BaseContextFunc is a function used to provide a base Context to Runnables
// managed by a Manager.
type BaseContextFunc func() context.Context
// Runnable allows a component to be started.
// It's very important that Start blocks until
// it's done running.
@@ -334,11 +344,21 @@ func New(config *rest.Config, options Options) (Manager, error) {
}
// Create the resource lock to enable leader election)
leaderConfig := options.LeaderElectionConfig
if leaderConfig == nil {
var leaderConfig *rest.Config
var leaderRecorderProvider *intrec.Provider
if options.LeaderElectionConfig == nil {
leaderConfig = rest.CopyConfig(config)
leaderRecorderProvider = recorderProvider
} else {
leaderConfig = rest.CopyConfig(options.LeaderElectionConfig)
leaderRecorderProvider, err = options.newRecorderProvider(leaderConfig, cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster)
if err != nil {
return nil, err
}
}
resourceLock, err := options.newResourceLock(leaderConfig, recorderProvider, leaderelection.Options{
resourceLock, err := options.newResourceLock(leaderConfig, leaderRecorderProvider, leaderelection.Options{
LeaderElection: options.LeaderElection,
LeaderElectionResourceLock: options.LeaderElectionResourceLock,
LeaderElectionID: options.LeaderElectionID,
@@ -365,8 +385,14 @@ func New(config *rest.Config, options Options) (Manager, error) {
return nil, err
}
errChan := make(chan error)
runnables := newRunnables(options.BaseContext, errChan)
return &controllerManager{
stopProcedureEngaged: pointer.Int64(0),
cluster: cluster,
runnables: runnables,
errChan: errChan,
recorderProvider: recorderProvider,
resourceLock: resourceLock,
metricsListener: metricsListener,
@@ -468,6 +494,11 @@ func (o Options) AndFromOrDie(loader config.ControllerManagerConfiguration) Opti
}
func (o Options) setLeaderElectionConfig(obj v1alpha1.ControllerManagerConfigurationSpec) Options {
if obj.LeaderElection == nil {
// The source does not have any configuration; noop
return o
}
if !o.LeaderElection && obj.LeaderElection.LeaderElect != nil {
o.LeaderElection = *obj.LeaderElection.LeaderElect
}
@@ -507,11 +538,17 @@ func defaultHealthProbeListener(addr string) (net.Listener, error) {
ln, err := net.Listen("tcp", addr)
if err != nil {
return nil, fmt.Errorf("error listening on %s: %v", addr, err)
return nil, fmt.Errorf("error listening on %s: %w", addr, err)
}
return ln, nil
}
// defaultBaseContext is used as the BaseContext value in Options if one
// has not already been set.
func defaultBaseContext() context.Context {
return context.Background()
}
// setOptionsDefaults set default values for Options fields.
func setOptionsDefaults(options Options) Options {
// Allow newResourceLock to be mocked
@@ -571,9 +608,13 @@ func setOptionsDefaults(options Options) Options {
options.GracefulShutdownTimeout = &gracefulShutdownTimeout
}
if options.Logger == nil {
if options.Logger.GetSink() == nil {
options.Logger = log.Log
}
if options.BaseContext == nil {
options.BaseContext = defaultBaseContext
}
return options
}

View File

@@ -0,0 +1,297 @@
package manager
import (
"context"
"errors"
"sync"
"sigs.k8s.io/controller-runtime/pkg/webhook"
)
var (
errRunnableGroupStopped = errors.New("can't accept new runnable as stop procedure is already engaged")
)
// readyRunnable encapsulates a runnable with
// a ready check.
type readyRunnable struct {
Runnable
Check runnableCheck
signalReady bool
}
// runnableCheck can be passed to Add() to let the runnable group determine that a
// runnable is ready. A runnable check should block until a runnable is ready,
// if the returned result is false, the runnable is considered not ready and failed.
type runnableCheck func(ctx context.Context) bool
// runnables handles all the runnables for a manager by grouping them accordingly to their
// type (webhooks, caches etc.).
type runnables struct {
Webhooks *runnableGroup
Caches *runnableGroup
LeaderElection *runnableGroup
Others *runnableGroup
}
// newRunnables creates a new runnables object.
func newRunnables(baseContext BaseContextFunc, errChan chan error) *runnables {
return &runnables{
Webhooks: newRunnableGroup(baseContext, errChan),
Caches: newRunnableGroup(baseContext, errChan),
LeaderElection: newRunnableGroup(baseContext, errChan),
Others: newRunnableGroup(baseContext, errChan),
}
}
// Add adds a runnable to closest group of runnable that they belong to.
//
// Add should be able to be called before and after Start, but not after StopAndWait.
// Add should return an error when called during StopAndWait.
// The runnables added before Start are started when Start is called.
// The runnables added after Start are started directly.
func (r *runnables) Add(fn Runnable) error {
switch runnable := fn.(type) {
case hasCache:
return r.Caches.Add(fn, func(ctx context.Context) bool {
return runnable.GetCache().WaitForCacheSync(ctx)
})
case *webhook.Server:
return r.Webhooks.Add(fn, nil)
case LeaderElectionRunnable:
if !runnable.NeedLeaderElection() {
return r.Others.Add(fn, nil)
}
return r.LeaderElection.Add(fn, nil)
default:
return r.LeaderElection.Add(fn, nil)
}
}
// runnableGroup manages a group of runnables that are
// meant to be running together until StopAndWait is called.
//
// Runnables can be added to a group after the group has started
// but not after it's stopped or while shutting down.
type runnableGroup struct {
ctx context.Context
cancel context.CancelFunc
start sync.Mutex
startOnce sync.Once
started bool
startQueue []*readyRunnable
startReadyCh chan *readyRunnable
stop sync.RWMutex
stopOnce sync.Once
stopped bool
// errChan is the error channel passed by the caller
// when the group is created.
// All errors are forwarded to this channel once they occur.
errChan chan error
// ch is the internal channel where the runnables are read off from.
ch chan *readyRunnable
// wg is an internal sync.WaitGroup that allows us to properly stop
// and wait for all the runnables to finish before returning.
wg *sync.WaitGroup
}
func newRunnableGroup(baseContext BaseContextFunc, errChan chan error) *runnableGroup {
r := &runnableGroup{
startReadyCh: make(chan *readyRunnable),
errChan: errChan,
ch: make(chan *readyRunnable),
wg: new(sync.WaitGroup),
}
r.ctx, r.cancel = context.WithCancel(baseContext())
return r
}
// Started returns true if the group has started.
func (r *runnableGroup) Started() bool {
r.start.Lock()
defer r.start.Unlock()
return r.started
}
// Start starts the group and waits for all
// initially registered runnables to start.
// It can only be called once, subsequent calls have no effect.
func (r *runnableGroup) Start(ctx context.Context) error {
var retErr error
r.startOnce.Do(func() {
defer close(r.startReadyCh)
// Start the internal reconciler.
go r.reconcile()
// Start the group and queue up all
// the runnables that were added prior.
r.start.Lock()
r.started = true
for _, rn := range r.startQueue {
rn.signalReady = true
r.ch <- rn
}
r.start.Unlock()
// If we don't have any queue, return.
if len(r.startQueue) == 0 {
return
}
// Wait for all runnables to signal.
for {
select {
case <-ctx.Done():
if err := ctx.Err(); !errors.Is(err, context.Canceled) {
retErr = err
}
case rn := <-r.startReadyCh:
for i, existing := range r.startQueue {
if existing == rn {
// Remove the item from the start queue.
r.startQueue = append(r.startQueue[:i], r.startQueue[i+1:]...)
break
}
}
// We're done waiting if the queue is empty, return.
if len(r.startQueue) == 0 {
return
}
}
}
})
return retErr
}
// reconcile is our main entrypoint for every runnable added
// to this group. Its primary job is to read off the internal channel
// and schedule runnables while tracking their state.
func (r *runnableGroup) reconcile() {
for runnable := range r.ch {
// Handle stop.
// If the shutdown has been called we want to avoid
// adding new goroutines to the WaitGroup because Wait()
// panics if Add() is called after it.
{
r.stop.RLock()
if r.stopped {
// Drop any runnables if we're stopped.
r.errChan <- errRunnableGroupStopped
r.stop.RUnlock()
continue
}
// Why is this here?
// When StopAndWait is called, if a runnable is in the process
// of being added, we could end up in a situation where
// the WaitGroup is incremented while StopAndWait has called Wait(),
// which would result in a panic.
r.wg.Add(1)
r.stop.RUnlock()
}
// Start the runnable.
go func(rn *readyRunnable) {
go func() {
if rn.Check(r.ctx) {
if rn.signalReady {
r.startReadyCh <- rn
}
}
}()
// If we return, the runnable ended cleanly
// or returned an error to the channel.
//
// We should always decrement the WaitGroup here.
defer r.wg.Done()
// Start the runnable.
if err := rn.Start(r.ctx); err != nil {
r.errChan <- err
}
}(runnable)
}
}
// Add should be able to be called before and after Start, but not after StopAndWait.
// Add should return an error when called during StopAndWait.
func (r *runnableGroup) Add(rn Runnable, ready runnableCheck) error {
r.stop.RLock()
if r.stopped {
r.stop.RUnlock()
return errRunnableGroupStopped
}
r.stop.RUnlock()
if ready == nil {
ready = func(_ context.Context) bool { return true }
}
readyRunnable := &readyRunnable{
Runnable: rn,
Check: ready,
}
// Handle start.
// If the overall runnable group isn't started yet
// we want to buffer the runnables and let Start()
// queue them up again later.
{
r.start.Lock()
// Check if we're already started.
if !r.started {
// Store the runnable in the internal if not.
r.startQueue = append(r.startQueue, readyRunnable)
r.start.Unlock()
return nil
}
r.start.Unlock()
}
// Enqueue the runnable.
r.ch <- readyRunnable
return nil
}
// StopAndWait waits for all the runnables to finish before returning.
func (r *runnableGroup) StopAndWait(ctx context.Context) {
r.stopOnce.Do(func() {
// Close the reconciler channel once we're done.
defer close(r.ch)
_ = r.Start(ctx)
r.stop.Lock()
// Store the stopped variable so we don't accept any new
// runnables for the time being.
r.stopped = true
r.stop.Unlock()
// Cancel the internal channel.
r.cancel()
done := make(chan struct{})
go func() {
defer close(done)
// Wait for all the runnables to finish.
r.wg.Wait()
}()
select {
case <-done:
// We're done, exit.
case <-ctx.Done():
// Calling context has expired, exit.
}
})
}

View File

@@ -24,8 +24,8 @@ import (
var onlyOneSignalHandler = make(chan struct{})
// SetupSignalHandler registers for SIGTERM and SIGINT. A stop channel is returned
// which is closed on one of these signals. If a second signal is caught, the program
// SetupSignalHandler registers for SIGTERM and SIGINT. A context is returned
// which is canceled on one of these signals. If a second signal is caught, the program
// is terminated with exit code 1.
func SetupSignalHandler() context.Context {
close(onlyOneSignalHandler) // panics when called twice

View File

@@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
/*

View File

@@ -52,7 +52,24 @@ const (
var (
// client metrics.
requestLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
// RequestLatency reports the request latency in seconds per verb/URL.
// Deprecated: This metric is deprecated for removal in a future release: using the URL as a
// dimension results in cardinality explosion for some consumers. It was deprecated upstream
// in k8s v1.14 and hidden in v1.17 via https://github.com/kubernetes/kubernetes/pull/83836.
// It is not registered by default. To register:
// import (
// clientmetrics "k8s.io/client-go/tools/metrics"
// clmetrics "sigs.k8s.io/controller-runtime/metrics"
// )
//
// func init() {
// clmetrics.Registry.MustRegister(clmetrics.RequestLatency)
// clientmetrics.Register(clientmetrics.RegisterOpts{
// RequestLatency: clmetrics.LatencyAdapter
// })
// }
RequestLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Subsystem: RestClientSubsystem,
Name: LatencyKey,
Help: "Request latency in seconds. Broken down by verb and URL.",
@@ -127,13 +144,11 @@ func init() {
// registerClientMetrics sets up the client latency metrics from client-go.
func registerClientMetrics() {
// register the metrics with our registry
Registry.MustRegister(requestLatency)
Registry.MustRegister(requestResult)
// register the metrics with client-go
clientmetrics.Register(clientmetrics.RegisterOpts{
RequestLatency: &latencyAdapter{metric: requestLatency},
RequestResult: &resultAdapter{metric: requestResult},
RequestResult: &resultAdapter{metric: requestResult},
})
}
@@ -159,11 +174,13 @@ func registerReflectorMetrics() {
// copied (more-or-less directly) from k8s.io/kubernetes setup code
// (which isn't anywhere in an easily-importable place).
type latencyAdapter struct {
// LatencyAdapter implements LatencyMetric.
type LatencyAdapter struct {
metric *prometheus.HistogramVec
}
func (l *latencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) {
// Observe increments the request latency metric for the given verb/URL.
func (l *LatencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) {
l.metric.WithLabelValues(verb, u.String()).Observe(latency.Seconds())
}

View File

@@ -41,7 +41,7 @@ func NewListener(addr string) (net.Listener, error) {
return nil, nil
}
log.Info("metrics server is starting to listen", "addr", addr)
log.Info("Metrics server is starting to listen", "addr", addr)
ln, err := net.Listen("tcp", addr)
if err != nil {
er := fmt.Errorf("error listening on %s: %w", addr, err)

View File

@@ -24,6 +24,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
)
var log = logf.RuntimeLog.WithName("predicate").WithName("eventFilters")
@@ -239,6 +240,15 @@ type and struct {
predicates []Predicate
}
func (a and) InjectFunc(f inject.Func) error {
for _, p := range a.predicates {
if err := f(p); err != nil {
return err
}
}
return nil
}
func (a and) Create(e event.CreateEvent) bool {
for _, p := range a.predicates {
if !p.Create(e) {
@@ -284,6 +294,15 @@ type or struct {
predicates []Predicate
}
func (o or) InjectFunc(f inject.Func) error {
for _, p := range o.predicates {
if err := f(p); err != nil {
return err
}
}
return nil
}
func (o or) Create(e event.CreateEvent) bool {
for _, p := range o.predicates {
if p.Create(e) {

View File

@@ -87,7 +87,7 @@ For example if responding to a Pod Delete Event, the Request won't contain that
instead the reconcile function observes this when reading the cluster state and seeing the Pod as missing.
*/
type Reconciler interface {
// Reconciler performs a full reconciliation for the object referred to by the Request.
// Reconcile performs a full reconciliation for the object referred to by the Request.
// The Controller will requeue the Request to be processed again if an error is non-nil or
// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
Reconcile(context.Context, Request) (Result, error)

View File

@@ -21,8 +21,11 @@ import (
"errors"
"fmt"
"sync"
"time"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
@@ -119,17 +122,39 @@ func (ks *Kind) Start(ctx context.Context, handler handler.EventHandler, queue w
ctx, ks.startCancel = context.WithCancel(ctx)
ks.started = make(chan error)
go func() {
// Lookup the Informer from the Cache and add an EventHandler which populates the Queue
i, err := ks.cache.GetInformer(ctx, ks.Type)
if err != nil {
kindMatchErr := &meta.NoKindMatchError{}
if errors.As(err, &kindMatchErr) {
log.Error(err, "if kind is a CRD, it should be installed before calling Start",
"kind", kindMatchErr.GroupKind)
var (
i cache.Informer
lastErr error
)
// Tries to get an informer until it returns true,
// an error or the specified context is cancelled or expired.
if err := wait.PollImmediateUntilWithContext(ctx, 10*time.Second, func(ctx context.Context) (bool, error) {
// Lookup the Informer from the Cache and add an EventHandler which populates the Queue
i, lastErr = ks.cache.GetInformer(ctx, ks.Type)
if lastErr != nil {
kindMatchErr := &meta.NoKindMatchError{}
switch {
case errors.As(lastErr, &kindMatchErr):
log.Error(lastErr, "if kind is a CRD, it should be installed before calling Start",
"kind", kindMatchErr.GroupKind)
case runtime.IsNotRegisteredError(lastErr):
log.Error(lastErr, "kind must be registered to the Scheme")
default:
log.Error(lastErr, "failed to get informer from cache")
}
return false, nil // Retry.
}
return true, nil
}); err != nil {
if lastErr != nil {
ks.started <- fmt.Errorf("failed to get informer from cache: %w", lastErr)
return
}
ks.started <- err
return
}
i.AddEventHandler(internal.EventHandler{Queue: queue, EventHandler: handler, Predicates: prct})
if !ks.cache.WaitForCacheSync(ctx) {
// Would be great to return something more informative here
@@ -142,10 +167,10 @@ func (ks *Kind) Start(ctx context.Context, handler handler.EventHandler, queue w
}
func (ks *Kind) String() string {
if ks.Type != nil && ks.Type.GetObjectKind() != nil {
return fmt.Sprintf("kind source: %v", ks.Type.GetObjectKind().GroupVersionKind().String())
if ks.Type != nil {
return fmt.Sprintf("kind source: %T", ks.Type)
}
return "kind source: unknown GVK"
return "kind source: unknown type"
}
// WaitForSync implements SyncingSource to allow controllers to wait with starting
@@ -156,6 +181,9 @@ func (ks *Kind) WaitForSync(ctx context.Context) error {
return err
case <-ctx.Done():
ks.startCancel()
if errors.Is(ctx.Err(), context.Canceled) {
return nil
}
return errors.New("timed out waiting for cache to be synced")
}
}

View File

@@ -0,0 +1,85 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package admission
import (
"context"
"encoding/json"
"errors"
"net/http"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
)
// CustomDefaulter defines functions for setting defaults on resources.
type CustomDefaulter interface {
Default(ctx context.Context, obj runtime.Object) error
}
// WithCustomDefaulter creates a new Webhook for a CustomDefaulter interface.
func WithCustomDefaulter(obj runtime.Object, defaulter CustomDefaulter) *Webhook {
return &Webhook{
Handler: &defaulterForType{object: obj, defaulter: defaulter},
}
}
type defaulterForType struct {
defaulter CustomDefaulter
object runtime.Object
decoder *Decoder
}
var _ DecoderInjector = &defaulterForType{}
func (h *defaulterForType) InjectDecoder(d *Decoder) error {
h.decoder = d
return nil
}
// Handle handles admission requests.
func (h *defaulterForType) Handle(ctx context.Context, req Request) Response {
if h.defaulter == nil {
panic("defaulter should never be nil")
}
if h.object == nil {
panic("object should never be nil")
}
// Get the object in the request
obj := h.object.DeepCopyObject()
if err := h.decoder.Decode(req, obj); err != nil {
return Errored(http.StatusBadRequest, err)
}
// Default the object
if err := h.defaulter.Default(ctx, obj); err != nil {
var apiStatus apierrors.APIStatus
if errors.As(err, &apiStatus) {
return validationResponseFromStatus(false, apiStatus.Status())
}
return Denied(err.Error())
}
// Create the patch
marshalled, err := json.Marshal(obj)
if err != nil {
return Errored(http.StatusInternalServerError, err)
}
return PatchResponseFromRaw(req.Object.Raw, marshalled)
}

View File

@@ -21,7 +21,6 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
v1 "k8s.io/api/admission/v1"
@@ -60,7 +59,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
defer r.Body.Close()
if body, err = ioutil.ReadAll(r.Body); err != nil {
if body, err = io.ReadAll(r.Body); err != nil {
wh.log.Error(err, "unable to read the body from the incoming request")
reviewResponse = Errored(http.StatusBadRequest, err)
wh.writeResponse(w, reviewResponse)

View File

@@ -0,0 +1,111 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package admission
import (
"context"
"errors"
"fmt"
"net/http"
v1 "k8s.io/api/admission/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
)
// CustomValidator defines functions for validating an operation.
type CustomValidator interface {
ValidateCreate(ctx context.Context, obj runtime.Object) error
ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error
ValidateDelete(ctx context.Context, obj runtime.Object) error
}
// WithCustomValidator creates a new Webhook for validating the provided type.
func WithCustomValidator(obj runtime.Object, validator CustomValidator) *Webhook {
return &Webhook{
Handler: &validatorForType{object: obj, validator: validator},
}
}
type validatorForType struct {
validator CustomValidator
object runtime.Object
decoder *Decoder
}
var _ DecoderInjector = &validatorForType{}
// InjectDecoder injects the decoder into a validatingHandler.
func (h *validatorForType) InjectDecoder(d *Decoder) error {
h.decoder = d
return nil
}
// Handle handles admission requests.
func (h *validatorForType) Handle(ctx context.Context, req Request) Response {
if h.validator == nil {
panic("validator should never be nil")
}
if h.object == nil {
panic("object should never be nil")
}
// Get the object in the request
obj := h.object.DeepCopyObject()
var err error
switch req.Operation {
case v1.Create:
if err := h.decoder.Decode(req, obj); err != nil {
return Errored(http.StatusBadRequest, err)
}
err = h.validator.ValidateCreate(ctx, obj)
case v1.Update:
oldObj := obj.DeepCopyObject()
if err := h.decoder.DecodeRaw(req.Object, obj); err != nil {
return Errored(http.StatusBadRequest, err)
}
if err := h.decoder.DecodeRaw(req.OldObject, oldObj); err != nil {
return Errored(http.StatusBadRequest, err)
}
err = h.validator.ValidateUpdate(ctx, oldObj, obj)
case v1.Delete:
// In reference to PR: https://github.com/kubernetes/kubernetes/pull/76346
// OldObject contains the object being deleted
if err := h.decoder.DecodeRaw(req.OldObject, obj); err != nil {
return Errored(http.StatusBadRequest, err)
}
err = h.validator.ValidateDelete(ctx, obj)
default:
return Errored(http.StatusBadRequest, fmt.Errorf("unknown operation request %q", req.Operation))
}
// Check the error message first.
if err != nil {
var apiStatus apierrors.APIStatus
if errors.As(err, &apiStatus) {
return validationResponseFromStatus(false, apiStatus.Status())
}
return Denied(err.Error())
}
// Return allowed if everything succeeded.
return Allowed("")
}

View File

@@ -243,7 +243,7 @@ func StandaloneWebhook(hook *Webhook, opts StandaloneOptions) (http.Handler, err
return nil, err
}
if opts.Logger == nil {
if opts.Logger.GetSink() == nil {
opts.Logger = logf.RuntimeLog.WithName("webhook")
}
hook.log = opts.Logger

View File

@@ -29,6 +29,12 @@ type Defaulter = admission.Defaulter
// Validator defines functions for validating an operation.
type Validator = admission.Validator
// CustomDefaulter defines functions for setting defaults on resources.
type CustomDefaulter = admission.CustomDefaulter
// CustomValidator defines functions for validating an operation.
type CustomValidator = admission.CustomValidator
// AdmissionRequest defines the input for an admission handler.
// It contains information to identify the object in
// question (group, version, kind, resource, subresource,

View File

@@ -26,7 +26,7 @@ import (
"fmt"
"net/http"
apix "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apix "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"

View File

@@ -21,7 +21,6 @@ import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
@@ -34,6 +33,7 @@ import (
kscheme "k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/internal/httpserver"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics"
)
@@ -142,7 +142,7 @@ func (s *Server) Register(path string, hook http.Handler) {
s.WebhookMux.Handle(path, metrics.InstrumentedHook(path, hook))
regLog := log.WithValues("path", path)
regLog.Info("registering webhook")
regLog.Info("Registering webhook")
// we've already been "started", inject dependencies here.
// Otherwise, InjectFunc will do this for us later.
@@ -210,7 +210,7 @@ func (s *Server) Start(ctx context.Context) error {
s.defaultingOnce.Do(s.setDefaults)
baseHookLog := log.WithName("webhooks")
baseHookLog.Info("starting webhook server")
baseHookLog.Info("Starting webhook server")
certPath := filepath.Join(s.CertDir, s.CertName)
keyPath := filepath.Join(s.CertDir, s.KeyName)
@@ -240,9 +240,9 @@ func (s *Server) Start(ctx context.Context) error {
// load CA to verify client certificate
if s.ClientCAName != "" {
certPool := x509.NewCertPool()
clientCABytes, err := ioutil.ReadFile(filepath.Join(s.CertDir, s.ClientCAName))
clientCABytes, err := os.ReadFile(filepath.Join(s.CertDir, s.ClientCAName))
if err != nil {
return fmt.Errorf("failed to read client CA cert: %v", err)
return fmt.Errorf("failed to read client CA cert: %w", err)
}
ok := certPool.AppendCertsFromPEM(clientCABytes)
@@ -259,11 +259,9 @@ func (s *Server) Start(ctx context.Context) error {
return err
}
log.Info("serving webhook server", "host", s.Host, "port", s.Port)
log.Info("Serving webhook server", "host", s.Host, "port", s.Port)
srv := &http.Server{
Handler: s.WebhookMux,
}
srv := httpserver.New(s.WebhookMux)
idleConnsClosed := make(chan struct{})
go func() {
@@ -306,11 +304,11 @@ func (s *Server) StartedChecker() healthz.Checker {
d := &net.Dialer{Timeout: 10 * time.Second}
conn, err := tls.DialWithDialer(d, "tcp", net.JoinHostPort(s.Host, strconv.Itoa(s.Port)), config)
if err != nil {
return fmt.Errorf("webhook server is not reachable: %v", err)
return fmt.Errorf("webhook server is not reachable: %w", err)
}
if err := conn.Close(); err != nil {
return fmt.Errorf("webhook server is not reachable: closing connection: %v", err)
return fmt.Errorf("webhook server is not reachable: closing connection: %w", err)
}
return nil