feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions

View File

@@ -19,11 +19,14 @@ package webhook
import (
"sync"
"k8s.io/api/admissionregistration/v1"
v1 "k8s.io/api/admissionregistration/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apiserver/pkg/admission/plugin/cel"
"k8s.io/apiserver/pkg/admission/plugin/webhook/matchconditions"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/namespace"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/object"
"k8s.io/apiserver/pkg/cel/environment"
webhookutil "k8s.io/apiserver/pkg/util/webhook"
"k8s.io/client-go/rest"
)
@@ -44,6 +47,9 @@ type WebhookAccessor interface {
// GetRESTClient gets the webhook client
GetRESTClient(clientManager *webhookutil.ClientManager) (*rest.RESTClient, error)
// GetCompiledMatcher gets the compiled matcher object
GetCompiledMatcher(compiler cel.FilterCompiler) matchconditions.Matcher
// GetName gets the webhook Name field. Note that the name is scoped to the webhook
// configuration and does not provide a globally unique identity, if a unique identity is
// needed, use GetUID.
@@ -67,10 +73,16 @@ type WebhookAccessor interface {
// GetAdmissionReviewVersions gets the webhook AdmissionReviewVersions field.
GetAdmissionReviewVersions() []string
// GetMatchConditions gets the webhook match conditions field.
GetMatchConditions() []v1.MatchCondition
// GetMutatingWebhook if the accessor contains a MutatingWebhook, returns it and true, else returns false.
GetMutatingWebhook() (*v1.MutatingWebhook, bool)
// GetValidatingWebhook if the accessor contains a ValidatingWebhook, returns it and true, else returns false.
GetValidatingWebhook() (*v1.ValidatingWebhook, bool)
// GetType returns the type of the accessor (validate or admit)
GetType() string
}
// NewMutatingWebhookAccessor creates an accessor for a MutatingWebhook.
@@ -94,6 +106,9 @@ type mutatingWebhookAccessor struct {
initClient sync.Once
client *rest.RESTClient
clientErr error
compileMatcher sync.Once
compiledMatcher matchconditions.Matcher
}
func (m *mutatingWebhookAccessor) GetUID() string {
@@ -111,6 +126,31 @@ func (m *mutatingWebhookAccessor) GetRESTClient(clientManager *webhookutil.Clien
return m.client, m.clientErr
}
func (m *mutatingWebhookAccessor) GetType() string {
return "admit"
}
func (m *mutatingWebhookAccessor) GetCompiledMatcher(compiler cel.FilterCompiler) matchconditions.Matcher {
m.compileMatcher.Do(func() {
expressions := make([]cel.ExpressionAccessor, len(m.MutatingWebhook.MatchConditions))
for i, matchCondition := range m.MutatingWebhook.MatchConditions {
expressions[i] = &matchconditions.MatchCondition{
Name: matchCondition.Name,
Expression: matchCondition.Expression,
}
}
m.compiledMatcher = matchconditions.NewMatcher(compiler.Compile(
expressions,
cel.OptionalVariableDeclarations{
HasParams: false,
HasAuthorizer: true,
},
environment.StoredExpressions,
), m.FailurePolicy, "webhook", "admit", m.Name)
})
return m.compiledMatcher
}
func (m *mutatingWebhookAccessor) GetParsedNamespaceSelector() (labels.Selector, error) {
m.initNamespaceSelector.Do(func() {
m.namespaceSelector, m.namespaceSelectorErr = metav1.LabelSelectorAsSelector(m.NamespaceSelector)
@@ -165,6 +205,10 @@ func (m *mutatingWebhookAccessor) GetAdmissionReviewVersions() []string {
return m.AdmissionReviewVersions
}
func (m *mutatingWebhookAccessor) GetMatchConditions() []v1.MatchCondition {
return m.MatchConditions
}
func (m *mutatingWebhookAccessor) GetMutatingWebhook() (*v1.MutatingWebhook, bool) {
return m.MutatingWebhook, true
}
@@ -194,6 +238,9 @@ type validatingWebhookAccessor struct {
initClient sync.Once
client *rest.RESTClient
clientErr error
compileMatcher sync.Once
compiledMatcher matchconditions.Matcher
}
func (v *validatingWebhookAccessor) GetUID() string {
@@ -211,6 +258,27 @@ func (v *validatingWebhookAccessor) GetRESTClient(clientManager *webhookutil.Cli
return v.client, v.clientErr
}
func (v *validatingWebhookAccessor) GetCompiledMatcher(compiler cel.FilterCompiler) matchconditions.Matcher {
v.compileMatcher.Do(func() {
expressions := make([]cel.ExpressionAccessor, len(v.ValidatingWebhook.MatchConditions))
for i, matchCondition := range v.ValidatingWebhook.MatchConditions {
expressions[i] = &matchconditions.MatchCondition{
Name: matchCondition.Name,
Expression: matchCondition.Expression,
}
}
v.compiledMatcher = matchconditions.NewMatcher(compiler.Compile(
expressions,
cel.OptionalVariableDeclarations{
HasParams: false,
HasAuthorizer: true,
},
environment.StoredExpressions,
), v.FailurePolicy, "webhook", "validating", v.Name)
})
return v.compiledMatcher
}
func (v *validatingWebhookAccessor) GetParsedNamespaceSelector() (labels.Selector, error) {
v.initNamespaceSelector.Do(func() {
v.namespaceSelector, v.namespaceSelectorErr = metav1.LabelSelectorAsSelector(v.NamespaceSelector)
@@ -225,6 +293,10 @@ func (v *validatingWebhookAccessor) GetParsedObjectSelector() (labels.Selector,
return v.objectSelector, v.objectSelectorErr
}
func (m *validatingWebhookAccessor) GetType() string {
return "validate"
}
func (v *validatingWebhookAccessor) GetName() string {
return v.Name
}
@@ -265,6 +337,10 @@ func (v *validatingWebhookAccessor) GetAdmissionReviewVersions() []string {
return v.AdmissionReviewVersions
}
func (v *validatingWebhookAccessor) GetMatchConditions() []v1.MatchCondition {
return v.MatchConditions
}
func (v *validatingWebhookAccessor) GetMutatingWebhook() (*v1.MutatingWebhook, bool) {
return nil, false
}

View File

@@ -19,7 +19,6 @@ package config
import (
"fmt"
"io"
"io/ioutil"
"path"
"k8s.io/apimachinery/pkg/runtime"
@@ -47,7 +46,7 @@ func LoadConfig(configFile io.Reader) (string, error) {
var kubeconfigFile string
if configFile != nil {
// we have a config so parse it.
data, err := ioutil.ReadAll(configFile)
data, err := io.ReadAll(configFile)
if err != nil {
return "", err
}

View File

@@ -1,112 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package generic
import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
)
// ConvertToGVK converts object to the desired gvk.
func ConvertToGVK(obj runtime.Object, gvk schema.GroupVersionKind, o admission.ObjectInterfaces) (runtime.Object, error) {
// Unlike other resources, custom resources do not have internal version, so
// if obj is a custom resource, it should not need conversion.
if obj.GetObjectKind().GroupVersionKind() == gvk {
return obj, nil
}
out, err := o.GetObjectCreater().New(gvk)
if err != nil {
return nil, err
}
err = o.GetObjectConvertor().Convert(obj, out, nil)
if err != nil {
return nil, err
}
// Explicitly set the GVK
out.GetObjectKind().SetGroupVersionKind(gvk)
return out, nil
}
// NewVersionedAttributes returns versioned attributes with the old and new object (if non-nil) converted to the requested kind
func NewVersionedAttributes(attr admission.Attributes, gvk schema.GroupVersionKind, o admission.ObjectInterfaces) (*VersionedAttributes, error) {
// convert the old and new objects to the requested version
versionedAttr := &VersionedAttributes{
Attributes: attr,
VersionedKind: gvk,
}
if oldObj := attr.GetOldObject(); oldObj != nil {
out, err := ConvertToGVK(oldObj, gvk, o)
if err != nil {
return nil, err
}
versionedAttr.VersionedOldObject = out
}
if obj := attr.GetObject(); obj != nil {
out, err := ConvertToGVK(obj, gvk, o)
if err != nil {
return nil, err
}
versionedAttr.VersionedObject = out
}
return versionedAttr, nil
}
// ConvertVersionedAttributes converts VersionedObject and VersionedOldObject to the specified kind, if needed.
// If attr.VersionedKind already matches the requested kind, no conversion is performed.
// If conversion is required:
// * attr.VersionedObject is used as the source for the new object if Dirty=true (and is round-tripped through attr.Attributes.Object, clearing Dirty in the process)
// * attr.Attributes.Object is used as the source for the new object if Dirty=false
// * attr.Attributes.OldObject is used as the source for the old object
func ConvertVersionedAttributes(attr *VersionedAttributes, gvk schema.GroupVersionKind, o admission.ObjectInterfaces) error {
// we already have the desired kind, we're done
if attr.VersionedKind == gvk {
return nil
}
// convert the original old object to the desired GVK
if oldObj := attr.Attributes.GetOldObject(); oldObj != nil {
out, err := ConvertToGVK(oldObj, gvk, o)
if err != nil {
return err
}
attr.VersionedOldObject = out
}
if attr.VersionedObject != nil {
// convert the existing versioned object to internal
if attr.Dirty {
err := o.GetObjectConvertor().Convert(attr.VersionedObject, attr.Attributes.GetObject(), nil)
if err != nil {
return err
}
}
// and back to external
out, err := ConvertToGVK(attr.Attributes.GetObject(), gvk, o)
if err != nil {
return err
}
attr.VersionedObject = out
}
// Remember we converted to this version
attr.VersionedKind = gvk
attr.Dirty = false
return nil
}

View File

@@ -19,43 +19,21 @@ package generic
import (
"context"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/webhook"
)
type VersionedAttributeAccessor interface {
VersionedAttribute(gvk schema.GroupVersionKind) (*admission.VersionedAttributes, error)
}
// Source can list dynamic webhook plugins.
type Source interface {
Webhooks() []webhook.WebhookAccessor
HasSynced() bool
}
// VersionedAttributes is a wrapper around the original admission attributes, adding versioned
// variants of the object and old object.
type VersionedAttributes struct {
// Attributes holds the original admission attributes
admission.Attributes
// VersionedOldObject holds Attributes.OldObject (if non-nil), converted to VersionedKind.
// It must never be mutated.
VersionedOldObject runtime.Object
// VersionedObject holds Attributes.Object (if non-nil), converted to VersionedKind.
// If mutated, Dirty must be set to true by the mutator.
VersionedObject runtime.Object
// VersionedKind holds the fully qualified kind
VersionedKind schema.GroupVersionKind
// Dirty indicates VersionedObject has been modified since being converted from Attributes.Object
Dirty bool
}
// GetObject overrides the Attributes.GetObject()
func (v *VersionedAttributes) GetObject() runtime.Object {
if v.VersionedObject != nil {
return v.VersionedObject
}
return v.Attributes.GetObject()
}
// WebhookInvocation describes how to call a webhook, including the resource and subresource the webhook registered for,
// and the kind that should be sent to the webhook.
type WebhookInvocation struct {

View File

@@ -21,18 +21,24 @@ import (
"fmt"
"io"
admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
"k8s.io/klog/v2"
admissionv1 "k8s.io/api/admission/v1"
admissionv1beta1 "k8s.io/api/admission/v1beta1"
"k8s.io/api/admissionregistration/v1"
v1 "k8s.io/api/admissionregistration/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
genericadmissioninit "k8s.io/apiserver/pkg/admission/initializer"
"k8s.io/apiserver/pkg/admission/plugin/cel"
"k8s.io/apiserver/pkg/admission/plugin/webhook"
"k8s.io/apiserver/pkg/admission/plugin/webhook/config"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/namespace"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/object"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/rules"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/cel/environment"
webhookutil "k8s.io/apiserver/pkg/util/webhook"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
@@ -49,6 +55,8 @@ type Webhook struct {
namespaceMatcher *namespace.Matcher
objectMatcher *object.Matcher
dispatcher Dispatcher
filterCompiler cel.FilterCompiler
authorizer authorizer.Authorizer
}
var (
@@ -92,6 +100,7 @@ func NewWebhook(handler *admission.Handler, configFile io.Reader, sourceFactory
namespaceMatcher: &namespace.Matcher{},
objectMatcher: &object.Matcher{},
dispatcher: dispatcherFactory(&cm),
filterCompiler: cel.NewFilterCompiler(environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion())),
}, nil
}
@@ -124,6 +133,10 @@ func (a *Webhook) SetExternalKubeInformerFactory(f informers.SharedInformerFacto
})
}
func (a *Webhook) SetAuthorizer(authorizer authorizer.Authorizer) {
a.authorizer = authorizer
}
// ValidateInitialization implements the InitializationValidator interface.
func (a *Webhook) ValidateInitialization() error {
if a.hookSource == nil {
@@ -140,7 +153,7 @@ func (a *Webhook) ValidateInitialization() error {
// ShouldCallHook returns invocation details if the webhook should be called, nil if the webhook should not be called,
// or an error if an error was encountered during evaluation.
func (a *Webhook) ShouldCallHook(h webhook.WebhookAccessor, attr admission.Attributes, o admission.ObjectInterfaces) (*WebhookInvocation, *apierrors.StatusError) {
func (a *Webhook) ShouldCallHook(ctx context.Context, h webhook.WebhookAccessor, attr admission.Attributes, o admission.ObjectInterfaces, v VersionedAttributeAccessor) (*WebhookInvocation, *apierrors.StatusError) {
matches, matchNsErr := a.namespaceMatcher.MatchNamespaceSelector(h, attr)
// Should not return an error here for webhooks which do not apply to the request, even if err is an unexpected scenario.
if !matches && matchNsErr == nil {
@@ -206,6 +219,25 @@ func (a *Webhook) ShouldCallHook(h webhook.WebhookAccessor, attr admission.Attri
if matchObjErr != nil {
return nil, matchObjErr
}
matchConditions := h.GetMatchConditions()
if len(matchConditions) > 0 {
versionedAttr, err := v.VersionedAttribute(invocation.Kind)
if err != nil {
return nil, apierrors.NewInternalError(err)
}
matcher := h.GetCompiledMatcher(a.filterCompiler)
matchResult := matcher.Match(ctx, versionedAttr, nil, a.authorizer)
if matchResult.Error != nil {
klog.Warningf("Failed evaluating match conditions, failing closed %v: %v", h.GetName(), matchResult.Error)
return nil, apierrors.NewForbidden(attr.GetResource().GroupResource(), attr.GetName(), matchResult.Error)
} else if !matchResult.Matches {
admissionmetrics.Metrics.ObserveMatchConditionExclusion(ctx, h.GetName(), "webhook", h.GetType(), string(attr.GetOperation()))
// if no match, always skip webhook
return nil, nil
}
}
return invocation, nil
}

View File

@@ -0,0 +1,37 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package matchconditions
import (
"context"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authorization/authorizer"
)
type MatchResult struct {
Matches bool
Error error
FailedConditionName string
}
// Matcher contains logic for converting Evaluations to bool of matches or does not match
type Matcher interface {
// Match is used to take cel evaluations and convert into decisions
Match(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, authz authorizer.Authorizer) MatchResult
}

View File

@@ -0,0 +1,144 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package matchconditions
import (
"context"
"errors"
"fmt"
"time"
"github.com/google/cel-go/cel"
celtypes "github.com/google/cel-go/common/types"
v1 "k8s.io/api/admissionregistration/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apiserver/pkg/admission"
admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
celplugin "k8s.io/apiserver/pkg/admission/plugin/cel"
celconfig "k8s.io/apiserver/pkg/apis/cel"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/klog/v2"
)
var _ celplugin.ExpressionAccessor = &MatchCondition{}
// MatchCondition contains the inputs needed to compile, evaluate and match a cel expression
type MatchCondition v1.MatchCondition
func (v *MatchCondition) GetExpression() string {
return v.Expression
}
func (v *MatchCondition) ReturnTypes() []*cel.Type {
return []*cel.Type{cel.BoolType}
}
var _ Matcher = &matcher{}
// matcher evaluates compiled cel expressions and determines if they match the given request or not
type matcher struct {
filter celplugin.Filter
failPolicy v1.FailurePolicyType
matcherType string
matcherKind string
objectName string
}
func NewMatcher(filter celplugin.Filter, failPolicy *v1.FailurePolicyType, matcherKind, matcherType, objectName string) Matcher {
var f v1.FailurePolicyType
if failPolicy == nil {
f = v1.Fail
} else {
f = *failPolicy
}
return &matcher{
filter: filter,
failPolicy: f,
matcherKind: matcherKind,
matcherType: matcherType,
objectName: objectName,
}
}
func (m *matcher) Match(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, authz authorizer.Authorizer) MatchResult {
t := time.Now()
evalResults, _, err := m.filter.ForInput(ctx, versionedAttr, celplugin.CreateAdmissionRequest(versionedAttr.Attributes, metav1.GroupVersionResource(versionedAttr.GetResource()), metav1.GroupVersionKind(versionedAttr.VersionedKind)), celplugin.OptionalVariableBindings{
VersionedParams: versionedParams,
Authorizer: authz,
}, nil, celconfig.RuntimeCELCostBudgetMatchConditions)
if err != nil {
admissionmetrics.Metrics.ObserveMatchConditionEvaluationTime(ctx, time.Since(t), m.objectName, m.matcherKind, m.matcherType, string(versionedAttr.GetOperation()))
// filter returning error is unexpected and not an evaluation error so not incrementing metric here
if m.failPolicy == v1.Fail {
return MatchResult{
Error: err,
}
} else if m.failPolicy == v1.Ignore {
return MatchResult{
Matches: false,
}
}
//TODO: add default so that if in future we add different failure types it doesn't fall through
}
errorList := []error{}
for _, evalResult := range evalResults {
matchCondition, ok := evalResult.ExpressionAccessor.(*MatchCondition)
if !ok {
// This shouldnt happen, but if it does treat same as eval error
klog.Error("Invalid type conversion to MatchCondition")
errorList = append(errorList, errors.New(fmt.Sprintf("internal error converting ExpressionAccessor to MatchCondition")))
continue
}
if evalResult.Error != nil {
errorList = append(errorList, evalResult.Error)
admissionmetrics.Metrics.ObserveMatchConditionEvalError(ctx, m.objectName, m.matcherKind, m.matcherType, string(versionedAttr.GetOperation()))
}
if evalResult.EvalResult == celtypes.False {
admissionmetrics.Metrics.ObserveMatchConditionEvaluationTime(ctx, time.Since(t), m.objectName, m.matcherKind, m.matcherType, string(versionedAttr.GetOperation()))
// If any condition false, skip calling webhook always
return MatchResult{
Matches: false,
FailedConditionName: matchCondition.Name,
}
}
}
if len(errorList) > 0 {
admissionmetrics.Metrics.ObserveMatchConditionEvaluationTime(ctx, time.Since(t), m.objectName, m.matcherKind, m.matcherType, string(versionedAttr.GetOperation()))
// If mix of true and eval errors then resort to fail policy
if m.failPolicy == v1.Fail {
// mix of true and errors with fail policy fail should fail request without calling webhook
err = utilerrors.NewAggregate(errorList)
return MatchResult{
Error: err,
}
} else if m.failPolicy == v1.Ignore {
// if fail policy ignore then skip call to webhook
return MatchResult{
Matches: false,
}
}
}
// if no results eval to false, return matches true with list of any errors encountered
return MatchResult{
Matches: true,
}
}

View File

@@ -20,20 +20,20 @@ package mutating
import (
"context"
"errors"
"fmt"
"time"
jsonpatch "github.com/evanphx/json-patch"
"go.opentelemetry.io/otel/attribute"
apiequality "k8s.io/apimachinery/pkg/api/equality"
"k8s.io/klog/v2"
admissionv1 "k8s.io/api/admission/v1"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
utiljson "k8s.io/apimachinery/pkg/util/json"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -48,6 +48,7 @@ import (
webhookutil "k8s.io/apiserver/pkg/util/webhook"
"k8s.io/apiserver/pkg/warning"
"k8s.io/component-base/tracing"
"k8s.io/klog/v2"
)
const (
@@ -75,6 +76,30 @@ func newMutatingDispatcher(p *Plugin) func(cm *webhookutil.ClientManager) generi
}
}
var _ generic.VersionedAttributeAccessor = &versionedAttributeAccessor{}
type versionedAttributeAccessor struct {
versionedAttr *admission.VersionedAttributes
attr admission.Attributes
objectInterfaces admission.ObjectInterfaces
}
func (v *versionedAttributeAccessor) VersionedAttribute(gvk schema.GroupVersionKind) (*admission.VersionedAttributes, error) {
if v.versionedAttr == nil {
// First call, create versioned attributes
var err error
if v.versionedAttr, err = admission.NewVersionedAttributes(v.attr, gvk, v.objectInterfaces); err != nil {
return nil, apierrors.NewInternalError(err)
}
} else {
// Subsequent call, convert existing versioned attributes to the requested version
if err := admission.ConvertVersionedAttributes(v.versionedAttr, gvk, v.objectInterfaces); err != nil {
return nil, apierrors.NewInternalError(err)
}
}
return v.versionedAttr, nil
}
var _ generic.Dispatcher = &mutatingDispatcher{}
func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces, hooks []webhook.WebhookAccessor) error {
@@ -95,19 +120,24 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
defer func() {
webhookReinvokeCtx.SetLastWebhookInvocationOutput(attr.GetObject())
}()
var versionedAttr *generic.VersionedAttributes
v := &versionedAttributeAccessor{
attr: attr,
objectInterfaces: o,
}
for i, hook := range hooks {
attrForCheck := attr
if versionedAttr != nil {
attrForCheck = versionedAttr
if v.versionedAttr != nil {
attrForCheck = v.versionedAttr
}
invocation, statusErr := a.plugin.ShouldCallHook(hook, attrForCheck, o)
invocation, statusErr := a.plugin.ShouldCallHook(ctx, hook, attrForCheck, o, v)
if statusErr != nil {
return statusErr
}
if invocation == nil {
continue
}
hook, ok := invocation.Webhook.GetMutatingWebhook()
if !ok {
return fmt.Errorf("mutating webhook dispatch requires v1.MutatingWebhook, but got %T", hook)
@@ -121,17 +151,9 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
continue
}
if versionedAttr == nil {
// First webhook, create versioned attributes
var err error
if versionedAttr, err = generic.NewVersionedAttributes(attr, invocation.Kind, o); err != nil {
return apierrors.NewInternalError(err)
}
} else {
// Subsequent webhook, convert existing versioned attributes to this webhook's version
if err := generic.ConvertVersionedAttributes(versionedAttr, invocation.Kind, o); err != nil {
return apierrors.NewInternalError(err)
}
versionedAttr, err := v.VersionedAttribute(invocation.Kind)
if err != nil {
return apierrors.NewInternalError(err)
}
t := time.Now()
@@ -149,7 +171,10 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
case *webhookutil.ErrCallingWebhook:
if !ignoreClientCallFailures {
rejected = true
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, int(err.Status.ErrStatus.Code))
// Ignore context cancelled from webhook metrics
if !errors.Is(err.Reason, context.Canceled) {
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, int(err.Status.ErrStatus.Code))
}
}
admissionmetrics.Metrics.ObserveWebhook(ctx, hook.Name, time.Since(t), rejected, versionedAttr.Attributes, "admit", int(err.Status.ErrStatus.Code))
case *webhookutil.ErrWebhookRejection:
@@ -178,10 +203,14 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
if ignoreClientCallFailures {
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "admit")
annotator.addFailedOpenAnnotation()
// Ignore context cancelled from webhook metrics
if errors.Is(callErr.Reason, context.Canceled) {
klog.Warningf("Context canceled when calling webhook %v", hook.Name)
} else {
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "admit")
annotator.addFailedOpenAnnotation()
}
utilruntime.HandleError(callErr)
select {
@@ -203,8 +232,8 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
}
// convert versionedAttr.VersionedObject to the internal version in the underlying admission.Attributes
if versionedAttr != nil && versionedAttr.VersionedObject != nil && versionedAttr.Dirty {
return o.GetObjectConvertor().Convert(versionedAttr.VersionedObject, versionedAttr.Attributes.GetObject(), nil)
if v.versionedAttr != nil && v.versionedAttr.VersionedObject != nil && v.versionedAttr.Dirty {
return o.GetObjectConvertor().Convert(v.versionedAttr.VersionedObject, v.versionedAttr.Attributes.GetObject(), nil)
}
return nil
@@ -212,7 +241,7 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
// note that callAttrMutatingHook updates attr
func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *admissionregistrationv1.MutatingWebhook, invocation *generic.WebhookInvocation, attr *generic.VersionedAttributes, annotator *webhookAnnotator, o admission.ObjectInterfaces, round, idx int) (bool, error) {
func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *admissionregistrationv1.MutatingWebhook, invocation *generic.WebhookInvocation, attr *admission.VersionedAttributes, annotator *webhookAnnotator, o admission.ObjectInterfaces, round, idx int) (bool, error) {
configurationName := invocation.Webhook.GetConfigurationName()
changed := false
defer func() { annotator.addMutationAnnotation(changed) }()
@@ -363,7 +392,7 @@ func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *admiss
}
type webhookAnnotator struct {
attr *generic.VersionedAttributes
attr *admission.VersionedAttributes
failedOpenAnnotationKey string
patchAnnotationKey string
mutationAnnotationKey string
@@ -371,7 +400,7 @@ type webhookAnnotator struct {
configuration string
}
func newWebhookAnnotator(attr *generic.VersionedAttributes, round, idx int, webhook, configuration string) *webhookAnnotator {
func newWebhookAnnotator(attr *admission.VersionedAttributes, round, idx int, webhook, configuration string) *webhookAnnotator {
return &webhookAnnotator{
attr: attr,
failedOpenAnnotationKey: fmt.Sprintf("%sround_%d_index_%d", MutationAuditAnnotationFailedOpenKeyPrefix, round, idx),

View File

@@ -20,6 +20,8 @@ import (
"context"
"fmt"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -42,6 +44,10 @@ type Matcher struct {
Client clientset.Interface
}
func (m *Matcher) GetNamespace(name string) (*v1.Namespace, error) {
return m.NamespaceLister.Get(name)
}
// Validate checks if the Matcher has a NamespaceLister and Client.
func (m *Matcher) Validate() error {
var errs []error
@@ -116,7 +122,7 @@ func (m *Matcher) MatchNamespaceSelector(p NamespaceSelectorProvider, attr admis
if !ok {
return false, apierrors.NewInternalError(err)
}
return false, &apierrors.StatusError{status.Status()}
return false, &apierrors.StatusError{ErrStatus: status.Status()}
}
if err != nil {
return false, apierrors.NewInternalError(err)

View File

@@ -26,6 +26,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/webhook/generic"
)
@@ -130,7 +131,7 @@ func VerifyAdmissionResponse(uid types.UID, mutating bool, review runtime.Object
// CreateAdmissionObjects returns the unique request uid, the AdmissionReview object to send the webhook and to decode the response into,
// or an error if the webhook does not support receiving any of the admission review versions we know to send
func CreateAdmissionObjects(versionedAttributes *generic.VersionedAttributes, invocation *generic.WebhookInvocation) (uid types.UID, request, response runtime.Object, err error) {
func CreateAdmissionObjects(versionedAttributes *admission.VersionedAttributes, invocation *generic.WebhookInvocation) (uid types.UID, request, response runtime.Object, err error) {
for _, version := range invocation.Webhook.GetAdmissionReviewVersions() {
switch version {
case admissionv1.SchemeGroupVersion.Version:
@@ -151,7 +152,7 @@ func CreateAdmissionObjects(versionedAttributes *generic.VersionedAttributes, in
}
// CreateV1AdmissionReview creates an AdmissionReview for the provided admission.Attributes
func CreateV1AdmissionReview(uid types.UID, versionedAttributes *generic.VersionedAttributes, invocation *generic.WebhookInvocation) *admissionv1.AdmissionReview {
func CreateV1AdmissionReview(uid types.UID, versionedAttributes *admission.VersionedAttributes, invocation *generic.WebhookInvocation) *admissionv1.AdmissionReview {
attr := versionedAttributes.Attributes
gvk := invocation.Kind
gvr := invocation.Resource
@@ -217,7 +218,7 @@ func CreateV1AdmissionReview(uid types.UID, versionedAttributes *generic.Version
}
// CreateV1beta1AdmissionReview creates an AdmissionReview for the provided admission.Attributes
func CreateV1beta1AdmissionReview(uid types.UID, versionedAttributes *generic.VersionedAttributes, invocation *generic.WebhookInvocation) *admissionv1beta1.AdmissionReview {
func CreateV1beta1AdmissionReview(uid types.UID, versionedAttributes *admission.VersionedAttributes, invocation *generic.WebhookInvocation) *admissionv1beta1.AdmissionReview {
attr := versionedAttributes.Attributes
gvk := invocation.Kind
gvr := invocation.Resource

View File

@@ -18,6 +18,7 @@ package validating
import (
"context"
"errors"
"fmt"
"sync"
"time"
@@ -62,30 +63,51 @@ func newValidatingDispatcher(p *Plugin) func(cm *webhookutil.ClientManager) gene
}
}
var _ generic.VersionedAttributeAccessor = &versionedAttributeAccessor{}
type versionedAttributeAccessor struct {
versionedAttrs map[schema.GroupVersionKind]*admission.VersionedAttributes
attr admission.Attributes
objectInterfaces admission.ObjectInterfaces
}
func (v *versionedAttributeAccessor) VersionedAttribute(gvk schema.GroupVersionKind) (*admission.VersionedAttributes, error) {
if val, ok := v.versionedAttrs[gvk]; ok {
return val, nil
}
versionedAttr, err := admission.NewVersionedAttributes(v.attr, gvk, v.objectInterfaces)
if err != nil {
return nil, err
}
v.versionedAttrs[gvk] = versionedAttr
return versionedAttr, nil
}
var _ generic.Dispatcher = &validatingDispatcher{}
func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces, hooks []webhook.WebhookAccessor) error {
var relevantHooks []*generic.WebhookInvocation
// Construct all the versions we need to call our webhooks
versionedAttrs := map[schema.GroupVersionKind]*generic.VersionedAttributes{}
versionedAttrAccessor := &versionedAttributeAccessor{
versionedAttrs: map[schema.GroupVersionKind]*admission.VersionedAttributes{},
attr: attr,
objectInterfaces: o,
}
for _, hook := range hooks {
invocation, statusError := d.plugin.ShouldCallHook(hook, attr, o)
invocation, statusError := d.plugin.ShouldCallHook(ctx, hook, attr, o, versionedAttrAccessor)
if statusError != nil {
return statusError
}
if invocation == nil {
continue
}
relevantHooks = append(relevantHooks, invocation)
// If we already have this version, continue
if _, ok := versionedAttrs[invocation.Kind]; ok {
continue
}
versionedAttr, err := generic.NewVersionedAttributes(attr, invocation.Kind, o)
// VersionedAttr result will be cached and reused later during parallel webhook calls
_, err := versionedAttrAccessor.VersionedAttribute(invocation.Kind)
if err != nil {
return apierrors.NewInternalError(err)
}
versionedAttrs[invocation.Kind] = versionedAttr
}
if len(relevantHooks) == 0 {
@@ -108,7 +130,7 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
go func(invocation *generic.WebhookInvocation, idx int) {
ignoreClientCallFailures := false
hookName := "unknown"
versionedAttr := versionedAttrs[invocation.Kind]
versionedAttr := versionedAttrAccessor.versionedAttrs[invocation.Kind]
// The ordering of these two defers is critical. The wg.Done will release the parent go func to close the errCh
// that is used by the second defer to report errors. The recovery and error reporting must be done first.
defer wg.Done()
@@ -154,7 +176,10 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
case *webhookutil.ErrCallingWebhook:
if !ignoreClientCallFailures {
rejected = true
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "validating", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, int(err.Status.ErrStatus.Code))
// Ignore context cancelled from webhook metrics
if !errors.Is(err.Reason, context.Canceled) {
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "validating", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, int(err.Status.ErrStatus.Code))
}
}
admissionmetrics.Metrics.ObserveWebhook(ctx, hook.Name, time.Since(t), rejected, versionedAttr.Attributes, "validating", int(err.Status.ErrStatus.Code))
case *webhookutil.ErrWebhookRejection:
@@ -173,12 +198,17 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
if ignoreClientCallFailures {
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "validating")
key := fmt.Sprintf("%sround_0_index_%d", ValidatingAuditAnnotationFailedOpenKeyPrefix, idx)
value := hook.Name
if err := versionedAttr.Attributes.AddAnnotation(key, value); err != nil {
klog.Warningf("Failed to set admission audit annotation %s to %s for validating webhook %s: %v", key, value, hook.Name, err)
// Ignore context cancelled from webhook metrics
if errors.Is(callErr.Reason, context.Canceled) {
klog.Warningf("Context canceled when calling webhook %v", hook.Name)
} else {
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "validating")
key := fmt.Sprintf("%sround_0_index_%d", ValidatingAuditAnnotationFailedOpenKeyPrefix, idx)
value := hook.Name
if err := versionedAttr.Attributes.AddAnnotation(key, value); err != nil {
klog.Warningf("Failed to set admission audit annotation %s to %s for validating webhook %s: %v", key, value, hook.Name, err)
}
}
utilruntime.HandleError(callErr)
return
@@ -215,7 +245,7 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
return errs[0]
}
func (d *validatingDispatcher) callHook(ctx context.Context, h *v1.ValidatingWebhook, invocation *generic.WebhookInvocation, attr *generic.VersionedAttributes) error {
func (d *validatingDispatcher) callHook(ctx context.Context, h *v1.ValidatingWebhook, invocation *generic.WebhookInvocation, attr *admission.VersionedAttributes) error {
if attr.Attributes.IsDryRun() {
if h.SideEffects == nil {
return &webhookutil.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("Webhook SideEffects is nil"), Status: apierrors.NewBadRequest("Webhook SideEffects is nil")}