feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions

View File

@@ -109,3 +109,15 @@ func (m *ValidatingAdmissionPolicyMetrics) ObserveRejection(ctx context.Context,
m.policyCheck.WithContext(ctx).WithLabelValues(policy, binding, "deny", state).Inc()
m.policyLatency.WithContext(ctx).WithLabelValues(policy, binding, "deny", state).Observe(elapsed.Seconds())
}
// ObserveAudit observes a policy validation audit annotation was published for a validation failure.
func (m *ValidatingAdmissionPolicyMetrics) ObserveAudit(ctx context.Context, elapsed time.Duration, policy, binding, state string) {
m.policyCheck.WithContext(ctx).WithLabelValues(policy, binding, "audit", state).Inc()
m.policyLatency.WithContext(ctx).WithLabelValues(policy, binding, "audit", state).Observe(elapsed.Seconds())
}
// ObserveWarn observes a policy validation warning was published for a validation failure.
func (m *ValidatingAdmissionPolicyMetrics) ObserveWarn(ctx context.Context, elapsed time.Duration, policy, binding, state string) {
m.policyCheck.WithContext(ctx).WithLabelValues(policy, binding, "warn", state).Inc()
m.policyLatency.WithContext(ctx).WithLabelValues(policy, binding, "warn", state).Observe(elapsed.Seconds())
}

View File

@@ -20,7 +20,6 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
@@ -60,7 +59,7 @@ func ReadAdmissionConfiguration(pluginNames []string, configFilePath string, con
return configProvider{config: &apiserver.AdmissionConfiguration{}}, nil
}
// a file was provided, so we just read it.
data, err := ioutil.ReadFile(configFilePath)
data, err := os.ReadFile(configFilePath)
if err != nil {
return nil, fmt.Errorf("unable to read admission control configuration from %q [%v]", configFilePath, err)
}
@@ -141,7 +140,7 @@ func GetAdmissionPluginConfigurationFor(pluginCfg apiserver.AdmissionPluginConfi
}
// there is nothing nested, so we delegate to path
if pluginCfg.Path != "" {
content, err := ioutil.ReadFile(pluginCfg.Path)
content, err := os.ReadFile(pluginCfg.Path)
if err != nil {
klog.Fatalf("Couldn't open admission plugin configuration %s: %#v", pluginCfg.Path, err)
return nil, err

View File

@@ -19,9 +19,9 @@ package configuration
import (
"fmt"
"sort"
"sync/atomic"
"sync"
"k8s.io/api/admissionregistration/v1"
v1 "k8s.io/api/admissionregistration/v1"
"k8s.io/apimachinery/pkg/labels"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/admission/plugin/webhook"
@@ -29,18 +29,23 @@ import (
"k8s.io/client-go/informers"
admissionregistrationlisters "k8s.io/client-go/listers/admissionregistration/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/cache/synctrack"
"k8s.io/klog/v2"
)
// Type for test injection.
type mutatingWebhookAccessorCreator func(uid string, configurationName string, h *v1.MutatingWebhook) webhook.WebhookAccessor
// mutatingWebhookConfigurationManager collects the mutating webhook objects so that they can be called.
type mutatingWebhookConfigurationManager struct {
configuration *atomic.Value
lister admissionregistrationlisters.MutatingWebhookConfigurationLister
hasSynced func() bool
// initialConfigurationSynced tracks if
// the existing webhook configs have been synced (honored) by the
// manager at startup-- the informer has synced and either has no items
// or has finished executing updateConfiguration() once.
initialConfigurationSynced *atomic.Bool
lister admissionregistrationlisters.MutatingWebhookConfigurationLister
hasSynced func() bool
lazy synctrack.Lazy[[]webhook.WebhookAccessor]
configurationsCache sync.Map
// createMutatingWebhookAccessor is used to instantiate webhook accessors.
// This function is defined as field instead of a struct method to allow injection
// during tests
createMutatingWebhookAccessor mutatingWebhookAccessorCreator
}
var _ generic.Source = &mutatingWebhookConfigurationManager{}
@@ -48,80 +53,99 @@ var _ generic.Source = &mutatingWebhookConfigurationManager{}
func NewMutatingWebhookConfigurationManager(f informers.SharedInformerFactory) generic.Source {
informer := f.Admissionregistration().V1().MutatingWebhookConfigurations()
manager := &mutatingWebhookConfigurationManager{
configuration: &atomic.Value{},
lister: informer.Lister(),
hasSynced: informer.Informer().HasSynced,
initialConfigurationSynced: &atomic.Bool{},
lister: informer.Lister(),
createMutatingWebhookAccessor: webhook.NewMutatingWebhookAccessor,
}
manager.lazy.Evaluate = manager.getConfiguration
// Start with an empty list
manager.configuration.Store([]webhook.WebhookAccessor{})
manager.initialConfigurationSynced.Store(false)
// On any change, rebuild the config
informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(_ interface{}) { manager.updateConfiguration() },
UpdateFunc: func(_, _ interface{}) { manager.updateConfiguration() },
DeleteFunc: func(_ interface{}) { manager.updateConfiguration() },
handle, _ := informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(_ interface{}) { manager.lazy.Notify() },
UpdateFunc: func(old, new interface{}) {
obj := new.(*v1.MutatingWebhookConfiguration)
manager.configurationsCache.Delete(obj.GetName())
manager.lazy.Notify()
},
DeleteFunc: func(obj interface{}) {
vwc, ok := obj.(*v1.MutatingWebhookConfiguration)
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
if !ok {
klog.V(2).Infof("Couldn't get object from tombstone %#v", obj)
return
}
vwc, ok = tombstone.Obj.(*v1.MutatingWebhookConfiguration)
if !ok {
klog.V(2).Infof("Tombstone contained object that is not expected %#v", obj)
return
}
}
manager.configurationsCache.Delete(vwc.Name)
manager.lazy.Notify()
},
})
manager.hasSynced = handle.HasSynced
return manager
}
// Webhooks returns the merged MutatingWebhookConfiguration.
func (m *mutatingWebhookConfigurationManager) Webhooks() []webhook.WebhookAccessor {
return m.configuration.Load().([]webhook.WebhookAccessor)
out, err := m.lazy.Get()
if err != nil {
utilruntime.HandleError(fmt.Errorf("error getting webhook configuration: %v", err))
}
return out
}
// HasSynced returns true when the manager is synced with existing webhookconfig
// objects at startup-- which means the informer is synced and either has no items
// or updateConfiguration() has completed.
func (m *mutatingWebhookConfigurationManager) HasSynced() bool {
if !m.hasSynced() {
return false
}
if m.initialConfigurationSynced.Load() {
// the informer has synced and configuration has been updated
return true
}
if configurations, err := m.lister.List(labels.Everything()); err == nil && len(configurations) == 0 {
// the empty list we initially stored is valid to use.
// Setting initialConfigurationSynced to true, so subsequent checks
// would be able to take the fast path on the atomic boolean in a
// cluster without any admission webhooks configured.
m.initialConfigurationSynced.Store(true)
// the informer has synced and we don't have any items
return true
}
return false
}
// HasSynced returns true if the initial set of mutating webhook configurations
// has been loaded.
func (m *mutatingWebhookConfigurationManager) HasSynced() bool { return m.hasSynced() }
func (m *mutatingWebhookConfigurationManager) updateConfiguration() {
func (m *mutatingWebhookConfigurationManager) getConfiguration() ([]webhook.WebhookAccessor, error) {
configurations, err := m.lister.List(labels.Everything())
if err != nil {
utilruntime.HandleError(fmt.Errorf("error updating configuration: %v", err))
return
return []webhook.WebhookAccessor{}, err
}
m.configuration.Store(mergeMutatingWebhookConfigurations(configurations))
m.initialConfigurationSynced.Store(true)
return m.getMutatingWebhookConfigurations(configurations), nil
}
func mergeMutatingWebhookConfigurations(configurations []*v1.MutatingWebhookConfiguration) []webhook.WebhookAccessor {
// getMutatingWebhookConfigurations returns the webhook accessors for a given list of
// mutating webhook configurations.
//
// This function will, first, try to load the webhook accessors from the cache and avoid
// recreating them, which can be expessive (requiring CEL expression recompilation).
func (m *mutatingWebhookConfigurationManager) getMutatingWebhookConfigurations(configurations []*v1.MutatingWebhookConfiguration) []webhook.WebhookAccessor {
// The internal order of webhooks for each configuration is provided by the user
// but configurations themselves can be in any order. As we are going to run these
// webhooks in serial, they are sorted here to have a deterministic order.
sort.SliceStable(configurations, MutatingWebhookConfigurationSorter(configurations).ByName)
accessors := []webhook.WebhookAccessor{}
size := 0
for _, cfg := range configurations {
size += len(cfg.Webhooks)
}
accessors := make([]webhook.WebhookAccessor, 0, size)
for _, c := range configurations {
cachedConfigurationAccessors, ok := m.configurationsCache.Load(c.Name)
if ok {
// Pick an already cached webhookAccessor
accessors = append(accessors, cachedConfigurationAccessors.([]webhook.WebhookAccessor)...)
continue
}
// webhook names are not validated for uniqueness, so we check for duplicates and
// add a int suffix to distinguish between them
names := map[string]int{}
configurationAccessors := make([]webhook.WebhookAccessor, 0, len(c.Webhooks))
for i := range c.Webhooks {
n := c.Webhooks[i].Name
uid := fmt.Sprintf("%s/%s/%d", c.Name, n, names[n])
names[n]++
accessors = append(accessors, webhook.NewMutatingWebhookAccessor(uid, c.Name, &c.Webhooks[i]))
configurationAccessor := m.createMutatingWebhookAccessor(uid, c.Name, &c.Webhooks[i])
configurationAccessors = append(configurationAccessors, configurationAccessor)
}
accessors = append(accessors, configurationAccessors...)
m.configurationsCache.Store(c.Name, configurationAccessors)
}
return accessors
}

View File

@@ -19,9 +19,9 @@ package configuration
import (
"fmt"
"sort"
"sync/atomic"
"sync"
"k8s.io/api/admissionregistration/v1"
v1 "k8s.io/api/admissionregistration/v1"
"k8s.io/apimachinery/pkg/labels"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/admission/plugin/webhook"
@@ -29,18 +29,23 @@ import (
"k8s.io/client-go/informers"
admissionregistrationlisters "k8s.io/client-go/listers/admissionregistration/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/cache/synctrack"
"k8s.io/klog/v2"
)
// Type for test injection.
type validatingWebhookAccessorCreator func(uid string, configurationName string, h *v1.ValidatingWebhook) webhook.WebhookAccessor
// validatingWebhookConfigurationManager collects the validating webhook objects so that they can be called.
type validatingWebhookConfigurationManager struct {
configuration *atomic.Value
lister admissionregistrationlisters.ValidatingWebhookConfigurationLister
hasSynced func() bool
// initialConfigurationSynced tracks if
// the existing webhook configs have been synced (honored) by the
// manager at startup-- the informer has synced and either has no items
// or has finished executing updateConfiguration() once.
initialConfigurationSynced *atomic.Bool
lister admissionregistrationlisters.ValidatingWebhookConfigurationLister
hasSynced func() bool
lazy synctrack.Lazy[[]webhook.WebhookAccessor]
configurationsCache sync.Map
// createValidatingWebhookAccessor is used to instantiate webhook accessors.
// This function is defined as field instead of a struct method to allow injection
// during tests
createValidatingWebhookAccessor validatingWebhookAccessorCreator
}
var _ generic.Source = &validatingWebhookConfigurationManager{}
@@ -48,79 +53,98 @@ var _ generic.Source = &validatingWebhookConfigurationManager{}
func NewValidatingWebhookConfigurationManager(f informers.SharedInformerFactory) generic.Source {
informer := f.Admissionregistration().V1().ValidatingWebhookConfigurations()
manager := &validatingWebhookConfigurationManager{
configuration: &atomic.Value{},
lister: informer.Lister(),
hasSynced: informer.Informer().HasSynced,
initialConfigurationSynced: &atomic.Bool{},
lister: informer.Lister(),
createValidatingWebhookAccessor: webhook.NewValidatingWebhookAccessor,
}
manager.lazy.Evaluate = manager.getConfiguration
// Start with an empty list
manager.configuration.Store([]webhook.WebhookAccessor{})
manager.initialConfigurationSynced.Store(false)
// On any change, rebuild the config
informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(_ interface{}) { manager.updateConfiguration() },
UpdateFunc: func(_, _ interface{}) { manager.updateConfiguration() },
DeleteFunc: func(_ interface{}) { manager.updateConfiguration() },
handle, _ := informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(_ interface{}) { manager.lazy.Notify() },
UpdateFunc: func(old, new interface{}) {
obj := new.(*v1.ValidatingWebhookConfiguration)
manager.configurationsCache.Delete(obj.GetName())
manager.lazy.Notify()
},
DeleteFunc: func(obj interface{}) {
vwc, ok := obj.(*v1.ValidatingWebhookConfiguration)
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
if !ok {
klog.V(2).Infof("Couldn't get object from tombstone %#v", obj)
return
}
vwc, ok = tombstone.Obj.(*v1.ValidatingWebhookConfiguration)
if !ok {
klog.V(2).Infof("Tombstone contained object that is not expected %#v", obj)
return
}
}
manager.configurationsCache.Delete(vwc.Name)
manager.lazy.Notify()
},
})
manager.hasSynced = handle.HasSynced
return manager
}
// Webhooks returns the merged ValidatingWebhookConfiguration.
func (v *validatingWebhookConfigurationManager) Webhooks() []webhook.WebhookAccessor {
return v.configuration.Load().([]webhook.WebhookAccessor)
out, err := v.lazy.Get()
if err != nil {
utilruntime.HandleError(fmt.Errorf("error getting webhook configuration: %v", err))
}
return out
}
// HasSynced returns true when the manager is synced with existing webhookconfig
// objects at startup-- which means the informer is synced and either has no items
// or updateConfiguration() has completed.
func (v *validatingWebhookConfigurationManager) HasSynced() bool {
if !v.hasSynced() {
return false
}
if v.initialConfigurationSynced.Load() {
// the informer has synced and configuration has been updated
return true
}
if configurations, err := v.lister.List(labels.Everything()); err == nil && len(configurations) == 0 {
// the empty list we initially stored is valid to use.
// Setting initialConfigurationSynced to true, so subsequent checks
// would be able to take the fast path on the atomic boolean in a
// cluster without any admission webhooks configured.
v.initialConfigurationSynced.Store(true)
// the informer has synced and we don't have any items
return true
}
return false
// HasSynced returns true if the initial set of validating webhook configurations
// has been loaded.
func (v *validatingWebhookConfigurationManager) HasSynced() bool { return v.hasSynced() }
}
func (v *validatingWebhookConfigurationManager) updateConfiguration() {
func (v *validatingWebhookConfigurationManager) getConfiguration() ([]webhook.WebhookAccessor, error) {
configurations, err := v.lister.List(labels.Everything())
if err != nil {
utilruntime.HandleError(fmt.Errorf("error updating configuration: %v", err))
return
return []webhook.WebhookAccessor{}, err
}
v.configuration.Store(mergeValidatingWebhookConfigurations(configurations))
v.initialConfigurationSynced.Store(true)
return v.getValidatingWebhookConfigurations(configurations), nil
}
func mergeValidatingWebhookConfigurations(configurations []*v1.ValidatingWebhookConfiguration) []webhook.WebhookAccessor {
// getMutatingWebhookConfigurations returns the webhook accessors for a given list of
// mutating webhook configurations.
//
// This function will, first, try to load the webhook accessors from the cache and avoid
// recreating them, which can be expessive (requiring CEL expression recompilation).
func (v *validatingWebhookConfigurationManager) getValidatingWebhookConfigurations(configurations []*v1.ValidatingWebhookConfiguration) []webhook.WebhookAccessor {
sort.SliceStable(configurations, ValidatingWebhookConfigurationSorter(configurations).ByName)
accessors := []webhook.WebhookAccessor{}
size := 0
for _, cfg := range configurations {
size += len(cfg.Webhooks)
}
accessors := make([]webhook.WebhookAccessor, 0, size)
for _, c := range configurations {
cachedConfigurationAccessors, ok := v.configurationsCache.Load(c.Name)
if ok {
// Pick an already cached webhookAccessor
accessors = append(accessors, cachedConfigurationAccessors.([]webhook.WebhookAccessor)...)
continue
}
// webhook names are not validated for uniqueness, so we check for duplicates and
// add a int suffix to distinguish between them
names := map[string]int{}
configurationAccessors := make([]webhook.WebhookAccessor, 0, len(c.Webhooks))
for i := range c.Webhooks {
n := c.Webhooks[i].Name
uid := fmt.Sprintf("%s/%s/%d", c.Name, n, names[n])
names[n]++
accessors = append(accessors, webhook.NewValidatingWebhookAccessor(uid, c.Name, &c.Webhooks[i]))
configurationAccessor := v.createValidatingWebhookAccessor(uid, c.Name, &c.Webhooks[i])
configurationAccessors = append(configurationAccessors, configurationAccessor)
}
accessors = append(accessors, configurationAccessors...)
v.configurationsCache.Store(c.Name, configurationAccessors)
}
return accessors
}

View File

@@ -14,16 +14,40 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package generic
package admission
import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
)
// VersionedAttributes is a wrapper around the original admission attributes, adding versioned
// variants of the object and old object.
type VersionedAttributes struct {
// Attributes holds the original admission attributes
Attributes
// VersionedOldObject holds Attributes.OldObject (if non-nil), converted to VersionedKind.
// It must never be mutated.
VersionedOldObject runtime.Object
// VersionedObject holds Attributes.Object (if non-nil), converted to VersionedKind.
// If mutated, Dirty must be set to true by the mutator.
VersionedObject runtime.Object
// VersionedKind holds the fully qualified kind
VersionedKind schema.GroupVersionKind
// Dirty indicates VersionedObject has been modified since being converted from Attributes.Object
Dirty bool
}
// GetObject overrides the Attributes.GetObject()
func (v *VersionedAttributes) GetObject() runtime.Object {
if v.VersionedObject != nil {
return v.VersionedObject
}
return v.Attributes.GetObject()
}
// ConvertToGVK converts object to the desired gvk.
func ConvertToGVK(obj runtime.Object, gvk schema.GroupVersionKind, o admission.ObjectInterfaces) (runtime.Object, error) {
func ConvertToGVK(obj runtime.Object, gvk schema.GroupVersionKind, o ObjectInterfaces) (runtime.Object, error) {
// Unlike other resources, custom resources do not have internal version, so
// if obj is a custom resource, it should not need conversion.
if obj.GetObjectKind().GroupVersionKind() == gvk {
@@ -43,7 +67,7 @@ func ConvertToGVK(obj runtime.Object, gvk schema.GroupVersionKind, o admission.O
}
// NewVersionedAttributes returns versioned attributes with the old and new object (if non-nil) converted to the requested kind
func NewVersionedAttributes(attr admission.Attributes, gvk schema.GroupVersionKind, o admission.ObjectInterfaces) (*VersionedAttributes, error) {
func NewVersionedAttributes(attr Attributes, gvk schema.GroupVersionKind, o ObjectInterfaces) (*VersionedAttributes, error) {
// convert the old and new objects to the requested version
versionedAttr := &VersionedAttributes{
Attributes: attr,
@@ -72,7 +96,7 @@ func NewVersionedAttributes(attr admission.Attributes, gvk schema.GroupVersionKi
// * attr.VersionedObject is used as the source for the new object if Dirty=true (and is round-tripped through attr.Attributes.Object, clearing Dirty in the process)
// * attr.Attributes.Object is used as the source for the new object if Dirty=false
// * attr.Attributes.OldObject is used as the source for the old object
func ConvertVersionedAttributes(attr *VersionedAttributes, gvk schema.GroupVersionKind, o admission.ObjectInterfaces) error {
func ConvertVersionedAttributes(attr *VersionedAttributes, gvk schema.GroupVersionKind, o ObjectInterfaces) error {
// we already have the desired kind, we're done
if attr.VersionedKind == gvk {
return nil

View File

@@ -20,6 +20,7 @@ import (
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/cel/openapi/resolver"
quota "k8s.io/apiserver/pkg/quota/v1"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/informers"
@@ -81,3 +82,10 @@ type WantsRESTMapper interface {
SetRESTMapper(meta.RESTMapper)
admission.InitializationValidator
}
// WantsSchemaResolver defines a function which sets the SchemaResolver for
// an admission plugin that needs it.
type WantsSchemaResolver interface {
SetSchemaResolver(resolver resolver.SchemaResolver)
admission.InitializationValidator
}

View File

@@ -54,6 +54,8 @@ var (
type ObserverFunc func(ctx context.Context, elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, extraLabels ...string)
const (
kindWebhook = "webhook"
kindPolicy = "policy"
stepValidate = "validate"
stepAdmit = "admit"
)
@@ -112,12 +114,15 @@ func (p pluginHandlerWithMetrics) Validate(ctx context.Context, a admission.Attr
// AdmissionMetrics instruments admission with prometheus metrics.
type AdmissionMetrics struct {
step *metricSet
controller *metricSet
webhook *metricSet
webhookRejection *metrics.CounterVec
webhookFailOpen *metrics.CounterVec
webhookRequest *metrics.CounterVec
step *metricSet
controller *metricSet
webhook *metricSet
webhookRejection *metrics.CounterVec
webhookFailOpen *metrics.CounterVec
webhookRequest *metrics.CounterVec
matchConditionEvalErrors *metrics.CounterVec
matchConditionExclusions *metrics.CounterVec
matchConditionEvaluationSeconds *metricSet
}
// newAdmissionMetrics create a new AdmissionMetrics, configured with default metric names.
@@ -178,7 +183,7 @@ func newAdmissionMetrics() *AdmissionMetrics {
Subsystem: subsystem,
Name: "webhook_admission_duration_seconds",
Help: "Admission webhook latency histogram in seconds, identified by name and broken out for each operation and API resource and type (validate or admit).",
Buckets: []float64{0.005, 0.025, 0.1, 0.5, 1.0, 2.5},
Buckets: []float64{0.005, 0.025, 0.1, 0.5, 1.0, 2.5, 10, 25},
StabilityLevel: metrics.STABLE,
},
[]string{"name", "type", "operation", "rejected"},
@@ -217,13 +222,51 @@ func newAdmissionMetrics() *AdmissionMetrics {
},
[]string{"name", "type", "operation", "code", "rejected"})
matchConditionEvalError := metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "match_condition_evaluation_errors_total",
Help: "Admission match condition evaluation errors count, identified by name of resource containing the match condition and broken out for each kind containing matchConditions (webhook or policy), operation and admission type (validate or admit).",
StabilityLevel: metrics.ALPHA,
},
[]string{"name", "kind", "type", "operation"})
matchConditionExclusions := metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "match_condition_exclusions_total",
Help: "Admission match condition evaluation exclusions count, identified by name of resource containing the match condition and broken out for each kind containing matchConditions (webhook or policy), operation and admission type (validate or admit).",
StabilityLevel: metrics.ALPHA,
},
[]string{"name", "kind", "type", "operation"})
matchConditionEvaluationSeconds := &metricSet{
latencies: metrics.NewHistogramVec(
&metrics.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "match_condition_evaluation_seconds",
Help: "Admission match condition evaluation time in seconds, identified by name and broken out for each kind containing matchConditions (webhook or policy), operation and type (validate or admit).",
Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.1, 0.2, 0.25},
StabilityLevel: metrics.ALPHA,
},
[]string{"name", "kind", "type", "operation"},
),
latenciesSummary: nil,
}
step.mustRegister()
controller.mustRegister()
webhook.mustRegister()
matchConditionEvaluationSeconds.mustRegister()
legacyregistry.MustRegister(webhookRejection)
legacyregistry.MustRegister(webhookFailOpen)
legacyregistry.MustRegister(webhookRequest)
return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookFailOpen: webhookFailOpen, webhookRequest: webhookRequest}
legacyregistry.MustRegister(matchConditionEvalError)
legacyregistry.MustRegister(matchConditionExclusions)
return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookFailOpen: webhookFailOpen, webhookRequest: webhookRequest, matchConditionEvalErrors: matchConditionEvalError, matchConditionExclusions: matchConditionExclusions, matchConditionEvaluationSeconds: matchConditionEvaluationSeconds}
}
func (m *AdmissionMetrics) reset() {
@@ -267,6 +310,21 @@ func (m *AdmissionMetrics) ObserveWebhookFailOpen(ctx context.Context, name, ste
m.webhookFailOpen.WithContext(ctx).WithLabelValues(name, stepType).Inc()
}
// ObserveMatchConditionEvalError records validating or mutating webhook that are not called due to match conditions
func (m *AdmissionMetrics) ObserveMatchConditionEvalError(ctx context.Context, name, kind, stepType, operation string) {
m.matchConditionEvalErrors.WithContext(ctx).WithLabelValues(name, kind, stepType, operation).Inc()
}
// ObserveMatchConditionExclusion records validating or mutating webhook that are not called due to match conditions
func (m *AdmissionMetrics) ObserveMatchConditionExclusion(ctx context.Context, name, kind, stepType, operation string) {
m.matchConditionExclusions.WithContext(ctx).WithLabelValues(name, kind, stepType, operation).Inc()
}
// ObserveMatchConditionEvaluationTime records duration of match condition evaluation process.
func (m *AdmissionMetrics) ObserveMatchConditionEvaluationTime(ctx context.Context, elapsed time.Duration, name, kind, stepType, operation string) {
m.matchConditionEvaluationSeconds.observe(ctx, elapsed, name, kind, stepType, operation)
}
type metricSet struct {
latencies *metrics.HistogramVec
latenciesSummary *metrics.SummaryVec

View File

@@ -0,0 +1,10 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- jpbetz
- cici37
- alexzielenski
reviewers:
- jpbetz
- cici37
- alexzielenski

View File

@@ -0,0 +1,262 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
import (
"fmt"
"github.com/google/cel-go/cel"
"k8s.io/apimachinery/pkg/util/version"
celconfig "k8s.io/apiserver/pkg/apis/cel"
apiservercel "k8s.io/apiserver/pkg/cel"
"k8s.io/apiserver/pkg/cel/environment"
"k8s.io/apiserver/pkg/cel/library"
)
const (
ObjectVarName = "object"
OldObjectVarName = "oldObject"
ParamsVarName = "params"
RequestVarName = "request"
NamespaceVarName = "namespaceObject"
AuthorizerVarName = "authorizer"
RequestResourceAuthorizerVarName = "authorizer.requestResource"
VariableVarName = "variables"
)
// BuildRequestType generates a DeclType for AdmissionRequest. This may be replaced with a utility that
// converts the native type definition to apiservercel.DeclType once such a utility becomes available.
// The 'uid' field is omitted since it is not needed for in-process admission review.
// The 'object' and 'oldObject' fields are omitted since they are exposed as root level CEL variables.
func BuildRequestType() *apiservercel.DeclType {
field := func(name string, declType *apiservercel.DeclType, required bool) *apiservercel.DeclField {
return apiservercel.NewDeclField(name, declType, required, nil, nil)
}
fields := func(fields ...*apiservercel.DeclField) map[string]*apiservercel.DeclField {
result := make(map[string]*apiservercel.DeclField, len(fields))
for _, f := range fields {
result[f.Name] = f
}
return result
}
gvkType := apiservercel.NewObjectType("kubernetes.GroupVersionKind", fields(
field("group", apiservercel.StringType, true),
field("version", apiservercel.StringType, true),
field("kind", apiservercel.StringType, true),
))
gvrType := apiservercel.NewObjectType("kubernetes.GroupVersionResource", fields(
field("group", apiservercel.StringType, true),
field("version", apiservercel.StringType, true),
field("resource", apiservercel.StringType, true),
))
userInfoType := apiservercel.NewObjectType("kubernetes.UserInfo", fields(
field("username", apiservercel.StringType, false),
field("uid", apiservercel.StringType, false),
field("groups", apiservercel.NewListType(apiservercel.StringType, -1), false),
field("extra", apiservercel.NewMapType(apiservercel.StringType, apiservercel.NewListType(apiservercel.StringType, -1), -1), false),
))
return apiservercel.NewObjectType("kubernetes.AdmissionRequest", fields(
field("kind", gvkType, true),
field("resource", gvrType, true),
field("subResource", apiservercel.StringType, false),
field("requestKind", gvkType, true),
field("requestResource", gvrType, true),
field("requestSubResource", apiservercel.StringType, false),
field("name", apiservercel.StringType, true),
field("namespace", apiservercel.StringType, false),
field("operation", apiservercel.StringType, true),
field("userInfo", userInfoType, true),
field("dryRun", apiservercel.BoolType, false),
field("options", apiservercel.DynType, false),
))
}
// BuildNamespaceType generates a DeclType for Namespace.
// Certain nested fields in Namespace (e.g. managedFields, ownerReferences etc.) are omitted in the generated DeclType
// by design.
func BuildNamespaceType() *apiservercel.DeclType {
field := func(name string, declType *apiservercel.DeclType, required bool) *apiservercel.DeclField {
return apiservercel.NewDeclField(name, declType, required, nil, nil)
}
fields := func(fields ...*apiservercel.DeclField) map[string]*apiservercel.DeclField {
result := make(map[string]*apiservercel.DeclField, len(fields))
for _, f := range fields {
result[f.Name] = f
}
return result
}
specType := apiservercel.NewObjectType("kubernetes.NamespaceSpec", fields(
field("finalizers", apiservercel.NewListType(apiservercel.StringType, -1), true),
))
conditionType := apiservercel.NewObjectType("kubernetes.NamespaceCondition", fields(
field("status", apiservercel.StringType, true),
field("type", apiservercel.StringType, true),
field("lastTransitionTime", apiservercel.TimestampType, true),
field("message", apiservercel.StringType, true),
field("reason", apiservercel.StringType, true),
))
statusType := apiservercel.NewObjectType("kubernetes.NamespaceStatus", fields(
field("conditions", apiservercel.NewListType(conditionType, -1), true),
field("phase", apiservercel.StringType, true),
))
metadataType := apiservercel.NewObjectType("kubernetes.NamespaceMetadata", fields(
field("name", apiservercel.StringType, true),
field("generateName", apiservercel.StringType, true),
field("namespace", apiservercel.StringType, true),
field("labels", apiservercel.NewMapType(apiservercel.StringType, apiservercel.StringType, -1), true),
field("annotations", apiservercel.NewMapType(apiservercel.StringType, apiservercel.StringType, -1), true),
field("UID", apiservercel.StringType, true),
field("creationTimestamp", apiservercel.TimestampType, true),
field("deletionGracePeriodSeconds", apiservercel.IntType, true),
field("deletionTimestamp", apiservercel.TimestampType, true),
field("generation", apiservercel.IntType, true),
field("resourceVersion", apiservercel.StringType, true),
field("finalizers", apiservercel.NewListType(apiservercel.StringType, -1), true),
))
return apiservercel.NewObjectType("kubernetes.Namespace", fields(
field("metadata", metadataType, true),
field("spec", specType, true),
field("status", statusType, true),
))
}
// CompilationResult represents a compiled validations expression.
type CompilationResult struct {
Program cel.Program
Error *apiservercel.Error
ExpressionAccessor ExpressionAccessor
OutputType *cel.Type
}
// Compiler provides a CEL expression compiler configured with the desired admission related CEL variables and
// environment mode.
type Compiler interface {
CompileCELExpression(expressionAccessor ExpressionAccessor, options OptionalVariableDeclarations, mode environment.Type) CompilationResult
}
type compiler struct {
varEnvs variableDeclEnvs
}
func NewCompiler(env *environment.EnvSet) Compiler {
return &compiler{varEnvs: mustBuildEnvs(env)}
}
type variableDeclEnvs map[OptionalVariableDeclarations]*environment.EnvSet
// CompileCELExpression returns a compiled CEL expression.
// perCallLimit was added for testing purpose only. Callers should always use const PerCallLimit from k8s.io/apiserver/pkg/apis/cel/config.go as input.
func (c compiler) CompileCELExpression(expressionAccessor ExpressionAccessor, options OptionalVariableDeclarations, envType environment.Type) CompilationResult {
resultError := func(errorString string, errType apiservercel.ErrorType) CompilationResult {
return CompilationResult{
Error: &apiservercel.Error{
Type: errType,
Detail: errorString,
},
ExpressionAccessor: expressionAccessor,
}
}
env, err := c.varEnvs[options].Env(envType)
if err != nil {
return resultError(fmt.Sprintf("unexpected error loading CEL environment: %v", err), apiservercel.ErrorTypeInternal)
}
ast, issues := env.Compile(expressionAccessor.GetExpression())
if issues != nil {
return resultError("compilation failed: "+issues.String(), apiservercel.ErrorTypeInvalid)
}
found := false
returnTypes := expressionAccessor.ReturnTypes()
for _, returnType := range returnTypes {
if ast.OutputType() == returnType || cel.AnyType == returnType {
found = true
break
}
}
if !found {
var reason string
if len(returnTypes) == 1 {
reason = fmt.Sprintf("must evaluate to %v", returnTypes[0].String())
} else {
reason = fmt.Sprintf("must evaluate to one of %v", returnTypes)
}
return resultError(reason, apiservercel.ErrorTypeInvalid)
}
_, err = cel.AstToCheckedExpr(ast)
if err != nil {
// should be impossible since env.Compile returned no issues
return resultError("unexpected compilation error: "+err.Error(), apiservercel.ErrorTypeInternal)
}
prog, err := env.Program(ast,
cel.InterruptCheckFrequency(celconfig.CheckFrequency),
)
if err != nil {
return resultError("program instantiation failed: "+err.Error(), apiservercel.ErrorTypeInternal)
}
return CompilationResult{
Program: prog,
ExpressionAccessor: expressionAccessor,
OutputType: ast.OutputType(),
}
}
func mustBuildEnvs(baseEnv *environment.EnvSet) variableDeclEnvs {
requestType := BuildRequestType()
namespaceType := BuildNamespaceType()
envs := make(variableDeclEnvs, 4) // since the number of variable combinations is small, pre-build a environment for each
for _, hasParams := range []bool{false, true} {
for _, hasAuthorizer := range []bool{false, true} {
var envOpts []cel.EnvOption
if hasParams {
envOpts = append(envOpts, cel.Variable(ParamsVarName, cel.DynType))
}
if hasAuthorizer {
envOpts = append(envOpts,
cel.Variable(AuthorizerVarName, library.AuthorizerType),
cel.Variable(RequestResourceAuthorizerVarName, library.ResourceCheckType))
}
envOpts = append(envOpts,
cel.Variable(ObjectVarName, cel.DynType),
cel.Variable(OldObjectVarName, cel.DynType),
cel.Variable(NamespaceVarName, namespaceType.CelType()),
cel.Variable(RequestVarName, requestType.CelType()))
extended, err := baseEnv.Extend(
environment.VersionedOptions{
// Feature epoch was actually 1.26, but we artificially set it to 1.0 because these
// options should always be present.
IntroducedVersion: version.MajorMinor(1, 0),
EnvOptions: envOpts,
DeclTypes: []*apiservercel.DeclType{
namespaceType,
requestType,
},
},
)
if err != nil {
panic(fmt.Sprintf("environment misconfigured: %v", err))
}
envs[OptionalVariableDeclarations{HasParams: hasParams, HasAuthorizer: hasAuthorizer}] = extended
}
}
return envs
}

View File

@@ -0,0 +1,244 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
import (
"context"
"math"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
"github.com/google/cel-go/common/types/traits"
v1 "k8s.io/api/admission/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/admission"
apiservercel "k8s.io/apiserver/pkg/cel"
"k8s.io/apiserver/pkg/cel/environment"
"k8s.io/apiserver/pkg/cel/lazy"
)
const VariablesTypeName = "kubernetes.variables"
type CompositedCompiler struct {
Compiler
FilterCompiler
CompositionEnv *CompositionEnv
}
type CompositedFilter struct {
Filter
compositionEnv *CompositionEnv
}
func NewCompositedCompiler(envSet *environment.EnvSet) (*CompositedCompiler, error) {
compositionContext, err := NewCompositionEnv(VariablesTypeName, envSet)
if err != nil {
return nil, err
}
compiler := NewCompiler(compositionContext.EnvSet)
filterCompiler := NewFilterCompiler(compositionContext.EnvSet)
return &CompositedCompiler{
Compiler: compiler,
FilterCompiler: filterCompiler,
CompositionEnv: compositionContext,
}, nil
}
func (c *CompositedCompiler) CompileAndStoreVariables(variables []NamedExpressionAccessor, options OptionalVariableDeclarations, mode environment.Type) {
for _, v := range variables {
_ = c.CompileAndStoreVariable(v, options, mode)
}
}
func (c *CompositedCompiler) CompileAndStoreVariable(variable NamedExpressionAccessor, options OptionalVariableDeclarations, mode environment.Type) CompilationResult {
result := c.Compiler.CompileCELExpression(variable, options, mode)
c.CompositionEnv.AddField(variable.GetName(), result.OutputType)
c.CompositionEnv.CompiledVariables[variable.GetName()] = result
return result
}
func (c *CompositedCompiler) Compile(expressions []ExpressionAccessor, optionalDecls OptionalVariableDeclarations, envType environment.Type) Filter {
filter := c.FilterCompiler.Compile(expressions, optionalDecls, envType)
return &CompositedFilter{
Filter: filter,
compositionEnv: c.CompositionEnv,
}
}
type CompositionEnv struct {
*environment.EnvSet
MapType *apiservercel.DeclType
CompiledVariables map[string]CompilationResult
}
func (c *CompositionEnv) AddField(name string, celType *cel.Type) {
c.MapType.Fields[name] = apiservercel.NewDeclField(name, convertCelTypeToDeclType(celType), true, nil, nil)
}
func NewCompositionEnv(typeName string, baseEnvSet *environment.EnvSet) (*CompositionEnv, error) {
declType := apiservercel.NewObjectType(typeName, map[string]*apiservercel.DeclField{})
envSet, err := baseEnvSet.Extend(environment.VersionedOptions{
// set to 1.0 because composition is one of the fundamental components
IntroducedVersion: version.MajorMinor(1, 0),
EnvOptions: []cel.EnvOption{
cel.Variable("variables", declType.CelType()),
},
DeclTypes: []*apiservercel.DeclType{
declType,
},
})
if err != nil {
return nil, err
}
return &CompositionEnv{
MapType: declType,
EnvSet: envSet,
CompiledVariables: map[string]CompilationResult{},
}, nil
}
func (c *CompositionEnv) CreateContext(parent context.Context) CompositionContext {
return &compositionContext{
Context: parent,
compositionEnv: c,
}
}
type CompositionContext interface {
context.Context
Variables(activation any) ref.Val
GetAndResetCost() int64
}
type compositionContext struct {
context.Context
compositionEnv *CompositionEnv
accumulatedCost int64
}
func (c *compositionContext) Variables(activation any) ref.Val {
lazyMap := lazy.NewMapValue(c.compositionEnv.MapType)
for name, result := range c.compositionEnv.CompiledVariables {
accessor := &variableAccessor{
name: name,
result: result,
activation: activation,
context: c,
}
lazyMap.Append(name, accessor.Callback)
}
return lazyMap
}
func (f *CompositedFilter) ForInput(ctx context.Context, versionedAttr *admission.VersionedAttributes, request *v1.AdmissionRequest, optionalVars OptionalVariableBindings, namespace *corev1.Namespace, runtimeCELCostBudget int64) ([]EvaluationResult, int64, error) {
ctx = f.compositionEnv.CreateContext(ctx)
return f.Filter.ForInput(ctx, versionedAttr, request, optionalVars, namespace, runtimeCELCostBudget)
}
func (c *compositionContext) reportCost(cost int64) {
c.accumulatedCost += cost
}
func (c *compositionContext) GetAndResetCost() int64 {
cost := c.accumulatedCost
c.accumulatedCost = 0
return cost
}
type variableAccessor struct {
name string
result CompilationResult
activation any
context *compositionContext
}
func (a *variableAccessor) Callback(_ *lazy.MapValue) ref.Val {
if a.result.Error != nil {
return types.NewErr("composited variable %q fails to compile: %v", a.name, a.result.Error)
}
v, details, err := a.result.Program.ContextEval(a.context, a.activation)
if details == nil {
return types.NewErr("unable to get evaluation details of variable %q", a.name)
}
costPtr := details.ActualCost()
if costPtr == nil {
return types.NewErr("unable to calculate cost of variable %q", a.name)
}
cost := int64(*costPtr)
if *costPtr > math.MaxInt64 {
cost = math.MaxInt64
}
a.context.reportCost(cost)
if err != nil {
return types.NewErr("composited variable %q fails to evaluate: %v", a.name, err)
}
return v
}
// convertCelTypeToDeclType converts a cel.Type to DeclType, for the use of
// the TypeProvider and the cost estimator.
// List and map types are created on-demand with their parameters converted recursively.
func convertCelTypeToDeclType(celType *cel.Type) *apiservercel.DeclType {
if celType == nil {
return apiservercel.DynType
}
switch celType {
case cel.AnyType:
return apiservercel.AnyType
case cel.BoolType:
return apiservercel.BoolType
case cel.BytesType:
return apiservercel.BytesType
case cel.DoubleType:
return apiservercel.DoubleType
case cel.DurationType:
return apiservercel.DurationType
case cel.IntType:
return apiservercel.IntType
case cel.NullType:
return apiservercel.NullType
case cel.StringType:
return apiservercel.StringType
case cel.TimestampType:
return apiservercel.TimestampType
case cel.UintType:
return apiservercel.UintType
default:
if celType.HasTrait(traits.ContainerType) && celType.HasTrait(traits.IndexerType) {
parameters := celType.Parameters()
switch len(parameters) {
case 1:
elemType := convertCelTypeToDeclType(parameters[0])
return apiservercel.NewListType(elemType, -1)
case 2:
keyType := convertCelTypeToDeclType(parameters[0])
valueType := convertCelTypeToDeclType(parameters[1])
return apiservercel.NewMapType(keyType, valueType, -1)
}
}
return apiservercel.DynType
}
}

View File

@@ -0,0 +1,357 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
import (
"context"
"fmt"
"math"
"reflect"
"time"
"github.com/google/cel-go/interpreter"
admissionv1 "k8s.io/api/admission/v1"
authenticationv1 "k8s.io/api/authentication/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/cel"
"k8s.io/apiserver/pkg/cel/environment"
"k8s.io/apiserver/pkg/cel/library"
)
// filterCompiler implement the interface FilterCompiler.
type filterCompiler struct {
compiler Compiler
}
func NewFilterCompiler(env *environment.EnvSet) FilterCompiler {
return &filterCompiler{compiler: NewCompiler(env)}
}
type evaluationActivation struct {
object, oldObject, params, request, namespace, authorizer, requestResourceAuthorizer, variables interface{}
}
// ResolveName returns a value from the activation by qualified name, or false if the name
// could not be found.
func (a *evaluationActivation) ResolveName(name string) (interface{}, bool) {
switch name {
case ObjectVarName:
return a.object, true
case OldObjectVarName:
return a.oldObject, true
case ParamsVarName:
return a.params, true // params may be null
case RequestVarName:
return a.request, true
case NamespaceVarName:
return a.namespace, true
case AuthorizerVarName:
return a.authorizer, a.authorizer != nil
case RequestResourceAuthorizerVarName:
return a.requestResourceAuthorizer, a.requestResourceAuthorizer != nil
case VariableVarName: // variables always present
return a.variables, true
default:
return nil, false
}
}
// Parent returns the parent of the current activation, may be nil.
// If non-nil, the parent will be searched during resolve calls.
func (a *evaluationActivation) Parent() interpreter.Activation {
return nil
}
// Compile compiles the cel expressions defined in the ExpressionAccessors into a Filter
func (c *filterCompiler) Compile(expressionAccessors []ExpressionAccessor, options OptionalVariableDeclarations, mode environment.Type) Filter {
compilationResults := make([]CompilationResult, len(expressionAccessors))
for i, expressionAccessor := range expressionAccessors {
if expressionAccessor == nil {
continue
}
compilationResults[i] = c.compiler.CompileCELExpression(expressionAccessor, options, mode)
}
return NewFilter(compilationResults)
}
// filter implements the Filter interface
type filter struct {
compilationResults []CompilationResult
}
func NewFilter(compilationResults []CompilationResult) Filter {
return &filter{
compilationResults,
}
}
func convertObjectToUnstructured(obj interface{}) (*unstructured.Unstructured, error) {
if obj == nil || reflect.ValueOf(obj).IsNil() {
return &unstructured.Unstructured{Object: nil}, nil
}
ret, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
if err != nil {
return nil, err
}
return &unstructured.Unstructured{Object: ret}, nil
}
func objectToResolveVal(r runtime.Object) (interface{}, error) {
if r == nil || reflect.ValueOf(r).IsNil() {
return nil, nil
}
v, err := convertObjectToUnstructured(r)
if err != nil {
return nil, err
}
return v.Object, nil
}
// ForInput evaluates the compiled CEL expressions converting them into CELEvaluations
// errors per evaluation are returned on the Evaluation object
// runtimeCELCostBudget was added for testing purpose only. Callers should always use const RuntimeCELCostBudget from k8s.io/apiserver/pkg/apis/cel/config.go as input.
func (f *filter) ForInput(ctx context.Context, versionedAttr *admission.VersionedAttributes, request *admissionv1.AdmissionRequest, inputs OptionalVariableBindings, namespace *v1.Namespace, runtimeCELCostBudget int64) ([]EvaluationResult, int64, error) {
// TODO: replace unstructured with ref.Val for CEL variables when native type support is available
evaluations := make([]EvaluationResult, len(f.compilationResults))
var err error
oldObjectVal, err := objectToResolveVal(versionedAttr.VersionedOldObject)
if err != nil {
return nil, -1, err
}
objectVal, err := objectToResolveVal(versionedAttr.VersionedObject)
if err != nil {
return nil, -1, err
}
var paramsVal, authorizerVal, requestResourceAuthorizerVal any
if inputs.VersionedParams != nil {
paramsVal, err = objectToResolveVal(inputs.VersionedParams)
if err != nil {
return nil, -1, err
}
}
if inputs.Authorizer != nil {
authorizerVal = library.NewAuthorizerVal(versionedAttr.GetUserInfo(), inputs.Authorizer)
requestResourceAuthorizerVal = library.NewResourceAuthorizerVal(versionedAttr.GetUserInfo(), inputs.Authorizer, versionedAttr)
}
requestVal, err := convertObjectToUnstructured(request)
if err != nil {
return nil, -1, err
}
namespaceVal, err := objectToResolveVal(namespace)
if err != nil {
return nil, -1, err
}
va := &evaluationActivation{
object: objectVal,
oldObject: oldObjectVal,
params: paramsVal,
request: requestVal.Object,
namespace: namespaceVal,
authorizer: authorizerVal,
requestResourceAuthorizer: requestResourceAuthorizerVal,
}
// composition is an optional feature that only applies for ValidatingAdmissionPolicy.
// check if the context allows composition
var compositionCtx CompositionContext
var ok bool
if compositionCtx, ok = ctx.(CompositionContext); ok {
va.variables = compositionCtx.Variables(va)
}
remainingBudget := runtimeCELCostBudget
for i, compilationResult := range f.compilationResults {
var evaluation = &evaluations[i]
if compilationResult.ExpressionAccessor == nil { // in case of placeholder
continue
}
evaluation.ExpressionAccessor = compilationResult.ExpressionAccessor
if compilationResult.Error != nil {
evaluation.Error = &cel.Error{
Type: cel.ErrorTypeInvalid,
Detail: fmt.Sprintf("compilation error: %v", compilationResult.Error),
}
continue
}
if compilationResult.Program == nil {
evaluation.Error = &cel.Error{
Type: cel.ErrorTypeInternal,
Detail: fmt.Sprintf("unexpected internal error compiling expression"),
}
continue
}
t1 := time.Now()
evalResult, evalDetails, err := compilationResult.Program.ContextEval(ctx, va)
// budget may be spent due to lazy evaluation of composited variables
if compositionCtx != nil {
compositionCost := compositionCtx.GetAndResetCost()
if compositionCost > remainingBudget {
return nil, -1, &cel.Error{
Type: cel.ErrorTypeInvalid,
Detail: fmt.Sprintf("validation failed due to running out of cost budget, no further validation rules will be run"),
}
}
remainingBudget -= compositionCost
}
elapsed := time.Since(t1)
evaluation.Elapsed = elapsed
if evalDetails == nil {
return nil, -1, &cel.Error{
Type: cel.ErrorTypeInternal,
Detail: fmt.Sprintf("runtime cost could not be calculated for expression: %v, no further expression will be run", compilationResult.ExpressionAccessor.GetExpression()),
}
} else {
rtCost := evalDetails.ActualCost()
if rtCost == nil {
return nil, -1, &cel.Error{
Type: cel.ErrorTypeInvalid,
Detail: fmt.Sprintf("runtime cost could not be calculated for expression: %v, no further expression will be run", compilationResult.ExpressionAccessor.GetExpression()),
}
} else {
if *rtCost > math.MaxInt64 || int64(*rtCost) > remainingBudget {
return nil, -1, &cel.Error{
Type: cel.ErrorTypeInvalid,
Detail: fmt.Sprintf("validation failed due to running out of cost budget, no further validation rules will be run"),
}
}
remainingBudget -= int64(*rtCost)
}
}
if err != nil {
evaluation.Error = &cel.Error{
Type: cel.ErrorTypeInvalid,
Detail: fmt.Sprintf("expression '%v' resulted in error: %v", compilationResult.ExpressionAccessor.GetExpression(), err),
}
} else {
evaluation.EvalResult = evalResult
}
}
return evaluations, remainingBudget, nil
}
// TODO: to reuse https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/request/admissionreview.go#L154
func CreateAdmissionRequest(attr admission.Attributes, equivalentGVR metav1.GroupVersionResource, equivalentKind metav1.GroupVersionKind) *admissionv1.AdmissionRequest {
// Attempting to use same logic as webhook for constructing resource
// GVK, GVR, subresource
// Use the GVK, GVR that the matcher decided was equivalent to that of the request
// https://github.com/kubernetes/kubernetes/blob/90c362b3430bcbbf8f245fadbcd521dab39f1d7c/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go#L182-L210
gvk := equivalentKind
gvr := equivalentGVR
subresource := attr.GetSubresource()
requestGVK := attr.GetKind()
requestGVR := attr.GetResource()
requestSubResource := attr.GetSubresource()
aUserInfo := attr.GetUserInfo()
var userInfo authenticationv1.UserInfo
if aUserInfo != nil {
userInfo = authenticationv1.UserInfo{
Extra: make(map[string]authenticationv1.ExtraValue),
Groups: aUserInfo.GetGroups(),
UID: aUserInfo.GetUID(),
Username: aUserInfo.GetName(),
}
// Convert the extra information in the user object
for key, val := range aUserInfo.GetExtra() {
userInfo.Extra[key] = authenticationv1.ExtraValue(val)
}
}
dryRun := attr.IsDryRun()
return &admissionv1.AdmissionRequest{
Kind: metav1.GroupVersionKind{
Group: gvk.Group,
Kind: gvk.Kind,
Version: gvk.Version,
},
Resource: metav1.GroupVersionResource{
Group: gvr.Group,
Resource: gvr.Resource,
Version: gvr.Version,
},
SubResource: subresource,
RequestKind: &metav1.GroupVersionKind{
Group: requestGVK.Group,
Kind: requestGVK.Kind,
Version: requestGVK.Version,
},
RequestResource: &metav1.GroupVersionResource{
Group: requestGVR.Group,
Resource: requestGVR.Resource,
Version: requestGVR.Version,
},
RequestSubResource: requestSubResource,
Name: attr.GetName(),
Namespace: attr.GetNamespace(),
Operation: admissionv1.Operation(attr.GetOperation()),
UserInfo: userInfo,
// Leave Object and OldObject unset since we don't provide access to them via request
DryRun: &dryRun,
Options: runtime.RawExtension{
Object: attr.GetOperationOptions(),
},
}
}
// CreateNamespaceObject creates a Namespace object that is suitable for the CEL evaluation.
// If the namespace is nil, CreateNamespaceObject returns nil
func CreateNamespaceObject(namespace *v1.Namespace) *v1.Namespace {
if namespace == nil {
return nil
}
return &v1.Namespace{
Status: namespace.Status,
Spec: namespace.Spec,
ObjectMeta: metav1.ObjectMeta{
Name: namespace.Name,
GenerateName: namespace.GenerateName,
Namespace: namespace.Namespace,
UID: namespace.UID,
ResourceVersion: namespace.ResourceVersion,
Generation: namespace.Generation,
CreationTimestamp: namespace.CreationTimestamp,
DeletionTimestamp: namespace.DeletionTimestamp,
DeletionGracePeriodSeconds: namespace.DeletionGracePeriodSeconds,
Labels: namespace.Labels,
Annotations: namespace.Annotations,
Finalizers: namespace.Finalizers,
},
}
}
// CompilationErrors returns a list of all the errors from the compilation of the evaluator
func (e *filter) CompilationErrors() []error {
compilationErrors := []error{}
for _, result := range e.compilationResults {
if result.Error != nil {
compilationErrors = append(compilationErrors, result.Error)
}
}
return compilationErrors
}

View File

@@ -0,0 +1,95 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
import (
"context"
"time"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/common/types/ref"
v1 "k8s.io/api/admission/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/cel/environment"
)
type ExpressionAccessor interface {
GetExpression() string
ReturnTypes() []*cel.Type
}
// NamedExpressionAccessor extends NamedExpressionAccessor with a name.
type NamedExpressionAccessor interface {
ExpressionAccessor
GetName() string // follows the naming convention of ExpressionAccessor
}
// EvaluationResult contains the minimal required fields and metadata of a cel evaluation
type EvaluationResult struct {
EvalResult ref.Val
ExpressionAccessor ExpressionAccessor
Elapsed time.Duration
Error error
}
// OptionalVariableDeclarations declares which optional CEL variables
// are declared for an expression.
type OptionalVariableDeclarations struct {
// HasParams specifies if the "params" variable is declared.
// The "params" variable may still be bound to "null" when declared.
HasParams bool
// HasAuthorizer specifies if the"authorizer" and "authorizer.requestResource"
// variables are declared. When declared, the authorizer variables are
// expected to be non-null.
HasAuthorizer bool
}
// FilterCompiler contains a function to assist with converting types and values to/from CEL-typed values.
type FilterCompiler interface {
// Compile is used for the cel expression compilation
Compile(expressions []ExpressionAccessor, optionalDecls OptionalVariableDeclarations, envType environment.Type) Filter
}
// OptionalVariableBindings provides expression bindings for optional CEL variables.
type OptionalVariableBindings struct {
// VersionedParams provides the "params" variable binding. This variable binding may
// be set to nil even when OptionalVariableDeclarations.HashParams is set to true.
VersionedParams runtime.Object
// Authorizer provides the authorizer used for the "authorizer" and
// "authorizer.requestResource" variable bindings. If the expression was compiled with
// OptionalVariableDeclarations.HasAuthorizer set to true this must be non-nil.
Authorizer authorizer.Authorizer
}
// Filter contains a function to evaluate compiled CEL-typed values
// It expects the inbound object to already have been converted to the version expected
// by the underlying CEL code (which is indicated by the match criteria of a policy definition).
// versionedParams may be nil.
type Filter interface {
// ForInput converts compiled CEL-typed values into evaluated CEL-typed value.
// runtimeCELCostBudget was added for testing purpose only. Callers should always use const RuntimeCELCostBudget from k8s.io/apiserver/pkg/apis/cel/config.go as input.
// If cost budget is calculated, the filter should return the remaining budget.
ForInput(ctx context.Context, versionedAttr *admission.VersionedAttributes, request *v1.AdmissionRequest, optionalVars OptionalVariableBindings, namespace *corev1.Namespace, runtimeCELCostBudget int64) ([]EvaluationResult, int64, error)
// CompilationErrors returns a list of errors from the compilation of the evaluator
CompilationErrors() []error
}

View File

@@ -23,6 +23,7 @@ import (
"io"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/features"
"k8s.io/client-go/dynamic"
"k8s.io/component-base/featuregate"
@@ -71,6 +72,7 @@ type celAdmissionPlugin struct {
restMapper meta.RESTMapper
dynamicClient dynamic.Interface
stopCh <-chan struct{}
authorizer authorizer.Authorizer
}
var _ initializer.WantsExternalKubeInformerFactory = &celAdmissionPlugin{}
@@ -78,7 +80,7 @@ var _ initializer.WantsExternalKubeClientSet = &celAdmissionPlugin{}
var _ initializer.WantsRESTMapper = &celAdmissionPlugin{}
var _ initializer.WantsDynamicClient = &celAdmissionPlugin{}
var _ initializer.WantsDrainedNotification = &celAdmissionPlugin{}
var _ initializer.WantsAuthorizer = &celAdmissionPlugin{}
var _ admission.InitializationValidator = &celAdmissionPlugin{}
var _ admission.ValidationInterface = &celAdmissionPlugin{}
@@ -108,6 +110,9 @@ func (c *celAdmissionPlugin) SetDrainedNotification(stopCh <-chan struct{}) {
c.stopCh = stopCh
}
func (c *celAdmissionPlugin) SetAuthorizer(authorizer authorizer.Authorizer) {
c.authorizer = authorizer
}
func (c *celAdmissionPlugin) InspectFeatureGates(featureGates featuregate.FeatureGate) {
if featureGates.Enabled(features.ValidatingAdmissionPolicy) {
c.enabled = true
@@ -138,7 +143,10 @@ func (c *celAdmissionPlugin) ValidateInitialization() error {
if c.stopCh == nil {
return errors.New("missing stop channel")
}
c.evaluator = NewAdmissionController(c.informerFactory, c.client, c.restMapper, c.dynamicClient)
if c.authorizer == nil {
return errors.New("missing authorizer")
}
c.evaluator = NewAdmissionController(c.informerFactory, c.client, c.restMapper, c.dynamicClient, c.authorizer)
if err := c.evaluator.ValidateInitialization(); err != nil {
return err
}

View File

@@ -0,0 +1,133 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"context"
"encoding/json"
"sort"
"strings"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
)
type authzResult struct {
authorized authorizer.Decision
reason string
err error
}
type cachingAuthorizer struct {
authorizer authorizer.Authorizer
decisions map[string]authzResult
}
func newCachingAuthorizer(in authorizer.Authorizer) authorizer.Authorizer {
return &cachingAuthorizer{
authorizer: in,
decisions: make(map[string]authzResult),
}
}
// The attribute accessors known to cache key construction. If this fails to compile, the cache
// implementation may need to be updated.
var _ authorizer.Attributes = (interface {
GetUser() user.Info
GetVerb() string
IsReadOnly() bool
GetNamespace() string
GetResource() string
GetSubresource() string
GetName() string
GetAPIGroup() string
GetAPIVersion() string
IsResourceRequest() bool
GetPath() string
})(nil)
// The user info accessors known to cache key construction. If this fails to compile, the cache
// implementation may need to be updated.
var _ user.Info = (interface {
GetName() string
GetUID() string
GetGroups() []string
GetExtra() map[string][]string
})(nil)
// Authorize returns an authorization decision by delegating to another Authorizer. If an equivalent
// check has already been performed, a cached result is returned. Not safe for concurrent use.
func (ca *cachingAuthorizer) Authorize(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
serializableAttributes := authorizer.AttributesRecord{
Verb: a.GetVerb(),
Namespace: a.GetNamespace(),
APIGroup: a.GetAPIGroup(),
APIVersion: a.GetAPIVersion(),
Resource: a.GetResource(),
Subresource: a.GetSubresource(),
Name: a.GetName(),
ResourceRequest: a.IsResourceRequest(),
Path: a.GetPath(),
}
if u := a.GetUser(); u != nil {
di := &user.DefaultInfo{
Name: u.GetName(),
UID: u.GetUID(),
}
// Differently-ordered groups or extras could cause otherwise-equivalent checks to
// have distinct cache keys.
if groups := u.GetGroups(); len(groups) > 0 {
di.Groups = make([]string, len(groups))
copy(di.Groups, groups)
sort.Strings(di.Groups)
}
if extra := u.GetExtra(); len(extra) > 0 {
di.Extra = make(map[string][]string, len(extra))
for k, vs := range extra {
vdupe := make([]string, len(vs))
copy(vdupe, vs)
sort.Strings(vdupe)
di.Extra[k] = vdupe
}
}
serializableAttributes.User = di
}
var b strings.Builder
if err := json.NewEncoder(&b).Encode(serializableAttributes); err != nil {
return authorizer.DecisionNoOpinion, "", err
}
key := b.String()
if cached, ok := ca.decisions[key]; ok {
return cached.authorized, cached.reason, cached.err
}
authorized, reason, err := ca.authorizer.Authorize(ctx, a)
ca.decisions[key] = authzResult{
authorized: authorized,
reason: reason,
err: err,
}
return authorized, reason, err
}

View File

@@ -1,231 +0,0 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"sync"
"github.com/google/cel-go/cel"
apiservercel "k8s.io/apiserver/pkg/cel"
"k8s.io/apiserver/pkg/cel/library"
)
const (
ObjectVarName = "object"
OldObjectVarName = "oldObject"
ParamsVarName = "params"
RequestVarName = "request"
checkFrequency = 100
)
type envs struct {
noParams *cel.Env
withParams *cel.Env
}
var (
initEnvsOnce sync.Once
initEnvs *envs
initEnvsErr error
)
func getEnvs() (*envs, error) {
initEnvsOnce.Do(func() {
base, err := buildBaseEnv()
if err != nil {
initEnvsErr = err
return
}
noParams, err := buildNoParamsEnv(base)
if err != nil {
initEnvsErr = err
return
}
withParams, err := buildWithParamsEnv(noParams)
if err != nil {
initEnvsErr = err
return
}
initEnvs = &envs{noParams: noParams, withParams: withParams}
})
return initEnvs, initEnvsErr
}
// This is a similar code as in k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/compilation.go
// If any changes are made here, consider to make the same changes there as well.
func buildBaseEnv() (*cel.Env, error) {
var opts []cel.EnvOption
opts = append(opts, cel.HomogeneousAggregateLiterals())
// Validate function declarations once during base env initialization,
// so they don't need to be evaluated each time a CEL rule is compiled.
// This is a relatively expensive operation.
opts = append(opts, cel.EagerlyValidateDeclarations(true), cel.DefaultUTCTimeZone(true))
opts = append(opts, library.ExtensionLibs...)
return cel.NewEnv(opts...)
}
func buildNoParamsEnv(baseEnv *cel.Env) (*cel.Env, error) {
var propDecls []cel.EnvOption
reg := apiservercel.NewRegistry(baseEnv)
requestType := buildRequestType()
rt, err := apiservercel.NewRuleTypes(requestType.TypeName(), requestType, reg)
if err != nil {
return nil, err
}
if rt == nil {
return nil, nil
}
opts, err := rt.EnvOptions(baseEnv.TypeProvider())
if err != nil {
return nil, err
}
propDecls = append(propDecls, cel.Variable(ObjectVarName, cel.DynType))
propDecls = append(propDecls, cel.Variable(OldObjectVarName, cel.DynType))
propDecls = append(propDecls, cel.Variable(RequestVarName, requestType.CelType()))
opts = append(opts, propDecls...)
env, err := baseEnv.Extend(opts...)
if err != nil {
return nil, err
}
return env, nil
}
func buildWithParamsEnv(noParams *cel.Env) (*cel.Env, error) {
return noParams.Extend(cel.Variable(ParamsVarName, cel.DynType))
}
// buildRequestType generates a DeclType for AdmissionRequest. This may be replaced with a utility that
// converts the native type definition to apiservercel.DeclType once such a utility becomes available.
// The 'uid' field is omitted since it is not needed for in-process admission review.
// The 'object' and 'oldObject' fields are omitted since they are exposed as root level CEL variables.
func buildRequestType() *apiservercel.DeclType {
field := func(name string, declType *apiservercel.DeclType, required bool) *apiservercel.DeclField {
return apiservercel.NewDeclField(name, declType, required, nil, nil)
}
fields := func(fields ...*apiservercel.DeclField) map[string]*apiservercel.DeclField {
result := make(map[string]*apiservercel.DeclField, len(fields))
for _, f := range fields {
result[f.Name] = f
}
return result
}
gvkType := apiservercel.NewObjectType("kubernetes.GroupVersionKind", fields(
field("group", apiservercel.StringType, true),
field("version", apiservercel.StringType, true),
field("kind", apiservercel.StringType, true),
))
gvrType := apiservercel.NewObjectType("kubernetes.GroupVersionResource", fields(
field("group", apiservercel.StringType, true),
field("version", apiservercel.StringType, true),
field("resource", apiservercel.StringType, true),
))
userInfoType := apiservercel.NewObjectType("kubernetes.UserInfo", fields(
field("username", apiservercel.StringType, false),
field("uid", apiservercel.StringType, false),
field("groups", apiservercel.NewListType(apiservercel.StringType, -1), false),
field("extra", apiservercel.NewMapType(apiservercel.StringType, apiservercel.NewListType(apiservercel.StringType, -1), -1), false),
))
return apiservercel.NewObjectType("kubernetes.AdmissionRequest", fields(
field("kind", gvkType, true),
field("resource", gvrType, true),
field("subResource", apiservercel.StringType, false),
field("requestKind", gvkType, true),
field("requestResource", gvrType, true),
field("requestSubResource", apiservercel.StringType, false),
field("name", apiservercel.StringType, true),
field("namespace", apiservercel.StringType, false),
field("operation", apiservercel.StringType, true),
field("userInfo", userInfoType, true),
field("dryRun", apiservercel.BoolType, false),
field("options", apiservercel.DynType, false),
))
}
// CompilationResult represents a compiled ValidatingAdmissionPolicy validation expression.
type CompilationResult struct {
Program cel.Program
Error *apiservercel.Error
}
// CompileValidatingPolicyExpression returns a compiled vaalidating policy CEL expression.
func CompileValidatingPolicyExpression(validationExpression string, hasParams bool) CompilationResult {
var env *cel.Env
envs, err := getEnvs()
if err != nil {
return CompilationResult{
Error: &apiservercel.Error{
Type: apiservercel.ErrorTypeInternal,
Detail: "compiler initialization failed: " + err.Error(),
},
}
}
if hasParams {
env = envs.withParams
} else {
env = envs.noParams
}
ast, issues := env.Compile(validationExpression)
if issues != nil {
return CompilationResult{
Error: &apiservercel.Error{
Type: apiservercel.ErrorTypeInvalid,
Detail: "compilation failed: " + issues.String(),
},
}
}
if ast.OutputType() != cel.BoolType {
return CompilationResult{
Error: &apiservercel.Error{
Type: apiservercel.ErrorTypeInvalid,
Detail: "cel expression must evaluate to a bool",
},
}
}
_, err = cel.AstToCheckedExpr(ast)
if err != nil {
// should be impossible since env.Compile returned no issues
return CompilationResult{
Error: &apiservercel.Error{
Type: apiservercel.ErrorTypeInternal,
Detail: "unexpected compilation error: " + err.Error(),
},
}
}
prog, err := env.Program(ast,
cel.EvalOptions(cel.OptOptimize),
cel.OptimizeRegex(library.ExtensionLibRegexOptimizations...),
cel.InterruptCheckFrequency(checkFrequency),
)
if err != nil {
return CompilationResult{
Error: &apiservercel.Error{
Type: apiservercel.ErrorTypeInvalid,
Detail: "program instantiation failed: " + err.Error(),
},
}
}
return CompilationResult{
Program: prog,
}
}

View File

@@ -20,25 +20,34 @@ import (
"context"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/matching"
"k8s.io/api/admissionregistration/v1alpha1"
"k8s.io/api/admissionregistration/v1beta1"
v1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utiljson "k8s.io/apimachinery/pkg/util/json"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/admission"
celmetrics "k8s.io/apiserver/pkg/admission/cel"
"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/internal/generic"
"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/matching"
celconfig "k8s.io/apiserver/pkg/apis/cel"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/warning"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
)
var _ CELPolicyEvaluator = &celAdmissionController{}
@@ -46,44 +55,34 @@ var _ CELPolicyEvaluator = &celAdmissionController{}
// celAdmissionController is the top-level controller for admission control using CEL
// it is responsible for watching policy definitions, bindings, and config param CRDs
type celAdmissionController struct {
// Context under which the controller runs
runningContext context.Context
// Controller which manages book-keeping for the cluster's dynamic policy
// information.
policyController *policyController
policyDefinitionsController generic.Controller[*v1alpha1.ValidatingAdmissionPolicy]
policyBindingController generic.Controller[*v1alpha1.ValidatingAdmissionPolicyBinding]
// atomic []policyData
// list of every known policy definition, and all informatoin required to
// validate its bindings against an object.
// A snapshot of the current policy configuration is synced with this field
// asynchronously
definitions atomic.Value
// dynamicclient used to create informers to watch the param crd types
dynamicClient dynamic.Interface
restMapper meta.RESTMapper
authz authorizer.Authorizer
}
// Provided to the policy's Compile function as an injected dependency to
// assist with compiling its expressions to CEL
validatorCompiler ValidatorCompiler
// Everything someone might need to validate a single ValidatingPolicyDefinition
// against all of its registered bindings.
type policyData struct {
definitionInfo
paramInfo
bindings []bindingInfo
}
// Lock which protects:
// - definitionInfo
// - bindingInfos
// - paramCRDControllers
// - definitionsToBindings
// All other fields should be assumed constant
mutex sync.RWMutex
// controller and metadata
paramsCRDControllers map[v1alpha1.ParamKind]*paramInfo
// Index for each definition namespace/name, contains all binding
// namespace/names known to exist for that definition
definitionInfo map[namespacedName]*definitionInfo
// Index for each bindings namespace/name. Contains compiled templates
// for the binding depending on the policy/param combination.
bindingInfos map[namespacedName]*bindingInfo
// Map from namespace/name of a definition to a set of namespace/name
// of bindings which depend on it.
// All keys must have at least one dependent binding
// All binding names MUST exist as a key bindingInfos
definitionsToBindings map[namespacedName]sets.Set[namespacedName]
// contains the cel PolicyDecisions along with the ValidatingAdmissionPolicy and ValidatingAdmissionPolicyBinding
// that determined the decision
type policyDecisionWithMetadata struct {
PolicyDecision
Definition *v1beta1.ValidatingAdmissionPolicy
Binding *v1beta1.ValidatingAdmissionPolicyBinding
}
// namespaceName is used as a key in definitionInfo and bindingInfos
@@ -99,25 +98,28 @@ type definitionInfo struct {
// Last value seen by this controller to be used in policy enforcement
// May not be nil
lastReconciledValue *v1alpha1.ValidatingAdmissionPolicy
lastReconciledValue *v1beta1.ValidatingAdmissionPolicy
}
type bindingInfo struct {
// Compiled CEL expression turned into an validator
validator atomic.Pointer[Validator]
validator Validator
// Last value seen by this controller to be used in policy enforcement
// May not be nil
lastReconciledValue *v1alpha1.ValidatingAdmissionPolicyBinding
lastReconciledValue *v1beta1.ValidatingAdmissionPolicyBinding
}
type paramInfo struct {
// Controller which is watching this param CRD
controller generic.Controller[*unstructured.Unstructured]
controller generic.Controller[runtime.Object]
// Function to call to stop the informer and clean up the controller
stop func()
// Whether this param is cluster or namespace scoped
scope meta.RESTScope
// Policy Definitions which refer to this param CRD
dependentDefinitions sets.Set[namespacedName]
}
@@ -128,66 +130,48 @@ func NewAdmissionController(
client kubernetes.Interface,
restMapper meta.RESTMapper,
dynamicClient dynamic.Interface,
authz authorizer.Authorizer,
) CELPolicyEvaluator {
matcher := matching.NewMatcher(informerFactory.Core().V1().Namespaces().Lister(), client)
validatorCompiler := &CELValidatorCompiler{
Matcher: matcher,
return &celAdmissionController{
definitions: atomic.Value{},
policyController: newPolicyController(
restMapper,
client,
dynamicClient,
informerFactory,
nil,
NewMatcher(matching.NewMatcher(informerFactory.Core().V1().Namespaces().Lister(), client)),
generic.NewInformer[*v1beta1.ValidatingAdmissionPolicy](
informerFactory.Admissionregistration().V1beta1().ValidatingAdmissionPolicies().Informer()),
generic.NewInformer[*v1beta1.ValidatingAdmissionPolicyBinding](
informerFactory.Admissionregistration().V1beta1().ValidatingAdmissionPolicyBindings().Informer()),
),
authz: authz,
}
c := &celAdmissionController{
definitionInfo: make(map[namespacedName]*definitionInfo),
bindingInfos: make(map[namespacedName]*bindingInfo),
paramsCRDControllers: make(map[v1alpha1.ParamKind]*paramInfo),
definitionsToBindings: make(map[namespacedName]sets.Set[namespacedName]),
dynamicClient: dynamicClient,
validatorCompiler: validatorCompiler,
restMapper: restMapper,
}
c.policyDefinitionsController = generic.NewController(
generic.NewInformer[*v1alpha1.ValidatingAdmissionPolicy](
informerFactory.Admissionregistration().V1alpha1().ValidatingAdmissionPolicies().Informer()),
c.reconcilePolicyDefinition,
generic.ControllerOptions{
Workers: 1,
Name: "cel-policy-definitions",
},
)
c.policyBindingController = generic.NewController(
generic.NewInformer[*v1alpha1.ValidatingAdmissionPolicyBinding](
informerFactory.Admissionregistration().V1alpha1().ValidatingAdmissionPolicyBindings().Informer()),
c.reconcilePolicyBinding,
generic.ControllerOptions{
Workers: 1,
Name: "cel-policy-bindings",
},
)
return c
}
func (c *celAdmissionController) Run(stopCh <-chan struct{}) {
if c.runningContext != nil {
return
}
ctx, cancel := context.WithCancel(context.Background())
c.runningContext = ctx
defer func() {
c.runningContext = nil
}()
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
c.policyDefinitionsController.Run(ctx)
c.policyController.Run(ctx)
}()
wg.Add(1)
go func() {
defer wg.Done()
c.policyBindingController.Run(ctx)
// Wait indefinitely until policies/bindings are listed & handled before
// allowing policies to be refreshed
if !cache.WaitForNamedCacheSync("cel-admission-controller", ctx.Done(), c.policyController.HasSynced) {
return
}
// Loop every 1 second until context is cancelled, refreshing policies
wait.Until(c.refreshPolicies, 1*time.Second, ctx.Done())
}()
<-stopCh
@@ -195,31 +179,34 @@ func (c *celAdmissionController) Run(stopCh <-chan struct{}) {
wg.Wait()
}
const maxAuditAnnotationValueLength = 10 * 1024
func (c *celAdmissionController) Validate(
ctx context.Context,
a admission.Attributes,
o admission.ObjectInterfaces,
) (err error) {
c.mutex.RLock()
defer c.mutex.RUnlock()
if !c.HasSynced() {
return admission.NewForbidden(a, fmt.Errorf("not yet ready to handle request"))
}
var deniedDecisions []policyDecisionWithMetadata
addConfigError := func(err error, definition *v1alpha1.ValidatingAdmissionPolicy, binding *v1alpha1.ValidatingAdmissionPolicyBinding) {
addConfigError := func(err error, definition *v1beta1.ValidatingAdmissionPolicy, binding *v1beta1.ValidatingAdmissionPolicyBinding) {
// we always default the FailurePolicy if it is unset and validate it in API level
var policy v1alpha1.FailurePolicyType
var policy v1beta1.FailurePolicyType
if definition.Spec.FailurePolicy == nil {
policy = v1alpha1.Fail
policy = v1beta1.Fail
} else {
policy = *definition.Spec.FailurePolicy
}
// apply FailurePolicy specified in ValidatingAdmissionPolicy, the default would be Fail
switch policy {
case v1alpha1.Ignore:
case v1beta1.Ignore:
// TODO: add metrics for ignored error here
return
case v1alpha1.Fail:
case v1beta1.Fail:
var message string
if binding == nil {
message = fmt.Errorf("failed to configure policy: %w", err).Error()
@@ -227,27 +214,37 @@ func (c *celAdmissionController) Validate(
message = fmt.Errorf("failed to configure binding: %w", err).Error()
}
deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{
policyDecision: policyDecision{
action: actionDeny,
message: message,
PolicyDecision: PolicyDecision{
Action: ActionDeny,
Message: message,
},
definition: definition,
binding: binding,
Definition: definition,
Binding: binding,
})
default:
deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{
policyDecision: policyDecision{
action: actionDeny,
message: fmt.Errorf("unrecognized failure policy: '%v'", policy).Error(),
PolicyDecision: PolicyDecision{
Action: ActionDeny,
Message: fmt.Errorf("unrecognized failure policy: '%v'", policy).Error(),
},
definition: definition,
binding: binding,
Definition: definition,
Binding: binding,
})
}
}
for definitionNamespacedName, definitionInfo := range c.definitionInfo {
policyDatas := c.definitions.Load().([]policyData)
authz := newCachingAuthorizer(c.authz)
for _, definitionInfo := range policyDatas {
// versionedAttributes will be set to non-nil inside of the loop, but
// is scoped outside of the param loop so we only convert once. We defer
// conversion so that it is only performed when we know a policy matches,
// saving the cost of converting non-matching requests.
var versionedAttr *admission.VersionedAttributes
definition := definitionInfo.lastReconciledValue
matches, matchKind, err := c.validatorCompiler.DefinitionMatches(a, o, definition)
matches, matchResource, matchKind, err := c.policyController.matcher.DefinitionMatches(a, o, definition)
if err != nil {
// Configuration error.
addConfigError(err, definition, nil)
@@ -262,17 +259,12 @@ func (c *celAdmissionController) Validate(
continue
}
dependentBindings := c.definitionsToBindings[definitionNamespacedName]
if len(dependentBindings) == 0 {
continue
}
for namespacedBindingName := range dependentBindings {
auditAnnotationCollector := newAuditAnnotationCollector()
for _, bindingInfo := range definitionInfo.bindings {
// If the key is inside dependentBindings, there is guaranteed to
// be a bindingInfo for it
bindingInfo := c.bindingInfos[namespacedBindingName]
binding := bindingInfo.lastReconciledValue
matches, err := c.validatorCompiler.BindingMatches(a, o, binding)
matches, err := c.policyController.matcher.BindingMatches(a, o, binding)
if err != nil {
// Configuration error.
addConfigError(err, definition, binding)
@@ -282,109 +274,131 @@ func (c *celAdmissionController) Validate(
continue
}
var param *unstructured.Unstructured
// If definition has paramKind, paramRef is required in binding.
// If definition has no paramKind, paramRef set in binding will be ignored.
paramKind := definition.Spec.ParamKind
paramRef := binding.Spec.ParamRef
if paramKind != nil && paramRef != nil {
// Find the params referred by the binding by looking its name up
// in our informer for its CRD
paramInfo, ok := c.paramsCRDControllers[*paramKind]
if !ok {
addConfigError(fmt.Errorf("paramKind kind `%v` not known",
paramKind.String()), definition, binding)
continue
}
// If the param informer for this admission policy has not yet
// had time to perform an initial listing, don't attempt to use
// it.
//!TOOD(alexzielenski): add a wait for a very short amount of
// time for the cache to sync
if !paramInfo.controller.HasSynced() {
addConfigError(fmt.Errorf("paramKind kind `%v` not yet synced to use for admission",
paramKind.String()), definition, binding)
continue
}
if len(paramRef.Namespace) == 0 {
param, err = paramInfo.controller.Informer().Get(paramRef.Name)
} else {
param, err = paramInfo.controller.Informer().Namespaced(paramRef.Namespace).Get(paramRef.Name)
}
if err != nil {
// Apply failure policy
addConfigError(err, definition, binding)
if k8serrors.IsInvalid(err) {
// Param mis-configured
// require to set paramRef.namespace for namespaced resource and unset paramRef.namespace for cluster scoped resource
continue
} else if k8serrors.IsNotFound(err) {
// Param not yet available. User may need to wait a bit
// before being able to use it for validation.
continue
}
// There was a bad internal error
utilruntime.HandleError(err)
continue
}
}
validator := bindingInfo.validator.Load()
if validator == nil {
// Compile policy definition using binding
newValidator := c.validatorCompiler.Compile(definition)
validator = &newValidator
bindingInfo.validator.Store(validator)
}
decisions, err := (*validator).Validate(a, o, param, matchKind)
params, err := c.collectParams(definition.Spec.ParamKind, definitionInfo.paramInfo, binding.Spec.ParamRef, a.GetNamespace())
if err != nil {
// runtime error. Apply failure policy
wrappedError := fmt.Errorf("failed to evaluate CEL expression: %w", err)
addConfigError(wrappedError, definition, binding)
addConfigError(err, definition, binding)
continue
} else if versionedAttr == nil && len(params) > 0 {
// As optimization versionedAttr creation is deferred until
// first use. Since > 0 params, we will validate
va, err := admission.NewVersionedAttributes(a, matchKind, o)
if err != nil {
wrappedErr := fmt.Errorf("failed to convert object version: %w", err)
addConfigError(wrappedErr, definition, binding)
continue
}
versionedAttr = va
}
for _, decision := range decisions {
switch decision.action {
case actionAdmit:
if decision.evaluation == evalError {
celmetrics.Metrics.ObserveAdmissionWithError(ctx, decision.elapsed, definition.Name, binding.Name, "active")
var validationResults []ValidateResult
var namespace *v1.Namespace
namespaceName := a.GetNamespace()
// Special case, the namespace object has the namespace of itself (maybe a bug).
// unset it if the incoming object is a namespace
if gvk := a.GetKind(); gvk.Kind == "Namespace" && gvk.Version == "v1" && gvk.Group == "" {
namespaceName = ""
}
// if it is cluster scoped, namespaceName will be empty
// Otherwise, get the Namespace resource.
if namespaceName != "" {
namespace, err = c.policyController.matcher.GetNamespace(namespaceName)
if err != nil {
return err
}
}
for _, param := range params {
var p runtime.Object = param
if p != nil && p.GetObjectKind().GroupVersionKind().Empty() {
// Make sure param has TypeMeta populated
// This is a simple hack to make sure typeMeta is
// available to CEL without making copies of objects, etc.
p = &wrappedParam{
TypeMeta: metav1.TypeMeta{
APIVersion: definition.Spec.ParamKind.APIVersion,
Kind: definition.Spec.ParamKind.Kind,
},
nested: param,
}
}
validationResults = append(validationResults, bindingInfo.validator.Validate(ctx, matchResource, versionedAttr, p, namespace, celconfig.RuntimeCELCostBudget, authz))
}
for _, validationResult := range validationResults {
for i, decision := range validationResult.Decisions {
switch decision.Action {
case ActionAdmit:
if decision.Evaluation == EvalError {
celmetrics.Metrics.ObserveAdmissionWithError(ctx, decision.Elapsed, definition.Name, binding.Name, "active")
}
case ActionDeny:
for _, action := range binding.Spec.ValidationActions {
switch action {
case v1beta1.Deny:
deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{
Definition: definition,
Binding: binding,
PolicyDecision: decision,
})
celmetrics.Metrics.ObserveRejection(ctx, decision.Elapsed, definition.Name, binding.Name, "active")
case v1beta1.Audit:
c.publishValidationFailureAnnotation(binding, i, decision, versionedAttr)
celmetrics.Metrics.ObserveAudit(ctx, decision.Elapsed, definition.Name, binding.Name, "active")
case v1beta1.Warn:
warning.AddWarning(ctx, "", fmt.Sprintf("Validation failed for ValidatingAdmissionPolicy '%s' with binding '%s': %s", definition.Name, binding.Name, decision.Message))
celmetrics.Metrics.ObserveWarn(ctx, decision.Elapsed, definition.Name, binding.Name, "active")
}
}
default:
return fmt.Errorf("unrecognized evaluation decision '%s' for ValidatingAdmissionPolicyBinding '%s' with ValidatingAdmissionPolicy '%s'",
decision.Action, binding.Name, definition.Name)
}
}
for _, auditAnnotation := range validationResult.AuditAnnotations {
switch auditAnnotation.Action {
case AuditAnnotationActionPublish:
value := auditAnnotation.Value
if len(auditAnnotation.Value) > maxAuditAnnotationValueLength {
value = value[:maxAuditAnnotationValueLength]
}
auditAnnotationCollector.add(auditAnnotation.Key, value)
case AuditAnnotationActionError:
// When failurePolicy=fail, audit annotation errors result in deny
deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{
Definition: definition,
Binding: binding,
PolicyDecision: PolicyDecision{
Action: ActionDeny,
Evaluation: EvalError,
Message: auditAnnotation.Error,
Elapsed: auditAnnotation.Elapsed,
},
})
celmetrics.Metrics.ObserveRejection(ctx, auditAnnotation.Elapsed, definition.Name, binding.Name, "active")
case AuditAnnotationActionExclude: // skip it
default:
return fmt.Errorf("unsupported AuditAnnotation Action: %s", auditAnnotation.Action)
}
case actionDeny:
deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{
definition: definition,
binding: binding,
policyDecision: decision,
})
celmetrics.Metrics.ObserveRejection(ctx, decision.elapsed, definition.Name, binding.Name, "active")
default:
return fmt.Errorf("unrecognized evaluation decision '%s' for ValidatingAdmissionPolicyBinding '%s' with ValidatingAdmissionPolicy '%s'",
decision.action, binding.Name, definition.Name)
}
}
}
auditAnnotationCollector.publish(definition.Name, a)
}
if len(deniedDecisions) > 0 {
// TODO: refactor admission.NewForbidden so the name extraction is reusable but the code/reason is customizable
var message string
deniedDecision := deniedDecisions[0]
if deniedDecision.binding != nil {
message = fmt.Sprintf("ValidatingAdmissionPolicy '%s' with binding '%s' denied request: %s", deniedDecision.definition.Name, deniedDecision.binding.Name, deniedDecision.message)
if deniedDecision.Binding != nil {
message = fmt.Sprintf("ValidatingAdmissionPolicy '%s' with binding '%s' denied request: %s", deniedDecision.Definition.Name, deniedDecision.Binding.Name, deniedDecision.Message)
} else {
message = fmt.Sprintf("ValidatingAdmissionPolicy '%s' denied request: %s", deniedDecision.definition.Name, deniedDecision.message)
message = fmt.Sprintf("ValidatingAdmissionPolicy '%s' denied request: %s", deniedDecision.Definition.Name, deniedDecision.Message)
}
err := admission.NewForbidden(a, errors.New(message)).(*k8serrors.StatusError)
reason := deniedDecision.reason
reason := deniedDecision.Reason
if len(reason) == 0 {
reason = metav1.StatusReasonInvalid
}
@@ -396,11 +410,240 @@ func (c *celAdmissionController) Validate(
return nil
}
// Returns objects to use to evaluate the policy
func (c *celAdmissionController) collectParams(
paramKind *v1beta1.ParamKind,
info paramInfo,
paramRef *v1beta1.ParamRef,
namespace string,
) ([]runtime.Object, error) {
// If definition has paramKind, paramRef is required in binding.
// If definition has no paramKind, paramRef set in binding will be ignored.
var params []runtime.Object
var paramStore generic.NamespacedLister[runtime.Object]
// Make sure the param kind is ready to use
if paramKind != nil && paramRef != nil {
if info.controller == nil {
return nil, fmt.Errorf("paramKind kind `%v` not known",
paramKind.String())
}
// Set up cluster-scoped, or namespaced access to the params
// "default" if not provided, and paramKind is namespaced
paramStore = info.controller.Informer()
if info.scope.Name() == meta.RESTScopeNameNamespace {
paramsNamespace := namespace
if len(paramRef.Namespace) > 0 {
paramsNamespace = paramRef.Namespace
} else if len(paramsNamespace) == 0 {
// You must supply namespace if your matcher can possibly
// match a cluster-scoped resource
return nil, fmt.Errorf("cannot use namespaced paramRef in policy binding that matches cluster-scoped resources")
}
paramStore = info.controller.Informer().Namespaced(paramsNamespace)
}
// If the param informer for this admission policy has not yet
// had time to perform an initial listing, don't attempt to use
// it.
timeoutCtx, cancel := context.WithTimeout(c.policyController.context, 1*time.Second)
defer cancel()
if !cache.WaitForCacheSync(timeoutCtx.Done(), info.controller.HasSynced) {
return nil, fmt.Errorf("paramKind kind `%v` not yet synced to use for admission",
paramKind.String())
}
}
// Find params to use with policy
switch {
case paramKind == nil:
// ParamKind is unset. Ignore any globalParamRef or namespaceParamRef
// setting.
return []runtime.Object{nil}, nil
case paramRef == nil:
// Policy ParamKind is set, but binding does not use it.
// Validate with nil params
return []runtime.Object{nil}, nil
case len(paramRef.Namespace) > 0 && info.scope.Name() == meta.RESTScopeRoot.Name():
// Not allowed to set namespace for cluster-scoped param
return nil, fmt.Errorf("paramRef.namespace must not be provided for a cluster-scoped `paramKind`")
case len(paramRef.Name) > 0:
if paramRef.Selector != nil {
// This should be validated, but just in case.
return nil, fmt.Errorf("paramRef.name and paramRef.selector are mutually exclusive")
}
switch param, err := paramStore.Get(paramRef.Name); {
case err == nil:
params = []runtime.Object{param}
case k8serrors.IsNotFound(err):
// Param not yet available. User may need to wait a bit
// before being able to use it for validation.
//
// Set params to nil to prepare for not found action
params = nil
case k8serrors.IsInvalid(err):
// Param mis-configured
// require to set namespace for namespaced resource
// and unset namespace for cluster scoped resource
return nil, err
default:
// Internal error
utilruntime.HandleError(err)
return nil, err
}
case paramRef.Selector != nil:
// Select everything by default if empty name and selector
selector, err := metav1.LabelSelectorAsSelector(paramRef.Selector)
if err != nil {
// Cannot parse label selector: configuration error
return nil, err
}
paramList, err := paramStore.List(selector)
if err != nil {
// There was a bad internal error
utilruntime.HandleError(err)
return nil, err
}
// Successfully grabbed params
params = paramList
default:
// Should be unreachable due to validation
return nil, fmt.Errorf("one of name or selector must be provided")
}
// Apply fail action for params not found case
if len(params) == 0 && paramRef.ParameterNotFoundAction != nil && *paramRef.ParameterNotFoundAction == v1beta1.DenyAction {
return nil, errors.New("no params found for policy binding with `Deny` parameterNotFoundAction")
}
return params, nil
}
func (c *celAdmissionController) publishValidationFailureAnnotation(binding *v1beta1.ValidatingAdmissionPolicyBinding, expressionIndex int, decision PolicyDecision, attributes admission.Attributes) {
key := "validation.policy.admission.k8s.io/validation_failure"
// Marshal to a list of failures since, in the future, we may need to support multiple failures
valueJson, err := utiljson.Marshal([]validationFailureValue{{
ExpressionIndex: expressionIndex,
Message: decision.Message,
ValidationActions: binding.Spec.ValidationActions,
Binding: binding.Name,
Policy: binding.Spec.PolicyName,
}})
if err != nil {
klog.Warningf("Failed to set admission audit annotation %s for ValidatingAdmissionPolicy %s and ValidatingAdmissionPolicyBinding %s: %v", key, binding.Spec.PolicyName, binding.Name, err)
}
value := string(valueJson)
if err := attributes.AddAnnotation(key, value); err != nil {
klog.Warningf("Failed to set admission audit annotation %s to %s for ValidatingAdmissionPolicy %s and ValidatingAdmissionPolicyBinding %s: %v", key, value, binding.Spec.PolicyName, binding.Name, err)
}
}
func (c *celAdmissionController) HasSynced() bool {
return c.policyBindingController.HasSynced() &&
c.policyDefinitionsController.HasSynced()
return c.policyController.HasSynced() && c.definitions.Load() != nil
}
func (c *celAdmissionController) ValidateInitialization() error {
return c.validatorCompiler.ValidateInitialization()
return c.policyController.matcher.ValidateInitialization()
}
func (c *celAdmissionController) refreshPolicies() {
c.definitions.Store(c.policyController.latestPolicyData())
}
// validationFailureValue defines the JSON format of a "validation.policy.admission.k8s.io/validation_failure" audit
// annotation value.
type validationFailureValue struct {
Message string `json:"message"`
Policy string `json:"policy"`
Binding string `json:"binding"`
ExpressionIndex int `json:"expressionIndex"`
ValidationActions []v1beta1.ValidationAction `json:"validationActions"`
}
type auditAnnotationCollector struct {
annotations map[string][]string
}
func newAuditAnnotationCollector() auditAnnotationCollector {
return auditAnnotationCollector{annotations: map[string][]string{}}
}
func (a auditAnnotationCollector) add(key, value string) {
// If multiple bindings produces the exact same key and value for an audit annotation,
// ignore the duplicates.
for _, v := range a.annotations[key] {
if v == value {
return
}
}
a.annotations[key] = append(a.annotations[key], value)
}
func (a auditAnnotationCollector) publish(policyName string, attributes admission.Attributes) {
for key, bindingAnnotations := range a.annotations {
var value string
if len(bindingAnnotations) == 1 {
value = bindingAnnotations[0]
} else {
// Multiple distinct values can exist when binding params are used in the valueExpression of an auditAnnotation.
// When this happens, the values are concatenated into a comma-separated list.
value = strings.Join(bindingAnnotations, ", ")
}
if err := attributes.AddAnnotation(policyName+"/"+key, value); err != nil {
klog.Warningf("Failed to set admission audit annotation %s to %s for ValidatingAdmissionPolicy %s: %v", key, value, policyName, err)
}
}
}
// A workaround to fact that native types do not have TypeMeta populated, which
// is needed for CEL expressions to be able to access the value.
type wrappedParam struct {
metav1.TypeMeta
nested runtime.Object
}
func (w *wrappedParam) MarshalJSON() ([]byte, error) {
return nil, errors.New("MarshalJSON unimplemented for wrappedParam")
}
func (w *wrappedParam) UnmarshalJSON(data []byte) error {
return errors.New("UnmarshalJSON unimplemented for wrappedParam")
}
func (w *wrappedParam) ToUnstructured() interface{} {
res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(w.nested)
if err != nil {
return nil
}
metaRes, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&w.TypeMeta)
if err != nil {
return nil
}
for k, v := range metaRes {
res[k] = v
}
return res
}
func (w *wrappedParam) DeepCopyObject() runtime.Object {
return &wrappedParam{
TypeMeta: w.TypeMeta,
nested: w.nested.DeepCopyObject(),
}
}
func (w *wrappedParam) GetObjectKind() schema.ObjectKind {
return w
}

View File

@@ -19,22 +19,156 @@ package validatingadmissionpolicy
import (
"context"
"fmt"
"sync"
"time"
"k8s.io/api/admissionregistration/v1alpha1"
v1 "k8s.io/api/admissionregistration/v1"
"k8s.io/api/admissionregistration/v1beta1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
apiequality "k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
celmetrics "k8s.io/apiserver/pkg/admission/cel"
"k8s.io/apiserver/pkg/admission/plugin/cel"
"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/internal/generic"
"k8s.io/apiserver/pkg/admission/plugin/webhook/matchconditions"
"k8s.io/apiserver/pkg/cel/environment"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/dynamic/dynamicinformer"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
)
func (c *celAdmissionController) reconcilePolicyDefinition(namespace, name string, definition *v1alpha1.ValidatingAdmissionPolicy) error {
type policyController struct {
once sync.Once
context context.Context
dynamicClient dynamic.Interface
informerFactory informers.SharedInformerFactory
restMapper meta.RESTMapper
policyDefinitionsController generic.Controller[*v1beta1.ValidatingAdmissionPolicy]
policyBindingController generic.Controller[*v1beta1.ValidatingAdmissionPolicyBinding]
// Provided to the policy's Compile function as an injected dependency to
// assist with compiling its expressions to CEL
// pass nil to create filter compiler in demand
filterCompiler cel.FilterCompiler
matcher Matcher
newValidator
client kubernetes.Interface
// Lock which protects
// All Below fields
// All above fields should be assumed constant
mutex sync.RWMutex
cachedPolicies []policyData
// controller and metadata
paramsCRDControllers map[v1beta1.ParamKind]*paramInfo
// Index for each definition namespace/name, contains all binding
// namespace/names known to exist for that definition
definitionInfo map[namespacedName]*definitionInfo
// Index for each bindings namespace/name. Contains compiled templates
// for the binding depending on the policy/param combination.
bindingInfos map[namespacedName]*bindingInfo
// Map from namespace/name of a definition to a set of namespace/name
// of bindings which depend on it.
// All keys must have at least one dependent binding
// All binding names MUST exist as a key bindingInfos
definitionsToBindings map[namespacedName]sets.Set[namespacedName]
}
type newValidator func(validationFilter cel.Filter, celMatcher matchconditions.Matcher, auditAnnotationFilter, messageFilter cel.Filter, failurePolicy *v1.FailurePolicyType) Validator
func newPolicyController(
restMapper meta.RESTMapper,
client kubernetes.Interface,
dynamicClient dynamic.Interface,
informerFactory informers.SharedInformerFactory,
filterCompiler cel.FilterCompiler,
matcher Matcher,
policiesInformer generic.Informer[*v1beta1.ValidatingAdmissionPolicy],
bindingsInformer generic.Informer[*v1beta1.ValidatingAdmissionPolicyBinding],
) *policyController {
res := &policyController{}
*res = policyController{
filterCompiler: filterCompiler,
definitionInfo: make(map[namespacedName]*definitionInfo),
bindingInfos: make(map[namespacedName]*bindingInfo),
paramsCRDControllers: make(map[v1beta1.ParamKind]*paramInfo),
definitionsToBindings: make(map[namespacedName]sets.Set[namespacedName]),
matcher: matcher,
newValidator: NewValidator,
policyDefinitionsController: generic.NewController(
policiesInformer,
res.reconcilePolicyDefinition,
generic.ControllerOptions{
Workers: 1,
Name: "cel-policy-definitions",
},
),
policyBindingController: generic.NewController(
bindingsInformer,
res.reconcilePolicyBinding,
generic.ControllerOptions{
Workers: 1,
Name: "cel-policy-bindings",
},
),
restMapper: restMapper,
dynamicClient: dynamicClient,
informerFactory: informerFactory,
client: client,
}
return res
}
func (c *policyController) Run(ctx context.Context) {
// Only support being run once
c.once.Do(func() {
c.context = ctx
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
c.policyDefinitionsController.Run(ctx)
}()
wg.Add(1)
go func() {
defer wg.Done()
c.policyBindingController.Run(ctx)
}()
<-ctx.Done()
wg.Wait()
})
}
func (c *policyController) HasSynced() bool {
return c.policyDefinitionsController.HasSynced() && c.policyBindingController.HasSynced()
}
func (c *policyController) reconcilePolicyDefinition(namespace, name string, definition *v1beta1.ValidatingAdmissionPolicy) error {
c.mutex.Lock()
defer c.mutex.Unlock()
err := c.reconcilePolicyDefinitionSpec(namespace, name, definition)
return err
}
func (c *policyController) reconcilePolicyDefinitionSpec(namespace, name string, definition *v1beta1.ValidatingAdmissionPolicy) error {
c.cachedPolicies = nil // invalidate cachedPolicies
// Namespace for policydefinition is empty.
nn := getNamespaceName(namespace, name)
@@ -46,7 +180,14 @@ func (c *celAdmissionController) reconcilePolicyDefinition(namespace, name strin
celmetrics.Metrics.ObserveDefinition(context.TODO(), "active", "deny")
}
var paramSource *v1alpha1.ParamKind
// Skip reconcile if the spec of the definition is unchanged and had a
// successful previous sync
if info.configurationError == nil && info.lastReconciledValue != nil && definition != nil &&
apiequality.Semantic.DeepEqual(info.lastReconciledValue.Spec, definition.Spec) {
return nil
}
var paramSource *v1beta1.ParamKind
if definition != nil {
paramSource = definition.Spec.ParamKind
}
@@ -75,7 +216,7 @@ func (c *celAdmissionController) reconcilePolicyDefinition(namespace, name strin
// definition has changed.
for key := range c.definitionsToBindings[nn] {
bindingInfo := c.bindingInfos[key]
bindingInfo.validator.Store(nil)
bindingInfo.validator = nil
c.bindingInfos[key] = bindingInfo
}
@@ -92,7 +233,6 @@ func (c *celAdmissionController) reconcilePolicyDefinition(namespace, name strin
// Skip setting up controller for empty param type
return nil
}
// find GVR for params
// Parse param source into a GVK
@@ -119,46 +259,83 @@ func (c *celAdmissionController) reconcilePolicyDefinition(namespace, name strin
return info.configurationError
}
// Start watching the param CRD
if _, ok := c.paramsCRDControllers[*paramSource]; !ok {
instanceContext, instanceCancel := context.WithCancel(c.runningContext)
// Watch for new instances of this policy
informer := dynamicinformer.NewFilteredDynamicInformer(
c.dynamicClient,
paramsGVR.Resource,
corev1.NamespaceAll,
30*time.Second,
cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
nil,
)
controller := generic.NewController(
generic.NewInformer[*unstructured.Unstructured](informer.Informer()),
c.reconcileParams,
generic.ControllerOptions{
Workers: 1,
Name: paramSource.String() + "-controller",
},
)
c.paramsCRDControllers[*paramSource] = &paramInfo{
controller: controller,
stop: instanceCancel,
dependentDefinitions: sets.New(nn),
}
go informer.Informer().Run(instanceContext.Done())
go controller.Run(instanceContext)
}
paramInfo := c.ensureParamInfo(paramSource, paramsGVR)
paramInfo.dependentDefinitions.Insert(nn)
return nil
}
func (c *celAdmissionController) reconcilePolicyBinding(namespace, name string, binding *v1alpha1.ValidatingAdmissionPolicyBinding) error {
// Ensures that there is an informer started for the given GVK to be used as a
// param
func (c *policyController) ensureParamInfo(paramSource *v1beta1.ParamKind, mapping *meta.RESTMapping) *paramInfo {
if info, ok := c.paramsCRDControllers[*paramSource]; ok {
return info
}
// We are not watching this param. Start an informer for it.
instanceContext, instanceCancel := context.WithCancel(c.context)
var informer cache.SharedIndexInformer
// Try to see if our provided informer factory has an informer for this type.
// We assume the informer is already started, and starts all types associated
// with it.
if genericInformer, err := c.informerFactory.ForResource(mapping.Resource); err == nil {
informer = genericInformer.Informer()
// Ensure the informer is started
// Use policyController's context rather than the instance context.
// PolicyController context is expected to last until app shutdown
// This is due to behavior of informerFactory which would cause the
// informer to stop running once the context is cancelled, and
// never started again.
c.informerFactory.Start(c.context.Done())
} else {
// Dynamic JSON informer fallback.
// Cannot use shared dynamic informer since it would be impossible
// to clean CRD informers properly with multiple dependents
// (cannot start ahead of time, and cannot track dependencies via stopCh)
informer = dynamicinformer.NewFilteredDynamicInformer(
c.dynamicClient,
mapping.Resource,
corev1.NamespaceAll,
// Use same interval as is used for k8s typed sharedInformerFactory
// https://github.com/kubernetes/kubernetes/blob/7e0923899fed622efbc8679cca6b000d43633e38/cmd/kube-apiserver/app/server.go#L430
10*time.Minute,
cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
nil,
).Informer()
go informer.Run(instanceContext.Done())
}
controller := generic.NewController(
generic.NewInformer[runtime.Object](informer),
c.reconcileParams,
generic.ControllerOptions{
Workers: 1,
Name: paramSource.String() + "-controller",
},
)
ret := &paramInfo{
controller: controller,
stop: instanceCancel,
scope: mapping.Scope,
dependentDefinitions: sets.New[namespacedName](),
}
c.paramsCRDControllers[*paramSource] = ret
go controller.Run(instanceContext)
return ret
}
func (c *policyController) reconcilePolicyBinding(namespace, name string, binding *v1beta1.ValidatingAdmissionPolicyBinding) error {
c.mutex.Lock()
defer c.mutex.Unlock()
c.cachedPolicies = nil // invalidate cachedPolicies
// Namespace for PolicyBinding is empty. In the future a namespaced binding
// may be added
// https://github.com/kubernetes/enhancements/blob/bf5c3c81ea2081d60c1dc7c832faa98479e06209/keps/sig-api-machinery/3488-cel-admission-control/README.md?plain=1#L1042
@@ -169,6 +346,12 @@ func (c *celAdmissionController) reconcilePolicyBinding(namespace, name string,
c.bindingInfos[nn] = info
}
// Skip if the spec of the binding is unchanged.
if info.lastReconciledValue != nil && binding != nil &&
apiequality.Semantic.DeepEqual(info.lastReconciledValue.Spec, binding.Spec) {
return nil
}
var oldNamespacedDefinitionName namespacedName
if info.lastReconciledValue != nil {
// All validating policies are cluster-scoped so have empty namespace
@@ -208,12 +391,12 @@ func (c *celAdmissionController) reconcilePolicyBinding(namespace, name string,
}
// Remove compiled template for old binding
info.validator.Store(nil)
info.validator = nil
info.lastReconciledValue = binding
return nil
}
func (c *celAdmissionController) reconcileParams(namespace, name string, params *unstructured.Unstructured) error {
func (c *policyController) reconcileParams(namespace, name string, params runtime.Object) error {
// Do nothing.
// When we add informational type checking we will need to compile in the
// reconcile loops instead of lazily so we can add compiler errors / type
@@ -221,6 +404,145 @@ func (c *celAdmissionController) reconcileParams(namespace, name string, params
return nil
}
// Fetches the latest set of policy data or recalculates it if it has changed
// since it was last fetched
func (c *policyController) latestPolicyData() []policyData {
existing := func() []policyData {
c.mutex.RLock()
defer c.mutex.RUnlock()
return c.cachedPolicies
}()
if existing != nil {
return existing
}
c.mutex.Lock()
defer c.mutex.Unlock()
var res []policyData
for definitionNN, definitionInfo := range c.definitionInfo {
var bindingInfos []bindingInfo
for bindingNN := range c.definitionsToBindings[definitionNN] {
bindingInfo := c.bindingInfos[bindingNN]
if bindingInfo.validator == nil && definitionInfo.configurationError == nil {
hasParam := false
if definitionInfo.lastReconciledValue.Spec.ParamKind != nil {
hasParam = true
}
optionalVars := cel.OptionalVariableDeclarations{HasParams: hasParam, HasAuthorizer: true}
expressionOptionalVars := cel.OptionalVariableDeclarations{HasParams: hasParam, HasAuthorizer: false}
failurePolicy := convertv1beta1FailurePolicyTypeTov1FailurePolicyType(definitionInfo.lastReconciledValue.Spec.FailurePolicy)
var matcher matchconditions.Matcher = nil
matchConditions := definitionInfo.lastReconciledValue.Spec.MatchConditions
filterCompiler := c.filterCompiler
if filterCompiler == nil {
compositedCompiler, err := cel.NewCompositedCompiler(environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion()))
if err == nil {
filterCompiler = compositedCompiler
compositedCompiler.CompileAndStoreVariables(convertv1beta1Variables(definitionInfo.lastReconciledValue.Spec.Variables), optionalVars, environment.StoredExpressions)
} else {
utilruntime.HandleError(err)
}
}
if len(matchConditions) > 0 {
matchExpressionAccessors := make([]cel.ExpressionAccessor, len(matchConditions))
for i := range matchConditions {
matchExpressionAccessors[i] = (*matchconditions.MatchCondition)(&matchConditions[i])
}
matcher = matchconditions.NewMatcher(filterCompiler.Compile(matchExpressionAccessors, optionalVars, environment.StoredExpressions), failurePolicy, "policy", "validate", definitionInfo.lastReconciledValue.Name)
}
bindingInfo.validator = c.newValidator(
filterCompiler.Compile(convertv1beta1Validations(definitionInfo.lastReconciledValue.Spec.Validations), optionalVars, environment.StoredExpressions),
matcher,
filterCompiler.Compile(convertv1beta1AuditAnnotations(definitionInfo.lastReconciledValue.Spec.AuditAnnotations), optionalVars, environment.StoredExpressions),
filterCompiler.Compile(convertv1beta1MessageExpressions(definitionInfo.lastReconciledValue.Spec.Validations), expressionOptionalVars, environment.StoredExpressions),
failurePolicy,
)
}
bindingInfos = append(bindingInfos, *bindingInfo)
}
var pInfo paramInfo
if paramKind := definitionInfo.lastReconciledValue.Spec.ParamKind; paramKind != nil {
if info, ok := c.paramsCRDControllers[*paramKind]; ok {
pInfo = *info
}
}
res = append(res, policyData{
definitionInfo: *definitionInfo,
paramInfo: pInfo,
bindings: bindingInfos,
})
}
c.cachedPolicies = res
return res
}
func convertv1beta1FailurePolicyTypeTov1FailurePolicyType(policyType *v1beta1.FailurePolicyType) *v1.FailurePolicyType {
if policyType == nil {
return nil
}
var v1FailPolicy v1.FailurePolicyType
if *policyType == v1beta1.Fail {
v1FailPolicy = v1.Fail
} else if *policyType == v1beta1.Ignore {
v1FailPolicy = v1.Ignore
}
return &v1FailPolicy
}
func convertv1beta1Validations(inputValidations []v1beta1.Validation) []cel.ExpressionAccessor {
celExpressionAccessor := make([]cel.ExpressionAccessor, len(inputValidations))
for i, validation := range inputValidations {
validation := ValidationCondition{
Expression: validation.Expression,
Message: validation.Message,
Reason: validation.Reason,
}
celExpressionAccessor[i] = &validation
}
return celExpressionAccessor
}
func convertv1beta1MessageExpressions(inputValidations []v1beta1.Validation) []cel.ExpressionAccessor {
celExpressionAccessor := make([]cel.ExpressionAccessor, len(inputValidations))
for i, validation := range inputValidations {
if validation.MessageExpression != "" {
condition := MessageExpressionCondition{
MessageExpression: validation.MessageExpression,
}
celExpressionAccessor[i] = &condition
}
}
return celExpressionAccessor
}
func convertv1beta1AuditAnnotations(inputValidations []v1beta1.AuditAnnotation) []cel.ExpressionAccessor {
celExpressionAccessor := make([]cel.ExpressionAccessor, len(inputValidations))
for i, validation := range inputValidations {
validation := AuditAnnotationCondition{
Key: validation.Key,
ValueExpression: validation.ValueExpression,
}
celExpressionAccessor[i] = &validation
}
return celExpressionAccessor
}
func convertv1beta1Variables(variables []v1beta1.Variable) []cel.NamedExpressionAccessor {
namedExpressions := make([]cel.NamedExpressionAccessor, len(variables))
for i, variable := range variables {
namedExpressions[i] = &Variable{Name: variable.Name, Expression: variable.Expression}
}
return namedExpressions
}
func getNamespaceName(namespace, name string) namespacedName {
return namespacedName{
namespace: namespace,

View File

@@ -18,6 +18,7 @@ package validatingadmissionpolicy
import (
"context"
"k8s.io/apiserver/pkg/admission"
)

View File

@@ -17,34 +17,97 @@ limitations under the License.
package validatingadmissionpolicy
import (
"k8s.io/api/admissionregistration/v1alpha1"
"context"
celgo "github.com/google/cel-go/cel"
"k8s.io/api/admissionregistration/v1beta1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/cel"
"k8s.io/apiserver/pkg/authorization/authorizer"
)
// Validator defines the func used to validate the cel expressions
// matchKind provides the GroupVersionKind that the object should be
// validated by CEL expressions as.
type Validator interface {
Validate(a admission.Attributes, o admission.ObjectInterfaces, versionedParams runtime.Object, matchKind schema.GroupVersionKind) ([]policyDecision, error)
var _ cel.ExpressionAccessor = &ValidationCondition{}
// ValidationCondition contains the inputs needed to compile, evaluate and validate a cel expression
type ValidationCondition struct {
Expression string
Message string
Reason *metav1.StatusReason
}
// ValidatorCompiler is Dependency Injected into the PolicyDefinition's `Compile`
// function to assist with converting types and values to/from CEL-typed values.
type ValidatorCompiler interface {
func (v *ValidationCondition) GetExpression() string {
return v.Expression
}
func (v *ValidationCondition) ReturnTypes() []*celgo.Type {
return []*celgo.Type{celgo.BoolType}
}
// AuditAnnotationCondition contains the inputs needed to compile, evaluate and publish a cel audit annotation
type AuditAnnotationCondition struct {
Key string
ValueExpression string
}
func (v *AuditAnnotationCondition) GetExpression() string {
return v.ValueExpression
}
func (v *AuditAnnotationCondition) ReturnTypes() []*celgo.Type {
return []*celgo.Type{celgo.StringType, celgo.NullType}
}
// Variable is a named expression for composition.
type Variable struct {
Name string
Expression string
}
func (v *Variable) GetExpression() string {
return v.Expression
}
func (v *Variable) ReturnTypes() []*celgo.Type {
return []*celgo.Type{celgo.AnyType, celgo.DynType}
}
func (v *Variable) GetName() string {
return v.Name
}
// Matcher is used for matching ValidatingAdmissionPolicy and ValidatingAdmissionPolicyBinding to attributes
type Matcher interface {
admission.InitializationValidator
// Matches says whether this policy definition matches the provided admission
// DefinitionMatches says whether this policy definition matches the provided admission
// resource request
DefinitionMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1alpha1.ValidatingAdmissionPolicy) (bool, schema.GroupVersionKind, error)
DefinitionMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1beta1.ValidatingAdmissionPolicy) (bool, schema.GroupVersionResource, schema.GroupVersionKind, error)
// Matches says whether this policy definition matches the provided admission
// BindingMatches says whether this policy definition matches the provided admission
// resource request
BindingMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1alpha1.ValidatingAdmissionPolicyBinding) (bool, error)
BindingMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1beta1.ValidatingAdmissionPolicyBinding) (bool, error)
// Compile is used for the cel expression compilation
Compile(
policy *v1alpha1.ValidatingAdmissionPolicy,
) Validator
// GetNamespace retrieves the Namespace resource by the given name. The name may be empty, in which case
// GetNamespace must return nil, nil
GetNamespace(name string) (*corev1.Namespace, error)
}
// ValidateResult defines the result of a Validator.Validate operation.
type ValidateResult struct {
// Decisions specifies the outcome of the validation as well as the details about the decision.
Decisions []PolicyDecision
// AuditAnnotations specifies the audit annotations that should be recorded for the validation.
AuditAnnotations []PolicyAuditAnnotation
}
// Validator is contains logic for converting ValidationEvaluation to PolicyDecisions
type Validator interface {
// Validate is used to take cel evaluations and convert into decisions
// runtimeCELCostBudget was added for testing purpose only. Callers should always use const RuntimeCELCostBudget from k8s.io/apiserver/pkg/apis/cel/config.go as input.
Validate(ctx context.Context, matchedResource schema.GroupVersionResource, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, namespace *corev1.Namespace, runtimeCELCostBudget int64, authz authorizer.Authorizer) ValidateResult
}

View File

@@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -30,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/cache/synctrack"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog/v2"
)
@@ -45,6 +47,11 @@ type controller[T runtime.Object] struct {
reconciler func(namespace, name string, newObj T) error
options ControllerOptions
// must hold a func() bool or nil
notificationsDelivered atomic.Value
hasProcessed synctrack.AsyncTracker[string]
}
type ControllerOptions struct {
@@ -69,12 +76,20 @@ func NewController[T runtime.Object](
options.Name = fmt.Sprintf("%T-controller", *new(T))
}
return &controller[T]{
c := &controller[T]{
options: options,
informer: informer,
reconciler: reconciler,
queue: nil,
}
c.hasProcessed.UpstreamHasSynced = func() bool {
f := c.notificationsDelivered.Load()
if f == nil {
return false
}
return f.(func() bool)()
}
return c
}
// Runs the controller and returns an error explaining why running was stopped.
@@ -92,20 +107,22 @@ func (c *controller[T]) Run(ctx context.Context) error {
// would never shut down the workqueue
defer c.queue.ShutDown()
enqueue := func(obj interface{}) {
enqueue := func(obj interface{}, isInInitialList bool) {
var key string
var err error
if key, err = cache.DeletionHandlingMetaNamespaceKeyFunc(obj); err != nil {
utilruntime.HandleError(err)
return
}
if isInInitialList {
c.hasProcessed.Start(key)
}
c.queue.Add(key)
}
registration, err := c.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
enqueue(obj)
},
registration, err := c.informer.AddEventHandler(cache.ResourceEventHandlerDetailedFuncs{
AddFunc: enqueue,
UpdateFunc: func(oldObj, newObj interface{}) {
oldMeta, err1 := meta.Accessor(oldObj)
newMeta, err2 := meta.Accessor(newObj)
@@ -126,11 +143,11 @@ func (c *controller[T]) Run(ctx context.Context) error {
return
}
enqueue(newObj)
enqueue(newObj, false)
},
DeleteFunc: func(obj interface{}) {
// Enqueue
enqueue(obj)
enqueue(obj, false)
},
})
@@ -139,9 +156,12 @@ func (c *controller[T]) Run(ctx context.Context) error {
return err
}
c.notificationsDelivered.Store(registration.HasSynced)
// Make sure event handler is removed from informer in case return early from
// an error
defer func() {
c.notificationsDelivered.Store(func() bool { return false })
// Remove event handler and Handle Error here. Error should only be raised
// for improper usage of event handler API.
if err := c.informer.RemoveEventHandler(registration); err != nil {
@@ -166,8 +186,8 @@ func (c *controller[T]) Run(ctx context.Context) error {
for i := uint(0); i < c.options.Workers; i++ {
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
wait.Until(c.runWorker, time.Second, ctx.Done())
waitGroup.Done()
}()
}
@@ -188,7 +208,7 @@ func (c *controller[T]) Run(ctx context.Context) error {
}
func (c *controller[T]) HasSynced() bool {
return c.informer.HasSynced()
return c.hasProcessed.HasSynced()
}
func (c *controller[T]) runWorker() {
@@ -220,6 +240,7 @@ func (c *controller[T]) runWorker() {
// but the key is invalid so there is no point in doing that)
return fmt.Errorf("expected string in workqueue but got %#v", obj)
}
defer c.hasProcessed.Finished(key)
if err := c.reconcile(key); err != nil {
// Put the item back on the workqueue to handle any transient errors.

View File

@@ -0,0 +1,83 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"k8s.io/api/admissionregistration/v1beta1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/matching"
)
var _ matching.MatchCriteria = &matchCriteria{}
type matchCriteria struct {
constraints *v1beta1.MatchResources
}
// GetParsedNamespaceSelector returns the converted LabelSelector which implements labels.Selector
func (m *matchCriteria) GetParsedNamespaceSelector() (labels.Selector, error) {
return metav1.LabelSelectorAsSelector(m.constraints.NamespaceSelector)
}
// GetParsedObjectSelector returns the converted LabelSelector which implements labels.Selector
func (m *matchCriteria) GetParsedObjectSelector() (labels.Selector, error) {
return metav1.LabelSelectorAsSelector(m.constraints.ObjectSelector)
}
// GetMatchResources returns the matchConstraints
func (m *matchCriteria) GetMatchResources() v1beta1.MatchResources {
return *m.constraints
}
type matcher struct {
Matcher *matching.Matcher
}
func NewMatcher(m *matching.Matcher) Matcher {
return &matcher{
Matcher: m,
}
}
// ValidateInitialization checks if Matcher is initialized.
func (c *matcher) ValidateInitialization() error {
return c.Matcher.ValidateInitialization()
}
// DefinitionMatches returns whether this ValidatingAdmissionPolicy matches the provided admission resource request
func (c *matcher) DefinitionMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1beta1.ValidatingAdmissionPolicy) (bool, schema.GroupVersionResource, schema.GroupVersionKind, error) {
criteria := matchCriteria{constraints: definition.Spec.MatchConstraints}
return c.Matcher.Matches(a, o, &criteria)
}
// BindingMatches returns whether this ValidatingAdmissionPolicyBinding matches the provided admission resource request
func (c *matcher) BindingMatches(a admission.Attributes, o admission.ObjectInterfaces, binding *v1beta1.ValidatingAdmissionPolicyBinding) (bool, error) {
if binding.Spec.MatchResources == nil {
return true, nil
}
criteria := matchCriteria{constraints: binding.Spec.MatchResources}
isMatch, _, _, err := c.Matcher.Matches(a, o, &criteria)
return isMatch, err
}
func (c *matcher) GetNamespace(name string) (*corev1.Namespace, error) {
return c.Matcher.GetNamespace(name)
}

View File

@@ -20,7 +20,8 @@ import (
"fmt"
v1 "k8s.io/api/admissionregistration/v1"
"k8s.io/api/admissionregistration/v1alpha1"
"k8s.io/api/admissionregistration/v1beta1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/client-go/kubernetes"
@@ -35,7 +36,7 @@ type MatchCriteria interface {
namespace.NamespaceSelectorProvider
object.ObjectSelectorProvider
GetMatchResources() v1alpha1.MatchResources
GetMatchResources() v1beta1.MatchResources
}
// Matcher decides if a request matches against matchCriteria
@@ -44,6 +45,10 @@ type Matcher struct {
objectMatcher *object.Matcher
}
func (m *Matcher) GetNamespace(name string) (*corev1.Namespace, error) {
return m.namespaceMatcher.GetNamespace(name)
}
// NewMatcher initialize the matcher with dependencies requires
func NewMatcher(
namespaceLister listersv1.NamespaceLister,
@@ -66,56 +71,60 @@ func (m *Matcher) ValidateInitialization() error {
return nil
}
func (m *Matcher) Matches(attr admission.Attributes, o admission.ObjectInterfaces, criteria MatchCriteria) (bool, schema.GroupVersionKind, error) {
func (m *Matcher) Matches(attr admission.Attributes, o admission.ObjectInterfaces, criteria MatchCriteria) (bool, schema.GroupVersionResource, schema.GroupVersionKind, error) {
matches, matchNsErr := m.namespaceMatcher.MatchNamespaceSelector(criteria, attr)
// Should not return an error here for policy which do not apply to the request, even if err is an unexpected scenario.
if !matches && matchNsErr == nil {
return false, schema.GroupVersionKind{}, nil
return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, nil
}
matches, matchObjErr := m.objectMatcher.MatchObjectSelector(criteria, attr)
// Should not return an error here for policy which do not apply to the request, even if err is an unexpected scenario.
if !matches && matchObjErr == nil {
return false, schema.GroupVersionKind{}, nil
return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, nil
}
matchResources := criteria.GetMatchResources()
matchPolicy := matchResources.MatchPolicy
if isExcluded, _, err := matchesResourceRules(matchResources.ExcludeResourceRules, matchPolicy, attr, o); isExcluded || err != nil {
return false, schema.GroupVersionKind{}, err
if isExcluded, _, _, err := matchesResourceRules(matchResources.ExcludeResourceRules, matchPolicy, attr, o); isExcluded || err != nil {
return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, err
}
var (
isMatch bool
matchKind schema.GroupVersionKind
matchErr error
isMatch bool
matchResource schema.GroupVersionResource
matchKind schema.GroupVersionKind
matchErr error
)
if len(matchResources.ResourceRules) == 0 {
isMatch = true
matchKind = attr.GetKind()
matchResource = attr.GetResource()
} else {
isMatch, matchKind, matchErr = matchesResourceRules(matchResources.ResourceRules, matchPolicy, attr, o)
isMatch, matchResource, matchKind, matchErr = matchesResourceRules(matchResources.ResourceRules, matchPolicy, attr, o)
}
if matchErr != nil {
return false, schema.GroupVersionKind{}, matchErr
return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, matchErr
}
if !isMatch {
return false, schema.GroupVersionKind{}, nil
return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, nil
}
// now that we know this applies to this request otherwise, if there were selector errors, return them
if matchNsErr != nil {
return false, schema.GroupVersionKind{}, matchNsErr
return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, matchNsErr
}
if matchObjErr != nil {
return false, schema.GroupVersionKind{}, matchObjErr
return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, matchObjErr
}
return true, matchKind, nil
return true, matchResource, matchKind, nil
}
func matchesResourceRules(namedRules []v1alpha1.NamedRuleWithOperations, matchPolicy *v1alpha1.MatchPolicyType, attr admission.Attributes, o admission.ObjectInterfaces) (bool, schema.GroupVersionKind, error) {
func matchesResourceRules(namedRules []v1beta1.NamedRuleWithOperations, matchPolicy *v1beta1.MatchPolicyType, attr admission.Attributes, o admission.ObjectInterfaces) (bool, schema.GroupVersionResource, schema.GroupVersionKind, error) {
matchKind := attr.GetKind()
matchResource := attr.GetResource()
for _, namedRule := range namedRules {
rule := v1.RuleWithOperations(namedRule.RuleWithOperations)
ruleMatcher := rules.Matcher{
@@ -127,22 +136,22 @@ func matchesResourceRules(namedRules []v1alpha1.NamedRuleWithOperations, matchPo
}
// an empty name list always matches
if len(namedRule.ResourceNames) == 0 {
return true, matchKind, nil
return true, matchResource, matchKind, nil
}
// TODO: GetName() can return an empty string if the user is relying on
// the API server to generate the name... figure out what to do for this edge case
name := attr.GetName()
for _, matchedName := range namedRule.ResourceNames {
if name == matchedName {
return true, matchKind, nil
return true, matchResource, matchKind, nil
}
}
}
// if match policy is undefined or exact, don't perform fuzzy matching
// note that defaulting to fuzzy matching is set by the API
if matchPolicy == nil || *matchPolicy == v1alpha1.Exact {
return false, schema.GroupVersionKind{}, nil
if matchPolicy == nil || *matchPolicy == v1beta1.Exact {
return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, nil
}
attrWithOverride := &attrWithResourceOverride{Attributes: attr}
@@ -164,11 +173,11 @@ func matchesResourceRules(namedRules []v1alpha1.NamedRuleWithOperations, matchPo
}
matchKind = o.GetEquivalentResourceMapper().KindFor(equivalent, attr.GetSubresource())
if matchKind.Empty() {
return false, schema.GroupVersionKind{}, fmt.Errorf("unable to convert to %v: unknown kind", equivalent)
return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, fmt.Errorf("unable to convert to %v: unknown kind", equivalent)
}
// an empty name list always matches
if len(namedRule.ResourceNames) == 0 {
return true, matchKind, nil
return true, equivalent, matchKind, nil
}
// TODO: GetName() can return an empty string if the user is relying on
@@ -176,12 +185,12 @@ func matchesResourceRules(namedRules []v1alpha1.NamedRuleWithOperations, matchPo
name := attr.GetName()
for _, matchedName := range namedRule.ResourceNames {
if name == matchedName {
return true, matchKind, nil
return true, equivalent, matchKind, nil
}
}
}
}
return false, schema.GroupVersionKind{}, nil
return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, nil
}
type attrWithResourceOverride struct {

View File

@@ -0,0 +1,36 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
celgo "github.com/google/cel-go/cel"
"k8s.io/apiserver/pkg/admission/plugin/cel"
)
var _ cel.ExpressionAccessor = (*MessageExpressionCondition)(nil)
type MessageExpressionCondition struct {
MessageExpression string
}
func (m *MessageExpressionCondition) GetExpression() string {
return m.MessageExpression
}
func (m *MessageExpressionCondition) ReturnTypes() []*celgo.Type {
return []*celgo.Type{celgo.StringType}
}

View File

@@ -20,37 +20,54 @@ import (
"net/http"
"time"
"k8s.io/api/admissionregistration/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type policyDecisionAction string
type PolicyDecisionAction string
const (
actionAdmit policyDecisionAction = "admit"
actionDeny policyDecisionAction = "deny"
ActionAdmit PolicyDecisionAction = "admit"
ActionDeny PolicyDecisionAction = "deny"
)
type policyDecisionEvaluation string
type PolicyDecisionEvaluation string
const (
evalAdmit policyDecisionEvaluation = "admit"
evalError policyDecisionEvaluation = "error"
evalDeny policyDecisionEvaluation = "deny"
EvalAdmit PolicyDecisionEvaluation = "admit"
EvalError PolicyDecisionEvaluation = "error"
EvalDeny PolicyDecisionEvaluation = "deny"
)
type policyDecision struct {
action policyDecisionAction
evaluation policyDecisionEvaluation
message string
reason metav1.StatusReason
elapsed time.Duration
// PolicyDecision contains the action determined from a cel evaluation along with metadata such as message, reason and duration
type PolicyDecision struct {
Action PolicyDecisionAction
Evaluation PolicyDecisionEvaluation
Message string
Reason metav1.StatusReason
Elapsed time.Duration
}
type policyDecisionWithMetadata struct {
policyDecision
definition *v1alpha1.ValidatingAdmissionPolicy
binding *v1alpha1.ValidatingAdmissionPolicyBinding
type PolicyAuditAnnotationAction string
const (
// AuditAnnotationActionPublish indicates that the audit annotation should be
// published with the audit event.
AuditAnnotationActionPublish PolicyAuditAnnotationAction = "publish"
// AuditAnnotationActionError indicates that the valueExpression resulted
// in an error.
AuditAnnotationActionError PolicyAuditAnnotationAction = "error"
// AuditAnnotationActionExclude indicates that the audit annotation should be excluded
// because the valueExpression evaluated to null, or because FailurePolicy is Ignore
// and the expression failed with a parse error, type check error, or runtime error.
AuditAnnotationActionExclude PolicyAuditAnnotationAction = "exclude"
)
type PolicyAuditAnnotation struct {
Key string
Value string
Elapsed time.Duration
Action PolicyAuditAnnotationAction
Error string
}
func reasonToCode(r metav1.StatusReason) int32 {

View File

@@ -0,0 +1,423 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/google/cel-go/cel"
"k8s.io/api/admissionregistration/v1beta1"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apimachinery/pkg/util/version"
plugincel "k8s.io/apiserver/pkg/admission/plugin/cel"
apiservercel "k8s.io/apiserver/pkg/cel"
"k8s.io/apiserver/pkg/cel/common"
"k8s.io/apiserver/pkg/cel/environment"
"k8s.io/apiserver/pkg/cel/library"
"k8s.io/apiserver/pkg/cel/openapi"
"k8s.io/apiserver/pkg/cel/openapi/resolver"
"k8s.io/klog/v2"
)
const maxTypesToCheck = 10
type TypeChecker struct {
SchemaResolver resolver.SchemaResolver
RestMapper meta.RESTMapper
}
// TypeCheckingContext holds information about the policy being type-checked.
// The struct is opaque to the caller.
type TypeCheckingContext struct {
gvks []schema.GroupVersionKind
declTypes []*apiservercel.DeclType
paramGVK schema.GroupVersionKind
paramDeclType *apiservercel.DeclType
}
type typeOverwrite struct {
object *apiservercel.DeclType
params *apiservercel.DeclType
}
// TypeCheckingResult holds the issues found during type checking, any returned
// error, and the gvk that the type checking is performed against.
type TypeCheckingResult struct {
// GVK is the associated GVK
GVK schema.GroupVersionKind
// Issues contain machine-readable information about the typechecking result.
Issues *cel.Issues
// Err is the possible error that was encounter during type checking.
Err error
}
// TypeCheckingResults is a collection of TypeCheckingResult
type TypeCheckingResults []*TypeCheckingResult
func (rs TypeCheckingResults) String() string {
var messages []string
for _, r := range rs {
message := r.String()
if message != "" {
messages = append(messages, message)
}
}
return strings.Join(messages, "\n")
}
// String converts the result to human-readable form as a string.
func (r *TypeCheckingResult) String() string {
if r.Issues == nil && r.Err == nil {
return ""
}
if r.Err != nil {
return fmt.Sprintf("%v: type checking error: %v\n", r.GVK, r.Err)
}
return fmt.Sprintf("%v: %s\n", r.GVK, r.Issues)
}
// Check preforms the type check against the given policy, and format the result
// as []ExpressionWarning that is ready to be set in policy.Status
// The result is nil if type checking returns no warning.
// The policy object is NOT mutated. The caller should update Status accordingly
func (c *TypeChecker) Check(policy *v1beta1.ValidatingAdmissionPolicy) []v1beta1.ExpressionWarning {
ctx := c.CreateContext(policy)
// warnings to return, note that the capacity is optimistically set to zero
var warnings []v1beta1.ExpressionWarning // intentionally not setting capacity
// check main validation expressions and their message expressions, located in spec.validations[*]
fieldRef := field.NewPath("spec", "validations")
for i, v := range policy.Spec.Validations {
results := c.CheckExpression(ctx, v.Expression)
if len(results) != 0 {
warnings = append(warnings, v1beta1.ExpressionWarning{
FieldRef: fieldRef.Index(i).Child("expression").String(),
Warning: results.String(),
})
}
// Note that MessageExpression is optional
if v.MessageExpression == "" {
continue
}
results = c.CheckExpression(ctx, v.MessageExpression)
if len(results) != 0 {
warnings = append(warnings, v1beta1.ExpressionWarning{
FieldRef: fieldRef.Index(i).Child("messageExpression").String(),
Warning: results.String(),
})
}
}
return warnings
}
// CreateContext resolves all types and their schemas from a policy definition and creates the context.
func (c *TypeChecker) CreateContext(policy *v1beta1.ValidatingAdmissionPolicy) *TypeCheckingContext {
ctx := new(TypeCheckingContext)
allGvks := c.typesToCheck(policy)
gvks := make([]schema.GroupVersionKind, 0, len(allGvks))
declTypes := make([]*apiservercel.DeclType, 0, len(allGvks))
for _, gvk := range allGvks {
declType, err := c.declType(gvk)
if err != nil {
// type checking errors MUST NOT alter the behavior of the policy
// even if an error occurs.
if !errors.Is(err, resolver.ErrSchemaNotFound) {
// Anything except ErrSchemaNotFound is an internal error
klog.V(2).ErrorS(err, "internal error: schema resolution failure", "gvk", gvk)
}
// skip for not found or internal error
continue
}
gvks = append(gvks, gvk)
declTypes = append(declTypes, declType)
}
ctx.gvks = gvks
ctx.declTypes = declTypes
paramsGVK := c.paramsGVK(policy) // maybe empty, correctly handled
paramsDeclType, err := c.declType(paramsGVK)
if err != nil {
if !errors.Is(err, resolver.ErrSchemaNotFound) {
klog.V(2).ErrorS(err, "internal error: cannot resolve schema for params", "gvk", paramsGVK)
}
paramsDeclType = nil
}
ctx.paramGVK = paramsGVK
ctx.paramDeclType = paramsDeclType
return ctx
}
// CheckExpression type checks a single expression, given the context
func (c *TypeChecker) CheckExpression(ctx *TypeCheckingContext, expression string) TypeCheckingResults {
var results TypeCheckingResults
for i, gvk := range ctx.gvks {
declType := ctx.declTypes[i]
// TODO(jiahuif) hasAuthorizer always true for now, will change after expending type checking to all fields.
issues, err := c.checkExpression(expression, ctx.paramDeclType != nil, true, typeOverwrite{
object: declType,
params: ctx.paramDeclType,
})
if issues != nil || err != nil {
results = append(results, &TypeCheckingResult{Issues: issues, Err: err, GVK: gvk})
}
}
return results
}
func generateUniqueTypeName(kind string) string {
return fmt.Sprintf("%s%d", kind, time.Now().Nanosecond())
}
func (c *TypeChecker) declType(gvk schema.GroupVersionKind) (*apiservercel.DeclType, error) {
if gvk.Empty() {
return nil, nil
}
s, err := c.SchemaResolver.ResolveSchema(gvk)
if err != nil {
return nil, err
}
return common.SchemaDeclType(&openapi.Schema{Schema: s}, true).MaybeAssignTypeName(generateUniqueTypeName(gvk.Kind)), nil
}
func (c *TypeChecker) paramsGVK(policy *v1beta1.ValidatingAdmissionPolicy) schema.GroupVersionKind {
if policy.Spec.ParamKind == nil {
return schema.GroupVersionKind{}
}
gv, err := schema.ParseGroupVersion(policy.Spec.ParamKind.APIVersion)
if err != nil {
return schema.GroupVersionKind{}
}
return gv.WithKind(policy.Spec.ParamKind.Kind)
}
func (c *TypeChecker) checkExpression(expression string, hasParams, hasAuthorizer bool, types typeOverwrite) (*cel.Issues, error) {
env, err := buildEnv(hasParams, hasAuthorizer, types)
if err != nil {
return nil, err
}
// We cannot reuse an AST that is parsed by another env, so reparse it here.
// Compile = Parse + Check, we especially want the results of Check.
//
// Paradoxically, we discard the type-checked result and let the admission
// controller use the dynamic typed program.
// This is a compromise that is defined in the KEP. We can revisit this
// decision and expect a change with limited size.
_, issues := env.Compile(expression)
return issues, nil
}
// typesToCheck extracts a list of GVKs that needs type checking from the policy
// the result is sorted in the order of Group, Version, and Kind
func (c *TypeChecker) typesToCheck(p *v1beta1.ValidatingAdmissionPolicy) []schema.GroupVersionKind {
gvks := sets.New[schema.GroupVersionKind]()
if p.Spec.MatchConstraints == nil || len(p.Spec.MatchConstraints.ResourceRules) == 0 {
return nil
}
restMapperRefreshAttempted := false // at most once per policy, refresh RESTMapper and retry resolution.
for _, rule := range p.Spec.MatchConstraints.ResourceRules {
groups := extractGroups(&rule.Rule)
if len(groups) == 0 {
continue
}
versions := extractVersions(&rule.Rule)
if len(versions) == 0 {
continue
}
resources := extractResources(&rule.Rule)
if len(resources) == 0 {
continue
}
// sort GVRs so that the loop below provides
// consistent results.
sort.Strings(groups)
sort.Strings(versions)
sort.Strings(resources)
count := 0
for _, group := range groups {
for _, version := range versions {
for _, resource := range resources {
gvr := schema.GroupVersionResource{
Group: group,
Version: version,
Resource: resource,
}
resolved, err := c.RestMapper.KindsFor(gvr)
if err != nil {
if restMapperRefreshAttempted {
// RESTMapper refresh happens at most once per policy
continue
}
c.tryRefreshRESTMapper()
restMapperRefreshAttempted = true
resolved, err = c.RestMapper.KindsFor(gvr)
if err != nil {
continue
}
}
for _, r := range resolved {
if !r.Empty() {
gvks.Insert(r)
count++
// early return if maximum number of types are already
// collected
if count == maxTypesToCheck {
if gvks.Len() == 0 {
return nil
}
return sortGVKList(gvks.UnsortedList())
}
}
}
}
}
}
}
if gvks.Len() == 0 {
return nil
}
return sortGVKList(gvks.UnsortedList())
}
func extractGroups(rule *v1beta1.Rule) []string {
groups := make([]string, 0, len(rule.APIGroups))
for _, group := range rule.APIGroups {
// give up if wildcard
if strings.ContainsAny(group, "*") {
return nil
}
groups = append(groups, group)
}
return groups
}
func extractVersions(rule *v1beta1.Rule) []string {
versions := make([]string, 0, len(rule.APIVersions))
for _, version := range rule.APIVersions {
if strings.ContainsAny(version, "*") {
return nil
}
versions = append(versions, version)
}
return versions
}
func extractResources(rule *v1beta1.Rule) []string {
resources := make([]string, 0, len(rule.Resources))
for _, resource := range rule.Resources {
// skip wildcard and subresources
if strings.ContainsAny(resource, "*/") {
continue
}
resources = append(resources, resource)
}
return resources
}
// sortGVKList sorts the list by Group, Version, and Kind
// returns the list itself.
func sortGVKList(list []schema.GroupVersionKind) []schema.GroupVersionKind {
sort.Slice(list, func(i, j int) bool {
if g := strings.Compare(list[i].Group, list[j].Group); g != 0 {
return g < 0
}
if v := strings.Compare(list[i].Version, list[j].Version); v != 0 {
return v < 0
}
return strings.Compare(list[i].Kind, list[j].Kind) < 0
})
return list
}
// tryRefreshRESTMapper refreshes the RESTMapper if it supports refreshing.
func (c *TypeChecker) tryRefreshRESTMapper() {
if r, ok := c.RestMapper.(meta.ResettableRESTMapper); ok {
r.Reset()
}
}
func buildEnv(hasParams bool, hasAuthorizer bool, types typeOverwrite) (*cel.Env, error) {
baseEnv := environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion())
requestType := plugincel.BuildRequestType()
namespaceType := plugincel.BuildNamespaceType()
var varOpts []cel.EnvOption
var declTypes []*apiservercel.DeclType
// namespace, hand-crafted type
declTypes = append(declTypes, namespaceType)
varOpts = append(varOpts, createVariableOpts(namespaceType, plugincel.NamespaceVarName)...)
// request, hand-crafted type
declTypes = append(declTypes, requestType)
varOpts = append(varOpts, createVariableOpts(requestType, plugincel.RequestVarName)...)
// object and oldObject, same type, type(s) resolved from constraints
declTypes = append(declTypes, types.object)
varOpts = append(varOpts, createVariableOpts(types.object, plugincel.ObjectVarName, plugincel.OldObjectVarName)...)
// params, defined by ParamKind
if hasParams && types.params != nil {
declTypes = append(declTypes, types.params)
varOpts = append(varOpts, createVariableOpts(types.params, plugincel.ParamsVarName)...)
}
// authorizer, implicitly available to all expressions of a policy
if hasAuthorizer {
// we only need its structure but not the variable itself
varOpts = append(varOpts, cel.Variable("authorizer", library.AuthorizerType))
}
env, err := baseEnv.Extend(
environment.VersionedOptions{
// Feature epoch was actually 1.26, but we artificially set it to 1.0 because these
// options should always be present.
IntroducedVersion: version.MajorMinor(1, 0),
EnvOptions: varOpts,
DeclTypes: declTypes,
},
)
if err != nil {
return nil, err
}
return env.Env(environment.StoredExpressions)
}
// createVariableOpts creates a slice of EnvOption
// that can be used for creating a CEL env containing variables of declType.
// declType can be nil, in which case the variables will be of DynType.
func createVariableOpts(declType *apiservercel.DeclType, variables ...string) []cel.EnvOption {
opts := make([]cel.EnvOption, 0, len(variables))
t := cel.DynType
if declType != nil {
t = declType.CelType()
}
for _, v := range variables {
opts = append(opts, cel.Variable(v, t))
}
return opts
}

View File

@@ -17,302 +17,235 @@ limitations under the License.
package validatingadmissionpolicy
import (
"context"
"fmt"
"reflect"
"strings"
"time"
celtypes "github.com/google/cel-go/common/types"
"github.com/google/cel-go/interpreter"
admissionv1 "k8s.io/api/admission/v1"
"k8s.io/api/admissionregistration/v1alpha1"
authenticationv1 "k8s.io/api/authentication/v1"
v1 "k8s.io/api/admissionregistration/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/matching"
"k8s.io/apiserver/pkg/admission/plugin/webhook/generic"
"k8s.io/apiserver/pkg/admission/plugin/cel"
"k8s.io/apiserver/pkg/admission/plugin/webhook/matchconditions"
celconfig "k8s.io/apiserver/pkg/apis/cel"
"k8s.io/apiserver/pkg/authorization/authorizer"
apiservercel "k8s.io/apiserver/pkg/cel"
"k8s.io/klog/v2"
)
var _ ValidatorCompiler = &CELValidatorCompiler{}
var _ matching.MatchCriteria = &matchCriteria{}
type matchCriteria struct {
constraints *v1alpha1.MatchResources
// validator implements the Validator interface
type validator struct {
celMatcher matchconditions.Matcher
validationFilter cel.Filter
auditAnnotationFilter cel.Filter
messageFilter cel.Filter
failPolicy *v1.FailurePolicyType
}
// GetParsedNamespaceSelector returns the converted LabelSelector which implements labels.Selector
func (m *matchCriteria) GetParsedNamespaceSelector() (labels.Selector, error) {
return metav1.LabelSelectorAsSelector(m.constraints.NamespaceSelector)
}
// GetParsedObjectSelector returns the converted LabelSelector which implements labels.Selector
func (m *matchCriteria) GetParsedObjectSelector() (labels.Selector, error) {
return metav1.LabelSelectorAsSelector(m.constraints.ObjectSelector)
}
// GetMatchResources returns the matchConstraints
func (m *matchCriteria) GetMatchResources() v1alpha1.MatchResources {
return *m.constraints
}
// CELValidatorCompiler implement the interface ValidatorCompiler.
type CELValidatorCompiler struct {
Matcher *matching.Matcher
}
// DefinitionMatches returns whether this ValidatingAdmissionPolicy matches the provided admission resource request
func (c *CELValidatorCompiler) DefinitionMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1alpha1.ValidatingAdmissionPolicy) (bool, schema.GroupVersionKind, error) {
criteria := matchCriteria{constraints: definition.Spec.MatchConstraints}
return c.Matcher.Matches(a, o, &criteria)
}
// BindingMatches returns whether this ValidatingAdmissionPolicyBinding matches the provided admission resource request
func (c *CELValidatorCompiler) BindingMatches(a admission.Attributes, o admission.ObjectInterfaces, binding *v1alpha1.ValidatingAdmissionPolicyBinding) (bool, error) {
if binding.Spec.MatchResources == nil {
return true, nil
}
criteria := matchCriteria{constraints: binding.Spec.MatchResources}
isMatch, _, err := c.Matcher.Matches(a, o, &criteria)
return isMatch, err
}
// ValidateInitialization checks if Matcher is initialized.
func (c *CELValidatorCompiler) ValidateInitialization() error {
return c.Matcher.ValidateInitialization()
}
type validationActivation struct {
object, oldObject, params, request interface{}
}
// ResolveName returns a value from the activation by qualified name, or false if the name
// could not be found.
func (a *validationActivation) ResolveName(name string) (interface{}, bool) {
switch name {
case ObjectVarName:
return a.object, true
case OldObjectVarName:
return a.oldObject, true
case ParamsVarName:
return a.params, true
case RequestVarName:
return a.request, true
default:
return nil, false
func NewValidator(validationFilter cel.Filter, celMatcher matchconditions.Matcher, auditAnnotationFilter, messageFilter cel.Filter, failPolicy *v1.FailurePolicyType) Validator {
return &validator{
celMatcher: celMatcher,
validationFilter: validationFilter,
auditAnnotationFilter: auditAnnotationFilter,
messageFilter: messageFilter,
failPolicy: failPolicy,
}
}
// Parent returns the parent of the current activation, may be nil.
// If non-nil, the parent will be searched during resolve calls.
func (a *validationActivation) Parent() interpreter.Activation {
return nil
func policyDecisionActionForError(f v1.FailurePolicyType) PolicyDecisionAction {
if f == v1.Ignore {
return ActionAdmit
}
return ActionDeny
}
// Compile compiles the cel expression defined in ValidatingAdmissionPolicy
func (c *CELValidatorCompiler) Compile(p *v1alpha1.ValidatingAdmissionPolicy) Validator {
if len(p.Spec.Validations) == 0 {
return nil
func auditAnnotationEvaluationForError(f v1.FailurePolicyType) PolicyAuditAnnotationAction {
if f == v1.Ignore {
return AuditAnnotationActionExclude
}
hasParam := false
if p.Spec.ParamKind != nil {
hasParam = true
}
compilationResults := make([]CompilationResult, len(p.Spec.Validations))
for i, validation := range p.Spec.Validations {
compilationResults[i] = CompileValidatingPolicyExpression(validation.Expression, hasParam)
}
return &CELValidator{policy: p, compilationResults: compilationResults}
return AuditAnnotationActionError
}
// CELValidator implements the Validator interface
type CELValidator struct {
policy *v1alpha1.ValidatingAdmissionPolicy
compilationResults []CompilationResult
}
// Validate takes a list of Evaluation and a failure policy and converts them into actionable PolicyDecisions
// runtimeCELCostBudget was added for testing purpose only. Callers should always use const RuntimeCELCostBudget from k8s.io/apiserver/pkg/apis/cel/config.go as input.
func convertObjectToUnstructured(obj interface{}) (*unstructured.Unstructured, error) {
if obj == nil || reflect.ValueOf(obj).IsNil() {
return &unstructured.Unstructured{Object: nil}, nil
}
ret, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
if err != nil {
return nil, err
}
return &unstructured.Unstructured{Object: ret}, nil
}
func objectToResolveVal(r runtime.Object) (interface{}, error) {
if r == nil || reflect.ValueOf(r).IsNil() {
return nil, nil
}
v, err := convertObjectToUnstructured(r)
if err != nil {
return nil, err
}
return v.Object, nil
}
func policyDecisionActionForError(f v1alpha1.FailurePolicyType) policyDecisionAction {
if f == v1alpha1.Ignore {
return actionAdmit
}
return actionDeny
}
// Validate validates all cel expressions in Validator and returns a PolicyDecision for each CEL expression or returns an error.
// An error will be returned if failed to convert the object/oldObject/params/request to unstructured.
// Each PolicyDecision will have a decision and a message.
// policyDecision.message will be empty if the decision is allowed and no error met.
func (v *CELValidator) Validate(a admission.Attributes, o admission.ObjectInterfaces, versionedParams runtime.Object, matchKind schema.GroupVersionKind) ([]policyDecision, error) {
// TODO: replace unstructured with ref.Val for CEL variables when native type support is available
decisions := make([]policyDecision, len(v.compilationResults))
var err error
versionedAttr, err := generic.NewVersionedAttributes(a, matchKind, o)
if err != nil {
return nil, err
}
oldObjectVal, err := objectToResolveVal(versionedAttr.VersionedOldObject)
if err != nil {
return nil, err
}
objectVal, err := objectToResolveVal(versionedAttr.VersionedObject)
if err != nil {
return nil, err
}
paramsVal, err := objectToResolveVal(versionedParams)
if err != nil {
return nil, err
}
request := createAdmissionRequest(versionedAttr.Attributes)
requestVal, err := convertObjectToUnstructured(request)
if err != nil {
return nil, err
}
va := &validationActivation{
object: objectVal,
oldObject: oldObjectVal,
params: paramsVal,
request: requestVal.Object,
}
var f v1alpha1.FailurePolicyType
if v.policy.Spec.FailurePolicy == nil {
f = v1alpha1.Fail
func (v *validator) Validate(ctx context.Context, matchedResource schema.GroupVersionResource, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, namespace *corev1.Namespace, runtimeCELCostBudget int64, authz authorizer.Authorizer) ValidateResult {
var f v1.FailurePolicyType
if v.failPolicy == nil {
f = v1.Fail
} else {
f = *v.policy.Spec.FailurePolicy
f = *v.failPolicy
}
for i, compilationResult := range v.compilationResults {
validation := v.policy.Spec.Validations[i]
if v.celMatcher != nil {
matchResults := v.celMatcher.Match(ctx, versionedAttr, versionedParams, authz)
if matchResults.Error != nil {
return ValidateResult{
Decisions: []PolicyDecision{
{
Action: policyDecisionActionForError(f),
Evaluation: EvalError,
Message: matchResults.Error.Error(),
},
},
}
}
var policyDecision = &decisions[i]
// if preconditions are not met, then do not return any validations
if !matchResults.Matches {
return ValidateResult{}
}
}
if compilationResult.Error != nil {
policyDecision.action = policyDecisionActionForError(f)
policyDecision.evaluation = evalError
policyDecision.message = fmt.Sprintf("compilation error: %v", compilationResult.Error)
optionalVars := cel.OptionalVariableBindings{VersionedParams: versionedParams, Authorizer: authz}
expressionOptionalVars := cel.OptionalVariableBindings{VersionedParams: versionedParams}
admissionRequest := cel.CreateAdmissionRequest(versionedAttr.Attributes, metav1.GroupVersionResource(matchedResource), metav1.GroupVersionKind(versionedAttr.VersionedKind))
// Decide which fields are exposed
ns := cel.CreateNamespaceObject(namespace)
evalResults, remainingBudget, err := v.validationFilter.ForInput(ctx, versionedAttr, admissionRequest, optionalVars, ns, runtimeCELCostBudget)
if err != nil {
return ValidateResult{
Decisions: []PolicyDecision{
{
Action: policyDecisionActionForError(f),
Evaluation: EvalError,
Message: err.Error(),
},
},
}
}
decisions := make([]PolicyDecision, len(evalResults))
messageResults, _, err := v.messageFilter.ForInput(ctx, versionedAttr, admissionRequest, expressionOptionalVars, ns, remainingBudget)
for i, evalResult := range evalResults {
var decision = &decisions[i]
// TODO: move this to generics
validation, ok := evalResult.ExpressionAccessor.(*ValidationCondition)
if !ok {
klog.Error("Invalid type conversion to ValidationCondition")
decision.Action = policyDecisionActionForError(f)
decision.Evaluation = EvalError
decision.Message = "Invalid type sent to validator, expected ValidationCondition"
continue
}
if compilationResult.Program == nil {
policyDecision.action = policyDecisionActionForError(f)
policyDecision.evaluation = evalError
policyDecision.message = "unexpected internal error compiling expression"
continue
var messageResult *cel.EvaluationResult
var messageError *apiservercel.Error
if len(messageResults) > i {
messageResult = &messageResults[i]
}
t1 := time.Now()
evalResult, _, err := compilationResult.Program.Eval(va)
elapsed := time.Since(t1)
policyDecision.elapsed = elapsed
if err != nil {
policyDecision.action = policyDecisionActionForError(f)
policyDecision.evaluation = evalError
policyDecision.message = fmt.Sprintf("expression '%v' resulted in error: %v", v.policy.Spec.Validations[i].Expression, err)
} else if evalResult != celtypes.True {
policyDecision.action = actionDeny
messageError, _ = err.(*apiservercel.Error)
if evalResult.Error != nil {
decision.Action = policyDecisionActionForError(f)
decision.Evaluation = EvalError
decision.Message = evalResult.Error.Error()
} else if messageError != nil &&
(messageError.Type == apiservercel.ErrorTypeInternal ||
(messageError.Type == apiservercel.ErrorTypeInvalid &&
strings.HasPrefix(messageError.Detail, "validation failed due to running out of cost budget"))) {
decision.Action = policyDecisionActionForError(f)
decision.Evaluation = EvalError
decision.Message = fmt.Sprintf("failed messageExpression: %s", err)
} else if evalResult.EvalResult != celtypes.True {
decision.Action = ActionDeny
if validation.Reason == nil {
policyDecision.reason = metav1.StatusReasonInvalid
decision.Reason = metav1.StatusReasonInvalid
} else {
policyDecision.reason = *validation.Reason
decision.Reason = *validation.Reason
}
if len(validation.Message) > 0 {
policyDecision.message = strings.TrimSpace(validation.Message)
} else {
policyDecision.message = fmt.Sprintf("failed expression: %v", strings.TrimSpace(validation.Expression))
// decide the failure message
var message string
// attempt to set message with messageExpression result
if messageResult != nil && messageResult.Error == nil && messageResult.EvalResult != nil {
// also fallback if the eval result is non-string (including null) or
// whitespaces.
if message, ok = messageResult.EvalResult.Value().(string); ok {
message = strings.TrimSpace(message)
// deny excessively long message from EvalResult
if len(message) > celconfig.MaxEvaluatedMessageExpressionSizeBytes {
klog.V(2).InfoS("excessively long message denied", "message", message)
message = ""
}
// deny message that contains newlines
if strings.ContainsAny(message, "\n") {
klog.V(2).InfoS("multi-line message denied", "message", message)
message = ""
}
}
}
if messageResult != nil && messageResult.Error != nil {
// log any error with messageExpression
klog.V(2).ErrorS(messageResult.Error, "error while evaluating messageExpression")
}
// fallback to set message to the custom message
if message == "" && len(validation.Message) > 0 {
message = strings.TrimSpace(validation.Message)
}
// fallback to use the expression to compose a message
if message == "" {
message = fmt.Sprintf("failed expression: %v", strings.TrimSpace(validation.Expression))
}
decision.Message = message
} else {
policyDecision.action = actionAdmit
policyDecision.evaluation = evalAdmit
decision.Action = ActionAdmit
decision.Evaluation = EvalAdmit
}
}
return decisions, nil
}
func createAdmissionRequest(attr admission.Attributes) *admissionv1.AdmissionRequest {
// FIXME: how to get resource GVK, GVR and subresource?
gvk := attr.GetKind()
gvr := attr.GetResource()
subresource := attr.GetSubresource()
requestGVK := attr.GetKind()
requestGVR := attr.GetResource()
requestSubResource := attr.GetSubresource()
aUserInfo := attr.GetUserInfo()
var userInfo authenticationv1.UserInfo
if aUserInfo != nil {
userInfo = authenticationv1.UserInfo{
Extra: make(map[string]authenticationv1.ExtraValue),
Groups: aUserInfo.GetGroups(),
UID: aUserInfo.GetUID(),
Username: aUserInfo.GetName(),
}
// Convert the extra information in the user object
for key, val := range aUserInfo.GetExtra() {
userInfo.Extra[key] = authenticationv1.ExtraValue(val)
}
}
dryRun := attr.IsDryRun()
return &admissionv1.AdmissionRequest{
Kind: metav1.GroupVersionKind{
Group: gvk.Group,
Kind: gvk.Kind,
Version: gvk.Version,
},
Resource: metav1.GroupVersionResource{
Group: gvr.Group,
Resource: gvr.Resource,
Version: gvr.Version,
},
SubResource: subresource,
RequestKind: &metav1.GroupVersionKind{
Group: requestGVK.Group,
Kind: requestGVK.Kind,
Version: requestGVK.Version,
},
RequestResource: &metav1.GroupVersionResource{
Group: requestGVR.Group,
Resource: requestGVR.Resource,
Version: requestGVR.Version,
},
RequestSubResource: requestSubResource,
Name: attr.GetName(),
Namespace: attr.GetNamespace(),
Operation: admissionv1.Operation(attr.GetOperation()),
UserInfo: userInfo,
// Leave Object and OldObject unset since we don't provide access to them via request
DryRun: &dryRun,
Options: runtime.RawExtension{
Object: attr.GetOperationOptions(),
},
}
options := cel.OptionalVariableBindings{VersionedParams: versionedParams}
auditAnnotationEvalResults, _, err := v.auditAnnotationFilter.ForInput(ctx, versionedAttr, admissionRequest, options, namespace, runtimeCELCostBudget)
if err != nil {
return ValidateResult{
Decisions: []PolicyDecision{
{
Action: policyDecisionActionForError(f),
Evaluation: EvalError,
Message: err.Error(),
},
},
}
}
auditAnnotationResults := make([]PolicyAuditAnnotation, len(auditAnnotationEvalResults))
for i, evalResult := range auditAnnotationEvalResults {
if evalResult.ExpressionAccessor == nil {
continue
}
var auditAnnotationResult = &auditAnnotationResults[i]
// TODO: move this to generics
validation, ok := evalResult.ExpressionAccessor.(*AuditAnnotationCondition)
if !ok {
klog.Error("Invalid type conversion to AuditAnnotationCondition")
auditAnnotationResult.Action = auditAnnotationEvaluationForError(f)
auditAnnotationResult.Error = fmt.Sprintf("Invalid type sent to validator, expected AuditAnnotationCondition but got %T", evalResult.ExpressionAccessor)
continue
}
auditAnnotationResult.Key = validation.Key
if evalResult.Error != nil {
auditAnnotationResult.Action = auditAnnotationEvaluationForError(f)
auditAnnotationResult.Error = evalResult.Error.Error()
} else {
switch evalResult.EvalResult.Type() {
case celtypes.StringType:
value := strings.TrimSpace(evalResult.EvalResult.Value().(string))
if len(value) == 0 {
auditAnnotationResult.Action = AuditAnnotationActionExclude
} else {
auditAnnotationResult.Action = AuditAnnotationActionPublish
auditAnnotationResult.Value = value
}
case celtypes.NullType:
auditAnnotationResult.Action = AuditAnnotationActionExclude
default:
auditAnnotationResult.Action = AuditAnnotationActionError
auditAnnotationResult.Error = fmt.Sprintf("valueExpression '%v' resulted in unsupported return type: %v. "+
"Return type must be either string or null.", validation.ValueExpression, evalResult.EvalResult.Type())
}
}
}
return ValidateResult{Decisions: decisions, AuditAnnotations: auditAnnotationResults}
}

View File

@@ -19,11 +19,14 @@ package webhook
import (
"sync"
"k8s.io/api/admissionregistration/v1"
v1 "k8s.io/api/admissionregistration/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apiserver/pkg/admission/plugin/cel"
"k8s.io/apiserver/pkg/admission/plugin/webhook/matchconditions"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/namespace"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/object"
"k8s.io/apiserver/pkg/cel/environment"
webhookutil "k8s.io/apiserver/pkg/util/webhook"
"k8s.io/client-go/rest"
)
@@ -44,6 +47,9 @@ type WebhookAccessor interface {
// GetRESTClient gets the webhook client
GetRESTClient(clientManager *webhookutil.ClientManager) (*rest.RESTClient, error)
// GetCompiledMatcher gets the compiled matcher object
GetCompiledMatcher(compiler cel.FilterCompiler) matchconditions.Matcher
// GetName gets the webhook Name field. Note that the name is scoped to the webhook
// configuration and does not provide a globally unique identity, if a unique identity is
// needed, use GetUID.
@@ -67,10 +73,16 @@ type WebhookAccessor interface {
// GetAdmissionReviewVersions gets the webhook AdmissionReviewVersions field.
GetAdmissionReviewVersions() []string
// GetMatchConditions gets the webhook match conditions field.
GetMatchConditions() []v1.MatchCondition
// GetMutatingWebhook if the accessor contains a MutatingWebhook, returns it and true, else returns false.
GetMutatingWebhook() (*v1.MutatingWebhook, bool)
// GetValidatingWebhook if the accessor contains a ValidatingWebhook, returns it and true, else returns false.
GetValidatingWebhook() (*v1.ValidatingWebhook, bool)
// GetType returns the type of the accessor (validate or admit)
GetType() string
}
// NewMutatingWebhookAccessor creates an accessor for a MutatingWebhook.
@@ -94,6 +106,9 @@ type mutatingWebhookAccessor struct {
initClient sync.Once
client *rest.RESTClient
clientErr error
compileMatcher sync.Once
compiledMatcher matchconditions.Matcher
}
func (m *mutatingWebhookAccessor) GetUID() string {
@@ -111,6 +126,31 @@ func (m *mutatingWebhookAccessor) GetRESTClient(clientManager *webhookutil.Clien
return m.client, m.clientErr
}
func (m *mutatingWebhookAccessor) GetType() string {
return "admit"
}
func (m *mutatingWebhookAccessor) GetCompiledMatcher(compiler cel.FilterCompiler) matchconditions.Matcher {
m.compileMatcher.Do(func() {
expressions := make([]cel.ExpressionAccessor, len(m.MutatingWebhook.MatchConditions))
for i, matchCondition := range m.MutatingWebhook.MatchConditions {
expressions[i] = &matchconditions.MatchCondition{
Name: matchCondition.Name,
Expression: matchCondition.Expression,
}
}
m.compiledMatcher = matchconditions.NewMatcher(compiler.Compile(
expressions,
cel.OptionalVariableDeclarations{
HasParams: false,
HasAuthorizer: true,
},
environment.StoredExpressions,
), m.FailurePolicy, "webhook", "admit", m.Name)
})
return m.compiledMatcher
}
func (m *mutatingWebhookAccessor) GetParsedNamespaceSelector() (labels.Selector, error) {
m.initNamespaceSelector.Do(func() {
m.namespaceSelector, m.namespaceSelectorErr = metav1.LabelSelectorAsSelector(m.NamespaceSelector)
@@ -165,6 +205,10 @@ func (m *mutatingWebhookAccessor) GetAdmissionReviewVersions() []string {
return m.AdmissionReviewVersions
}
func (m *mutatingWebhookAccessor) GetMatchConditions() []v1.MatchCondition {
return m.MatchConditions
}
func (m *mutatingWebhookAccessor) GetMutatingWebhook() (*v1.MutatingWebhook, bool) {
return m.MutatingWebhook, true
}
@@ -194,6 +238,9 @@ type validatingWebhookAccessor struct {
initClient sync.Once
client *rest.RESTClient
clientErr error
compileMatcher sync.Once
compiledMatcher matchconditions.Matcher
}
func (v *validatingWebhookAccessor) GetUID() string {
@@ -211,6 +258,27 @@ func (v *validatingWebhookAccessor) GetRESTClient(clientManager *webhookutil.Cli
return v.client, v.clientErr
}
func (v *validatingWebhookAccessor) GetCompiledMatcher(compiler cel.FilterCompiler) matchconditions.Matcher {
v.compileMatcher.Do(func() {
expressions := make([]cel.ExpressionAccessor, len(v.ValidatingWebhook.MatchConditions))
for i, matchCondition := range v.ValidatingWebhook.MatchConditions {
expressions[i] = &matchconditions.MatchCondition{
Name: matchCondition.Name,
Expression: matchCondition.Expression,
}
}
v.compiledMatcher = matchconditions.NewMatcher(compiler.Compile(
expressions,
cel.OptionalVariableDeclarations{
HasParams: false,
HasAuthorizer: true,
},
environment.StoredExpressions,
), v.FailurePolicy, "webhook", "validating", v.Name)
})
return v.compiledMatcher
}
func (v *validatingWebhookAccessor) GetParsedNamespaceSelector() (labels.Selector, error) {
v.initNamespaceSelector.Do(func() {
v.namespaceSelector, v.namespaceSelectorErr = metav1.LabelSelectorAsSelector(v.NamespaceSelector)
@@ -225,6 +293,10 @@ func (v *validatingWebhookAccessor) GetParsedObjectSelector() (labels.Selector,
return v.objectSelector, v.objectSelectorErr
}
func (m *validatingWebhookAccessor) GetType() string {
return "validate"
}
func (v *validatingWebhookAccessor) GetName() string {
return v.Name
}
@@ -265,6 +337,10 @@ func (v *validatingWebhookAccessor) GetAdmissionReviewVersions() []string {
return v.AdmissionReviewVersions
}
func (v *validatingWebhookAccessor) GetMatchConditions() []v1.MatchCondition {
return v.MatchConditions
}
func (v *validatingWebhookAccessor) GetMutatingWebhook() (*v1.MutatingWebhook, bool) {
return nil, false
}

View File

@@ -19,7 +19,6 @@ package config
import (
"fmt"
"io"
"io/ioutil"
"path"
"k8s.io/apimachinery/pkg/runtime"
@@ -47,7 +46,7 @@ func LoadConfig(configFile io.Reader) (string, error) {
var kubeconfigFile string
if configFile != nil {
// we have a config so parse it.
data, err := ioutil.ReadAll(configFile)
data, err := io.ReadAll(configFile)
if err != nil {
return "", err
}

View File

@@ -19,43 +19,21 @@ package generic
import (
"context"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/webhook"
)
type VersionedAttributeAccessor interface {
VersionedAttribute(gvk schema.GroupVersionKind) (*admission.VersionedAttributes, error)
}
// Source can list dynamic webhook plugins.
type Source interface {
Webhooks() []webhook.WebhookAccessor
HasSynced() bool
}
// VersionedAttributes is a wrapper around the original admission attributes, adding versioned
// variants of the object and old object.
type VersionedAttributes struct {
// Attributes holds the original admission attributes
admission.Attributes
// VersionedOldObject holds Attributes.OldObject (if non-nil), converted to VersionedKind.
// It must never be mutated.
VersionedOldObject runtime.Object
// VersionedObject holds Attributes.Object (if non-nil), converted to VersionedKind.
// If mutated, Dirty must be set to true by the mutator.
VersionedObject runtime.Object
// VersionedKind holds the fully qualified kind
VersionedKind schema.GroupVersionKind
// Dirty indicates VersionedObject has been modified since being converted from Attributes.Object
Dirty bool
}
// GetObject overrides the Attributes.GetObject()
func (v *VersionedAttributes) GetObject() runtime.Object {
if v.VersionedObject != nil {
return v.VersionedObject
}
return v.Attributes.GetObject()
}
// WebhookInvocation describes how to call a webhook, including the resource and subresource the webhook registered for,
// and the kind that should be sent to the webhook.
type WebhookInvocation struct {

View File

@@ -21,18 +21,24 @@ import (
"fmt"
"io"
admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
"k8s.io/klog/v2"
admissionv1 "k8s.io/api/admission/v1"
admissionv1beta1 "k8s.io/api/admission/v1beta1"
"k8s.io/api/admissionregistration/v1"
v1 "k8s.io/api/admissionregistration/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
genericadmissioninit "k8s.io/apiserver/pkg/admission/initializer"
"k8s.io/apiserver/pkg/admission/plugin/cel"
"k8s.io/apiserver/pkg/admission/plugin/webhook"
"k8s.io/apiserver/pkg/admission/plugin/webhook/config"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/namespace"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/object"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/rules"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/cel/environment"
webhookutil "k8s.io/apiserver/pkg/util/webhook"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
@@ -49,6 +55,8 @@ type Webhook struct {
namespaceMatcher *namespace.Matcher
objectMatcher *object.Matcher
dispatcher Dispatcher
filterCompiler cel.FilterCompiler
authorizer authorizer.Authorizer
}
var (
@@ -92,6 +100,7 @@ func NewWebhook(handler *admission.Handler, configFile io.Reader, sourceFactory
namespaceMatcher: &namespace.Matcher{},
objectMatcher: &object.Matcher{},
dispatcher: dispatcherFactory(&cm),
filterCompiler: cel.NewFilterCompiler(environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion())),
}, nil
}
@@ -124,6 +133,10 @@ func (a *Webhook) SetExternalKubeInformerFactory(f informers.SharedInformerFacto
})
}
func (a *Webhook) SetAuthorizer(authorizer authorizer.Authorizer) {
a.authorizer = authorizer
}
// ValidateInitialization implements the InitializationValidator interface.
func (a *Webhook) ValidateInitialization() error {
if a.hookSource == nil {
@@ -140,7 +153,7 @@ func (a *Webhook) ValidateInitialization() error {
// ShouldCallHook returns invocation details if the webhook should be called, nil if the webhook should not be called,
// or an error if an error was encountered during evaluation.
func (a *Webhook) ShouldCallHook(h webhook.WebhookAccessor, attr admission.Attributes, o admission.ObjectInterfaces) (*WebhookInvocation, *apierrors.StatusError) {
func (a *Webhook) ShouldCallHook(ctx context.Context, h webhook.WebhookAccessor, attr admission.Attributes, o admission.ObjectInterfaces, v VersionedAttributeAccessor) (*WebhookInvocation, *apierrors.StatusError) {
matches, matchNsErr := a.namespaceMatcher.MatchNamespaceSelector(h, attr)
// Should not return an error here for webhooks which do not apply to the request, even if err is an unexpected scenario.
if !matches && matchNsErr == nil {
@@ -206,6 +219,25 @@ func (a *Webhook) ShouldCallHook(h webhook.WebhookAccessor, attr admission.Attri
if matchObjErr != nil {
return nil, matchObjErr
}
matchConditions := h.GetMatchConditions()
if len(matchConditions) > 0 {
versionedAttr, err := v.VersionedAttribute(invocation.Kind)
if err != nil {
return nil, apierrors.NewInternalError(err)
}
matcher := h.GetCompiledMatcher(a.filterCompiler)
matchResult := matcher.Match(ctx, versionedAttr, nil, a.authorizer)
if matchResult.Error != nil {
klog.Warningf("Failed evaluating match conditions, failing closed %v: %v", h.GetName(), matchResult.Error)
return nil, apierrors.NewForbidden(attr.GetResource().GroupResource(), attr.GetName(), matchResult.Error)
} else if !matchResult.Matches {
admissionmetrics.Metrics.ObserveMatchConditionExclusion(ctx, h.GetName(), "webhook", h.GetType(), string(attr.GetOperation()))
// if no match, always skip webhook
return nil, nil
}
}
return invocation, nil
}

View File

@@ -0,0 +1,37 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package matchconditions
import (
"context"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authorization/authorizer"
)
type MatchResult struct {
Matches bool
Error error
FailedConditionName string
}
// Matcher contains logic for converting Evaluations to bool of matches or does not match
type Matcher interface {
// Match is used to take cel evaluations and convert into decisions
Match(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, authz authorizer.Authorizer) MatchResult
}

View File

@@ -0,0 +1,144 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package matchconditions
import (
"context"
"errors"
"fmt"
"time"
"github.com/google/cel-go/cel"
celtypes "github.com/google/cel-go/common/types"
v1 "k8s.io/api/admissionregistration/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apiserver/pkg/admission"
admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
celplugin "k8s.io/apiserver/pkg/admission/plugin/cel"
celconfig "k8s.io/apiserver/pkg/apis/cel"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/klog/v2"
)
var _ celplugin.ExpressionAccessor = &MatchCondition{}
// MatchCondition contains the inputs needed to compile, evaluate and match a cel expression
type MatchCondition v1.MatchCondition
func (v *MatchCondition) GetExpression() string {
return v.Expression
}
func (v *MatchCondition) ReturnTypes() []*cel.Type {
return []*cel.Type{cel.BoolType}
}
var _ Matcher = &matcher{}
// matcher evaluates compiled cel expressions and determines if they match the given request or not
type matcher struct {
filter celplugin.Filter
failPolicy v1.FailurePolicyType
matcherType string
matcherKind string
objectName string
}
func NewMatcher(filter celplugin.Filter, failPolicy *v1.FailurePolicyType, matcherKind, matcherType, objectName string) Matcher {
var f v1.FailurePolicyType
if failPolicy == nil {
f = v1.Fail
} else {
f = *failPolicy
}
return &matcher{
filter: filter,
failPolicy: f,
matcherKind: matcherKind,
matcherType: matcherType,
objectName: objectName,
}
}
func (m *matcher) Match(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, authz authorizer.Authorizer) MatchResult {
t := time.Now()
evalResults, _, err := m.filter.ForInput(ctx, versionedAttr, celplugin.CreateAdmissionRequest(versionedAttr.Attributes, metav1.GroupVersionResource(versionedAttr.GetResource()), metav1.GroupVersionKind(versionedAttr.VersionedKind)), celplugin.OptionalVariableBindings{
VersionedParams: versionedParams,
Authorizer: authz,
}, nil, celconfig.RuntimeCELCostBudgetMatchConditions)
if err != nil {
admissionmetrics.Metrics.ObserveMatchConditionEvaluationTime(ctx, time.Since(t), m.objectName, m.matcherKind, m.matcherType, string(versionedAttr.GetOperation()))
// filter returning error is unexpected and not an evaluation error so not incrementing metric here
if m.failPolicy == v1.Fail {
return MatchResult{
Error: err,
}
} else if m.failPolicy == v1.Ignore {
return MatchResult{
Matches: false,
}
}
//TODO: add default so that if in future we add different failure types it doesn't fall through
}
errorList := []error{}
for _, evalResult := range evalResults {
matchCondition, ok := evalResult.ExpressionAccessor.(*MatchCondition)
if !ok {
// This shouldnt happen, but if it does treat same as eval error
klog.Error("Invalid type conversion to MatchCondition")
errorList = append(errorList, errors.New(fmt.Sprintf("internal error converting ExpressionAccessor to MatchCondition")))
continue
}
if evalResult.Error != nil {
errorList = append(errorList, evalResult.Error)
admissionmetrics.Metrics.ObserveMatchConditionEvalError(ctx, m.objectName, m.matcherKind, m.matcherType, string(versionedAttr.GetOperation()))
}
if evalResult.EvalResult == celtypes.False {
admissionmetrics.Metrics.ObserveMatchConditionEvaluationTime(ctx, time.Since(t), m.objectName, m.matcherKind, m.matcherType, string(versionedAttr.GetOperation()))
// If any condition false, skip calling webhook always
return MatchResult{
Matches: false,
FailedConditionName: matchCondition.Name,
}
}
}
if len(errorList) > 0 {
admissionmetrics.Metrics.ObserveMatchConditionEvaluationTime(ctx, time.Since(t), m.objectName, m.matcherKind, m.matcherType, string(versionedAttr.GetOperation()))
// If mix of true and eval errors then resort to fail policy
if m.failPolicy == v1.Fail {
// mix of true and errors with fail policy fail should fail request without calling webhook
err = utilerrors.NewAggregate(errorList)
return MatchResult{
Error: err,
}
} else if m.failPolicy == v1.Ignore {
// if fail policy ignore then skip call to webhook
return MatchResult{
Matches: false,
}
}
}
// if no results eval to false, return matches true with list of any errors encountered
return MatchResult{
Matches: true,
}
}

View File

@@ -20,20 +20,20 @@ package mutating
import (
"context"
"errors"
"fmt"
"time"
jsonpatch "github.com/evanphx/json-patch"
"go.opentelemetry.io/otel/attribute"
apiequality "k8s.io/apimachinery/pkg/api/equality"
"k8s.io/klog/v2"
admissionv1 "k8s.io/api/admission/v1"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
utiljson "k8s.io/apimachinery/pkg/util/json"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -48,6 +48,7 @@ import (
webhookutil "k8s.io/apiserver/pkg/util/webhook"
"k8s.io/apiserver/pkg/warning"
"k8s.io/component-base/tracing"
"k8s.io/klog/v2"
)
const (
@@ -75,6 +76,30 @@ func newMutatingDispatcher(p *Plugin) func(cm *webhookutil.ClientManager) generi
}
}
var _ generic.VersionedAttributeAccessor = &versionedAttributeAccessor{}
type versionedAttributeAccessor struct {
versionedAttr *admission.VersionedAttributes
attr admission.Attributes
objectInterfaces admission.ObjectInterfaces
}
func (v *versionedAttributeAccessor) VersionedAttribute(gvk schema.GroupVersionKind) (*admission.VersionedAttributes, error) {
if v.versionedAttr == nil {
// First call, create versioned attributes
var err error
if v.versionedAttr, err = admission.NewVersionedAttributes(v.attr, gvk, v.objectInterfaces); err != nil {
return nil, apierrors.NewInternalError(err)
}
} else {
// Subsequent call, convert existing versioned attributes to the requested version
if err := admission.ConvertVersionedAttributes(v.versionedAttr, gvk, v.objectInterfaces); err != nil {
return nil, apierrors.NewInternalError(err)
}
}
return v.versionedAttr, nil
}
var _ generic.Dispatcher = &mutatingDispatcher{}
func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces, hooks []webhook.WebhookAccessor) error {
@@ -95,19 +120,24 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
defer func() {
webhookReinvokeCtx.SetLastWebhookInvocationOutput(attr.GetObject())
}()
var versionedAttr *generic.VersionedAttributes
v := &versionedAttributeAccessor{
attr: attr,
objectInterfaces: o,
}
for i, hook := range hooks {
attrForCheck := attr
if versionedAttr != nil {
attrForCheck = versionedAttr
if v.versionedAttr != nil {
attrForCheck = v.versionedAttr
}
invocation, statusErr := a.plugin.ShouldCallHook(hook, attrForCheck, o)
invocation, statusErr := a.plugin.ShouldCallHook(ctx, hook, attrForCheck, o, v)
if statusErr != nil {
return statusErr
}
if invocation == nil {
continue
}
hook, ok := invocation.Webhook.GetMutatingWebhook()
if !ok {
return fmt.Errorf("mutating webhook dispatch requires v1.MutatingWebhook, but got %T", hook)
@@ -121,17 +151,9 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
continue
}
if versionedAttr == nil {
// First webhook, create versioned attributes
var err error
if versionedAttr, err = generic.NewVersionedAttributes(attr, invocation.Kind, o); err != nil {
return apierrors.NewInternalError(err)
}
} else {
// Subsequent webhook, convert existing versioned attributes to this webhook's version
if err := generic.ConvertVersionedAttributes(versionedAttr, invocation.Kind, o); err != nil {
return apierrors.NewInternalError(err)
}
versionedAttr, err := v.VersionedAttribute(invocation.Kind)
if err != nil {
return apierrors.NewInternalError(err)
}
t := time.Now()
@@ -149,7 +171,10 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
case *webhookutil.ErrCallingWebhook:
if !ignoreClientCallFailures {
rejected = true
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, int(err.Status.ErrStatus.Code))
// Ignore context cancelled from webhook metrics
if !errors.Is(err.Reason, context.Canceled) {
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, int(err.Status.ErrStatus.Code))
}
}
admissionmetrics.Metrics.ObserveWebhook(ctx, hook.Name, time.Since(t), rejected, versionedAttr.Attributes, "admit", int(err.Status.ErrStatus.Code))
case *webhookutil.ErrWebhookRejection:
@@ -178,10 +203,14 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
if ignoreClientCallFailures {
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "admit")
annotator.addFailedOpenAnnotation()
// Ignore context cancelled from webhook metrics
if errors.Is(callErr.Reason, context.Canceled) {
klog.Warningf("Context canceled when calling webhook %v", hook.Name)
} else {
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "admit")
annotator.addFailedOpenAnnotation()
}
utilruntime.HandleError(callErr)
select {
@@ -203,8 +232,8 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
}
// convert versionedAttr.VersionedObject to the internal version in the underlying admission.Attributes
if versionedAttr != nil && versionedAttr.VersionedObject != nil && versionedAttr.Dirty {
return o.GetObjectConvertor().Convert(versionedAttr.VersionedObject, versionedAttr.Attributes.GetObject(), nil)
if v.versionedAttr != nil && v.versionedAttr.VersionedObject != nil && v.versionedAttr.Dirty {
return o.GetObjectConvertor().Convert(v.versionedAttr.VersionedObject, v.versionedAttr.Attributes.GetObject(), nil)
}
return nil
@@ -212,7 +241,7 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
// note that callAttrMutatingHook updates attr
func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *admissionregistrationv1.MutatingWebhook, invocation *generic.WebhookInvocation, attr *generic.VersionedAttributes, annotator *webhookAnnotator, o admission.ObjectInterfaces, round, idx int) (bool, error) {
func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *admissionregistrationv1.MutatingWebhook, invocation *generic.WebhookInvocation, attr *admission.VersionedAttributes, annotator *webhookAnnotator, o admission.ObjectInterfaces, round, idx int) (bool, error) {
configurationName := invocation.Webhook.GetConfigurationName()
changed := false
defer func() { annotator.addMutationAnnotation(changed) }()
@@ -363,7 +392,7 @@ func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *admiss
}
type webhookAnnotator struct {
attr *generic.VersionedAttributes
attr *admission.VersionedAttributes
failedOpenAnnotationKey string
patchAnnotationKey string
mutationAnnotationKey string
@@ -371,7 +400,7 @@ type webhookAnnotator struct {
configuration string
}
func newWebhookAnnotator(attr *generic.VersionedAttributes, round, idx int, webhook, configuration string) *webhookAnnotator {
func newWebhookAnnotator(attr *admission.VersionedAttributes, round, idx int, webhook, configuration string) *webhookAnnotator {
return &webhookAnnotator{
attr: attr,
failedOpenAnnotationKey: fmt.Sprintf("%sround_%d_index_%d", MutationAuditAnnotationFailedOpenKeyPrefix, round, idx),

View File

@@ -20,6 +20,8 @@ import (
"context"
"fmt"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -42,6 +44,10 @@ type Matcher struct {
Client clientset.Interface
}
func (m *Matcher) GetNamespace(name string) (*v1.Namespace, error) {
return m.NamespaceLister.Get(name)
}
// Validate checks if the Matcher has a NamespaceLister and Client.
func (m *Matcher) Validate() error {
var errs []error
@@ -116,7 +122,7 @@ func (m *Matcher) MatchNamespaceSelector(p NamespaceSelectorProvider, attr admis
if !ok {
return false, apierrors.NewInternalError(err)
}
return false, &apierrors.StatusError{status.Status()}
return false, &apierrors.StatusError{ErrStatus: status.Status()}
}
if err != nil {
return false, apierrors.NewInternalError(err)

View File

@@ -26,6 +26,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/webhook/generic"
)
@@ -130,7 +131,7 @@ func VerifyAdmissionResponse(uid types.UID, mutating bool, review runtime.Object
// CreateAdmissionObjects returns the unique request uid, the AdmissionReview object to send the webhook and to decode the response into,
// or an error if the webhook does not support receiving any of the admission review versions we know to send
func CreateAdmissionObjects(versionedAttributes *generic.VersionedAttributes, invocation *generic.WebhookInvocation) (uid types.UID, request, response runtime.Object, err error) {
func CreateAdmissionObjects(versionedAttributes *admission.VersionedAttributes, invocation *generic.WebhookInvocation) (uid types.UID, request, response runtime.Object, err error) {
for _, version := range invocation.Webhook.GetAdmissionReviewVersions() {
switch version {
case admissionv1.SchemeGroupVersion.Version:
@@ -151,7 +152,7 @@ func CreateAdmissionObjects(versionedAttributes *generic.VersionedAttributes, in
}
// CreateV1AdmissionReview creates an AdmissionReview for the provided admission.Attributes
func CreateV1AdmissionReview(uid types.UID, versionedAttributes *generic.VersionedAttributes, invocation *generic.WebhookInvocation) *admissionv1.AdmissionReview {
func CreateV1AdmissionReview(uid types.UID, versionedAttributes *admission.VersionedAttributes, invocation *generic.WebhookInvocation) *admissionv1.AdmissionReview {
attr := versionedAttributes.Attributes
gvk := invocation.Kind
gvr := invocation.Resource
@@ -217,7 +218,7 @@ func CreateV1AdmissionReview(uid types.UID, versionedAttributes *generic.Version
}
// CreateV1beta1AdmissionReview creates an AdmissionReview for the provided admission.Attributes
func CreateV1beta1AdmissionReview(uid types.UID, versionedAttributes *generic.VersionedAttributes, invocation *generic.WebhookInvocation) *admissionv1beta1.AdmissionReview {
func CreateV1beta1AdmissionReview(uid types.UID, versionedAttributes *admission.VersionedAttributes, invocation *generic.WebhookInvocation) *admissionv1beta1.AdmissionReview {
attr := versionedAttributes.Attributes
gvk := invocation.Kind
gvr := invocation.Resource

View File

@@ -18,6 +18,7 @@ package validating
import (
"context"
"errors"
"fmt"
"sync"
"time"
@@ -62,30 +63,51 @@ func newValidatingDispatcher(p *Plugin) func(cm *webhookutil.ClientManager) gene
}
}
var _ generic.VersionedAttributeAccessor = &versionedAttributeAccessor{}
type versionedAttributeAccessor struct {
versionedAttrs map[schema.GroupVersionKind]*admission.VersionedAttributes
attr admission.Attributes
objectInterfaces admission.ObjectInterfaces
}
func (v *versionedAttributeAccessor) VersionedAttribute(gvk schema.GroupVersionKind) (*admission.VersionedAttributes, error) {
if val, ok := v.versionedAttrs[gvk]; ok {
return val, nil
}
versionedAttr, err := admission.NewVersionedAttributes(v.attr, gvk, v.objectInterfaces)
if err != nil {
return nil, err
}
v.versionedAttrs[gvk] = versionedAttr
return versionedAttr, nil
}
var _ generic.Dispatcher = &validatingDispatcher{}
func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces, hooks []webhook.WebhookAccessor) error {
var relevantHooks []*generic.WebhookInvocation
// Construct all the versions we need to call our webhooks
versionedAttrs := map[schema.GroupVersionKind]*generic.VersionedAttributes{}
versionedAttrAccessor := &versionedAttributeAccessor{
versionedAttrs: map[schema.GroupVersionKind]*admission.VersionedAttributes{},
attr: attr,
objectInterfaces: o,
}
for _, hook := range hooks {
invocation, statusError := d.plugin.ShouldCallHook(hook, attr, o)
invocation, statusError := d.plugin.ShouldCallHook(ctx, hook, attr, o, versionedAttrAccessor)
if statusError != nil {
return statusError
}
if invocation == nil {
continue
}
relevantHooks = append(relevantHooks, invocation)
// If we already have this version, continue
if _, ok := versionedAttrs[invocation.Kind]; ok {
continue
}
versionedAttr, err := generic.NewVersionedAttributes(attr, invocation.Kind, o)
// VersionedAttr result will be cached and reused later during parallel webhook calls
_, err := versionedAttrAccessor.VersionedAttribute(invocation.Kind)
if err != nil {
return apierrors.NewInternalError(err)
}
versionedAttrs[invocation.Kind] = versionedAttr
}
if len(relevantHooks) == 0 {
@@ -108,7 +130,7 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
go func(invocation *generic.WebhookInvocation, idx int) {
ignoreClientCallFailures := false
hookName := "unknown"
versionedAttr := versionedAttrs[invocation.Kind]
versionedAttr := versionedAttrAccessor.versionedAttrs[invocation.Kind]
// The ordering of these two defers is critical. The wg.Done will release the parent go func to close the errCh
// that is used by the second defer to report errors. The recovery and error reporting must be done first.
defer wg.Done()
@@ -154,7 +176,10 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
case *webhookutil.ErrCallingWebhook:
if !ignoreClientCallFailures {
rejected = true
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "validating", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, int(err.Status.ErrStatus.Code))
// Ignore context cancelled from webhook metrics
if !errors.Is(err.Reason, context.Canceled) {
admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "validating", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, int(err.Status.ErrStatus.Code))
}
}
admissionmetrics.Metrics.ObserveWebhook(ctx, hook.Name, time.Since(t), rejected, versionedAttr.Attributes, "validating", int(err.Status.ErrStatus.Code))
case *webhookutil.ErrWebhookRejection:
@@ -173,12 +198,17 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
if callErr, ok := err.(*webhookutil.ErrCallingWebhook); ok {
if ignoreClientCallFailures {
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "validating")
key := fmt.Sprintf("%sround_0_index_%d", ValidatingAuditAnnotationFailedOpenKeyPrefix, idx)
value := hook.Name
if err := versionedAttr.Attributes.AddAnnotation(key, value); err != nil {
klog.Warningf("Failed to set admission audit annotation %s to %s for validating webhook %s: %v", key, value, hook.Name, err)
// Ignore context cancelled from webhook metrics
if errors.Is(callErr.Reason, context.Canceled) {
klog.Warningf("Context canceled when calling webhook %v", hook.Name)
} else {
klog.Warningf("Failed calling webhook, failing open %v: %v", hook.Name, callErr)
admissionmetrics.Metrics.ObserveWebhookFailOpen(ctx, hook.Name, "validating")
key := fmt.Sprintf("%sround_0_index_%d", ValidatingAuditAnnotationFailedOpenKeyPrefix, idx)
value := hook.Name
if err := versionedAttr.Attributes.AddAnnotation(key, value); err != nil {
klog.Warningf("Failed to set admission audit annotation %s to %s for validating webhook %s: %v", key, value, hook.Name, err)
}
}
utilruntime.HandleError(callErr)
return
@@ -215,7 +245,7 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
return errs[0]
}
func (d *validatingDispatcher) callHook(ctx context.Context, h *v1.ValidatingWebhook, invocation *generic.WebhookInvocation, attr *generic.VersionedAttributes) error {
func (d *validatingDispatcher) callHook(ctx context.Context, h *v1.ValidatingWebhook, invocation *generic.WebhookInvocation, attr *admission.VersionedAttributes) error {
if attr.Attributes.IsDryRun() {
if h.SideEffects == nil {
return &webhookutil.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("Webhook SideEffects is nil"), Status: apierrors.NewBadRequest("Webhook SideEffects is nil")}

View File

@@ -20,7 +20,6 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"reflect"
"sort"
"strings"
@@ -115,7 +114,7 @@ func splitStream(config io.Reader) (io.Reader, io.Reader, error) {
return nil, nil, nil
}
configBytes, err := ioutil.ReadAll(config)
configBytes, err := io.ReadAll(config)
if err != nil {
return nil, nil, err
}