Bump sigs.k8s.io/controller-runtime to v0.14.4 (#5507)

* Bump sigs.k8s.io/controller-runtime to v0.14.4

* Update gofmt
This commit is contained in:
hongming
2023-02-08 14:06:15 +08:00
committed by GitHub
parent 129e6fbec3
commit 1c49fcd57e
1404 changed files with 141422 additions and 47769 deletions

111
vendor/k8s.io/apiserver/pkg/admission/cel/metrics.go generated vendored Normal file
View File

@@ -0,0 +1,111 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
import (
"context"
"time"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
const (
metricsNamespace = "apiserver"
metricsSubsystem = "validating_admission_policy"
)
var (
// Metrics provides access to validation admission metrics.
Metrics = newValidationAdmissionMetrics()
)
// ValidatingAdmissionPolicyMetrics aggregates Prometheus metrics related to validation admission control.
type ValidatingAdmissionPolicyMetrics struct {
policyCheck *metrics.CounterVec
policyDefinition *metrics.CounterVec
policyLatency *metrics.HistogramVec
}
func newValidationAdmissionMetrics() *ValidatingAdmissionPolicyMetrics {
check := metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "check_total",
Help: "Validation admission policy check total, labeled by policy and further identified by binding, enforcement action taken, and state.",
StabilityLevel: metrics.ALPHA,
},
[]string{"policy", "policy_binding", "enforcement_action", "state"},
)
definition := metrics.NewCounterVec(&metrics.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "definition_total",
Help: "Validation admission policy count total, labeled by state and enforcement action.",
StabilityLevel: metrics.ALPHA,
},
[]string{"state", "enforcement_action"},
)
latency := metrics.NewHistogramVec(&metrics.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "check_duration_seconds",
Help: "Validation admission latency for individual validation expressions in seconds, labeled by policy and further including binding, state and enforcement action taken.",
// the bucket distribution here is based oo the benchmark suite at
// github.com/DangerOnTheRanger/cel-benchmark performed on 16-core Intel Xeon
// the lowest bucket was based around the 180ns/op figure for BenchmarkAccess,
// plus some additional leeway to account for the apiserver doing other things
// the largest bucket was chosen based on the fact that benchmarks indicate the
// same Xeon running a CEL expression close to the estimated cost limit takes
// around 760ms, so that bucket should only ever have the slowest CEL expressions
// in it
Buckets: []float64{0.0000005, 0.001, 0.01, 0.1, 1.0},
StabilityLevel: metrics.ALPHA,
},
[]string{"policy", "policy_binding", "enforcement_action", "state"},
)
legacyregistry.MustRegister(check)
legacyregistry.MustRegister(definition)
legacyregistry.MustRegister(latency)
return &ValidatingAdmissionPolicyMetrics{policyCheck: check, policyDefinition: definition, policyLatency: latency}
}
// Reset resets all validation admission-related Prometheus metrics.
func (m *ValidatingAdmissionPolicyMetrics) Reset() {
m.policyCheck.Reset()
m.policyDefinition.Reset()
m.policyLatency.Reset()
}
// ObserveDefinition observes a policy definition.
func (m *ValidatingAdmissionPolicyMetrics) ObserveDefinition(ctx context.Context, state, enforcementAction string) {
m.policyDefinition.WithContext(ctx).WithLabelValues(state, enforcementAction).Inc()
}
// ObserveAdmissionWithError observes a policy validation error that was ignored due to failure policy.
func (m *ValidatingAdmissionPolicyMetrics) ObserveAdmissionWithError(ctx context.Context, elapsed time.Duration, policy, binding, state string) {
m.policyCheck.WithContext(ctx).WithLabelValues(policy, binding, "allow", state).Inc()
m.policyLatency.WithContext(ctx).WithLabelValues(policy, binding, "allow", state).Observe(elapsed.Seconds())
}
// ObserveRejection observes a policy validation error that was at least one of the reasons for a deny.
func (m *ValidatingAdmissionPolicyMetrics) ObserveRejection(ctx context.Context, elapsed time.Duration, policy, binding, state string) {
m.policyCheck.WithContext(ctx).WithLabelValues(policy, binding, "deny", state).Inc()
m.policyLatency.WithContext(ctx).WithLabelValues(policy, binding, "deny", state).Observe(elapsed.Seconds())
}

View File

@@ -36,11 +36,11 @@ type mutatingWebhookConfigurationManager struct {
configuration *atomic.Value
lister admissionregistrationlisters.MutatingWebhookConfigurationLister
hasSynced func() bool
// initialConfigurationSynced stores a boolean value, which tracks if
// initialConfigurationSynced tracks if
// the existing webhook configs have been synced (honored) by the
// manager at startup-- the informer has synced and either has no items
// or has finished executing updateConfiguration() once.
initialConfigurationSynced *atomic.Value
initialConfigurationSynced *atomic.Bool
}
var _ generic.Source = &mutatingWebhookConfigurationManager{}
@@ -51,7 +51,7 @@ func NewMutatingWebhookConfigurationManager(f informers.SharedInformerFactory) g
configuration: &atomic.Value{},
lister: informer.Lister(),
hasSynced: informer.Informer().HasSynced,
initialConfigurationSynced: &atomic.Value{},
initialConfigurationSynced: &atomic.Bool{},
}
// Start with an empty list
@@ -80,7 +80,7 @@ func (m *mutatingWebhookConfigurationManager) HasSynced() bool {
if !m.hasSynced() {
return false
}
if m.initialConfigurationSynced.Load().(bool) {
if m.initialConfigurationSynced.Load() {
// the informer has synced and configuration has been updated
return true
}

View File

@@ -36,11 +36,11 @@ type validatingWebhookConfigurationManager struct {
configuration *atomic.Value
lister admissionregistrationlisters.ValidatingWebhookConfigurationLister
hasSynced func() bool
// initialConfigurationSynced stores a boolean value, which tracks if
// initialConfigurationSynced tracks if
// the existing webhook configs have been synced (honored) by the
// manager at startup-- the informer has synced and either has no items
// or has finished executing updateConfiguration() once.
initialConfigurationSynced *atomic.Value
initialConfigurationSynced *atomic.Bool
}
var _ generic.Source = &validatingWebhookConfigurationManager{}
@@ -51,7 +51,7 @@ func NewValidatingWebhookConfigurationManager(f informers.SharedInformerFactory)
configuration: &atomic.Value{},
lister: informer.Lister(),
hasSynced: informer.Informer().HasSynced,
initialConfigurationSynced: &atomic.Value{},
initialConfigurationSynced: &atomic.Bool{},
}
// Start with an empty list
@@ -80,7 +80,7 @@ func (v *validatingWebhookConfigurationManager) HasSynced() bool {
if !v.hasSynced() {
return false
}
if v.initialConfigurationSynced.Load().(bool) {
if v.initialConfigurationSynced.Load() {
// the informer has synced and configuration has been updated
return true
}

View File

@@ -19,6 +19,7 @@ package initializer
import (
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/component-base/featuregate"
@@ -26,6 +27,7 @@ import (
type pluginInitializer struct {
externalClient kubernetes.Interface
dynamicClient dynamic.Interface
externalInformers informers.SharedInformerFactory
authorizer authorizer.Authorizer
featureGates featuregate.FeatureGate
@@ -37,6 +39,7 @@ type pluginInitializer struct {
// during compilation when they update a level.
func New(
extClientset kubernetes.Interface,
dynamicClient dynamic.Interface,
extInformers informers.SharedInformerFactory,
authz authorizer.Authorizer,
featureGates featuregate.FeatureGate,
@@ -44,6 +47,7 @@ func New(
) pluginInitializer {
return pluginInitializer{
externalClient: extClientset,
dynamicClient: dynamicClient,
externalInformers: extInformers,
authorizer: authz,
featureGates: featureGates,
@@ -68,6 +72,10 @@ func (i pluginInitializer) Initialize(plugin admission.Interface) {
wants.SetExternalKubeClientSet(i.externalClient)
}
if wants, ok := plugin.(WantsDynamicClient); ok {
wants.SetDynamicClient(i.dynamicClient)
}
if wants, ok := plugin.(WantsExternalKubeInformerFactory); ok {
wants.SetExternalKubeInformerFactory(i.externalInformers)
}

View File

@@ -17,9 +17,11 @@ limitations under the License.
package initializer
import (
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authorization/authorizer"
quota "k8s.io/apiserver/pkg/quota/v1"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/component-base/featuregate"
@@ -68,3 +70,14 @@ type WantsFeatures interface {
InspectFeatureGates(featuregate.FeatureGate)
admission.InitializationValidator
}
type WantsDynamicClient interface {
SetDynamicClient(dynamic.Interface)
admission.InitializationValidator
}
// WantsRESTMapper defines a function which sets RESTMapper for admission plugins that need it.
type WantsRESTMapper interface {
SetRESTMapper(meta.RESTMapper)
admission.InitializationValidator
}

View File

@@ -0,0 +1,10 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- jpbetz
- cici37
- alexzielenski
reviewers:
- jpbetz
- cici37
- alexzielenski

View File

@@ -0,0 +1,189 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"context"
"errors"
"fmt"
"io"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apiserver/pkg/features"
"k8s.io/client-go/dynamic"
"k8s.io/component-base/featuregate"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/initializer"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
)
////////////////////////////////////////////////////////////////////////////////
// Plugin Definition
////////////////////////////////////////////////////////////////////////////////
// Definition for CEL admission plugin. This is the entry point into the
// CEL admission control system.
//
// Each plugin is asked to validate every object update.
const (
// PluginName indicates the name of admission plug-in
PluginName = "ValidatingAdmissionPolicy"
)
// Register registers a plugin
func Register(plugins *admission.Plugins) {
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
return NewPlugin()
})
}
////////////////////////////////////////////////////////////////////////////////
// Plugin Initialization & Dependency Injection
////////////////////////////////////////////////////////////////////////////////
type celAdmissionPlugin struct {
*admission.Handler
evaluator CELPolicyEvaluator
inspectedFeatureGates bool
enabled bool
// Injected Dependencies
informerFactory informers.SharedInformerFactory
client kubernetes.Interface
restMapper meta.RESTMapper
dynamicClient dynamic.Interface
stopCh <-chan struct{}
}
var _ initializer.WantsExternalKubeInformerFactory = &celAdmissionPlugin{}
var _ initializer.WantsExternalKubeClientSet = &celAdmissionPlugin{}
var _ initializer.WantsRESTMapper = &celAdmissionPlugin{}
var _ initializer.WantsDynamicClient = &celAdmissionPlugin{}
var _ initializer.WantsDrainedNotification = &celAdmissionPlugin{}
var _ admission.InitializationValidator = &celAdmissionPlugin{}
var _ admission.ValidationInterface = &celAdmissionPlugin{}
func NewPlugin() (admission.Interface, error) {
return &celAdmissionPlugin{
Handler: admission.NewHandler(admission.Connect, admission.Create, admission.Delete, admission.Update),
}, nil
}
func (c *celAdmissionPlugin) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) {
c.informerFactory = f
}
func (c *celAdmissionPlugin) SetExternalKubeClientSet(client kubernetes.Interface) {
c.client = client
}
func (c *celAdmissionPlugin) SetRESTMapper(mapper meta.RESTMapper) {
c.restMapper = mapper
}
func (c *celAdmissionPlugin) SetDynamicClient(client dynamic.Interface) {
c.dynamicClient = client
}
func (c *celAdmissionPlugin) SetDrainedNotification(stopCh <-chan struct{}) {
c.stopCh = stopCh
}
func (c *celAdmissionPlugin) InspectFeatureGates(featureGates featuregate.FeatureGate) {
if featureGates.Enabled(features.ValidatingAdmissionPolicy) {
c.enabled = true
}
c.inspectedFeatureGates = true
}
// ValidateInitialization - once clientset and informer factory are provided, creates and starts the admission controller
func (c *celAdmissionPlugin) ValidateInitialization() error {
if !c.inspectedFeatureGates {
return fmt.Errorf("%s did not see feature gates", PluginName)
}
if !c.enabled {
return nil
}
if c.informerFactory == nil {
return errors.New("missing informer factory")
}
if c.client == nil {
return errors.New("missing kubernetes client")
}
if c.restMapper == nil {
return errors.New("missing rest mapper")
}
if c.dynamicClient == nil {
return errors.New("missing dynamic client")
}
if c.stopCh == nil {
return errors.New("missing stop channel")
}
c.evaluator = NewAdmissionController(c.informerFactory, c.client, c.restMapper, c.dynamicClient)
if err := c.evaluator.ValidateInitialization(); err != nil {
return err
}
c.SetReadyFunc(c.evaluator.HasSynced)
go c.evaluator.Run(c.stopCh)
return nil
}
////////////////////////////////////////////////////////////////////////////////
// admission.ValidationInterface
////////////////////////////////////////////////////////////////////////////////
func (c *celAdmissionPlugin) Handles(operation admission.Operation) bool {
return true
}
func (c *celAdmissionPlugin) Validate(
ctx context.Context,
a admission.Attributes,
o admission.ObjectInterfaces,
) (err error) {
if !c.enabled {
return nil
}
// isPolicyResource determines if an admission.Attributes object is describing
// the admission of a ValidatingAdmissionPolicy or a ValidatingAdmissionPolicyBinding
if isPolicyResource(a) {
return
}
if !c.WaitForReady() {
return admission.NewForbidden(a, fmt.Errorf("not yet ready to handle request"))
}
return c.evaluator.Validate(ctx, a, o)
}
func isPolicyResource(attr admission.Attributes) bool {
gvk := attr.GetResource()
if gvk.Group == "admissionregistration.k8s.io" {
if gvk.Resource == "validatingadmissionpolicies" || gvk.Resource == "validatingadmissionpolicybindings" {
return true
}
}
return false
}

View File

@@ -0,0 +1,231 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"sync"
"github.com/google/cel-go/cel"
apiservercel "k8s.io/apiserver/pkg/cel"
"k8s.io/apiserver/pkg/cel/library"
)
const (
ObjectVarName = "object"
OldObjectVarName = "oldObject"
ParamsVarName = "params"
RequestVarName = "request"
checkFrequency = 100
)
type envs struct {
noParams *cel.Env
withParams *cel.Env
}
var (
initEnvsOnce sync.Once
initEnvs *envs
initEnvsErr error
)
func getEnvs() (*envs, error) {
initEnvsOnce.Do(func() {
base, err := buildBaseEnv()
if err != nil {
initEnvsErr = err
return
}
noParams, err := buildNoParamsEnv(base)
if err != nil {
initEnvsErr = err
return
}
withParams, err := buildWithParamsEnv(noParams)
if err != nil {
initEnvsErr = err
return
}
initEnvs = &envs{noParams: noParams, withParams: withParams}
})
return initEnvs, initEnvsErr
}
// This is a similar code as in k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/compilation.go
// If any changes are made here, consider to make the same changes there as well.
func buildBaseEnv() (*cel.Env, error) {
var opts []cel.EnvOption
opts = append(opts, cel.HomogeneousAggregateLiterals())
// Validate function declarations once during base env initialization,
// so they don't need to be evaluated each time a CEL rule is compiled.
// This is a relatively expensive operation.
opts = append(opts, cel.EagerlyValidateDeclarations(true), cel.DefaultUTCTimeZone(true))
opts = append(opts, library.ExtensionLibs...)
return cel.NewEnv(opts...)
}
func buildNoParamsEnv(baseEnv *cel.Env) (*cel.Env, error) {
var propDecls []cel.EnvOption
reg := apiservercel.NewRegistry(baseEnv)
requestType := buildRequestType()
rt, err := apiservercel.NewRuleTypes(requestType.TypeName(), requestType, reg)
if err != nil {
return nil, err
}
if rt == nil {
return nil, nil
}
opts, err := rt.EnvOptions(baseEnv.TypeProvider())
if err != nil {
return nil, err
}
propDecls = append(propDecls, cel.Variable(ObjectVarName, cel.DynType))
propDecls = append(propDecls, cel.Variable(OldObjectVarName, cel.DynType))
propDecls = append(propDecls, cel.Variable(RequestVarName, requestType.CelType()))
opts = append(opts, propDecls...)
env, err := baseEnv.Extend(opts...)
if err != nil {
return nil, err
}
return env, nil
}
func buildWithParamsEnv(noParams *cel.Env) (*cel.Env, error) {
return noParams.Extend(cel.Variable(ParamsVarName, cel.DynType))
}
// buildRequestType generates a DeclType for AdmissionRequest. This may be replaced with a utility that
// converts the native type definition to apiservercel.DeclType once such a utility becomes available.
// The 'uid' field is omitted since it is not needed for in-process admission review.
// The 'object' and 'oldObject' fields are omitted since they are exposed as root level CEL variables.
func buildRequestType() *apiservercel.DeclType {
field := func(name string, declType *apiservercel.DeclType, required bool) *apiservercel.DeclField {
return apiservercel.NewDeclField(name, declType, required, nil, nil)
}
fields := func(fields ...*apiservercel.DeclField) map[string]*apiservercel.DeclField {
result := make(map[string]*apiservercel.DeclField, len(fields))
for _, f := range fields {
result[f.Name] = f
}
return result
}
gvkType := apiservercel.NewObjectType("kubernetes.GroupVersionKind", fields(
field("group", apiservercel.StringType, true),
field("version", apiservercel.StringType, true),
field("kind", apiservercel.StringType, true),
))
gvrType := apiservercel.NewObjectType("kubernetes.GroupVersionResource", fields(
field("group", apiservercel.StringType, true),
field("version", apiservercel.StringType, true),
field("resource", apiservercel.StringType, true),
))
userInfoType := apiservercel.NewObjectType("kubernetes.UserInfo", fields(
field("username", apiservercel.StringType, false),
field("uid", apiservercel.StringType, false),
field("groups", apiservercel.NewListType(apiservercel.StringType, -1), false),
field("extra", apiservercel.NewMapType(apiservercel.StringType, apiservercel.NewListType(apiservercel.StringType, -1), -1), false),
))
return apiservercel.NewObjectType("kubernetes.AdmissionRequest", fields(
field("kind", gvkType, true),
field("resource", gvrType, true),
field("subResource", apiservercel.StringType, false),
field("requestKind", gvkType, true),
field("requestResource", gvrType, true),
field("requestSubResource", apiservercel.StringType, false),
field("name", apiservercel.StringType, true),
field("namespace", apiservercel.StringType, false),
field("operation", apiservercel.StringType, true),
field("userInfo", userInfoType, true),
field("dryRun", apiservercel.BoolType, false),
field("options", apiservercel.DynType, false),
))
}
// CompilationResult represents a compiled ValidatingAdmissionPolicy validation expression.
type CompilationResult struct {
Program cel.Program
Error *apiservercel.Error
}
// CompileValidatingPolicyExpression returns a compiled vaalidating policy CEL expression.
func CompileValidatingPolicyExpression(validationExpression string, hasParams bool) CompilationResult {
var env *cel.Env
envs, err := getEnvs()
if err != nil {
return CompilationResult{
Error: &apiservercel.Error{
Type: apiservercel.ErrorTypeInternal,
Detail: "compiler initialization failed: " + err.Error(),
},
}
}
if hasParams {
env = envs.withParams
} else {
env = envs.noParams
}
ast, issues := env.Compile(validationExpression)
if issues != nil {
return CompilationResult{
Error: &apiservercel.Error{
Type: apiservercel.ErrorTypeInvalid,
Detail: "compilation failed: " + issues.String(),
},
}
}
if ast.OutputType() != cel.BoolType {
return CompilationResult{
Error: &apiservercel.Error{
Type: apiservercel.ErrorTypeInvalid,
Detail: "cel expression must evaluate to a bool",
},
}
}
_, err = cel.AstToCheckedExpr(ast)
if err != nil {
// should be impossible since env.Compile returned no issues
return CompilationResult{
Error: &apiservercel.Error{
Type: apiservercel.ErrorTypeInternal,
Detail: "unexpected compilation error: " + err.Error(),
},
}
}
prog, err := env.Program(ast,
cel.EvalOptions(cel.OptOptimize),
cel.OptimizeRegex(library.ExtensionLibRegexOptimizations...),
cel.InterruptCheckFrequency(checkFrequency),
)
if err != nil {
return CompilationResult{
Error: &apiservercel.Error{
Type: apiservercel.ErrorTypeInvalid,
Detail: "program instantiation failed: " + err.Error(),
},
}
}
return CompilationResult{
Program: prog,
}
}

View File

@@ -0,0 +1,406 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/matching"
"k8s.io/api/admissionregistration/v1alpha1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
celmetrics "k8s.io/apiserver/pkg/admission/cel"
"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/internal/generic"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
)
var _ CELPolicyEvaluator = &celAdmissionController{}
// celAdmissionController is the top-level controller for admission control using CEL
// it is responsible for watching policy definitions, bindings, and config param CRDs
type celAdmissionController struct {
// Context under which the controller runs
runningContext context.Context
policyDefinitionsController generic.Controller[*v1alpha1.ValidatingAdmissionPolicy]
policyBindingController generic.Controller[*v1alpha1.ValidatingAdmissionPolicyBinding]
// dynamicclient used to create informers to watch the param crd types
dynamicClient dynamic.Interface
restMapper meta.RESTMapper
// Provided to the policy's Compile function as an injected dependency to
// assist with compiling its expressions to CEL
validatorCompiler ValidatorCompiler
// Lock which protects:
// - definitionInfo
// - bindingInfos
// - paramCRDControllers
// - definitionsToBindings
// All other fields should be assumed constant
mutex sync.RWMutex
// controller and metadata
paramsCRDControllers map[v1alpha1.ParamKind]*paramInfo
// Index for each definition namespace/name, contains all binding
// namespace/names known to exist for that definition
definitionInfo map[namespacedName]*definitionInfo
// Index for each bindings namespace/name. Contains compiled templates
// for the binding depending on the policy/param combination.
bindingInfos map[namespacedName]*bindingInfo
// Map from namespace/name of a definition to a set of namespace/name
// of bindings which depend on it.
// All keys must have at least one dependent binding
// All binding names MUST exist as a key bindingInfos
definitionsToBindings map[namespacedName]sets.Set[namespacedName]
}
// namespaceName is used as a key in definitionInfo and bindingInfos
type namespacedName struct {
namespace, name string
}
type definitionInfo struct {
// Error about the state of the definition's configuration and the cluster
// preventing its enforcement or compilation.
// Reset every reconciliation
configurationError error
// Last value seen by this controller to be used in policy enforcement
// May not be nil
lastReconciledValue *v1alpha1.ValidatingAdmissionPolicy
}
type bindingInfo struct {
// Compiled CEL expression turned into an validator
validator atomic.Pointer[Validator]
// Last value seen by this controller to be used in policy enforcement
// May not be nil
lastReconciledValue *v1alpha1.ValidatingAdmissionPolicyBinding
}
type paramInfo struct {
// Controller which is watching this param CRD
controller generic.Controller[*unstructured.Unstructured]
// Function to call to stop the informer and clean up the controller
stop func()
// Policy Definitions which refer to this param CRD
dependentDefinitions sets.Set[namespacedName]
}
func NewAdmissionController(
// Injected Dependencies
informerFactory informers.SharedInformerFactory,
client kubernetes.Interface,
restMapper meta.RESTMapper,
dynamicClient dynamic.Interface,
) CELPolicyEvaluator {
matcher := matching.NewMatcher(informerFactory.Core().V1().Namespaces().Lister(), client)
validatorCompiler := &CELValidatorCompiler{
Matcher: matcher,
}
c := &celAdmissionController{
definitionInfo: make(map[namespacedName]*definitionInfo),
bindingInfos: make(map[namespacedName]*bindingInfo),
paramsCRDControllers: make(map[v1alpha1.ParamKind]*paramInfo),
definitionsToBindings: make(map[namespacedName]sets.Set[namespacedName]),
dynamicClient: dynamicClient,
validatorCompiler: validatorCompiler,
restMapper: restMapper,
}
c.policyDefinitionsController = generic.NewController(
generic.NewInformer[*v1alpha1.ValidatingAdmissionPolicy](
informerFactory.Admissionregistration().V1alpha1().ValidatingAdmissionPolicies().Informer()),
c.reconcilePolicyDefinition,
generic.ControllerOptions{
Workers: 1,
Name: "cel-policy-definitions",
},
)
c.policyBindingController = generic.NewController(
generic.NewInformer[*v1alpha1.ValidatingAdmissionPolicyBinding](
informerFactory.Admissionregistration().V1alpha1().ValidatingAdmissionPolicyBindings().Informer()),
c.reconcilePolicyBinding,
generic.ControllerOptions{
Workers: 1,
Name: "cel-policy-bindings",
},
)
return c
}
func (c *celAdmissionController) Run(stopCh <-chan struct{}) {
if c.runningContext != nil {
return
}
ctx, cancel := context.WithCancel(context.Background())
c.runningContext = ctx
defer func() {
c.runningContext = nil
}()
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
c.policyDefinitionsController.Run(ctx)
}()
wg.Add(1)
go func() {
defer wg.Done()
c.policyBindingController.Run(ctx)
}()
<-stopCh
cancel()
wg.Wait()
}
func (c *celAdmissionController) Validate(
ctx context.Context,
a admission.Attributes,
o admission.ObjectInterfaces,
) (err error) {
c.mutex.RLock()
defer c.mutex.RUnlock()
var deniedDecisions []policyDecisionWithMetadata
addConfigError := func(err error, definition *v1alpha1.ValidatingAdmissionPolicy, binding *v1alpha1.ValidatingAdmissionPolicyBinding) {
// we always default the FailurePolicy if it is unset and validate it in API level
var policy v1alpha1.FailurePolicyType
if definition.Spec.FailurePolicy == nil {
policy = v1alpha1.Fail
} else {
policy = *definition.Spec.FailurePolicy
}
// apply FailurePolicy specified in ValidatingAdmissionPolicy, the default would be Fail
switch policy {
case v1alpha1.Ignore:
// TODO: add metrics for ignored error here
return
case v1alpha1.Fail:
var message string
if binding == nil {
message = fmt.Errorf("failed to configure policy: %w", err).Error()
} else {
message = fmt.Errorf("failed to configure binding: %w", err).Error()
}
deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{
policyDecision: policyDecision{
action: actionDeny,
message: message,
},
definition: definition,
binding: binding,
})
default:
deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{
policyDecision: policyDecision{
action: actionDeny,
message: fmt.Errorf("unrecognized failure policy: '%v'", policy).Error(),
},
definition: definition,
binding: binding,
})
}
}
for definitionNamespacedName, definitionInfo := range c.definitionInfo {
definition := definitionInfo.lastReconciledValue
matches, matchKind, err := c.validatorCompiler.DefinitionMatches(a, o, definition)
if err != nil {
// Configuration error.
addConfigError(err, definition, nil)
continue
}
if !matches {
// Policy definition does not match request
continue
} else if definitionInfo.configurationError != nil {
// Configuration error.
addConfigError(definitionInfo.configurationError, definition, nil)
continue
}
dependentBindings := c.definitionsToBindings[definitionNamespacedName]
if len(dependentBindings) == 0 {
continue
}
for namespacedBindingName := range dependentBindings {
// If the key is inside dependentBindings, there is guaranteed to
// be a bindingInfo for it
bindingInfo := c.bindingInfos[namespacedBindingName]
binding := bindingInfo.lastReconciledValue
matches, err := c.validatorCompiler.BindingMatches(a, o, binding)
if err != nil {
// Configuration error.
addConfigError(err, definition, binding)
continue
}
if !matches {
continue
}
var param *unstructured.Unstructured
// If definition has paramKind, paramRef is required in binding.
// If definition has no paramKind, paramRef set in binding will be ignored.
paramKind := definition.Spec.ParamKind
paramRef := binding.Spec.ParamRef
if paramKind != nil && paramRef != nil {
// Find the params referred by the binding by looking its name up
// in our informer for its CRD
paramInfo, ok := c.paramsCRDControllers[*paramKind]
if !ok {
addConfigError(fmt.Errorf("paramKind kind `%v` not known",
paramKind.String()), definition, binding)
continue
}
// If the param informer for this admission policy has not yet
// had time to perform an initial listing, don't attempt to use
// it.
//!TOOD(alexzielenski): add a wait for a very short amount of
// time for the cache to sync
if !paramInfo.controller.HasSynced() {
addConfigError(fmt.Errorf("paramKind kind `%v` not yet synced to use for admission",
paramKind.String()), definition, binding)
continue
}
if len(paramRef.Namespace) == 0 {
param, err = paramInfo.controller.Informer().Get(paramRef.Name)
} else {
param, err = paramInfo.controller.Informer().Namespaced(paramRef.Namespace).Get(paramRef.Name)
}
if err != nil {
// Apply failure policy
addConfigError(err, definition, binding)
if k8serrors.IsInvalid(err) {
// Param mis-configured
// require to set paramRef.namespace for namespaced resource and unset paramRef.namespace for cluster scoped resource
continue
} else if k8serrors.IsNotFound(err) {
// Param not yet available. User may need to wait a bit
// before being able to use it for validation.
continue
}
// There was a bad internal error
utilruntime.HandleError(err)
continue
}
}
validator := bindingInfo.validator.Load()
if validator == nil {
// Compile policy definition using binding
newValidator := c.validatorCompiler.Compile(definition)
validator = &newValidator
bindingInfo.validator.Store(validator)
}
decisions, err := (*validator).Validate(a, o, param, matchKind)
if err != nil {
// runtime error. Apply failure policy
wrappedError := fmt.Errorf("failed to evaluate CEL expression: %w", err)
addConfigError(wrappedError, definition, binding)
continue
}
for _, decision := range decisions {
switch decision.action {
case actionAdmit:
if decision.evaluation == evalError {
celmetrics.Metrics.ObserveAdmissionWithError(ctx, decision.elapsed, definition.Name, binding.Name, "active")
}
case actionDeny:
deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{
definition: definition,
binding: binding,
policyDecision: decision,
})
celmetrics.Metrics.ObserveRejection(ctx, decision.elapsed, definition.Name, binding.Name, "active")
default:
return fmt.Errorf("unrecognized evaluation decision '%s' for ValidatingAdmissionPolicyBinding '%s' with ValidatingAdmissionPolicy '%s'",
decision.action, binding.Name, definition.Name)
}
}
}
}
if len(deniedDecisions) > 0 {
// TODO: refactor admission.NewForbidden so the name extraction is reusable but the code/reason is customizable
var message string
deniedDecision := deniedDecisions[0]
if deniedDecision.binding != nil {
message = fmt.Sprintf("ValidatingAdmissionPolicy '%s' with binding '%s' denied request: %s", deniedDecision.definition.Name, deniedDecision.binding.Name, deniedDecision.message)
} else {
message = fmt.Sprintf("ValidatingAdmissionPolicy '%s' denied request: %s", deniedDecision.definition.Name, deniedDecision.message)
}
err := admission.NewForbidden(a, errors.New(message)).(*k8serrors.StatusError)
reason := deniedDecision.reason
if len(reason) == 0 {
reason = metav1.StatusReasonInvalid
}
err.ErrStatus.Reason = reason
err.ErrStatus.Code = reasonToCode(reason)
err.ErrStatus.Details.Causes = append(err.ErrStatus.Details.Causes, metav1.StatusCause{Message: message})
return err
}
return nil
}
func (c *celAdmissionController) HasSynced() bool {
return c.policyBindingController.HasSynced() &&
c.policyDefinitionsController.HasSynced()
}
func (c *celAdmissionController) ValidateInitialization() error {
return c.validatorCompiler.ValidateInitialization()
}

View File

@@ -0,0 +1,229 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"context"
"fmt"
"time"
"k8s.io/api/admissionregistration/v1alpha1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
celmetrics "k8s.io/apiserver/pkg/admission/cel"
"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/internal/generic"
"k8s.io/client-go/dynamic/dynamicinformer"
"k8s.io/client-go/tools/cache"
)
func (c *celAdmissionController) reconcilePolicyDefinition(namespace, name string, definition *v1alpha1.ValidatingAdmissionPolicy) error {
c.mutex.Lock()
defer c.mutex.Unlock()
// Namespace for policydefinition is empty.
nn := getNamespaceName(namespace, name)
info, ok := c.definitionInfo[nn]
if !ok {
info = &definitionInfo{}
c.definitionInfo[nn] = info
// TODO(DangerOnTheRanger): add support for "warn" being a valid enforcementAction
celmetrics.Metrics.ObserveDefinition(context.TODO(), "active", "deny")
}
var paramSource *v1alpha1.ParamKind
if definition != nil {
paramSource = definition.Spec.ParamKind
}
// If param source has changed, remove definition as dependent of old params
// If there are no more dependents of old param, stop and clean up controller
if info.lastReconciledValue != nil && info.lastReconciledValue.Spec.ParamKind != nil {
oldParamSource := *info.lastReconciledValue.Spec.ParamKind
// If we are:
// - switching from having a param to not having a param (includes deletion)
// - or from having a param to a different one
// we remove dependency on the controller.
if paramSource == nil || *paramSource != oldParamSource {
if oldParamInfo, ok := c.paramsCRDControllers[oldParamSource]; ok {
oldParamInfo.dependentDefinitions.Delete(nn)
if len(oldParamInfo.dependentDefinitions) == 0 {
oldParamInfo.stop()
delete(c.paramsCRDControllers, oldParamSource)
}
}
}
}
// Reset all previously compiled evaluators in case something relevant in
// definition has changed.
for key := range c.definitionsToBindings[nn] {
bindingInfo := c.bindingInfos[key]
bindingInfo.validator.Store(nil)
c.bindingInfos[key] = bindingInfo
}
if definition == nil {
delete(c.definitionInfo, nn)
return nil
}
// Update definition info
info.lastReconciledValue = definition
info.configurationError = nil
if paramSource == nil {
// Skip setting up controller for empty param type
return nil
}
// find GVR for params
// Parse param source into a GVK
paramSourceGV, err := schema.ParseGroupVersion(paramSource.APIVersion)
if err != nil {
// Failed to resolve. Return error so we retry again (rate limited)
// Save a record of this definition with an evaluator that unconditionally
info.configurationError = fmt.Errorf("failed to parse apiVersion of paramKind '%v' with error: %w", paramSource.String(), err)
// Return nil, since this error cannot be resolved by waiting more time
return nil
}
paramsGVR, err := c.restMapper.RESTMapping(schema.GroupKind{
Group: paramSourceGV.Group,
Kind: paramSource.Kind,
}, paramSourceGV.Version)
if err != nil {
// Failed to resolve. Return error so we retry again (rate limited)
// Save a record of this definition with an evaluator that unconditionally
//
info.configurationError = fmt.Errorf("failed to find resource referenced by paramKind: '%v'", paramSourceGV.WithKind(paramSource.Kind))
return info.configurationError
}
// Start watching the param CRD
if _, ok := c.paramsCRDControllers[*paramSource]; !ok {
instanceContext, instanceCancel := context.WithCancel(c.runningContext)
// Watch for new instances of this policy
informer := dynamicinformer.NewFilteredDynamicInformer(
c.dynamicClient,
paramsGVR.Resource,
corev1.NamespaceAll,
30*time.Second,
cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
nil,
)
controller := generic.NewController(
generic.NewInformer[*unstructured.Unstructured](informer.Informer()),
c.reconcileParams,
generic.ControllerOptions{
Workers: 1,
Name: paramSource.String() + "-controller",
},
)
c.paramsCRDControllers[*paramSource] = &paramInfo{
controller: controller,
stop: instanceCancel,
dependentDefinitions: sets.New(nn),
}
go informer.Informer().Run(instanceContext.Done())
go controller.Run(instanceContext)
}
return nil
}
func (c *celAdmissionController) reconcilePolicyBinding(namespace, name string, binding *v1alpha1.ValidatingAdmissionPolicyBinding) error {
c.mutex.Lock()
defer c.mutex.Unlock()
// Namespace for PolicyBinding is empty. In the future a namespaced binding
// may be added
// https://github.com/kubernetes/enhancements/blob/bf5c3c81ea2081d60c1dc7c832faa98479e06209/keps/sig-api-machinery/3488-cel-admission-control/README.md?plain=1#L1042
nn := getNamespaceName(namespace, name)
info, ok := c.bindingInfos[nn]
if !ok {
info = &bindingInfo{}
c.bindingInfos[nn] = info
}
var oldNamespacedDefinitionName namespacedName
if info.lastReconciledValue != nil {
// All validating policies are cluster-scoped so have empty namespace
oldNamespacedDefinitionName = getNamespaceName("", info.lastReconciledValue.Spec.PolicyName)
}
var namespacedDefinitionName namespacedName
if binding != nil {
// All validating policies are cluster-scoped so have empty namespace
namespacedDefinitionName = getNamespaceName("", binding.Spec.PolicyName)
}
// Remove record of binding from old definition if the referred policy
// has changed
if oldNamespacedDefinitionName != namespacedDefinitionName {
if dependentBindings, ok := c.definitionsToBindings[oldNamespacedDefinitionName]; ok {
dependentBindings.Delete(nn)
// if there are no more dependent bindings, remove knowledge of the
// definition altogether
if len(dependentBindings) == 0 {
delete(c.definitionsToBindings, oldNamespacedDefinitionName)
}
}
}
if binding == nil {
delete(c.bindingInfos, nn)
return nil
}
// Add record of binding to new definition
if dependentBindings, ok := c.definitionsToBindings[namespacedDefinitionName]; ok {
dependentBindings.Insert(nn)
} else {
c.definitionsToBindings[namespacedDefinitionName] = sets.New(nn)
}
// Remove compiled template for old binding
info.validator.Store(nil)
info.lastReconciledValue = binding
return nil
}
func (c *celAdmissionController) reconcileParams(namespace, name string, params *unstructured.Unstructured) error {
// Do nothing.
// When we add informational type checking we will need to compile in the
// reconcile loops instead of lazily so we can add compiler errors / type
// checker errors to the status of the resources.
return nil
}
func getNamespaceName(namespace, name string) namespacedName {
return namespacedName{
namespace: namespace,
name: name,
}
}

View File

@@ -14,5 +14,17 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// Package v2alpha1 contains definition of kms-plugin's gRPC service.
package v2alpha1
package validatingadmissionpolicy
import (
"context"
"k8s.io/apiserver/pkg/admission"
)
type CELPolicyEvaluator interface {
admission.InitializationValidator
Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error
HasSynced() bool
Run(stopCh <-chan struct{})
}

View File

@@ -0,0 +1,50 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"k8s.io/api/admissionregistration/v1alpha1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
)
// Validator defines the func used to validate the cel expressions
// matchKind provides the GroupVersionKind that the object should be
// validated by CEL expressions as.
type Validator interface {
Validate(a admission.Attributes, o admission.ObjectInterfaces, versionedParams runtime.Object, matchKind schema.GroupVersionKind) ([]policyDecision, error)
}
// ValidatorCompiler is Dependency Injected into the PolicyDefinition's `Compile`
// function to assist with converting types and values to/from CEL-typed values.
type ValidatorCompiler interface {
admission.InitializationValidator
// Matches says whether this policy definition matches the provided admission
// resource request
DefinitionMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1alpha1.ValidatingAdmissionPolicy) (bool, schema.GroupVersionKind, error)
// Matches says whether this policy definition matches the provided admission
// resource request
BindingMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1alpha1.ValidatingAdmissionPolicyBinding) (bool, error)
// Compile is used for the cel expression compilation
Compile(
policy *v1alpha1.ValidatingAdmissionPolicy,
) Validator
}

View File

@@ -0,0 +1,272 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package generic
import (
"context"
"errors"
"fmt"
"sync"
"time"
kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog/v2"
)
var _ Controller[runtime.Object] = &controller[runtime.Object]{}
type controller[T runtime.Object] struct {
informer Informer[T]
queue workqueue.RateLimitingInterface
// Returns an error if there was a transient error during reconciliation
// and the object should be tried again later.
reconciler func(namespace, name string, newObj T) error
options ControllerOptions
}
type ControllerOptions struct {
Name string
Workers uint
}
func (c *controller[T]) Informer() Informer[T] {
return c.informer
}
func NewController[T runtime.Object](
informer Informer[T],
reconciler func(namepace, name string, newObj T) error,
options ControllerOptions,
) Controller[T] {
if options.Workers == 0 {
options.Workers = 2
}
if len(options.Name) == 0 {
options.Name = fmt.Sprintf("%T-controller", *new(T))
}
return &controller[T]{
options: options,
informer: informer,
reconciler: reconciler,
queue: nil,
}
}
// Runs the controller and returns an error explaining why running was stopped.
// Reconciliation ends as soon as the context completes. If there are events
// waiting to be processed at that itme, they will be dropped.
func (c *controller[T]) Run(ctx context.Context) error {
klog.Infof("starting %s", c.options.Name)
defer klog.Infof("stopping %s", c.options.Name)
c.queue = workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), c.options.Name)
// Forcefully shutdown workqueue. Drop any enqueued items.
// Important to do this in a `defer` at the start of `Run`.
// Otherwise, if there are any early returns without calling this, we
// would never shut down the workqueue
defer c.queue.ShutDown()
enqueue := func(obj interface{}) {
var key string
var err error
if key, err = cache.DeletionHandlingMetaNamespaceKeyFunc(obj); err != nil {
utilruntime.HandleError(err)
return
}
c.queue.Add(key)
}
registration, err := c.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
enqueue(obj)
},
UpdateFunc: func(oldObj, newObj interface{}) {
oldMeta, err1 := meta.Accessor(oldObj)
newMeta, err2 := meta.Accessor(newObj)
if err1 != nil || err2 != nil {
if err1 != nil {
utilruntime.HandleError(err1)
}
if err2 != nil {
utilruntime.HandleError(err2)
}
return
} else if oldMeta.GetResourceVersion() == newMeta.GetResourceVersion() {
if len(oldMeta.GetResourceVersion()) == 0 {
klog.Warningf("%v throwing out update with empty RV. this is likely to happen if a test did not supply a resource version on an updated object", c.options.Name)
}
return
}
enqueue(newObj)
},
DeleteFunc: func(obj interface{}) {
// Enqueue
enqueue(obj)
},
})
// Error might be raised if informer was started and stopped already
if err != nil {
return err
}
// Make sure event handler is removed from informer in case return early from
// an error
defer func() {
// Remove event handler and Handle Error here. Error should only be raised
// for improper usage of event handler API.
if err := c.informer.RemoveEventHandler(registration); err != nil {
utilruntime.HandleError(err)
}
}()
// Wait for initial cache list to complete before beginning to reconcile
// objects.
if !cache.WaitForNamedCacheSync(c.options.Name, ctx.Done(), c.informer.HasSynced) {
// ctx cancelled during cache sync. return early
err := ctx.Err()
if err == nil {
// if context wasnt cancelled then the sync failed for another reason
err = errors.New("cache sync failed")
}
return err
}
waitGroup := sync.WaitGroup{}
for i := uint(0); i < c.options.Workers; i++ {
waitGroup.Add(1)
go func() {
wait.Until(c.runWorker, time.Second, ctx.Done())
waitGroup.Done()
}()
}
klog.Infof("Started %v workers for %v", c.options.Workers, c.options.Name)
// Wait for context cancel.
<-ctx.Done()
// Forcefully shutdown workqueue. Drop any enqueued items.
c.queue.ShutDown()
// Workqueue shutdown signals for workers to stop. Wait for all workers to
// clean up
waitGroup.Wait()
// Only way for workers to ever stop is for caller to cancel the context
return ctx.Err()
}
func (c *controller[T]) HasSynced() bool {
return c.informer.HasSynced()
}
func (c *controller[T]) runWorker() {
for {
key, shutdown := c.queue.Get()
if shutdown {
return
}
// We wrap this block in a func so we can defer c.workqueue.Done.
err := func(obj interface{}) error {
// We call Done here so the workqueue knows we have finished
// processing this item. We also must remember to call Forget if we
// do not want this work item being re-queued. For example, we do
// not call Forget if a transient error occurs, instead the item is
// put back on the workqueue and attempted again after a back-off
// period.
defer c.queue.Done(obj)
var key string
var ok bool
// We expect strings to come off the workqueue. These are of the
// form namespace/name. We do this as the delayed nature of the
// workqueue means the items in the informer cache may actually be
// more up to date that when the item was initially put onto the
// workqueue.
if key, ok = obj.(string); !ok {
// How did an incorrectly formatted key get in the workqueue?
// Done is sufficient. (Forget resets rate limiter for the key,
// but the key is invalid so there is no point in doing that)
return fmt.Errorf("expected string in workqueue but got %#v", obj)
}
if err := c.reconcile(key); err != nil {
// Put the item back on the workqueue to handle any transient errors.
c.queue.AddRateLimited(key)
return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error())
}
// Finally, if no error occurs we Forget this item so it is allowed
// to be re-enqueued without a long rate limit
c.queue.Forget(obj)
klog.V(4).Infof("syncAdmissionPolicy(%q)", key)
return nil
}(key)
if err != nil {
utilruntime.HandleError(err)
}
}
}
func (c *controller[T]) reconcile(key string) error {
var newObj T
var err error
var namespace string
var name string
var lister NamespacedLister[T]
// Convert the namespace/name string into a distinct namespace and name
namespace, name, err = cache.SplitMetaNamespaceKey(key)
if err != nil {
utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key))
return nil
}
if len(namespace) > 0 {
lister = c.informer.Namespaced(namespace)
} else {
lister = c.informer
}
newObj, err = lister.Get(name)
if err != nil {
if !kerrors.IsNotFound(err) {
return err
}
// Deleted object. Inform reconciler with empty
}
return c.reconciler(namespace, name, newObj)
}

View File

@@ -0,0 +1,29 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package generic contains a typed wrapper over cache SharedIndexInformer
// and Lister (maybe eventually should have a home there?)
//
// This interface is being experimented with as an easier way to write controllers
// with a bit less boilerplate.
//
// Informer/Lister classes are thin wrappers providing a type-safe interface
// over regular interface{}-based Informers/Listers
//
// Controller[T] provides a reusable way to reconcile objects out of an informer
// using the tried and true controller design pattern found all over k8s
// codebase based upon syncFunc/reconcile
package generic

View File

@@ -1,5 +1,5 @@
/*
Copyright 2019 The Kubernetes Authors.
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -14,10 +14,23 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// Package v1beta1 contains definition of kms-plugin's gRPC service.
package v1beta1
package generic
// IsVersionCheckMethod determines whether the supplied method is a version check against kms-plugin.
func IsVersionCheckMethod(method string) bool {
return method == "/v1beta1.KeyManagementService/Version"
import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/cache"
)
var _ Informer[runtime.Object] = informer[runtime.Object]{}
type informer[T runtime.Object] struct {
cache.SharedIndexInformer
lister[T]
}
func NewInformer[T runtime.Object](informe cache.SharedIndexInformer) Informer[T] {
return informer[T]{
SharedIndexInformer: informe,
lister: NewLister[T](informe.GetIndexer()),
}
}

View File

@@ -0,0 +1,62 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package generic
import (
"context"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/cache"
)
type Controller[T runtime.Object] interface {
// Meant to be run inside a goroutine
// Waits for and reacts to changes in whatever type the controller
// is concerned with.
//
// Returns an error always non-nil explaining why the worker stopped
Run(ctx context.Context) error
// Retrieves the informer used to back this controller
Informer() Informer[T]
// Returns true if the informer cache has synced, and all the objects from
// the initial list have been reconciled at least once.
HasSynced() bool
}
type NamespacedLister[T any] interface {
// List lists all ValidationRuleSets in the indexer for a given namespace.
// Objects returned here must be treated as read-only.
List(selector labels.Selector) (ret []T, err error)
// Get retrieves the ValidationRuleSet from the indexer for a given namespace and name.
// Objects returned here must be treated as read-only.
Get(name string) (T, error)
}
type Informer[T any] interface {
cache.SharedIndexInformer
Lister[T]
}
// Lister[T] helps list Ts.
// All objects returned here must be treated as read-only.
type Lister[T any] interface {
NamespacedLister[T]
Namespaced(namespace string) NamespacedLister[T]
}

View File

@@ -0,0 +1,100 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package generic
import (
"fmt"
"net/http"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/cache"
)
var _ Lister[runtime.Object] = lister[runtime.Object]{}
type namespacedLister[T runtime.Object] struct {
indexer cache.Indexer
namespace string
}
func (w namespacedLister[T]) List(selector labels.Selector) (ret []T, err error) {
err = cache.ListAllByNamespace(w.indexer, w.namespace, selector, func(m interface{}) {
ret = append(ret, m.(T))
})
return ret, err
}
func (w namespacedLister[T]) Get(name string) (T, error) {
var result T
obj, exists, err := w.indexer.GetByKey(w.namespace + "/" + name)
if err != nil {
return result, err
}
if !exists {
return result, &kerrors.StatusError{ErrStatus: metav1.Status{
Status: metav1.StatusFailure,
Code: http.StatusNotFound,
Reason: metav1.StatusReasonNotFound,
Message: fmt.Sprintf("%s not found", name),
}}
}
result = obj.(T)
return result, nil
}
type lister[T runtime.Object] struct {
indexer cache.Indexer
}
func (w lister[T]) List(selector labels.Selector) (ret []T, err error) {
err = cache.ListAll(w.indexer, selector, func(m interface{}) {
ret = append(ret, m.(T))
})
return ret, err
}
func (w lister[T]) Get(name string) (T, error) {
var result T
obj, exists, err := w.indexer.GetByKey(name)
if err != nil {
return result, err
}
if !exists {
// kerrors.StatusNotFound requires a GroupResource we cannot provide
return result, &kerrors.StatusError{ErrStatus: metav1.Status{
Status: metav1.StatusFailure,
Code: http.StatusNotFound,
Reason: metav1.StatusReasonNotFound,
Message: fmt.Sprintf("%s not found", name),
}}
}
result = obj.(T)
return result, nil
}
func (w lister[T]) Namespaced(namespace string) NamespacedLister[T] {
return namespacedLister[T]{namespace: namespace, indexer: w.indexer}
}
func NewLister[T runtime.Object](indexer cache.Indexer) lister[T] {
return lister[T]{indexer: indexer}
}

View File

@@ -0,0 +1,192 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package matching
import (
"fmt"
v1 "k8s.io/api/admissionregistration/v1"
"k8s.io/api/admissionregistration/v1alpha1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/client-go/kubernetes"
listersv1 "k8s.io/client-go/listers/core/v1"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/namespace"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/object"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/rules"
)
type MatchCriteria interface {
namespace.NamespaceSelectorProvider
object.ObjectSelectorProvider
GetMatchResources() v1alpha1.MatchResources
}
// Matcher decides if a request matches against matchCriteria
type Matcher struct {
namespaceMatcher *namespace.Matcher
objectMatcher *object.Matcher
}
// NewMatcher initialize the matcher with dependencies requires
func NewMatcher(
namespaceLister listersv1.NamespaceLister,
client kubernetes.Interface,
) *Matcher {
return &Matcher{
namespaceMatcher: &namespace.Matcher{
NamespaceLister: namespaceLister,
Client: client,
},
objectMatcher: &object.Matcher{},
}
}
// ValidateInitialization verify if the matcher is ready before use
func (m *Matcher) ValidateInitialization() error {
if err := m.namespaceMatcher.Validate(); err != nil {
return fmt.Errorf("namespaceMatcher is not properly setup: %v", err)
}
return nil
}
func (m *Matcher) Matches(attr admission.Attributes, o admission.ObjectInterfaces, criteria MatchCriteria) (bool, schema.GroupVersionKind, error) {
matches, matchNsErr := m.namespaceMatcher.MatchNamespaceSelector(criteria, attr)
// Should not return an error here for policy which do not apply to the request, even if err is an unexpected scenario.
if !matches && matchNsErr == nil {
return false, schema.GroupVersionKind{}, nil
}
matches, matchObjErr := m.objectMatcher.MatchObjectSelector(criteria, attr)
// Should not return an error here for policy which do not apply to the request, even if err is an unexpected scenario.
if !matches && matchObjErr == nil {
return false, schema.GroupVersionKind{}, nil
}
matchResources := criteria.GetMatchResources()
matchPolicy := matchResources.MatchPolicy
if isExcluded, _, err := matchesResourceRules(matchResources.ExcludeResourceRules, matchPolicy, attr, o); isExcluded || err != nil {
return false, schema.GroupVersionKind{}, err
}
var (
isMatch bool
matchKind schema.GroupVersionKind
matchErr error
)
if len(matchResources.ResourceRules) == 0 {
isMatch = true
matchKind = attr.GetKind()
} else {
isMatch, matchKind, matchErr = matchesResourceRules(matchResources.ResourceRules, matchPolicy, attr, o)
}
if matchErr != nil {
return false, schema.GroupVersionKind{}, matchErr
}
if !isMatch {
return false, schema.GroupVersionKind{}, nil
}
// now that we know this applies to this request otherwise, if there were selector errors, return them
if matchNsErr != nil {
return false, schema.GroupVersionKind{}, matchNsErr
}
if matchObjErr != nil {
return false, schema.GroupVersionKind{}, matchObjErr
}
return true, matchKind, nil
}
func matchesResourceRules(namedRules []v1alpha1.NamedRuleWithOperations, matchPolicy *v1alpha1.MatchPolicyType, attr admission.Attributes, o admission.ObjectInterfaces) (bool, schema.GroupVersionKind, error) {
matchKind := attr.GetKind()
for _, namedRule := range namedRules {
rule := v1.RuleWithOperations(namedRule.RuleWithOperations)
ruleMatcher := rules.Matcher{
Rule: rule,
Attr: attr,
}
if !ruleMatcher.Matches() {
continue
}
// an empty name list always matches
if len(namedRule.ResourceNames) == 0 {
return true, matchKind, nil
}
// TODO: GetName() can return an empty string if the user is relying on
// the API server to generate the name... figure out what to do for this edge case
name := attr.GetName()
for _, matchedName := range namedRule.ResourceNames {
if name == matchedName {
return true, matchKind, nil
}
}
}
// if match policy is undefined or exact, don't perform fuzzy matching
// note that defaulting to fuzzy matching is set by the API
if matchPolicy == nil || *matchPolicy == v1alpha1.Exact {
return false, schema.GroupVersionKind{}, nil
}
attrWithOverride := &attrWithResourceOverride{Attributes: attr}
equivalents := o.GetEquivalentResourceMapper().EquivalentResourcesFor(attr.GetResource(), attr.GetSubresource())
for _, namedRule := range namedRules {
for _, equivalent := range equivalents {
if equivalent == attr.GetResource() {
// we have already checked the original resource
continue
}
attrWithOverride.resource = equivalent
rule := v1.RuleWithOperations(namedRule.RuleWithOperations)
m := rules.Matcher{
Rule: rule,
Attr: attrWithOverride,
}
if !m.Matches() {
continue
}
matchKind = o.GetEquivalentResourceMapper().KindFor(equivalent, attr.GetSubresource())
if matchKind.Empty() {
return false, schema.GroupVersionKind{}, fmt.Errorf("unable to convert to %v: unknown kind", equivalent)
}
// an empty name list always matches
if len(namedRule.ResourceNames) == 0 {
return true, matchKind, nil
}
// TODO: GetName() can return an empty string if the user is relying on
// the API server to generate the name... figure out what to do for this edge case
name := attr.GetName()
for _, matchedName := range namedRule.ResourceNames {
if name == matchedName {
return true, matchKind, nil
}
}
}
}
return false, schema.GroupVersionKind{}, nil
}
type attrWithResourceOverride struct {
admission.Attributes
resource schema.GroupVersionResource
}
func (a *attrWithResourceOverride) GetResource() schema.GroupVersionResource { return a.resource }

View File

@@ -0,0 +1,70 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"net/http"
"time"
"k8s.io/api/admissionregistration/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type policyDecisionAction string
const (
actionAdmit policyDecisionAction = "admit"
actionDeny policyDecisionAction = "deny"
)
type policyDecisionEvaluation string
const (
evalAdmit policyDecisionEvaluation = "admit"
evalError policyDecisionEvaluation = "error"
evalDeny policyDecisionEvaluation = "deny"
)
type policyDecision struct {
action policyDecisionAction
evaluation policyDecisionEvaluation
message string
reason metav1.StatusReason
elapsed time.Duration
}
type policyDecisionWithMetadata struct {
policyDecision
definition *v1alpha1.ValidatingAdmissionPolicy
binding *v1alpha1.ValidatingAdmissionPolicyBinding
}
func reasonToCode(r metav1.StatusReason) int32 {
switch r {
case metav1.StatusReasonForbidden:
return http.StatusForbidden
case metav1.StatusReasonUnauthorized:
return http.StatusUnauthorized
case metav1.StatusReasonRequestEntityTooLarge:
return http.StatusRequestEntityTooLarge
case metav1.StatusReasonInvalid:
return http.StatusUnprocessableEntity
default:
// It should not reach here since we only allow above reason to be set from API level
return http.StatusUnprocessableEntity
}
}

View File

@@ -0,0 +1,318 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"fmt"
"reflect"
"strings"
"time"
celtypes "github.com/google/cel-go/common/types"
"github.com/google/cel-go/interpreter"
admissionv1 "k8s.io/api/admission/v1"
"k8s.io/api/admissionregistration/v1alpha1"
authenticationv1 "k8s.io/api/authentication/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/matching"
"k8s.io/apiserver/pkg/admission/plugin/webhook/generic"
)
var _ ValidatorCompiler = &CELValidatorCompiler{}
var _ matching.MatchCriteria = &matchCriteria{}
type matchCriteria struct {
constraints *v1alpha1.MatchResources
}
// GetParsedNamespaceSelector returns the converted LabelSelector which implements labels.Selector
func (m *matchCriteria) GetParsedNamespaceSelector() (labels.Selector, error) {
return metav1.LabelSelectorAsSelector(m.constraints.NamespaceSelector)
}
// GetParsedObjectSelector returns the converted LabelSelector which implements labels.Selector
func (m *matchCriteria) GetParsedObjectSelector() (labels.Selector, error) {
return metav1.LabelSelectorAsSelector(m.constraints.ObjectSelector)
}
// GetMatchResources returns the matchConstraints
func (m *matchCriteria) GetMatchResources() v1alpha1.MatchResources {
return *m.constraints
}
// CELValidatorCompiler implement the interface ValidatorCompiler.
type CELValidatorCompiler struct {
Matcher *matching.Matcher
}
// DefinitionMatches returns whether this ValidatingAdmissionPolicy matches the provided admission resource request
func (c *CELValidatorCompiler) DefinitionMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1alpha1.ValidatingAdmissionPolicy) (bool, schema.GroupVersionKind, error) {
criteria := matchCriteria{constraints: definition.Spec.MatchConstraints}
return c.Matcher.Matches(a, o, &criteria)
}
// BindingMatches returns whether this ValidatingAdmissionPolicyBinding matches the provided admission resource request
func (c *CELValidatorCompiler) BindingMatches(a admission.Attributes, o admission.ObjectInterfaces, binding *v1alpha1.ValidatingAdmissionPolicyBinding) (bool, error) {
if binding.Spec.MatchResources == nil {
return true, nil
}
criteria := matchCriteria{constraints: binding.Spec.MatchResources}
isMatch, _, err := c.Matcher.Matches(a, o, &criteria)
return isMatch, err
}
// ValidateInitialization checks if Matcher is initialized.
func (c *CELValidatorCompiler) ValidateInitialization() error {
return c.Matcher.ValidateInitialization()
}
type validationActivation struct {
object, oldObject, params, request interface{}
}
// ResolveName returns a value from the activation by qualified name, or false if the name
// could not be found.
func (a *validationActivation) ResolveName(name string) (interface{}, bool) {
switch name {
case ObjectVarName:
return a.object, true
case OldObjectVarName:
return a.oldObject, true
case ParamsVarName:
return a.params, true
case RequestVarName:
return a.request, true
default:
return nil, false
}
}
// Parent returns the parent of the current activation, may be nil.
// If non-nil, the parent will be searched during resolve calls.
func (a *validationActivation) Parent() interpreter.Activation {
return nil
}
// Compile compiles the cel expression defined in ValidatingAdmissionPolicy
func (c *CELValidatorCompiler) Compile(p *v1alpha1.ValidatingAdmissionPolicy) Validator {
if len(p.Spec.Validations) == 0 {
return nil
}
hasParam := false
if p.Spec.ParamKind != nil {
hasParam = true
}
compilationResults := make([]CompilationResult, len(p.Spec.Validations))
for i, validation := range p.Spec.Validations {
compilationResults[i] = CompileValidatingPolicyExpression(validation.Expression, hasParam)
}
return &CELValidator{policy: p, compilationResults: compilationResults}
}
// CELValidator implements the Validator interface
type CELValidator struct {
policy *v1alpha1.ValidatingAdmissionPolicy
compilationResults []CompilationResult
}
func convertObjectToUnstructured(obj interface{}) (*unstructured.Unstructured, error) {
if obj == nil || reflect.ValueOf(obj).IsNil() {
return &unstructured.Unstructured{Object: nil}, nil
}
ret, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
if err != nil {
return nil, err
}
return &unstructured.Unstructured{Object: ret}, nil
}
func objectToResolveVal(r runtime.Object) (interface{}, error) {
if r == nil || reflect.ValueOf(r).IsNil() {
return nil, nil
}
v, err := convertObjectToUnstructured(r)
if err != nil {
return nil, err
}
return v.Object, nil
}
func policyDecisionActionForError(f v1alpha1.FailurePolicyType) policyDecisionAction {
if f == v1alpha1.Ignore {
return actionAdmit
}
return actionDeny
}
// Validate validates all cel expressions in Validator and returns a PolicyDecision for each CEL expression or returns an error.
// An error will be returned if failed to convert the object/oldObject/params/request to unstructured.
// Each PolicyDecision will have a decision and a message.
// policyDecision.message will be empty if the decision is allowed and no error met.
func (v *CELValidator) Validate(a admission.Attributes, o admission.ObjectInterfaces, versionedParams runtime.Object, matchKind schema.GroupVersionKind) ([]policyDecision, error) {
// TODO: replace unstructured with ref.Val for CEL variables when native type support is available
decisions := make([]policyDecision, len(v.compilationResults))
var err error
versionedAttr, err := generic.NewVersionedAttributes(a, matchKind, o)
if err != nil {
return nil, err
}
oldObjectVal, err := objectToResolveVal(versionedAttr.VersionedOldObject)
if err != nil {
return nil, err
}
objectVal, err := objectToResolveVal(versionedAttr.VersionedObject)
if err != nil {
return nil, err
}
paramsVal, err := objectToResolveVal(versionedParams)
if err != nil {
return nil, err
}
request := createAdmissionRequest(versionedAttr.Attributes)
requestVal, err := convertObjectToUnstructured(request)
if err != nil {
return nil, err
}
va := &validationActivation{
object: objectVal,
oldObject: oldObjectVal,
params: paramsVal,
request: requestVal.Object,
}
var f v1alpha1.FailurePolicyType
if v.policy.Spec.FailurePolicy == nil {
f = v1alpha1.Fail
} else {
f = *v.policy.Spec.FailurePolicy
}
for i, compilationResult := range v.compilationResults {
validation := v.policy.Spec.Validations[i]
var policyDecision = &decisions[i]
if compilationResult.Error != nil {
policyDecision.action = policyDecisionActionForError(f)
policyDecision.evaluation = evalError
policyDecision.message = fmt.Sprintf("compilation error: %v", compilationResult.Error)
continue
}
if compilationResult.Program == nil {
policyDecision.action = policyDecisionActionForError(f)
policyDecision.evaluation = evalError
policyDecision.message = "unexpected internal error compiling expression"
continue
}
t1 := time.Now()
evalResult, _, err := compilationResult.Program.Eval(va)
elapsed := time.Since(t1)
policyDecision.elapsed = elapsed
if err != nil {
policyDecision.action = policyDecisionActionForError(f)
policyDecision.evaluation = evalError
policyDecision.message = fmt.Sprintf("expression '%v' resulted in error: %v", v.policy.Spec.Validations[i].Expression, err)
} else if evalResult != celtypes.True {
policyDecision.action = actionDeny
if validation.Reason == nil {
policyDecision.reason = metav1.StatusReasonInvalid
} else {
policyDecision.reason = *validation.Reason
}
if len(validation.Message) > 0 {
policyDecision.message = strings.TrimSpace(validation.Message)
} else {
policyDecision.message = fmt.Sprintf("failed expression: %v", strings.TrimSpace(validation.Expression))
}
} else {
policyDecision.action = actionAdmit
policyDecision.evaluation = evalAdmit
}
}
return decisions, nil
}
func createAdmissionRequest(attr admission.Attributes) *admissionv1.AdmissionRequest {
// FIXME: how to get resource GVK, GVR and subresource?
gvk := attr.GetKind()
gvr := attr.GetResource()
subresource := attr.GetSubresource()
requestGVK := attr.GetKind()
requestGVR := attr.GetResource()
requestSubResource := attr.GetSubresource()
aUserInfo := attr.GetUserInfo()
var userInfo authenticationv1.UserInfo
if aUserInfo != nil {
userInfo = authenticationv1.UserInfo{
Extra: make(map[string]authenticationv1.ExtraValue),
Groups: aUserInfo.GetGroups(),
UID: aUserInfo.GetUID(),
Username: aUserInfo.GetName(),
}
// Convert the extra information in the user object
for key, val := range aUserInfo.GetExtra() {
userInfo.Extra[key] = authenticationv1.ExtraValue(val)
}
}
dryRun := attr.IsDryRun()
return &admissionv1.AdmissionRequest{
Kind: metav1.GroupVersionKind{
Group: gvk.Group,
Kind: gvk.Kind,
Version: gvk.Version,
},
Resource: metav1.GroupVersionResource{
Group: gvr.Group,
Resource: gvr.Resource,
Version: gvr.Version,
},
SubResource: subresource,
RequestKind: &metav1.GroupVersionKind{
Group: requestGVK.Group,
Kind: requestGVK.Kind,
Version: requestGVK.Version,
},
RequestResource: &metav1.GroupVersionResource{
Group: requestGVR.Group,
Resource: requestGVR.Resource,
Version: requestGVR.Version,
},
RequestSubResource: requestSubResource,
Name: attr.GetName(),
Namespace: attr.GetNamespace(),
Operation: admissionv1.Operation(attr.GetOperation()),
UserInfo: userInfo,
// Leave Object and OldObject unset since we don't provide access to them via request
DryRun: &dryRun,
Options: runtime.RawExtension{
Object: attr.GetOperationOptions(),
},
}
}

View File

@@ -22,12 +22,19 @@ import (
"k8s.io/api/admissionregistration/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/namespace"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/object"
webhookutil "k8s.io/apiserver/pkg/util/webhook"
"k8s.io/client-go/rest"
)
// WebhookAccessor provides a common interface to both mutating and validating webhook types.
type WebhookAccessor interface {
// This accessor provides the methods needed to support matching against webhook
// predicates
namespace.NamespaceSelectorProvider
object.ObjectSelectorProvider
// GetUID gets a string that uniquely identifies the webhook.
GetUID() string
@@ -36,10 +43,6 @@ type WebhookAccessor interface {
// GetRESTClient gets the webhook client
GetRESTClient(clientManager *webhookutil.ClientManager) (*rest.RESTClient, error)
// GetParsedNamespaceSelector gets the webhook NamespaceSelector field.
GetParsedNamespaceSelector() (labels.Selector, error)
// GetParsedObjectSelector gets the webhook ObjectSelector field.
GetParsedObjectSelector() (labels.Selector, error)
// GetName gets the webhook Name field. Note that the name is scoped to the webhook
// configuration and does not provide a globally unique identity, if a unique identity is

View File

@@ -30,9 +30,9 @@ import (
genericadmissioninit "k8s.io/apiserver/pkg/admission/initializer"
"k8s.io/apiserver/pkg/admission/plugin/webhook"
"k8s.io/apiserver/pkg/admission/plugin/webhook/config"
"k8s.io/apiserver/pkg/admission/plugin/webhook/namespace"
"k8s.io/apiserver/pkg/admission/plugin/webhook/object"
"k8s.io/apiserver/pkg/admission/plugin/webhook/rules"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/namespace"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/object"
"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/rules"
webhookutil "k8s.io/apiserver/pkg/util/webhook"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
@@ -219,7 +219,7 @@ func (a *attrWithResourceOverride) GetResource() schema.GroupVersionResource { r
// Dispatch is called by the downstream Validate or Admit methods.
func (a *Webhook) Dispatch(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces) error {
if rules.IsWebhookConfigurationResource(attr) {
if rules.IsExemptAdmissionConfigurationResource(attr) {
return nil
}
if !a.WaitForReady() {

View File

@@ -24,6 +24,7 @@ import (
"time"
jsonpatch "github.com/evanphx/json-patch"
"go.opentelemetry.io/otel/attribute"
apiequality "k8s.io/apimachinery/pkg/api/equality"
"k8s.io/klog/v2"
@@ -46,7 +47,7 @@ import (
endpointsrequest "k8s.io/apiserver/pkg/endpoints/request"
webhookutil "k8s.io/apiserver/pkg/util/webhook"
"k8s.io/apiserver/pkg/warning"
utiltrace "k8s.io/utils/trace"
"k8s.io/component-base/tracing"
)
const (
@@ -233,14 +234,14 @@ func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *admiss
if err != nil {
return false, &webhookutil.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("could not get REST client: %w", err), Status: apierrors.NewBadRequest("error getting REST client")}
}
trace := utiltrace.New("Call mutating webhook",
utiltrace.Field{"configuration", configurationName},
utiltrace.Field{"webhook", h.Name},
utiltrace.Field{"resource", attr.GetResource()},
utiltrace.Field{"subresource", attr.GetSubresource()},
utiltrace.Field{"operation", attr.GetOperation()},
utiltrace.Field{"UID", uid})
defer trace.LogIfLong(500 * time.Millisecond)
ctx, span := tracing.Start(ctx, "Call mutating webhook",
attribute.String("configuration", configurationName),
attribute.String("webhook", h.Name),
attribute.Stringer("resource", attr.GetResource()),
attribute.String("subresource", attr.GetSubresource()),
attribute.String("operation", string(attr.GetOperation())),
attribute.String("UID", string(uid)))
defer span.End(500 * time.Millisecond)
// if the webhook has a specific timeout, wrap the context to apply it
if h.TimeoutSeconds != nil {
@@ -279,7 +280,7 @@ func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *admiss
}
return false, &webhookutil.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("failed to call webhook: %w", err), Status: status}
}
trace.Step("Request completed")
span.AddEvent("Request completed")
result, err := webhookrequest.VerifyAdmissionResponse(uid, true, response)
if err != nil {
@@ -353,7 +354,7 @@ func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *admiss
}
changed = !apiequality.Semantic.DeepEqual(attr.VersionedObject, newVersionedObject)
trace.Step("Patch applied")
span.AddEvent("Patch applied")
annotator.addPatchAnnotation(patchObj, result.PatchType)
attr.Dirty = true
attr.VersionedObject = newVersionedObject

View File

@@ -17,4 +17,4 @@ limitations under the License.
// Package namespace defines the utilities that are used by the webhook
// plugin to decide if a webhook should be applied to an object based on its
// namespace.
package namespace // import "k8s.io/apiserver/pkg/admission/plugin/webhook/namespace"
package namespace // import "k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/namespace"

View File

@@ -26,11 +26,15 @@ import (
"k8s.io/apimachinery/pkg/labels"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/webhook"
clientset "k8s.io/client-go/kubernetes"
corelisters "k8s.io/client-go/listers/core/v1"
)
type NamespaceSelectorProvider interface {
// GetNamespaceSelector gets the webhook NamespaceSelector field.
GetParsedNamespaceSelector() (labels.Selector, error)
}
// Matcher decides if a request is exempted by the NamespaceSelector of a
// webhook configuration.
type Matcher struct {
@@ -87,7 +91,7 @@ func (m *Matcher) GetNamespaceLabels(attr admission.Attributes) (map[string]stri
// MatchNamespaceSelector decideds whether the request matches the
// namespaceSelctor of the webhook. Only when they match, the webhook is called.
func (m *Matcher) MatchNamespaceSelector(h webhook.WebhookAccessor, attr admission.Attributes) (bool, *apierrors.StatusError) {
func (m *Matcher) MatchNamespaceSelector(p NamespaceSelectorProvider, attr admission.Attributes) (bool, *apierrors.StatusError) {
namespaceName := attr.GetNamespace()
if len(namespaceName) == 0 && attr.GetResource().Resource != "namespaces" {
// If the request is about a cluster scoped resource, and it is not a
@@ -96,7 +100,7 @@ func (m *Matcher) MatchNamespaceSelector(h webhook.WebhookAccessor, attr admissi
// Also update the comment in types.go
return true, nil
}
selector, err := h.GetParsedNamespaceSelector()
selector, err := p.GetParsedNamespaceSelector()
if err != nil {
return false, apierrors.NewInternalError(err)
}

View File

@@ -17,4 +17,4 @@ limitations under the License.
// Package object defines the utilities that are used by the webhook plugin to
// decide if a webhook should run, as long as either the old object or the new
// object has labels matching the webhook config's objectSelector.
package object // import "k8s.io/apiserver/pkg/admission/plugin/webhook/object"
package object // import "k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/object"

View File

@@ -22,10 +22,14 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/webhook"
"k8s.io/klog/v2"
)
type ObjectSelectorProvider interface {
// GetObjectSelector gets the webhook ObjectSelector field.
GetParsedObjectSelector() (labels.Selector, error)
}
// Matcher decides if a request selected by the ObjectSelector.
type Matcher struct {
}
@@ -45,8 +49,8 @@ func matchObject(obj runtime.Object, selector labels.Selector) bool {
// MatchObjectSelector decideds whether the request matches the ObjectSelector
// of the webhook. Only when they match, the webhook is called.
func (m *Matcher) MatchObjectSelector(h webhook.WebhookAccessor, attr admission.Attributes) (bool, *apierrors.StatusError) {
selector, err := h.GetParsedObjectSelector()
func (m *Matcher) MatchObjectSelector(p ObjectSelectorProvider, attr admission.Attributes) (bool, *apierrors.StatusError) {
selector, err := p.GetParsedObjectSelector()
if err != nil {
return false, apierrors.NewInternalError(err)
}

View File

@@ -116,12 +116,12 @@ func (r *Matcher) resource() bool {
return false
}
// IsWebhookConfigurationResource determines if an admission.Attributes object is describing
// the admission of a ValidatingWebhookConfiguration or a MutatingWebhookConfiguration
func IsWebhookConfigurationResource(attr admission.Attributes) bool {
// IsExemptAdmissionConfigurationResource determines if an admission.Attributes object is describing
// the admission of a ValidatingWebhookConfiguration or a MutatingWebhookConfiguration or a ValidatingAdmissionPolicy or a ValidatingAdmissionPolicyBinding
func IsExemptAdmissionConfigurationResource(attr admission.Attributes) bool {
gvk := attr.GetKind()
if gvk.Group == "admissionregistration.k8s.io" {
if gvk.Kind == "ValidatingWebhookConfiguration" || gvk.Kind == "MutatingWebhookConfiguration" {
if gvk.Kind == "ValidatingWebhookConfiguration" || gvk.Kind == "MutatingWebhookConfiguration" || gvk.Kind == "ValidatingAdmissionPolicy" || gvk.Kind == "ValidatingAdmissionPolicyBinding" {
return true
}
}

View File

@@ -22,6 +22,8 @@ import (
"sync"
"time"
"go.opentelemetry.io/otel/attribute"
v1 "k8s.io/api/admissionregistration/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -35,8 +37,8 @@ import (
endpointsrequest "k8s.io/apiserver/pkg/endpoints/request"
webhookutil "k8s.io/apiserver/pkg/util/webhook"
"k8s.io/apiserver/pkg/warning"
"k8s.io/component-base/tracing"
"k8s.io/klog/v2"
utiltrace "k8s.io/utils/trace"
)
const (
@@ -232,14 +234,14 @@ func (d *validatingDispatcher) callHook(ctx context.Context, h *v1.ValidatingWeb
if err != nil {
return &webhookutil.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("could not get REST client: %w", err), Status: apierrors.NewBadRequest("error getting REST client")}
}
trace := utiltrace.New("Call validating webhook",
utiltrace.Field{"configuration", invocation.Webhook.GetConfigurationName()},
utiltrace.Field{"webhook", h.Name},
utiltrace.Field{"resource", attr.GetResource()},
utiltrace.Field{"subresource", attr.GetSubresource()},
utiltrace.Field{"operation", attr.GetOperation()},
utiltrace.Field{"UID", uid})
defer trace.LogIfLong(500 * time.Millisecond)
ctx, span := tracing.Start(ctx, "Call validating webhook",
attribute.String("configuration", invocation.Webhook.GetConfigurationName()),
attribute.String("webhook", h.Name),
attribute.Stringer("resource", attr.GetResource()),
attribute.String("subresource", attr.GetSubresource()),
attribute.String("operation", string(attr.GetOperation())),
attribute.String("UID", string(uid)))
defer span.End(500 * time.Millisecond)
// if the webhook has a specific timeout, wrap the context to apply it
if h.TimeoutSeconds != nil {
@@ -278,7 +280,7 @@ func (d *validatingDispatcher) callHook(ctx context.Context, h *v1.ValidatingWeb
}
return &webhookutil.ErrCallingWebhook{WebhookName: h.Name, Reason: fmt.Errorf("failed to call webhook: %w", err), Status: status}
}
trace.Step("Request completed")
span.AddEvent("Request completed")
result, err := webhookrequest.VerifyAdmissionResponse(uid, false, response)
if err != nil {

View File

@@ -85,7 +85,7 @@ func (ps *Plugins) Register(name string, plugin Factory) {
ps.registry[name] = plugin
}
// getPlugin creates an instance of the named plugin. It returns `false` if the
// getPlugin creates an instance of the named plugin. It returns `false` if
// the name is not known. The error is returned only when the named provider was
// known but failed to initialize. The config parameter specifies the io.Reader
// handler of the configuration file for the cloud provider, or nil for no configuration.

View File

@@ -23,6 +23,7 @@ import (
"net/url"
"strings"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/apis/config"
)
@@ -40,18 +41,21 @@ const (
nonZeroErrFmt = "%s should be a positive value, or negative to disable"
encryptionConfigNilErr = "EncryptionConfiguration can't be nil"
invalidKMSConfigNameErrFmt = "invalid KMS provider name %s, must not contain ':'"
duplicateKMSConfigNameErrFmt = "duplicate KMS provider name %s, names must be unique"
)
var (
aesKeySizes = []int{16, 24, 32}
// See https://golang.org/pkg/crypto/aes/#NewCipher for details on supported key sizes for AES.
secretBoxKeySizes = []int{32}
aesKeySizes = []int{16, 24, 32}
// See https://godoc.org/golang.org/x/crypto/nacl/secretbox#Open for details on the supported key sizes for Secretbox.
secretBoxKeySizes = []int{32}
root = field.NewPath("resources")
)
// ValidateEncryptionConfiguration validates a v1.EncryptionConfiguration.
func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration) field.ErrorList {
func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload bool) field.ErrorList {
allErrs := field.ErrorList{}
if c == nil {
@@ -64,6 +68,8 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration) field.Er
return allErrs
}
// kmsProviderNames is used to track config names to ensure they are unique.
kmsProviderNames := sets.NewString()
for i, conf := range c.Resources {
r := root.Index(i).Child("resources")
p := root.Index(i).Child("providers")
@@ -82,7 +88,8 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration) field.Er
switch {
case provider.KMS != nil:
allErrs = append(allErrs, validateKMSConfiguration(provider.KMS, path.Child("kms"))...)
allErrs = append(allErrs, validateKMSConfiguration(provider.KMS, path.Child("kms"), kmsProviderNames, reload)...)
kmsProviderNames.Insert(provider.KMS.Name)
case provider.AESGCM != nil:
allErrs = append(allErrs, validateKeys(provider.AESGCM.Keys, path.Child("aesgcm").Child("keys"), aesKeySizes)...)
case provider.AESCBC != nil:
@@ -96,7 +103,7 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration) field.Er
return allErrs
}
func validateSingleProvider(provider config.ProviderConfiguration, filedPath *field.Path) field.ErrorList {
func validateSingleProvider(provider config.ProviderConfiguration, fieldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
found := 0
@@ -117,11 +124,11 @@ func validateSingleProvider(provider config.ProviderConfiguration, filedPath *fi
}
if found == 0 {
return append(allErrs, field.Invalid(filedPath, provider, "provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity"))
return append(allErrs, field.Invalid(fieldPath, provider, "provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity"))
}
if found > 1 {
return append(allErrs, field.Invalid(filedPath, provider, moreThanOneElementErr))
return append(allErrs, field.Invalid(fieldPath, provider, moreThanOneElementErr))
}
return allErrs
@@ -175,10 +182,10 @@ func validateKey(key config.Key, fieldPath *field.Path, expectedLen []int) field
return allErrs
}
func validateKMSConfiguration(c *config.KMSConfiguration, fieldPath *field.Path) field.ErrorList {
func validateKMSConfiguration(c *config.KMSConfiguration, fieldPath *field.Path, kmsProviderNames sets.String, reload bool) field.ErrorList {
allErrs := field.ErrorList{}
allErrs = append(allErrs, validateKMSConfigName(c, fieldPath.Child("name"))...)
allErrs = append(allErrs, validateKMSConfigName(c, fieldPath.Child("name"), kmsProviderNames, reload)...)
allErrs = append(allErrs, validateKMSTimeout(c, fieldPath.Child("timeout"))...)
allErrs = append(allErrs, validateKMSEndpoint(c, fieldPath.Child("endpoint"))...)
allErrs = append(allErrs, validateKMSCacheSize(c, fieldPath.Child("cachesize"))...)
@@ -231,15 +238,24 @@ func validateKMSAPIVersion(c *config.KMSConfiguration, fieldPath *field.Path) fi
return allErrs
}
func validateKMSConfigName(c *config.KMSConfiguration, fieldPath *field.Path) field.ErrorList {
func validateKMSConfigName(c *config.KMSConfiguration, fieldPath *field.Path, kmsProviderNames sets.String, reload bool) field.ErrorList {
allErrs := field.ErrorList{}
if c.Name == "" {
allErrs = append(allErrs, field.Required(fieldPath, fmt.Sprintf(mandatoryFieldErrFmt, "name", "provider")))
}
// kms v2 providers are not allowed to have a ":" in their name
if c.APIVersion != "v1" && strings.Contains(c.Name, ":") {
allErrs = append(allErrs, field.Invalid(fieldPath, c.Name, fmt.Sprintf(invalidKMSConfigNameErrFmt, c.Name)))
}
// kms v2 providers name must always be unique across all kms providers (v1 and v2)
// kms v1 provider names must be unique across all kms providers (v1 and v2) when hot reloading of encryption configuration is enabled (reload=true)
if reload || c.APIVersion != "v1" {
if kmsProviderNames.Has(c.Name) {
allErrs = append(allErrs, field.Invalid(fieldPath, c.Name, fmt.Sprintf(duplicateKMSConfigNameErrFmt, c.Name)))
}
}
return allErrs
}

View File

@@ -19,10 +19,11 @@ package bootstrap
import (
coordinationv1 "k8s.io/api/coordination/v1"
corev1 "k8s.io/api/core/v1"
flowcontrol "k8s.io/api/flowcontrol/v1beta2"
flowcontrol "k8s.io/api/flowcontrol/v1beta3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/authentication/serviceaccount"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/utils/pointer"
)
// The objects that define an apiserver's initial behavior. The
@@ -95,7 +96,8 @@ var (
flowcontrol.PriorityLevelConfigurationSpec{
Type: flowcontrol.PriorityLevelEnablementLimited,
Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
AssuredConcurrencyShares: 5,
NominalConcurrencyShares: 5,
LendablePercent: pointer.Int32(0),
LimitResponse: flowcontrol.LimitResponse{
Type: flowcontrol.LimitResponseTypeReject,
},
@@ -167,7 +169,8 @@ var (
flowcontrol.PriorityLevelConfigurationSpec{
Type: flowcontrol.PriorityLevelEnablementLimited,
Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
AssuredConcurrencyShares: 30,
NominalConcurrencyShares: 30,
LendablePercent: pointer.Int32(33),
LimitResponse: flowcontrol.LimitResponse{
Type: flowcontrol.LimitResponseTypeQueue,
Queuing: &flowcontrol.QueuingConfiguration{
@@ -183,7 +186,8 @@ var (
flowcontrol.PriorityLevelConfigurationSpec{
Type: flowcontrol.PriorityLevelEnablementLimited,
Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
AssuredConcurrencyShares: 40,
NominalConcurrencyShares: 40,
LendablePercent: pointer.Int32(25),
LimitResponse: flowcontrol.LimitResponse{
Type: flowcontrol.LimitResponseTypeQueue,
Queuing: &flowcontrol.QueuingConfiguration{
@@ -200,7 +204,8 @@ var (
flowcontrol.PriorityLevelConfigurationSpec{
Type: flowcontrol.PriorityLevelEnablementLimited,
Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
AssuredConcurrencyShares: 10,
NominalConcurrencyShares: 10,
LendablePercent: pointer.Int32(0),
LimitResponse: flowcontrol.LimitResponse{
Type: flowcontrol.LimitResponseTypeQueue,
Queuing: &flowcontrol.QueuingConfiguration{
@@ -217,7 +222,8 @@ var (
flowcontrol.PriorityLevelConfigurationSpec{
Type: flowcontrol.PriorityLevelEnablementLimited,
Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
AssuredConcurrencyShares: 40,
NominalConcurrencyShares: 40,
LendablePercent: pointer.Int32(50),
LimitResponse: flowcontrol.LimitResponse{
Type: flowcontrol.LimitResponseTypeQueue,
Queuing: &flowcontrol.QueuingConfiguration{
@@ -234,7 +240,8 @@ var (
flowcontrol.PriorityLevelConfigurationSpec{
Type: flowcontrol.PriorityLevelEnablementLimited,
Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
AssuredConcurrencyShares: 100,
NominalConcurrencyShares: 100,
LendablePercent: pointer.Int32(90),
LimitResponse: flowcontrol.LimitResponse{
Type: flowcontrol.LimitResponseTypeQueue,
Queuing: &flowcontrol.QueuingConfiguration{
@@ -251,7 +258,8 @@ var (
flowcontrol.PriorityLevelConfigurationSpec{
Type: flowcontrol.PriorityLevelEnablementLimited,
Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
AssuredConcurrencyShares: 20,
NominalConcurrencyShares: 20,
LendablePercent: pointer.Int32(50),
LimitResponse: flowcontrol.LimitResponse{
Type: flowcontrol.LimitResponseTypeQueue,
Queuing: &flowcontrol.QueuingConfiguration{

View File

@@ -20,6 +20,7 @@ import (
"context"
"sync"
"k8s.io/apimachinery/pkg/types"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/klog/v2"
@@ -28,38 +29,31 @@ import (
// The key type is unexported to prevent collisions
type key int
const (
// auditAnnotationsKey is the context key for the audit annotations.
// TODO: consolidate all audit info under the AuditContext, rather than storing 3 separate keys.
auditAnnotationsKey key = iota
// auditKey is the context key for storing the audit context that is being
// captured and the evaluated policy that applies to the given request.
const auditKey key = iota
// auditKey is the context key for storing the audit event that is being
// captured and the evaluated policy that applies to the given request.
auditKey
// AuditContext holds the information for constructing the audit events for the current request.
type AuditContext struct {
// RequestAuditConfig is the audit configuration that applies to the request
RequestAuditConfig RequestAuditConfig
// auditAnnotationsMutexKey is the context key for the audit annotations mutex.
auditAnnotationsMutexKey
)
// Event is the audit Event object that is being captured to be written in
// the API audit log. It is set to nil when the request is not being audited.
Event *auditinternal.Event
// annotations = *[]annotation instead of a map to preserve order of insertions
type annotation struct {
key, value string
// annotations holds audit annotations that are recorded before the event has been initialized.
// This is represented as a slice rather than a map to preserve order.
annotations []annotation
// annotationMutex guards annotations AND event.Annotations
annotationMutex sync.Mutex
// auditID is the Audit ID associated with this request.
auditID types.UID
}
// WithAuditAnnotations returns a new context that can store audit annotations
// via the AddAuditAnnotation function. This function is meant to be called from
// an early request handler to allow all later layers to set audit annotations.
// This is required to support flows where handlers that come before WithAudit
// (such as WithAuthentication) wish to set audit annotations.
func WithAuditAnnotations(parent context.Context) context.Context {
// this should never really happen, but prevent double registration of this slice
if _, ok := parent.Value(auditAnnotationsKey).(*[]annotation); ok {
return parent
}
parent = withAuditAnnotationsMutex(parent)
var annotations []annotation // avoid allocations until we actually need it
return genericapirequest.WithValue(parent, auditAnnotationsKey, &annotations)
type annotation struct {
key, value string
}
// AddAuditAnnotation sets the audit annotation for the given key, value pair.
@@ -70,102 +64,79 @@ func WithAuditAnnotations(parent context.Context) context.Context {
// Handlers that are unaware of their position in the overall request flow should
// prefer AddAuditAnnotation over LogAnnotation to avoid dropping annotations.
func AddAuditAnnotation(ctx context.Context, key, value string) {
mutex, ok := auditAnnotationsMutex(ctx)
if !ok {
ac := AuditContextFrom(ctx)
if ac == nil {
// auditing is not enabled
return
}
mutex.Lock()
defer mutex.Unlock()
ac.annotationMutex.Lock()
defer ac.annotationMutex.Unlock()
ae := AuditEventFrom(ctx)
var ctxAnnotations *[]annotation
if ae == nil {
ctxAnnotations, _ = ctx.Value(auditAnnotationsKey).(*[]annotation)
}
addAuditAnnotationLocked(ae, ctxAnnotations, key, value)
addAuditAnnotationLocked(ac, key, value)
}
// AddAuditAnnotations is a bulk version of AddAuditAnnotation. Refer to AddAuditAnnotation for
// restrictions on when this can be called.
// keysAndValues are the key-value pairs to add, and must have an even number of items.
func AddAuditAnnotations(ctx context.Context, keysAndValues ...string) {
mutex, ok := auditAnnotationsMutex(ctx)
if !ok {
ac := AuditContextFrom(ctx)
if ac == nil {
// auditing is not enabled
return
}
mutex.Lock()
defer mutex.Unlock()
ae := AuditEventFrom(ctx)
var ctxAnnotations *[]annotation
if ae == nil {
ctxAnnotations, _ = ctx.Value(auditAnnotationsKey).(*[]annotation)
}
ac.annotationMutex.Lock()
defer ac.annotationMutex.Unlock()
if len(keysAndValues)%2 != 0 {
klog.Errorf("Dropping mismatched audit annotation %q", keysAndValues[len(keysAndValues)-1])
}
for i := 0; i < len(keysAndValues); i += 2 {
addAuditAnnotationLocked(ae, ctxAnnotations, keysAndValues[i], keysAndValues[i+1])
addAuditAnnotationLocked(ac, keysAndValues[i], keysAndValues[i+1])
}
}
// AddAuditAnnotationsMap is a bulk version of AddAuditAnnotation. Refer to AddAuditAnnotation for
// restrictions on when this can be called.
func AddAuditAnnotationsMap(ctx context.Context, annotations map[string]string) {
mutex, ok := auditAnnotationsMutex(ctx)
if !ok {
ac := AuditContextFrom(ctx)
if ac == nil {
// auditing is not enabled
return
}
mutex.Lock()
defer mutex.Unlock()
ae := AuditEventFrom(ctx)
var ctxAnnotations *[]annotation
if ae == nil {
ctxAnnotations, _ = ctx.Value(auditAnnotationsKey).(*[]annotation)
}
ac.annotationMutex.Lock()
defer ac.annotationMutex.Unlock()
for k, v := range annotations {
addAuditAnnotationLocked(ae, ctxAnnotations, k, v)
addAuditAnnotationLocked(ac, k, v)
}
}
// addAuditAnnotationLocked is the shared code for recording an audit annotation. This method should
// only be called while the auditAnnotationsMutex is locked.
func addAuditAnnotationLocked(ae *auditinternal.Event, annotations *[]annotation, key, value string) {
if ae != nil {
logAnnotation(ae, key, value)
} else if annotations != nil {
*annotations = append(*annotations, annotation{key: key, value: value})
func addAuditAnnotationLocked(ac *AuditContext, key, value string) {
if ac.Event != nil {
logAnnotation(ac.Event, key, value)
} else {
ac.annotations = append(ac.annotations, annotation{key: key, value: value})
}
}
// This is private to prevent reads/write to the slice from outside of this package.
// The audit event should be directly read to get access to the annotations.
func addAuditAnnotationsFrom(ctx context.Context, ev *auditinternal.Event) {
mutex, ok := auditAnnotationsMutex(ctx)
if !ok {
ac := AuditContextFrom(ctx)
if ac == nil {
// auditing is not enabled
return
}
mutex.Lock()
defer mutex.Unlock()
ac.annotationMutex.Lock()
defer ac.annotationMutex.Unlock()
annotations, ok := ctx.Value(auditAnnotationsKey).(*[]annotation)
if !ok {
return // no annotations to copy
}
for _, kv := range *annotations {
for _, kv := range ac.annotations {
logAnnotation(ev, kv.key, kv.value)
}
}
@@ -185,12 +156,13 @@ func logAnnotation(ae *auditinternal.Event, key, value string) {
ae.Annotations[key] = value
}
// WithAuditContext returns a new context that stores the pair of the audit
// configuration object that applies to the given request and
// the audit event that is going to be written to the API audit log.
func WithAuditContext(parent context.Context, ev *AuditContext) context.Context {
parent = withAuditAnnotationsMutex(parent)
return genericapirequest.WithValue(parent, auditKey, ev)
// WithAuditContext returns a new context that stores the AuditContext.
func WithAuditContext(parent context.Context) context.Context {
if AuditContextFrom(parent) != nil {
return parent // Avoid double registering.
}
return genericapirequest.WithValue(parent, auditKey, &AuditContext{})
}
// AuditEventFrom returns the audit event struct on the ctx
@@ -209,17 +181,46 @@ func AuditContextFrom(ctx context.Context) *AuditContext {
return ev
}
// WithAuditAnnotationMutex adds a mutex for guarding context.AddAuditAnnotation.
func withAuditAnnotationsMutex(parent context.Context) context.Context {
if _, ok := parent.Value(auditAnnotationsMutexKey).(*sync.Mutex); ok {
return parent
// WithAuditID sets the AuditID on the AuditContext. The AuditContext must already be present in the
// request context. If the specified auditID is empty, no value is set.
func WithAuditID(ctx context.Context, auditID types.UID) {
if auditID == "" {
return
}
ac := AuditContextFrom(ctx)
if ac == nil {
return
}
ac.auditID = auditID
if ac.Event != nil {
ac.Event.AuditID = auditID
}
var mutex sync.Mutex
return genericapirequest.WithValue(parent, auditAnnotationsMutexKey, &mutex)
}
// AuditAnnotationsMutex returns the audit annotations mutex from the context.
func auditAnnotationsMutex(ctx context.Context) (*sync.Mutex, bool) {
mutex, ok := ctx.Value(auditAnnotationsMutexKey).(*sync.Mutex)
return mutex, ok
// AuditIDFrom returns the value of the audit ID from the request context.
func AuditIDFrom(ctx context.Context) (types.UID, bool) {
if ac := AuditContextFrom(ctx); ac != nil {
return ac.auditID, ac.auditID != ""
}
return "", false
}
// GetAuditIDTruncated returns the audit ID (truncated) from the request context.
// If the length of the Audit-ID value exceeds the limit, we truncate it to keep
// the first N (maxAuditIDLength) characters.
// This is intended to be used in logging only.
func GetAuditIDTruncated(ctx context.Context) string {
auditID, ok := AuditIDFrom(ctx)
if !ok {
return ""
}
// if the user has specified a very long audit ID then we will use the first N characters
// Note: assuming Audit-ID header is in ASCII
const maxAuditIDLength = 64
if len(auditID) > maxAuditIDLength {
auditID = auditID[:maxAuditIDLength]
}
return string(auditID)
}

View File

@@ -21,18 +21,6 @@ import (
"k8s.io/apiserver/pkg/authorization/authorizer"
)
// AuditContext is a pair of the audit configuration object that applies to
// a given request and the audit Event object that is being captured.
// It's a convenient placeholder to store both these objects in the request context.
type AuditContext struct {
// RequestAuditConfig is the audit configuration that applies to the request
RequestAuditConfig RequestAuditConfig
// Event is the audit Event object that is being captured to be written in
// the API audit log. It is set to nil when the request is not being audited.
Event *audit.Event
}
// RequestAuditConfig is the evaluated audit configuration that is applicable to
// a given request. PolicyRuleEvaluator evaluates the audit policy against the
// authorizer attributes and returns a RequestAuditConfig that applies to the request.

View File

@@ -20,12 +20,13 @@ import (
"fmt"
"io/ioutil"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
"k8s.io/apiserver/pkg/apis/audit/validation"
"k8s.io/apiserver/pkg/audit"
"k8s.io/klog/v2"
)
@@ -61,11 +62,23 @@ func LoadPolicyFromFile(filePath string) (*auditinternal.Policy, error) {
func LoadPolicyFromBytes(policyDef []byte) (*auditinternal.Policy, error) {
policy := &auditinternal.Policy{}
decoder := audit.Codecs.UniversalDecoder(apiGroupVersions...)
strictDecoder := serializer.NewCodecFactory(audit.Scheme, serializer.EnableStrict).UniversalDecoder()
_, gvk, err := decoder.Decode(policyDef, nil, policy)
// Try strict decoding first.
_, gvk, err := strictDecoder.Decode(policyDef, nil, policy)
if err != nil {
return nil, fmt.Errorf("failed decoding: %v", err)
if !runtime.IsStrictDecodingError(err) {
return nil, fmt.Errorf("failed decoding: %w", err)
}
var (
lenientDecoder = audit.Codecs.UniversalDecoder(apiGroupVersions...)
lenientErr error
)
_, gvk, lenientErr = lenientDecoder.Decode(policyDef, nil, policy)
if lenientErr != nil {
return nil, fmt.Errorf("failed lenient decoding: %w", lenientErr)
}
klog.Warningf("Audit policy contains errors, falling back to lenient decoding: %v", err)
}
// Ensure the policy file contained an apiVersion and kind.

View File

@@ -21,7 +21,6 @@ import (
"context"
"fmt"
"net/http"
"reflect"
"time"
authnv1 "k8s.io/api/authentication/v1"
@@ -34,7 +33,6 @@ import (
auditinternal "k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/klog/v2"
"github.com/google/uuid"
@@ -54,7 +52,7 @@ func NewEventFromRequest(req *http.Request, requestReceivedTimestamp time.Time,
Level: level,
}
auditID, found := request.AuditIDFrom(req.Context())
auditID, found := AuditIDFrom(req.Context())
if !found {
auditID = types.UID(uuid.New().String())
}
@@ -154,7 +152,7 @@ func LogRequestObject(ctx context.Context, obj runtime.Object, objGV schema.Grou
if shouldOmitManagedFields(ctx) {
copy, ok, err := copyWithoutManagedFields(obj)
if err != nil {
klog.Warningf("error while dropping managed fields from the request for %q error: %v", reflect.TypeOf(obj).Name(), err)
klog.ErrorS(err, "Error while dropping managed fields from the request", "auditID", ae.AuditID)
}
if ok {
obj = copy
@@ -166,7 +164,7 @@ func LogRequestObject(ctx context.Context, obj runtime.Object, objGV schema.Grou
ae.RequestObject, err = encodeObject(obj, objGV, s)
if err != nil {
// TODO(audit): add error slice to audit event struct
klog.Warningf("Auditing failed of %v request: %v", reflect.TypeOf(obj).Name(), err)
klog.ErrorS(err, "Encoding failed of request object", "auditID", ae.AuditID, "gvr", gvr.String(), "obj", obj)
return
}
}
@@ -209,7 +207,7 @@ func LogResponseObject(ctx context.Context, obj runtime.Object, gv schema.GroupV
if shouldOmitManagedFields(ctx) {
copy, ok, err := copyWithoutManagedFields(obj)
if err != nil {
klog.Warningf("error while dropping managed fields from the response for %q error: %v", reflect.TypeOf(obj).Name(), err)
klog.ErrorS(err, "Error while dropping managed fields from the response", "auditID", ae.AuditID)
}
if ok {
obj = copy
@@ -220,7 +218,7 @@ func LogResponseObject(ctx context.Context, obj runtime.Object, gv schema.GroupV
var err error
ae.ResponseObject, err = encodeObject(obj, gv, s)
if err != nil {
klog.Warningf("Audit failed for %q response: %v", reflect.TypeOf(obj).Name(), err)
klog.ErrorS(err, "Encoding failed of response object", "auditID", ae.AuditID, "obj", obj)
}
}

View File

@@ -49,19 +49,19 @@ var clientCertificateExpirationHistogram = metrics.NewHistogram(
Help: "Distribution of the remaining lifetime on the certificate used to authenticate a request.",
Buckets: []float64{
0,
(30 * time.Minute).Seconds(),
(1 * time.Hour).Seconds(),
(2 * time.Hour).Seconds(),
(6 * time.Hour).Seconds(),
(12 * time.Hour).Seconds(),
(24 * time.Hour).Seconds(),
(2 * 24 * time.Hour).Seconds(),
(4 * 24 * time.Hour).Seconds(),
(7 * 24 * time.Hour).Seconds(),
(30 * 24 * time.Hour).Seconds(),
(3 * 30 * 24 * time.Hour).Seconds(),
(6 * 30 * 24 * time.Hour).Seconds(),
(12 * 30 * 24 * time.Hour).Seconds(),
1800, // 30 minutes
3600, // 1 hour
7200, // 2 hours
21600, // 6 hours
43200, // 12 hours
86400, // 1 day
172800, // 2 days
345600, // 4 days
604800, // 1 week
2592000, // 1 month
7776000, // 3 months
15552000, // 6 months
31104000, // 1 year
},
StabilityLevel: metrics.ALPHA,
},

View File

@@ -36,6 +36,7 @@ import (
auditinternal "k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/warning"
"k8s.io/klog/v2"
"k8s.io/utils/clock"
)
@@ -59,6 +60,12 @@ type cacheRecord struct {
// based on the current time, but that may be okay since cache TTLs are generally
// small (seconds).
annotations map[string]string
warnings []*cacheWarning
}
type cacheWarning struct {
agent string
text string
}
type cachedTokenAuthenticator struct {
@@ -128,6 +135,9 @@ func (a *cachedTokenAuthenticator) AuthenticateToken(ctx context.Context, token
for key, value := range record.annotations {
audit.AddAuditAnnotation(ctx, key, value)
}
for _, w := range record.warnings {
warning.AddWarning(ctx, w.agent, w.text)
}
return record.resp, true, nil
}
@@ -184,14 +194,19 @@ func (a *cachedTokenAuthenticator) doAuthenticateToken(ctx context.Context, toke
if audsOk {
ctx = authenticator.WithAudiences(ctx, auds)
}
recorder := &recorder{}
ctx = warning.WithWarningRecorder(ctx, recorder)
// since this is shared work between multiple requests, we have no way of knowing if any
// particular request supports audit annotations. thus we always attempt to record them.
ev := &auditinternal.Event{Level: auditinternal.LevelMetadata}
ctx = audit.WithAuditContext(ctx, &audit.AuditContext{Event: ev})
ctx = audit.WithAuditContext(ctx)
ac := audit.AuditContextFrom(ctx)
ac.Event = ev
record.resp, record.ok, record.err = a.authenticator.AuthenticateToken(ctx, token)
record.annotations = ev.Annotations
record.warnings = recorder.extractWarnings()
if !a.cacheErrs && record.err != nil {
return record, nil
@@ -269,3 +284,24 @@ func toBytes(s string) []byte {
func toString(b []byte) string {
return *(*string)(unsafe.Pointer(&b))
}
// simple recorder that only appends warning
type recorder struct {
mu sync.Mutex
warnings []*cacheWarning
}
// AddWarning adds a warning to recorder.
func (r *recorder) AddWarning(agent, text string) {
r.mu.Lock()
defer r.mu.Unlock()
r.warnings = append(r.warnings, &cacheWarning{agent: agent, text: text})
}
func (r *recorder) extractWarnings() []*cacheWarning {
r.mu.Lock()
defer r.mu.Unlock()
warnings := r.warnings
r.warnings = nil
return warnings
}

47
vendor/k8s.io/apiserver/pkg/cel/errors.go generated vendored Normal file
View File

@@ -0,0 +1,47 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
// Error is an implementation of the 'error' interface, which represents a
// XValidation error.
type Error struct {
Type ErrorType
Detail string
}
var _ error = &Error{}
// Error implements the error interface.
func (v *Error) Error() string {
return v.Detail
}
// ErrorType is a machine readable value providing more detail about why
// a XValidation is invalid.
type ErrorType string
const (
// ErrorTypeRequired is used to report withNullable values that are not
// provided (e.g. empty strings, null values, or empty arrays). See
// Required().
ErrorTypeRequired ErrorType = "RuleRequired"
// ErrorTypeInvalid is used to report malformed values
ErrorTypeInvalid ErrorType = "RuleInvalid"
// ErrorTypeInternal is used to report other errors that are not related
// to user input. See InternalError().
ErrorTypeInternal ErrorType = "InternalError"
)

170
vendor/k8s.io/apiserver/pkg/cel/escaping.go generated vendored Normal file
View File

@@ -0,0 +1,170 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
import (
"regexp"
"k8s.io/apimachinery/pkg/util/sets"
)
// celReservedSymbols is a list of RESERVED symbols defined in the CEL lexer.
// No identifiers are allowed to collide with these symbols.
// https://github.com/google/cel-spec/blob/master/doc/langdef.md#syntax
var celReservedSymbols = sets.NewString(
"true", "false", "null", "in",
"as", "break", "const", "continue", "else",
"for", "function", "if", "import", "let",
"loop", "package", "namespace", "return", // !! 'namespace' is used heavily in Kubernetes
"var", "void", "while",
)
// expandMatcher matches the escape sequence, characters that are escaped, and characters that are unsupported
var expandMatcher = regexp.MustCompile(`(__|[-./]|[^a-zA-Z0-9-./_])`)
// newCharacterFilter returns a boolean array to indicate the allowed characters
func newCharacterFilter(characters string) []bool {
maxChar := 0
for _, c := range characters {
if maxChar < int(c) {
maxChar = int(c)
}
}
filter := make([]bool, maxChar+1)
for _, c := range characters {
filter[int(c)] = true
}
return filter
}
type escapeCheck struct {
canSkipRegex bool
invalidCharFound bool
}
// skipRegexCheck checks if escape would be skipped.
// if invalidCharFound is true, it must have invalid character; if invalidCharFound is false, not sure if it has invalid character or not
func skipRegexCheck(ident string) escapeCheck {
escapeCheck := escapeCheck{canSkipRegex: true, invalidCharFound: false}
// skip escape if possible
previous_underscore := false
for _, c := range ident {
if c == '/' || c == '-' || c == '.' {
escapeCheck.canSkipRegex = false
return escapeCheck
}
intc := int(c)
if intc < 0 || intc >= len(validCharacterFilter) || !validCharacterFilter[intc] {
escapeCheck.invalidCharFound = true
return escapeCheck
}
if c == '_' && previous_underscore {
escapeCheck.canSkipRegex = false
return escapeCheck
}
previous_underscore = c == '_'
}
return escapeCheck
}
// validCharacterFilter indicates the allowed characters.
var validCharacterFilter = newCharacterFilter("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_")
// Escape escapes ident and returns a CEL identifier (of the form '[a-zA-Z_][a-zA-Z0-9_]*'), or returns
// false if the ident does not match the supported input format of `[a-zA-Z_.-/][a-zA-Z0-9_.-/]*`.
// Escaping Rules:
// - '__' escapes to '__underscores__'
// - '.' escapes to '__dot__'
// - '-' escapes to '__dash__'
// - '/' escapes to '__slash__'
// - Identifiers that exactly match a CEL RESERVED keyword escape to '__{keyword}__'. The keywords are: "true", "false",
// "null", "in", "as", "break", "const", "continue", "else", "for", "function", "if", "import", "let", loop", "package",
// "namespace", "return".
func Escape(ident string) (string, bool) {
if len(ident) == 0 || ('0' <= ident[0] && ident[0] <= '9') {
return "", false
}
if celReservedSymbols.Has(ident) {
return "__" + ident + "__", true
}
escapeCheck := skipRegexCheck(ident)
if escapeCheck.invalidCharFound {
return "", false
}
if escapeCheck.canSkipRegex {
return ident, true
}
ok := true
ident = expandMatcher.ReplaceAllStringFunc(ident, func(s string) string {
switch s {
case "__":
return "__underscores__"
case ".":
return "__dot__"
case "-":
return "__dash__"
case "/":
return "__slash__"
default: // matched a unsupported supported
ok = false
return ""
}
})
if !ok {
return "", false
}
return ident, true
}
var unexpandMatcher = regexp.MustCompile(`(_{2}[^_]+_{2})`)
// Unescape unescapes an CEL identifier containing the escape sequences described in Escape, or return false if the
// string contains invalid escape sequences. The escaped input is expected to be a valid CEL identifier, but is
// not checked.
func Unescape(escaped string) (string, bool) {
ok := true
escaped = unexpandMatcher.ReplaceAllStringFunc(escaped, func(s string) string {
contents := s[2 : len(s)-2]
switch contents {
case "underscores":
return "__"
case "dot":
return "."
case "dash":
return "-"
case "slash":
return "/"
}
if celReservedSymbols.Has(contents) {
if len(s) != len(escaped) {
ok = false
}
return contents
}
ok = false
return ""
})
if !ok {
return "", false
}
return escaped, true
}

268
vendor/k8s.io/apiserver/pkg/cel/library/cost.go generated vendored Normal file
View File

@@ -0,0 +1,268 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package library
import (
"math"
"github.com/google/cel-go/checker"
"github.com/google/cel-go/common"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
"github.com/google/cel-go/common/types/traits"
exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
)
// CostEstimator implements CEL's interpretable.ActualCostEstimator and checker.CostEstimator.
type CostEstimator struct {
// SizeEstimator provides a CostEstimator.EstimateSize that this CostEstimator will delegate size estimation
// calculations to if the size is not well known (i.e. a constant).
SizeEstimator checker.CostEstimator
}
func (l *CostEstimator) CallCost(function, overloadId string, args []ref.Val, result ref.Val) *uint64 {
switch function {
case "isSorted", "sum", "max", "min", "indexOf", "lastIndexOf":
var cost uint64
if len(args) > 0 {
cost += traversalCost(args[0]) // these O(n) operations all cost roughly the cost of a single traversal
}
return &cost
case "url", "lowerAscii", "upperAscii", "substring", "trim":
if len(args) >= 1 {
cost := uint64(math.Ceil(float64(actualSize(args[0])) * common.StringTraversalCostFactor))
return &cost
}
case "replace", "split":
if len(args) >= 1 {
// cost is the traversal plus the construction of the result
cost := uint64(math.Ceil(float64(actualSize(args[0])) * 2 * common.StringTraversalCostFactor))
return &cost
}
case "join":
if len(args) >= 1 {
cost := uint64(math.Ceil(float64(actualSize(result)) * 2 * common.StringTraversalCostFactor))
return &cost
}
case "find", "findAll":
if len(args) >= 2 {
strCost := uint64(math.Ceil((1.0 + float64(actualSize(args[0]))) * common.StringTraversalCostFactor))
// We don't know how many expressions are in the regex, just the string length (a huge
// improvement here would be to somehow get a count the number of expressions in the regex or
// how many states are in the regex state machine and use that to measure regex cost).
// For now, we're making a guess that each expression in a regex is typically at least 4 chars
// in length.
regexCost := uint64(math.Ceil(float64(actualSize(args[1])) * common.RegexStringLengthCostFactor))
cost := strCost * regexCost
return &cost
}
}
return nil
}
func (l *CostEstimator) EstimateCallCost(function, overloadId string, target *checker.AstNode, args []checker.AstNode) *checker.CallEstimate {
// WARNING: Any changes to this code impact API compatibility! The estimated cost is used to determine which CEL rules may be written to a
// CRD and any change (cost increases and cost decreases) are breaking.
switch function {
case "isSorted", "sum", "max", "min", "indexOf", "lastIndexOf":
if target != nil {
// Charge 1 cost for comparing each element in the list
elCost := checker.CostEstimate{Min: 1, Max: 1}
// If the list contains strings or bytes, add the cost of traversing all the strings/bytes as a way
// of estimating the additional comparison cost.
if elNode := l.listElementNode(*target); elNode != nil {
t := elNode.Type().GetPrimitive()
if t == exprpb.Type_STRING || t == exprpb.Type_BYTES {
sz := l.sizeEstimate(elNode)
elCost = elCost.Add(sz.MultiplyByCostFactor(common.StringTraversalCostFactor))
}
return &checker.CallEstimate{CostEstimate: l.sizeEstimate(*target).MultiplyByCost(elCost)}
} else { // the target is a string, which is supported by indexOf and lastIndexOf
return &checker.CallEstimate{CostEstimate: l.sizeEstimate(*target).MultiplyByCostFactor(common.StringTraversalCostFactor)}
}
}
case "url":
if len(args) == 1 {
sz := l.sizeEstimate(args[0])
return &checker.CallEstimate{CostEstimate: sz.MultiplyByCostFactor(common.StringTraversalCostFactor)}
}
case "lowerAscii", "upperAscii", "substring", "trim":
if target != nil {
sz := l.sizeEstimate(*target)
return &checker.CallEstimate{CostEstimate: sz.MultiplyByCostFactor(common.StringTraversalCostFactor), ResultSize: &sz}
}
case "replace":
if target != nil && len(args) >= 2 {
sz := l.sizeEstimate(*target)
toReplaceSz := l.sizeEstimate(args[0])
replaceWithSz := l.sizeEstimate(args[1])
// smallest possible result: smallest input size composed of the largest possible substrings being replaced by smallest possible replacement
minSz := uint64(math.Ceil(float64(sz.Min)/float64(toReplaceSz.Max))) * replaceWithSz.Min
// largest possible result: largest input size composed of the smallest possible substrings being replaced by largest possible replacement
maxSz := uint64(math.Ceil(float64(sz.Max)/float64(toReplaceSz.Min))) * replaceWithSz.Max
// cost is the traversal plus the construction of the result
return &checker.CallEstimate{CostEstimate: sz.MultiplyByCostFactor(2 * common.StringTraversalCostFactor), ResultSize: &checker.SizeEstimate{Min: minSz, Max: maxSz}}
}
case "split":
if target != nil {
sz := l.sizeEstimate(*target)
// Worst case size is where is that a separator of "" is used, and each char is returned as a list element.
max := sz.Max
if len(args) > 1 {
if c := args[1].Expr().GetConstExpr(); c != nil {
max = uint64(c.GetInt64Value())
}
}
// Cost is the traversal plus the construction of the result.
return &checker.CallEstimate{CostEstimate: sz.MultiplyByCostFactor(2 * common.StringTraversalCostFactor), ResultSize: &checker.SizeEstimate{Min: 0, Max: max}}
}
case "join":
if target != nil {
var sz checker.SizeEstimate
listSize := l.sizeEstimate(*target)
if elNode := l.listElementNode(*target); elNode != nil {
elemSize := l.sizeEstimate(elNode)
sz = listSize.Multiply(elemSize)
}
if len(args) > 0 {
sepSize := l.sizeEstimate(args[0])
minSeparators := uint64(0)
maxSeparators := uint64(0)
if listSize.Min > 0 {
minSeparators = listSize.Min - 1
}
if listSize.Max > 0 {
maxSeparators = listSize.Max - 1
}
sz = sz.Add(sepSize.Multiply(checker.SizeEstimate{Min: minSeparators, Max: maxSeparators}))
}
return &checker.CallEstimate{CostEstimate: sz.MultiplyByCostFactor(common.StringTraversalCostFactor), ResultSize: &sz}
}
case "find", "findAll":
if target != nil && len(args) >= 1 {
sz := l.sizeEstimate(*target)
// Add one to string length for purposes of cost calculation to prevent product of string and regex to be 0
// in case where string is empty but regex is still expensive.
strCost := sz.Add(checker.SizeEstimate{Min: 1, Max: 1}).MultiplyByCostFactor(common.StringTraversalCostFactor)
// We don't know how many expressions are in the regex, just the string length (a huge
// improvement here would be to somehow get a count the number of expressions in the regex or
// how many states are in the regex state machine and use that to measure regex cost).
// For now, we're making a guess that each expression in a regex is typically at least 4 chars
// in length.
regexCost := l.sizeEstimate(args[0]).MultiplyByCostFactor(common.RegexStringLengthCostFactor)
// worst case size of result is that every char is returned as separate find result.
return &checker.CallEstimate{CostEstimate: strCost.Multiply(regexCost), ResultSize: &checker.SizeEstimate{Min: 0, Max: sz.Max}}
}
}
return nil
}
func actualSize(value ref.Val) uint64 {
if sz, ok := value.(traits.Sizer); ok {
return uint64(sz.Size().(types.Int))
}
return 1
}
func (l *CostEstimator) sizeEstimate(t checker.AstNode) checker.SizeEstimate {
if sz := t.ComputedSize(); sz != nil {
return *sz
}
if sz := l.EstimateSize(t); sz != nil {
return *sz
}
return checker.SizeEstimate{Min: 0, Max: math.MaxUint64}
}
func (l *CostEstimator) listElementNode(list checker.AstNode) checker.AstNode {
if lt := list.Type().GetListType(); lt != nil {
nodePath := list.Path()
if nodePath != nil {
// Provide path if we have it so that a OpenAPIv3 maxLength validation can be looked up, if it exists
// for this node.
path := make([]string, len(nodePath)+1)
copy(path, nodePath)
path[len(nodePath)] = "@items"
return &itemsNode{path: path, t: lt.GetElemType(), expr: nil}
} else {
// Provide just the type if no path is available so that worst case size can be looked up based on type.
return &itemsNode{t: lt.GetElemType(), expr: nil}
}
}
return nil
}
func (l *CostEstimator) EstimateSize(element checker.AstNode) *checker.SizeEstimate {
if l.SizeEstimator != nil {
return l.SizeEstimator.EstimateSize(element)
}
return nil
}
type itemsNode struct {
path []string
t *exprpb.Type
expr *exprpb.Expr
}
func (i *itemsNode) Path() []string {
return i.path
}
func (i *itemsNode) Type() *exprpb.Type {
return i.t
}
func (i *itemsNode) Expr() *exprpb.Expr {
return i.expr
}
func (i *itemsNode) ComputedSize() *checker.SizeEstimate {
return nil
}
// traversalCost computes the cost of traversing a ref.Val as a data tree.
func traversalCost(v ref.Val) uint64 {
// TODO: This could potentially be optimized by sampling maps and lists instead of traversing.
switch vt := v.(type) {
case types.String:
return uint64(float64(len(string(vt))) * common.StringTraversalCostFactor)
case types.Bytes:
return uint64(float64(len([]byte(vt))) * common.StringTraversalCostFactor)
case traits.Lister:
cost := uint64(0)
for it := vt.Iterator(); it.HasNext() == types.True; {
i := it.Next()
cost += traversalCost(i)
}
return cost
case traits.Mapper: // maps and objects
cost := uint64(0)
for it := vt.Iterator(); it.HasNext() == types.True; {
k := it.Next()
cost += traversalCost(k) + traversalCost(vt.Get(k))
}
return cost
default:
return 1
}
}

34
vendor/k8s.io/apiserver/pkg/cel/library/libraries.go generated vendored Normal file
View File

@@ -0,0 +1,34 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package library
import (
"github.com/google/cel-go/cel"
"github.com/google/cel-go/ext"
"github.com/google/cel-go/interpreter"
)
// ExtensionLibs declares the set of CEL extension libraries available everywhere CEL is used in Kubernetes.
var ExtensionLibs = append(k8sExtensionLibs, ext.Strings())
var k8sExtensionLibs = []cel.EnvOption{
URLs(),
Regex(),
Lists(),
}
var ExtensionLibRegexOptimizations = []*interpreter.RegexOptimization{FindRegexOptimization, FindAllRegexOptimization}

312
vendor/k8s.io/apiserver/pkg/cel/library/lists.go generated vendored Normal file
View File

@@ -0,0 +1,312 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package library
import (
"fmt"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
"github.com/google/cel-go/common/types/traits"
"github.com/google/cel-go/interpreter/functions"
)
// Lists provides a CEL function library extension of list utility functions.
//
// isSorted
//
// Returns true if the provided list of comparable elements is sorted, else returns false.
//
// <list<T>>.isSorted() <bool>, T must be a comparable type
//
// Examples:
//
// [1, 2, 3].isSorted() // return true
// ['a', 'b', 'b', 'c'].isSorted() // return true
// [2.0, 1.0].isSorted() // return false
// [1].isSorted() // return true
// [].isSorted() // return true
//
// sum
//
// Returns the sum of the elements of the provided list. Supports CEL number (int, uint, double) and duration types.
//
// <list<T>>.sum() <T>, T must be a numeric type or a duration
//
// Examples:
//
// [1, 3].sum() // returns 4
// [1.0, 3.0].sum() // returns 4.0
// ['1m', '1s'].sum() // returns '1m1s'
// emptyIntList.sum() // returns 0
// emptyDoubleList.sum() // returns 0.0
// [].sum() // returns 0
//
// min / max
//
// Returns the minimum/maximum valued element of the provided list. Supports all comparable types.
// If the list is empty, an error is returned.
//
// <list<T>>.min() <T>, T must be a comparable type
// <list<T>>.max() <T>, T must be a comparable type
//
// Examples:
//
// [1, 3].min() // returns 1
// [1, 3].max() // returns 3
// [].min() // error
// [1].min() // returns 1
// ([0] + emptyList).min() // returns 0
//
// indexOf / lastIndexOf
//
// Returns either the first or last positional index of the provided element in the list.
// If the element is not found, -1 is returned. Supports all equatable types.
//
// <list<T>>.indexOf(<T>) <int>, T must be an equatable type
// <list<T>>.lastIndexOf(<T>) <int>, T must be an equatable type
//
// Examples:
//
// [1, 2, 2, 3].indexOf(2) // returns 1
// ['a', 'b', 'b', 'c'].lastIndexOf('b') // returns 2
// [1.0].indexOf(1.1) // returns -1
// [].indexOf('string') // returns -1
func Lists() cel.EnvOption {
return cel.Lib(listsLib)
}
var listsLib = &lists{}
type lists struct{}
var paramA = cel.TypeParamType("A")
// CEL typeParams can be used to constraint to a specific trait (e.g. traits.ComparableType) if the 1st operand is the type to constrain.
// But the functions we need to constrain are <list<paramType>>, not just <paramType>.
// Make sure the order of overload set is deterministic
type namedCELType struct {
typeName string
celType *cel.Type
}
var summableTypes = []namedCELType{
{typeName: "int", celType: cel.IntType},
{typeName: "uint", celType: cel.UintType},
{typeName: "double", celType: cel.DoubleType},
{typeName: "duration", celType: cel.DurationType},
}
var zeroValuesOfSummableTypes = map[string]ref.Val{
"int": types.Int(0),
"uint": types.Uint(0),
"double": types.Double(0.0),
"duration": types.Duration{Duration: 0},
}
var comparableTypes = []namedCELType{
{typeName: "int", celType: cel.IntType},
{typeName: "uint", celType: cel.UintType},
{typeName: "double", celType: cel.DoubleType},
{typeName: "bool", celType: cel.BoolType},
{typeName: "duration", celType: cel.DurationType},
{typeName: "timestamp", celType: cel.TimestampType},
{typeName: "string", celType: cel.StringType},
{typeName: "bytes", celType: cel.BytesType},
}
// WARNING: All library additions or modifications must follow
// https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/2876-crd-validation-expression-language#function-library-updates
var listsLibraryDecls = map[string][]cel.FunctionOpt{
"isSorted": templatedOverloads(comparableTypes, func(name string, paramType *cel.Type) cel.FunctionOpt {
return cel.MemberOverload(fmt.Sprintf("list_%s_is_sorted_bool", name),
[]*cel.Type{cel.ListType(paramType)}, cel.BoolType, cel.UnaryBinding(isSorted))
}),
"sum": templatedOverloads(summableTypes, func(name string, paramType *cel.Type) cel.FunctionOpt {
return cel.MemberOverload(fmt.Sprintf("list_%s_sum_%s", name, name),
[]*cel.Type{cel.ListType(paramType)}, paramType, cel.UnaryBinding(func(list ref.Val) ref.Val {
return sum(
func() ref.Val {
return zeroValuesOfSummableTypes[name]
})(list)
}))
}),
"max": templatedOverloads(comparableTypes, func(name string, paramType *cel.Type) cel.FunctionOpt {
return cel.MemberOverload(fmt.Sprintf("list_%s_max_%s", name, name),
[]*cel.Type{cel.ListType(paramType)}, paramType, cel.UnaryBinding(max()))
}),
"min": templatedOverloads(comparableTypes, func(name string, paramType *cel.Type) cel.FunctionOpt {
return cel.MemberOverload(fmt.Sprintf("list_%s_min_%s", name, name),
[]*cel.Type{cel.ListType(paramType)}, paramType, cel.UnaryBinding(min()))
}),
"indexOf": {
cel.MemberOverload("list_a_index_of_int", []*cel.Type{cel.ListType(paramA), paramA}, cel.IntType,
cel.BinaryBinding(indexOf)),
},
"lastIndexOf": {
cel.MemberOverload("list_a_last_index_of_int", []*cel.Type{cel.ListType(paramA), paramA}, cel.IntType,
cel.BinaryBinding(lastIndexOf)),
},
}
func (*lists) CompileOptions() []cel.EnvOption {
options := []cel.EnvOption{}
for name, overloads := range listsLibraryDecls {
options = append(options, cel.Function(name, overloads...))
}
return options
}
func (*lists) ProgramOptions() []cel.ProgramOption {
return []cel.ProgramOption{}
}
func isSorted(val ref.Val) ref.Val {
var prev traits.Comparer
iterable, ok := val.(traits.Iterable)
if !ok {
return types.MaybeNoSuchOverloadErr(val)
}
for it := iterable.Iterator(); it.HasNext() == types.True; {
next := it.Next()
nextCmp, ok := next.(traits.Comparer)
if !ok {
return types.MaybeNoSuchOverloadErr(next)
}
if prev != nil {
cmp := prev.Compare(next)
if cmp == types.IntOne {
return types.False
}
}
prev = nextCmp
}
return types.True
}
func sum(init func() ref.Val) functions.UnaryOp {
return func(val ref.Val) ref.Val {
i := init()
acc, ok := i.(traits.Adder)
if !ok {
// Should never happen since all passed in init values are valid
return types.MaybeNoSuchOverloadErr(i)
}
iterable, ok := val.(traits.Iterable)
if !ok {
return types.MaybeNoSuchOverloadErr(val)
}
for it := iterable.Iterator(); it.HasNext() == types.True; {
next := it.Next()
nextAdder, ok := next.(traits.Adder)
if !ok {
// Should never happen for type checked CEL programs
return types.MaybeNoSuchOverloadErr(next)
}
if acc != nil {
s := acc.Add(next)
sum, ok := s.(traits.Adder)
if !ok {
// Should never happen for type checked CEL programs
return types.MaybeNoSuchOverloadErr(s)
}
acc = sum
} else {
acc = nextAdder
}
}
return acc.(ref.Val)
}
}
func min() functions.UnaryOp {
return cmp("min", types.IntOne)
}
func max() functions.UnaryOp {
return cmp("max", types.IntNegOne)
}
func cmp(opName string, opPreferCmpResult ref.Val) functions.UnaryOp {
return func(val ref.Val) ref.Val {
var result traits.Comparer
iterable, ok := val.(traits.Iterable)
if !ok {
return types.MaybeNoSuchOverloadErr(val)
}
for it := iterable.Iterator(); it.HasNext() == types.True; {
next := it.Next()
nextCmp, ok := next.(traits.Comparer)
if !ok {
// Should never happen for type checked CEL programs
return types.MaybeNoSuchOverloadErr(next)
}
if result == nil {
result = nextCmp
} else {
cmp := result.Compare(next)
if cmp == opPreferCmpResult {
result = nextCmp
}
}
}
if result == nil {
return types.NewErr("%s called on empty list", opName)
}
return result.(ref.Val)
}
}
func indexOf(list ref.Val, item ref.Val) ref.Val {
lister, ok := list.(traits.Lister)
if !ok {
return types.MaybeNoSuchOverloadErr(list)
}
sz := lister.Size().(types.Int)
for i := types.Int(0); i < sz; i++ {
if lister.Get(types.Int(i)).Equal(item) == types.True {
return types.Int(i)
}
}
return types.Int(-1)
}
func lastIndexOf(list ref.Val, item ref.Val) ref.Val {
lister, ok := list.(traits.Lister)
if !ok {
return types.MaybeNoSuchOverloadErr(list)
}
sz := lister.Size().(types.Int)
for i := sz - 1; i >= 0; i-- {
if lister.Get(types.Int(i)).Equal(item) == types.True {
return types.Int(i)
}
}
return types.Int(-1)
}
// templatedOverloads returns overloads for each of the provided types. The template function is called with each type
// name (map key) and type to construct the overloads.
func templatedOverloads(types []namedCELType, template func(name string, t *cel.Type) cel.FunctionOpt) []cel.FunctionOpt {
overloads := make([]cel.FunctionOpt, len(types))
i := 0
for _, t := range types {
overloads[i] = template(t.typeName, t.celType)
i++
}
return overloads
}

187
vendor/k8s.io/apiserver/pkg/cel/library/regex.go generated vendored Normal file
View File

@@ -0,0 +1,187 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package library
import (
"regexp"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
"github.com/google/cel-go/interpreter"
)
// Regex provides a CEL function library extension of regex utility functions.
//
// find / findAll
//
// Returns substrings that match the provided regular expression. find returns the first match. findAll may optionally
// be provided a limit. If the limit is set and >= 0, no more than the limit number of matches are returned.
//
// <string>.find(<string>) <string>
// <string>.findAll(<string>) <list <string>>
// <string>.findAll(<string>, <int>) <list <string>>
//
// Examples:
//
// "abc 123".find('[0-9]*') // returns '123'
// "abc 123".find('xyz') // returns ''
// "123 abc 456".findAll('[0-9]*') // returns ['123', '456']
// "123 abc 456".findAll('[0-9]*', 1) // returns ['123']
// "123 abc 456".findAll('xyz') // returns []
func Regex() cel.EnvOption {
return cel.Lib(regexLib)
}
var regexLib = &regex{}
type regex struct{}
var regexLibraryDecls = map[string][]cel.FunctionOpt{
"find": {
cel.MemberOverload("string_find_string", []*cel.Type{cel.StringType, cel.StringType}, cel.StringType,
cel.BinaryBinding(find))},
"findAll": {
cel.MemberOverload("string_find_all_string", []*cel.Type{cel.StringType, cel.StringType},
cel.ListType(cel.StringType),
cel.BinaryBinding(func(str, regex ref.Val) ref.Val {
return findAll(str, regex, types.Int(-1))
})),
cel.MemberOverload("string_find_all_string_int",
[]*cel.Type{cel.StringType, cel.StringType, cel.IntType},
cel.ListType(cel.StringType),
cel.FunctionBinding(findAll)),
},
}
func (*regex) CompileOptions() []cel.EnvOption {
options := []cel.EnvOption{}
for name, overloads := range regexLibraryDecls {
options = append(options, cel.Function(name, overloads...))
}
return options
}
func (*regex) ProgramOptions() []cel.ProgramOption {
return []cel.ProgramOption{}
}
func find(strVal ref.Val, regexVal ref.Val) ref.Val {
str, ok := strVal.Value().(string)
if !ok {
return types.MaybeNoSuchOverloadErr(strVal)
}
regex, ok := regexVal.Value().(string)
if !ok {
return types.MaybeNoSuchOverloadErr(regexVal)
}
re, err := regexp.Compile(regex)
if err != nil {
return types.NewErr("Illegal regex: %v", err.Error())
}
result := re.FindString(str)
return types.String(result)
}
func findAll(args ...ref.Val) ref.Val {
argn := len(args)
if argn < 2 || argn > 3 {
return types.NoSuchOverloadErr()
}
str, ok := args[0].Value().(string)
if !ok {
return types.MaybeNoSuchOverloadErr(args[0])
}
regex, ok := args[1].Value().(string)
if !ok {
return types.MaybeNoSuchOverloadErr(args[1])
}
n := int64(-1)
if argn == 3 {
n, ok = args[2].Value().(int64)
if !ok {
return types.MaybeNoSuchOverloadErr(args[2])
}
}
re, err := regexp.Compile(regex)
if err != nil {
return types.NewErr("Illegal regex: %v", err.Error())
}
result := re.FindAllString(str, int(n))
return types.NewStringList(types.DefaultTypeAdapter, result)
}
// FindRegexOptimization optimizes the 'find' function by compiling the regex pattern and
// reporting any compilation errors at program creation time, and using the compiled regex pattern for all function
// call invocations.
var FindRegexOptimization = &interpreter.RegexOptimization{
Function: "find",
RegexIndex: 1,
Factory: func(call interpreter.InterpretableCall, regexPattern string) (interpreter.InterpretableCall, error) {
compiledRegex, err := regexp.Compile(regexPattern)
if err != nil {
return nil, err
}
return interpreter.NewCall(call.ID(), call.Function(), call.OverloadID(), call.Args(), func(args ...ref.Val) ref.Val {
if len(args) != 2 {
return types.NoSuchOverloadErr()
}
in, ok := args[0].Value().(string)
if !ok {
return types.MaybeNoSuchOverloadErr(args[0])
}
return types.String(compiledRegex.FindString(in))
}), nil
},
}
// FindAllRegexOptimization optimizes the 'findAll' function by compiling the regex pattern and
// reporting any compilation errors at program creation time, and using the compiled regex pattern for all function
// call invocations.
var FindAllRegexOptimization = &interpreter.RegexOptimization{
Function: "findAll",
RegexIndex: 1,
Factory: func(call interpreter.InterpretableCall, regexPattern string) (interpreter.InterpretableCall, error) {
compiledRegex, err := regexp.Compile(regexPattern)
if err != nil {
return nil, err
}
return interpreter.NewCall(call.ID(), call.Function(), call.OverloadID(), call.Args(), func(args ...ref.Val) ref.Val {
argn := len(args)
if argn < 2 || argn > 3 {
return types.NoSuchOverloadErr()
}
str, ok := args[0].Value().(string)
if !ok {
return types.MaybeNoSuchOverloadErr(args[0])
}
n := int64(-1)
if argn == 3 {
n, ok = args[2].Value().(int64)
if !ok {
return types.MaybeNoSuchOverloadErr(args[2])
}
}
result := compiledRegex.FindAllString(str, int(n))
return types.NewStringList(types.DefaultTypeAdapter, result)
}), nil
},
}

236
vendor/k8s.io/apiserver/pkg/cel/library/urls.go generated vendored Normal file
View File

@@ -0,0 +1,236 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package library
import (
"net/url"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
apiservercel "k8s.io/apiserver/pkg/cel"
)
// URLs provides a CEL function library extension of URL parsing functions.
//
// url
//
// Converts a string to a URL or results in an error if the string is not a valid URL. The URL must be an absolute URI
// or an absolute path.
//
// url(<string>) <URL>
//
// Examples:
//
// url('https://user:pass@example.com:80/path?query=val#fragment') // returns a URL
// url('/absolute-path') // returns a URL
// url('https://a:b:c/') // error
// url('../relative-path') // error
//
// isURL
//
// Returns true if a string is a valid URL. The URL must be an absolute URI or an absolute path.
//
// isURL( <string>) <bool>
//
// Examples:
//
// isURL('https://user:pass@example.com:80/path?query=val#fragment') // returns true
// isURL('/absolute-path') // returns true
// isURL('https://a:b:c/') // returns false
// isURL('../relative-path') // returns false
//
// getScheme / getHost / getHostname / getPort / getEscapedPath / getQuery
//
// Return the parsed components of a URL.
//
// - getScheme: If absent in the URL, returns an empty string.
//
// - getHostname: IPv6 addresses are returned with braces, e.g. "[::1]". If absent in the URL, returns an empty string.
//
// - getHost: IPv6 addresses are returned without braces, e.g. "::1". If absent in the URL, returns an empty string.
//
// - getEscapedPath: The string returned by getEscapedPath is URL escaped, e.g. "with space" becomes "with%20space".
// If absent in the URL, returns an empty string.
//
// - getPort: If absent in the URL, returns an empty string.
//
// - getQuery: Returns the query parameters in "matrix" form where a repeated query key is interpreted to
// mean that there are multiple values for that key. The keys and values are returned unescaped.
// If absent in the URL, returns an empty map.
//
// <URL>.getScheme() <string>
// <URL>.getHost() <string>
// <URL>.getHostname() <string>
// <URL>.getPort() <string>
// <URL>.getEscapedPath() <string>
// <URL>.getQuery() <map <string>, <list <string>>
//
// Examples:
//
// url('/path').getScheme() // returns ''
// url('https://example.com/').getScheme() // returns 'https'
// url('https://example.com:80/').getHost() // returns 'example.com:80'
// url('https://example.com/').getHost() // returns 'example.com'
// url('https://[::1]:80/').getHost() // returns '[::1]:80'
// url('https://[::1]/').getHost() // returns '[::1]'
// url('/path').getHost() // returns ''
// url('https://example.com:80/').getHostname() // returns 'example.com'
// url('https://127.0.0.1:80/').getHostname() // returns '127.0.0.1'
// url('https://[::1]:80/').getHostname() // returns '::1'
// url('/path').getHostname() // returns ''
// url('https://example.com:80/').getPort() // returns '80'
// url('https://example.com/').getPort() // returns ''
// url('/path').getPort() // returns ''
// url('https://example.com/path').getEscapedPath() // returns '/path'
// url('https://example.com/path with spaces/').getEscapedPath() // returns '/path%20with%20spaces/'
// url('https://example.com').getEscapedPath() // returns ''
// url('https://example.com/path?k1=a&k2=b&k2=c').getQuery() // returns { 'k1': ['a'], 'k2': ['b', 'c']}
// url('https://example.com/path?key with spaces=value with spaces').getQuery() // returns { 'key with spaces': ['value with spaces']}
// url('https://example.com/path?').getQuery() // returns {}
// url('https://example.com/path').getQuery() // returns {}
func URLs() cel.EnvOption {
return cel.Lib(urlsLib)
}
var urlsLib = &urls{}
type urls struct{}
var urlLibraryDecls = map[string][]cel.FunctionOpt{
"url": {
cel.Overload("string_to_url", []*cel.Type{cel.StringType}, apiservercel.URLType,
cel.UnaryBinding(stringToUrl))},
"getScheme": {
cel.MemberOverload("url_get_scheme", []*cel.Type{apiservercel.URLType}, cel.StringType,
cel.UnaryBinding(getScheme))},
"getHost": {
cel.MemberOverload("url_get_host", []*cel.Type{apiservercel.URLType}, cel.StringType,
cel.UnaryBinding(getHost))},
"getHostname": {
cel.MemberOverload("url_get_hostname", []*cel.Type{apiservercel.URLType}, cel.StringType,
cel.UnaryBinding(getHostname))},
"getPort": {
cel.MemberOverload("url_get_port", []*cel.Type{apiservercel.URLType}, cel.StringType,
cel.UnaryBinding(getPort))},
"getEscapedPath": {
cel.MemberOverload("url_get_escaped_path", []*cel.Type{apiservercel.URLType}, cel.StringType,
cel.UnaryBinding(getEscapedPath))},
"getQuery": {
cel.MemberOverload("url_get_query", []*cel.Type{apiservercel.URLType},
cel.MapType(cel.StringType, cel.ListType(cel.StringType)),
cel.UnaryBinding(getQuery))},
"isURL": {
cel.Overload("is_url_string", []*cel.Type{cel.StringType}, cel.BoolType,
cel.UnaryBinding(isURL))},
}
func (*urls) CompileOptions() []cel.EnvOption {
options := []cel.EnvOption{}
for name, overloads := range urlLibraryDecls {
options = append(options, cel.Function(name, overloads...))
}
return options
}
func (*urls) ProgramOptions() []cel.ProgramOption {
return []cel.ProgramOption{}
}
func stringToUrl(arg ref.Val) ref.Val {
s, ok := arg.Value().(string)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
// Use ParseRequestURI to check the URL before conversion.
// ParseRequestURI requires absolute URLs and is used by the OpenAPIv3 'uri' format.
_, err := url.ParseRequestURI(s)
if err != nil {
return types.NewErr("URL parse error during conversion from string: %v", err)
}
// We must parse again with Parse since ParseRequestURI incorrectly parses URLs that contain a fragment
// part and will incorrectly append the fragment to either the path or the query, depending on which it was adjacent to.
u, err := url.Parse(s)
if err != nil {
// Errors are not expected here since Parse is a more lenient parser than ParseRequestURI.
return types.NewErr("URL parse error during conversion from string: %v", err)
}
return apiservercel.URL{URL: u}
}
func getScheme(arg ref.Val) ref.Val {
u, ok := arg.Value().(*url.URL)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
return types.String(u.Scheme)
}
func getHost(arg ref.Val) ref.Val {
u, ok := arg.Value().(*url.URL)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
return types.String(u.Host)
}
func getHostname(arg ref.Val) ref.Val {
u, ok := arg.Value().(*url.URL)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
return types.String(u.Hostname())
}
func getPort(arg ref.Val) ref.Val {
u, ok := arg.Value().(*url.URL)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
return types.String(u.Port())
}
func getEscapedPath(arg ref.Val) ref.Val {
u, ok := arg.Value().(*url.URL)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
return types.String(u.EscapedPath())
}
func getQuery(arg ref.Val) ref.Val {
u, ok := arg.Value().(*url.URL)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
result := map[ref.Val]ref.Val{}
for k, v := range u.Query() {
result[types.String(k)] = types.NewStringList(types.DefaultTypeAdapter, v)
}
return types.NewRefValMap(types.DefaultTypeAdapter, result)
}
func isURL(arg ref.Val) ref.Val {
s, ok := arg.Value().(string)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
_, err := url.ParseRequestURI(s)
return types.Bool(err == nil)
}

48
vendor/k8s.io/apiserver/pkg/cel/limits.go generated vendored Normal file
View File

@@ -0,0 +1,48 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
const (
// DefaultMaxRequestSizeBytes is the size of the largest request that will be accepted
DefaultMaxRequestSizeBytes = int64(3 * 1024 * 1024)
// MaxDurationSizeJSON
// OpenAPI duration strings follow RFC 3339, section 5.6 - see the comment on maxDatetimeSizeJSON
MaxDurationSizeJSON = 32
// MaxDatetimeSizeJSON
// OpenAPI datetime strings follow RFC 3339, section 5.6, and the longest possible
// such string is 9999-12-31T23:59:59.999999999Z, which has length 30 - we add 2
// to allow for quotation marks
MaxDatetimeSizeJSON = 32
// MinDurationSizeJSON
// Golang allows a string of 0 to be parsed as a duration, so that plus 2 to account for
// quotation marks makes 3
MinDurationSizeJSON = 3
// JSONDateSize is the size of a date serialized as part of a JSON object
// RFC 3339 dates require YYYY-MM-DD, and then we add 2 to allow for quotation marks
JSONDateSize = 12
// MinDatetimeSizeJSON is the minimal length of a datetime formatted as RFC 3339
// RFC 3339 datetimes require a full date (YYYY-MM-DD) and full time (HH:MM:SS), and we add 3 for
// quotation marks like always in addition to the capital T that separates the date and time
MinDatetimeSizeJSON = 21
// MinStringSize is the size of literal ""
MinStringSize = 2
// MinBoolSize is the length of literal true
MinBoolSize = 4
// MinNumberSize is the length of literal 0
MinNumberSize = 1
)

79
vendor/k8s.io/apiserver/pkg/cel/registry.go generated vendored Normal file
View File

@@ -0,0 +1,79 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
import (
"sync"
"github.com/google/cel-go/cel"
)
// Resolver declares methods to find policy templates and related configuration objects.
type Resolver interface {
// FindType returns a DeclType instance corresponding to the given fully-qualified name, if
// present.
FindType(name string) (*DeclType, bool)
}
// NewRegistry create a registry for keeping track of environments and types
// from a base cel.Env expression environment.
func NewRegistry(stdExprEnv *cel.Env) *Registry {
return &Registry{
exprEnvs: map[string]*cel.Env{"": stdExprEnv},
types: map[string]*DeclType{
BoolType.TypeName(): BoolType,
BytesType.TypeName(): BytesType,
DoubleType.TypeName(): DoubleType,
DurationType.TypeName(): DurationType,
IntType.TypeName(): IntType,
NullType.TypeName(): NullType,
StringType.TypeName(): StringType,
TimestampType.TypeName(): TimestampType,
UintType.TypeName(): UintType,
ListType.TypeName(): ListType,
MapType.TypeName(): MapType,
},
}
}
// Registry defines a repository of environment, schema, template, and type definitions.
//
// Registry instances are concurrency-safe.
type Registry struct {
rwMux sync.RWMutex
exprEnvs map[string]*cel.Env
types map[string]*DeclType
}
// FindType implements the Resolver interface method.
func (r *Registry) FindType(name string) (*DeclType, bool) {
r.rwMux.RLock()
defer r.rwMux.RUnlock()
typ, found := r.types[name]
if found {
return typ, true
}
return typ, found
}
// SetType registers a DeclType descriptor by its fully qualified name.
func (r *Registry) SetType(name string, declType *DeclType) error {
r.rwMux.Lock()
defer r.rwMux.Unlock()
r.types[name] = declType
return nil
}

552
vendor/k8s.io/apiserver/pkg/cel/types.go generated vendored Normal file
View File

@@ -0,0 +1,552 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
import (
"fmt"
"math"
"time"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
"github.com/google/cel-go/common/types/traits"
exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
"google.golang.org/protobuf/proto"
)
const (
noMaxLength = math.MaxInt
)
// NewListType returns a parameterized list type with a specified element type.
func NewListType(elem *DeclType, maxItems int64) *DeclType {
return &DeclType{
name: "list",
ElemType: elem,
MaxElements: maxItems,
celType: cel.ListType(elem.CelType()),
defaultValue: NewListValue(),
// a list can always be represented as [] in JSON, so hardcode the min size
// to 2
MinSerializedSize: 2,
}
}
// NewMapType returns a parameterized map type with the given key and element types.
func NewMapType(key, elem *DeclType, maxProperties int64) *DeclType {
return &DeclType{
name: "map",
KeyType: key,
ElemType: elem,
MaxElements: maxProperties,
celType: cel.MapType(key.CelType(), elem.CelType()),
defaultValue: NewMapValue(),
// a map can always be represented as {} in JSON, so hardcode the min size
// to 2
MinSerializedSize: 2,
}
}
// NewObjectType creates an object type with a qualified name and a set of field declarations.
func NewObjectType(name string, fields map[string]*DeclField) *DeclType {
t := &DeclType{
name: name,
Fields: fields,
celType: cel.ObjectType(name),
traitMask: traits.FieldTesterType | traits.IndexerType,
// an object could potentially be larger than the min size we default to here ({}),
// but we rely upon the caller to change MinSerializedSize accordingly if they add
// properties to the object
MinSerializedSize: 2,
}
t.defaultValue = NewObjectValue(t)
return t
}
func NewSimpleTypeWithMinSize(name string, celType *cel.Type, zeroVal ref.Val, minSize int64) *DeclType {
return &DeclType{
name: name,
celType: celType,
defaultValue: zeroVal,
MinSerializedSize: minSize,
}
}
// DeclType represents the universal type descriptor for OpenAPIv3 types.
type DeclType struct {
fmt.Stringer
name string
// Fields contains a map of escaped CEL identifier field names to field declarations.
Fields map[string]*DeclField
KeyType *DeclType
ElemType *DeclType
TypeParam bool
Metadata map[string]string
MaxElements int64
// MinSerializedSize represents the smallest possible size in bytes that
// the DeclType could be serialized to in JSON.
MinSerializedSize int64
celType *cel.Type
traitMask int
defaultValue ref.Val
}
// MaybeAssignTypeName attempts to set the DeclType name to a fully qualified name, if the type
// is of `object` type.
//
// The DeclType must return true for `IsObject` or this assignment will error.
func (t *DeclType) MaybeAssignTypeName(name string) *DeclType {
if t.IsObject() {
objUpdated := false
if t.name != "object" {
name = t.name
} else {
objUpdated = true
}
fieldMap := make(map[string]*DeclField, len(t.Fields))
for fieldName, field := range t.Fields {
fieldType := field.Type
fieldTypeName := fmt.Sprintf("%s.%s", name, fieldName)
updated := fieldType.MaybeAssignTypeName(fieldTypeName)
if updated == fieldType {
fieldMap[fieldName] = field
continue
}
objUpdated = true
fieldMap[fieldName] = &DeclField{
Name: fieldName,
Type: updated,
Required: field.Required,
enumValues: field.enumValues,
defaultValue: field.defaultValue,
}
}
if !objUpdated {
return t
}
return &DeclType{
name: name,
Fields: fieldMap,
KeyType: t.KeyType,
ElemType: t.ElemType,
TypeParam: t.TypeParam,
Metadata: t.Metadata,
celType: cel.ObjectType(name),
traitMask: t.traitMask,
defaultValue: t.defaultValue,
MinSerializedSize: t.MinSerializedSize,
}
}
if t.IsMap() {
elemTypeName := fmt.Sprintf("%s.@elem", name)
updated := t.ElemType.MaybeAssignTypeName(elemTypeName)
if updated == t.ElemType {
return t
}
return NewMapType(t.KeyType, updated, t.MaxElements)
}
if t.IsList() {
elemTypeName := fmt.Sprintf("%s.@idx", name)
updated := t.ElemType.MaybeAssignTypeName(elemTypeName)
if updated == t.ElemType {
return t
}
return NewListType(updated, t.MaxElements)
}
return t
}
// ExprType returns the CEL expression type of this declaration.
func (t *DeclType) ExprType() (*exprpb.Type, error) {
return cel.TypeToExprType(t.celType)
}
// CelType returns the CEL type of this declaration.
func (t *DeclType) CelType() *cel.Type {
return t.celType
}
// FindField returns the DeclField with the given name if present.
func (t *DeclType) FindField(name string) (*DeclField, bool) {
f, found := t.Fields[name]
return f, found
}
// HasTrait implements the CEL ref.Type interface making this type declaration suitable for use
// within the CEL evaluator.
func (t *DeclType) HasTrait(trait int) bool {
if t.traitMask&trait == trait {
return true
}
if t.defaultValue == nil {
return false
}
_, isDecl := t.defaultValue.Type().(*DeclType)
if isDecl {
return false
}
return t.defaultValue.Type().HasTrait(trait)
}
// IsList returns whether the declaration is a `list` type which defines a parameterized element
// type, but not a parameterized key type or fields.
func (t *DeclType) IsList() bool {
return t.KeyType == nil && t.ElemType != nil && t.Fields == nil
}
// IsMap returns whether the declaration is a 'map' type which defines parameterized key and
// element types, but not fields.
func (t *DeclType) IsMap() bool {
return t.KeyType != nil && t.ElemType != nil && t.Fields == nil
}
// IsObject returns whether the declartion is an 'object' type which defined a set of typed fields.
func (t *DeclType) IsObject() bool {
return t.KeyType == nil && t.ElemType == nil && t.Fields != nil
}
// String implements the fmt.Stringer interface method.
func (t *DeclType) String() string {
return t.name
}
// TypeName returns the fully qualified type name for the DeclType.
func (t *DeclType) TypeName() string {
return t.name
}
// DefaultValue returns the CEL ref.Val representing the default value for this object type,
// if one exists.
func (t *DeclType) DefaultValue() ref.Val {
return t.defaultValue
}
// FieldTypeMap constructs a map of the field and object types nested within a given type.
func FieldTypeMap(path string, t *DeclType) map[string]*DeclType {
if t.IsObject() && t.TypeName() != "object" {
path = t.TypeName()
}
types := make(map[string]*DeclType)
buildDeclTypes(path, t, types)
return types
}
func buildDeclTypes(path string, t *DeclType, types map[string]*DeclType) {
// Ensure object types are properly named according to where they appear in the schema.
if t.IsObject() {
// Hack to ensure that names are uniquely qualified and work well with the type
// resolution steps which require fully qualified type names for field resolution
// to function properly.
types[t.TypeName()] = t
for name, field := range t.Fields {
fieldPath := fmt.Sprintf("%s.%s", path, name)
buildDeclTypes(fieldPath, field.Type, types)
}
}
// Map element properties to type names if needed.
if t.IsMap() {
mapElemPath := fmt.Sprintf("%s.@elem", path)
buildDeclTypes(mapElemPath, t.ElemType, types)
types[path] = t
}
// List element properties.
if t.IsList() {
listIdxPath := fmt.Sprintf("%s.@idx", path)
buildDeclTypes(listIdxPath, t.ElemType, types)
types[path] = t
}
}
// DeclField describes the name, ordinal, and optionality of a field declaration within a type.
type DeclField struct {
Name string
Type *DeclType
Required bool
enumValues []interface{}
defaultValue interface{}
}
func NewDeclField(name string, declType *DeclType, required bool, enumValues []interface{}, defaultValue interface{}) *DeclField {
return &DeclField{
Name: name,
Type: declType,
Required: required,
enumValues: enumValues,
defaultValue: defaultValue,
}
}
// TypeName returns the string type name of the field.
func (f *DeclField) TypeName() string {
return f.Type.TypeName()
}
// DefaultValue returns the zero value associated with the field.
func (f *DeclField) DefaultValue() ref.Val {
if f.defaultValue != nil {
return types.DefaultTypeAdapter.NativeToValue(f.defaultValue)
}
return f.Type.DefaultValue()
}
// EnumValues returns the set of values that this field may take.
func (f *DeclField) EnumValues() []ref.Val {
if f.enumValues == nil || len(f.enumValues) == 0 {
return []ref.Val{}
}
ev := make([]ref.Val, len(f.enumValues))
for i, e := range f.enumValues {
ev[i] = types.DefaultTypeAdapter.NativeToValue(e)
}
return ev
}
// NewRuleTypes returns an Open API Schema-based type-system which is CEL compatible.
func NewRuleTypes(kind string,
declType *DeclType,
res Resolver) (*RuleTypes, error) {
// Note, if the schema indicates that it's actually based on another proto
// then prefer the proto definition. For expressions in the proto, a new field
// annotation will be needed to indicate the expected environment and type of
// the expression.
schemaTypes, err := newSchemaTypeProvider(kind, declType)
if err != nil {
return nil, err
}
if schemaTypes == nil {
return nil, nil
}
return &RuleTypes{
ruleSchemaDeclTypes: schemaTypes,
resolver: res,
}, nil
}
// RuleTypes extends the CEL ref.TypeProvider interface and provides an Open API Schema-based
// type-system.
type RuleTypes struct {
ref.TypeProvider
ruleSchemaDeclTypes *schemaTypeProvider
typeAdapter ref.TypeAdapter
resolver Resolver
}
// EnvOptions returns a set of cel.EnvOption values which includes the declaration set
// as well as a custom ref.TypeProvider.
//
// Note, the standard declaration set includes 'rule' which is defined as the top-level rule-schema
// type if one is configured.
//
// If the RuleTypes value is nil, an empty []cel.EnvOption set is returned.
func (rt *RuleTypes) EnvOptions(tp ref.TypeProvider) ([]cel.EnvOption, error) {
if rt == nil {
return []cel.EnvOption{}, nil
}
var ta ref.TypeAdapter = types.DefaultTypeAdapter
tpa, ok := tp.(ref.TypeAdapter)
if ok {
ta = tpa
}
rtWithTypes := &RuleTypes{
TypeProvider: tp,
typeAdapter: ta,
ruleSchemaDeclTypes: rt.ruleSchemaDeclTypes,
resolver: rt.resolver,
}
for name, declType := range rt.ruleSchemaDeclTypes.types {
tpType, found := tp.FindType(name)
expT, err := declType.ExprType()
if err != nil {
return nil, fmt.Errorf("fail to get cel type: %s", err)
}
if found && !proto.Equal(tpType, expT) {
return nil, fmt.Errorf(
"type %s definition differs between CEL environment and rule", name)
}
}
return []cel.EnvOption{
cel.CustomTypeProvider(rtWithTypes),
cel.CustomTypeAdapter(rtWithTypes),
cel.Variable("rule", rt.ruleSchemaDeclTypes.root.CelType()),
}, nil
}
// FindType attempts to resolve the typeName provided from the rule's rule-schema, or if not
// from the embedded ref.TypeProvider.
//
// FindType overrides the default type-finding behavior of the embedded TypeProvider.
//
// Note, when the type name is based on the Open API Schema, the name will reflect the object path
// where the type definition appears.
func (rt *RuleTypes) FindType(typeName string) (*exprpb.Type, bool) {
if rt == nil {
return nil, false
}
declType, found := rt.findDeclType(typeName)
if found {
expT, err := declType.ExprType()
if err != nil {
return expT, false
}
return expT, found
}
return rt.TypeProvider.FindType(typeName)
}
// FindDeclType returns the CPT type description which can be mapped to a CEL type.
func (rt *RuleTypes) FindDeclType(typeName string) (*DeclType, bool) {
if rt == nil {
return nil, false
}
return rt.findDeclType(typeName)
}
// FindFieldType returns a field type given a type name and field name, if found.
//
// Note, the type name for an Open API Schema type is likely to be its qualified object path.
// If, in the future an object instance rather than a type name were provided, the field
// resolution might more accurately reflect the expected type model. However, in this case
// concessions were made to align with the existing CEL interfaces.
func (rt *RuleTypes) FindFieldType(typeName, fieldName string) (*ref.FieldType, bool) {
st, found := rt.findDeclType(typeName)
if !found {
return rt.TypeProvider.FindFieldType(typeName, fieldName)
}
f, found := st.Fields[fieldName]
if found {
ft := f.Type
expT, err := ft.ExprType()
if err != nil {
return nil, false
}
return &ref.FieldType{
Type: expT,
}, true
}
// This could be a dynamic map.
if st.IsMap() {
et := st.ElemType
expT, err := et.ExprType()
if err != nil {
return nil, false
}
return &ref.FieldType{
Type: expT,
}, true
}
return nil, false
}
// NativeToValue is an implementation of the ref.TypeAdapater interface which supports conversion
// of rule values to CEL ref.Val instances.
func (rt *RuleTypes) NativeToValue(val interface{}) ref.Val {
return rt.typeAdapter.NativeToValue(val)
}
// TypeNames returns the list of type names declared within the RuleTypes object.
func (rt *RuleTypes) TypeNames() []string {
typeNames := make([]string, len(rt.ruleSchemaDeclTypes.types))
i := 0
for name := range rt.ruleSchemaDeclTypes.types {
typeNames[i] = name
i++
}
return typeNames
}
func (rt *RuleTypes) findDeclType(typeName string) (*DeclType, bool) {
declType, found := rt.ruleSchemaDeclTypes.types[typeName]
if found {
return declType, true
}
declType, found = rt.resolver.FindType(typeName)
if found {
return declType, true
}
return nil, false
}
func newSchemaTypeProvider(kind string, declType *DeclType) (*schemaTypeProvider, error) {
if declType == nil {
return nil, nil
}
root := declType.MaybeAssignTypeName(kind)
types := FieldTypeMap(kind, root)
return &schemaTypeProvider{
root: root,
types: types,
}, nil
}
type schemaTypeProvider struct {
root *DeclType
types map[string]*DeclType
}
var (
// AnyType is equivalent to the CEL 'protobuf.Any' type in that the value may have any of the
// types supported.
AnyType = NewSimpleTypeWithMinSize("any", cel.AnyType, nil, 1)
// BoolType is equivalent to the CEL 'bool' type.
BoolType = NewSimpleTypeWithMinSize("bool", cel.BoolType, types.False, MinBoolSize)
// BytesType is equivalent to the CEL 'bytes' type.
BytesType = NewSimpleTypeWithMinSize("bytes", cel.BytesType, types.Bytes([]byte{}), MinStringSize)
// DoubleType is equivalent to the CEL 'double' type which is a 64-bit floating point value.
DoubleType = NewSimpleTypeWithMinSize("double", cel.DoubleType, types.Double(0), MinNumberSize)
// DurationType is equivalent to the CEL 'duration' type.
DurationType = NewSimpleTypeWithMinSize("duration", cel.DurationType, types.Duration{Duration: time.Duration(0)}, MinDurationSizeJSON)
// DateType is equivalent to the CEL 'date' type.
DateType = NewSimpleTypeWithMinSize("date", cel.TimestampType, types.Timestamp{Time: time.Time{}}, JSONDateSize)
// DynType is the equivalent of the CEL 'dyn' concept which indicates that the type will be
// determined at runtime rather than compile time.
DynType = NewSimpleTypeWithMinSize("dyn", cel.DynType, nil, 1)
// IntType is equivalent to the CEL 'int' type which is a 64-bit signed int.
IntType = NewSimpleTypeWithMinSize("int", cel.IntType, types.IntZero, MinNumberSize)
// NullType is equivalent to the CEL 'null_type'.
NullType = NewSimpleTypeWithMinSize("null_type", cel.NullType, types.NullValue, 4)
// StringType is equivalent to the CEL 'string' type which is expected to be a UTF-8 string.
// StringType values may either be string literals or expression strings.
StringType = NewSimpleTypeWithMinSize("string", cel.StringType, types.String(""), MinStringSize)
// TimestampType corresponds to the well-known protobuf.Timestamp type supported within CEL.
// Note that both the OpenAPI date and date-time types map onto TimestampType, so not all types
// labeled as Timestamp will necessarily have the same MinSerializedSize.
TimestampType = NewSimpleTypeWithMinSize("timestamp", cel.TimestampType, types.Timestamp{Time: time.Time{}}, JSONDateSize)
// UintType is equivalent to the CEL 'uint' type.
UintType = NewSimpleTypeWithMinSize("uint", cel.UintType, types.Uint(0), 1)
// ListType is equivalent to the CEL 'list' type.
ListType = NewListType(AnyType, noMaxLength)
// MapType is equivalent to the CEL 'map' type.
MapType = NewMapType(AnyType, AnyType, noMaxLength)
)

80
vendor/k8s.io/apiserver/pkg/cel/url.go generated vendored Normal file
View File

@@ -0,0 +1,80 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
import (
"fmt"
"net/url"
"reflect"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/checker/decls"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
)
// URL provides a CEL representation of a URL.
type URL struct {
*url.URL
}
var (
URLObject = decls.NewObjectType("kubernetes.URL")
typeValue = types.NewTypeValue("kubernetes.URL")
URLType = cel.ObjectType("kubernetes.URL")
)
// ConvertToNative implements ref.Val.ConvertToNative.
func (d URL) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
if reflect.TypeOf(d.URL).AssignableTo(typeDesc) {
return d.URL, nil
}
if reflect.TypeOf("").AssignableTo(typeDesc) {
return d.URL.String(), nil
}
return nil, fmt.Errorf("type conversion error from 'URL' to '%v'", typeDesc)
}
// ConvertToType implements ref.Val.ConvertToType.
func (d URL) ConvertToType(typeVal ref.Type) ref.Val {
switch typeVal {
case typeValue:
return d
case types.TypeType:
return typeValue
}
return types.NewErr("type conversion error from '%s' to '%s'", typeValue, typeVal)
}
// Equal implements ref.Val.Equal.
func (d URL) Equal(other ref.Val) ref.Val {
otherDur, ok := other.(URL)
if !ok {
return types.MaybeNoSuchOverloadErr(other)
}
return types.Bool(d.URL.String() == otherDur.URL.String())
}
// Type implements ref.Val.Type.
func (d URL) Type() ref.Type {
return typeValue
}
// Value implements ref.Val.Value.
func (d URL) Value() interface{} {
return d.URL
}

769
vendor/k8s.io/apiserver/pkg/cel/value.go generated vendored Normal file
View File

@@ -0,0 +1,769 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
import (
"fmt"
"reflect"
"sync"
"time"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
"github.com/google/cel-go/common/types/traits"
)
// EncodeStyle is a hint for string encoding of parsed values.
type EncodeStyle int
const (
// BlockValueStyle is the default string encoding which preserves whitespace and newlines.
BlockValueStyle EncodeStyle = iota
// FlowValueStyle indicates that the string is an inline representation of complex types.
FlowValueStyle
// FoldedValueStyle is a multiline string with whitespace and newlines trimmed to a single
// a whitespace. Repeated newlines are replaced with a single newline rather than a single
// whitespace.
FoldedValueStyle
// LiteralStyle is a multiline string that preserves newlines, but trims all other whitespace
// to a single character.
LiteralStyle
)
// NewEmptyDynValue returns the zero-valued DynValue.
func NewEmptyDynValue() *DynValue {
// note: 0 is not a valid parse node identifier.
dv, _ := NewDynValue(0, nil)
return dv
}
// NewDynValue returns a DynValue that corresponds to a parse node id and value.
func NewDynValue(id int64, val interface{}) (*DynValue, error) {
dv := &DynValue{ID: id}
err := dv.SetValue(val)
return dv, err
}
// DynValue is a dynamically typed value used to describe unstructured content.
// Whether the value has the desired type is determined by where it is used within the Instance or
// Template, and whether there are schemas which might enforce a more rigid type definition.
type DynValue struct {
ID int64
EncodeStyle EncodeStyle
value interface{}
exprValue ref.Val
declType *DeclType
}
// DeclType returns the policy model type of the dyn value.
func (dv *DynValue) DeclType() *DeclType {
return dv.declType
}
// ConvertToNative is an implementation of the CEL ref.Val method used to adapt between CEL types
// and Go-native types.
//
// The default behavior of this method is to first convert to a CEL type which has a well-defined
// set of conversion behaviors and proxy to the CEL ConvertToNative method for the type.
func (dv *DynValue) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
ev := dv.ExprValue()
if types.IsError(ev) {
return nil, ev.(*types.Err)
}
return ev.ConvertToNative(typeDesc)
}
// Equal returns whether the dyn value is equal to a given CEL value.
func (dv *DynValue) Equal(other ref.Val) ref.Val {
dvType := dv.Type()
otherType := other.Type()
// Preserve CEL's homogeneous equality constraint.
if dvType.TypeName() != otherType.TypeName() {
return types.MaybeNoSuchOverloadErr(other)
}
switch v := dv.value.(type) {
case ref.Val:
return v.Equal(other)
case PlainTextValue:
return celBool(string(v) == other.Value().(string))
case *MultilineStringValue:
return celBool(v.Value == other.Value().(string))
case time.Duration:
otherDuration := other.Value().(time.Duration)
return celBool(v == otherDuration)
case time.Time:
otherTimestamp := other.Value().(time.Time)
return celBool(v.Equal(otherTimestamp))
default:
return celBool(reflect.DeepEqual(v, other.Value()))
}
}
// ExprValue converts the DynValue into a CEL value.
func (dv *DynValue) ExprValue() ref.Val {
return dv.exprValue
}
// Value returns the underlying value held by this reference.
func (dv *DynValue) Value() interface{} {
return dv.value
}
// SetValue updates the underlying value held by this reference.
func (dv *DynValue) SetValue(value interface{}) error {
dv.value = value
var err error
dv.exprValue, dv.declType, err = exprValue(value)
return err
}
// Type returns the CEL type for the given value.
func (dv *DynValue) Type() ref.Type {
return dv.ExprValue().Type()
}
func exprValue(value interface{}) (ref.Val, *DeclType, error) {
switch v := value.(type) {
case bool:
return types.Bool(v), BoolType, nil
case []byte:
return types.Bytes(v), BytesType, nil
case float64:
return types.Double(v), DoubleType, nil
case int64:
return types.Int(v), IntType, nil
case string:
return types.String(v), StringType, nil
case uint64:
return types.Uint(v), UintType, nil
case time.Duration:
return types.Duration{Duration: v}, DurationType, nil
case time.Time:
return types.Timestamp{Time: v}, TimestampType, nil
case types.Null:
return v, NullType, nil
case *ListValue:
return v, ListType, nil
case *MapValue:
return v, MapType, nil
case *ObjectValue:
return v, v.objectType, nil
default:
return nil, unknownType, fmt.Errorf("unsupported type: (%T)%v", v, v)
}
}
// PlainTextValue is a text string literal which must not be treated as an expression.
type PlainTextValue string
// MultilineStringValue is a multiline string value which has been parsed in a way which omits
// whitespace as well as a raw form which preserves whitespace.
type MultilineStringValue struct {
Value string
Raw string
}
func newStructValue() *structValue {
return &structValue{
Fields: []*Field{},
fieldMap: map[string]*Field{},
}
}
type structValue struct {
Fields []*Field
fieldMap map[string]*Field
}
// AddField appends a MapField to the MapValue and indexes the field by name.
func (sv *structValue) AddField(field *Field) {
sv.Fields = append(sv.Fields, field)
sv.fieldMap[field.Name] = field
}
// ConvertToNative converts the MapValue type to a native go types.
func (sv *structValue) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
if typeDesc.Kind() != reflect.Map &&
typeDesc.Kind() != reflect.Struct &&
typeDesc.Kind() != reflect.Pointer &&
typeDesc.Kind() != reflect.Interface {
return nil, fmt.Errorf("type conversion error from object to '%v'", typeDesc)
}
// Unwrap pointers, but track their use.
isPtr := false
if typeDesc.Kind() == reflect.Pointer {
tk := typeDesc
typeDesc = typeDesc.Elem()
if typeDesc.Kind() == reflect.Pointer {
return nil, fmt.Errorf("unsupported type conversion to '%v'", tk)
}
isPtr = true
}
if typeDesc.Kind() == reflect.Map {
keyType := typeDesc.Key()
if keyType.Kind() != reflect.String && keyType.Kind() != reflect.Interface {
return nil, fmt.Errorf("object fields cannot be converted to type '%v'", keyType)
}
elemType := typeDesc.Elem()
sz := len(sv.fieldMap)
ntvMap := reflect.MakeMapWithSize(typeDesc, sz)
for name, val := range sv.fieldMap {
refVal, err := val.Ref.ConvertToNative(elemType)
if err != nil {
return nil, err
}
ntvMap.SetMapIndex(reflect.ValueOf(name), reflect.ValueOf(refVal))
}
return ntvMap.Interface(), nil
}
if typeDesc.Kind() == reflect.Struct {
ntvObjPtr := reflect.New(typeDesc)
ntvObj := ntvObjPtr.Elem()
for name, val := range sv.fieldMap {
f := ntvObj.FieldByName(name)
if !f.IsValid() {
return nil, fmt.Errorf("type conversion error, no such field %s in type %v",
name, typeDesc)
}
fv, err := val.Ref.ConvertToNative(f.Type())
if err != nil {
return nil, err
}
f.Set(reflect.ValueOf(fv))
}
if isPtr {
return ntvObjPtr.Interface(), nil
}
return ntvObj.Interface(), nil
}
return nil, fmt.Errorf("type conversion error from object to '%v'", typeDesc)
}
// GetField returns a MapField by name if one exists.
func (sv *structValue) GetField(name string) (*Field, bool) {
field, found := sv.fieldMap[name]
return field, found
}
// IsSet returns whether the given field, which is defined, has also been set.
func (sv *structValue) IsSet(key ref.Val) ref.Val {
k, ok := key.(types.String)
if !ok {
return types.MaybeNoSuchOverloadErr(key)
}
name := string(k)
_, found := sv.fieldMap[name]
return celBool(found)
}
// NewObjectValue creates a struct value with a schema type and returns the empty ObjectValue.
func NewObjectValue(sType *DeclType) *ObjectValue {
return &ObjectValue{
structValue: newStructValue(),
objectType: sType,
}
}
// ObjectValue is a struct with a custom schema type which indicates the fields and types
// associated with the structure.
type ObjectValue struct {
*structValue
objectType *DeclType
}
// ConvertToType is an implementation of the CEL ref.Val interface method.
func (o *ObjectValue) ConvertToType(t ref.Type) ref.Val {
if t == types.TypeType {
return types.NewObjectTypeValue(o.objectType.TypeName())
}
if t.TypeName() == o.objectType.TypeName() {
return o
}
return types.NewErr("type conversion error from '%s' to '%s'", o.Type(), t)
}
// Equal returns true if the two object types are equal and their field values are equal.
func (o *ObjectValue) Equal(other ref.Val) ref.Val {
// Preserve CEL's homogeneous equality semantics.
if o.objectType.TypeName() != other.Type().TypeName() {
return types.MaybeNoSuchOverloadErr(other)
}
o2 := other.(traits.Indexer)
for name := range o.objectType.Fields {
k := types.String(name)
v := o.Get(k)
ov := o2.Get(k)
vEq := v.Equal(ov)
if vEq != types.True {
return vEq
}
}
return types.True
}
// Get returns the value of the specified field.
//
// If the field is set, its value is returned. If the field is not set, the default value for the
// field is returned thus allowing for safe-traversal and preserving proto-like field traversal
// semantics for Open API Schema backed types.
func (o *ObjectValue) Get(name ref.Val) ref.Val {
n, ok := name.(types.String)
if !ok {
return types.MaybeNoSuchOverloadErr(n)
}
nameStr := string(n)
field, found := o.fieldMap[nameStr]
if found {
return field.Ref.ExprValue()
}
fieldDef, found := o.objectType.Fields[nameStr]
if !found {
return types.NewErr("no such field: %s", nameStr)
}
defValue := fieldDef.DefaultValue()
if defValue != nil {
return defValue
}
return types.NewErr("no default for type: %s", fieldDef.TypeName())
}
// Type returns the CEL type value of the object.
func (o *ObjectValue) Type() ref.Type {
return o.objectType
}
// Value returns the Go-native representation of the object.
func (o *ObjectValue) Value() interface{} {
return o
}
// NewMapValue returns an empty MapValue.
func NewMapValue() *MapValue {
return &MapValue{
structValue: newStructValue(),
}
}
// MapValue declares an object with a set of named fields whose values are dynamically typed.
type MapValue struct {
*structValue
}
// ConvertToObject produces an ObjectValue from the MapValue with the associated schema type.
//
// The conversion is shallow and the memory shared between the Object and Map as all references
// to the map are expected to be replaced with the Object reference.
func (m *MapValue) ConvertToObject(declType *DeclType) *ObjectValue {
return &ObjectValue{
structValue: m.structValue,
objectType: declType,
}
}
// Contains returns whether the given key is contained in the MapValue.
func (m *MapValue) Contains(key ref.Val) ref.Val {
v, found := m.Find(key)
if v != nil && types.IsUnknownOrError(v) {
return v
}
return celBool(found)
}
// ConvertToType converts the MapValue to another CEL type, if possible.
func (m *MapValue) ConvertToType(t ref.Type) ref.Val {
switch t {
case types.MapType:
return m
case types.TypeType:
return types.MapType
}
return types.NewErr("type conversion error from '%s' to '%s'", m.Type(), t)
}
// Equal returns true if the maps are of the same size, have the same keys, and the key-values
// from each map are equal.
func (m *MapValue) Equal(other ref.Val) ref.Val {
oMap, isMap := other.(traits.Mapper)
if !isMap {
return types.MaybeNoSuchOverloadErr(other)
}
if m.Size() != oMap.Size() {
return types.False
}
for name, field := range m.fieldMap {
k := types.String(name)
ov, found := oMap.Find(k)
if !found {
return types.False
}
v := field.Ref.ExprValue()
vEq := v.Equal(ov)
if vEq != types.True {
return vEq
}
}
return types.True
}
// Find returns the value for the key in the map, if found.
func (m *MapValue) Find(name ref.Val) (ref.Val, bool) {
// Currently only maps with string keys are supported as this is best aligned with JSON,
// and also much simpler to support.
n, ok := name.(types.String)
if !ok {
return types.MaybeNoSuchOverloadErr(n), true
}
nameStr := string(n)
field, found := m.fieldMap[nameStr]
if found {
return field.Ref.ExprValue(), true
}
return nil, false
}
// Get returns the value for the key in the map, or error if not found.
func (m *MapValue) Get(key ref.Val) ref.Val {
v, found := m.Find(key)
if found {
return v
}
return types.ValOrErr(key, "no such key: %v", key)
}
// Iterator produces a traits.Iterator which walks over the map keys.
//
// The Iterator is frequently used within comprehensions.
func (m *MapValue) Iterator() traits.Iterator {
keys := make([]ref.Val, len(m.fieldMap))
i := 0
for k := range m.fieldMap {
keys[i] = types.String(k)
i++
}
return &baseMapIterator{
baseVal: &baseVal{},
keys: keys,
}
}
// Size returns the number of keys in the map.
func (m *MapValue) Size() ref.Val {
return types.Int(len(m.Fields))
}
// Type returns the CEL ref.Type for the map.
func (m *MapValue) Type() ref.Type {
return types.MapType
}
// Value returns the Go-native representation of the MapValue.
func (m *MapValue) Value() interface{} {
return m
}
type baseMapIterator struct {
*baseVal
keys []ref.Val
idx int
}
// HasNext implements the traits.Iterator interface method.
func (it *baseMapIterator) HasNext() ref.Val {
if it.idx < len(it.keys) {
return types.True
}
return types.False
}
// Next implements the traits.Iterator interface method.
func (it *baseMapIterator) Next() ref.Val {
key := it.keys[it.idx]
it.idx++
return key
}
// Type implements the CEL ref.Val interface metohd.
func (it *baseMapIterator) Type() ref.Type {
return types.IteratorType
}
// NewField returns a MapField instance with an empty DynValue that refers to the
// specified parse node id and field name.
func NewField(id int64, name string) *Field {
return &Field{
ID: id,
Name: name,
Ref: NewEmptyDynValue(),
}
}
// Field specifies a field name and a reference to a dynamic value.
type Field struct {
ID int64
Name string
Ref *DynValue
}
// NewListValue returns an empty ListValue instance.
func NewListValue() *ListValue {
return &ListValue{
Entries: []*DynValue{},
}
}
// ListValue contains a list of dynamically typed entries.
type ListValue struct {
Entries []*DynValue
initValueSet sync.Once
valueSet map[ref.Val]struct{}
}
// Add concatenates two lists together to produce a new CEL list value.
func (lv *ListValue) Add(other ref.Val) ref.Val {
oArr, isArr := other.(traits.Lister)
if !isArr {
return types.MaybeNoSuchOverloadErr(other)
}
szRight := len(lv.Entries)
szLeft := int(oArr.Size().(types.Int))
sz := szRight + szLeft
combo := make([]ref.Val, sz)
for i := 0; i < szRight; i++ {
combo[i] = lv.Entries[i].ExprValue()
}
for i := 0; i < szLeft; i++ {
combo[i+szRight] = oArr.Get(types.Int(i))
}
return types.DefaultTypeAdapter.NativeToValue(combo)
}
// Append adds another entry into the ListValue.
func (lv *ListValue) Append(entry *DynValue) {
lv.Entries = append(lv.Entries, entry)
// The append resets all previously built indices.
lv.initValueSet = sync.Once{}
}
// Contains returns whether the input `val` is equal to an element in the list.
//
// If any pair-wise comparison between the input value and the list element is an error, the
// operation will return an error.
func (lv *ListValue) Contains(val ref.Val) ref.Val {
if types.IsUnknownOrError(val) {
return val
}
lv.initValueSet.Do(lv.finalizeValueSet)
if lv.valueSet != nil {
_, found := lv.valueSet[val]
if found {
return types.True
}
// Instead of returning false, ensure that CEL's heterogeneous equality constraint
// is satisfied by allowing pair-wise equality behavior to determine the outcome.
}
var err ref.Val
sz := len(lv.Entries)
for i := 0; i < sz; i++ {
elem := lv.Entries[i]
cmp := elem.Equal(val)
b, ok := cmp.(types.Bool)
if !ok && err == nil {
err = types.MaybeNoSuchOverloadErr(cmp)
}
if b == types.True {
return types.True
}
}
if err != nil {
return err
}
return types.False
}
// ConvertToNative is an implementation of the CEL ref.Val method used to adapt between CEL types
// and Go-native array-like types.
func (lv *ListValue) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
// Non-list conversion.
if typeDesc.Kind() != reflect.Slice &&
typeDesc.Kind() != reflect.Array &&
typeDesc.Kind() != reflect.Interface {
return nil, fmt.Errorf("type conversion error from list to '%v'", typeDesc)
}
// If the list is already assignable to the desired type return it.
if reflect.TypeOf(lv).AssignableTo(typeDesc) {
return lv, nil
}
// List conversion.
otherElem := typeDesc.Elem()
// Allow the element ConvertToNative() function to determine whether conversion is possible.
sz := len(lv.Entries)
nativeList := reflect.MakeSlice(typeDesc, int(sz), int(sz))
for i := 0; i < sz; i++ {
elem := lv.Entries[i]
nativeElemVal, err := elem.ConvertToNative(otherElem)
if err != nil {
return nil, err
}
nativeList.Index(int(i)).Set(reflect.ValueOf(nativeElemVal))
}
return nativeList.Interface(), nil
}
// ConvertToType converts the ListValue to another CEL type.
func (lv *ListValue) ConvertToType(t ref.Type) ref.Val {
switch t {
case types.ListType:
return lv
case types.TypeType:
return types.ListType
}
return types.NewErr("type conversion error from '%s' to '%s'", ListType, t)
}
// Equal returns true if two lists are of the same size, and the values at each index are also
// equal.
func (lv *ListValue) Equal(other ref.Val) ref.Val {
oArr, isArr := other.(traits.Lister)
if !isArr {
return types.MaybeNoSuchOverloadErr(other)
}
sz := types.Int(len(lv.Entries))
if sz != oArr.Size() {
return types.False
}
for i := types.Int(0); i < sz; i++ {
cmp := lv.Get(i).Equal(oArr.Get(i))
if cmp != types.True {
return cmp
}
}
return types.True
}
// Get returns the value at the given index.
//
// If the index is negative or greater than the size of the list, an error is returned.
func (lv *ListValue) Get(idx ref.Val) ref.Val {
iv, isInt := idx.(types.Int)
if !isInt {
return types.ValOrErr(idx, "unsupported index: %v", idx)
}
i := int(iv)
if i < 0 || i >= len(lv.Entries) {
return types.NewErr("index out of bounds: %v", idx)
}
return lv.Entries[i].ExprValue()
}
// Iterator produces a traits.Iterator suitable for use in CEL comprehension macros.
func (lv *ListValue) Iterator() traits.Iterator {
return &baseListIterator{
getter: lv.Get,
sz: len(lv.Entries),
}
}
// Size returns the number of elements in the list.
func (lv *ListValue) Size() ref.Val {
return types.Int(len(lv.Entries))
}
// Type returns the CEL ref.Type for the list.
func (lv *ListValue) Type() ref.Type {
return types.ListType
}
// Value returns the Go-native value.
func (lv *ListValue) Value() interface{} {
return lv
}
// finalizeValueSet inspects the ListValue entries in order to make internal optimizations once all list
// entries are known.
func (lv *ListValue) finalizeValueSet() {
valueSet := make(map[ref.Val]struct{})
for _, e := range lv.Entries {
switch e.value.(type) {
case bool, float64, int64, string, uint64, types.Null, PlainTextValue:
valueSet[e.ExprValue()] = struct{}{}
default:
lv.valueSet = nil
return
}
}
lv.valueSet = valueSet
}
type baseVal struct{}
func (*baseVal) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
return nil, fmt.Errorf("unsupported native conversion to: %v", typeDesc)
}
func (*baseVal) ConvertToType(t ref.Type) ref.Val {
return types.NewErr("unsupported type conversion to: %v", t)
}
func (*baseVal) Equal(other ref.Val) ref.Val {
return types.NewErr("unsupported equality test between instances")
}
func (v *baseVal) Value() interface{} {
return nil
}
type baseListIterator struct {
*baseVal
getter func(idx ref.Val) ref.Val
sz int
idx int
}
func (it *baseListIterator) HasNext() ref.Val {
if it.idx < it.sz {
return types.True
}
return types.False
}
func (it *baseListIterator) Next() ref.Val {
v := it.getter(types.Int(it.idx))
it.idx++
return v
}
func (it *baseListIterator) Type() ref.Type {
return types.IteratorType
}
func celBool(pred bool) ref.Val {
if pred {
return types.True
}
return types.False
}
var unknownType = &DeclType{name: "unknown", MinSerializedSize: 1}

View File

@@ -0,0 +1,84 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aggregated
import (
"crypto/sha512"
"encoding/json"
"fmt"
"net/http"
"strconv"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
)
// This file exposes helper functions used for calculating the E-Tag header
// used in discovery endpoint responses
// Attaches Cache-Busting functionality to an endpoint
// - Sets ETag header to provided hash
// - Replies with 304 Not Modified, if If-None-Match header matches hash
//
// hash should be the value of calculateETag on object. If hash is empty, then
//
// the object is simply serialized without E-Tag functionality
func ServeHTTPWithETag(
object runtime.Object,
hash string,
serializer runtime.NegotiatedSerializer,
w http.ResponseWriter,
req *http.Request,
) {
// ETag must be enclosed in double quotes:
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
quotedHash := strconv.Quote(hash)
w.Header().Set("ETag", quotedHash)
w.Header().Set("Vary", "Accept")
w.Header().Set("Cache-Control", "public")
// If Request includes If-None-Match and matches hash, reply with 304
// Otherwise, we delegate to the handler for actual content
//
// According to documentation, An Etag within an If-None-Match
// header will be enclosed within doule quotes:
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match#directives
if clientCachedHash := req.Header.Get("If-None-Match"); quotedHash == clientCachedHash {
w.WriteHeader(http.StatusNotModified)
return
}
responsewriters.WriteObjectNegotiated(
serializer,
DiscoveryEndpointRestrictions,
AggregatedDiscoveryGV,
w,
req,
http.StatusOK,
object,
true,
)
}
func calculateETag(resources interface{}) (string, error) {
serialized, err := json.Marshal(resources)
if err != nil {
return "", err
}
return fmt.Sprintf("%X", sha512.Sum512(serialized)), nil
}

View File

@@ -0,0 +1,171 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aggregated
import (
"context"
"errors"
"net/http"
"reflect"
"sync"
"time"
"github.com/emicklei/go-restful/v3"
"github.com/google/go-cmp/cmp"
apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
)
type FakeResourceManager interface {
ResourceManager
Expect() ResourceManager
HasExpectedNumberActions() bool
Validate() error
WaitForActions(ctx context.Context, timeout time.Duration) error
}
func NewFakeResourceManager() FakeResourceManager {
return &fakeResourceManager{}
}
// a resource manager with helper functions for checking the actions
// match expected. For Use in tests
type fakeResourceManager struct {
recorderResourceManager
expect recorderResourceManager
}
// a resource manager which instead of managing a discovery document,
// simply records the calls to its interface functoins for testing
type recorderResourceManager struct {
lock sync.RWMutex
Actions []recorderResourceManagerAction
}
var _ ResourceManager = &fakeResourceManager{}
var _ ResourceManager = &recorderResourceManager{}
// Storage type for a call to the resource manager
type recorderResourceManagerAction struct {
Type string
Group string
Version string
Value interface{}
}
func (f *fakeResourceManager) Expect() ResourceManager {
return &f.expect
}
func (f *fakeResourceManager) HasExpectedNumberActions() bool {
f.lock.RLock()
defer f.lock.RUnlock()
f.expect.lock.RLock()
defer f.expect.lock.RUnlock()
return len(f.Actions) >= len(f.expect.Actions)
}
func (f *fakeResourceManager) Validate() error {
f.lock.RLock()
defer f.lock.RUnlock()
f.expect.lock.RLock()
defer f.expect.lock.RUnlock()
if !reflect.DeepEqual(f.expect.Actions, f.Actions) {
return errors.New(cmp.Diff(f.expect.Actions, f.Actions))
}
return nil
}
func (f *fakeResourceManager) WaitForActions(ctx context.Context, timeout time.Duration) error {
err := wait.PollImmediateWithContext(
ctx,
100*time.Millisecond, // try every 100ms
timeout, // timeout after timeout
func(ctx context.Context) (done bool, err error) {
if f.HasExpectedNumberActions() {
return true, f.Validate()
}
return false, nil
})
return err
}
func (f *recorderResourceManager) SetGroupVersionPriority(gv metav1.GroupVersion, grouppriority, versionpriority int) {
f.lock.Lock()
defer f.lock.Unlock()
f.Actions = append(f.Actions, recorderResourceManagerAction{
Type: "SetGroupVersionPriority",
Group: gv.Group,
Version: gv.Version,
Value: versionpriority,
})
}
func (f *recorderResourceManager) AddGroupVersion(groupName string, value apidiscoveryv2beta1.APIVersionDiscovery) {
f.lock.Lock()
defer f.lock.Unlock()
f.Actions = append(f.Actions, recorderResourceManagerAction{
Type: "AddGroupVersion",
Group: groupName,
Value: value,
})
}
func (f *recorderResourceManager) RemoveGroup(groupName string) {
f.lock.Lock()
defer f.lock.Unlock()
f.Actions = append(f.Actions, recorderResourceManagerAction{
Type: "RemoveGroup",
Group: groupName,
})
}
func (f *recorderResourceManager) RemoveGroupVersion(gv metav1.GroupVersion) {
f.lock.Lock()
defer f.lock.Unlock()
f.Actions = append(f.Actions, recorderResourceManagerAction{
Type: "RemoveGroupVersion",
Group: gv.Group,
Version: gv.Version,
})
}
func (f *recorderResourceManager) SetGroups(values []apidiscoveryv2beta1.APIGroupDiscovery) {
f.lock.Lock()
defer f.lock.Unlock()
f.Actions = append(f.Actions, recorderResourceManagerAction{
Type: "SetGroups",
Value: values,
})
}
func (f *recorderResourceManager) WebService() *restful.WebService {
panic("unimplemented")
}
func (f *recorderResourceManager) ServeHTTP(http.ResponseWriter, *http.Request) {
panic("unimplemented")
}

View File

@@ -0,0 +1,368 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aggregated
import (
"net/http"
"reflect"
"sort"
"sync"
apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/version"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"sync/atomic"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/klog/v2"
)
// This handler serves the /apis endpoint for an aggregated list of
// api resources indexed by their group version.
type ResourceManager interface {
// Adds knowledge of the given groupversion to the discovery document
// If it was already being tracked, updates the stored APIVersionDiscovery
// Thread-safe
AddGroupVersion(groupName string, value apidiscoveryv2beta1.APIVersionDiscovery)
// Sets a priority to be used while sorting a specific group and
// group-version. If two versions report different priorities for
// the group, the higher one will be used. If the group is not
// known, the priority is ignored. The priority for this version
// is forgotten once the group-version is forgotten
SetGroupVersionPriority(gv metav1.GroupVersion, grouppriority, versionpriority int)
// Removes all group versions for a given group
// Thread-safe
RemoveGroup(groupName string)
// Removes a specific groupversion. If all versions of a group have been
// removed, then the entire group is unlisted.
// Thread-safe
RemoveGroupVersion(gv metav1.GroupVersion)
// Resets the manager's known list of group-versions and replaces them
// with the given groups
// Thread-Safe
SetGroups([]apidiscoveryv2beta1.APIGroupDiscovery)
http.Handler
}
type resourceDiscoveryManager struct {
serializer runtime.NegotiatedSerializer
// cache is an atomic pointer to avoid the use of locks
cache atomic.Pointer[cachedGroupList]
// Writes protected by the lock.
// List of all apigroups & resources indexed by the resource manager
lock sync.RWMutex
apiGroups map[string]*apidiscoveryv2beta1.APIGroupDiscovery
versionPriorities map[metav1.GroupVersion]priorityInfo
}
type priorityInfo struct {
GroupPriorityMinimum int
VersionPriority int
}
func NewResourceManager() ResourceManager {
scheme := runtime.NewScheme()
codecs := serializer.NewCodecFactory(scheme)
utilruntime.Must(apidiscoveryv2beta1.AddToScheme(scheme))
return &resourceDiscoveryManager{serializer: codecs, versionPriorities: make(map[metav1.GroupVersion]priorityInfo)}
}
func (rdm *resourceDiscoveryManager) SetGroupVersionPriority(gv metav1.GroupVersion, groupPriorityMinimum, versionPriority int) {
rdm.lock.Lock()
defer rdm.lock.Unlock()
rdm.versionPriorities[gv] = priorityInfo{
GroupPriorityMinimum: groupPriorityMinimum,
VersionPriority: versionPriority,
}
rdm.cache.Store(nil)
}
func (rdm *resourceDiscoveryManager) SetGroups(groups []apidiscoveryv2beta1.APIGroupDiscovery) {
rdm.lock.Lock()
defer rdm.lock.Unlock()
rdm.apiGroups = nil
rdm.cache.Store(nil)
for _, group := range groups {
for _, version := range group.Versions {
rdm.addGroupVersionLocked(group.Name, version)
}
}
// Filter unused out priority entries
for gv := range rdm.versionPriorities {
entry, exists := rdm.apiGroups[gv.Group]
if !exists {
delete(rdm.versionPriorities, gv)
continue
}
containsVersion := false
for _, v := range entry.Versions {
if v.Version == gv.Version {
containsVersion = true
break
}
}
if !containsVersion {
delete(rdm.versionPriorities, gv)
}
}
}
func (rdm *resourceDiscoveryManager) AddGroupVersion(groupName string, value apidiscoveryv2beta1.APIVersionDiscovery) {
rdm.lock.Lock()
defer rdm.lock.Unlock()
rdm.addGroupVersionLocked(groupName, value)
}
func (rdm *resourceDiscoveryManager) addGroupVersionLocked(groupName string, value apidiscoveryv2beta1.APIVersionDiscovery) {
klog.Infof("Adding GroupVersion %s %s to ResourceManager", groupName, value.Version)
if rdm.apiGroups == nil {
rdm.apiGroups = make(map[string]*apidiscoveryv2beta1.APIGroupDiscovery)
}
if existing, groupExists := rdm.apiGroups[groupName]; groupExists {
// If this version already exists, replace it
versionExists := false
// Not very efficient, but in practice there are generally not many versions
for i := range existing.Versions {
if existing.Versions[i].Version == value.Version {
// The new gv is the exact same as what is already in
// the map. This is a noop and cache should not be
// invalidated.
if reflect.DeepEqual(existing.Versions[i], value) {
return
}
existing.Versions[i] = value
versionExists = true
break
}
}
if !versionExists {
existing.Versions = append(existing.Versions, value)
}
} else {
group := &apidiscoveryv2beta1.APIGroupDiscovery{
ObjectMeta: metav1.ObjectMeta{
Name: groupName,
},
Versions: []apidiscoveryv2beta1.APIVersionDiscovery{value},
}
rdm.apiGroups[groupName] = group
}
gv := metav1.GroupVersion{Group: groupName, Version: value.Version}
if _, ok := rdm.versionPriorities[gv]; !ok {
rdm.versionPriorities[gv] = priorityInfo{
GroupPriorityMinimum: 1000,
VersionPriority: 15,
}
}
// Reset response document so it is recreated lazily
rdm.cache.Store(nil)
}
func (rdm *resourceDiscoveryManager) RemoveGroupVersion(apiGroup metav1.GroupVersion) {
rdm.lock.Lock()
defer rdm.lock.Unlock()
group, exists := rdm.apiGroups[apiGroup.Group]
if !exists {
return
}
modified := false
for i := range group.Versions {
if group.Versions[i].Version == apiGroup.Version {
group.Versions = append(group.Versions[:i], group.Versions[i+1:]...)
modified = true
break
}
}
// If no modification was done, cache does not need to be cleared
if !modified {
return
}
delete(rdm.versionPriorities, apiGroup)
if len(group.Versions) == 0 {
delete(rdm.apiGroups, group.Name)
}
// Reset response document so it is recreated lazily
rdm.cache.Store(nil)
}
func (rdm *resourceDiscoveryManager) RemoveGroup(groupName string) {
rdm.lock.Lock()
defer rdm.lock.Unlock()
delete(rdm.apiGroups, groupName)
for k := range rdm.versionPriorities {
if k.Group == groupName {
delete(rdm.versionPriorities, k)
}
}
// Reset response document so it is recreated lazily
rdm.cache.Store(nil)
}
// Prepares the api group list for serving by converting them from map into
// list and sorting them according to insertion order
func (rdm *resourceDiscoveryManager) calculateAPIGroupsLocked() []apidiscoveryv2beta1.APIGroupDiscovery {
// Re-order the apiGroups by their priority.
groups := []apidiscoveryv2beta1.APIGroupDiscovery{}
for _, group := range rdm.apiGroups {
copied := *group.DeepCopy()
// Re-order versions based on their priority. Use kube-aware string
// comparison as a tie breaker
sort.SliceStable(copied.Versions, func(i, j int) bool {
iVersion := copied.Versions[i].Version
jVersion := copied.Versions[j].Version
iPriority := rdm.versionPriorities[metav1.GroupVersion{Group: group.Name, Version: iVersion}].VersionPriority
jPriority := rdm.versionPriorities[metav1.GroupVersion{Group: group.Name, Version: jVersion}].VersionPriority
// Sort by version string comparator if priority is equal
if iPriority == jPriority {
return version.CompareKubeAwareVersionStrings(iVersion, jVersion) > 0
}
// i sorts before j if it has a higher priority
return iPriority > jPriority
})
groups = append(groups, *copied.DeepCopy())
}
// For each group, determine the highest minimum group priority and use that
priorities := map[string]int{}
for gv, info := range rdm.versionPriorities {
if existing, exists := priorities[gv.Group]; exists {
if existing < info.GroupPriorityMinimum {
priorities[gv.Group] = info.GroupPriorityMinimum
}
} else {
priorities[gv.Group] = info.GroupPriorityMinimum
}
}
sort.SliceStable(groups, func(i, j int) bool {
iName := groups[i].Name
jName := groups[j].Name
// Default to 0 priority by default
iPriority := priorities[iName]
jPriority := priorities[jName]
// Sort discovery based on apiservice priority.
// Duplicated from staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helpers.go
if iPriority == jPriority {
// Equal priority uses name to break ties
return iName < jName
}
// i sorts before j if it has a higher priority
return iPriority > jPriority
})
return groups
}
// Fetches from cache if it exists. If cache is empty, create it.
func (rdm *resourceDiscoveryManager) fetchFromCache() *cachedGroupList {
rdm.lock.RLock()
defer rdm.lock.RUnlock()
cacheLoad := rdm.cache.Load()
if cacheLoad != nil {
return cacheLoad
}
response := apidiscoveryv2beta1.APIGroupDiscoveryList{
Items: rdm.calculateAPIGroupsLocked(),
}
etag, err := calculateETag(response)
if err != nil {
klog.Errorf("failed to calculate etag for discovery document: %s", etag)
etag = ""
}
cached := &cachedGroupList{
cachedResponse: response,
cachedResponseETag: etag,
}
rdm.cache.Store(cached)
return cached
}
type cachedGroupList struct {
cachedResponse apidiscoveryv2beta1.APIGroupDiscoveryList
cachedResponseETag string
}
func (rdm *resourceDiscoveryManager) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
cache := rdm.fetchFromCache()
response := cache.cachedResponse
etag := cache.cachedResponseETag
if len(etag) > 0 {
// Use proper e-tag headers if one is available
ServeHTTPWithETag(
&response,
etag,
rdm.serializer,
resp,
req,
)
} else {
// Default to normal response in rare case etag is
// not cached with the object for some reason.
responsewriters.WriteObjectNegotiated(
rdm.serializer,
DiscoveryEndpointRestrictions,
AggregatedDiscoveryGV,
resp,
req,
http.StatusOK,
&response,
true,
)
}
}

View File

@@ -0,0 +1,45 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aggregated
import (
"k8s.io/apimachinery/pkg/runtime/schema"
)
var AggregatedDiscoveryGV = schema.GroupVersion{Group: "apidiscovery.k8s.io", Version: "v2beta1"}
// Interface is from "k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
// DiscoveryEndpointRestrictions allows requests to /apis to provide a Content Negotiation GVK for aggregated discovery.
var DiscoveryEndpointRestrictions = discoveryEndpointRestrictions{}
type discoveryEndpointRestrictions struct{}
func (discoveryEndpointRestrictions) AllowsMediaTypeTransform(mimeType string, mimeSubType string, gvk *schema.GroupVersionKind) bool {
return IsAggregatedDiscoveryGVK(gvk)
}
func (discoveryEndpointRestrictions) AllowsServerVersion(string) bool { return false }
func (discoveryEndpointRestrictions) AllowsStreamSchema(s string) bool { return s == "watch" }
// IsAggregatedDiscoveryGVK checks if a provided GVK is the GVK for serving aggregated discovery.
func IsAggregatedDiscoveryGVK(gvk *schema.GroupVersionKind) bool {
if gvk != nil {
return gvk.Group == "apidiscovery.k8s.io" && gvk.Version == "v2beta1" && gvk.Kind == "APIGroupDiscoveryList"
}
return false
}

View File

@@ -0,0 +1,78 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package aggregated
import (
"net/http"
apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
"k8s.io/apimachinery/pkg/runtime/serializer"
"github.com/emicklei/go-restful/v3"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
genericfeatures "k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
)
type WrappedHandler struct {
s runtime.NegotiatedSerializer
handler http.Handler
aggHandler http.Handler
}
func (wrapped *WrappedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
mediaType, _ := negotiation.NegotiateMediaTypeOptions(req.Header.Get("Accept"), wrapped.s.SupportedMediaTypes(), DiscoveryEndpointRestrictions)
// mediaType.Convert looks at the request accept headers and is used to control whether the discovery document will be aggregated.
if IsAggregatedDiscoveryGVK(mediaType.Convert) {
wrapped.aggHandler.ServeHTTP(resp, req)
return
}
}
wrapped.handler.ServeHTTP(resp, req)
}
func (wrapped *WrappedHandler) restfulHandle(req *restful.Request, resp *restful.Response) {
wrapped.ServeHTTP(resp.ResponseWriter, req.Request)
}
func (wrapped *WrappedHandler) GenerateWebService(prefix string, returnType interface{}) *restful.WebService {
mediaTypes, _ := negotiation.MediaTypesForSerializer(wrapped.s)
ws := new(restful.WebService)
ws.Path(prefix)
ws.Doc("get available API versions")
ws.Route(ws.GET("/").To(wrapped.restfulHandle).
Doc("get available API versions").
Operation("getAPIVersions").
Produces(mediaTypes...).
Consumes(mediaTypes...).
Writes(returnType))
return ws
}
// WrapAggregatedDiscoveryToHandler wraps a handler with an option to
// emit the aggregated discovery by passing in the aggregated
// discovery type in content negotiation headers: eg: (Accept:
// application/json;v=v2beta1;g=apidiscovery.k8s.io;as=APIGroupDiscoveryList)
func WrapAggregatedDiscoveryToHandler(handler http.Handler, aggHandler http.Handler) *WrappedHandler {
scheme := runtime.NewScheme()
apidiscoveryv2beta1.AddToScheme(scheme)
codecs := serializer.NewCodecFactory(scheme)
return &WrappedHandler{codecs, handler, aggHandler}
}

View File

@@ -69,5 +69,5 @@ func (s *APIGroupHandler) handle(req *restful.Request, resp *restful.Response) {
}
func (s *APIGroupHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
responsewriters.WriteObjectNegotiated(s.serializer, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, w, req, http.StatusOK, &s.group)
responsewriters.WriteObjectNegotiated(s.serializer, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, w, req, http.StatusOK, &s.group, false)
}

View File

@@ -56,7 +56,7 @@ func (s *legacyRootAPIHandler) WebService() *restful.WebService {
ws := new(restful.WebService)
ws.Path(s.apiPrefix)
ws.Doc("get available API versions")
ws.Route(ws.GET("/").To(s.handle).
ws.Route(ws.GET("/").To(s.restfulHandle).
Doc("get available API versions").
Operation("getAPIVersions").
Produces(mediaTypes...).
@@ -65,12 +65,16 @@ func (s *legacyRootAPIHandler) WebService() *restful.WebService {
return ws
}
func (s *legacyRootAPIHandler) handle(req *restful.Request, resp *restful.Response) {
clientIP := utilnet.GetClientIP(req.Request)
func (s *legacyRootAPIHandler) restfulHandle(req *restful.Request, resp *restful.Response) {
s.ServeHTTP(resp.ResponseWriter, req.Request)
}
func (s *legacyRootAPIHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
clientIP := utilnet.GetClientIP(req)
apiVersions := &metav1.APIVersions{
ServerAddressByClientCIDRs: s.addresses.ServerAddressByClientCIDRs(clientIP),
Versions: []string{"v1"},
}
responsewriters.WriteObjectNegotiated(s.serializer, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, resp.ResponseWriter, req.Request, http.StatusOK, apiVersions)
responsewriters.WriteObjectNegotiated(s.serializer, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, resp, req, http.StatusOK, apiVersions, false)
}

View File

@@ -35,7 +35,7 @@ import (
type GroupManager interface {
AddGroup(apiGroup metav1.APIGroup)
RemoveGroup(groupName string)
ServeHTTP(resp http.ResponseWriter, req *http.Request)
WebService() *restful.WebService
}
@@ -111,7 +111,7 @@ func (s *rootAPIsHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request)
groups[i].ServerAddressByClientCIDRs = serverCIDR
}
responsewriters.WriteObjectNegotiated(s.serializer, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, resp, req, http.StatusOK, &metav1.APIGroupList{Groups: groups})
responsewriters.WriteObjectNegotiated(s.serializer, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, resp, req, http.StatusOK, &metav1.APIGroupList{Groups: groups}, false)
}
func (s *rootAPIsHandler) restfulHandle(req *restful.Request, resp *restful.Response) {

View File

@@ -79,5 +79,5 @@ func (s *APIVersionHandler) handle(req *restful.Request, resp *restful.Response)
func (s *APIVersionHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
responsewriters.WriteObjectNegotiated(s.serializer, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, w, req, http.StatusOK,
&metav1.APIResourceList{GroupVersion: s.groupVersion.String(), APIResources: s.apiResourceLister.ListAPIResources()})
&metav1.APIResourceList{GroupVersion: s.groupVersion.String(), APIResources: s.apiResourceLister.ListAPIResources()}, false)
}

View File

@@ -22,6 +22,8 @@ import (
"net/http"
"time"
"go.opentelemetry.io/otel/trace"
"k8s.io/apiserver/pkg/endpoints/metrics"
apirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/server/httplog"
@@ -54,8 +56,8 @@ func requestFilterRecordFrom(ctx context.Context) *requestFilterRecord {
// TrackStarted measures the timestamp the given handler has started execution
// by attaching a handler to the chain.
func TrackStarted(handler http.Handler, name string) http.Handler {
return trackStarted(handler, name, clock.RealClock{})
func TrackStarted(handler http.Handler, tp trace.TracerProvider, name string) http.Handler {
return trackStarted(handler, tp, name, clock.RealClock{})
}
// TrackCompleted measures the timestamp the given handler has completed execution and then
@@ -70,7 +72,9 @@ func TrackCompleted(handler http.Handler) http.Handler {
})
}
func trackStarted(handler http.Handler, name string, clock clock.PassiveClock) http.Handler {
func trackStarted(handler http.Handler, tp trace.TracerProvider, name string, clock clock.PassiveClock) http.Handler {
// This is a noop if the tracing is disabled, since tp will be a NoopTracerProvider
tracer := tp.Tracer("k8s.op/apiserver/pkg/endpoints/filterlatency")
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
if fr := requestFilterRecordFrom(ctx); fr != nil {
@@ -85,6 +89,7 @@ func trackStarted(handler http.Handler, name string, clock clock.PassiveClock) h
name: name,
startedTimestamp: clock.Now(),
}
ctx, _ = tracer.Start(ctx, name)
r = r.WithContext(withRequestFilterRecord(ctx, fr))
handler.ServeHTTP(w, r)
})
@@ -101,5 +106,6 @@ func trackCompleted(handler http.Handler, clock clock.PassiveClock, action func(
if fr := requestFilterRecordFrom(ctx); fr != nil {
action(ctx, fr, completedAt)
}
trace.SpanFromContext(ctx).End()
})
}

View File

@@ -44,23 +44,21 @@ func WithAudit(handler http.Handler, sink audit.Sink, policy audit.PolicyRuleEva
return handler
}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
auditContext, err := evaluatePolicyAndCreateAuditEvent(req, policy)
ac, err := evaluatePolicyAndCreateAuditEvent(req, policy)
if err != nil {
utilruntime.HandleError(fmt.Errorf("failed to create audit event: %v", err))
responsewriters.InternalError(w, req, errors.New("failed to create audit event"))
return
}
ev := auditContext.Event
if ev == nil || req.Context() == nil {
if ac == nil || ac.Event == nil {
handler.ServeHTTP(w, req)
return
}
req = req.WithContext(audit.WithAuditContext(req.Context(), auditContext))
ev := ac.Event
ctx := req.Context()
omitStages := auditContext.RequestAuditConfig.OmitStages
omitStages := ac.RequestAuditConfig.OmitStages
ev.Stage = auditinternal.StageRequestReceived
if processed := processAuditEvent(ctx, sink, ev, omitStages); !processed {
@@ -124,19 +122,23 @@ func WithAudit(handler http.Handler, sink audit.Sink, policy audit.PolicyRuleEva
// - error if anything bad happened
func evaluatePolicyAndCreateAuditEvent(req *http.Request, policy audit.PolicyRuleEvaluator) (*audit.AuditContext, error) {
ctx := req.Context()
ac := audit.AuditContextFrom(ctx)
if ac == nil {
// Auditing not enabled.
return nil, nil
}
attribs, err := GetAuthorizerAttributes(ctx)
if err != nil {
return nil, fmt.Errorf("failed to GetAuthorizerAttributes: %v", err)
return ac, fmt.Errorf("failed to GetAuthorizerAttributes: %v", err)
}
ls := policy.EvaluatePolicyRule(attribs)
audit.ObservePolicyLevel(ctx, ls.Level)
ac.RequestAuditConfig = ls.RequestAuditConfig
if ls.Level == auditinternal.LevelNone {
// Don't audit.
return &audit.AuditContext{
RequestAuditConfig: ls.RequestAuditConfig,
}, nil
return ac, nil
}
requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(ctx)
@@ -148,10 +150,9 @@ func evaluatePolicyAndCreateAuditEvent(req *http.Request, policy audit.PolicyRul
return nil, fmt.Errorf("failed to complete audit event from request: %v", err)
}
return &audit.AuditContext{
RequestAuditConfig: ls.RequestAuditConfig,
Event: ev,
}, nil
ac.Event = ev
return ac, nil
}
// writeLatencyToAnnotation writes the latency incurred in different

View File

@@ -1,38 +0,0 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filters
import (
"net/http"
"k8s.io/apiserver/pkg/audit"
)
// WithAuditAnnotations decorates a http.Handler with a []{key, value} that is merged
// with the audit.Event.Annotations map. This allows layers that run before WithAudit
// (such as authentication) to assert annotations.
// If sink or audit policy is nil, no decoration takes place.
func WithAuditAnnotations(handler http.Handler, sink audit.Sink, policy audit.PolicyRuleEvaluator) http.Handler {
// no need to wrap if auditing is disabled
if sink == nil || policy == nil {
return handler
}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
req = req.WithContext(audit.WithAuditAnnotations(req.Context()))
handler.ServeHTTP(w, req)
})
}

View File

@@ -21,28 +21,25 @@ import (
"k8s.io/apimachinery/pkg/types"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/audit"
"github.com/google/uuid"
)
// WithAuditID attaches the Audit-ID associated with a request to the context.
// WithAuditInit initializes the audit context and attaches the Audit-ID associated with a request.
//
// a. If the caller does not specify a value for Audit-ID in the request header, we generate a new audit ID
// b. We echo the Audit-ID value to the caller via the response Header 'Audit-ID'.
func WithAuditID(handler http.Handler) http.Handler {
return withAuditID(handler, func() string {
func WithAuditInit(handler http.Handler) http.Handler {
return withAuditInit(handler, func() string {
return uuid.New().String()
})
}
func withAuditID(handler http.Handler, newAuditIDFunc func() string) http.Handler {
if newAuditIDFunc == nil {
return handler
}
func withAuditInit(handler http.Handler, newAuditIDFunc func() string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ctx := audit.WithAuditContext(r.Context())
r = r.WithContext(ctx)
auditID := r.Header.Get(auditinternal.HeaderAuditID)
if len(auditID) == 0 {
@@ -50,7 +47,7 @@ func withAuditID(handler http.Handler, newAuditIDFunc func() string) http.Handle
}
// Note: we save the user specified value of the Audit-ID header as is, no truncation is performed.
r = r.WithContext(request.WithAuditID(ctx, types.UID(auditID)))
audit.WithAuditID(ctx, types.UID(auditID))
// We echo the Audit-ID in to the response header.
// It's not guaranteed Audit-ID http header is sent for all requests.

View File

@@ -36,26 +36,24 @@ func WithFailedAuthenticationAudit(failedHandler http.Handler, sink audit.Sink,
return failedHandler
}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
a, err := evaluatePolicyAndCreateAuditEvent(req, policy)
ac, err := evaluatePolicyAndCreateAuditEvent(req, policy)
if err != nil {
utilruntime.HandleError(fmt.Errorf("failed to create audit event: %v", err))
responsewriters.InternalError(w, req, errors.New("failed to create audit event"))
return
}
ev := a.Event
if ev == nil {
if ac == nil || ac.Event == nil {
failedHandler.ServeHTTP(w, req)
return
}
req = req.WithContext(audit.WithAuditContext(req.Context(), a))
ev := ac.Event
ev.ResponseStatus = &metav1.Status{}
ev.ResponseStatus.Message = getAuthMethods(req)
ev.Stage = auditinternal.StageResponseStarted
rw := decorateResponseWriter(req.Context(), w, ev, sink, a.RequestAuditConfig.OmitStages)
rw := decorateResponseWriter(req.Context(), w, ev, sink, ac.RequestAuditConfig.OmitStages)
failedHandler.ServeHTTP(rw, req)
})
}

View File

@@ -108,20 +108,18 @@ func withFailedRequestAudit(failedHandler http.Handler, statusErr *apierrors.Sta
return failedHandler
}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
a, err := evaluatePolicyAndCreateAuditEvent(req, policy)
ac, err := evaluatePolicyAndCreateAuditEvent(req, policy)
if err != nil {
utilruntime.HandleError(fmt.Errorf("failed to create audit event: %v", err))
responsewriters.InternalError(w, req, errors.New("failed to create audit event"))
return
}
ev := a.Event
if ev == nil {
if ac == nil || ac.Event == nil {
failedHandler.ServeHTTP(w, req)
return
}
req = req.WithContext(audit.WithAuditContext(req.Context(), a))
ev := ac.Event
ev.ResponseStatus = &metav1.Status{}
ev.Stage = auditinternal.StageResponseStarted
@@ -129,7 +127,7 @@ func withFailedRequestAudit(failedHandler http.Handler, statusErr *apierrors.Sta
ev.ResponseStatus.Message = statusErr.Error()
}
rw := decorateResponseWriter(req.Context(), w, ev, sink, a.RequestAuditConfig.OmitStages)
rw := decorateResponseWriter(req.Context(), w, ev, sink, ac.RequestAuditConfig.OmitStages)
failedHandler.ServeHTTP(rw, req)
})
}

View File

@@ -22,6 +22,7 @@ import (
restful "github.com/emicklei/go-restful/v3"
apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -105,7 +106,7 @@ type APIGroupVersion struct {
// InstallREST registers the REST handlers (storage, watch, proxy and redirect) into a restful Container.
// It is expected that the provided path root prefix will serve all operations. Root MUST NOT end
// in a slash.
func (g *APIGroupVersion) InstallREST(container *restful.Container) ([]*storageversion.ResourceInfo, error) {
func (g *APIGroupVersion) InstallREST(container *restful.Container) ([]apidiscoveryv2beta1.APIResourceDiscovery, []*storageversion.ResourceInfo, error) {
prefix := path.Join(g.Root, g.GroupVersion.Group, g.GroupVersion.Version)
installer := &APIInstaller{
group: g,
@@ -117,7 +118,11 @@ func (g *APIGroupVersion) InstallREST(container *restful.Container) ([]*storagev
versionDiscoveryHandler := discovery.NewAPIVersionHandler(g.Serializer, g.GroupVersion, staticLister{apiResources})
versionDiscoveryHandler.AddToWebService(ws)
container.Add(ws)
return removeNonPersistedResources(resourceInfos), utilerrors.NewAggregate(registrationErrors)
aggregatedDiscoveryResources, err := ConvertGroupVersionIntoToDiscovery(apiResources)
if err != nil {
registrationErrors = append(registrationErrors, err)
}
return aggregatedDiscoveryResources, removeNonPersistedResources(resourceInfos), utilerrors.NewAggregate(registrationErrors)
}
func removeNonPersistedResources(infos []*storageversion.ResourceInfo) []*storageversion.ResourceInfo {

View File

@@ -26,6 +26,8 @@ import (
"unicode"
"unicode/utf8"
"go.opentelemetry.io/otel/attribute"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme"
@@ -37,28 +39,23 @@ import (
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/util/dryrun"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/tracing"
"k8s.io/klog/v2"
utiltrace "k8s.io/utils/trace"
)
var namespaceGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "namespaces"}
func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Interface, includeName bool) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
ctx := req.Context()
// For performance tracking purposes.
trace := utiltrace.New("Create", traceFields(req)...)
defer trace.LogIfLong(500 * time.Millisecond)
if isDryRun(req.URL) && !utilfeature.DefaultFeatureGate.Enabled(features.DryRun) {
scope.err(errors.NewBadRequest("the dryRun feature is disabled"), w, req)
return
}
ctx, span := tracing.Start(ctx, "Create", traceFields(req)...)
defer span.End(500 * time.Millisecond)
namespace, name, err := scope.Namer.Name(req)
if err != nil {
@@ -78,7 +75,7 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
// enforce a timeout of at most requestTimeoutUpperBound (34s) or less if the user-provided
// timeout inside the parent context is lower than requestTimeoutUpperBound.
ctx, cancel := context.WithTimeout(req.Context(), requestTimeoutUpperBound)
ctx, cancel := context.WithTimeout(ctx, requestTimeoutUpperBound)
defer cancel()
outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope)
if err != nil {
@@ -93,12 +90,13 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
return
}
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
trace.Step("limitedReadBody done", utiltrace.Field{"len", len(body)}, utiltrace.Field{"err", err})
body, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.Create)
if err != nil {
span.AddEvent("limitedReadBody failed", attribute.Int("len", len(body)), attribute.String("err", err.Error()))
scope.err(err, w, req)
return
}
span.AddEvent("limitedReadBody succeeded", attribute.Int("len", len(body)))
options := &metav1.CreateOptions{}
values := req.URL.Query()
@@ -124,7 +122,7 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
}
decoder := scope.Serializer.DecoderToVersion(decodeSerializer, scope.HubGroupVersion)
trace.Step("About to convert to expected version")
span.AddEvent("About to convert to expected version")
obj, gvk, err := decoder.Decode(body, &defaultGVK, original)
if err != nil {
strictError, isStrictError := runtime.AsStrictDecodingError(err)
@@ -147,7 +145,7 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
scope.err(err, w, req)
return
}
trace.Step("Conversion done")
span.AddEvent("Conversion done")
// On create, get name from new object if unset
if len(name) == 0 {
@@ -174,7 +172,7 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
}
}
trace.Step("About to store object in database")
span.AddEvent("About to store object in database")
admissionAttributes := admission.NewAttributesRecord(obj, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Create, options, dryrun.IsDryRun(options.DryRun), userInfo)
requestFunc := func() (runtime.Object, error) {
return r.Create(
@@ -214,11 +212,12 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
}
return result, err
})
trace.Step("Write to database call finished", utiltrace.Field{"len", len(body)}, utiltrace.Field{"err", err})
if err != nil {
span.AddEvent("Write to database call failed", attribute.Int("len", len(body)), attribute.String("err", err.Error()))
scope.err(err, w, req)
return
}
span.AddEvent("Write to database call succeeded", attribute.Int("len", len(body)))
code := http.StatusCreated
status, ok := result.(*metav1.Status)
@@ -226,9 +225,9 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
status.Code = int32(code)
}
trace.Step("About to write a response")
defer trace.Step("Writing http response done")
transformResponseObject(ctx, scope, trace, req, w, code, outputMediaType, result)
span.AddEvent("About to write a response")
defer span.AddEvent("Writing http response done")
transformResponseObject(ctx, scope, req, w, code, outputMediaType, result)
}
}

View File

@@ -22,6 +22,8 @@ import (
"net/http"
"time"
"go.opentelemetry.io/otel/attribute"
"k8s.io/apimachinery/pkg/api/errors"
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme"
@@ -33,27 +35,22 @@ import (
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/util/dryrun"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utiltrace "k8s.io/utils/trace"
"k8s.io/component-base/tracing"
)
// DeleteResource returns a function that will handle a resource deletion
// TODO admission here becomes solely validating admission
func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestScope, admit admission.Interface) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
ctx := req.Context()
// For performance tracking purposes.
trace := utiltrace.New("Delete", traceFields(req)...)
defer trace.LogIfLong(500 * time.Millisecond)
if isDryRun(req.URL) && !utilfeature.DefaultFeatureGate.Enabled(features.DryRun) {
scope.err(errors.NewBadRequest("the dryRun feature is disabled"), w, req)
return
}
ctx, span := tracing.Start(ctx, "Delete", traceFields(req)...)
defer span.End(500 * time.Millisecond)
namespace, name, err := scope.Namer.Name(req)
if err != nil {
@@ -63,7 +60,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
// enforce a timeout of at most requestTimeoutUpperBound (34s) or less if the user-provided
// timeout inside the parent context is lower than requestTimeoutUpperBound.
ctx, cancel := context.WithTimeout(req.Context(), requestTimeoutUpperBound)
ctx, cancel := context.WithTimeout(ctx, requestTimeoutUpperBound)
defer cancel()
ctx = request.WithNamespace(ctx, namespace)
@@ -77,11 +74,13 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
options := &metav1.DeleteOptions{}
if allowsOptions {
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
body, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.Delete)
if err != nil {
span.AddEvent("limitedReadBody failed", attribute.Int("len", len(body)), attribute.String("err", err.Error()))
scope.err(err, w, req)
return
}
span.AddEvent("limitedReadBody succeeded", attribute.Int("len", len(body)))
if len(body) > 0 {
s, err := negotiation.NegotiateInputSerializer(req, false, metainternalversionscheme.Codecs)
if err != nil {
@@ -100,11 +99,11 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
scope.err(fmt.Errorf("decoded object cannot be converted to DeleteOptions"), w, req)
return
}
trace.Step("Decoded delete options")
span.AddEvent("Decoded delete options")
objGV := gvk.GroupVersion()
audit.LogRequestObject(req.Context(), obj, objGV, scope.Resource, scope.Subresource, metainternalversionscheme.Codecs)
trace.Step("Recorded the audit event")
span.AddEvent("Recorded the audit event")
} else {
if err := metainternalversionscheme.ParameterCodec.DecodeParameters(req.URL.Query(), scope.MetaGroupVersion, options); err != nil {
err = errors.NewBadRequest(err.Error())
@@ -120,7 +119,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
}
options.TypeMeta.SetGroupVersionKind(metav1.SchemeGroupVersion.WithKind("DeleteOptions"))
trace.Step("About to delete object from database")
span.AddEvent("About to delete object from database")
wasDeleted := true
userInfo, _ := request.UserFrom(ctx)
staticAdmissionAttrs := admission.NewAttributesRecord(nil, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Delete, options, dryrun.IsDryRun(options.DryRun), userInfo)
@@ -133,7 +132,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
scope.err(err, w, req)
return
}
trace.Step("Object deleted from database")
span.AddEvent("Object deleted from database")
status := http.StatusOK
// Return http.StatusAccepted if the resource was not deleted immediately and
@@ -160,22 +159,18 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
}
}
trace.Step("About to write a response")
defer trace.Step("Writing http response done")
transformResponseObject(ctx, scope, trace, req, w, status, outputMediaType, result)
span.AddEvent("About to write a response")
defer span.AddEvent("Writing http response done")
transformResponseObject(ctx, scope, req, w, status, outputMediaType, result)
}
}
// DeleteCollection returns a function that will handle a collection deletion
func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestScope, admit admission.Interface) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
trace := utiltrace.New("Delete", traceFields(req)...)
defer trace.LogIfLong(500 * time.Millisecond)
if isDryRun(req.URL) && !utilfeature.DefaultFeatureGate.Enabled(features.DryRun) {
scope.err(errors.NewBadRequest("the dryRun feature is disabled"), w, req)
return
}
ctx := req.Context()
ctx, span := tracing.Start(ctx, "Delete", traceFields(req)...)
defer span.End(500 * time.Millisecond)
namespace, err := scope.Namer.Namespace(req)
if err != nil {
@@ -185,7 +180,7 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc
// enforce a timeout of at most requestTimeoutUpperBound (34s) or less if the user-provided
// timeout inside the parent context is lower than requestTimeoutUpperBound.
ctx, cancel := context.WithTimeout(req.Context(), requestTimeoutUpperBound)
ctx, cancel := context.WithTimeout(ctx, requestTimeoutUpperBound)
defer cancel()
ctx = request.WithNamespace(ctx, namespace)
@@ -225,13 +220,15 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc
options := &metav1.DeleteOptions{}
if checkBody {
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
body, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.DeleteCollection)
if err != nil {
span.AddEvent("limitedReadBody failed", attribute.Int("len", len(body)), attribute.String("err", err.Error()))
scope.err(err, w, req)
return
}
span.AddEvent("limitedReadBody succeeded", attribute.Int("len", len(body)))
if len(body) > 0 {
s, err := negotiation.NegotiateInputSerializer(req, false, scope.Serializer)
s, err := negotiation.NegotiateInputSerializer(req, false, metainternalversionscheme.Codecs)
if err != nil {
scope.err(err, w, req)
return
@@ -289,8 +286,8 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc
}
}
trace.Step("About to write a response")
defer trace.Step("Writing http response done")
transformResponseObject(ctx, scope, trace, req, w, http.StatusOK, outputMediaType, result)
span.AddEvent("About to write a response")
defer span.AddEvent("Writing http response done")
transformResponseObject(ctx, scope, req, w, http.StatusOK, outputMediaType, result)
}
}

View File

@@ -22,6 +22,7 @@ import (
"os"
"reflect"
"strconv"
"sync"
"time"
"k8s.io/apimachinery/pkg/api/equality"
@@ -33,42 +34,41 @@ import (
"k8s.io/klog/v2"
)
func determineAvoidNoopTimestampUpdatesEnabled() bool {
if avoidNoopTimestampUpdatesString, exists := os.LookupEnv("KUBE_APISERVER_AVOID_NOOP_SSA_TIMESTAMP_UPDATES"); exists {
if ret, err := strconv.ParseBool(avoidNoopTimestampUpdatesString); err == nil {
return ret
} else {
klog.Errorf("failed to parse envar KUBE_APISERVER_AVOID_NOOP_SSA_TIMESTAMP_UPDATES: %v", err)
}
}
// enabled by default
return true
}
var (
avoidNoopTimestampUpdatesEnabled = determineAvoidNoopTimestampUpdatesEnabled()
avoidTimestampEqualities conversion.Equalities
initAvoidTimestampEqualities sync.Once
)
var avoidTimestampEqualities = func() conversion.Equalities {
var eqs = equality.Semantic.Copy()
func getAvoidTimestampEqualities() conversion.Equalities {
initAvoidTimestampEqualities.Do(func() {
if avoidNoopTimestampUpdatesString, exists := os.LookupEnv("KUBE_APISERVER_AVOID_NOOP_SSA_TIMESTAMP_UPDATES"); exists {
if ret, err := strconv.ParseBool(avoidNoopTimestampUpdatesString); err == nil && !ret {
// leave avoidTimestampEqualities empty.
return
} else {
klog.Errorf("failed to parse envar KUBE_APISERVER_AVOID_NOOP_SSA_TIMESTAMP_UPDATES: %v", err)
}
}
err := eqs.AddFunc(
func(a, b metav1.ManagedFieldsEntry) bool {
// Two objects' managed fields are equivalent if, ignoring timestamp,
// the objects are deeply equal.
a.Time = nil
b.Time = nil
return reflect.DeepEqual(a, b)
},
)
var eqs = equality.Semantic.Copy()
err := eqs.AddFunc(
func(a, b metav1.ManagedFieldsEntry) bool {
// Two objects' managed fields are equivalent if, ignoring timestamp,
// the objects are deeply equal.
a.Time = nil
b.Time = nil
return reflect.DeepEqual(a, b)
},
)
if err != nil {
panic(err)
}
if err != nil {
panic(fmt.Errorf("failed to instantiate semantic equalities: %w", err))
}
return eqs
}()
avoidTimestampEqualities = eqs
})
return avoidTimestampEqualities
}
// IgnoreManagedFieldsTimestampsTransformer reverts timestamp updates
// if the non-managed parts of the object are equivalent
@@ -77,7 +77,8 @@ func IgnoreManagedFieldsTimestampsTransformer(
newObj runtime.Object,
oldObj runtime.Object,
) (res runtime.Object, err error) {
if !avoidNoopTimestampUpdatesEnabled {
equalities := getAvoidTimestampEqualities()
if len(equalities.Equalities) == 0 {
return newObj, nil
}
@@ -154,11 +155,11 @@ func IgnoreManagedFieldsTimestampsTransformer(
// This condition ensures the managed fields are always compared first. If
// this check fails, the if statement will short circuit. If the check
// succeeds the slow path is taken which compares entire objects.
if !avoidTimestampEqualities.DeepEqualWithNilDifferentFromEmpty(oldManagedFields, newManagedFields) {
if !equalities.DeepEqualWithNilDifferentFromEmpty(oldManagedFields, newManagedFields) {
return newObj, nil
}
if avoidTimestampEqualities.DeepEqualWithNilDifferentFromEmpty(newObj, oldObj) {
if equalities.DeepEqualWithNilDifferentFromEmpty(newObj, oldObj) {
// Remove any changed timestamps, so that timestamp is not the only
// change seen by etcd.
//

View File

@@ -202,8 +202,7 @@ func (f *FieldManager) UpdateNoErrors(liveObj, newObj runtime.Object, manager st
if err != nil {
atMostEverySecond.Do(func() {
ns, name := "unknown", "unknown"
accessor, err := meta.Accessor(newObj)
if err == nil {
if accessor, err := meta.Accessor(newObj); err == nil {
ns = accessor.GetNamespace()
name = accessor.GetName()
}

View File

@@ -25,42 +25,42 @@ import (
"strings"
"time"
metainternalversionvalidation "k8s.io/apimachinery/pkg/apis/meta/internalversion/validation"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/klog/v2"
"go.opentelemetry.io/otel/attribute"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme"
metainternalversionvalidation "k8s.io/apimachinery/pkg/apis/meta/internalversion/validation"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
"k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/rest"
utiltrace "k8s.io/utils/trace"
"k8s.io/component-base/tracing"
"k8s.io/klog/v2"
)
// getterFunc performs a get request with the given context and object name. The request
// may be used to deserialize an options object to pass to the getter.
type getterFunc func(ctx context.Context, name string, req *http.Request, trace *utiltrace.Trace) (runtime.Object, error)
type getterFunc func(ctx context.Context, name string, req *http.Request) (runtime.Object, error)
// getResourceHandler is an HTTP handler function for get requests. It delegates to the
// passed-in getterFunc to perform the actual get.
func getResourceHandler(scope *RequestScope, getter getterFunc) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
trace := utiltrace.New("Get", traceFields(req)...)
defer trace.LogIfLong(500 * time.Millisecond)
ctx := req.Context()
ctx, span := tracing.Start(ctx, "Get", traceFields(req)...)
defer span.End(500 * time.Millisecond)
namespace, name, err := scope.Namer.Name(req)
if err != nil {
scope.err(err, w, req)
return
}
ctx := req.Context()
ctx = request.WithNamespace(ctx, namespace)
outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope)
@@ -69,22 +69,22 @@ func getResourceHandler(scope *RequestScope, getter getterFunc) http.HandlerFunc
return
}
result, err := getter(ctx, name, req, trace)
result, err := getter(ctx, name, req)
if err != nil {
scope.err(err, w, req)
return
}
trace.Step("About to write a response")
defer trace.Step("Writing http response done")
transformResponseObject(ctx, scope, trace, req, w, http.StatusOK, outputMediaType, result)
span.AddEvent("About to write a response")
defer span.AddEvent("Writing http response done")
transformResponseObject(ctx, scope, req, w, http.StatusOK, outputMediaType, result)
}
}
// GetResource returns a function that handles retrieving a single resource from a rest.Storage object.
func GetResource(r rest.Getter, scope *RequestScope) http.HandlerFunc {
return getResourceHandler(scope,
func(ctx context.Context, name string, req *http.Request, trace *utiltrace.Trace) (runtime.Object, error) {
func(ctx context.Context, name string, req *http.Request) (runtime.Object, error) {
// check for export
options := metav1.GetOptions{}
if values := req.URL.Query(); len(values) > 0 {
@@ -104,9 +104,7 @@ func GetResource(r rest.Getter, scope *RequestScope) http.HandlerFunc {
return nil, err
}
}
if trace != nil {
trace.Step("About to Get from storage")
}
tracing.SpanFromContext(ctx).AddEvent("About to Get from storage")
return r.Get(ctx, name, &options)
})
}
@@ -114,16 +112,15 @@ func GetResource(r rest.Getter, scope *RequestScope) http.HandlerFunc {
// GetResourceWithOptions returns a function that handles retrieving a single resource from a rest.Storage object.
func GetResourceWithOptions(r rest.GetterWithOptions, scope *RequestScope, isSubresource bool) http.HandlerFunc {
return getResourceHandler(scope,
func(ctx context.Context, name string, req *http.Request, trace *utiltrace.Trace) (runtime.Object, error) {
func(ctx context.Context, name string, req *http.Request) (runtime.Object, error) {
opts, subpath, subpathKey := r.NewGetOptions()
trace.Step("About to process Get options")
span := tracing.SpanFromContext(ctx)
span.AddEvent("About to process Get options")
if err := getRequestOptions(req, scope, opts, subpath, subpathKey, isSubresource); err != nil {
err = errors.NewBadRequest(err.Error())
return nil, err
}
if trace != nil {
trace.Step("About to Get from storage")
}
span.AddEvent("About to Get from storage")
return r.Get(ctx, name, opts)
})
}
@@ -168,8 +165,9 @@ func getRequestOptions(req *http.Request, scope *RequestScope, into runtime.Obje
func ListResource(r rest.Lister, rw rest.Watcher, scope *RequestScope, forceWatch bool, minRequestTimeout time.Duration) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
ctx := req.Context()
// For performance tracking purposes.
trace := utiltrace.New("List", traceFields(req)...)
ctx, span := tracing.Start(ctx, "List", traceFields(req)...)
namespace, err := scope.Namer.Namespace(req)
if err != nil {
@@ -185,7 +183,6 @@ func ListResource(r rest.Lister, rw rest.Watcher, scope *RequestScope, forceWatc
hasName = false
}
ctx := req.Context()
ctx = request.WithNamespace(ctx, namespace)
outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope)
@@ -273,15 +270,15 @@ func ListResource(r rest.Lister, rw rest.Watcher, scope *RequestScope, forceWatc
}
// Log only long List requests (ignore Watch).
defer trace.LogIfLong(500 * time.Millisecond)
trace.Step("About to List from storage")
defer span.End(500 * time.Millisecond)
span.AddEvent("About to List from storage")
result, err := r.List(ctx, &opts)
if err != nil {
scope.err(err, w, req)
return
}
trace.Step("Listing from storage done")
defer trace.Step("Writing http response done", utiltrace.Field{"count", meta.LenList(result)})
transformResponseObject(ctx, scope, trace, req, w, http.StatusOK, outputMediaType, result)
span.AddEvent("Listing from storage done")
defer span.AddEvent("Writing http response done", attribute.Int("count", meta.LenList(result)))
transformResponseObject(ctx, scope, req, w, http.StatusOK, outputMediaType, result)
}
}

View File

@@ -20,7 +20,9 @@ import (
"net/http"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/endpoints/metrics"
apirequest "k8s.io/apiserver/pkg/endpoints/request"
)
const (
@@ -83,7 +85,56 @@ type lazyAuditID struct {
func (lazy *lazyAuditID) String() string {
if lazy.req != nil {
return request.GetAuditIDTruncated(lazy.req.Context())
return audit.GetAuditIDTruncated(lazy.req.Context())
}
return "unknown"
}
// lazyVerb implements String() string and it will
// lazily get normalized Verb
type lazyVerb struct {
req *http.Request
}
func (lazy *lazyVerb) String() string {
if lazy.req == nil {
return "unknown"
}
return metrics.NormalizedVerb(lazy.req)
}
// lazyResource implements String() string and it will
// lazily get Resource from request info
type lazyResource struct {
req *http.Request
}
func (lazy *lazyResource) String() string {
if lazy.req != nil {
ctx := lazy.req.Context()
requestInfo, ok := apirequest.RequestInfoFrom(ctx)
if ok {
return requestInfo.Resource
}
}
return "unknown"
}
// lazyScope implements String() string and it will
// lazily get Scope from request info
type lazyScope struct {
req *http.Request
}
func (lazy *lazyScope) String() string {
if lazy.req != nil {
ctx := lazy.req.Context()
requestInfo, ok := apirequest.RequestInfoFrom(ctx)
if ok {
return metrics.CleanScope(requestInfo)
}
}
return "unknown"

View File

@@ -0,0 +1,4 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- logicalhan

View File

@@ -0,0 +1,51 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package metrics
import (
"context"
"k8s.io/component-base/metrics"
)
type RequestBodyVerb string
const (
Patch RequestBodyVerb = "patch"
Delete RequestBodyVerb = "delete"
Update RequestBodyVerb = "update"
Create RequestBodyVerb = "create"
DeleteCollection RequestBodyVerb = "delete_collection"
)
var (
RequestBodySizes = metrics.NewHistogramVec(
&metrics.HistogramOpts{
Subsystem: "apiserver",
Name: "request_body_sizes",
Help: "Apiserver request body sizes broken out by size.",
// we use 0.05 KB as the smallest bucket with 0.1 KB increments up to the
// apiserver limit.
Buckets: metrics.LinearBuckets(50000, 100000, 31),
StabilityLevel: metrics.ALPHA,
},
[]string{"resource", "verb"},
)
)
func RecordRequestBodySize(ctx context.Context, resource string, verb RequestBodyVerb, size int) {
RequestBodySizes.WithContext(ctx).WithLabelValues(resource, string(verb)).Observe(float64(size))
}

View File

@@ -23,9 +23,10 @@ import (
"strings"
"time"
jsonpatch "github.com/evanphx/json-patch"
"go.opentelemetry.io/otel/attribute"
kjson "sigs.k8s.io/json"
jsonpatch "github.com/evanphx/json-patch"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme"
@@ -45,13 +46,12 @@ import (
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/util/dryrun"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utiltrace "k8s.io/utils/trace"
"k8s.io/component-base/tracing"
)
const (
@@ -62,14 +62,10 @@ const (
// PatchResource returns a function that will handle a resource patch.
func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interface, patchTypes []string) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
ctx := req.Context()
// For performance tracking purposes.
trace := utiltrace.New("Patch", traceFields(req)...)
defer trace.LogIfLong(500 * time.Millisecond)
if isDryRun(req.URL) && !utilfeature.DefaultFeatureGate.Enabled(features.DryRun) {
scope.err(errors.NewBadRequest("the dryRun feature is disabled"), w, req)
return
}
ctx, span := tracing.Start(ctx, "Patch", traceFields(req)...)
defer span.End(500 * time.Millisecond)
// Do this first, otherwise name extraction can fail for unrecognized content types
// TODO: handle this in negotiation
@@ -94,7 +90,7 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac
// enforce a timeout of at most requestTimeoutUpperBound (34s) or less if the user-provided
// timeout inside the parent context is lower than requestTimeoutUpperBound.
ctx, cancel := context.WithTimeout(req.Context(), requestTimeoutUpperBound)
ctx, cancel := context.WithTimeout(ctx, requestTimeoutUpperBound)
defer cancel()
ctx = request.WithNamespace(ctx, namespace)
@@ -105,12 +101,13 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac
return
}
patchBytes, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
trace.Step("limitedReadBody done", utiltrace.Field{"len", len(patchBytes)}, utiltrace.Field{"err", err})
patchBytes, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.Patch)
if err != nil {
span.AddEvent("limitedReadBody failed", attribute.Int("len", len(patchBytes)), attribute.String("err", err.Error()))
scope.err(err, w, req)
return
}
span.AddEvent("limitedReadBody succeeded", attribute.Int("len", len(patchBytes)))
options := &metav1.PatchOptions{}
if err := metainternalversionscheme.ParameterCodec.DecodeParameters(req.URL.Query(), scope.MetaGroupVersion, options); err != nil {
@@ -128,7 +125,7 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac
admit = admission.WithAudit(admit)
audit.LogRequestPatch(req.Context(), patchBytes)
trace.Step("Recorded the audit event")
span.AddEvent("Recorded the audit event")
baseContentType := runtime.ContentTypeJSON
if patchType == types.ApplyPatchType {
@@ -225,8 +222,6 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac
patchType: patchType,
patchBytes: patchBytes,
userAgent: req.UserAgent(),
trace: trace,
}
result, wasCreated, err := p.patchResource(ctx, scope)
@@ -234,16 +229,16 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac
scope.err(err, w, req)
return
}
trace.Step("Object stored in database")
span.AddEvent("Object stored in database")
status := http.StatusOK
if wasCreated {
status = http.StatusCreated
}
trace.Step("About to write a response")
defer trace.Step("Writing http response done")
transformResponseObject(ctx, scope, trace, req, w, status, outputMediaType, result)
span.AddEvent("About to write a response")
defer span.AddEvent("Writing http response done")
transformResponseObject(ctx, scope, req, w, status, outputMediaType, result)
}
}
@@ -287,8 +282,6 @@ type patcher struct {
patchBytes []byte
userAgent string
trace *utiltrace.Trace
// Set at invocation-time (by applyPatch) and immutable thereafter
namespace string
updatedObjectInfo rest.UpdatedObjectInfo
@@ -550,7 +543,7 @@ func strategicPatchObject(
// TODO: rename this function because the name implies it is related to applyPatcher
func (p *patcher) applyPatch(ctx context.Context, _, currentObject runtime.Object) (objToUpdate runtime.Object, patchErr error) {
// Make sure we actually have a persisted currentObject
p.trace.Step("About to apply patch")
tracing.SpanFromContext(ctx).AddEvent("About to apply patch")
currentObjectHasUID, err := hasUID(currentObject)
if err != nil {
return nil, err
@@ -599,7 +592,7 @@ func (p *patcher) admissionAttributes(ctx context.Context, updatedObject runtime
// and is given the currently persisted object and the patched object as input.
// TODO: rename this function because the name implies it is related to applyPatcher
func (p *patcher) applyAdmission(ctx context.Context, patchedObject runtime.Object, currentObject runtime.Object) (runtime.Object, error) {
p.trace.Step("About to check admission control")
tracing.SpanFromContext(ctx).AddEvent("About to check admission control")
var operation admission.Operation
var options runtime.Object
if hasUID, err := hasUID(currentObject); err != nil {

View File

@@ -32,7 +32,6 @@ import (
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
endpointsrequest "k8s.io/apiserver/pkg/endpoints/request"
utiltrace "k8s.io/utils/trace"
)
// transformObject takes the object as returned by storage and ensures it is in
@@ -129,7 +128,7 @@ func targetEncodingForTransform(scope *RequestScope, mediaType negotiation.Media
// transformResponseObject takes an object loaded from storage and performs any necessary transformations.
// Will write the complete response object.
func transformResponseObject(ctx context.Context, scope *RequestScope, trace *utiltrace.Trace, req *http.Request, w http.ResponseWriter, statusCode int, mediaType negotiation.MediaTypeOptions, result runtime.Object) {
func transformResponseObject(ctx context.Context, scope *RequestScope, req *http.Request, w http.ResponseWriter, statusCode int, mediaType negotiation.MediaTypeOptions, result runtime.Object) {
options, err := optionsForTransform(mediaType, req)
if err != nil {
scope.err(err, w, req)
@@ -147,7 +146,7 @@ func transformResponseObject(ctx context.Context, scope *RequestScope, trace *ut
return
}
kind, serializer, _ := targetEncodingForTransform(scope, mediaType, req)
responsewriters.WriteObjectNegotiated(serializer, scope, kind.GroupVersion(), w, req, statusCode, obj)
responsewriters.WriteObjectNegotiated(serializer, scope, kind.GroupVersion(), w, req, statusCode, obj, false)
}
// errNotAcceptable indicates Accept negotiation has failed

View File

@@ -18,6 +18,7 @@ package responsewriters
import (
"compress/gzip"
"context"
"encoding/json"
"fmt"
"io"
@@ -27,6 +28,8 @@ import (
"sync"
"time"
"go.opentelemetry.io/otel/attribute"
"k8s.io/apiserver/pkg/features"
"k8s.io/apimachinery/pkg/runtime"
@@ -40,7 +43,7 @@ import (
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/util/flushwriter"
"k8s.io/apiserver/pkg/util/wsstream"
utiltrace "k8s.io/utils/trace"
"k8s.io/component-base/tracing"
)
// StreamObject performs input stream negotiation from a ResourceStreamer and writes that to the response.
@@ -87,21 +90,22 @@ func StreamObject(statusCode int, gv schema.GroupVersion, s runtime.NegotiatedSe
// The context is optional and can be nil. This method will perform optional content compression if requested by
// a client and the feature gate for APIResponseCompression is enabled.
func SerializeObject(mediaType string, encoder runtime.Encoder, hw http.ResponseWriter, req *http.Request, statusCode int, object runtime.Object) {
trace := utiltrace.New("SerializeObject",
utiltrace.Field{"audit-id", request.GetAuditIDTruncated(req.Context())},
utiltrace.Field{"method", req.Method},
utiltrace.Field{"url", req.URL.Path},
utiltrace.Field{"protocol", req.Proto},
utiltrace.Field{"mediaType", mediaType},
utiltrace.Field{"encoder", encoder.Identifier()})
defer trace.LogIfLong(5 * time.Second)
ctx := req.Context()
ctx, span := tracing.Start(ctx, "SerializeObject",
attribute.String("audit-id", audit.GetAuditIDTruncated(ctx)),
attribute.String("method", req.Method),
attribute.String("url", req.URL.Path),
attribute.String("protocol", req.Proto),
attribute.String("mediaType", mediaType),
attribute.String("encoder", string(encoder.Identifier())))
defer span.End(5 * time.Second)
w := &deferredResponseWriter{
mediaType: mediaType,
statusCode: statusCode,
contentEncoding: negotiateContentEncoding(req),
hw: hw,
trace: trace,
ctx: ctx,
}
err := encoder.Encode(object, w)
@@ -191,23 +195,30 @@ type deferredResponseWriter struct {
hw http.ResponseWriter
w io.Writer
trace *utiltrace.Trace
ctx context.Context
}
func (w *deferredResponseWriter) Write(p []byte) (n int, err error) {
if w.trace != nil {
// This Step usually wraps in-memory object serialization.
w.trace.Step("About to start writing response", utiltrace.Field{"size", len(p)})
ctx := w.ctx
span := tracing.SpanFromContext(ctx)
// This Step usually wraps in-memory object serialization.
span.AddEvent("About to start writing response", attribute.Int("size", len(p)))
firstWrite := !w.hasWritten
defer func() {
w.trace.Step("Write call finished",
utiltrace.Field{"writer", fmt.Sprintf("%T", w.w)},
utiltrace.Field{"size", len(p)},
utiltrace.Field{"firstWrite", firstWrite},
utiltrace.Field{"err", err})
}()
}
firstWrite := !w.hasWritten
defer func() {
if err != nil {
span.AddEvent("Write call failed",
attribute.String("writer", fmt.Sprintf("%T", w.w)),
attribute.Int("size", len(p)),
attribute.Bool("firstWrite", firstWrite),
attribute.String("err", err.Error()))
} else {
span.AddEvent("Write call succeeded",
attribute.String("writer", fmt.Sprintf("%T", w.w)),
attribute.Int("size", len(p)),
attribute.Bool("firstWrite", firstWrite))
}
}()
if w.hasWritten {
return w.w.Write(p)
}
@@ -248,7 +259,7 @@ func (w *deferredResponseWriter) Close() error {
}
// WriteObjectNegotiated renders an object in the content type negotiated by the client.
func WriteObjectNegotiated(s runtime.NegotiatedSerializer, restrictions negotiation.EndpointRestrictions, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request, statusCode int, object runtime.Object) {
func WriteObjectNegotiated(s runtime.NegotiatedSerializer, restrictions negotiation.EndpointRestrictions, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request, statusCode int, object runtime.Object, listGVKInContentType bool) {
stream, ok := object.(rest.ResourceStreamer)
if ok {
requestInfo, _ := request.RequestInfoFrom(req.Context())
@@ -258,7 +269,7 @@ func WriteObjectNegotiated(s runtime.NegotiatedSerializer, restrictions negotiat
return
}
_, serializer, err := negotiation.NegotiateOutputMediaType(req, s, restrictions)
mediaType, serializer, err := negotiation.NegotiateOutputMediaType(req, s, restrictions)
if err != nil {
// if original statusCode was not successful we need to return the original error
// we cannot hide it behind negotiation problems
@@ -275,10 +286,30 @@ func WriteObjectNegotiated(s runtime.NegotiatedSerializer, restrictions negotiat
encoder := s.EncoderForVersion(serializer.Serializer, gv)
request.TrackSerializeResponseObjectLatency(req.Context(), func() {
SerializeObject(serializer.MediaType, encoder, w, req, statusCode, object)
if listGVKInContentType {
SerializeObject(generateMediaTypeWithGVK(serializer.MediaType, mediaType.Convert), encoder, w, req, statusCode, object)
} else {
SerializeObject(serializer.MediaType, encoder, w, req, statusCode, object)
}
})
}
func generateMediaTypeWithGVK(mediaType string, gvk *schema.GroupVersionKind) string {
if gvk == nil {
return mediaType
}
if gvk.Group != "" {
mediaType += ";g=" + gvk.Group
}
if gvk.Version != "" {
mediaType += ";v=" + gvk.Version
}
if gvk.Kind != "" {
mediaType += ";as=" + gvk.Kind
}
return mediaType
}
// ErrorNegotiated renders an error to the response. Returns the HTTP status code of the error.
// The context is optional and may be nil.
func ErrorNegotiated(err error, s runtime.NegotiatedSerializer, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request) int {
@@ -295,7 +326,7 @@ func ErrorNegotiated(err error, s runtime.NegotiatedSerializer, gv schema.GroupV
return code
}
WriteObjectNegotiated(s, negotiation.DefaultEndpointRestrictions, gv, w, req, code, status)
WriteObjectNegotiated(s, negotiation.DefaultEndpointRestrictions, gv, w, req, code, status, false)
return code
}

View File

@@ -41,6 +41,7 @@ import (
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/apiserver/pkg/endpoints/request"
@@ -236,7 +237,7 @@ type responder struct {
}
func (r *responder) Object(statusCode int, obj runtime.Object) {
responsewriters.WriteObjectNegotiated(r.scope.Serializer, r.scope, r.scope.Kind.GroupVersion(), r.w, r.req, statusCode, obj)
responsewriters.WriteObjectNegotiated(r.scope.Serializer, r.scope, r.scope.Kind.GroupVersion(), r.w, r.req, statusCode, obj, false)
}
func (r *responder) Error(err error) {
@@ -390,6 +391,15 @@ func limitedReadBody(req *http.Request, limit int64) ([]byte, error) {
return data, nil
}
func limitedReadBodyWithRecordMetric(ctx context.Context, req *http.Request, limit int64, resourceGroup string, verb requestmetrics.RequestBodyVerb) ([]byte, error) {
readBody, err := limitedReadBody(req, limit)
if err == nil {
// only record if we've read successfully
requestmetrics.RecordRequestBodySize(ctx, resourceGroup, verb, len(readBody))
}
return readBody, err
}
func isDryRun(url *url.URL) bool {
return len(url.Query()["dryRun"]) != 0
}

View File

@@ -19,15 +19,19 @@ package handlers
import (
"net/http"
utiltrace "k8s.io/utils/trace"
"go.opentelemetry.io/otel/attribute"
)
func traceFields(req *http.Request) []utiltrace.Field {
return []utiltrace.Field{
{Key: "url", Value: req.URL.Path},
{Key: "user-agent", Value: &lazyTruncatedUserAgent{req: req}},
{Key: "audit-id", Value: &lazyAuditID{req: req}},
{Key: "client", Value: &lazyClientIP{req: req}},
{Key: "accept", Value: &lazyAccept{req: req}},
{Key: "protocol", Value: req.Proto}}
func traceFields(req *http.Request) []attribute.KeyValue {
return []attribute.KeyValue{
attribute.Stringer("accept", &lazyAccept{req: req}),
attribute.Stringer("audit-id", &lazyAuditID{req: req}),
attribute.Stringer("client", &lazyClientIP{req: req}),
attribute.String("protocol", req.Proto),
attribute.Stringer("resource", &lazyResource{req: req}),
attribute.Stringer("scope", &lazyScope{req: req}),
attribute.String("url", req.URL.Path),
attribute.Stringer("user-agent", &lazyTruncatedUserAgent{req: req}),
attribute.Stringer("verb", &lazyVerb{req: req}),
}
}

View File

@@ -23,6 +23,8 @@ import (
"sync"
"time"
"go.opentelemetry.io/otel/attribute"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme"
@@ -35,27 +37,22 @@ import (
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
requestmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/util/dryrun"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/tracing"
"k8s.io/klog/v2"
utiltrace "k8s.io/utils/trace"
)
// UpdateResource returns a function that will handle a resource update
func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interface) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
ctx := req.Context()
// For performance tracking purposes.
trace := utiltrace.New("Update", traceFields(req)...)
defer trace.LogIfLong(500 * time.Millisecond)
if isDryRun(req.URL) && !utilfeature.DefaultFeatureGate.Enabled(features.DryRun) {
scope.err(errors.NewBadRequest("the dryRun feature is disabled"), w, req)
return
}
ctx, span := tracing.Start(ctx, "Update", traceFields(req)...)
defer span.End(500 * time.Millisecond)
namespace, name, err := scope.Namer.Name(req)
if err != nil {
@@ -65,7 +62,7 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
// enforce a timeout of at most requestTimeoutUpperBound (34s) or less if the user-provided
// timeout inside the parent context is lower than requestTimeoutUpperBound.
ctx, cancel := context.WithTimeout(req.Context(), requestTimeoutUpperBound)
ctx, cancel := context.WithTimeout(ctx, requestTimeoutUpperBound)
defer cancel()
ctx = request.WithNamespace(ctx, namespace)
@@ -76,12 +73,13 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
return
}
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
trace.Step("limitedReadBody done", utiltrace.Field{"len", len(body)}, utiltrace.Field{"err", err})
body, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes, scope.Resource.GroupResource().String(), requestmetrics.Update)
if err != nil {
span.AddEvent("limitedReadBody failed", attribute.Int("len", len(body)), attribute.String("err", err.Error()))
scope.err(err, w, req)
return
}
span.AddEvent("limitedReadBody succeeded", attribute.Int("len", len(body)))
options := &metav1.UpdateOptions{}
if err := metainternalversionscheme.ParameterCodec.DecodeParameters(req.URL.Query(), scope.MetaGroupVersion, options); err != nil {
@@ -111,7 +109,7 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
}
decoder := scope.Serializer.DecoderToVersion(decodeSerializer, scope.HubGroupVersion)
trace.Step("About to convert to expected version")
span.AddEvent("About to convert to expected version")
obj, gvk, err := decoder.Decode(body, &defaultGVK, original)
if err != nil {
strictError, isStrictError := runtime.AsStrictDecodingError(err)
@@ -134,7 +132,7 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
scope.err(err, w, req)
return
}
trace.Step("Conversion done")
span.AddEvent("Conversion done")
audit.LogRequestObject(req.Context(), obj, objGV, scope.Resource, scope.Subresource, scope.Serializer)
admit = admission.WithAudit(admit)
@@ -213,7 +211,7 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
Name: name,
}
trace.Step("About to store object in database")
span.AddEvent("About to store object in database")
wasCreated := false
requestFunc := func() (runtime.Object, error) {
obj, created, err := r.Update(
@@ -248,20 +246,21 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
}
return result, err
})
trace.Step("Write to database call finished", utiltrace.Field{"len", len(body)}, utiltrace.Field{"err", err})
if err != nil {
span.AddEvent("Write to database call failed", attribute.Int("len", len(body)), attribute.String("err", err.Error()))
scope.err(err, w, req)
return
}
span.AddEvent("Write to database call succeeded", attribute.Int("len", len(body)))
status := http.StatusOK
if wasCreated {
status = http.StatusCreated
}
trace.Step("About to write a response")
defer trace.Step("Writing http response done")
transformResponseObject(ctx, scope, trace, req, w, status, outputMediaType, result)
span.AddEvent("About to write a response")
defer span.AddEvent("Writing http response done")
transformResponseObject(ctx, scope, req, w, status, outputMediaType, result)
}
}

View File

@@ -26,6 +26,7 @@ import (
"unicode"
restful "github.com/emicklei/go-restful/v3"
apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/conversion"
"k8s.io/apimachinery/pkg/runtime"
@@ -68,6 +69,91 @@ type action struct {
AllNamespaces bool // true iff the action is namespaced but works on aggregate result for all namespaces
}
func ConvertGroupVersionIntoToDiscovery(list []metav1.APIResource) ([]apidiscoveryv2beta1.APIResourceDiscovery, error) {
var apiResourceList []apidiscoveryv2beta1.APIResourceDiscovery
parentResources := make(map[string]int)
// Loop through all top-level resources
for _, r := range list {
if strings.Contains(r.Name, "/") {
// Skip subresources for now so we can get the list of resources
continue
}
var scope apidiscoveryv2beta1.ResourceScope
if r.Namespaced {
scope = apidiscoveryv2beta1.ScopeNamespace
} else {
scope = apidiscoveryv2beta1.ScopeCluster
}
resource := apidiscoveryv2beta1.APIResourceDiscovery{
Resource: r.Name,
Scope: scope,
ResponseKind: &metav1.GroupVersionKind{
Group: r.Group,
Version: r.Version,
Kind: r.Kind,
},
Verbs: r.Verbs,
ShortNames: r.ShortNames,
Categories: r.Categories,
SingularResource: r.SingularName,
}
apiResourceList = append(apiResourceList, resource)
parentResources[r.Name] = len(apiResourceList) - 1
}
// Loop through all subresources
for _, r := range list {
// Split resource name and subresource name
split := strings.SplitN(r.Name, "/", 2)
if len(split) != 2 {
// Skip parent resources
continue
}
var scope apidiscoveryv2beta1.ResourceScope
if r.Namespaced {
scope = apidiscoveryv2beta1.ScopeNamespace
} else {
scope = apidiscoveryv2beta1.ScopeCluster
}
parentidx, exists := parentResources[split[0]]
if !exists {
// If a subresource exists without a parent, create a parent
apiResourceList = append(apiResourceList, apidiscoveryv2beta1.APIResourceDiscovery{
Resource: split[0],
Scope: scope,
})
parentidx = len(apiResourceList) - 1
parentResources[split[0]] = parentidx
}
if apiResourceList[parentidx].Scope != scope {
return nil, fmt.Errorf("Error: Parent %s (scope: %s) and subresource %s (scope: %s) scope do not match", split[0], apiResourceList[parentidx].Scope, split[1], scope)
//
}
subresource := apidiscoveryv2beta1.APISubresourceDiscovery{
Subresource: split[1],
Verbs: r.Verbs,
}
if r.Kind != "" {
subresource.ResponseKind = &metav1.GroupVersionKind{
Group: r.Group,
Version: r.Version,
Kind: r.Kind,
}
}
apiResourceList[parentidx].Subresources = append(apiResourceList[parentidx].Subresources, subresource)
}
return apiResourceList, nil
}
// An interface to see if one storage supports override its default verb for monitoring
type StorageMetricsOverride interface {
// OverrideMetricsVerb gives a storage object an opportunity to override the verb reported to the metrics endpoint
@@ -260,7 +346,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
}
var resetFields map[fieldpath.APIVersion]*fieldpath.Set
if a.group.OpenAPIModels != nil && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
if a.group.OpenAPIModels != nil {
if resetFieldsStrategy, isResetFieldsStrategy := storage.(rest.ResetFieldsStrategy); isResetFieldsStrategy {
resetFields = resetFieldsStrategy.GetResetFields()
}
@@ -599,7 +685,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
if a.group.MetaGroupVersion != nil {
reqScope.MetaGroupVersion = *a.group.MetaGroupVersion
}
if a.group.OpenAPIModels != nil && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
if a.group.OpenAPIModels != nil {
reqScope.FieldManager, err = fieldmanager.NewDefaultFieldManager(
a.group.TypeConverter,
a.group.UnsafeConvertor,
@@ -785,9 +871,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
string(types.JSONPatchType),
string(types.MergePatchType),
string(types.StrategicMergePatchType),
}
if utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
supportedTypes = append(supportedTypes, string(types.ApplyPatchType))
string(types.ApplyPatchType),
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulPatchResource(patcher, reqScope, admit, supportedTypes))
handler = utilwarning.AddWarningsHandler(handler, warnings)

View File

@@ -62,7 +62,8 @@ const (
var (
deprecatedRequestGauge = compbasemetrics.NewGaugeVec(
&compbasemetrics.GaugeOpts{
Name: "apiserver_requested_deprecated_apis",
Subsystem: APIServerComponent,
Name: "requested_deprecated_apis",
Help: "Gauge of deprecated APIs that have been requested, broken out by API group, version, resource, subresource, and removed_release.",
StabilityLevel: compbasemetrics.STABLE,
},
@@ -73,7 +74,8 @@ var (
// the upstream library supports it.
requestCounter = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_request_total",
Subsystem: APIServerComponent,
Name: "request_total",
Help: "Counter of apiserver requests broken out for each verb, dry run value, group, version, resource, scope, component, and HTTP response code.",
StabilityLevel: compbasemetrics.STABLE,
},
@@ -81,7 +83,8 @@ var (
)
longRunningRequestsGauge = compbasemetrics.NewGaugeVec(
&compbasemetrics.GaugeOpts{
Name: "apiserver_longrunning_requests",
Subsystem: APIServerComponent,
Name: "longrunning_requests",
Help: "Gauge of all active long-running apiserver requests broken out by verb, group, version, resource, scope and component. Not all requests are tracked this way.",
StabilityLevel: compbasemetrics.STABLE,
},
@@ -89,8 +92,9 @@ var (
)
requestLatencies = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "apiserver_request_duration_seconds",
Help: "Response latency distribution in seconds for each verb, dry run value, group, version, resource, subresource, scope and component.",
Subsystem: APIServerComponent,
Name: "request_duration_seconds",
Help: "Response latency distribution in seconds for each verb, dry run value, group, version, resource, subresource, scope and component.",
// This metric is used for verifying api call latencies SLO,
// as well as tracking regressions in this aspects.
// Thus we customize buckets significantly, to empower both usecases.
@@ -102,8 +106,24 @@ var (
)
requestSloLatencies = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "apiserver_request_slo_duration_seconds",
Help: "Response latency distribution (not counting webhook duration) in seconds for each verb, group, version, resource, subresource, scope and component.",
Subsystem: APIServerComponent,
Name: "request_slo_duration_seconds",
Help: "Response latency distribution (not counting webhook duration) in seconds for each verb, group, version, resource, subresource, scope and component.",
// This metric is supplementary to the requestLatencies metric.
// It measures request duration excluding webhooks as they are mostly
// dependant on user configuration.
Buckets: []float64{0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.25, 1.5, 2, 3,
4, 5, 6, 8, 10, 15, 20, 30, 45, 60},
StabilityLevel: compbasemetrics.ALPHA,
DeprecatedVersion: "1.27.0",
},
[]string{"verb", "group", "version", "resource", "subresource", "scope", "component"},
)
requestSliLatencies = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Subsystem: APIServerComponent,
Name: "request_sli_duration_seconds",
Help: "Response latency distribution (not counting webhook duration) in seconds for each verb, group, version, resource, subresource, scope and component.",
// This metric is supplementary to the requestLatencies metric.
// It measures request duration excluding webhooks as they are mostly
// dependant on user configuration.
@@ -128,8 +148,9 @@ var (
)
responseSizes = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "apiserver_response_sizes",
Help: "Response size distribution in bytes for each group, version, verb, resource, subresource, scope and component.",
Subsystem: APIServerComponent,
Name: "response_sizes",
Help: "Response size distribution in bytes for each group, version, verb, resource, subresource, scope and component.",
// Use buckets ranging from 1000 bytes (1KB) to 10^9 bytes (1GB).
Buckets: compbasemetrics.ExponentialBuckets(1000, 10.0, 7),
StabilityLevel: compbasemetrics.STABLE,
@@ -139,14 +160,16 @@ var (
// TLSHandshakeErrors is a number of requests dropped with 'TLS handshake error from' error
TLSHandshakeErrors = compbasemetrics.NewCounter(
&compbasemetrics.CounterOpts{
Name: "apiserver_tls_handshake_errors_total",
Subsystem: APIServerComponent,
Name: "tls_handshake_errors_total",
Help: "Number of requests dropped with 'TLS handshake error from' error",
StabilityLevel: compbasemetrics.ALPHA,
},
)
WatchEvents = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_watch_events_total",
Subsystem: APIServerComponent,
Name: "watch_events_total",
Help: "Number of events sent in watch clients",
StabilityLevel: compbasemetrics.ALPHA,
},
@@ -154,7 +177,8 @@ var (
)
WatchEventsSizes = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "apiserver_watch_events_sizes",
Subsystem: APIServerComponent,
Name: "watch_events_sizes",
Help: "Watch event size distribution in bytes",
Buckets: compbasemetrics.ExponentialBuckets(1024, 2.0, 8), // 1K, 2K, 4K, 8K, ..., 128K.
StabilityLevel: compbasemetrics.ALPHA,
@@ -165,7 +189,8 @@ var (
// it reports maximal usage during the last second.
currentInflightRequests = compbasemetrics.NewGaugeVec(
&compbasemetrics.GaugeOpts{
Name: "apiserver_current_inflight_requests",
Subsystem: APIServerComponent,
Name: "current_inflight_requests",
Help: "Maximal number of currently used inflight request limit of this apiserver per request kind in last second.",
StabilityLevel: compbasemetrics.STABLE,
},
@@ -173,7 +198,8 @@ var (
)
currentInqueueRequests = compbasemetrics.NewGaugeVec(
&compbasemetrics.GaugeOpts{
Name: "apiserver_current_inqueue_requests",
Subsystem: APIServerComponent,
Name: "current_inqueue_requests",
Help: "Maximal number of queued requests in this apiserver per request kind in last second.",
StabilityLevel: compbasemetrics.ALPHA,
},
@@ -182,7 +208,8 @@ var (
requestTerminationsTotal = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_request_terminations_total",
Subsystem: APIServerComponent,
Name: "request_terminations_total",
Help: "Number of requests which apiserver terminated in self-defense.",
StabilityLevel: compbasemetrics.ALPHA,
},
@@ -191,7 +218,8 @@ var (
apiSelfRequestCounter = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_selfrequest_total",
Subsystem: APIServerComponent,
Name: "selfrequest_total",
Help: "Counter of apiserver self-requests broken out for each verb, API resource and subresource.",
StabilityLevel: compbasemetrics.ALPHA,
},
@@ -200,7 +228,8 @@ var (
requestFilterDuration = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "apiserver_request_filter_duration_seconds",
Subsystem: APIServerComponent,
Name: "request_filter_duration_seconds",
Help: "Request filter latency distribution in seconds, for each filter type",
Buckets: []float64{0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 5.0},
StabilityLevel: compbasemetrics.ALPHA,
@@ -211,7 +240,8 @@ var (
// requestAbortsTotal is a number of aborted requests with http.ErrAbortHandler
requestAbortsTotal = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_request_aborts_total",
Subsystem: APIServerComponent,
Name: "request_aborts_total",
Help: "Number of requests which apiserver aborted possibly due to a timeout, for each group, version, verb, resource, subresource and scope",
StabilityLevel: compbasemetrics.ALPHA,
},
@@ -231,7 +261,8 @@ var (
// within the wait threshold.
requestPostTimeoutTotal = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_request_post_timeout_total",
Subsystem: APIServerComponent,
Name: "request_post_timeout_total",
Help: "Tracks the activity of the request handlers after the associated requests have been timed out by the apiserver",
StabilityLevel: compbasemetrics.ALPHA,
},
@@ -240,7 +271,8 @@ var (
requestTimestampComparisonDuration = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "apiserver_request_timestamp_comparison_time",
Subsystem: APIServerComponent,
Name: "request_timestamp_comparison_time",
Help: "Time taken for comparison of old vs new objects in UPDATE or PATCH requests",
Buckets: []float64{0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 5.0},
StabilityLevel: compbasemetrics.ALPHA,
@@ -256,6 +288,7 @@ var (
longRunningRequestsGauge,
requestLatencies,
requestSloLatencies,
requestSliLatencies,
fieldValidationRequestLatencies,
responseSizes,
TLSHandshakeErrors,
@@ -502,8 +535,9 @@ func MonitorRequest(req *http.Request, verb, group, version, resource, subresour
fieldValidationRequestLatencies.WithContext(req.Context()).WithLabelValues(fieldValidation, fieldValidationEnabled)
if wd, ok := request.LatencyTrackersFrom(req.Context()); ok {
sloLatency := elapsedSeconds - (wd.MutatingWebhookTracker.GetLatency() + wd.ValidatingWebhookTracker.GetLatency()).Seconds()
requestSloLatencies.WithContext(req.Context()).WithLabelValues(reportedVerb, group, version, resource, subresource, scope, component).Observe(sloLatency)
sliLatency := elapsedSeconds - (wd.MutatingWebhookTracker.GetLatency() + wd.ValidatingWebhookTracker.GetLatency()).Seconds()
requestSloLatencies.WithContext(req.Context()).WithLabelValues(reportedVerb, group, version, resource, subresource, scope, component).Observe(sliLatency)
requestSliLatencies.WithContext(req.Context()).WithLabelValues(reportedVerb, group, version, resource, subresource, scope, component).Observe(sliLatency)
}
// We are only interested in response sizes of read requests.
if verb == "GET" || verb == "LIST" {
@@ -548,6 +582,20 @@ func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, c
}
}
// NormalizedVerb returns normalized verb
func NormalizedVerb(req *http.Request) string {
verb := req.Method
if requestInfo, ok := request.RequestInfoFrom(req.Context()); ok {
// If we can find a requestInfo, we can get a scope, and then
// we can convert GETs to LISTs when needed.
scope := CleanScope(requestInfo)
verb = CanonicalVerb(strings.ToUpper(verb), scope)
}
// mark APPLY requests and WATCH requests correctly.
return CleanVerb(verb, req)
}
// CleanScope returns the scope of the request.
func CleanScope(requestInfo *request.RequestInfo) string {
if requestInfo.Name != "" || requestInfo.Verb == "create" {
@@ -588,7 +636,7 @@ func CleanVerb(verb string, request *http.Request) string {
if verb == "WATCHLIST" {
reportedVerb = "WATCH"
}
if verb == "PATCH" && request.Header.Get("Content-Type") == string(types.ApplyPatchType) && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
if verb == "PATCH" && request.Header.Get("Content-Type") == string(types.ApplyPatchType) {
reportedVerb = "APPLY"
}
return reportedVerb

View File

@@ -1,65 +0,0 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package request
import (
"context"
"k8s.io/apimachinery/pkg/types"
)
type auditIDKeyType int
// auditIDKey is the key to associate the Audit-ID value of a request.
const auditIDKey auditIDKeyType = iota
// WithAuditID returns a copy of the parent context into which the Audit-ID
// associated with the request is set.
//
// If the specified auditID is empty, no value is set and the parent context is returned as is.
func WithAuditID(parent context.Context, auditID types.UID) context.Context {
if auditID == "" {
return parent
}
return WithValue(parent, auditIDKey, auditID)
}
// AuditIDFrom returns the value of the audit ID from the request context.
func AuditIDFrom(ctx context.Context) (types.UID, bool) {
auditID, ok := ctx.Value(auditIDKey).(types.UID)
return auditID, ok
}
// GetAuditIDTruncated returns the audit ID (truncated) from the request context.
// If the length of the Audit-ID value exceeds the limit, we truncate it to keep
// the first N (maxAuditIDLength) characters.
// This is intended to be used in logging only.
func GetAuditIDTruncated(ctx context.Context) string {
auditID, ok := AuditIDFrom(ctx)
if !ok {
return ""
}
// if the user has specified a very long audit ID then we will use the first N characters
// Note: assuming Audit-ID header is in ASCII
const maxAuditIDLength = 64
if len(auditID) > maxAuditIDLength {
auditID = auditID[0:maxAuditIDLength]
}
return string(auditID)
}

View File

@@ -35,6 +35,13 @@ const (
// of code conflicts because changes are more likely to be scattered
// across the file.
// owner: @jefftree @alexzielenski
// alpha: v1.26
//
// Enables an single HTTP endpoint /discovery/<version> which supports native HTTP
// caching with ETags containing all APIResources known to the apiserver.
AggregatedDiscoveryEndpoint featuregate.Feature = "AggregatedDiscoveryEndpoint"
// owner: @smarterclayton
// alpha: v1.8
// beta: v1.9
@@ -81,8 +88,15 @@ const (
// audited.
AdvancedAuditing featuregate.Feature = "AdvancedAuditing"
// owner: @cici37 @jpbetz
// kep: http://kep.k8s.io/3488
// alpha: v1.26
//
// Enables expression validation in Admission Control
ValidatingAdmissionPolicy featuregate.Feature = "ValidatingAdmissionPolicy"
// owner: @cici37
// kep: http://kep.k8s.io/2876
// kep: https://kep.k8s.io/2876
// alpha: v1.23
// beta: v1.25
//
@@ -108,14 +122,14 @@ const (
EfficientWatchResumption featuregate.Feature = "EfficientWatchResumption"
// owner: @aramase
// kep: http://kep.k8s.io/3299
// kep: https://kep.k8s.io/3299
// alpha: v1.25
//
// Enables KMS v2 API for encryption at rest.
KMSv2 featuregate.Feature = "KMSv2"
// owner: @jiahuif
// kep: http://kep.k8s.io/2887
// kep: https://kep.k8s.io/2887
// alpha: v1.23
// beta: v1.24
//
@@ -124,7 +138,7 @@ const (
OpenAPIEnums featuregate.Feature = "OpenAPIEnums"
// owner: @jefftree
// kep: http://kep.k8s.io/2896
// kep: https://kep.k8s.io/2896
// alpha: v1.23
// beta: v1.24
//
@@ -156,7 +170,7 @@ const (
ServerSideApply featuregate.Feature = "ServerSideApply"
// owner: @kevindelgado
// kep: http://kep.k8s.io/2885
// kep: https://kep.k8s.io/2885
// alpha: v1.23
// beta: v1.24
//
@@ -194,21 +208,25 @@ func init() {
// To add a new feature, define a key for it above and add it here. The features will be
// available throughout Kubernetes binaries.
var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
AggregatedDiscoveryEndpoint: {Default: false, PreRelease: featuregate.Alpha},
APIListChunking: {Default: true, PreRelease: featuregate.Beta},
APIPriorityAndFairness: {Default: true, PreRelease: featuregate.Beta},
APIResponseCompression: {Default: true, PreRelease: featuregate.Beta},
APIServerIdentity: {Default: false, PreRelease: featuregate.Alpha},
APIServerIdentity: {Default: true, PreRelease: featuregate.Beta},
APIServerTracing: {Default: false, PreRelease: featuregate.Alpha},
AdvancedAuditing: {Default: true, PreRelease: featuregate.GA},
ValidatingAdmissionPolicy: {Default: false, PreRelease: featuregate.Alpha},
CustomResourceValidationExpressions: {Default: true, PreRelease: featuregate.Beta},
DryRun: {Default: true, PreRelease: featuregate.GA},
DryRun: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.28
EfficientWatchResumption: {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
@@ -222,7 +240,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
RemoveSelfLink: {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
ServerSideApply: {Default: true, PreRelease: featuregate.GA},
ServerSideApply: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.29
ServerSideFieldValidation: {Default: true, PreRelease: featuregate.Beta},

View File

@@ -105,9 +105,5 @@ func (s *DryRunnableStorage) copyInto(in, out runtime.Object) error {
return err
}
_, _, err = s.Codec.Decode(data, nil, out)
if err != nil {
return err
}
return nil
return err
}

View File

@@ -56,6 +56,7 @@ func StorageWithCacher() generic.StorageDecorator {
cacherConfig := cacherstorage.Config{
Storage: s,
Versioner: storage.APIObjectVersioner{},
GroupResource: storageConfig.GroupResource,
ResourcePrefix: resourcePrefix,
KeyFunc: keyFunc,
NewFunc: newFunc,

View File

@@ -387,6 +387,9 @@ func (e *Store) Create(ctx context.Context, obj runtime.Object, createValidation
return nil, err
} else {
rest.FillObjectMetaSystemFields(objectMeta)
if len(objectMeta.GetGenerateName()) > 0 && len(objectMeta.GetName()) == 0 {
objectMeta.SetName(e.CreateStrategy.GenerateName(objectMeta.GetGenerateName()))
}
}
if e.BeginCreate != nil {

View File

@@ -30,9 +30,7 @@ import (
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/admission"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/storage/names"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/warning"
)
@@ -92,7 +90,7 @@ type RESTCreateStrategy interface {
}
// BeforeCreate ensures that common operations for all resources are performed on creation. It only returns
// errors that can be converted to api.Status. It invokes PrepareForCreate, then GenerateName, then Validate.
// errors that can be converted to api.Status. It invokes PrepareForCreate, then Validate.
// It returns nil if the object should be created.
func BeforeCreate(strategy RESTCreateStrategy, ctx context.Context, obj runtime.Object) error {
objectMeta, kind, kerr := objectMetaAndKind(strategy, obj)
@@ -105,6 +103,11 @@ func BeforeCreate(strategy RESTCreateStrategy, ctx context.Context, obj runtime.
return errors.NewInternalError(fmt.Errorf("system metadata was not initialized"))
}
// ensure the name has been generated
if len(objectMeta.GetGenerateName()) > 0 && len(objectMeta.GetName()) == 0 {
return errors.NewInternalError(fmt.Errorf("metadata.name was not generated"))
}
// ensure namespace on the object is correct, or error if a conflicting namespace was set in the object
requestNamespace, ok := genericapirequest.NamespaceFrom(ctx)
if !ok {
@@ -116,15 +119,6 @@ func BeforeCreate(strategy RESTCreateStrategy, ctx context.Context, obj runtime.
strategy.PrepareForCreate(ctx, obj)
if len(objectMeta.GetGenerateName()) > 0 && len(objectMeta.GetName()) == 0 {
objectMeta.SetName(strategy.GenerateName(objectMeta.GetGenerateName()))
}
// Ensure managedFields is not set unless the feature is enabled
if !utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
objectMeta.SetManagedFields(nil)
}
if errs := strategy.Validate(ctx, obj); len(errs) > 0 {
return errors.NewInvalid(kind.GroupKind(), objectMeta.GetName(), errs)
}

View File

@@ -29,8 +29,6 @@ import (
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/admission"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/warning"
)
@@ -128,12 +126,6 @@ func BeforeUpdate(strategy RESTUpdateStrategy, ctx context.Context, obj, old run
}
objectMeta.SetGeneration(oldMeta.GetGeneration())
// Ensure managedFields state is removed unless ServerSideApply is enabled
if !utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
oldMeta.SetManagedFields(nil)
objectMeta.SetManagedFields(nil)
}
strategy.PrepareForUpdate(ctx, obj, old)
// Use the existing UID if none is provided

View File

@@ -17,9 +17,13 @@ limitations under the License.
package server
import (
"context"
"crypto/sha256"
"encoding/base32"
"fmt"
"net"
"net/http"
"os"
goruntime "runtime"
"runtime/debug"
"sort"
@@ -30,11 +34,12 @@ import (
jsonpatch "github.com/evanphx/json-patch"
"github.com/google/uuid"
oteltrace "go.opentelemetry.io/otel/trace"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
"k8s.io/apimachinery/pkg/version"
@@ -45,9 +50,8 @@ import (
authenticatorunion "k8s.io/apiserver/pkg/authentication/request/union"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/authorization/authorizerfactory"
authorizerunion "k8s.io/apiserver/pkg/authorization/union"
"k8s.io/apiserver/pkg/endpoints/discovery"
discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
"k8s.io/apiserver/pkg/endpoints/filterlatency"
genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi"
@@ -67,6 +71,9 @@ import (
"k8s.io/client-go/informers"
restclient "k8s.io/client-go/rest"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics/features"
"k8s.io/component-base/metrics/prometheus/slis"
"k8s.io/component-base/tracing"
"k8s.io/klog/v2"
openapicommon "k8s.io/kube-openapi/pkg/common"
"k8s.io/kube-openapi/pkg/validation/spec"
@@ -120,6 +127,7 @@ type Config struct {
EnableIndex bool
EnableProfiling bool
EnableDiscovery bool
// Requires generic profiling enabled
EnableContentionProfiling bool
EnableMetrics bool
@@ -139,7 +147,7 @@ type Config struct {
ExternalAddress string
// TracerProvider can provide a tracer, which records spans for distributed tracing.
TracerProvider oteltrace.TracerProvider
TracerProvider tracing.TracerProvider
//===========================================================================
// Fields you probably don't care about changing
@@ -257,6 +265,9 @@ type Config struct {
// StorageVersionManager holds the storage versions of the API resources installed by this server.
StorageVersionManager storageversion.Manager
// AggregatedDiscoveryGroupManager serves /apis in an aggregated form.
AggregatedDiscoveryGroupManager discoveryendpoint.ResourceManager
}
type RecommendedConfig struct {
@@ -317,12 +328,22 @@ type AuthorizationInfo struct {
Authorizer authorizer.Authorizer
}
func init() {
utilruntime.Must(features.AddFeatureGates(utilfeature.DefaultMutableFeatureGate))
}
// NewConfig returns a Config struct with the default values
func NewConfig(codecs serializer.CodecFactory) *Config {
defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz}
var id string
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
id = "kube-apiserver-" + uuid.New().String()
hostname, err := os.Hostname()
if err != nil {
klog.Fatalf("error getting hostname for apiserver identity: %v", err)
}
hash := sha256.Sum256([]byte(hostname))
id = "kube-apiserver-" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:16]))
}
lifecycleSignals := newLifecycleSignals()
@@ -359,7 +380,7 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
// A request body might be encoded in json, and is converted to
// proto when persisted in etcd, so we allow 2x as the largest request
// body size to be accepted and decoded in a write request.
// If this constant is changed, maxRequestSizeBytes in apiextensions-apiserver/third_party/forked/celopenapi/model/schemas.go
// If this constant is changed, DefaultMaxRequestSizeBytes in k8s.io/apiserver/pkg/cel/limits.go
// should be changed to reflect the new value, if the two haven't
// been wired together already somehow.
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
@@ -372,7 +393,7 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
APIServerID: id,
StorageVersionManager: storageversion.NewDefaultManager(),
TracerProvider: oteltrace.NewNoopTracerProvider(),
TracerProvider: tracing.NewNoopTracerProvider(),
}
}
@@ -662,6 +683,14 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
}
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
manager := c.AggregatedDiscoveryGroupManager
if manager == nil {
manager = discoveryendpoint.NewResourceManager()
}
s.AggregatedDiscoveryGroupManager = manager
s.AggregatedLegacyDiscoveryGroupManager = discoveryendpoint.NewResourceManager()
}
for {
if c.JSONPatchMaxCopyBytes <= 0 {
break
@@ -782,6 +811,11 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
}
s.AddHealthChecks(delegateCheck)
}
s.RegisterDestroyFunc(func() {
if err := c.Config.TracerProvider.Shutdown(context.Background()); err != nil {
klog.Errorf("failed to shut down tracer provider: %v", err)
}
})
s.listedPathProvider = routes.ListedPathProviders{s.listedPathProvider, delegationTarget}
@@ -808,7 +842,7 @@ func BuildHandlerChainWithStorageVersionPrecondition(apiHandler http.Handler, c
func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
handler := filterlatency.TrackCompleted(apiHandler)
handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer)
handler = filterlatency.TrackStarted(handler, "authorization")
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authorization")
if c.FlowControl != nil {
workEstimatorCfg := flowcontrolrequest.DefaultWorkEstimatorConfig()
@@ -816,18 +850,18 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg)
handler = filterlatency.TrackCompleted(handler)
handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, requestWorkEstimator)
handler = filterlatency.TrackStarted(handler, "priorityandfairness")
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "priorityandfairness")
} else {
handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
}
handler = filterlatency.TrackCompleted(handler)
handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)
handler = filterlatency.TrackStarted(handler, "impersonation")
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "impersonation")
handler = filterlatency.TrackCompleted(handler)
handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator, c.LongRunningFunc)
handler = filterlatency.TrackStarted(handler, "audit")
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "audit")
failedHandler := genericapifilters.Unauthorized(c.Serializer)
failedHandler = genericapifilters.WithFailedAuthenticationAudit(failedHandler, c.AuditBackend, c.AuditPolicyRuleEvaluator)
@@ -835,7 +869,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
failedHandler = filterlatency.TrackCompleted(failedHandler)
handler = filterlatency.TrackCompleted(handler)
handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences)
handler = filterlatency.TrackStarted(handler, "authentication")
handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authentication")
handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")
@@ -849,7 +883,6 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
if c.SecureServing != nil && !c.SecureServing.DisableHTTP2 && c.GoawayChance > 0 {
handler = genericfilters.WithProbabilisticGoaway(handler, c.GoawayChance)
}
handler = genericapifilters.WithAuditAnnotations(handler, c.AuditBackend, c.AuditPolicyRuleEvaluator)
handler = genericapifilters.WithWarningRecorder(handler)
handler = genericapifilters.WithCacheControl(handler)
handler = genericfilters.WithHSTS(handler, c.HSTSDirectives)
@@ -865,7 +898,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
handler = genericapifilters.WithRequestReceivedTimestamp(handler)
handler = genericapifilters.WithMuxAndDiscoveryComplete(handler, c.lifecycleSignals.MuxAndDiscoveryComplete.Signaled())
handler = genericfilters.WithPanicRecovery(handler, c.RequestInfoResolver)
handler = genericapifilters.WithAuditID(handler)
handler = genericapifilters.WithAuditInit(handler)
return handler
}
@@ -881,18 +914,30 @@ func installAPI(s *GenericAPIServer, c *Config) {
// so far, only logging related endpoints are considered valid to add for these debug flags.
routes.DebugFlags{}.Install(s.Handler.NonGoRestfulMux, "v", routes.StringFlagPutHandler(logs.GlogSetter))
}
if c.EnableMetrics {
if c.EnableProfiling {
routes.MetricsWithReset{}.Install(s.Handler.NonGoRestfulMux)
if utilfeature.DefaultFeatureGate.Enabled(features.ComponentSLIs) {
slis.SLIMetricsWithReset{}.Install(s.Handler.NonGoRestfulMux)
}
} else {
routes.DefaultMetrics{}.Install(s.Handler.NonGoRestfulMux)
if utilfeature.DefaultFeatureGate.Enabled(features.ComponentSLIs) {
slis.SLIMetrics{}.Install(s.Handler.NonGoRestfulMux)
}
}
}
routes.Version{Version: c.Version}.Install(s.Handler.GoRestfulContainer)
if c.EnableDiscovery {
s.Handler.GoRestfulContainer.Add(s.DiscoveryGroupManager.WebService())
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(s.DiscoveryGroupManager, s.AggregatedDiscoveryGroupManager)
s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/apis", metav1.APIGroupList{}))
} else {
s.Handler.GoRestfulContainer.Add(s.DiscoveryGroupManager.WebService())
}
}
if c.FlowControl != nil && utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) {
c.FlowControl.Install(s.Handler.NonGoRestfulMux)
@@ -957,7 +1002,4 @@ func AuthorizeClientBearerToken(loopback *restclient.Config, authn *Authenticati
tokenAuthenticator := authenticatorfactory.NewFromTokens(tokens, authn.APIAudiences)
authn.Authenticator = authenticatorunion.New(tokenAuthenticator, authn.Authenticator)
tokenAuthorizer := authorizerfactory.NewPrivilegedGroups(user.SystemPrivilegedGroup)
authz.Authorizer = authorizerunion.New(tokenAuthorizer, authz.Authorizer)
}

View File

@@ -54,7 +54,9 @@ type ResourceExpirationEvaluator interface {
}
func NewResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (ResourceExpirationEvaluator, error) {
ret := &resourceExpirationEvaluator{}
ret := &resourceExpirationEvaluator{
strictRemovedHandlingInAlpha: false,
}
if len(currentVersion.Major) > 0 {
currentMajor64, err := strconv.ParseInt(currentVersion.Major, 10, 32)
if err != nil {

View File

@@ -204,7 +204,7 @@ func (c *DynamicFileCAContent) watchCAFile(stopCh <-chan struct{}) error {
func (c *DynamicFileCAContent) handleWatchEvent(e fsnotify.Event, w *fsnotify.Watcher) error {
// This should be executed after restarting the watch (if applicable) to ensure no file event will be missing.
defer c.queue.Add(workItemKey)
if e.Op&(fsnotify.Remove|fsnotify.Rename) == 0 {
if !e.Has(fsnotify.Remove) && !e.Has(fsnotify.Rename) {
return nil
}
if err := w.Remove(c.filename); err != nil {

View File

@@ -185,7 +185,7 @@ func (c *DynamicCertKeyPairContent) watchCertKeyFile(stopCh <-chan struct{}) err
func (c *DynamicCertKeyPairContent) handleWatchEvent(e fsnotify.Event, w *fsnotify.Watcher) error {
// This should be executed after restarting the watch (if applicable) to ensure no file event will be missing.
defer c.queue.Add(workItemKey)
if e.Op&(fsnotify.Remove|fsnotify.Rename) == 0 {
if !e.Has(fsnotify.Remove) && !e.Has(fsnotify.Rename) {
return nil
}
if err := w.Remove(e.Name); err != nil {

View File

@@ -29,19 +29,25 @@ import (
"strings"
"time"
"go.opentelemetry.io/otel/attribute"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apiserver/pkg/apis/apiserver"
egressmetrics "k8s.io/apiserver/pkg/server/egressselector/metrics"
compbasemetrics "k8s.io/component-base/metrics"
"k8s.io/component-base/tracing"
"k8s.io/klog/v2"
utiltrace "k8s.io/utils/trace"
client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client"
)
var directDialer utilnet.DialFunc = http.DefaultTransport.(*http.Transport).DialContext
func init() {
client.Metrics.RegisterMetrics(compbasemetrics.NewKubeRegistry().Registerer())
}
// EgressSelector is the map of network context type to context dialer, for network egress.
type EgressSelector struct {
egressToDialer map[EgressType]utilnet.DialFunc
@@ -216,6 +222,9 @@ func (u *udsGRPCConnector) connect(_ context.Context) (proxier, error) {
// See https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/357.
tunnelCtx := context.TODO()
tunnel, err := client.CreateSingleUseGrpcTunnel(tunnelCtx, udsName, dialOption,
grpc.WithBlock(),
grpc.WithReturnConnectionError(),
grpc.WithTimeout(30*time.Second), // matches http.DefaultTransport dial timeout
grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return nil, err
@@ -239,9 +248,10 @@ func (d *dialerCreator) createDialer() utilnet.DialFunc {
return directDialer
}
return func(ctx context.Context, network, addr string) (net.Conn, error) {
trace := utiltrace.New(fmt.Sprintf("Proxy via %s protocol over %s", d.options.protocol, d.options.transport), utiltrace.Field{Key: "address", Value: addr})
defer trace.LogIfLong(500 * time.Millisecond)
ctx, span := tracing.Start(ctx, fmt.Sprintf("Proxy via %s protocol over %s", d.options.protocol, d.options.transport), attribute.String("address", addr))
defer span.End(500 * time.Millisecond)
start := egressmetrics.Metrics.Clock().Now()
egressmetrics.Metrics.ObserveDialStart(d.options.protocol, d.options.transport)
proxier, err := d.connector.connect(ctx)
if err != nil {
egressmetrics.Metrics.ObserveDialFailure(d.options.protocol, d.options.transport, egressmetrics.StageConnect)

View File

@@ -53,12 +53,24 @@ var (
// DialMetrics instruments dials to proxy server with prometheus metrics.
type DialMetrics struct {
clock clock.Clock
starts *metrics.CounterVec
latencies *metrics.HistogramVec
failures *metrics.CounterVec
}
// newDialMetrics create a new DialMetrics, configured with default metric names.
func newDialMetrics() *DialMetrics {
starts := metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "dial_start_total",
Help: "Dial starts, labeled by the protocol (http-connect or grpc) and transport (tcp or uds).",
StabilityLevel: metrics.ALPHA,
},
[]string{"protocol", "transport"},
)
latencies := metrics.NewHistogramVec(
&metrics.HistogramOpts{
Namespace: namespace,
@@ -82,9 +94,10 @@ func newDialMetrics() *DialMetrics {
[]string{"protocol", "transport", "stage"},
)
legacyregistry.MustRegister(starts)
legacyregistry.MustRegister(latencies)
legacyregistry.MustRegister(failures)
return &DialMetrics{latencies: latencies, failures: failures, clock: clock.RealClock{}}
return &DialMetrics{starts: starts, latencies: latencies, failures: failures, clock: clock.RealClock{}}
}
// Clock returns the clock.
@@ -99,10 +112,16 @@ func (m *DialMetrics) SetClock(c clock.Clock) {
// Reset resets the metrics.
func (m *DialMetrics) Reset() {
m.starts.Reset()
m.latencies.Reset()
m.failures.Reset()
}
// ObserveDialStart records the start of a dial attempt, labeled by protocol, transport.
func (m *DialMetrics) ObserveDialStart(protocol, transport string) {
m.starts.WithLabelValues(protocol, transport).Inc()
}
// ObserveDialLatency records the latency of a dial, labeled by protocol, transport.
func (m *DialMetrics) ObserveDialLatency(elapsed time.Duration, protocol, transport string) {
m.latencies.WithLabelValues(protocol, transport).Observe(elapsed.Seconds())

View File

@@ -25,7 +25,7 @@ import (
"sync/atomic"
"time"
flowcontrol "k8s.io/api/flowcontrol/v1beta2"
flowcontrol "k8s.io/api/flowcontrol/v1beta3"
apitypes "k8s.io/apimachinery/pkg/types"
epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
apirequest "k8s.io/apiserver/pkg/endpoints/request"

View File

@@ -31,6 +31,7 @@ import (
"k8s.io/apiserver/pkg/endpoints/metrics"
apirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/endpoints/responsewriter"
"k8s.io/apiserver/pkg/server/httplog"
)
// WithTimeoutForNonLongRunningRequests times out non-long-running requests after the time given by timeout.
@@ -141,7 +142,9 @@ func (t *timeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
utilruntime.HandleError(err)
}()
}()
httplog.SetStacktracePredicate(r.Context(), func(status int) bool {
return false
})
defer postTimeoutFn()
tw.timeout(err)
}

View File

@@ -21,6 +21,7 @@ import (
"net/http"
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/server/httplog"
@@ -50,11 +51,11 @@ func WithPanicRecovery(handler http.Handler, resolver request.RequestInfoResolve
// This call can have different handlers, but the default chain rate limits. Call it after the metrics are updated
// in case the rate limit delays it. If you outrun the rate for this one timed out requests, something has gone
// seriously wrong with your server, but generally having a logging signal for timeouts is useful.
runtime.HandleError(fmt.Errorf("timeout or abort while handling: method=%v URI=%q audit-ID=%q", req.Method, req.RequestURI, request.GetAuditIDTruncated(req.Context())))
runtime.HandleError(fmt.Errorf("timeout or abort while handling: method=%v URI=%q audit-ID=%q", req.Method, req.RequestURI, audit.GetAuditIDTruncated(req.Context())))
return
}
http.Error(w, "This request caused apiserver to panic. Look in the logs for details.", http.StatusInternalServerError)
klog.ErrorS(nil, "apiserver panic'd", "method", req.Method, "URI", req.RequestURI, "audit-ID", request.GetAuditIDTruncated(req.Context()))
klog.ErrorS(nil, "apiserver panic'd", "method", req.Method, "URI", req.RequestURI, "audit-ID", audit.GetAuditIDTruncated(req.Context()))
})
}

View File

@@ -26,6 +26,7 @@ import (
systemd "github.com/coreos/go-systemd/v22/daemon"
apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -39,6 +40,7 @@ import (
"k8s.io/apiserver/pkg/authorization/authorizer"
genericapi "k8s.io/apiserver/pkg/endpoints"
"k8s.io/apiserver/pkg/endpoints/discovery"
discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/registry/rest"
@@ -137,9 +139,15 @@ type GenericAPIServer struct {
// listedPathProvider is a lister which provides the set of paths to show at /
listedPathProvider routes.ListedPathProvider
// DiscoveryGroupManager serves /apis
// DiscoveryGroupManager serves /apis in an unaggregated form.
DiscoveryGroupManager discovery.GroupManager
// AggregatedDiscoveryGroupManager serves /apis in an aggregated form.
AggregatedDiscoveryGroupManager discoveryendpoint.ResourceManager
// AggregatedLegacyDiscoveryGroupManager serves /api in an aggregated form.
AggregatedLegacyDiscoveryGroupManager discoveryendpoint.ResourceManager
// Enable swagger and/or OpenAPI if these configs are non-nil.
openAPIConfig *openapicommon.Config
@@ -418,44 +426,40 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
// or the secure port cannot be listened on initially.
// This is the diagram of what channels/signals are dependent on each other:
//
// stopCh
// |
// ---------------------------------------------------------
// | |
// ShutdownInitiated (shutdownInitiatedCh) |
// | |
//
// (ShutdownDelayDuration) (PreShutdownHooks)
//
// | |
// AfterShutdownDelayDuration (delayedStopCh) PreShutdownHooksStopped (preShutdownHooksHasStoppedCh)
// | |
// |-------------------------------------------------------|
// |
// |
// NotAcceptingNewRequest (notAcceptingNewRequestCh)
// |
// |
// |---------------------------------------------------------|
// | | | |
// [without [with | |
//
// ShutdownSendRetryAfter] ShutdownSendRetryAfter] | |
//
// | | | |
// | ---------------| |
// | | |
// | (HandlerChainWaitGroup::Wait) |
// | | |
// | InFlightRequestsDrained (drainedCh) |
// | | |
// ----------------------------------------|-----------------|
// | |
// stopHttpServerCh (AuditBackend::Shutdown())
// |
// listenerStoppedCh
// |
// HTTPServerStoppedListening (httpServerStoppedListeningCh)
// | stopCh
// | |
// | ---------------------------------------------------------
// | | |
// | ShutdownInitiated (shutdownInitiatedCh) |
// | | |
// | (ShutdownDelayDuration) (PreShutdownHooks)
// | | |
// | AfterShutdownDelayDuration (delayedStopCh) PreShutdownHooksStopped (preShutdownHooksHasStoppedCh)
// | | |
// | |-------------------------------------------------------|
// | |
// | |
// | NotAcceptingNewRequest (notAcceptingNewRequestCh)
// | |
// | |
// | |---------------------------------------------------------|
// | | | | |
// | [without [with | |
// | ShutdownSendRetryAfter] ShutdownSendRetryAfter] | |
// | | | | |
// | | ---------------| |
// | | | |
// | | (HandlerChainWaitGroup::Wait) |
// | | | |
// | | InFlightRequestsDrained (drainedCh) |
// | | | |
// | ----------------------------------------|-----------------|
// | | |
// | stopHttpServerCh (AuditBackend::Shutdown())
// | |
// | listenerStoppedCh
// | |
// | HTTPServerStoppedListening (httpServerStoppedListeningCh)
func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
delayedStopCh := s.lifecycleSignals.AfterShutdownDelayDuration
shutdownInitiatedCh := s.lifecycleSignals.ShutdownInitiated
@@ -666,7 +670,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
}
apiGroupVersion.OpenAPIModels = openAPIModels
if openAPIModels != nil && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
if openAPIModels != nil {
typeConverter, err := fieldmanager.NewTypeConverter(openAPIModels, false)
if err != nil {
return err
@@ -676,11 +680,35 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
apiGroupVersion.MaxRequestBodyBytes = s.maxRequestBodyBytes
r, err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer)
discoveryAPIResources, r, err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer)
if err != nil {
return fmt.Errorf("unable to setup API %v: %v", apiGroupInfo, err)
}
resourceInfos = append(resourceInfos, r...)
if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
// Aggregated discovery only aggregates resources under /apis
if apiPrefix == APIGroupPrefix {
s.AggregatedDiscoveryGroupManager.AddGroupVersion(
groupVersion.Group,
apidiscoveryv2beta1.APIVersionDiscovery{
Version: groupVersion.Version,
Resources: discoveryAPIResources,
},
)
} else {
// There is only one group version for legacy resources, priority can be defaulted to 0.
s.AggregatedLegacyDiscoveryGroupManager.AddGroupVersion(
groupVersion.Group,
apidiscoveryv2beta1.APIVersionDiscovery{
Version: groupVersion.Version,
Resources: discoveryAPIResources,
},
)
}
}
}
s.RegisterDestroyFunc(apiGroupInfo.destroyStorage)
@@ -715,7 +743,13 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo
// Install the version handler.
// Add a handler at /<apiPrefix> to enumerate the supported api versions.
s.Handler.GoRestfulContainer.Add(discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix).WebService())
legacyRootAPIHandler := discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix)
if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(legacyRootAPIHandler, s.AggregatedLegacyDiscoveryGroupManager)
s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/api", metav1.APIVersions{}))
} else {
s.Handler.GoRestfulContainer.Add(legacyRootAPIHandler.WebService())
}
return nil
}

View File

@@ -18,6 +18,7 @@ package healthz
import (
"bytes"
"context"
"fmt"
"net/http"
"reflect"
@@ -30,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/apiserver/pkg/server/httplog"
"k8s.io/component-base/metrics/prometheus/slis"
"k8s.io/klog/v2"
)
@@ -237,6 +239,7 @@ func handleRootHealth(name string, firstTimeHealthy func(), checks ...HealthChec
continue
}
if err := check.Check(r); err != nil {
slis.ObserveHealthcheck(context.Background(), check.Name(), name, slis.Error)
// don't include the error since this endpoint is public. If someone wants more detail
// they should have explicit permission to the detailed checks.
fmt.Fprintf(&individualCheckOutput, "[-]%s failed: reason withheld\n", check.Name())
@@ -244,12 +247,13 @@ func handleRootHealth(name string, firstTimeHealthy func(), checks ...HealthChec
fmt.Fprintf(&failedVerboseLogOutput, "[-]%s failed: %v\n", check.Name(), err)
failedChecks = append(failedChecks, check.Name())
} else {
slis.ObserveHealthcheck(context.Background(), check.Name(), name, slis.Success)
fmt.Fprintf(&individualCheckOutput, "[+]%s ok\n", check.Name())
}
}
if excluded.Len() > 0 {
fmt.Fprintf(&individualCheckOutput, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(excluded.List()...))
klog.Warningf("cannot exclude some health checks, no health checks are installed matching %s",
klog.V(6).Infof("cannot exclude some health checks, no health checks are installed matching %s",
formatQuoted(excluded.List()...))
}
// always be verbose on failure

Some files were not shown because too many files have changed in this diff Show More