admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: kubesphere 4.0 (#6115)

Browse Source 
* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
vendor/k8s.io/apiserver/pkg/server/options/OWNERS generated vendored
View File

@@ -1,5 +1,7 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- enj
reviewers:
- smarterclayton
- wojtek-t

15
vendor/k8s.io/apiserver/pkg/server/options/admission.go generated vendored
View File

@@ -39,7 +39,6 @@ import (
"k8s.io/client-go/dynamic"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/component-base/featuregate"
)
@@ -123,7 +122,8 @@ func (a *AdmissionOptions) AddFlags(fs *pflag.FlagSet) {
func (a *AdmissionOptions) ApplyTo(
c *server.Config,
informers informers.SharedInformerFactory,
kubeAPIServerClientConfig *rest.Config,
kubeClient kubernetes.Interface,
dynamicClient dynamic.Interface,
features featuregate.FeatureGate,
pluginInitializers ...admission.PluginInitializer,
) error {
@@ -143,15 +143,8 @@ func (a *AdmissionOptions) ApplyTo(
return fmt.Errorf("failed to read plugin config: %v", err)
}
clientset, err := kubernetes.NewForConfig(kubeAPIServerClientConfig)
if err != nil {
return err
}
dynamicClient, err := dynamic.NewForConfig(kubeAPIServerClientConfig)
if err != nil {
return err
}
genericInitializer := initializer.New(clientset, dynamicClient, informers, c.Authorization.Authorizer, features, c.DrainedNotify())
genericInitializer := initializer.New(kubeClient, dynamicClient, informers, c.Authorization.Authorizer, features,
c.DrainedNotify())
initializersChain := admission.PluginInitializers{genericInitializer}
initializersChain = append(initializersChain, pluginInitializers...)

4
vendor/k8s.io/apiserver/pkg/server/options/api_enablement.go generated vendored
View File

@@ -42,6 +42,9 @@ func NewAPIEnablementOptions() *APIEnablementOptions {
// AddFlags adds flags for a specific APIServer to the specified FlagSet
func (s *APIEnablementOptions) AddFlags(fs *pflag.FlagSet) {
if s == nil {
return
}
fs.Var(&s.RuntimeConfig, "runtime-config", ""+
"A set of key=value pairs that enable or disable built-in APIs. Supported options are:\n"+
"v1=true|false for the core API group\n"+
@@ -87,7 +90,6 @@ func (s *APIEnablementOptions) Validate(registries ...GroupRegistry) []error {
// ApplyTo override MergedResourceConfig with defaults and registry
func (s *APIEnablementOptions) ApplyTo(c *server.Config, defaultResourceConfig *serverstore.ResourceConfig, registry resourceconfig.GroupVersionRegistry) error {
if s == nil {
return nil
}

10
vendor/k8s.io/apiserver/pkg/server/options/audit.go generated vendored
View File

@@ -142,16 +142,6 @@ type AuditWebhookOptions struct {
GroupVersionString string
}
// AuditDynamicOptions control the configuration of dynamic backends for audit events
type AuditDynamicOptions struct {
// Enabled tells whether the dynamic audit capability is enabled.
Enabled bool
// Configuration for batching backend. This is currently only used as an override
// for integration tests
BatchConfig *pluginbuffered.BatchConfig
}
func NewAuditOptions() *AuditOptions {
return &AuditOptions{
WebhookOptions: AuditWebhookOptions{

20
vendor/k8s.io/apiserver/pkg/server/options/authentication.go generated vendored
View File

@@ -76,6 +76,16 @@ func (s *RequestHeaderAuthenticationOptions) Validate() []error {
allErrors = append(allErrors, err)
}
if len(s.UsernameHeaders) > 0 && !caseInsensitiveHas(s.UsernameHeaders, "X-Remote-User") {
klog.Warningf("--requestheader-username-headers is set without specifying the standard X-Remote-User header - API aggregation will not work")
}
if len(s.GroupHeaders) > 0 && !caseInsensitiveHas(s.GroupHeaders, "X-Remote-Group") {
klog.Warningf("--requestheader-group-headers is set without specifying the standard X-Remote-Group header - API aggregation will not work")
}
if len(s.ExtraHeaderPrefixes) > 0 && !caseInsensitiveHas(s.ExtraHeaderPrefixes, "X-Remote-Extra-") {
klog.Warningf("--requestheader-extra-headers-prefix is set without specifying the standard X-Remote-Extra- header prefix - API aggregation will not work")
}
return allErrors
}
@@ -89,6 +99,15 @@ func checkForWhiteSpaceOnly(flag string, headerNames ...string) error {
return nil
}
func caseInsensitiveHas(headers []string, header string) bool {
for _, h := range headers {
if strings.EqualFold(h, header) {
return true
}
}
return false
}
func (s *RequestHeaderAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
if s == nil {
return
@@ -357,6 +376,7 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.Aut
}
if requestHeaderConfig != nil {
cfg.RequestHeaderConfig = requestHeaderConfig
authenticationInfo.RequestHeaderConfig = requestHeaderConfig
if err = authenticationInfo.ApplyClientCert(cfg.RequestHeaderConfig.CAContentProvider, servingInfo); err != nil {
return fmt.Errorf("unable to load request-header-client-ca-file: %v", err)
}

5
vendor/k8s.io/apiserver/pkg/server/options/authorization.go generated vendored
View File

@@ -38,9 +38,8 @@ import (
// DelegatingAuthorizationOptions provides an easy way for composing API servers to delegate their authorization to
// the root kube API server.
// WARNING: never assume that every authenticated incoming request already does authorization.
//
// The aggregator in the kube API server does this today, but this behaviour is not
// guaranteed in the future.
// The aggregator in the kube API server does this today, but this behaviour is not
// guaranteed in the future.
type DelegatingAuthorizationOptions struct {
// RemoteKubeConfigFile is the file to use to connect to a "normal" kube API server which hosts the
// SubjectAccessReview.authorization.k8s.io endpoint for checking tokens.

4
vendor/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go generated vendored
View File

@@ -67,7 +67,7 @@ func (s *DeprecatedInsecureServingOptions) AddFlags(fs *pflag.FlagSet) {
}
fs.IPVar(&s.BindAddress, "insecure-bind-address", s.BindAddress, ""+
"The IP address on which to serve the --insecure-port (set to 0.0.0.0 or :: for listening in all interfaces and IP families).")
"The IP address on which to serve the --insecure-port (set to 0.0.0.0 or :: for listening on all interfaces and IP address families).")
// Though this flag is deprecated, we discovered security concerns over how to do health checks without it e.g. #43784
fs.MarkDeprecated("insecure-bind-address", "This flag will be removed in a future version.")
fs.Lookup("insecure-bind-address").Hidden = false
@@ -86,7 +86,7 @@ func (s *DeprecatedInsecureServingOptions) AddUnqualifiedFlags(fs *pflag.FlagSet
}
fs.IPVar(&s.BindAddress, "address", s.BindAddress,
"The IP address on which to serve the insecure --port (set to '0.0.0.0' or '::' for listening in all interfaces and IP families).")
"The IP address on which to serve the insecure --port (set to '0.0.0.0' or '::' for listening on all interfaces and IP address families).")
fs.MarkDeprecated("address", "see --bind-address instead.")
fs.Lookup("address").Hidden = false

564
vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go generated vendored
View File

@@ -36,19 +36,25 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
apiserverconfig "k8s.io/apiserver/pkg/apis/config"
apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
"k8s.io/apiserver/pkg/apis/config/validation"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/storage/value"
"k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
storagevalue "k8s.io/apiserver/pkg/storage/value"
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
envelopemetrics "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
"k8s.io/apiserver/pkg/storage/value/encrypt/identity"
"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
kmsservice "k8s.io/kms/pkg/service"
)
const (
@@ -57,13 +63,80 @@ const (
secretboxTransformerPrefixV1 = "k8s:enc:secretbox:v1:"
kmsTransformerPrefixV1 = "k8s:enc:kms:v1:"
kmsTransformerPrefixV2 = "k8s:enc:kms:v2:"
kmsPluginHealthzNegativeTTL = 3 * time.Second
kmsPluginHealthzPositiveTTL = 20 * time.Second
kmsAPIVersionV1 = "v1"
kmsAPIVersionV2 = "v2"
kmsReloadHealthCheckName = "kms-providers"
// these constants relate to how the KMS v2 plugin status poll logic
// and the DEK/seed generation logic behave. In particular, the positive
// interval and max TTL are closely related as the difference between
// these values defines the worst case window in which the write DEK/seed
// could expire due to the plugin going into an error state. The
// worst case window divided by the negative interval defines the
// minimum amount of times the server will attempt to return to a
// healthy state before the DEK/seed expires and writes begin to fail.
//
// For now, these values are kept small and hardcoded to support being
// able to perform a "passive" storage migration while tolerating some
// amount of plugin downtime.
//
// With the current approach, a user can update the key ID their plugin
// is using and then can simply schedule a migration for 3 + N + M minutes
// later where N is how long it takes their plugin to pick up new config
// and M is extra buffer to allow the API server to process the config.
// At that point, they are guaranteed to either migrate to the new key
// or get errors during the migration.
//
// If the API server coasted forever on the last DEK/seed, they would need
// to actively check if it had observed the new key ID before starting
// a migration - otherwise it could keep using the old DEK/seed and their
// storage migration would not do what they thought it did.
kmsv2PluginHealthzPositiveInterval = 1 * time.Minute
kmsv2PluginHealthzNegativeInterval = 10 * time.Second
kmsv2PluginWriteDEKSourceMaxTTL = 3 * time.Minute
kmsPluginHealthzNegativeTTL = 3 * time.Second
kmsPluginHealthzPositiveTTL = 20 * time.Second
kmsAPIVersionV1 = "v1"
kmsAPIVersionV2 = "v2"
// this name is used for two different healthz endpoints:
// - when one or more KMS v2 plugins are in use and no KMS v1 plugins are in use
// in this case, all v2 plugins are probed via this single endpoint
// - when automatic reload of encryption config is enabled
// in this case, all KMS plugins are probed via this single endpoint
// the endpoint is present even if there are no KMS plugins configured (it is a no-op then)
kmsReloadHealthCheckName = "kms-providers"
)
var codecs serializer.CodecFactory
// this atomic bool allows us to swap enablement of the KMSv2KDF feature in tests
// as the feature gate is now locked to true starting with v1.29
// Note: it cannot be set by an end user
var kdfDisabled atomic.Bool
// this function should only be called in tests to swap enablement of the KMSv2KDF feature
func SetKDFForTests(b bool) func() {
kdfDisabled.Store(!b)
return func() {
kdfDisabled.Store(false)
}
}
// this function should be used to determine enablement of the KMSv2KDF feature
// instead of getting it from DefaultFeatureGate as the feature gate is now locked
// to true starting with v1.29
func GetKDF() bool {
return !kdfDisabled.Load()
}
func init() {
configScheme := runtime.NewScheme()
utilruntime.Must(apiserverconfig.AddToScheme(configScheme))
utilruntime.Must(apiserverconfigv1.AddToScheme(configScheme))
codecs = serializer.NewCodecFactory(configScheme)
envelopemetrics.RegisterMetrics()
storagevalue.RegisterMetrics()
metrics.RegisterMetrics()
}
type kmsPluginHealthzResponse struct {
err error
received time.Time
@@ -78,11 +151,14 @@ type kmsPluginProbe struct {
}
type kmsv2PluginProbe struct {
state atomic.Pointer[envelopekmsv2.State]
name string
ttl time.Duration
service envelopekmsv2.Service
service kmsservice.Service
lastResponse *kmsPluginHealthzResponse
l *sync.Mutex
apiServerID string
version string
}
type kmsHealthChecker []healthz.HealthChecker
@@ -119,7 +195,7 @@ func (h *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
// EncryptionConfiguration represents the parsed and normalized encryption configuration for the apiserver.
type EncryptionConfiguration struct {
// Transformers is a list of value.Transformer that will be used to encrypt and decrypt data.
Transformers map[schema.GroupResource]value.Transformer
Transformers map[schema.GroupResource]storagevalue.Transformer
// HealthChecks is a list of healthz.HealthChecker that will be used to check the health of the encryption providers.
HealthChecks []healthz.HealthChecker
@@ -133,15 +209,16 @@ type EncryptionConfiguration struct {
}
// LoadEncryptionConfig parses and validates the encryption config specified by filepath.
// It may launch multiple go routines whose lifecycle is controlled by stopCh.
// It may launch multiple go routines whose lifecycle is controlled by ctx.
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
// If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1.
func LoadEncryptionConfig(filepath string, reload bool, stopCh <-chan struct{}) (*EncryptionConfiguration, error) {
func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool, apiServerID string) (*EncryptionConfiguration, error) {
config, contentHash, err := loadConfig(filepath, reload)
if err != nil {
return nil, fmt.Errorf("error while parsing file: %w", err)
}
transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh)
transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config, apiServerID)
if err != nil {
return nil, fmt.Errorf("error while building transformers: %w", err)
}
@@ -150,7 +227,7 @@ func LoadEncryptionConfig(filepath string, reload bool, stopCh <-chan struct{})
kmsHealthChecks = []healthz.HealthChecker{kmsHealthChecker(kmsHealthChecks)}
}
// KMSTimeout is the duration we will wait before closing old transformers.
// KMSCloseGracePeriod is the duration we will wait before closing old transformers.
// The way we calculate is as follows:
// 1. Sum all timeouts across all KMS plugins. (check kmsPrefixTransformer for differences between v1 and v2)
// 2. Multiply that by 2 (to allow for some buffer)
@@ -160,12 +237,15 @@ func LoadEncryptionConfig(filepath string, reload bool, stopCh <-chan struct{})
HealthChecks: kmsHealthChecks,
EncryptionFileContentHash: contentHash,
KMSCloseGracePeriod: 2 * kmsUsed.kmsTimeoutSum,
}, err
}, nil
}
func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, *kmsState, error) {
// getTransformerOverridesAndKMSPluginHealthzCheckers creates the set of transformers and KMS healthz checks based on the given config.
// It may launch multiple go routines whose lifecycle is controlled by ctx.
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
var kmsHealthChecks []healthz.HealthChecker
transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh)
transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, apiServerID)
if err != nil {
return nil, nil, nil, err
}
@@ -181,8 +261,11 @@ type healthChecker interface {
toHealthzCheck(idx int) healthz.HealthChecker
}
func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, *kmsState, error) {
resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{}
// getTransformerOverridesAndKMSPluginProbes creates the set of transformers and KMS probes based on the given config.
// It may launch multiple go routines whose lifecycle is controlled by ctx.
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
resourceToPrefixTransformer := map[schema.GroupResource][]storagevalue.PrefixTransformer{}
var probes []healthChecker
var kmsUsed kmsState
@@ -190,31 +273,42 @@ func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.Encryptio
for _, resourceConfig := range config.Resources {
resourceConfig := resourceConfig
transformers, p, used, err := prefixTransformersAndProbes(resourceConfig, stopCh)
transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig, apiServerID)
if err != nil {
return nil, nil, nil, err
}
kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used
kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used
kmsUsed.kmsTimeoutSum += used.kmsTimeoutSum
kmsUsed.accumulate(used)
// For each resource, create a list of providers to use
for _, resource := range resourceConfig.Resources {
resource := resource
gr := schema.ParseGroupResource(resource)
resourceToPrefixTransformer[gr] = append(
resourceToPrefixTransformer[gr], transformers...)
// check if resource is masked by *.group rule
anyResourceInGroup := schema.GroupResource{Group: gr.Group, Resource: "*"}
if _, masked := resourceToPrefixTransformer[anyResourceInGroup]; masked {
// an earlier rule already configured a transformer for *.group, masking this rule
// return error since this is not allowed
return nil, nil, nil, fmt.Errorf("resource %q is masked by earlier rule %q", grYAMLString(gr), grYAMLString(anyResourceInGroup))
}
if _, masked := resourceToPrefixTransformer[anyGroupAnyResource]; masked {
// an earlier rule already configured a transformer for *.*, masking this rule
// return error since this is not allowed
return nil, nil, nil, fmt.Errorf("resource %q is masked by earlier rule %q", grYAMLString(gr), grYAMLString(anyGroupAnyResource))
}
resourceToPrefixTransformer[gr] = append(resourceToPrefixTransformer[gr], transformers...)
}
probes = append(probes, p...)
}
transformers := make(map[schema.GroupResource]value.Transformer, len(resourceToPrefixTransformer))
transformers := make(map[schema.GroupResource]storagevalue.Transformer, len(resourceToPrefixTransformer))
for gr, transList := range resourceToPrefixTransformer {
gr := gr
transList := transList
transformers[gr] = value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)
transformers[gr] = storagevalue.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)
}
return transformers, probes, &kmsUsed, nil
@@ -252,7 +346,7 @@ func (h *kmsv2PluginProbe) check(ctx context.Context) error {
h.l.Lock()
defer h.l.Unlock()
if (time.Since(h.lastResponse.received)) < h.ttl {
if time.Since(h.lastResponse.received) < h.ttl {
return h.lastResponse.err
}
@@ -263,7 +357,7 @@ func (h *kmsv2PluginProbe) check(ctx context.Context) error {
return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %w", h.name, err)
}
if err := isKMSv2ProviderHealthy(h.name, p); err != nil {
if err := h.isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx, p); err != nil {
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
h.ttl = kmsPluginHealthzNegativeTTL
return err
@@ -274,27 +368,160 @@ func (h *kmsv2PluginProbe) check(ctx context.Context) error {
return nil
}
// isKMSv2ProviderHealthy checks if the KMSv2-Plugin is healthy.
func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse) error {
// rotateDEKOnKeyIDChange tries to rotate to a new DEK/seed if the key ID returned by Status does not match the
// current state. If a successful rotation is performed, the new DEK/seed and keyID overwrite the existing state.
// On any failure during rotation (including mismatch between status and encrypt calls), the current state is
// preserved and will remain valid to use for encryption until its expiration (the system attempts to coast).
// If the key ID returned by Status matches the current state, the expiration of the current state is extended
// and no rotation is performed.
func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKeyID, uid string) error {
// we do not check ValidateEncryptCapability here because it is fine to re-use an old key
// that was marked as expired during an unhealthy period. As long as the key ID matches
// what we expect then there is no need to rotate here.
state, errState := h.getCurrentState()
// allow reads indefinitely in all cases
// allow writes indefinitely as long as there is no error
// allow writes for only up to kmsv2PluginWriteDEKSourceMaxTTL from now when there are errors
// we start the timer before we make the network call because kmsv2PluginWriteDEKSourceMaxTTL is meant to be the upper bound
expirationTimestamp := envelopekmsv2.NowFunc().Add(kmsv2PluginWriteDEKSourceMaxTTL)
// dynamically check if we want to use KDF seed to derive DEKs or just a single DEK
// this gate can only change during tests, but the check is cheap enough to always make
// this allows us to easily exercise both modes without restarting the API server
// TODO integration test that this dynamically takes effect
useSeed := GetKDF()
stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
// state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it
// just move the expiration of the current state forward by the reuse interval
// useSeed can only change at runtime during tests, so we check it here to allow us to easily exercise both modes
if errState == nil && state.EncryptedObject.KeyID == statusKeyID && stateUseSeed == useSeed {
state.ExpirationTimestamp = expirationTimestamp
h.state.Store(&state)
return nil
}
transformer, encObject, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service, useSeed)
if encObject == nil {
encObject = &kmstypes.EncryptedObject{} // avoid nil panics
}
// happy path, should be the common case
// TODO maybe add success metrics?
if errGen == nil && encObject.KeyID == statusKeyID {
h.state.Store(&envelopekmsv2.State{
Transformer: transformer,
EncryptedObject: *encObject,
UID: uid,
ExpirationTimestamp: expirationTimestamp,
CacheKey: cacheKey,
})
// it should be logically impossible for the new state to be invalid but check just in case
_, errGen = h.getCurrentState()
if errGen == nil {
klogV6 := klog.V(6)
if klogV6.Enabled() {
klogV6.InfoS("successfully rotated DEK",
"uid", uid,
"useSeed", useSeed,
"newKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID),
"oldKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID),
"expirationTimestamp", expirationTimestamp.Format(time.RFC3339),
)
}
return nil
}
}
return fmt.Errorf("failed to rotate DEK uid=%q, useSeed=%v, errState=%v, errGen=%v, statusKeyIDHash=%q, encryptKeyIDHash=%q, stateKeyIDHash=%q, expirationTimestamp=%s",
uid, useSeed, errState, errGen, envelopekmsv2.GetHashIfNotEmpty(statusKeyID), envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID), envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID), state.ExpirationTimestamp.Format(time.RFC3339))
}
// getCurrentState returns the latest state from the last status and encrypt calls.
// If the returned error is nil, the state is considered valid indefinitely for read requests.
// For write requests, the caller must also check that state.ValidateEncryptCapability does not error.
func (h *kmsv2PluginProbe) getCurrentState() (envelopekmsv2.State, error) {
state := *h.state.Load()
if state.Transformer == nil {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected nil transformer")
}
encryptedObjectCopy := state.EncryptedObject
if len(encryptedObjectCopy.EncryptedData) != 0 {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected non-empty EncryptedData")
}
encryptedObjectCopy.EncryptedData = []byte{0} // any non-empty value to pass validation
if err := envelopekmsv2.ValidateEncryptedObject(&encryptedObjectCopy); err != nil {
return envelopekmsv2.State{}, fmt.Errorf("got invalid EncryptedObject: %w", err)
}
if state.ExpirationTimestamp.IsZero() {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected zero expirationTimestamp")
}
if len(state.CacheKey) == 0 {
return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty cacheKey")
}
return state, nil
}
func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.Context, response *kmsservice.StatusResponse) error {
var errs []error
if response.Healthz != "ok" {
errs = append(errs, fmt.Errorf("got unexpected healthz status: %s", response.Healthz))
}
if response.Version != envelopekmsv2.KMSAPIVersion {
errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersion, response.Version))
if response.Version != envelopekmsv2.KMSAPIVersionv2 && response.Version != envelopekmsv2.KMSAPIVersionv2beta1 {
errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersionv2, response.Version))
} else {
// set version for the first status response
if len(h.version) == 0 {
h.version = response.Version
}
if h.version != response.Version {
errs = append(errs, fmt.Errorf("KMSv2 API version should not change after the initial status response version %s, got %s", h.version, response.Version))
}
}
if len(response.KeyID) == 0 {
errs = append(errs, fmt.Errorf("expected KMSv2 KeyID to be set, got %s", response.KeyID))
if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil {
envelopemetrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode))
errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID hash %q: %w", envelopekmsv2.GetHashIfNotEmpty(response.KeyID), err))
} else {
envelopemetrics.RecordKeyIDFromStatus(h.name, response.KeyID, h.apiServerID)
// unconditionally append as we filter out nil errors below
errs = append(errs, h.rotateDEKOnKeyIDChange(ctx, response.KeyID, string(uuid.NewUUID())))
}
if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil {
return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %w", name, err)
return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %w", h.name, err)
}
return nil
}
// loadConfig parses the encryption configuration file at filepath and returns the parsed config and hash of the file.
func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfiguration, string, error) {
data, contentHash, err := loadDataAndHash(filepath)
if err != nil {
return nil, "", fmt.Errorf("error while loading file: %w", err)
}
configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil)
if err != nil {
return nil, "", fmt.Errorf("error decoding encryption provider configuration file %q: %w", filepath, err)
}
config, ok := configObj.(*apiserverconfig.EncryptionConfiguration)
if !ok {
return nil, "", fmt.Errorf("got unexpected config type: %v", gvk)
}
return config, contentHash, validation.ValidateEncryptionConfiguration(config, reload).ToAggregate()
}
func loadDataAndHash(filepath string) ([]byte, string, error) {
f, err := os.Open(filepath)
if err != nil {
return nil, "", fmt.Errorf("error opening encryption provider configuration file %q: %w", filepath, err)
@@ -309,32 +536,28 @@ func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfig
return nil, "", fmt.Errorf("encryption provider configuration file %q is empty", filepath)
}
scheme := runtime.NewScheme()
codecs := serializer.NewCodecFactory(scheme)
utilruntime.Must(apiserverconfig.AddToScheme(scheme))
utilruntime.Must(apiserverconfigv1.AddToScheme(scheme))
configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil)
if err != nil {
return nil, "", err
}
config, ok := configObj.(*apiserverconfig.EncryptionConfiguration)
if !ok {
return nil, "", fmt.Errorf("got unexpected config type: %v", gvk)
}
return config, computeEncryptionConfigHash(data), validation.ValidateEncryptionConfiguration(config, reload).ToAggregate()
return data, computeEncryptionConfigHash(data), nil
}
func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, *kmsState, error) {
var transformers []value.PrefixTransformer
// GetEncryptionConfigHash reads the encryption configuration file at filepath and returns the hash of the file.
// It does not attempt to decode or load the config, and serves as a cheap check to determine if the file has changed.
func GetEncryptionConfigHash(filepath string) (string, error) {
_, contentHash, err := loadDataAndHash(filepath)
return contentHash, err
}
// prefixTransformersAndProbes creates the set of transformers and KMS probes based on the given resource config.
// It may launch multiple go routines whose lifecycle is controlled by ctx.
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration, apiServerID string) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
var transformers []storagevalue.PrefixTransformer
var probes []healthChecker
var kmsUsed kmsState
for _, provider := range config.Providers {
provider := provider
var (
transformer value.PrefixTransformer
transformer storagevalue.PrefixTransformer
transformerErr error
probe healthChecker
used *kmsState
@@ -345,24 +568,23 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s
transformer, transformerErr = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
case provider.AESCBC != nil:
transformer, transformerErr = aesPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1)
cbcTransformer := func(block cipher.Block) (storagevalue.Transformer, error) {
return aestransformer.NewCBCTransformer(block), nil
}
transformer, transformerErr = aesPrefixTransformer(provider.AESCBC, cbcTransformer, aesCBCTransformerPrefixV1)
case provider.Secretbox != nil:
transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox)
case provider.KMS != nil:
transformer, probe, used, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh)
transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS, apiServerID)
if transformerErr == nil {
probes = append(probes, probe)
kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used
kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used
// calculate the maximum timeout for all KMS providers
kmsUsed.kmsTimeoutSum += used.kmsTimeoutSum
kmsUsed.accumulate(used)
}
case provider.Identity != nil:
transformer = value.PrefixTransformer{
transformer = storagevalue.PrefixTransformer{
Transformer: identity.NewEncryptCheckTransformer(),
Prefix: []byte{},
}
@@ -381,10 +603,10 @@ func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, s
return transformers, probes, &kmsUsed, nil
}
type blockTransformerFunc func(cipher.Block) value.Transformer
type blockTransformerFunc func(cipher.Block) (storagevalue.Transformer, error)
func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTransformerFunc, prefix string) (value.PrefixTransformer, error) {
var result value.PrefixTransformer
func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTransformerFunc, prefix string) (storagevalue.PrefixTransformer, error) {
var result storagevalue.PrefixTransformer
if len(config.Keys) == 0 {
return result, fmt.Errorf("aes provider has no valid keys")
@@ -399,41 +621,45 @@ func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTran
}
}
keyTransformers := []value.PrefixTransformer{}
keyTransformers := []storagevalue.PrefixTransformer{}
for _, keyData := range config.Keys {
keyData := keyData
key, err := base64.StdEncoding.DecodeString(keyData.Secret)
if err != nil {
return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err)
return result, fmt.Errorf("could not obtain secret for named key %s: %w", keyData.Name, err)
}
block, err := aes.NewCipher(key)
if err != nil {
return result, fmt.Errorf("error while creating cipher for named key %s: %s", keyData.Name, err)
return result, fmt.Errorf("error while creating cipher for named key %s: %w", keyData.Name, err)
}
transformer, err := fn(block)
if err != nil {
return result, fmt.Errorf("error while creating transformer for named key %s: %w", keyData.Name, err)
}
// Create a new PrefixTransformer for this key
keyTransformers = append(keyTransformers,
value.PrefixTransformer{
Transformer: fn(block),
storagevalue.PrefixTransformer{
Transformer: transformer,
Prefix: []byte(keyData.Name + ":"),
})
}
// Create a prefixTransformer which can choose between these keys
keyTransformer := value.NewPrefixTransformers(
keyTransformer := storagevalue.NewPrefixTransformers(
fmt.Errorf("no matching key was found for the provided AES transformer"), keyTransformers...)
// Create a PrefixTransformer which shall later be put in a list with other providers
result = value.PrefixTransformer{
result = storagevalue.PrefixTransformer{
Transformer: keyTransformer,
Prefix: []byte(prefix),
}
return result, nil
}
func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (value.PrefixTransformer, error) {
var result value.PrefixTransformer
func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (storagevalue.PrefixTransformer, error) {
var result storagevalue.PrefixTransformer
if len(config.Keys) == 0 {
return result, fmt.Errorf("secretbox provider has no valid keys")
@@ -448,7 +674,7 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
}
}
keyTransformers := []value.PrefixTransformer{}
keyTransformers := []storagevalue.PrefixTransformer{}
for _, keyData := range config.Keys {
keyData := keyData
@@ -466,18 +692,18 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
// Create a new PrefixTransformer for this key
keyTransformers = append(keyTransformers,
value.PrefixTransformer{
storagevalue.PrefixTransformer{
Transformer: secretbox.NewSecretboxTransformer(keyArray),
Prefix: []byte(keyData.Name + ":"),
})
}
// Create a prefixTransformer which can choose between these keys
keyTransformer := value.NewPrefixTransformers(
keyTransformer := storagevalue.NewPrefixTransformers(
fmt.Errorf("no matching key was found for the provided Secretbox transformer"), keyTransformers...)
// Create a PrefixTransformer which shall later be put in a list with other providers
result = value.PrefixTransformer{
result = storagevalue.PrefixTransformer{
Transformer: keyTransformer,
Prefix: []byte(secretboxTransformerPrefixV1),
}
@@ -497,16 +723,31 @@ type kmsState struct {
kmsTimeoutSum time.Duration
}
func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, *kmsState, error) {
// we ignore the cancel func because this context should only be canceled when stopCh is closed
ctx, _ := wait.ContextForChannel(stopCh)
// accumulate computes the KMS state by:
// - determining which KMS plugin versions are in use
// - calculating kmsTimeoutSum which is used as transformTracker.kmsCloseGracePeriod
// DynamicTransformers.Set waits for this period before closing old transformers after a config reload
func (s *kmsState) accumulate(other *kmsState) {
s.v1Used = s.v1Used || other.v1Used
s.v2Used = s.v2Used || other.v2Used
s.kmsTimeoutSum += other.kmsTimeoutSum
}
// kmsPrefixTransformer creates a KMS transformer and probe based on the given KMS config.
// It may launch multiple go routines whose lifecycle is controlled by ctx.
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration, apiServerID string) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
kmsName := config.Name
switch config.APIVersion {
case kmsAPIVersionV1:
if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv1) {
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead. Set --feature-gates=KMSv1=true to use the deprecated KMSv1 feature.")
}
klog.InfoS("KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead.")
envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
if err != nil {
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err)
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err)
}
probe := &kmsPluginProbe{
@@ -527,12 +768,12 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha
case kmsAPIVersionV2:
if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
}
envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Name, config.Timeout.Duration)
if err != nil {
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err)
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err)
}
probe := &kmsv2PluginProbe{
@@ -541,11 +782,14 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha
service: envelopeService,
l: &sync.Mutex{},
lastResponse: &kmsPluginHealthzResponse{},
apiServerID: apiServerID,
}
// initialize state so that Load always works
probe.state.Store(&envelopekmsv2.State{})
// using AES-GCM by default for encrypting data with KMSv2
transformer := value.PrefixTransformer{
Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), aestransformer.NewGCMTransformer),
primeAndProbeKMSv2(ctx, probe, kmsName)
transformer := storagevalue.PrefixTransformer{
Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState, apiServerID),
Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"),
}
@@ -555,28 +799,83 @@ func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-cha
}, nil
default:
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
}
}
func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) value.PrefixTransformer {
baseTransformerFunc := func(block cipher.Block) value.Transformer {
func primeAndProbeKMSv2(ctx context.Context, probe *kmsv2PluginProbe, kmsName string) {
runProbeCheckAndLog := func(ctx context.Context, depth int) error {
if err := probe.check(ctx); err != nil {
klog.VDepth(1+depth, 2).ErrorS(err, "kms plugin failed health check probe", "name", kmsName)
return err
}
return nil
}
blockAndProbeFastUntilSuccess := func(ctx context.Context) {
_ = wait.PollUntilWithContext(
ctx,
kmsv2PluginHealthzNegativeInterval,
func(ctx context.Context) (bool, error) {
return runProbeCheckAndLog(ctx, 1) == nil, nil
},
)
}
// on the happy path where the plugin is healthy and available on server start,
// prime keyID and DEK by running the check inline once (this also prevents unit tests from flaking)
errPrime := runProbeCheckAndLog(ctx, 0)
// if our initial attempt to prime failed, start trying to get to a valid state in the background ASAP
// this prevents a slow start when the external healthz checker is configured to ignore the KMS healthz endpoint
// since we want to support the plugin starting up async with the API server, this error is not fatal
if errPrime != nil {
go blockAndProbeFastUntilSuccess(ctx) // separate go routine to avoid blocking
}
// make sure that the plugin's key ID is reasonably up-to-date
// also, make sure that our DEK is up-to-date to with said key ID (if it expires the server will fail all writes)
// if this background loop ever stops running, the server will become unfunctional after kmsv2PluginWriteDEKSourceMaxTTL
go wait.PollUntilWithContext(
ctx,
kmsv2PluginHealthzPositiveInterval,
func(ctx context.Context) (bool, error) {
if err := runProbeCheckAndLog(ctx, 0); err == nil {
return false, nil
}
// TODO add integration test for quicker error poll on failure
// if we fail, block the outer polling and start a new quicker poll inline
// this limits the chance that our DEK expires during a transient failure
blockAndProbeFastUntilSuccess(ctx)
return false, nil
})
}
func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) storagevalue.PrefixTransformer {
baseTransformerFunc := func(block cipher.Block) (storagevalue.Transformer, error) {
gcm, err := aestransformer.NewGCMTransformer(block)
if err != nil {
return nil, err
}
// v1.24: write using AES-CBC only but support reads via AES-CBC and AES-GCM (so we can move to AES-GCM)
// v1.25: write using AES-GCM only but support reads via AES-GCM and fallback to AES-CBC for backwards compatibility
// TODO(aramase): Post v1.25: We cannot drop CBC read support until we automate storage migration.
// We could have a release note that hard requires users to perform storage migration.
return unionTransformers{aestransformer.NewGCMTransformer(block), aestransformer.NewCBCTransformer(block)}
return unionTransformers{gcm, aestransformer.NewCBCTransformer(block)}, nil
}
return value.PrefixTransformer{
return storagevalue.PrefixTransformer{
Transformer: envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc),
Prefix: []byte(prefix + config.Name + ":"),
}
}
type unionTransformers []value.Transformer
type unionTransformers []storagevalue.Transformer
func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, stale bool, err error) {
func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) (out []byte, stale bool, err error) {
var errs []error
for i := range u {
transformer := u[i]
@@ -595,7 +894,7 @@ func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte
return nil, false, fmt.Errorf("unionTransformers: unable to transform from storage")
}
func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, err error) {
func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) (out []byte, err error) {
return u[0].TransformToStorage(ctx, data, dataCtx)
}
@@ -606,6 +905,7 @@ func computeEncryptionConfigHash(data []byte) string {
return fmt.Sprintf("%x", sha256.Sum256(data))
}
var _ storagevalue.ResourceTransformers = &DynamicTransformers{}
var _ healthz.HealthChecker = &DynamicTransformers{}
// DynamicTransformers holds transformers that may be dynamically updated via a single external actor, likely a controller.
@@ -615,7 +915,7 @@ type DynamicTransformers struct {
}
type transformTracker struct {
transformerOverrides map[schema.GroupResource]value.Transformer
transformerOverrides map[schema.GroupResource]storagevalue.Transformer
kmsPluginHealthzCheck healthz.HealthChecker
closeTransformers context.CancelFunc
kmsCloseGracePeriod time.Duration
@@ -623,7 +923,7 @@ type transformTracker struct {
// NewDynamicTransformers returns transformers, health checks for kms providers and an ability to close transformers.
func NewDynamicTransformers(
transformerOverrides map[schema.GroupResource]value.Transformer,
transformerOverrides map[schema.GroupResource]storagevalue.Transformer,
kmsPluginHealthzCheck healthz.HealthChecker,
closeTransformers context.CancelFunc,
kmsCloseGracePeriod time.Duration,
@@ -654,7 +954,7 @@ func (d *DynamicTransformers) Name() string {
}
// TransformerForResource returns the transformer for the given resource.
func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResource) value.Transformer {
func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResource) storagevalue.Transformer {
return &resourceTransformer{
resource: resource,
transformTracker: d.transformTracker,
@@ -663,7 +963,7 @@ func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResour
// Set sets the transformer overrides. This method is not go routine safe and must only be called by the same, single caller throughout the lifetime of this object.
func (d *DynamicTransformers) Set(
transformerOverrides map[schema.GroupResource]value.Transformer,
transformerOverrides map[schema.GroupResource]storagevalue.Transformer,
closeTransformers context.CancelFunc,
kmsPluginHealthzCheck healthz.HealthChecker,
kmsCloseGracePeriod time.Duration,
@@ -688,43 +988,61 @@ func (d *DynamicTransformers) Set(
}()
}
var _ value.Transformer = &resourceTransformer{}
var _ storagevalue.Transformer = &resourceTransformer{}
type resourceTransformer struct {
resource schema.GroupResource
transformTracker *atomic.Value
}
func (r *resourceTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
func (r *resourceTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, bool, error) {
return r.transformer().TransformFromStorage(ctx, data, dataCtx)
}
func (r *resourceTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
func (r *resourceTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, error) {
return r.transformer().TransformToStorage(ctx, data, dataCtx)
}
func (r *resourceTransformer) transformer() value.Transformer {
transformer := r.transformTracker.Load().(*transformTracker).transformerOverrides[r.resource]
if transformer == nil {
return identity.NewEncryptCheckTransformer()
func (r *resourceTransformer) transformer() storagevalue.Transformer {
return transformerFromOverrides(r.transformTracker.Load().(*transformTracker).transformerOverrides, r.resource)
}
var _ storagevalue.ResourceTransformers = &StaticTransformers{}
type StaticTransformers map[schema.GroupResource]storagevalue.Transformer
func (s StaticTransformers) TransformerForResource(resource schema.GroupResource) storagevalue.Transformer {
return transformerFromOverrides(s, resource)
}
var anyGroupAnyResource = schema.GroupResource{
Group: "*",
Resource: "*",
}
func transformerFromOverrides(transformerOverrides map[schema.GroupResource]storagevalue.Transformer, resource schema.GroupResource) storagevalue.Transformer {
if transformer := transformerOverrides[resource]; transformer != nil {
return transformer
}
return transformer
}
type ResourceTransformers interface {
TransformerForResource(resource schema.GroupResource) value.Transformer
}
var _ ResourceTransformers = &DynamicTransformers{}
var _ ResourceTransformers = &StaticTransformers{}
type StaticTransformers map[schema.GroupResource]value.Transformer
// StaticTransformers
func (s StaticTransformers) TransformerForResource(resource schema.GroupResource) value.Transformer {
transformer := s[resource]
if transformer == nil {
return identity.NewEncryptCheckTransformer()
if transformer := transformerOverrides[schema.GroupResource{
Group: resource.Group,
Resource: "*",
}]; transformer != nil {
return transformer
}
return transformer
if transformer := transformerOverrides[anyGroupAnyResource]; transformer != nil {
return transformer
}
return identity.NewEncryptCheckTransformer()
}
func grYAMLString(gr schema.GroupResource) string {
if gr.Group == "" && gr.Resource == "*" {
return "*."
}
return gr.String()
}

208
vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go generated vendored
View File

@@ -20,13 +20,14 @@ import (
"context"
"fmt"
"net/http"
"sync"
"time"
"github.com/fsnotify/fsnotify"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
"k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog/v2"
)
@@ -34,8 +35,11 @@ import (
// workqueueKey is the dummy key used to process change in encryption config file.
const workqueueKey = "key"
// DynamicKMSEncryptionConfigContent which can dynamically handle changes in encryption config file.
type DynamicKMSEncryptionConfigContent struct {
// EncryptionConfigFileChangePollDuration is exposed so that integration tests can crank up the reload speed.
var EncryptionConfigFileChangePollDuration = time.Minute
// DynamicEncryptionConfigContent which can dynamically handle changes in encryption config file.
type DynamicEncryptionConfigContent struct {
name string
// filePath is the path of the file to read.
@@ -50,105 +54,90 @@ type DynamicKMSEncryptionConfigContent struct {
// dynamicTransformers updates the transformers when encryption config file changes.
dynamicTransformers *encryptionconfig.DynamicTransformers
// stopCh used here is a lifecycle signal of genericapiserver already drained while shutting down.
stopCh <-chan struct{}
// identity of the api server
apiServerID string
// can be swapped during testing
getEncryptionConfigHash func(ctx context.Context, filepath string) (string, error)
loadEncryptionConfig func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error)
}
// NewDynamicKMSEncryptionConfiguration returns controller that dynamically reacts to changes in encryption config file.
func NewDynamicKMSEncryptionConfiguration(
func init() {
metrics.RegisterMetrics()
}
// NewDynamicEncryptionConfiguration returns controller that dynamically reacts to changes in encryption config file.
func NewDynamicEncryptionConfiguration(
name, filePath string,
dynamicTransformers *encryptionconfig.DynamicTransformers,
configContentHash string,
stopCh <-chan struct{},
) *DynamicKMSEncryptionConfigContent {
encryptionConfig := &DynamicKMSEncryptionConfigContent{
apiServerID string,
) *DynamicEncryptionConfigContent {
return &DynamicEncryptionConfigContent{
name: name,
filePath: filePath,
lastLoadedEncryptionConfigHash: configContentHash,
dynamicTransformers: dynamicTransformers,
stopCh: stopCh,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("%s-hot-reload", name)),
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), name),
apiServerID: apiServerID,
getEncryptionConfigHash: func(_ context.Context, filepath string) (string, error) {
return encryptionconfig.GetEncryptionConfigHash(filepath)
},
loadEncryptionConfig: encryptionconfig.LoadEncryptionConfig,
}
encryptionConfig.queue.Add(workqueueKey)
return encryptionConfig
}
// Run starts the controller and blocks until stopCh is closed.
func (d *DynamicKMSEncryptionConfigContent) Run(ctx context.Context) {
// Run starts the controller and blocks until ctx is canceled.
func (d *DynamicEncryptionConfigContent) Run(ctx context.Context) {
defer utilruntime.HandleCrash()
defer d.queue.ShutDown()
klog.InfoS("Starting controller", "name", d.name)
defer klog.InfoS("Shutting down controller", "name", d.name)
// start worker for processing content
go wait.Until(d.runWorker, time.Second, ctx.Done())
var wg sync.WaitGroup
// start the loop that watches the encryption config file until stopCh is closed.
go wait.Until(func() {
if err := d.watchEncryptionConfigFile(ctx.Done()); err != nil {
// if there is an error while setting up or handling the watches, this will ensure that we will process the config file.
defer d.queue.Add(workqueueKey)
klog.ErrorS(err, "Failed to watch encryption config file, will retry later")
}
}, time.Second, ctx.Done())
wg.Add(1)
go func() {
defer utilruntime.HandleCrash()
defer wg.Done()
defer d.queue.ShutDown()
<-ctx.Done()
}()
<-ctx.Done()
}
wg.Add(1)
go func() {
defer utilruntime.HandleCrash()
defer wg.Done()
d.runWorker(ctx)
}()
func (d *DynamicKMSEncryptionConfigContent) watchEncryptionConfigFile(stopCh <-chan struct{}) error {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return fmt.Errorf("error creating fsnotify watcher: %w", err)
}
defer watcher.Close()
// this function polls changes in the encryption config file by placing a dummy key in the queue.
// the 'runWorker' function then picks up this dummy key and processes the changes.
// the goroutine terminates when 'ctx' is canceled.
_ = wait.PollUntilContextCancel(
ctx,
EncryptionConfigFileChangePollDuration,
true,
func(ctx context.Context) (bool, error) {
// add dummy item to the queue to trigger file content processing.
d.queue.Add(workqueueKey)
if err = watcher.Add(d.filePath); err != nil {
return fmt.Errorf("error adding watch for file %s: %w", d.filePath, err)
}
// return false to continue polling.
return false, nil
},
)
for {
select {
case event := <-watcher.Events:
if err := d.handleWatchEvent(event, watcher); err != nil {
return err
}
case err := <-watcher.Errors:
return fmt.Errorf("received fsnotify error: %w", err)
case <-stopCh:
return nil
}
}
}
func (d *DynamicKMSEncryptionConfigContent) handleWatchEvent(event fsnotify.Event, watcher *fsnotify.Watcher) error {
// This should be executed after restarting the watch (if applicable) to ensure no file event will be missing.
defer d.queue.Add(workqueueKey)
// return if file has not been removed or renamed.
if event.Op&(fsnotify.Remove|fsnotify.Rename) == 0 {
return nil
}
if err := watcher.Remove(d.filePath); err != nil {
klog.V(2).InfoS("Failed to remove file watch, it may have been deleted", "file", d.filePath, "err", err)
}
if err := watcher.Add(d.filePath); err != nil {
return fmt.Errorf("error adding watch for file %s: %w", d.filePath, err)
}
return nil
wg.Wait()
}
// runWorker to process file content
func (d *DynamicKMSEncryptionConfigContent) runWorker() {
for d.processNextWorkItem() {
func (d *DynamicEncryptionConfigContent) runWorker(ctx context.Context) {
for d.processNextWorkItem(ctx) {
}
}
// processNextWorkItem processes file content when there is a message in the queue.
func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem() bool {
func (d *DynamicEncryptionConfigContent) processNextWorkItem(serverCtx context.Context) bool {
// key here is dummy item in the queue to trigger file content processing.
key, quit := d.queue.Get()
if quit {
@@ -156,6 +145,12 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem() bool {
}
defer d.queue.Done(key)
d.processWorkItem(serverCtx, key)
return true
}
func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Context, workqueueKey interface{}) {
var (
updatedEffectiveConfig bool
err error
@@ -163,39 +158,45 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem() bool {
configChanged bool
)
// get context to close the new transformers.
ctx, closeTransformers := wait.ContextForChannel(d.stopCh)
// get context to close the new transformers (on error cases and on the next reload)
// serverCtx is attached to the API server's lifecycle so we will always close transformers on shut down
ctx, closeTransformers := context.WithCancel(serverCtx)
defer func() {
// TODO: increment success metric when updatedEffectiveConfig=true
// TODO can work queue metrics help here?
if !updatedEffectiveConfig {
// avoid leaking if we're not using the newly constructed transformers (due to an error or them not being changed)
closeTransformers()
}
if updatedEffectiveConfig && err == nil {
metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID)
}
if err != nil {
// TODO: increment failure metric
metrics.RecordEncryptionConfigAutomaticReloadFailure(d.apiServerID)
utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err))
// add dummy item back to the queue to trigger file content processing.
d.queue.AddRateLimited(key)
d.queue.AddRateLimited(workqueueKey)
}
}()
encryptionConfiguration, configChanged, err = d.processEncryptionConfig(ctx)
if err != nil {
return true
return
}
if !configChanged {
return true
return
}
if len(encryptionConfiguration.HealthChecks) != 1 {
err = fmt.Errorf("unexpected number of healthz checks: %d. Should have only one", len(encryptionConfiguration.HealthChecks))
return true
return
}
// get healthz checks for all new KMS plugins.
if err = d.validateNewTransformersHealth(ctx, encryptionConfiguration.HealthChecks[0], encryptionConfiguration.KMSCloseGracePeriod); err != nil {
return true
return
}
// update transformers.
@@ -212,30 +213,44 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem() bool {
klog.V(2).InfoS("Loaded new kms encryption config content", "name", d.name)
updatedEffectiveConfig = true
return true
}
// loadEncryptionConfig processes the next set of content from the file.
func (d *DynamicKMSEncryptionConfigContent) processEncryptionConfig(ctx context.Context) (
encryptionConfiguration *encryptionconfig.EncryptionConfiguration,
func (d *DynamicEncryptionConfigContent) processEncryptionConfig(ctx context.Context) (
_ *encryptionconfig.EncryptionConfiguration,
configChanged bool,
err error,
_ error,
) {
// this code path will only execute if reload=true. So passing true explicitly.
encryptionConfiguration, err = encryptionconfig.LoadEncryptionConfig(d.filePath, true, ctx.Done())
contentHash, err := d.getEncryptionConfigHash(ctx, d.filePath)
if err != nil {
return nil, false, err
}
// check if encryptionConfig is different from the current. Do nothing if they are the same.
if encryptionConfiguration.EncryptionFileContentHash == d.lastLoadedEncryptionConfigHash {
klog.V(4).InfoS("Encryption config has not changed", "name", d.name)
if contentHash == d.lastLoadedEncryptionConfigHash {
klog.V(4).InfoS("Encryption config has not changed (before load)", "name", d.name)
return nil, false, nil
}
// this code path will only execute if reload=true. So passing true explicitly.
encryptionConfiguration, err := d.loadEncryptionConfig(ctx, d.filePath, true, d.apiServerID)
if err != nil {
return nil, false, err
}
// check if encryptionConfig is different from the current (again to avoid TOCTOU). Do nothing if they are the same.
if encryptionConfiguration.EncryptionFileContentHash == d.lastLoadedEncryptionConfigHash {
klog.V(4).InfoS("Encryption config has not changed (after load)", "name", d.name)
return nil, false, nil
}
return encryptionConfiguration, true, nil
}
func (d *DynamicKMSEncryptionConfigContent) validateNewTransformersHealth(
// minKMSPluginCloseGracePeriod can be lowered in unit tests to make the health check poll faster
var minKMSPluginCloseGracePeriod = 10 * time.Second
func (d *DynamicEncryptionConfigContent) validateNewTransformersHealth(
ctx context.Context,
kmsPluginHealthzCheck healthz.HealthChecker,
kmsPluginCloseGracePeriod time.Duration,
@@ -243,11 +258,16 @@ func (d *DynamicKMSEncryptionConfigContent) validateNewTransformersHealth(
// test if new transformers are healthy
var healthCheckError error
if kmsPluginCloseGracePeriod < 10*time.Second {
kmsPluginCloseGracePeriod = 10 * time.Second
if kmsPluginCloseGracePeriod < minKMSPluginCloseGracePeriod {
kmsPluginCloseGracePeriod = minKMSPluginCloseGracePeriod
}
pollErr := wait.PollImmediate(100*time.Millisecond, kmsPluginCloseGracePeriod, func() (bool, error) {
// really make sure that the immediate check does not hang
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, kmsPluginCloseGracePeriod)
defer cancel()
pollErr := wait.PollImmediateWithContext(ctx, 100*time.Millisecond, kmsPluginCloseGracePeriod, func(ctx context.Context) (bool, error) {
// create a fake http get request to health check endpoint
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("/healthz/%s", kmsPluginHealthzCheck.Name()), nil)
if err != nil {

111
vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go generated vendored Normal file
View File

@@ -0,0 +1,111 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package metrics
import (
"crypto/sha256"
"fmt"
"hash"
"sync"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
const (
namespace = "apiserver"
subsystem = "encryption_config_controller"
)
var (
encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "automatic_reload_failures_total",
Help: "Total number of failed automatic reloads of encryption configuration split by apiserver identity.",
StabilityLevel: metrics.ALPHA,
},
[]string{"apiserver_id_hash"},
)
encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "automatic_reload_success_total",
Help: "Total number of successful automatic reloads of encryption configuration split by apiserver identity.",
StabilityLevel: metrics.ALPHA,
},
[]string{"apiserver_id_hash"},
)
encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec(
&metrics.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "automatic_reload_last_timestamp_seconds",
Help: "Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity.",
StabilityLevel: metrics.ALPHA,
},
[]string{"status", "apiserver_id_hash"},
)
)
var registerMetrics sync.Once
var hashPool *sync.Pool
func RegisterMetrics() {
registerMetrics.Do(func() {
hashPool = &sync.Pool{
New: func() interface{} {
return sha256.New()
},
}
legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal)
legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal)
legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
})
}
func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) {
apiServerIDHash := getHash(apiServerID)
encryptionConfigAutomaticReloadFailureTotal.WithLabelValues(apiServerIDHash).Inc()
recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash)
}
func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) {
apiServerIDHash := getHash(apiServerID)
encryptionConfigAutomaticReloadSuccessTotal.WithLabelValues(apiServerIDHash).Inc()
recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash)
}
func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) {
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime()
}
func getHash(data string) string {
if len(data) == 0 {
return ""
}
h := hashPool.Get().(hash.Hash)
h.Reset()
h.Write([]byte(data))
dataHash := fmt.Sprintf("sha256:%x", h.Sum(nil))
hashPool.Put(h)
return dataHash
}

234
vendor/k8s.io/apiserver/pkg/server/options/etcd.go generated vendored
View File

@@ -17,6 +17,7 @@ limitations under the License.
package options
import (
"context"
"fmt"
"net/http"
"strconv"
@@ -25,6 +26,7 @@ import (
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
@@ -33,17 +35,16 @@ import (
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
kmsconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/etcd3/metrics"
"k8s.io/apiserver/pkg/storage/storagebackend"
storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
storagevalue "k8s.io/apiserver/pkg/storage/value"
"k8s.io/klog/v2"
)
type EtcdOptions struct {
// The value of Paging on StorageConfig will be overridden by the
// calculated feature gate value.
StorageConfig storagebackend.Config
EncryptionProviderConfigFilepath string
EncryptionProviderConfigAutomaticReload bool
@@ -63,11 +64,6 @@ type EtcdOptions struct {
// WatchCacheSizes represents override to a given resource
WatchCacheSizes []string
// complete guards fields that must be initialized via Complete before the Apply methods can be used.
complete bool
resourceTransformers encryptionconfig.ResourceTransformers
kmsPluginHealthzChecks []healthz.HealthChecker
// SkipHealthEndpoints, when true, causes the Apply methods to not set up health endpoints.
// This allows multiple invocations of the Apply methods without duplication of said endpoints.
SkipHealthEndpoints bool
@@ -90,6 +86,12 @@ func NewEtcdOptions(backendConfig *storagebackend.Config) *EtcdOptions {
return options
}
var storageMediaTypes = sets.New(
runtime.ContentTypeJSON,
runtime.ContentTypeYAML,
runtime.ContentTypeProtobuf,
)
func (s *EtcdOptions) Validate() []error {
if s == nil {
return nil
@@ -123,6 +125,10 @@ func (s *EtcdOptions) Validate() []error {
allErrors = append(allErrors, fmt.Errorf("--encryption-provider-config-automatic-reload must be set with --encryption-provider-config"))
}
if s.DefaultStorageMediaType != "" && !storageMediaTypes.Has(s.DefaultStorageMediaType) {
allErrors = append(allErrors, fmt.Errorf("--storage-media-type %q invalid, allowed values: %s", s.DefaultStorageMediaType, strings.Join(sets.List(storageMediaTypes), ", ")))
}
return allErrors
}
@@ -211,90 +217,18 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
"The time in seconds that each lease is reused. A lower value could avoid large number of objects reusing the same lease. Notice that a too small value may cause performance problems at storage layer.")
}
// Complete must be called exactly once before using any of the Apply methods. It is responsible for setting
// up objects that must be created once and reused across multiple invocations such as storage transformers.
// This method mutates the receiver (EtcdOptions). It must never mutate the inputs.
func (s *EtcdOptions) Complete(
storageObjectCountTracker flowcontrolrequest.StorageObjectCountTracker,
stopCh <-chan struct{},
addPostStartHook func(name string, hook server.PostStartHookFunc) error,
) error {
if s == nil {
return nil
}
if s.complete {
return fmt.Errorf("EtcdOptions.Complete called more than once")
}
if len(s.EncryptionProviderConfigFilepath) != 0 {
ctxTransformers, closeTransformers := wait.ContextForChannel(stopCh)
ctxServer, _ := wait.ContextForChannel(stopCh) // explicitly ignore cancel here because we do not own the server's lifecycle
encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, ctxTransformers.Done())
if err != nil {
// in case of error, we want to close partially initialized (if any) transformers
closeTransformers()
return err
}
// enable kms hot reload controller only if the config file is set to be automatically reloaded
if s.EncryptionProviderConfigAutomaticReload {
// with reload=true we will always have 1 health check
if len(encryptionConfiguration.HealthChecks) != 1 {
// in case of error, we want to close partially initialized (if any) transformers
closeTransformers()
return fmt.Errorf("failed to start kms encryption config hot reload controller. only 1 health check should be available when reload is enabled")
}
dynamicTransformers := encryptionconfig.NewDynamicTransformers(encryptionConfiguration.Transformers, encryptionConfiguration.HealthChecks[0], closeTransformers, encryptionConfiguration.KMSCloseGracePeriod)
s.resourceTransformers = dynamicTransformers
s.kmsPluginHealthzChecks = []healthz.HealthChecker{dynamicTransformers}
// add post start hook to start hot reload controller
// adding this hook here will ensure that it gets configured exactly once
err = addPostStartHook(
"start-encryption-provider-config-automatic-reload",
func(hookContext server.PostStartHookContext) error {
kmsConfigController := kmsconfigcontroller.NewDynamicKMSEncryptionConfiguration(
"kms-encryption-config",
s.EncryptionProviderConfigFilepath,
dynamicTransformers,
encryptionConfiguration.EncryptionFileContentHash,
ctxServer.Done(),
)
go kmsConfigController.Run(ctxServer)
return nil
},
)
if err != nil {
// in case of error, we want to close partially initialized (if any) transformers
closeTransformers()
return fmt.Errorf("failed to add post start hook for kms encryption config hot reload controller: %w", err)
}
} else {
s.resourceTransformers = encryptionconfig.StaticTransformers(encryptionConfiguration.Transformers)
s.kmsPluginHealthzChecks = encryptionConfiguration.HealthChecks
}
}
s.StorageConfig.StorageObjectCountTracker = storageObjectCountTracker
s.complete = true
return nil
}
// ApplyTo mutates the provided server.Config. It must never mutate the receiver (EtcdOptions).
func (s *EtcdOptions) ApplyTo(c *server.Config) error {
if s == nil {
return nil
}
return s.ApplyWithStorageFactoryTo(&SimpleStorageFactory{StorageConfig: s.StorageConfig}, c)
storageConfigCopy := s.StorageConfig
if storageConfigCopy.StorageObjectCountTracker == nil {
storageConfigCopy.StorageObjectCountTracker = c.StorageObjectCountTracker
}
return s.ApplyWithStorageFactoryTo(&SimpleStorageFactory{StorageConfig: storageConfigCopy}, c)
}
// ApplyWithStorageFactoryTo mutates the provided server.Config. It must never mutate the receiver (EtcdOptions).
@@ -303,27 +237,127 @@ func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFac
return nil
}
if !s.complete {
return fmt.Errorf("EtcdOptions.Apply called without completion")
}
if !s.SkipHealthEndpoints {
if err := s.addEtcdHealthEndpoint(c); err != nil {
return err
}
}
if s.resourceTransformers != nil {
// setup encryption
if err := s.maybeApplyResourceTransformers(c); err != nil {
return err
}
metrics.SetStorageMonitorGetter(monitorGetter(factory))
c.RESTOptionsGetter = s.CreateRESTOptionsGetter(factory, c.ResourceTransformers)
return nil
}
func monitorGetter(factory serverstorage.StorageFactory) func() (monitors []metrics.Monitor, err error) {
return func() (monitors []metrics.Monitor, err error) {
defer func() {
if err != nil {
for _, m := range monitors {
m.Close()
}
}
}()
var m metrics.Monitor
for _, cfg := range factory.Configs() {
m, err = storagefactory.CreateMonitor(cfg)
if err != nil {
return nil, err
}
monitors = append(monitors, m)
}
return monitors, nil
}
}
func (s *EtcdOptions) CreateRESTOptionsGetter(factory serverstorage.StorageFactory, resourceTransformers storagevalue.ResourceTransformers) generic.RESTOptionsGetter {
if resourceTransformers != nil {
factory = &transformerStorageFactory{
delegate: factory,
resourceTransformers: s.resourceTransformers,
resourceTransformers: resourceTransformers,
}
}
return &StorageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory}
}
func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err error) {
if c.ResourceTransformers != nil {
return nil
}
if len(s.EncryptionProviderConfigFilepath) == 0 {
return nil
}
ctxServer := wait.ContextForChannel(c.DrainedNotify())
ctxTransformers, closeTransformers := context.WithCancel(ctxServer)
defer func() {
// in case of error, we want to close partially initialized (if any) transformers
if err != nil {
closeTransformers()
}
}()
encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, c.APIServerID)
if err != nil {
return err
}
if s.EncryptionProviderConfigAutomaticReload {
// with reload=true we will always have 1 health check
if len(encryptionConfiguration.HealthChecks) != 1 {
return fmt.Errorf("failed to start kms encryption config hot reload controller. only 1 health check should be available when reload is enabled")
}
// Here the dynamic transformers take ownership of the transformers and their cancellation.
dynamicTransformers := encryptionconfig.NewDynamicTransformers(encryptionConfiguration.Transformers, encryptionConfiguration.HealthChecks[0], closeTransformers, encryptionConfiguration.KMSCloseGracePeriod)
// add post start hook to start hot reload controller
// adding this hook here will ensure that it gets configured exactly once
err = c.AddPostStartHook(
"start-encryption-provider-config-automatic-reload",
func(_ server.PostStartHookContext) error {
dynamicEncryptionConfigController := encryptionconfigcontroller.NewDynamicEncryptionConfiguration(
"encryption-provider-config-automatic-reload-controller",
s.EncryptionProviderConfigFilepath,
dynamicTransformers,
encryptionConfiguration.EncryptionFileContentHash,
c.APIServerID,
)
go dynamicEncryptionConfigController.Run(ctxServer)
return nil
},
)
if err != nil {
return fmt.Errorf("failed to add post start hook for kms encryption config hot reload controller: %w", err)
}
c.ResourceTransformers = dynamicTransformers
if !s.SkipHealthEndpoints {
addHealthChecksWithoutLivez(c, dynamicTransformers)
}
} else {
c.ResourceTransformers = encryptionconfig.StaticTransformers(encryptionConfiguration.Transformers)
if !s.SkipHealthEndpoints {
addHealthChecksWithoutLivez(c, encryptionConfiguration.HealthChecks...)
}
}
c.RESTOptionsGetter = &StorageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory}
return nil
}
func addHealthChecksWithoutLivez(c *server.Config, healthChecks ...healthz.HealthChecker) {
c.HealthzChecks = append(c.HealthzChecks, healthChecks...)
c.ReadyzChecks = append(c.ReadyzChecks, healthChecks...)
}
func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
healthCheck, err := storagefactory.CreateHealthCheck(s.StorageConfig, c.DrainedNotify())
if err != nil {
@@ -341,8 +375,6 @@ func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
return readyCheck()
}))
c.AddHealthChecks(s.kmsPluginHealthzChecks...)
return nil
}
@@ -441,6 +473,10 @@ func (s *SimpleStorageFactory) ResourcePrefix(resource schema.GroupResource) str
return resource.Group + "/" + resource.Resource
}
func (s *SimpleStorageFactory) Configs() []storagebackend.Config {
return serverstorage.Configs(s.StorageConfig)
}
func (s *SimpleStorageFactory) Backends() []serverstorage.Backend {
// nothing should ever call this method but we still provide a functional implementation
return serverstorage.Backends(s.StorageConfig)
@@ -450,7 +486,7 @@ var _ serverstorage.StorageFactory = &transformerStorageFactory{}
type transformerStorageFactory struct {
delegate serverstorage.StorageFactory
resourceTransformers encryptionconfig.ResourceTransformers
resourceTransformers storagevalue.ResourceTransformers
}
func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
@@ -471,6 +507,10 @@ func (t *transformerStorageFactory) ResourcePrefix(resource schema.GroupResource
return t.delegate.ResourcePrefix(resource)
}
func (t *transformerStorageFactory) Configs() []storagebackend.Config {
return t.delegate.Configs()
}
func (t *transformerStorageFactory) Backends() []serverstorage.Backend {
return t.delegate.Backends()
}

30
vendor/k8s.io/apiserver/pkg/server/options/feature.go generated vendored
View File

@@ -17,15 +17,22 @@ limitations under the License.
package options
import (
"fmt"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apiserver/pkg/server"
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
)
type FeatureOptions struct {
EnableProfiling bool
DebugSocketPath string
EnableContentionProfiling bool
EnablePriorityAndFairness bool
}
func NewFeatureOptions() *FeatureOptions {
@@ -33,7 +40,9 @@ func NewFeatureOptions() *FeatureOptions {
return &FeatureOptions{
EnableProfiling: defaults.EnableProfiling,
DebugSocketPath: defaults.DebugSocketPath,
EnableContentionProfiling: defaults.EnableContentionProfiling,
EnablePriorityAndFairness: true,
}
}
@@ -45,17 +54,34 @@ func (o *FeatureOptions) AddFlags(fs *pflag.FlagSet) {
fs.BoolVar(&o.EnableProfiling, "profiling", o.EnableProfiling,
"Enable profiling via web interface host:port/debug/pprof/")
fs.BoolVar(&o.EnableContentionProfiling, "contention-profiling", o.EnableContentionProfiling,
"Enable lock contention profiling, if profiling is enabled")
"Enable block profiling, if profiling is enabled")
fs.StringVar(&o.DebugSocketPath, "debug-socket-path", o.DebugSocketPath,
"Use an unprotected (no authn/authz) unix-domain socket for profiling with the given path")
fs.BoolVar(&o.EnablePriorityAndFairness, "enable-priority-and-fairness", o.EnablePriorityAndFairness, ""+
"If true, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")
}
func (o *FeatureOptions) ApplyTo(c *server.Config) error {
func (o *FeatureOptions) ApplyTo(c *server.Config, clientset kubernetes.Interface, informers informers.SharedInformerFactory) error {
if o == nil {
return nil
}
c.EnableProfiling = o.EnableProfiling
c.DebugSocketPath = o.DebugSocketPath
c.EnableContentionProfiling = o.EnableContentionProfiling
if o.EnablePriorityAndFairness {
if c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight <= 0 {
return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight)
}
c.FlowControl = utilflowcontrol.New(
informers,
clientset.FlowcontrolV1(),
c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight,
)
}
return nil
}

47
vendor/k8s.io/apiserver/pkg/server/options/recommended.go generated vendored
View File

@@ -17,20 +17,15 @@ limitations under the License.
package options
import (
"fmt"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/apiserver/pkg/util/feature"
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/component-base/featuregate"
"k8s.io/klog/v2"
)
// RecommendedOptions contains the recommended options for running an API server.
@@ -101,9 +96,6 @@ func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) {
// ApplyTo adds RecommendedOptions to the server configuration.
// pluginInitializers can be empty, it is only need for additional initializers.
func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
if err := o.Etcd.Complete(config.Config.StorageObjectCountTracker, config.Config.DrainedNotify(), config.Config.AddPostStartHook); err != nil {
return err
}
if err := o.Etcd.ApplyTo(&config.Config); err != nil {
return err
}
@@ -125,32 +117,27 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
if err := o.Audit.ApplyTo(&config.Config); err != nil {
return err
}
if err := o.Features.ApplyTo(&config.Config); err != nil {
return err
}
if err := o.CoreAPI.ApplyTo(config); err != nil {
return err
}
if initializers, err := o.ExtraAdmissionInitializers(config); err != nil {
return err
} else if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, config.ClientConfig, o.FeatureGate, initializers...); err != nil {
kubeClient, err := kubernetes.NewForConfig(config.ClientConfig)
if err != nil {
return err
}
if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
if config.ClientConfig != nil {
if config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight <= 0 {
return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", config.MaxRequestsInFlight, config.MaxMutatingRequestsInFlight)
}
config.FlowControl = utilflowcontrol.New(
config.SharedInformerFactory,
kubernetes.NewForConfigOrDie(config.ClientConfig).FlowcontrolV1beta3(),
config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight,
config.RequestTimeout/4,
)
} else {
klog.Warningf("Neither kubeconfig is provided nor service-account is mounted, so APIPriorityAndFairness will be disabled")
}
if err := o.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil {
return err
}
initializers, err := o.ExtraAdmissionInitializers(config)
if err != nil {
return err
}
dynamicClient, err := dynamic.NewForConfig(config.ClientConfig)
if err != nil {
return err
}
if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, kubeClient, dynamicClient, o.FeatureGate,
initializers...); err != nil {
return err
}
return nil
}

128
vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go generated vendored
View File

@@ -19,10 +19,10 @@ package options
import (
"fmt"
"net"
"regexp"
"strings"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apiserver/pkg/server"
@@ -31,6 +31,16 @@ import (
"github.com/spf13/pflag"
)
const (
corsAllowedOriginsHelpText = "List of allowed origins for CORS, comma separated. " +
"An allowed origin can be a regular expression to support subdomain matching. " +
"If this list is empty CORS will not be enabled. " +
"Please ensure each expression matches the entire hostname by anchoring " +
"to the start with '^' or including the '//' prefix, and by anchoring to the " +
"end with '$' or including the ':' port separator suffix. " +
"Examples of valid expressions are '//example\\.com(:|$)' and '^https://example\\.com(:|$)'"
)
// ServerRunOptions contains the options while running a generic api server.
type ServerRunOptions struct {
AdvertiseAddress net.IP
@@ -52,8 +62,7 @@ type ServerRunOptions struct {
// decoded in a write request. 0 means no limit.
// We intentionally did not add a flag for this option. Users of the
// apiserver library can wire it to a flag.
MaxRequestBodyBytes int64
EnablePriorityAndFairness bool
MaxRequestBodyBytes int64
// ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP
// Server during the graceful termination of the apiserver. If true, we wait
@@ -63,21 +72,38 @@ type ServerRunOptions struct {
// If enabled, after ShutdownDelayDuration elapses, any incoming request is
// rejected with a 429 status code and a 'Retry-After' response.
ShutdownSendRetryAfter bool
// ShutdownWatchTerminationGracePeriod, if set to a positive value,
// is the maximum duration the apiserver will wait for all active
// watch request(s) to drain.
// Once this grace period elapses, the apiserver will no longer
// wait for any active watch request(s) in flight to drain, it will
// proceed to the next step in the graceful server shutdown process.
// If set to a positive value, the apiserver will keep track of the
// number of active watch request(s) in flight and during shutdown
// it will wait, at most, for the specified duration and allow these
// active watch requests to drain with some rate limiting in effect.
// The default is zero, which implies the apiserver will not keep
// track of active watch request(s) in flight and will not wait
// for them to drain, this maintains backward compatibility.
// This grace period is orthogonal to other grace periods, and
// it is not overridden by any other grace period.
ShutdownWatchTerminationGracePeriod time.Duration
}
func NewServerRunOptions() *ServerRunOptions {
defaults := server.NewConfig(serializer.CodecFactory{})
return &ServerRunOptions{
MaxRequestsInFlight: defaults.MaxRequestsInFlight,
MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
RequestTimeout: defaults.RequestTimeout,
LivezGracePeriod: defaults.LivezGracePeriod,
MinRequestTimeout: defaults.MinRequestTimeout,
ShutdownDelayDuration: defaults.ShutdownDelayDuration,
JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes,
MaxRequestBodyBytes: defaults.MaxRequestBodyBytes,
EnablePriorityAndFairness: true,
ShutdownSendRetryAfter: false,
MaxRequestsInFlight: defaults.MaxRequestsInFlight,
MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
RequestTimeout: defaults.RequestTimeout,
LivezGracePeriod: defaults.LivezGracePeriod,
MinRequestTimeout: defaults.MinRequestTimeout,
ShutdownDelayDuration: defaults.ShutdownDelayDuration,
ShutdownWatchTerminationGracePeriod: defaults.ShutdownWatchTerminationGracePeriod,
JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes,
MaxRequestBodyBytes: defaults.MaxRequestBodyBytes,
ShutdownSendRetryAfter: false,
}
}
@@ -97,6 +123,7 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
c.PublicAddress = s.AdvertiseAddress
c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod
return nil
}
@@ -150,6 +177,10 @@ func (s *ServerRunOptions) Validate() []error {
errors = append(errors, fmt.Errorf("--shutdown-delay-duration can not be negative value"))
}
if s.ShutdownWatchTerminationGracePeriod < 0 {
errors = append(errors, fmt.Errorf("shutdown-watch-termination-grace-period, if provided, can not be a negative value"))
}
if s.JSONPatchMaxCopyBytes < 0 {
errors = append(errors, fmt.Errorf("ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value"))
}
@@ -161,6 +192,10 @@ func (s *ServerRunOptions) Validate() []error {
if err := validateHSTSDirectives(s.HSTSDirectives); err != nil {
errors = append(errors, err)
}
if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil {
errors = append(errors, err)
}
return errors
}
@@ -183,6 +218,57 @@ func validateHSTSDirectives(hstsDirectives []string) error {
return errors.NewAggregate(allErrors)
}
func validateCorsAllowedOriginList(corsAllowedOriginList []string) error {
allErrors := []error{}
validateRegexFn := func(regexpStr string) error {
if _, err := regexp.Compile(regexpStr); err != nil {
return err
}
// the regular expression should pin to the start and end of the host
// in the origin header, this will prevent CVE-2022-1996.
// possible ways it can pin to the start of host in the origin header:
// - match the start of the origin with '^'
// - match what separates the scheme and host with '//' or '://',
// this pins to the start of host in the origin header.
// possible ways it can match the end of the host in the origin header:
// - match the end of the origin with '$'
// - with a capture group that matches the host and port separator '(:|$)'
// We will relax the validation to check if these regex markers
// are present in the user specified expression.
var pinStart, pinEnd bool
for _, prefix := range []string{"^", "//"} {
if strings.Contains(regexpStr, prefix) {
pinStart = true
break
}
}
for _, suffix := range []string{"$", ":"} {
if strings.Contains(regexpStr, suffix) {
pinEnd = true
break
}
}
if !pinStart || !pinEnd {
return fmt.Errorf("regular expression does not pin to start/end of host in the origin header")
}
return nil
}
for _, regexp := range corsAllowedOriginList {
if len(regexp) == 0 {
allErrors = append(allErrors, fmt.Errorf("empty value in --cors-allowed-origins, help: %s", corsAllowedOriginsHelpText))
continue
}
if err := validateRegexFn(regexp); err != nil {
err = fmt.Errorf("--cors-allowed-origins has an invalid regular expression: %v, help: %s", err, corsAllowedOriginsHelpText)
allErrors = append(allErrors, err)
}
}
return errors.NewAggregate(allErrors)
}
// AddUniversalFlags adds flags for a specific APIServer to the specified FlagSet
func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
@@ -194,9 +280,7 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
"will be used. If --bind-address is unspecified, the host's default interface will "+
"be used.")
fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, ""+
"List of allowed origins for CORS, comma separated. An allowed origin can be a regular "+
"expression to support subdomain matching. If this list is empty CORS will not be enabled.")
fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, corsAllowedOriginsHelpText)
fs.StringSliceVar(&s.HSTSDirectives, "strict-transport-security-directives", s.HSTSDirectives, ""+
"List of directives for HSTS, comma separated. If this list is empty, then HSTS directives will not "+
@@ -205,11 +289,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs or OpenID Discovery).")
deprecatedMasterServiceNamespace := metav1.NamespaceDefault
fs.StringVar(&deprecatedMasterServiceNamespace, "master-service-namespace", deprecatedMasterServiceNamespace, ""+
"DEPRECATED: the namespace from which the Kubernetes master services should be injected into pods.")
fs.MarkDeprecated("master-service-namespace", "This flag will be removed in v1.27")
fs.IntVar(&s.MaxRequestsInFlight, "max-requests-inflight", s.MaxRequestsInFlight, ""+
"This and --max-mutating-requests-inflight are summed to determine the server's total concurrency limit "+
"(which must be positive) if --enable-priority-and-fairness is true. "+
@@ -244,9 +323,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
"handler, which picks a randomized value above this number as the connection timeout, "+
"to spread out load.")
fs.BoolVar(&s.EnablePriorityAndFairness, "enable-priority-and-fairness", s.EnablePriorityAndFairness, ""+
"If true and the APIPriorityAndFairness feature gate is enabled, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")
fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
"Time to delay the termination. During that time the server keeps serving requests normally. The endpoints /healthz and /livez "+
"will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
@@ -257,5 +333,9 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
"during this window all incoming requests will be rejected with a status code 429 and a 'Retry-After' response header, "+
"in addition 'Connection: close' response header is set in order to tear down the TCP connection when idle.")
fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+
"This option, if set, represents the maximum amount of grace period the apiserver will wait "+
"for active watch request(s) to drain during the graceful server shutdown window.")
utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
}

36
vendor/k8s.io/apiserver/pkg/server/options/serving.go generated vendored
View File

@@ -153,7 +153,7 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
fs.IPVar(&s.BindAddress, "bind-address", s.BindAddress, ""+
"The IP address on which to listen for the --secure-port port. The "+
"associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+
"clients. If blank or an unspecified address (0.0.0.0 or ::), all interfaces will be used.")
"clients. If blank or an unspecified address (0.0.0.0 or ::), all interfaces and IP address families will be used.")
desc := "The port on which to serve HTTPS with authentication and authorization."
if s.Required {
@@ -260,7 +260,39 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
c := *config
serverCertFile, serverKeyFile := s.ServerCert.CertKey.CertFile, s.ServerCert.CertKey.KeyFile
// load main cert
// load main cert *original description until 2023-08-18*
/*
kubernetes mutual (2-way) x509 between client and apiserver:
>1. apiserver sending its apiserver certificate along with its publickey to client
2. client verifies the apiserver certificate sent against its cluster certificate authority data
3. client sending its client certificate along with its public key to the apiserver
4. apiserver verifies the client certificate sent against its cluster certificate authority data
description:
here, with this block,
apiserver certificate and pub key data (along with priv key)get loaded into server.SecureServingInfo
for client to later in the step 2 verify the apiserver certificate during the handshake
when making a request
normal args related to this stage:
--tls-cert-file string File containing the default x509 Certificate for HTTPS.
(CA cert, if any, concatenated after server cert). If HTTPS serving is enabled, and
--tls-cert-file and --tls-private-key-file are not provided, a self-signed certificate
and key are generated for the public address and saved to the directory specified by
--cert-dir
--tls-private-key-file string File containing the default x509 private key matching --tls-cert-file.
(retrievable from "kube-apiserver --help" command)
(suggested by @deads2k)
see also:
- for the step 2, see: staging/src/k8s.io/client-go/transport/transport.go
- for the step 3, see: staging/src/k8s.io/client-go/transport/transport.go
- for the step 4, see: staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go
*/
if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
var err error
c.Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile)

6
vendor/k8s.io/apiserver/pkg/server/options/tracing.go generated vendored
View File

@@ -154,9 +154,5 @@ func ReadTracingConfiguration(configFilePath string) (*tracingapi.TracingConfigu
if err := runtime.DecodeInto(codecs.UniversalDecoder(), data, internalConfig); err != nil {
return nil, fmt.Errorf("unable to decode tracing configuration data: %v", err)
}
tc := &tracingapi.TracingConfiguration{
Endpoint: internalConfig.Endpoint,
SamplingRatePerMillion: internalConfig.SamplingRatePerMillion,
}
return tc, nil
return &internalConfig.TracingConfiguration, nil
}