admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bump sigs.k8s.io/controller-runtime to v0.14.4 (#5507)

Browse Source 
* Bump sigs.k8s.io/controller-runtime to v0.14.4

* Update gofmt
This commit is contained in:
hongming
2023-02-08 14:06:15 +08:00
committed by GitHub
parent 129e6fbec3
commit 1c49fcd57e
1404 changed files with 141422 additions and 47769 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
vendor/k8s.io/apiserver/pkg/server/options/admission.go generated vendored
View File

@@ -29,12 +29,14 @@ import (
"k8s.io/apiserver/pkg/admission/initializer"
admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy"
mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
apiserverapi "k8s.io/apiserver/pkg/apis/apiserver"
apiserverapiv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
apiserverapiv1alpha1 "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
"k8s.io/apiserver/pkg/server"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
@@ -85,7 +87,7 @@ func NewAdmissionOptions() *AdmissionOptions {
// admission plugins. The apiserver always runs the validating ones
// after all the mutating ones, so their relative order in this list
// doesn't matter.
RecommendedPluginOrder: []string{lifecycle.PluginName, mutatingwebhook.PluginName, validatingwebhook.PluginName},
RecommendedPluginOrder: []string{lifecycle.PluginName, mutatingwebhook.PluginName, validatingadmissionpolicy.PluginName, validatingwebhook.PluginName},
DefaultOffPlugins: sets.NewString(),
}
server.RegisterAllAdmissionPlugins(options.Plugins)
@@ -145,7 +147,11 @@ func (a *AdmissionOptions) ApplyTo(
if err != nil {
return err
}
genericInitializer := initializer.New(clientset, informers, c.Authorization.Authorizer, features, c.DrainedNotify())
dynamicClient, err := dynamic.NewForConfig(kubeAPIServerClientConfig)
if err != nil {
return err
}
genericInitializer := initializer.New(clientset, dynamicClient, informers, c.Authorization.Authorizer, features, c.DrainedNotify())
initializersChain := admission.PluginInitializers{genericInitializer}
initializersChain = append(initializersChain, pluginInitializers...)

5
vendor/k8s.io/apiserver/pkg/server/options/authentication.go generated vendored
View File

@@ -202,6 +202,9 @@ type DelegatingAuthenticationOptions struct {
// CustomRoundTripperFn allows for specifying a middleware function for custom HTTP behaviour for the authentication webhook client.
CustomRoundTripperFn transport.WrapperFunc
// DisableAnonymous gives user an option to disable Anonymous authentication.
DisableAnonymous bool
}
func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
@@ -283,7 +286,7 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.Aut
}
cfg := authenticatorfactory.DelegatingAuthenticatorConfig{
Anonymous: true,
Anonymous: !s.DisableAnonymous,
CacheTTL: s.CacheTTL,
WebhookRetryBackoff: s.WebhookRetryBackoff,
TokenAccessReviewTimeout: s.TokenRequestTimeout,

43
vendor/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go generated vendored
View File

@@ -23,7 +23,6 @@ import (
"github.com/spf13/pflag"
"k8s.io/apiserver/pkg/server"
"k8s.io/client-go/rest"
)
// DeprecatedInsecureServingOptions are for creating an unauthenticated, unauthorized, insecure port.
@@ -125,45 +124,3 @@ func (s *DeprecatedInsecureServingOptions) ApplyTo(c **server.DeprecatedInsecure
return nil
}
// WithLoopback adds loopback functionality to the serving options.
func (o *DeprecatedInsecureServingOptions) WithLoopback() *DeprecatedInsecureServingOptionsWithLoopback {
return &DeprecatedInsecureServingOptionsWithLoopback{o}
}
// DeprecatedInsecureServingOptionsWithLoopback adds loopback functionality to the DeprecatedInsecureServingOptions.
// DEPRECATED: all insecure serving options will be removed in a future version, however note that
// there are security concerns over how health checks can work here - see e.g. #43784
type DeprecatedInsecureServingOptionsWithLoopback struct {
*DeprecatedInsecureServingOptions
}
// ApplyTo fills up serving information in the server configuration.
func (s *DeprecatedInsecureServingOptionsWithLoopback) ApplyTo(insecureServingInfo **server.DeprecatedInsecureServingInfo, loopbackClientConfig **rest.Config) error {
if s == nil || s.DeprecatedInsecureServingOptions == nil || insecureServingInfo == nil {
return nil
}
if err := s.DeprecatedInsecureServingOptions.ApplyTo(insecureServingInfo); err != nil {
return err
}
if *insecureServingInfo == nil || loopbackClientConfig == nil {
return nil
}
secureLoopbackClientConfig, err := (*insecureServingInfo).NewLoopbackClientConfig()
switch {
// if we failed and there's no fallback loopback client config, we need to fail
case err != nil && *loopbackClientConfig == nil:
return err
// if we failed, but we already have a fallback loopback client config (usually insecure), allow it
case err != nil && *loopbackClientConfig != nil:
default:
*loopbackClientConfig = secureLoopbackClientConfig
}
return nil
}

595
vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go generated vendored
View File

@@ -20,20 +20,23 @@ import (
"context"
"crypto/aes"
"crypto/cipher"
"crypto/sha256"
"encoding/base64"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"sync"
"sync/atomic"
"time"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
apiserverconfig "k8s.io/apiserver/pkg/apis/config"
apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
"k8s.io/apiserver/pkg/apis/config/validation"
@@ -58,6 +61,7 @@ const (
kmsPluginHealthzPositiveTTL = 20 * time.Second
kmsAPIVersionV1 = "v1"
kmsAPIVersionV2 = "v2"
kmsReloadHealthCheckName = "kms-providers"
)
type kmsPluginHealthzResponse struct {
@@ -66,122 +70,158 @@ type kmsPluginHealthzResponse struct {
}
type kmsPluginProbe struct {
name string
ttl time.Duration
envelope.Service
name string
ttl time.Duration
service envelope.Service
lastResponse *kmsPluginHealthzResponse
l *sync.Mutex
}
type kmsv2PluginProbe struct {
name string
ttl time.Duration
envelopekmsv2.Service
name string
ttl time.Duration
service envelopekmsv2.Service
lastResponse *kmsPluginHealthzResponse
l *sync.Mutex
}
type kmsHealthChecker []healthz.HealthChecker
func (k kmsHealthChecker) Name() string {
return kmsReloadHealthCheckName
}
func (k kmsHealthChecker) Check(req *http.Request) error {
var errs []error
for i := range k {
checker := k[i]
if err := checker.Check(req); err != nil {
errs = append(errs, fmt.Errorf("%s: %w", checker.Name(), err))
}
}
return utilerrors.Reduce(utilerrors.NewAggregate(errs))
}
func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
return h.Check()
return h.check()
})
}
func (p *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
func (h *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
return p.Check()
return h.check(r.Context())
})
}
// GetKMSPluginHealthzCheckers extracts KMSPluginProbes from the EncryptionConfig.
func GetKMSPluginHealthzCheckers(filepath string) ([]healthz.HealthChecker, error) {
f, err := os.Open(filepath)
if err != nil {
return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err)
}
defer f.Close()
// EncryptionConfiguration represents the parsed and normalized encryption configuration for the apiserver.
type EncryptionConfiguration struct {
// Transformers is a list of value.Transformer that will be used to encrypt and decrypt data.
Transformers map[schema.GroupResource]value.Transformer
var result []healthz.HealthChecker
probes, err := getKMSPluginProbes(f)
if err != nil {
return nil, err
}
for i, p := range probes {
probe := p
switch t := probe.(type) {
case *kmsPluginProbe:
result = append(result, t.toHealthzCheck(i))
case *kmsv2PluginProbe:
result = append(result, t.toHealthzCheck(i))
default:
return nil, fmt.Errorf("unsupported KMS plugin type: %T", t)
}
}
// HealthChecks is a list of healthz.HealthChecker that will be used to check the health of the encryption providers.
HealthChecks []healthz.HealthChecker
return result, nil
// EncryptionFileContentHash is the hash of the encryption config file.
EncryptionFileContentHash string
// KMSCloseGracePeriod is the duration we will wait before closing old transformers.
// We wait for any in-flight requests to finish by using the duration which is longer than their timeout.
KMSCloseGracePeriod time.Duration
}
func getKMSPluginProbes(reader io.Reader) ([]interface{}, error) {
var result []interface{}
configFileContents, err := ioutil.ReadAll(reader)
// LoadEncryptionConfig parses and validates the encryption config specified by filepath.
// It may launch multiple go routines whose lifecycle is controlled by stopCh.
// If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1.
func LoadEncryptionConfig(filepath string, reload bool, stopCh <-chan struct{}) (*EncryptionConfiguration, error) {
config, contentHash, err := loadConfig(filepath, reload)
if err != nil {
return nil, fmt.Errorf("could not read content of encryption provider configuration: %v", err)
return nil, fmt.Errorf("error while parsing file: %w", err)
}
config, err := loadConfig(configFileContents)
transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh)
if err != nil {
return nil, fmt.Errorf("error while parsing encryption provider configuration: %v", err)
return nil, fmt.Errorf("error while building transformers: %w", err)
}
for _, r := range config.Resources {
for _, p := range r.Providers {
if p.KMS != nil {
switch p.KMS.APIVersion {
case kmsAPIVersionV1:
s, err := envelope.NewGRPCService(p.KMS.Endpoint, p.KMS.Timeout.Duration)
if err != nil {
return nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %v", p.KMS.Name, err)
}
result = append(result, &kmsPluginProbe{
name: p.KMS.Name,
ttl: kmsPluginHealthzNegativeTTL,
Service: s,
l: &sync.Mutex{},
lastResponse: &kmsPluginHealthzResponse{},
})
case kmsAPIVersionV2:
if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
return nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, KMSv2 feature is not enabled", p.KMS.Name)
}
s, err := envelopekmsv2.NewGRPCService(p.KMS.Endpoint, p.KMS.Timeout.Duration)
if err != nil {
return nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %v", p.KMS.Name, err)
}
result = append(result, &kmsv2PluginProbe{
name: p.KMS.Name,
ttl: kmsPluginHealthzNegativeTTL,
Service: s,
l: &sync.Mutex{},
lastResponse: &kmsPluginHealthzResponse{},
})
default:
return nil, fmt.Errorf("could not configure KMS Plugin's probe %q, unsupported KMS API version %q", p.KMS.Name, p.KMS.APIVersion)
}
}
}
if reload || (kmsUsed.v2Used && !kmsUsed.v1Used) {
kmsHealthChecks = []healthz.HealthChecker{kmsHealthChecker(kmsHealthChecks)}
}
return result, nil
// KMSTimeout is the duration we will wait before closing old transformers.
// The way we calculate is as follows:
// 1. Sum all timeouts across all KMS plugins. (check kmsPrefixTransformer for differences between v1 and v2)
// 2. Multiply that by 2 (to allow for some buffer)
// The reason we sum all timeout is because kmsHealthChecker() will run all health checks serially
return &EncryptionConfiguration{
Transformers: transformers,
HealthChecks: kmsHealthChecks,
EncryptionFileContentHash: contentHash,
KMSCloseGracePeriod: 2 * kmsUsed.kmsTimeoutSum,
}, err
}
// Check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint.
func (h *kmsPluginProbe) Check() error {
func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, *kmsState, error) {
var kmsHealthChecks []healthz.HealthChecker
transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh)
if err != nil {
return nil, nil, nil, err
}
for i := range probes {
probe := probes[i]
kmsHealthChecks = append(kmsHealthChecks, probe.toHealthzCheck(i))
}
return transformers, kmsHealthChecks, kmsUsed, nil
}
type healthChecker interface {
toHealthzCheck(idx int) healthz.HealthChecker
}
func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, *kmsState, error) {
resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{}
var probes []healthChecker
var kmsUsed kmsState
// For each entry in the configuration
for _, resourceConfig := range config.Resources {
resourceConfig := resourceConfig
transformers, p, used, err := prefixTransformersAndProbes(resourceConfig, stopCh)
if err != nil {
return nil, nil, nil, err
}
kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used
kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used
kmsUsed.kmsTimeoutSum += used.kmsTimeoutSum
// For each resource, create a list of providers to use
for _, resource := range resourceConfig.Resources {
resource := resource
gr := schema.ParseGroupResource(resource)
resourceToPrefixTransformer[gr] = append(
resourceToPrefixTransformer[gr], transformers...)
}
probes = append(probes, p...)
}
transformers := make(map[schema.GroupResource]value.Transformer, len(resourceToPrefixTransformer))
for gr, transList := range resourceToPrefixTransformer {
gr := gr
transList := transList
transformers[gr] = value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)
}
return transformers, probes, &kmsUsed, nil
}
// check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint.
func (h *kmsPluginProbe) check() error {
h.l.Lock()
defer h.l.Unlock()
@@ -189,17 +229,17 @@ func (h *kmsPluginProbe) Check() error {
return h.lastResponse.err
}
p, err := h.Service.Encrypt([]byte("ping"))
p, err := h.service.Encrypt([]byte("ping"))
if err != nil {
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
h.ttl = kmsPluginHealthzNegativeTTL
return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err)
}
if _, err := h.Service.Decrypt(p); err != nil {
if _, err := h.service.Decrypt(p); err != nil {
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
h.ttl = kmsPluginHealthzNegativeTTL
return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err)
}
h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()}
@@ -207,8 +247,8 @@ func (h *kmsPluginProbe) Check() error {
return nil
}
// Check gets the healthz status of the KMSv2-Plugin using the Status() method.
func (h *kmsv2PluginProbe) Check() error {
// check gets the healthz status of the KMSv2-Plugin using the Status() method.
func (h *kmsv2PluginProbe) check(ctx context.Context) error {
h.l.Lock()
defer h.l.Unlock()
@@ -216,12 +256,11 @@ func (h *kmsv2PluginProbe) Check() error {
return h.lastResponse.err
}
ctx := context.Background()
p, err := h.Service.Status(ctx)
p, err := h.service.Status(ctx)
if err != nil {
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
h.ttl = kmsPluginHealthzNegativeTTL
return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %v", h.name, err)
return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %w", h.name, err)
}
if err := isKMSv2ProviderHealthy(h.name, p); err != nil {
@@ -249,139 +288,97 @@ func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse)
}
if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil {
return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %v", name, err)
return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %w", name, err)
}
return nil
}
// GetTransformerOverrides returns the transformer overrides by reading and parsing the encryption provider configuration file
func GetTransformerOverrides(filepath string) (map[schema.GroupResource]value.Transformer, error) {
// loadConfig parses the encryption configuration file at filepath and returns the parsed config and hash of the file.
func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfiguration, string, error) {
f, err := os.Open(filepath)
if err != nil {
return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err)
return nil, "", fmt.Errorf("error opening encryption provider configuration file %q: %w", filepath, err)
}
defer f.Close()
result, err := parseEncryptionConfiguration(f)
data, err := io.ReadAll(f)
if err != nil {
return nil, fmt.Errorf("error while parsing encryption provider configuration file %q: %v", filepath, err)
return nil, "", fmt.Errorf("could not read contents: %w", err)
}
return result, nil
}
func parseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.Transformer, error) {
configFileContents, err := ioutil.ReadAll(f)
if err != nil {
return nil, fmt.Errorf("could not read contents: %v", err)
if len(data) == 0 {
return nil, "", fmt.Errorf("encryption provider configuration file %q is empty", filepath)
}
config, err := loadConfig(configFileContents)
if err != nil {
return nil, fmt.Errorf("error while parsing file: %v", err)
}
resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{}
// For each entry in the configuration
for _, resourceConfig := range config.Resources {
transformers, err := prefixTransformers(&resourceConfig)
if err != nil {
return nil, err
}
// For each resource, create a list of providers to use
for _, resource := range resourceConfig.Resources {
gr := schema.ParseGroupResource(resource)
resourceToPrefixTransformer[gr] = append(
resourceToPrefixTransformer[gr], transformers...)
}
}
result := map[schema.GroupResource]value.Transformer{}
for gr, transList := range resourceToPrefixTransformer {
result[gr] = value.NewMutableTransformer(value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...))
}
return result, nil
}
func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) {
scheme := runtime.NewScheme()
codecs := serializer.NewCodecFactory(scheme)
apiserverconfig.AddToScheme(scheme)
apiserverconfigv1.AddToScheme(scheme)
utilruntime.Must(apiserverconfig.AddToScheme(scheme))
utilruntime.Must(apiserverconfigv1.AddToScheme(scheme))
configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil)
if err != nil {
return nil, err
return nil, "", err
}
config, ok := configObj.(*apiserverconfig.EncryptionConfiguration)
if !ok {
return nil, fmt.Errorf("got unexpected config type: %v", gvk)
return nil, "", fmt.Errorf("got unexpected config type: %v", gvk)
}
return config, validation.ValidateEncryptionConfiguration(config).ToAggregate()
return config, computeEncryptionConfigHash(data), validation.ValidateEncryptionConfiguration(config, reload).ToAggregate()
}
var (
// The factory to create kms service. This is to make writing test easier.
envelopeServiceFactory = envelope.NewGRPCService
func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, *kmsState, error) {
var transformers []value.PrefixTransformer
var probes []healthChecker
var kmsUsed kmsState
// The factory to create kmsv2 service.
envelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService
)
func prefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]value.PrefixTransformer, error) {
var result []value.PrefixTransformer
for _, provider := range config.Providers {
provider := provider
var (
transformer value.PrefixTransformer
err error
transformer value.PrefixTransformer
transformerErr error
probe healthChecker
used *kmsState
)
switch {
case provider.AESGCM != nil:
transformer, err = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
case provider.AESCBC != nil:
transformer, err = aesPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1)
case provider.Secretbox != nil:
transformer, err = secretboxPrefixTransformer(provider.Secretbox)
case provider.KMS != nil:
switch provider.KMS.APIVersion {
case kmsAPIVersionV1:
var envelopeService envelope.Service
if envelopeService, err = envelopeServiceFactory(provider.KMS.Endpoint, provider.KMS.Timeout.Duration); err != nil {
return nil, fmt.Errorf("could not configure KMS plugin %q, error: %v", provider.KMS.Name, err)
}
transformer, err = envelopePrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV1)
case kmsAPIVersionV2:
if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
return nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", provider.KMS.Name)
}
transformer, transformerErr = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
var envelopeService envelopekmsv2.Service
if envelopeService, err = envelopeKMSv2ServiceFactory(provider.KMS.Endpoint, provider.KMS.Timeout.Duration); err != nil {
return nil, fmt.Errorf("could not configure KMSv2 plugin %q, error: %v", provider.KMS.Name, err)
}
transformer, err = envelopekmsv2PrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV2)
default:
return nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", provider.KMS.Name, provider.KMS.APIVersion)
case provider.AESCBC != nil:
transformer, transformerErr = aesPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1)
case provider.Secretbox != nil:
transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox)
case provider.KMS != nil:
transformer, probe, used, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh)
if transformerErr == nil {
probes = append(probes, probe)
kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used
kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used
// calculate the maximum timeout for all KMS providers
kmsUsed.kmsTimeoutSum += used.kmsTimeoutSum
}
case provider.Identity != nil:
transformer = value.PrefixTransformer{
Transformer: identity.NewEncryptCheckTransformer(),
Prefix: []byte{},
}
default:
return nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity")
return nil, nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity")
}
if err != nil {
return result, err
if transformerErr != nil {
return nil, nil, nil, transformerErr
}
result = append(result, transformer)
transformers = append(transformers, transformer)
}
return result, nil
return transformers, probes, &kmsUsed, nil
}
type blockTransformerFunc func(cipher.Block) value.Transformer
@@ -393,6 +390,7 @@ func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTran
return result, fmt.Errorf("aes provider has no valid keys")
}
for _, key := range config.Keys {
key := key
if key.Name == "" {
return result, fmt.Errorf("key with invalid name provided")
}
@@ -404,6 +402,7 @@ func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTran
keyTransformers := []value.PrefixTransformer{}
for _, keyData := range config.Keys {
keyData := keyData
key, err := base64.StdEncoding.DecodeString(keyData.Secret)
if err != nil {
return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err)
@@ -440,6 +439,7 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
return result, fmt.Errorf("secretbox provider has no valid keys")
}
for _, key := range config.Keys {
key := key
if key.Name == "" {
return result, fmt.Errorf("key with invalid name provided")
}
@@ -451,6 +451,7 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
keyTransformers := []value.PrefixTransformer{}
for _, keyData := range config.Keys {
keyData := keyData
key, err := base64.StdEncoding.DecodeString(keyData.Secret)
if err != nil {
return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err)
@@ -483,7 +484,82 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
return result, nil
}
func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) {
var (
// The factory to create kms service. This is to make writing test easier.
envelopeServiceFactory = envelope.NewGRPCService
// The factory to create kmsv2 service. Exported for integration tests.
EnvelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService
)
type kmsState struct {
v1Used, v2Used bool
kmsTimeoutSum time.Duration
}
func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, *kmsState, error) {
// we ignore the cancel func because this context should only be canceled when stopCh is closed
ctx, _ := wait.ContextForChannel(stopCh)
kmsName := config.Name
switch config.APIVersion {
case kmsAPIVersionV1:
envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
if err != nil {
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err)
}
probe := &kmsPluginProbe{
name: kmsName,
ttl: kmsPluginHealthzNegativeTTL,
service: envelopeService,
l: &sync.Mutex{},
lastResponse: &kmsPluginHealthzResponse{},
}
transformer := envelopePrefixTransformer(config, envelopeService, kmsTransformerPrefixV1)
return transformer, probe, &kmsState{
v1Used: true,
// for v1 we will do encrypt and decrypt for health check. Since these are serial operations, we will double the timeout.
kmsTimeoutSum: 2 * config.Timeout.Duration,
}, nil
case kmsAPIVersionV2:
if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
}
envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
if err != nil {
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err)
}
probe := &kmsv2PluginProbe{
name: kmsName,
ttl: kmsPluginHealthzNegativeTTL,
service: envelopeService,
l: &sync.Mutex{},
lastResponse: &kmsPluginHealthzResponse{},
}
// using AES-GCM by default for encrypting data with KMSv2
transformer := value.PrefixTransformer{
Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), aestransformer.NewGCMTransformer),
Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"),
}
return transformer, probe, &kmsState{
v2Used: true,
kmsTimeoutSum: config.Timeout.Duration,
}, nil
default:
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
}
}
func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) value.PrefixTransformer {
baseTransformerFunc := func(block cipher.Block) value.Transformer {
// v1.24: write using AES-CBC only but support reads via AES-CBC and AES-GCM (so we can move to AES-GCM)
// v1.25: write using AES-GCM only but support reads via AES-GCM and fallback to AES-CBC for backwards compatibility
@@ -492,33 +568,18 @@ func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelop
return unionTransformers{aestransformer.NewGCMTransformer(block), aestransformer.NewCBCTransformer(block)}
}
envelopeTransformer, err := envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc)
if err != nil {
return value.PrefixTransformer{}, err
}
return value.PrefixTransformer{
Transformer: envelopeTransformer,
Transformer: envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc),
Prefix: []byte(prefix + config.Name + ":"),
}, nil
}
func envelopekmsv2PrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelopekmsv2.Service, prefix string) (value.PrefixTransformer, error) {
// using AES-GCM by default for encrypting data with KMSv2
envelopeTransformer, err := envelopekmsv2.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), aestransformer.NewGCMTransformer)
if err != nil {
return value.PrefixTransformer{}, err
}
return value.PrefixTransformer{
Transformer: envelopeTransformer,
Prefix: []byte(prefix + config.Name + ":"),
}, nil
}
type unionTransformers []value.Transformer
func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, stale bool, err error) {
var errs []error
for i, transformer := range u {
for i := range u {
transformer := u[i]
result, stale, err := transformer.TransformFromStorage(ctx, data, dataCtx)
if err != nil {
errs = append(errs, err)
@@ -537,3 +598,133 @@ func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte
func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, err error) {
return u[0].TransformToStorage(ctx, data, dataCtx)
}
// computeEncryptionConfigHash returns the expected hash for an encryption config file that has been loaded as bytes.
// We use a hash instead of the raw file contents when tracking changes to avoid holding any encryption keys in memory outside of their associated transformers.
// This hash must be used in-memory and not externalized to the process because it has no cross-release stability guarantees.
func computeEncryptionConfigHash(data []byte) string {
return fmt.Sprintf("%x", sha256.Sum256(data))
}
var _ healthz.HealthChecker = &DynamicTransformers{}
// DynamicTransformers holds transformers that may be dynamically updated via a single external actor, likely a controller.
// This struct must avoid locks (even read write locks) as it is inline to all calls to storage.
type DynamicTransformers struct {
transformTracker *atomic.Value
}
type transformTracker struct {
transformerOverrides map[schema.GroupResource]value.Transformer
kmsPluginHealthzCheck healthz.HealthChecker
closeTransformers context.CancelFunc
kmsCloseGracePeriod time.Duration
}
// NewDynamicTransformers returns transformers, health checks for kms providers and an ability to close transformers.
func NewDynamicTransformers(
transformerOverrides map[schema.GroupResource]value.Transformer,
kmsPluginHealthzCheck healthz.HealthChecker,
closeTransformers context.CancelFunc,
kmsCloseGracePeriod time.Duration,
) *DynamicTransformers {
dynamicTransformers := &DynamicTransformers{
transformTracker: &atomic.Value{},
}
tracker := &transformTracker{
transformerOverrides: transformerOverrides,
kmsPluginHealthzCheck: kmsPluginHealthzCheck,
closeTransformers: closeTransformers,
kmsCloseGracePeriod: kmsCloseGracePeriod,
}
dynamicTransformers.transformTracker.Store(tracker)
return dynamicTransformers
}
// Check implements healthz.HealthChecker
func (d *DynamicTransformers) Check(req *http.Request) error {
return d.transformTracker.Load().(*transformTracker).kmsPluginHealthzCheck.Check(req)
}
// Name implements healthz.HealthChecker
func (d *DynamicTransformers) Name() string {
return kmsReloadHealthCheckName
}
// TransformerForResource returns the transformer for the given resource.
func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResource) value.Transformer {
return &resourceTransformer{
resource: resource,
transformTracker: d.transformTracker,
}
}
// Set sets the transformer overrides. This method is not go routine safe and must only be called by the same, single caller throughout the lifetime of this object.
func (d *DynamicTransformers) Set(
transformerOverrides map[schema.GroupResource]value.Transformer,
closeTransformers context.CancelFunc,
kmsPluginHealthzCheck healthz.HealthChecker,
kmsCloseGracePeriod time.Duration,
) {
// store new values
newTransformTracker := &transformTracker{
transformerOverrides: transformerOverrides,
closeTransformers: closeTransformers,
kmsPluginHealthzCheck: kmsPluginHealthzCheck,
kmsCloseGracePeriod: kmsCloseGracePeriod,
}
// update new transformer overrides
oldTransformTracker := d.transformTracker.Swap(newTransformTracker).(*transformTracker)
// close old transformers once we wait for grpc request to finish any in-flight requests.
// by the time we spawn this go routine, the new transformers have already been set and will be used for new requests.
// if the server starts shutting down during sleep duration then the transformers will correctly closed early because their lifetime is tied to the api-server drain notifier.
go func() {
time.Sleep(oldTransformTracker.kmsCloseGracePeriod)
oldTransformTracker.closeTransformers()
}()
}
var _ value.Transformer = &resourceTransformer{}
type resourceTransformer struct {
resource schema.GroupResource
transformTracker *atomic.Value
}
func (r *resourceTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
return r.transformer().TransformFromStorage(ctx, data, dataCtx)
}
func (r *resourceTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
return r.transformer().TransformToStorage(ctx, data, dataCtx)
}
func (r *resourceTransformer) transformer() value.Transformer {
transformer := r.transformTracker.Load().(*transformTracker).transformerOverrides[r.resource]
if transformer == nil {
return identity.NewEncryptCheckTransformer()
}
return transformer
}
type ResourceTransformers interface {
TransformerForResource(resource schema.GroupResource) value.Transformer
}
var _ ResourceTransformers = &DynamicTransformers{}
var _ ResourceTransformers = &StaticTransformers{}
type StaticTransformers map[schema.GroupResource]value.Transformer
// StaticTransformers
func (s StaticTransformers) TransformerForResource(resource schema.GroupResource) value.Transformer {
transformer := s[resource]
if transformer == nil {
return identity.NewEncryptCheckTransformer()
}
return transformer
}

265
vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go generated vendored Normal file
View File

@@ -0,0 +1,265 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"context"
"fmt"
"net/http"
"time"
"github.com/fsnotify/fsnotify"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog/v2"
)
// workqueueKey is the dummy key used to process change in encryption config file.
const workqueueKey = "key"
// DynamicKMSEncryptionConfigContent which can dynamically handle changes in encryption config file.
type DynamicKMSEncryptionConfigContent struct {
name string
// filePath is the path of the file to read.
filePath string
// lastLoadedEncryptionConfigHash stores last successfully read encryption config file content.
lastLoadedEncryptionConfigHash string
// queue for processing changes in encryption config file.
queue workqueue.RateLimitingInterface
// dynamicTransformers updates the transformers when encryption config file changes.
dynamicTransformers *encryptionconfig.DynamicTransformers
// stopCh used here is a lifecycle signal of genericapiserver already drained while shutting down.
stopCh <-chan struct{}
}
// NewDynamicKMSEncryptionConfiguration returns controller that dynamically reacts to changes in encryption config file.
func NewDynamicKMSEncryptionConfiguration(
name, filePath string,
dynamicTransformers *encryptionconfig.DynamicTransformers,
configContentHash string,
stopCh <-chan struct{},
) *DynamicKMSEncryptionConfigContent {
encryptionConfig := &DynamicKMSEncryptionConfigContent{
name: name,
filePath: filePath,
lastLoadedEncryptionConfigHash: configContentHash,
dynamicTransformers: dynamicTransformers,
stopCh: stopCh,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("%s-hot-reload", name)),
}
encryptionConfig.queue.Add(workqueueKey)
return encryptionConfig
}
// Run starts the controller and blocks until stopCh is closed.
func (d *DynamicKMSEncryptionConfigContent) Run(ctx context.Context) {
defer utilruntime.HandleCrash()
defer d.queue.ShutDown()
klog.InfoS("Starting controller", "name", d.name)
defer klog.InfoS("Shutting down controller", "name", d.name)
// start worker for processing content
go wait.Until(d.runWorker, time.Second, ctx.Done())
// start the loop that watches the encryption config file until stopCh is closed.
go wait.Until(func() {
if err := d.watchEncryptionConfigFile(ctx.Done()); err != nil {
// if there is an error while setting up or handling the watches, this will ensure that we will process the config file.
defer d.queue.Add(workqueueKey)
klog.ErrorS(err, "Failed to watch encryption config file, will retry later")
}
}, time.Second, ctx.Done())
<-ctx.Done()
}
func (d *DynamicKMSEncryptionConfigContent) watchEncryptionConfigFile(stopCh <-chan struct{}) error {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return fmt.Errorf("error creating fsnotify watcher: %w", err)
}
defer watcher.Close()
if err = watcher.Add(d.filePath); err != nil {
return fmt.Errorf("error adding watch for file %s: %w", d.filePath, err)
}
for {
select {
case event := <-watcher.Events:
if err := d.handleWatchEvent(event, watcher); err != nil {
return err
}
case err := <-watcher.Errors:
return fmt.Errorf("received fsnotify error: %w", err)
case <-stopCh:
return nil
}
}
}
func (d *DynamicKMSEncryptionConfigContent) handleWatchEvent(event fsnotify.Event, watcher *fsnotify.Watcher) error {
// This should be executed after restarting the watch (if applicable) to ensure no file event will be missing.
defer d.queue.Add(workqueueKey)
// return if file has not been removed or renamed.
if event.Op&(fsnotify.Remove|fsnotify.Rename) == 0 {
return nil
}
if err := watcher.Remove(d.filePath); err != nil {
klog.V(2).InfoS("Failed to remove file watch, it may have been deleted", "file", d.filePath, "err", err)
}
if err := watcher.Add(d.filePath); err != nil {
return fmt.Errorf("error adding watch for file %s: %w", d.filePath, err)
}
return nil
}
// runWorker to process file content
func (d *DynamicKMSEncryptionConfigContent) runWorker() {
for d.processNextWorkItem() {
}
}
// processNextWorkItem processes file content when there is a message in the queue.
func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem() bool {
// key here is dummy item in the queue to trigger file content processing.
key, quit := d.queue.Get()
if quit {
return false
}
defer d.queue.Done(key)
var (
updatedEffectiveConfig bool
err error
encryptionConfiguration *encryptionconfig.EncryptionConfiguration
configChanged bool
)
// get context to close the new transformers.
ctx, closeTransformers := wait.ContextForChannel(d.stopCh)
defer func() {
// TODO: increment success metric when updatedEffectiveConfig=true
if !updatedEffectiveConfig {
// avoid leaking if we're not using the newly constructed transformers (due to an error or them not being changed)
closeTransformers()
}
if err != nil {
// TODO: increment failure metric
utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err))
// add dummy item back to the queue to trigger file content processing.
d.queue.AddRateLimited(key)
}
}()
encryptionConfiguration, configChanged, err = d.processEncryptionConfig(ctx)
if err != nil {
return true
}
if !configChanged {
return true
}
if len(encryptionConfiguration.HealthChecks) != 1 {
err = fmt.Errorf("unexpected number of healthz checks: %d. Should have only one", len(encryptionConfiguration.HealthChecks))
return true
}
// get healthz checks for all new KMS plugins.
if err = d.validateNewTransformersHealth(ctx, encryptionConfiguration.HealthChecks[0], encryptionConfiguration.KMSCloseGracePeriod); err != nil {
return true
}
// update transformers.
// when reload=true there must always be one healthz check.
d.dynamicTransformers.Set(
encryptionConfiguration.Transformers,
closeTransformers,
encryptionConfiguration.HealthChecks[0],
encryptionConfiguration.KMSCloseGracePeriod,
)
// update local copy of recent config content once update is successful.
d.lastLoadedEncryptionConfigHash = encryptionConfiguration.EncryptionFileContentHash
klog.V(2).InfoS("Loaded new kms encryption config content", "name", d.name)
updatedEffectiveConfig = true
return true
}
// loadEncryptionConfig processes the next set of content from the file.
func (d *DynamicKMSEncryptionConfigContent) processEncryptionConfig(ctx context.Context) (
encryptionConfiguration *encryptionconfig.EncryptionConfiguration,
configChanged bool,
err error,
) {
// this code path will only execute if reload=true. So passing true explicitly.
encryptionConfiguration, err = encryptionconfig.LoadEncryptionConfig(d.filePath, true, ctx.Done())
if err != nil {
return nil, false, err
}
// check if encryptionConfig is different from the current. Do nothing if they are the same.
if encryptionConfiguration.EncryptionFileContentHash == d.lastLoadedEncryptionConfigHash {
klog.V(4).InfoS("Encryption config has not changed", "name", d.name)
return nil, false, nil
}
return encryptionConfiguration, true, nil
}
func (d *DynamicKMSEncryptionConfigContent) validateNewTransformersHealth(
ctx context.Context,
kmsPluginHealthzCheck healthz.HealthChecker,
kmsPluginCloseGracePeriod time.Duration,
) error {
// test if new transformers are healthy
var healthCheckError error
if kmsPluginCloseGracePeriod < 10*time.Second {
kmsPluginCloseGracePeriod = 10 * time.Second
}
pollErr := wait.PollImmediate(100*time.Millisecond, kmsPluginCloseGracePeriod, func() (bool, error) {
// create a fake http get request to health check endpoint
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("/healthz/%s", kmsPluginHealthzCheck.Name()), nil)
if err != nil {
return false, err
}
healthCheckError = kmsPluginHealthzCheck.Check(req)
return healthCheckError == nil, nil
})
if pollErr != nil {
return fmt.Errorf("health check for new transformers failed, polling error %v: %w", pollErr, healthCheckError)
}
klog.V(2).InfoS("Health check succeeded")
return nil
}

255
vendor/k8s.io/apiserver/pkg/server/options/etcd.go generated vendored
View File

@@ -27,23 +27,26 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/registry/generic"
genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
kmsconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
"k8s.io/apiserver/pkg/storage/value"
flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
"k8s.io/klog/v2"
)
type EtcdOptions struct {
// The value of Paging on StorageConfig will be overridden by the
// calculated feature gate value.
StorageConfig storagebackend.Config
EncryptionProviderConfigFilepath string
StorageConfig storagebackend.Config
EncryptionProviderConfigFilepath string
EncryptionProviderConfigAutomaticReload bool
EtcdServersOverrides []string
@@ -59,6 +62,15 @@ type EtcdOptions struct {
DefaultWatchCacheSize int
// WatchCacheSizes represents override to a given resource
WatchCacheSizes []string
// complete guards fields that must be initialized via Complete before the Apply methods can be used.
complete bool
resourceTransformers encryptionconfig.ResourceTransformers
kmsPluginHealthzChecks []healthz.HealthChecker
// SkipHealthEndpoints, when true, causes the Apply methods to not set up health endpoints.
// This allows multiple invocations of the Apply methods without duplication of said endpoints.
SkipHealthEndpoints bool
}
var storageTypes = sets.NewString(
@@ -107,10 +119,14 @@ func (s *EtcdOptions) Validate() []error {
}
if len(s.EncryptionProviderConfigFilepath) == 0 && s.EncryptionProviderConfigAutomaticReload {
allErrors = append(allErrors, fmt.Errorf("--encryption-provider-config-automatic-reload must be set with --encryption-provider-config"))
}
return allErrors
}
// AddEtcdFlags adds flags related to etcd storage for a specific APIServer to the specified FlagSet
// AddFlags adds flags related to etcd storage for a specific APIServer to the specified FlagSet
func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
if s == nil {
return
@@ -123,7 +139,8 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.DefaultStorageMediaType, "storage-media-type", s.DefaultStorageMediaType, ""+
"The media type to use to store objects in storage. "+
"Some resources or storage backends may only support a specific media type and will ignore this setting.")
"Some resources or storage backends may only support a specific media type and will ignore this setting. "+
"Supported media types: [application/json, application/yaml, application/vnd.kubernetes.protobuf]")
fs.IntVar(&s.DeleteCollectionWorkers, "delete-collection-workers", s.DeleteCollectionWorkers,
"Number of workers spawned for DeleteCollection call. These are used to speed up namespace cleanup.")
@@ -171,6 +188,10 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.EncryptionProviderConfigFilepath, "encryption-provider-config", s.EncryptionProviderConfigFilepath,
"The file containing configuration for encryption providers to be used for storing secrets in etcd")
fs.BoolVar(&s.EncryptionProviderConfigAutomaticReload, "encryption-provider-config-automatic-reload", s.EncryptionProviderConfigAutomaticReload,
"Determines if the file set by --encryption-provider-config should be automatically reloaded if the disk contents change. "+
"Setting this to true disables the ability to uniquely identify distinct KMS plugins via the API server healthz endpoints.")
fs.DurationVar(&s.StorageConfig.CompactionInterval, "etcd-compaction-interval", s.StorageConfig.CompactionInterval,
"The interval of compaction requests. If 0, the compaction request from apiserver is disabled.")
@@ -190,39 +211,114 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
"The time in seconds that each lease is reused. A lower value could avoid large number of objects reusing the same lease. Notice that a too small value may cause performance problems at storage layer.")
}
// Complete must be called exactly once before using any of the Apply methods. It is responsible for setting
// up objects that must be created once and reused across multiple invocations such as storage transformers.
// This method mutates the receiver (EtcdOptions). It must never mutate the inputs.
func (s *EtcdOptions) Complete(
storageObjectCountTracker flowcontrolrequest.StorageObjectCountTracker,
stopCh <-chan struct{},
addPostStartHook func(name string, hook server.PostStartHookFunc) error,
) error {
if s == nil {
return nil
}
if s.complete {
return fmt.Errorf("EtcdOptions.Complete called more than once")
}
if len(s.EncryptionProviderConfigFilepath) != 0 {
ctxTransformers, closeTransformers := wait.ContextForChannel(stopCh)
ctxServer, _ := wait.ContextForChannel(stopCh) // explicitly ignore cancel here because we do not own the server's lifecycle
encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, ctxTransformers.Done())
if err != nil {
// in case of error, we want to close partially initialized (if any) transformers
closeTransformers()
return err
}
// enable kms hot reload controller only if the config file is set to be automatically reloaded
if s.EncryptionProviderConfigAutomaticReload {
// with reload=true we will always have 1 health check
if len(encryptionConfiguration.HealthChecks) != 1 {
// in case of error, we want to close partially initialized (if any) transformers
closeTransformers()
return fmt.Errorf("failed to start kms encryption config hot reload controller. only 1 health check should be available when reload is enabled")
}
dynamicTransformers := encryptionconfig.NewDynamicTransformers(encryptionConfiguration.Transformers, encryptionConfiguration.HealthChecks[0], closeTransformers, encryptionConfiguration.KMSCloseGracePeriod)
s.resourceTransformers = dynamicTransformers
s.kmsPluginHealthzChecks = []healthz.HealthChecker{dynamicTransformers}
// add post start hook to start hot reload controller
// adding this hook here will ensure that it gets configured exactly once
err = addPostStartHook(
"start-encryption-provider-config-automatic-reload",
func(hookContext server.PostStartHookContext) error {
kmsConfigController := kmsconfigcontroller.NewDynamicKMSEncryptionConfiguration(
"kms-encryption-config",
s.EncryptionProviderConfigFilepath,
dynamicTransformers,
encryptionConfiguration.EncryptionFileContentHash,
ctxServer.Done(),
)
go kmsConfigController.Run(ctxServer)
return nil
},
)
if err != nil {
// in case of error, we want to close partially initialized (if any) transformers
closeTransformers()
return fmt.Errorf("failed to add post start hook for kms encryption config hot reload controller: %w", err)
}
} else {
s.resourceTransformers = encryptionconfig.StaticTransformers(encryptionConfiguration.Transformers)
s.kmsPluginHealthzChecks = encryptionConfiguration.HealthChecks
}
}
s.StorageConfig.StorageObjectCountTracker = storageObjectCountTracker
s.complete = true
return nil
}
// ApplyTo mutates the provided server.Config. It must never mutate the receiver (EtcdOptions).
func (s *EtcdOptions) ApplyTo(c *server.Config) error {
if s == nil {
return nil
}
if err := s.addEtcdHealthEndpoint(c); err != nil {
return err
return s.ApplyWithStorageFactoryTo(&SimpleStorageFactory{StorageConfig: s.StorageConfig}, c)
}
// ApplyWithStorageFactoryTo mutates the provided server.Config. It must never mutate the receiver (EtcdOptions).
func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFactory, c *server.Config) error {
if s == nil {
return nil
}
transformerOverrides := make(map[schema.GroupResource]value.Transformer)
if len(s.EncryptionProviderConfigFilepath) > 0 {
var err error
transformerOverrides, err = encryptionconfig.GetTransformerOverrides(s.EncryptionProviderConfigFilepath)
if err != nil {
if !s.complete {
return fmt.Errorf("EtcdOptions.Apply called without completion")
}
if !s.SkipHealthEndpoints {
if err := s.addEtcdHealthEndpoint(c); err != nil {
return err
}
}
// use the StorageObjectCountTracker interface instance from server.Config
s.StorageConfig.StorageObjectCountTracker = c.StorageObjectCountTracker
c.RESTOptionsGetter = &SimpleRestOptionsFactory{
Options: *s,
TransformerOverrides: transformerOverrides,
if s.resourceTransformers != nil {
factory = &transformerStorageFactory{
delegate: factory,
resourceTransformers: s.resourceTransformers,
}
}
return nil
}
func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFactory, c *server.Config) error {
if err := s.addEtcdHealthEndpoint(c); err != nil {
return err
}
// use the StorageObjectCountTracker interface instance from server.Config
s.StorageConfig.StorageObjectCountTracker = c.StorageObjectCountTracker
c.RESTOptionsGetter = &StorageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory}
return nil
@@ -245,57 +341,11 @@ func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
return readyCheck()
}))
if s.EncryptionProviderConfigFilepath != "" {
kmsPluginHealthzChecks, err := encryptionconfig.GetKMSPluginHealthzCheckers(s.EncryptionProviderConfigFilepath)
if err != nil {
return err
}
c.AddHealthChecks(kmsPluginHealthzChecks...)
}
c.AddHealthChecks(s.kmsPluginHealthzChecks...)
return nil
}
type SimpleRestOptionsFactory struct {
Options EtcdOptions
TransformerOverrides map[schema.GroupResource]value.Transformer
}
func (f *SimpleRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
ret := generic.RESTOptions{
StorageConfig: f.Options.StorageConfig.ForResource(resource),
Decorator: generic.UndecoratedStorage,
EnableGarbageCollection: f.Options.EnableGarbageCollection,
DeleteCollectionWorkers: f.Options.DeleteCollectionWorkers,
ResourcePrefix: resource.Group + "/" + resource.Resource,
CountMetricPollPeriod: f.Options.StorageConfig.CountMetricPollPeriod,
StorageObjectCountTracker: f.Options.StorageConfig.StorageObjectCountTracker,
}
if f.TransformerOverrides != nil {
if transformer, ok := f.TransformerOverrides[resource]; ok {
ret.StorageConfig.Transformer = transformer
}
}
if f.Options.EnableWatchCache {
sizes, err := ParseWatchCacheSizes(f.Options.WatchCacheSizes)
if err != nil {
return generic.RESTOptions{}, err
}
size, ok := sizes[resource]
if ok && size > 0 {
klog.Warningf("Dropping watch-cache-size for %v - watchCache size is now dynamic", resource)
}
if ok && size <= 0 {
klog.V(3).InfoS("Not using watch cache", "resource", resource)
ret.Decorator = generic.UndecoratedStorage
} else {
klog.V(3).InfoS("Using watch cache", "resource", resource)
ret.Decorator = genericregistry.StorageWithCacher()
}
}
return ret, nil
}
type StorageFactoryRestOptionsFactory struct {
Options EtcdOptions
StorageFactory serverstorage.StorageFactory
@@ -316,6 +366,7 @@ func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupR
CountMetricPollPeriod: f.Options.StorageConfig.CountMetricPollPeriod,
StorageObjectCountTracker: f.Options.StorageConfig.StorageObjectCountTracker,
}
if f.Options.EnableWatchCache {
sizes, err := ParseWatchCacheSizes(f.Options.WatchCacheSizes)
if err != nil {
@@ -326,8 +377,10 @@ func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupR
klog.Warningf("Dropping watch-cache-size for %v - watchCache size is now dynamic", resource)
}
if ok && size <= 0 {
klog.V(3).InfoS("Not using watch cache", "resource", resource)
ret.Decorator = generic.UndecoratedStorage
} else {
klog.V(3).InfoS("Using watch cache", "resource", resource)
ret.Decorator = genericregistry.StorageWithCacher()
}
}
@@ -369,3 +422,55 @@ func WriteWatchCacheSizes(watchCacheSizes map[schema.GroupResource]int) ([]strin
}
return cacheSizes, nil
}
var _ serverstorage.StorageFactory = &SimpleStorageFactory{}
// SimpleStorageFactory provides a StorageFactory implementation that should be used when different
// resources essentially share the same storage config (as defined by the given storagebackend.Config).
// It assumes the resources are stored at a path that is purely based on the schema.GroupResource.
// Users that need flexibility and per resource overrides should use DefaultStorageFactory instead.
type SimpleStorageFactory struct {
StorageConfig storagebackend.Config
}
func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
return s.StorageConfig.ForResource(resource), nil
}
func (s *SimpleStorageFactory) ResourcePrefix(resource schema.GroupResource) string {
return resource.Group + "/" + resource.Resource
}
func (s *SimpleStorageFactory) Backends() []serverstorage.Backend {
// nothing should ever call this method but we still provide a functional implementation
return serverstorage.Backends(s.StorageConfig)
}
var _ serverstorage.StorageFactory = &transformerStorageFactory{}
type transformerStorageFactory struct {
delegate serverstorage.StorageFactory
resourceTransformers encryptionconfig.ResourceTransformers
}
func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
config, err := t.delegate.NewConfig(resource)
if err != nil {
return nil, err
}
configCopy := *config
resourceConfig := configCopy.Config
resourceConfig.Transformer = t.resourceTransformers.TransformerForResource(resource)
configCopy.Config = resourceConfig
return &configCopy, nil
}
func (t *transformerStorageFactory) ResourcePrefix(resource schema.GroupResource) string {
return t.delegate.ResourcePrefix(resource)
}
func (t *transformerStorageFactory) Backends() []serverstorage.Backend {
return t.delegate.Backends()
}

5
vendor/k8s.io/apiserver/pkg/server/options/recommended.go generated vendored
View File

@@ -101,6 +101,9 @@ func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) {
// ApplyTo adds RecommendedOptions to the server configuration.
// pluginInitializers can be empty, it is only need for additional initializers.
func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
if err := o.Etcd.Complete(config.Config.StorageObjectCountTracker, config.Config.DrainedNotify(), config.Config.AddPostStartHook); err != nil {
return err
}
if err := o.Etcd.ApplyTo(&config.Config); err != nil {
return err
}
@@ -141,7 +144,7 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
}
config.FlowControl = utilflowcontrol.New(
config.SharedInformerFactory,
kubernetes.NewForConfigOrDie(config.ClientConfig).FlowcontrolV1beta2(),
kubernetes.NewForConfigOrDie(config.ClientConfig).FlowcontrolV1beta3(),
config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight,
config.RequestTimeout/4,
)

1
vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go generated vendored
View File

@@ -208,6 +208,7 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
deprecatedMasterServiceNamespace := metav1.NamespaceDefault
fs.StringVar(&deprecatedMasterServiceNamespace, "master-service-namespace", deprecatedMasterServiceNamespace, ""+
"DEPRECATED: the namespace from which the Kubernetes master services should be injected into pods.")
fs.MarkDeprecated("master-service-namespace", "This flag will be removed in v1.27")
fs.IntVar(&s.MaxRequestsInFlight, "max-requests-inflight", s.MaxRequestsInFlight, ""+
"This and --max-mutating-requests-inflight are summed to determine the server's total concurrency limit "+

15
vendor/k8s.io/apiserver/pkg/server/options/tracing.go generated vendored
View File

@@ -23,9 +23,9 @@ import (
"net"
"github.com/spf13/pflag"
"go.opentelemetry.io/otel/exporters/otlp/otlpgrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/sdk/resource"
"go.opentelemetry.io/otel/semconv"
"go.opentelemetry.io/otel/semconv/v1.12.0"
"google.golang.org/grpc"
"k8s.io/apimachinery/pkg/runtime"
@@ -93,7 +93,7 @@ func (o *TracingOptions) ApplyTo(es *egressselector.EgressSelector, c *server.Co
return fmt.Errorf("failed to validate tracing configuration: %v", errs.ToAggregate())
}
opts := []otlpgrpc.Option{}
opts := []otlptracegrpc.Option{}
if es != nil {
// Only use the egressselector dialer if egressselector is enabled.
// Endpoint is on the "ControlPlane" network
@@ -101,11 +101,12 @@ func (o *TracingOptions) ApplyTo(es *egressselector.EgressSelector, c *server.Co
if err != nil {
return err
}
otelDialer := func(ctx context.Context, addr string) (net.Conn, error) {
return egressDialer(ctx, "tcp", addr)
if egressDialer != nil {
otelDialer := func(ctx context.Context, addr string) (net.Conn, error) {
return egressDialer(ctx, "tcp", addr)
}
opts = append(opts, otlptracegrpc.WithDialOption(grpc.WithContextDialer(otelDialer)))
}
opts = append(opts, otlpgrpc.WithDialOption(grpc.WithContextDialer(otelDialer)))
}
resourceOpts := []resource.Option{