feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

vendor/k8s.io/apiserver/pkg/apis/config/types.go

@@ -24,7 +24,49 @@ import (
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
-// EncryptionConfiguration stores the complete configuration for encryption providers.
+/*
+EncryptionConfiguration stores the complete configuration for encryption providers.
+It also allows the use of wildcards to specify the resources that should be encrypted.
+Use '*.<group>' to encrypt all resources within a group or '*.*' to encrypt all resources.
+'*.' can be used to encrypt all resource in the core group.  '*.*' will encrypt all
+resources, even custom resources that are added after API server start.
+Use of wildcards that overlap within the same resource list or across multiple
+entries are not allowed since part of the configuration would be ineffective.
+Resource lists are processed in order, with earlier lists taking precedence.
+
+Example:
+
+	kind: EncryptionConfiguration
+	apiVersion: apiserver.config.k8s.io/v1
+	resources:
+	- resources:
+	  - events
+	  providers:
+	  - identity: {}  # do not encrypt events even though *.* is specified below
+	- resources:
+	  - secrets
+	  - configmaps
+	  - pandas.awesome.bears.example
+	  providers:
+	  - aescbc:
+	      keys:
+	      - name: key1
+	        secret: c2VjcmV0IGlzIHNlY3VyZQ==
+	- resources:
+	  - '*.apps'
+	  providers:
+	  - aescbc:
+	      keys:
+	      - name: key2
+	        secret: c2VjcmV0IGlzIHNlY3VyZSwgb3IgaXMgaXQ/Cg==
+	- resources:
+	  - '*.*'
+	  providers:
+	  - aescbc:
+	      keys:
+	      - name: key3
+	        secret: c2VjcmV0IGlzIHNlY3VyZSwgSSB0aGluaw==
+*/
 type EncryptionConfiguration struct {
 	metav1.TypeMeta
 	// resources is a list containing resources, and their corresponding encryption providers.
@@ -33,10 +75,14 @@ type EncryptionConfiguration struct {
 
 // ResourceConfiguration stores per resource configuration.
 type ResourceConfiguration struct {
-	// resources is a list of kubernetes resources which have to be encrypted.
+	// resources is a list of kubernetes resources which have to be encrypted. The resource names are derived from `resource` or `resource.group` of the group/version/resource.
+	// eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas.
+	// Use '*.*' to encrypt all resources and '*.<group>' to encrypt all resources in a specific group.
+	// eg: '*.awesome.bears.example' will encrypt all resources in the group 'awesome.bears.example'.
+	// eg: '*.' will encrypt all resources in the core group (such as pods, configmaps, etc).
 	Resources []string
 	// providers is a list of transformers to be used for reading and writing the resources to disk.
-	// eg: aesgcm, aescbc, secretbox, identity.
+	// eg: aesgcm, aescbc, secretbox, identity, kms.
 	Providers []ProviderConfiguration
 }
 
@@ -92,7 +138,7 @@ type KMSConfiguration struct {
 	// name is the name of the KMS plugin to be used.
 	Name string
 	// cachesize is the maximum number of secrets which are cached in memory. The default value is 1000.
-	// Set to a negative value to disable caching.
+	// Set to a negative value to disable caching. This field is only allowed for KMS v1 providers.
 	// +optional
 	CacheSize *int32
 	// endpoint is the gRPC server listening address, for example "unix:///var/run/kms-provider.sock".

vendor/k8s.io/apiserver/pkg/apis/config/v1/defaults.go

@@ -39,11 +39,12 @@ func SetDefaults_KMSConfiguration(obj *KMSConfiguration) {
 		obj.Timeout = defaultTimeout
 	}
 
-	if obj.CacheSize == nil {
-		obj.CacheSize = &defaultCacheSize
-	}
-
 	if obj.APIVersion == "" {
 		obj.APIVersion = defaultAPIVersion
 	}
+
+	// cacheSize is relevant only for kms v1
+	if obj.CacheSize == nil && obj.APIVersion == "v1" {
+		obj.CacheSize = &defaultCacheSize
+	}
 }

vendor/k8s.io/apiserver/pkg/apis/config/v1/types.go

@@ -24,7 +24,49 @@ import (
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
-// EncryptionConfiguration stores the complete configuration for encryption providers.
+/*
+EncryptionConfiguration stores the complete configuration for encryption providers.
+It also allows the use of wildcards to specify the resources that should be encrypted.
+Use '*.<group>' to encrypt all resources within a group or '*.*' to encrypt all resources.
+'*.' can be used to encrypt all resource in the core group.  '*.*' will encrypt all
+resources, even custom resources that are added after API server start.
+Use of wildcards that overlap within the same resource list or across multiple
+entries are not allowed since part of the configuration would be ineffective.
+Resource lists are processed in order, with earlier lists taking precedence.
+
+Example:
+
+	kind: EncryptionConfiguration
+	apiVersion: apiserver.config.k8s.io/v1
+	resources:
+	- resources:
+	  - events
+	  providers:
+	  - identity: {}  # do not encrypt events even though *.* is specified below
+	- resources:
+	  - secrets
+	  - configmaps
+	  - pandas.awesome.bears.example
+	  providers:
+	  - aescbc:
+	      keys:
+	      - name: key1
+	        secret: c2VjcmV0IGlzIHNlY3VyZQ==
+	- resources:
+	  - '*.apps'
+	  providers:
+	  - aescbc:
+	      keys:
+	      - name: key2
+	        secret: c2VjcmV0IGlzIHNlY3VyZSwgb3IgaXMgaXQ/Cg==
+	- resources:
+	  - '*.*'
+	  providers:
+	  - aescbc:
+	      keys:
+	      - name: key3
+	        secret: c2VjcmV0IGlzIHNlY3VyZSwgSSB0aGluaw==
+*/
 type EncryptionConfiguration struct {
 	metav1.TypeMeta
 	// resources is a list containing resources, and their corresponding encryption providers.
@@ -33,10 +75,14 @@ type EncryptionConfiguration struct {
 
 // ResourceConfiguration stores per resource configuration.
 type ResourceConfiguration struct {
-	// resources is a list of kubernetes resources which have to be encrypted.
+	// resources is a list of kubernetes resources which have to be encrypted. The resource names are derived from `resource` or `resource.group` of the group/version/resource.
+	// eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas.
+	// Use '*.*' to encrypt all resources and '*.<group>' to encrypt all resources in a specific group.
+	// eg: '*.awesome.bears.example' will encrypt all resources in the group 'awesome.bears.example'.
+	// eg: '*.' will encrypt all resources in the core group (such as pods, configmaps, etc).
 	Resources []string `json:"resources"`
 	// providers is a list of transformers to be used for reading and writing the resources to disk.
-	// eg: aesgcm, aescbc, secretbox, identity.
+	// eg: aesgcm, aescbc, secretbox, identity, kms.
 	Providers []ProviderConfiguration `json:"providers"`
 }
 
@@ -92,7 +138,7 @@ type KMSConfiguration struct {
 	// name is the name of the KMS plugin to be used.
 	Name string `json:"name"`
 	// cachesize is the maximum number of secrets which are cached in memory. The default value is 1000.
-	// Set to a negative value to disable caching.
+	// Set to a negative value to disable caching. This field is only allowed for KMS v1 providers.
 	// +optional
 	CacheSize *int32 `json:"cachesize,omitempty"`
 	// endpoint is the gRPC server listening address, for example "unix:///var/run/kms-provider.sock".

vendor/k8s.io/apiserver/pkg/apis/config/validation/validation.go

@@ -23,6 +23,7 @@ import (
 	"net/url"
 	"strings"
 
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/apiserver/pkg/apis/config"
@@ -34,7 +35,7 @@ const (
 	unsupportedSchemeErrFmt        = "unsupported scheme %q for KMS provider, only unix is supported"
 	unsupportedKMSAPIVersionErrFmt = "unsupported apiVersion %s for KMS provider, only v1 and v2 are supported"
 	atLeastOneRequiredErrFmt       = "at least one %s is required"
-	invalidURLErrFmt               = "invalid endpoint for kms provider, error: parse %s: net/url: invalid control character in URL"
+	invalidURLErrFmt               = "invalid endpoint for kms provider, error: %v"
 	mandatoryFieldErrFmt           = "%s is a mandatory field for a %s"
 	base64EncodingErr              = "secrets must be base64 encoded"
 	zeroOrNegativeErrFmt           = "%s should be a positive value"
@@ -42,6 +43,14 @@ const (
 	encryptionConfigNilErr         = "EncryptionConfiguration can't be nil"
 	invalidKMSConfigNameErrFmt     = "invalid KMS provider name %s, must not contain ':'"
 	duplicateKMSConfigNameErrFmt   = "duplicate KMS provider name %s, names must be unique"
+	eventsGroupErr                 = "'*.events.k8s.io' objects are stored using the 'events' API group in etcd. Use 'events' instead in the config file"
+	extensionsGroupErr             = "'extensions' group has been removed and cannot be used for encryption"
+	starResourceErr                = "use '*.' to encrypt all the resources from core API group or *.* to encrypt all resources"
+	overlapErr                     = "using overlapping resources such as 'secrets' and '*.' in the same resource list is not allowed as they will be masked"
+	nonRESTAPIResourceErr          = "resources which do not have REST API/s cannot be encrypted"
+	resourceNameErr                = "resource name should not contain capital letters"
+	resourceAcrossGroupErr         = "encrypting the same resource across groups is not supported"
+	duplicateResourceErr           = "the same resource cannot be specified multiple times"
 )
 
 var (
@@ -59,7 +68,7 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload b
 	allErrs := field.ErrorList{}
 
 	if c == nil {
-		allErrs = append(allErrs, field.Required(root, "EncryptionConfiguration can't be nil"))
+		allErrs = append(allErrs, field.Required(root, encryptionConfigNilErr))
 		return allErrs
 	}
 
@@ -78,6 +87,9 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload b
 			allErrs = append(allErrs, field.Required(r, fmt.Sprintf(atLeastOneRequiredErrFmt, r)))
 		}
 
+		allErrs = append(allErrs, validateResourceOverlap(conf.Resources, r)...)
+		allErrs = append(allErrs, validateResourceNames(conf.Resources, r)...)
+
 		if len(conf.Providers) == 0 {
 			allErrs = append(allErrs, field.Required(p, fmt.Sprintf(atLeastOneRequiredErrFmt, p)))
 		}
@@ -103,6 +115,175 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload b
 	return allErrs
 }
 
+var anyGroupAnyResource = schema.GroupResource{
+	Group:    "*",
+	Resource: "*",
+}
+
+func validateResourceOverlap(resources []string, fieldPath *field.Path) field.ErrorList {
+	if len(resources) < 2 { // cannot have overlap with a single resource
+		return nil
+	}
+
+	var allErrs field.ErrorList
+
+	r := make([]schema.GroupResource, 0, len(resources))
+	for _, resource := range resources {
+		r = append(r, schema.ParseGroupResource(resource))
+	}
+
+	var hasOverlap, hasDuplicate bool
+
+	for i, r1 := range r {
+		for j, r2 := range r {
+			if i == j {
+				continue
+			}
+
+			if r1 == r2 && !hasDuplicate {
+				hasDuplicate = true
+				continue
+			}
+
+			if hasOverlap {
+				continue
+			}
+
+			if r1 == anyGroupAnyResource {
+				hasOverlap = true
+				continue
+			}
+
+			if r1.Group != r2.Group {
+				continue
+			}
+
+			if r1.Resource == "*" || r2.Resource == "*" {
+				hasOverlap = true
+				continue
+			}
+		}
+	}
+
+	if hasDuplicate {
+		allErrs = append(
+			allErrs,
+			field.Invalid(
+				fieldPath,
+				resources,
+				duplicateResourceErr,
+			),
+		)
+	}
+
+	if hasOverlap {
+		allErrs = append(
+			allErrs,
+			field.Invalid(
+				fieldPath,
+				resources,
+				overlapErr,
+			),
+		)
+	}
+
+	return allErrs
+}
+
+func validateResourceNames(resources []string, fieldPath *field.Path) field.ErrorList {
+	var allErrs field.ErrorList
+
+	for j, res := range resources {
+		jj := fieldPath.Index(j)
+
+		// check if resource name has capital letters
+		if hasCapital(res) {
+			allErrs = append(
+				allErrs,
+				field.Invalid(
+					jj,
+					resources[j],
+					resourceNameErr,
+				),
+			)
+			continue
+		}
+
+		// check if resource is '*'
+		if res == "*" {
+			allErrs = append(
+				allErrs,
+				field.Invalid(
+					jj,
+					resources[j],
+					starResourceErr,
+				),
+			)
+			continue
+		}
+
+		// check if resource is:
+		// 'apiserveripinfo' OR
+		// 'serviceipallocations' OR
+		// 'servicenodeportallocations' OR
+		if res == "apiserveripinfo" ||
+			res == "serviceipallocations" ||
+			res == "servicenodeportallocations" {
+			allErrs = append(
+				allErrs,
+				field.Invalid(
+					jj,
+					resources[j],
+					nonRESTAPIResourceErr,
+				),
+			)
+			continue
+		}
+
+		// check if group is 'events.k8s.io'
+		gr := schema.ParseGroupResource(res)
+		if gr.Group == "events.k8s.io" {
+			allErrs = append(
+				allErrs,
+				field.Invalid(
+					jj,
+					resources[j],
+					eventsGroupErr,
+				),
+			)
+			continue
+		}
+
+		// check if group is 'extensions'
+		if gr.Group == "extensions" {
+			allErrs = append(
+				allErrs,
+				field.Invalid(
+					jj,
+					resources[j],
+					extensionsGroupErr,
+				),
+			)
+			continue
+		}
+
+		// disallow resource.* as encrypting the same resource across groups does not make sense
+		if gr.Group == "*" && gr.Resource != "*" {
+			allErrs = append(
+				allErrs,
+				field.Invalid(
+					jj,
+					resources[j],
+					resourceAcrossGroupErr,
+				),
+			)
+			continue
+		}
+	}
+
+	return allErrs
+}
+
 func validateSingleProvider(provider config.ProviderConfiguration, fieldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 	found := 0
@@ -195,7 +376,13 @@ func validateKMSConfiguration(c *config.KMSConfiguration, fieldPath *field.Path,
 
 func validateKMSCacheSize(c *config.KMSConfiguration, fieldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
-	if *c.CacheSize == 0 {
+
+	// In defaulting, we set the cache size to the default value only when API version is v1.
+	// So, for v2 API version, we expect the cache size field to be nil.
+	if c.APIVersion != "v1" && c.CacheSize != nil {
+		allErrs = append(allErrs, field.Invalid(fieldPath, *c.CacheSize, "cachesize is not supported in v2"))
+	}
+	if c.APIVersion == "v1" && *c.CacheSize == 0 {
 		allErrs = append(allErrs, field.Invalid(fieldPath, *c.CacheSize, fmt.Sprintf(nonZeroErrFmt, "cachesize")))
 	}
 
@@ -219,7 +406,7 @@ func validateKMSEndpoint(c *config.KMSConfiguration, fieldPath *field.Path) fiel
 
 	u, err := url.Parse(c.Endpoint)
 	if err != nil {
-		return append(allErrs, field.Invalid(fieldPath, c.Endpoint, fmt.Sprintf("invalid endpoint for kms provider, error: %v", err)))
+		return append(allErrs, field.Invalid(fieldPath, c.Endpoint, fmt.Sprintf(invalidURLErrFmt, err)))
 	}
 
 	if u.Scheme != "unix" {
@@ -259,3 +446,7 @@ func validateKMSConfigName(c *config.KMSConfiguration, fieldPath *field.Path, km
 
 	return allErrs
 }
+
+func hasCapital(input string) bool {
+	return strings.ToLower(input) != input
+}