feat: support service account token auth mode

Signed-off-by: hongming <coder.scala@gmail.com>

cmd/controller-manager/app/options/options.go

@@ -152,6 +152,9 @@ func (s *ControllerManagerOptions) Merge(conf *config.Config) {
 	if conf.TerminalOptions != nil {
 		s.TerminalOptions = conf.TerminalOptions
 	}
+	if conf.KubeconfigOptions != nil {
+		s.KubeconfigOptions = conf.KubeconfigOptions
+	}
 	if conf.HelmExecutorOptions != nil {
 		s.HelmExecutorOptions = conf.HelmExecutorOptions
 	}

cmd/controller-manager/app/server.go

@@ -45,6 +45,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/controller/roletemplate"
 	"kubesphere.io/kubesphere/pkg/controller/secret"
 	"kubesphere.io/kubesphere/pkg/controller/serviceaccount"
+	"kubesphere.io/kubesphere/pkg/controller/serviceaccounttoken"
 	"kubesphere.io/kubesphere/pkg/controller/storageclass"
 	"kubesphere.io/kubesphere/pkg/controller/telemetry"
 	"kubesphere.io/kubesphere/pkg/controller/user"
@@ -118,6 +119,7 @@ func init() {
 	runtime.Must(controller.Register(&application.ReleaseWebhook{}))
 	// kubectl
 	runtime.Must(controller.Register(&kubectl.Reconciler{}))
+	runtime.Must(controller.Register(&serviceaccounttoken.Reconciler{}))
 }
 
 func NewControllerManagerCommand() *cobra.Command {

config/ks-core/templates/kubesphere-config.yaml

@@ -37,6 +37,9 @@ data:
     multicluster:
       clusterRole: {{ include "multicluster.role" . | quote }}
       hostClusterName: {{ include "multicluster.hostClusterName" . | include "validateHostClusterName" | quote }}
+    kubeconfig:
+      # service-account-token client-certificate oidc-token webhook-token
+      authMode: {{ .Values.kubeconfig.authMode | default "client-certificate" }}
     terminal:
       kubectl:
         image: {{ template "kubectl.image" . }}

pkg/config/config.go

@@ -21,6 +21,8 @@ import (
 	"kubesphere.io/kubesphere/pkg/constants"
 	"kubesphere.io/kubesphere/pkg/controller/options"
 	"kubesphere.io/kubesphere/pkg/models/composedapp"
+	"kubesphere.io/kubesphere/pkg/models/kubeconfig"
+	"kubesphere.io/kubesphere/pkg/models/ratelimiter"
 	"kubesphere.io/kubesphere/pkg/models/terminal"
 	"kubesphere.io/kubesphere/pkg/multicluster"
 	"kubesphere.io/kubesphere/pkg/simple/client/cache"
@@ -111,6 +113,7 @@ type Config struct {
 	AuthorizationOptions  *authorization.Options       `json:"authorization,omitempty" yaml:"authorization,omitempty" mapstructure:"authorization"`
 	MultiClusterOptions   *multicluster.Options        `json:"multicluster,omitempty" yaml:"multicluster,omitempty" mapstructure:"multicluster"`
 	AuditingOptions       *auditing.Options            `json:"auditing,omitempty" yaml:"auditing,omitempty" mapstructure:"auditing"`
+	KubeconfigOptions     *kubeconfig.Options          `json:"kubeconfig,omitempty" yaml:"kubeconfig,omitempty" mapstructure:"kubeconfig"`
 	TerminalOptions       *terminal.Options            `json:"terminal,omitempty" yaml:"terminal,omitempty" mapstructure:"terminal"`
 	HelmExecutorOptions   *options.HelmExecutorOptions `json:"helmExecutor,omitempty" yaml:"helmExecutor,omitempty" mapstructure:"helmExecutor"`
 	ExtensionOptions      *options.ExtensionOptions    `json:"extension,omitempty" yaml:"extension,omitempty" mapstructure:"extension"`
@@ -129,6 +132,7 @@ func New() *Config {
 		AuthorizationOptions:  authorization.NewOptions(),
 		MultiClusterOptions:   multicluster.NewOptions(),
 		TerminalOptions:       terminal.NewOptions(),
+		KubeconfigOptions:     kubeconfig.NewOptions(),
 		AuditingOptions:       auditing.NewAuditingOptions(),
 		HelmExecutorOptions:   options.NewHelmExecutorOptions(),
 		ExtensionOptions:      options.NewExtensionOptions(),

pkg/config/config_test.go

@@ -19,6 +19,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/apiserver/authorization"
 	"kubesphere.io/kubesphere/pkg/controller/options"
 	"kubesphere.io/kubesphere/pkg/models/composedapp"
+	"kubesphere.io/kubesphere/pkg/models/kubeconfig"
 	"kubesphere.io/kubesphere/pkg/models/terminal"
 	"kubesphere.io/kubesphere/pkg/multicluster"
 	"kubesphere.io/kubesphere/pkg/simple/client/cache"
@@ -33,6 +34,7 @@ func newTestConfig() (*Config, error) {
 		AuthenticationOptions: authentication.NewOptions(),
 		MultiClusterOptions:   multicluster.NewOptions(),
 		AuditingOptions:       auditing.NewAuditingOptions(),
+		KubeconfigOptions:     kubeconfig.NewOptions(),
 		TerminalOptions:       terminal.NewOptions(),
 		HelmExecutorOptions:   options.NewHelmExecutorOptions(),
 		ExtensionOptions:      options.NewExtensionOptions(),

pkg/controller/certificatesigningrequest/certificatesigningrequest_controller.go

@@ -27,6 +27,7 @@ import (
 
 	"kubesphere.io/kubesphere/pkg/constants"
 	kscontroller "kubesphere.io/kubesphere/pkg/controller"
+	"kubesphere.io/kubesphere/pkg/models/kubeconfig"
 )
 
 const (
@@ -49,6 +50,11 @@ func (r *Reconciler) Name() string {
 }
 
 func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error {
+	if mgr.KubeconfigOptions.AuthMode != kubeconfig.AuthModeClientCertificate {
+		klog.Infof("Skip %s controller as the auth mode is not client certificate", controllerName)
+		return nil
+	}
+
 	r.recorder = mgr.GetEventRecorderFor(controllerName)
 	r.Client = mgr.GetClient()
 	return builder.

pkg/controller/globalrolebinding/globalrolebinding_controller.go

@@ -28,9 +28,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
+	"kubesphere.io/kubesphere/pkg/constants"
 	kscontroller "kubesphere.io/kubesphere/pkg/controller"
 	"kubesphere.io/kubesphere/pkg/controller/cluster/predicate"
 	clusterutils "kubesphere.io/kubesphere/pkg/controller/cluster/utils"
+	"kubesphere.io/kubesphere/pkg/models/kubeconfig"
 	"kubesphere.io/kubesphere/pkg/utils/clusterclient"
 )
 
@@ -178,6 +180,11 @@ func (r *Reconciler) assignClusterAdminRole(ctx context.Context, clusterName str
 				APIGroup: rbacv1.GroupName,
 				Name:     username,
 			},
+			{
+				Kind:      rbacv1.ServiceAccountKind,
+				Name:      fmt.Sprintf(kubeconfig.UserKubeConfigServiceAccountNameFormat, username),
+				Namespace: constants.KubeSphereNamespace,
+			},
 		}
 		clusterRoleBinding.RoleRef = rbacv1.RoleRef{
 			APIGroup: rbacv1.GroupName,

pkg/controller/kubeconfig/kubeconfig_controller.go

@@ -17,6 +17,7 @@ import (
 
 	certificatesv1 "k8s.io/api/certificates/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/client-go/rest"
@@ -24,19 +25,17 @@ import (
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/klog/v2"
-	"sigs.k8s.io/controller-runtime/pkg/builder"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
-
-	"kubesphere.io/kubesphere/pkg/constants"
-	"kubesphere.io/kubesphere/pkg/models/kubeconfig"
-	"kubesphere.io/kubesphere/pkg/utils/pkiutil"
-
-	kscontroller "kubesphere.io/kubesphere/pkg/controller"
-
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	"kubesphere.io/kubesphere/pkg/constants"
+	kscontroller "kubesphere.io/kubesphere/pkg/controller"
+	"kubesphere.io/kubesphere/pkg/models/kubeconfig"
+	"kubesphere.io/kubesphere/pkg/utils/pkiutil"
 )
 
 const (
@@ -51,6 +50,7 @@ var _ reconcile.Reconciler = &Reconciler{}
 type Reconciler struct {
 	client.Client
 	config  *rest.Config
+	options *kubeconfig.Options
 }
 
 func (r *Reconciler) Name() string {
@@ -60,6 +60,7 @@ func (r *Reconciler) Name() string {
 func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error {
 	r.Client = mgr.GetClient()
 	r.config = mgr.K8sClient.Config()
+	r.options = mgr.KubeconfigOptions
 	return ctrl.NewControllerManagedBy(mgr).
 		Named(controllerName).
 		WithOptions(controller.Options{MaxConcurrentReconciles: 1}).
@@ -92,7 +93,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco
 
 func (r *Reconciler) UpdateSecret(ctx context.Context, secret *corev1.Secret) error {
 	// already exist and cert will not expire in 3 days
-	if isValid(secret) {
+	if r.isValid(secret) {
 		return nil
 	}
 
@@ -146,15 +147,24 @@ func (r *Reconciler) UpdateSecret(ctx context.Context, secret *corev1.Secret) er
 		return err
 	}
 
+	if r.options.AuthMode == kubeconfig.AuthModeClientCertificate {
 		if err = r.createCSR(ctx, username); err != nil {
 			klog.Errorf("Failed to create CSR for user %s: %v", username, err)
 			return err
 		}
+	}
+
+	if r.options.AuthMode == kubeconfig.AuthModeServiceAccountToken {
+		if err = r.createServiceAccount(ctx, username); err != nil {
+			klog.Errorf("Failed to create sa for user %s: %v", username, err)
+			return err
+		}
+	}
 
 	return nil
 }
 
-func isValid(secret *corev1.Secret) bool {
+func (r *Reconciler) isValid(secret *corev1.Secret) bool {
 	username := secret.Labels[constants.UsernameLabelKey]
 
 	data := secret.Data[kubeconfig.FileName]
@@ -169,6 +179,10 @@ func isValid(secret *corev1.Secret) bool {
 	}
 
 	if authInfo, ok := config.AuthInfos[username]; ok {
+		if r.options.AuthMode == kubeconfig.AuthModeServiceAccountToken && authInfo.Token != "" {
+			return true
+		}
+		if r.options.AuthMode == kubeconfig.AuthModeClientCertificate {
 			clientCert, err := certutil.ParseCertsPEM(authInfo.ClientCertificateData)
 			if err != nil {
 				klog.Warningf("Failed to parse client certificate for user %s: %v", username, err)
@@ -179,6 +193,7 @@ func isValid(secret *corev1.Secret) bool {
 					return true
 				}
 			}
+		}
 	} else {
 		// in process
 		return true
@@ -247,3 +262,43 @@ func (r *Reconciler) createCSR(ctx context.Context, username string) error {
 
 	return nil
 }
+
+func (r *Reconciler) createServiceAccount(ctx context.Context, username string) error {
+	saName := fmt.Sprintf("kubesphere.users.%s", username)
+	sa := &corev1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      saName,
+			Namespace: constants.KubeSphereNamespace,
+			Labels:    map[string]string{constants.UsernameLabelKey: username},
+		},
+	}
+
+	if err := r.Create(ctx, sa); err != nil {
+		if !errors.IsAlreadyExists(err) {
+			klog.Errorf("Failed to create service account for user %s: %v", username, err)
+			return err
+		}
+	}
+
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s.token", saName),
+			Namespace: constants.KubeSphereNamespace,
+			Annotations: map[string]string{
+				corev1.ServiceAccountNameKey: saName,
+			},
+			Labels: map[string]string{
+				constants.UsernameLabelKey: username,
+			},
+		},
+		Type: corev1.SecretTypeServiceAccountToken,
+	}
+
+	if err := r.Create(ctx, secret); err != nil {
+		if !errors.IsAlreadyExists(err) {
+			klog.Errorf("Failed to create service account for user %s: %v", username, err)
+			return err
+		}
+	}
+	return nil
+}

pkg/controller/options/options.go

@@ -15,6 +15,7 @@ import (
 
 	"kubesphere.io/kubesphere/pkg/apiserver/authentication"
 	"kubesphere.io/kubesphere/pkg/models/composedapp"
+	"kubesphere.io/kubesphere/pkg/models/kubeconfig"
 	"kubesphere.io/kubesphere/pkg/models/terminal"
 	"kubesphere.io/kubesphere/pkg/multicluster"
 	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
@@ -24,6 +25,7 @@ type Options struct {
 	KubernetesOptions     *k8s.Options
 	AuthenticationOptions *authentication.Options
 	MultiClusterOptions   *multicluster.Options
+	KubeconfigOptions     *kubeconfig.Options
 	TerminalOptions       *terminal.Options
 	ComposedAppOptions    *composedapp.Options
 	HelmExecutorOptions   *HelmExecutorOptions

pkg/controller/rolebinding/rolebinding_controller.go

@@ -86,10 +86,12 @@ func (r *Reconciler) syncToKubernetes(ctx context.Context, roleBinding *iamv1bet
 		for _, subject := range roleBinding.Subjects {
 			newSubject := rbacv1.Subject{
 				Kind:      subject.Kind,
-				APIGroup:  rbacv1.GroupName,
 				Name:      subject.Name,
 				Namespace: subject.Namespace,
 			}
+			if subject.APIGroup != "" {
+				newSubject.APIGroup = rbacv1.GroupName
+			}
 			subjects = append(subjects, newSubject)
 		}
 		k8sRolBinding.Subjects = subjects

pkg/controller/serviceaccounttoken/serviceaccount_token_controller.go

@@ -0,0 +1,146 @@
+/*
+Copyright 2019 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package serviceaccounttoken
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/klog/v2"
+
+	"kubesphere.io/kubesphere/pkg/models/kubeconfig"
+
+	kscontroller "kubesphere.io/kubesphere/pkg/controller"
+
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/tools/record"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+
+	"kubesphere.io/kubesphere/pkg/constants"
+)
+
+const (
+	controllerName                 = "service-account-token"
+	userKubeConfigSecretNameFormat = "kubeconfig-%s"
+	kubeconfigFileName             = "config"
+)
+
+var _ kscontroller.Controller = &Reconciler{}
+var _ reconcile.Reconciler = &Reconciler{}
+
+type Reconciler struct {
+	client.Client
+	recorder record.EventRecorder
+}
+
+func (r *Reconciler) Name() string {
+	return controllerName
+}
+
+func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error {
+	if mgr.KubeconfigOptions.AuthMode != kubeconfig.AuthModeServiceAccountToken {
+		klog.Infof("Skip %s controller as the auth mode is not service account token", controllerName)
+		return nil
+	}
+
+	r.recorder = mgr.GetEventRecorderFor(controllerName)
+	r.Client = mgr.GetClient()
+	return builder.
+		ControllerManagedBy(mgr).
+		For(&corev1.Secret{},
+			builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+				if object.GetNamespace() == constants.KubeSphereNamespace && object.GetLabels()[constants.UsernameLabelKey] != "" {
+					return true
+				}
+				return false
+			})),
+		).
+		Named(controllerName).
+		Complete(r)
+}
+
+func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	secret := &corev1.Secret{}
+	if err := r.Get(ctx, req.NamespacedName, secret); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if username := secret.Labels[constants.UsernameLabelKey]; username != "" {
+		if secret.Data == nil {
+			return ctrl.Result{}, nil
+		}
+		token := secret.Data["token"]
+
+		if len(token) > 0 {
+			if err := r.UpdateKubeConfigServiceAccountToken(ctx, username, string(token)); err != nil {
+				// kubeconfig not generated
+				return ctrl.Result{}, err
+			}
+		}
+	}
+
+	r.recorder.Event(secret, corev1.EventTypeNormal, kscontroller.Synced, kscontroller.MessageResourceSynced)
+	return ctrl.Result{}, nil
+}
+
+func (r *Reconciler) UpdateKubeConfigServiceAccountToken(ctx context.Context, username string, token string) error {
+	secretName := fmt.Sprintf(userKubeConfigSecretNameFormat, username)
+	kubeconfigSecret := &corev1.Secret{}
+	if err := r.Get(ctx, types.NamespacedName{Namespace: constants.KubeSphereNamespace, Name: secretName}, kubeconfigSecret); err != nil {
+		return client.IgnoreNotFound(err)
+	}
+
+	kubeconfigSecret = applyToken(kubeconfigSecret, token)
+
+	if err := r.Update(ctx, kubeconfigSecret); err != nil {
+		klog.Errorf("Failed to update secret %s: %v", secretName, err)
+		return err
+	}
+	return nil
+}
+
+func applyToken(secret *corev1.Secret, token string) *corev1.Secret {
+	data := secret.Data[kubeconfigFileName]
+	kubeconfig, err := clientcmd.Load(data)
+	if err != nil {
+		klog.Error(err)
+		return secret
+	}
+
+	username := secret.Labels[constants.UsernameLabelKey]
+	kubeconfig.AuthInfos = map[string]*clientcmdapi.AuthInfo{
+		username: {
+			Token: token,
+		},
+	}
+
+	data, err = clientcmd.Write(*kubeconfig)
+	if err != nil {
+		return secret
+	}
+
+	secret.StringData = map[string]string{kubeconfigFileName: string(data)}
+	return secret
+}

pkg/models/iam/am/am.go

@@ -23,6 +23,8 @@ import (
 
 	"kubesphere.io/kubesphere/pkg/api"
 	"kubesphere.io/kubesphere/pkg/apiserver/query"
+	"kubesphere.io/kubesphere/pkg/constants"
+	"kubesphere.io/kubesphere/pkg/models/kubeconfig"
 	resourcev1beta1 "kubesphere.io/kubesphere/pkg/models/resources/v1beta1"
 	"kubesphere.io/kubesphere/pkg/utils/sliceutil"
 )
@@ -588,6 +590,11 @@ func (am *amOperator) CreateOrUpdateNamespaceRoleBinding(username string, namesp
 				APIGroup: iamv1beta1.SchemeGroupVersion.Group,
 				Name:     username,
 			},
+			{
+				Kind:      rbacv1.ServiceAccountKind,
+				Name:      fmt.Sprintf(kubeconfig.UserKubeConfigServiceAccountNameFormat, username),
+				Namespace: constants.KubeSphereNamespace,
+			},
 		},
 		RoleRef: rbacv1.RoleRef{
 			APIGroup: iamv1beta1.SchemeGroupVersion.Group,
@@ -637,6 +644,11 @@ func (am *amOperator) CreateOrUpdateClusterRoleBinding(username string, role str
 				APIGroup: iamv1beta1.SchemeGroupVersion.Group,
 				Name:     username,
 			},
+			{
+				Kind:      rbacv1.ServiceAccountKind,
+				Name:      fmt.Sprintf(kubeconfig.UserKubeConfigServiceAccountNameFormat, username),
+				Namespace: constants.KubeSphereNamespace,
+			},
 		},
 		RoleRef: rbacv1.RoleRef{
 			APIGroup: iamv1beta1.SchemeGroupVersion.Group,

pkg/models/kubeconfig/kubeconfig.go

@@ -26,6 +26,7 @@ const (
 	InClusterCAFilePath                    = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
 	PrivateKeyAnnotation                   = "kubesphere.io/private-key"
 	UserKubeConfigSecretNameFormat         = "kubeconfig-%s"
+	UserKubeConfigServiceAccountNameFormat = "kubesphere.users.%s"
 )
 
 type Interface interface {

pkg/models/kubeconfig/options.go

@@ -0,0 +1,18 @@
+package kubeconfig
+
+const (
+	AuthModeServiceAccountToken AuthMode = "service-account-token"
+	AuthModeClientCertificate   AuthMode = "client-certificate"
+	AuthModeOIDCToken           AuthMode = "oidc-token"
+	AuthModeWebhookToken        AuthMode = "webhook-token"
+)
+
+type AuthMode string
+
+type Options struct {
+	AuthMode AuthMode `json:"authMode" yaml:"authMode" mapstructure:"authMode"`
+}
+
+func NewOptions() *Options {
+	return &Options{AuthMode: AuthModeClientCertificate}
+}