feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

pkg/controller/workspacerolebinding/workspacerolebinding_controller.go

@@ -1,122 +1,267 @@
 /*
-Copyright 2019 The KubeSphere Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
 
 package workspacerolebinding
 
 import (
 	"context"
+	"fmt"
 	"reflect"
+	"sort"
+	"strings"
+
+	kscontroller "kubesphere.io/kubesphere/pkg/controller"
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/kubernetes/scheme"
+	errorutils "k8s.io/apimachinery/pkg/util/errors"
+	toolscache "k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
+	"k8s.io/klog/v2"
+	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
+	iamv1beta1 "kubesphere.io/api/iam/v1beta1"
+	tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-
-	iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
-	tenantv1alpha2 "kubesphere.io/api/tenant/v1alpha2"
-	typesv1beta1 "kubesphere.io/api/types/v1beta1"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"kubesphere.io/kubesphere/pkg/constants"
-	controllerutils "kubesphere.io/kubesphere/pkg/controller/utils/controller"
+	clusterutils "kubesphere.io/kubesphere/pkg/controller/cluster/utils"
+	"kubesphere.io/kubesphere/pkg/controller/workspacetemplate/utils"
+	"kubesphere.io/kubesphere/pkg/utils/clusterclient"
 	"kubesphere.io/kubesphere/pkg/utils/k8sutil"
 )
 
 const (
-	controllerName = "workspacerolebinding-controller"
+	controllerName = "workspacerolebinding"
+	finalizer      = "finalizers.kubesphere.io/workspacerolebindings"
 )
 
+var _ kscontroller.Controller = &Reconciler{}
+var _ reconcile.Reconciler = &Reconciler{}
+
+func (r *Reconciler) Name() string {
+	return controllerName
+}
+
 // Reconciler reconciles a WorkspaceRoleBinding object
 type Reconciler struct {
 	client.Client
-	Logger                  logr.Logger
-	Scheme                  *runtime.Scheme
-	Recorder                record.EventRecorder
-	MaxConcurrentReconciles int
-	MultiClusterEnabled     bool
+	cache         cache.Cache
+	logger        logr.Logger
+	recorder      record.EventRecorder
+	ClusterClient clusterclient.Interface
 }
 
-func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
-	if r.Client == nil {
-		r.Client = mgr.GetClient()
+func (r *Reconciler) Start(ctx context.Context) error {
+	informer, err := r.cache.GetInformer(ctx, &clusterv1alpha1.Cluster{})
+	if err != nil {
+		return err
 	}
-	if r.Logger.GetSink() == nil {
-		r.Logger = ctrl.Log.WithName("controllers").WithName(controllerName)
+	var hostCluster *clusterv1alpha1.Cluster
+	clusters, err := r.ClusterClient.ListClusters(ctx)
+	if err != nil {
+		return err
 	}
-	if r.Scheme == nil {
-		r.Scheme = mgr.GetScheme()
+	for i := range clusters {
+		if clusterutils.IsHostCluster(&clusters[i]) {
+			hostCluster = &clusters[i]
+			break
+		}
 	}
-	if r.Recorder == nil {
-		r.Recorder = mgr.GetEventRecorderFor(controllerName)
-	}
-	if r.MaxConcurrentReconciles <= 0 {
-		r.MaxConcurrentReconciles = 1
+	_, err = informer.AddEventHandler(toolscache.ResourceEventHandlerFuncs{
+		UpdateFunc: func(oldObj, newObj interface{}) {
+			oldCluster := oldObj.(*clusterv1alpha1.Cluster)
+			newCluster := newObj.(*clusterv1alpha1.Cluster)
+			// cluster is ready
+			if !clusterutils.IsHostCluster(newCluster) &&
+				(!clusterutils.IsClusterReady(oldCluster) && clusterutils.IsClusterReady(newCluster)) {
+				err := r.CompletelySync(ctx, *hostCluster, *newCluster)
+				if err != nil {
+					r.logger.Error(err, "failed to sync workspacerolebinding for cluster",
+						"cluster", newCluster.Name)
+					return
+				}
+			}
+		},
+	})
+	return err
+}
+
+func (r *Reconciler) Enabled(clusterRole string) bool {
+	return strings.EqualFold(clusterRole, string(clusterv1alpha1.ClusterRoleHost))
+}
+
+func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error {
+	r.ClusterClient = mgr.ClusterClient
+	r.Client = mgr.GetClient()
+	r.cache = mgr.GetCache()
+	r.logger = ctrl.Log.WithName("controllers").WithName(controllerName)
+	r.recorder = mgr.GetEventRecorderFor(controllerName)
+
+	err := mgr.Add(r)
+	if err != nil {
+		return err
 	}
+
 	return ctrl.NewControllerManagedBy(mgr).
 		Named(controllerName).
-		WithOptions(controller.Options{
-			MaxConcurrentReconciles: r.MaxConcurrentReconciles,
-		}).
-		For(&iamv1alpha2.WorkspaceRoleBinding{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: 2}).
+		For(&iamv1beta1.WorkspaceRoleBinding{}).
 		Complete(r)
 }
 
 // +kubebuilder:rbac:groups=iam.kubesphere.io,resources=workspacerolebindings,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=types.kubefed.io,resources=federatedworkspacerolebindings,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=tenant.kubesphere.io,resources=workspaces,verbs=get;list;watch;
+
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	logger := r.Logger.WithValues("workspacerolebinding", req.NamespacedName)
-	rootCtx := context.Background()
-	workspaceRoleBinding := &iamv1alpha2.WorkspaceRoleBinding{}
-	if err := r.Get(rootCtx, req.NamespacedName, workspaceRoleBinding); err != nil {
+	workspaceRoleBinding := &iamv1beta1.WorkspaceRoleBinding{}
+	if err := r.Get(ctx, req.NamespacedName, workspaceRoleBinding); err != nil {
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
-	// controlled kubefed-controller-manager
-	if workspaceRoleBinding.Labels[constants.KubefedManagedLabel] == "true" {
+	if workspaceRoleBinding.ObjectMeta.DeletionTimestamp.IsZero() {
+		// The object is not being deleted, so if it does not have our finalizer,
+		// then lets add the finalizer and update the object.
+		if !controllerutil.ContainsFinalizer(workspaceRoleBinding, finalizer) {
+			expected := workspaceRoleBinding.DeepCopy()
+			controllerutil.AddFinalizer(expected, finalizer)
+			return ctrl.Result{}, r.Patch(ctx, expected, client.MergeFrom(workspaceRoleBinding))
+		}
+	} else {
+		// The object is being deleted
+		if controllerutil.ContainsFinalizer(workspaceRoleBinding, finalizer) {
+			if err := r.deleteRelatedResources(ctx, workspaceRoleBinding); err != nil {
+				return ctrl.Result{}, fmt.Errorf("failed to delete related resources: %s", err)
+			}
+			// remove our finalizer from the list and update it.
+			controllerutil.RemoveFinalizer(workspaceRoleBinding, finalizer)
+			if err := r.Update(ctx, workspaceRoleBinding, &client.UpdateOptions{}); err != nil {
+				return ctrl.Result{}, err
+			}
+		}
 		return ctrl.Result{}, nil
 	}
 
-	if err := r.bindWorkspace(rootCtx, logger, workspaceRoleBinding); err != nil {
+	if err := r.bindWorkspace(ctx, workspaceRoleBinding); err != nil {
 		return ctrl.Result{}, err
 	}
 
-	if r.MultiClusterEnabled {
-		if err := r.multiClusterSync(rootCtx, logger, workspaceRoleBinding); err != nil {
-			return ctrl.Result{}, err
-		}
+	if err := r.multiClusterSync(ctx, workspaceRoleBinding); err != nil {
+		return ctrl.Result{}, err
 	}
 
-	r.Recorder.Event(workspaceRoleBinding, corev1.EventTypeNormal, controllerutils.SuccessSynced, controllerutils.MessageResourceSynced)
+	r.recorder.Event(workspaceRoleBinding, corev1.EventTypeNormal, kscontroller.Synced, kscontroller.MessageResourceSynced)
 	return ctrl.Result{}, nil
 }
 
-func (r *Reconciler) bindWorkspace(ctx context.Context, logger logr.Logger, workspaceRoleBinding *iamv1alpha2.WorkspaceRoleBinding) error {
+func (r *Reconciler) deleteRelatedResources(ctx context.Context, workspaceRoleBinding *iamv1beta1.WorkspaceRoleBinding) error {
+	clusters, err := r.ClusterClient.ListClusters(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to list clusters: %s", err)
+	}
+	var notReadyClusters []string
+	for _, cluster := range clusters {
+		if clusterutils.IsHostCluster(&cluster) {
+			continue
+		}
+		// skip if cluster is not ready
+		if !clusterutils.IsClusterReady(&cluster) {
+			notReadyClusters = append(notReadyClusters, cluster.Name)
+			continue
+		}
+		clusterClient, err := r.ClusterClient.GetRuntimeClient(cluster.Name)
+		if err != nil {
+			return fmt.Errorf("failed to get cluster client: %s", err)
+		}
+		if err = clusterClient.Delete(ctx, &iamv1beta1.WorkspaceRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: workspaceRoleBinding.Name}}); err != nil {
+			if errors.IsNotFound(err) {
+				continue
+			}
+			logr.FromContextOrDiscard(ctx).Error(err, "failed to delete related resources")
+		}
+	}
+	if len(notReadyClusters) > 0 {
+		err = fmt.Errorf("cluster not ready: %s", strings.Join(notReadyClusters, ","))
+		logr.FromContextOrDiscard(ctx).Error(err, "failed to delete related resources")
+	}
+	return nil
+}
+
+func (r *Reconciler) multiClusterSync(ctx context.Context, workspaceRoleBinding *iamv1beta1.WorkspaceRoleBinding) error {
+	clusters, err := r.ClusterClient.ListClusters(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to list clusters: %s", err)
+	}
+	var notReadyClusters []string
+	for _, cluster := range clusters {
+		// skip if cluster is not ready
+		if !clusterutils.IsClusterReady(&cluster) {
+			notReadyClusters = append(notReadyClusters, cluster.Name)
+			continue
+		}
+		if clusterutils.IsHostCluster(&cluster) {
+			continue
+		}
+		// skip the sync failed , cause error will break process
+		if err := r.syncWorkspaceRoleBinding(ctx, cluster, workspaceRoleBinding); err != nil {
+			r.recorder.Event(workspaceRoleBinding, corev1.EventTypeWarning, kscontroller.SyncFailed,
+				fmt.Sprintf("failed to sync workspace role binding %s to cluster %s: %s", workspaceRoleBinding.Name, cluster.Name, err))
+		}
+	}
+	if len(notReadyClusters) > 0 {
+		klog.FromContext(ctx).V(4).Info("cluster not ready", "clusters", strings.Join(notReadyClusters, ","))
+		r.recorder.Event(workspaceRoleBinding, corev1.EventTypeWarning, kscontroller.SyncFailed, fmt.Sprintf("cluster not ready: %s", strings.Join(notReadyClusters, ",")))
+	}
+	return nil
+}
+
+func (r *Reconciler) syncWorkspaceRoleBinding(ctx context.Context, cluster clusterv1alpha1.Cluster, workspaceRoleBinding *iamv1beta1.WorkspaceRoleBinding) error {
+	clusterClient, err := r.ClusterClient.GetRuntimeClient(cluster.Name)
+	if err != nil {
+		return err
+	}
+
+	workspaceTemplate := &tenantv1beta1.WorkspaceTemplate{}
+	if err := r.Get(ctx, types.NamespacedName{Name: workspaceRoleBinding.Labels[tenantv1beta1.WorkspaceLabel]}, workspaceTemplate); err != nil {
+		return client.IgnoreNotFound(err)
+	}
+
+	if utils.WorkspaceTemplateMatchTargetCluster(workspaceTemplate, &cluster) {
+		target := &iamv1beta1.WorkspaceRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: workspaceRoleBinding.Name}}
+		op, err := controllerutil.CreateOrUpdate(ctx, clusterClient, target, func() error {
+			target.Labels = workspaceRoleBinding.Labels
+			target.Annotations = workspaceRoleBinding.Annotations
+			target.RoleRef = workspaceRoleBinding.RoleRef
+			target.Subjects = workspaceRoleBinding.Subjects
+			return nil
+		})
+		if err != nil {
+			return fmt.Errorf("failed to update workspace role binding: %s", err)
+		}
+		klog.FromContext(ctx).V(4).Info("workspace role binding successfully synced", "cluster", cluster.Name, "operation", op, "name", workspaceRoleBinding.Name)
+	} else {
+		return client.IgnoreNotFound(clusterClient.DeleteAllOf(ctx, &iamv1beta1.WorkspaceRole{}, client.MatchingLabels{tenantv1beta1.WorkspaceLabel: workspaceTemplate.Name}))
+	}
+	return nil
+}
+
+func (r *Reconciler) bindWorkspace(ctx context.Context, workspaceRoleBinding *iamv1beta1.WorkspaceRoleBinding) error {
 	workspaceName := workspaceRoleBinding.Labels[constants.WorkspaceLabelKey]
 	if workspaceName == "" {
 		return nil
 	}
-	workspace := &tenantv1alpha2.WorkspaceTemplate{}
+	workspace := &tenantv1beta1.WorkspaceTemplate{}
 	if err := r.Get(ctx, types.NamespacedName{Name: workspaceName}, workspace); err != nil {
 		// skip if workspace not found
 		return client.IgnoreNotFound(err)
@@ -124,93 +269,113 @@ func (r *Reconciler) bindWorkspace(ctx context.Context, logger logr.Logger, work
 	// owner reference not match workspace label
 	if !metav1.IsControlledBy(workspaceRoleBinding, workspace) {
 		workspaceRoleBinding.OwnerReferences = k8sutil.RemoveWorkspaceOwnerReference(workspaceRoleBinding.OwnerReferences)
-		if err := controllerutil.SetControllerReference(workspace, workspaceRoleBinding, r.Scheme); err != nil {
-			logger.Error(err, "set controller reference failed")
+		if err := controllerutil.SetControllerReference(workspace, workspaceRoleBinding, r.Scheme()); err != nil {
 			return err
 		}
-		logger.V(4).Info("update owner reference")
 		if err := r.Update(ctx, workspaceRoleBinding); err != nil {
-			logger.Error(err, "update owner reference failed")
 			return err
 		}
 	}
 	return nil
 }
 
-func (r *Reconciler) multiClusterSync(ctx context.Context, logger logr.Logger, workspaceRoleBinding *iamv1alpha2.WorkspaceRoleBinding) error {
-	if err := r.ensureNotControlledByKubefed(ctx, logger, workspaceRoleBinding); err != nil {
+func (r *Reconciler) CompletelySync(ctx context.Context, hostCluster, cluster clusterv1alpha1.Cluster) error {
+	searchMap := map[string]iamv1beta1.WorkspaceRoleBinding{}
+	// list resources at host cluster
+	hostList := &iamv1beta1.WorkspaceRoleBindingList{}
+	hostClusterClient, err := r.ClusterClient.GetRuntimeClient(hostCluster.Name)
+	if err != nil {
 		return err
 	}
-	federatedWorkspaceRoleBinding := &typesv1beta1.FederatedWorkspaceRoleBinding{}
-	if err := r.Client.Get(ctx, types.NamespacedName{Name: workspaceRoleBinding.Name}, federatedWorkspaceRoleBinding); err != nil {
-		if errors.IsNotFound(err) {
-			if federatedWorkspaceRoleBinding, err := newFederatedWorkspaceRoleBinding(workspaceRoleBinding); err != nil {
-				logger.Error(err, "generate federated workspace role binding failed")
-				return err
-			} else {
-				if err := r.Create(ctx, federatedWorkspaceRoleBinding); err != nil {
-					logger.Error(err, "create federated workspace role binding failed")
-					return err
-				}
-				return nil
+	err = hostClusterClient.List(ctx, hostList)
+	if err != nil {
+		return err
+	}
+
+	// list resources at member cluster
+	clusterClient, err := r.ClusterClient.GetRuntimeClient(cluster.Name)
+	if err != nil {
+		return err
+	}
+	memberList := &iamv1beta1.WorkspaceRoleBindingList{}
+	err = clusterClient.List(ctx, memberList)
+	if err != nil {
+		return err
+	}
+
+	for _, item := range memberList.Items {
+		searchMap[item.GetName()] = item
+	}
+
+	var errList []error
+	// check and update
+	for _, item := range hostList.Items {
+		workspaceTemplate := &tenantv1beta1.WorkspaceTemplate{}
+		if err := r.Get(ctx, types.NamespacedName{Name: item.Labels[tenantv1beta1.WorkspaceLabel]}, workspaceTemplate); err != nil {
+			return client.IgnoreNotFound(err)
+		}
+		if !utils.WorkspaceTemplateMatchTargetCluster(workspaceTemplate, &cluster) {
+			continue
+		}
+
+		memObj, exist := searchMap[item.GetName()]
+		if !exist {
+			if err := clusterClient.Create(ctx, &item); err != nil {
+				err = fmt.Errorf("create worspaceRoleBinding %s at cluster %s failed", item.Name, cluster.Name)
+				errList = append(errList, err)
+			}
+			continue
+		}
+		if !bindingEqual(item, memObj) {
+			memObj.Labels = item.Labels
+			memObj.Annotations = item.Annotations
+			memObj.RoleRef = item.RoleRef
+			memObj.Subjects = item.Subjects
+			if err := clusterClient.Update(ctx, &memObj); err != nil {
+				err = fmt.Errorf("update worspaceRoleBinding %s at cluster %s failed", item.Name, cluster.Name)
+				errList = append(errList, err)
 			}
 		}
-		logger.Error(err, "get federated workspace role binding failed")
-		return err
+		delete(searchMap, memObj.GetName())
 	}
 
-	if !reflect.DeepEqual(federatedWorkspaceRoleBinding.Spec.Template.RoleRef, workspaceRoleBinding.RoleRef) ||
-		!reflect.DeepEqual(federatedWorkspaceRoleBinding.Spec.Template.Subjects, workspaceRoleBinding.Subjects) ||
-		!reflect.DeepEqual(federatedWorkspaceRoleBinding.Spec.Template.Labels, workspaceRoleBinding.Labels) {
+	for _, obj := range searchMap {
+		err := clusterClient.Delete(ctx, &obj)
+		if err != nil {
+			err = fmt.Errorf("delete worspaceRoleBinding %s at cluster %s failed", obj.Name, cluster.Name)
+			errList = append(errList, err)
+		}
+	}
+	return errorutils.NewAggregate(errList)
+}
 
-		federatedWorkspaceRoleBinding.Spec.Template.RoleRef = workspaceRoleBinding.RoleRef
-		federatedWorkspaceRoleBinding.Spec.Template.Subjects = workspaceRoleBinding.Subjects
-		federatedWorkspaceRoleBinding.Spec.Template.Labels = workspaceRoleBinding.Labels
+func bindingEqual(a, b iamv1beta1.WorkspaceRoleBinding) bool {
+	if a.Name != b.Name || len(a.Subjects) != len(b.Subjects) || !reflect.DeepEqual(a.RoleRef, b.RoleRef) {
+		return false
+	}
 
-		if err := r.Update(ctx, federatedWorkspaceRoleBinding); err != nil {
-			logger.Error(err, "update federated workspace role failed")
-			return err
+	sort.Sort(SortableSubjectSlice(a.Subjects))
+	sort.Sort(SortableSubjectSlice(b.Subjects))
+
+	for i, subject := range a.Subjects {
+		if !reflect.DeepEqual(subject, b.Subjects[i]) {
+			return false
 		}
 	}
 
-	return nil
+	return true
 }
 
-func newFederatedWorkspaceRoleBinding(workspaceRoleBinding *iamv1alpha2.WorkspaceRoleBinding) (*typesv1beta1.FederatedWorkspaceRoleBinding, error) {
-	federatedWorkspaceRoleBinding := &typesv1beta1.FederatedWorkspaceRoleBinding{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: workspaceRoleBinding.Name,
-		},
-		Spec: typesv1beta1.FederatedWorkspaceRoleBindingSpec{
-			Template: typesv1beta1.WorkspaceRoleBindingTemplate{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: workspaceRoleBinding.Labels,
-				},
-				RoleRef:  workspaceRoleBinding.RoleRef,
-				Subjects: workspaceRoleBinding.Subjects,
-			},
-			Placement: typesv1beta1.GenericPlacementFields{
-				ClusterSelector: &metav1.LabelSelector{},
-			},
-		},
-	}
-	if err := controllerutil.SetControllerReference(workspaceRoleBinding, federatedWorkspaceRoleBinding, scheme.Scheme); err != nil {
-		return nil, err
-	}
-	return federatedWorkspaceRoleBinding, nil
+type SortableSubjectSlice []v1.Subject
+
+func (s SortableSubjectSlice) Len() int {
+	return len(s)
 }
 
-func (r *Reconciler) ensureNotControlledByKubefed(ctx context.Context, logger logr.Logger, workspaceRoleBinding *iamv1alpha2.WorkspaceRoleBinding) error {
-	if workspaceRoleBinding.Labels[constants.KubefedManagedLabel] != "false" {
-		if workspaceRoleBinding.Labels == nil {
-			workspaceRoleBinding.Labels = make(map[string]string)
-		}
-		workspaceRoleBinding.Labels[constants.KubefedManagedLabel] = "false"
-		logger.V(4).Info("update kubefed managed label")
-		if err := r.Update(ctx, workspaceRoleBinding); err != nil {
-			logger.Error(err, "update kubefed managed label failed")
-			return err
-		}
-	}
-	return nil
+func (s SortableSubjectSlice) Less(i, j int) bool {
+	return s[i].Name < s[j].Name
+}
+
+func (s SortableSubjectSlice) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
 }

pkg/controller/workspacerolebinding/workspacerolebinding_controller_suite_test.go

@@ -1,39 +1,36 @@
 /*
-Copyright 2019 The KubeSphere Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
 
 package workspacerolebinding
 
 import (
+	"context"
 	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
-	"github.com/onsi/gomega/gexec"
-	"k8s.io/client-go/kubernetes/scheme"
+	"kubesphere.io/kubesphere/pkg/controller/controllertest"
+
+	"kubesphere.io/kubesphere/pkg/controller"
+	kscontroller "kubesphere.io/kubesphere/pkg/controller/options"
+
+	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
+
+	"kubesphere.io/kubesphere/pkg/multicluster"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	logf "sigs.k8s.io/controller-runtime/pkg/log"
-
-	"kubesphere.io/kubesphere/pkg/apis"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+
+	"kubesphere.io/kubesphere/pkg/scheme"
+	"kubesphere.io/kubesphere/pkg/utils/clusterclient"
 )
 
 // These tests use Ginkgo (BDD-style Go testing framework). Refer to
@@ -42,13 +39,15 @@ import (
 var k8sClient client.Client
 var k8sManager ctrl.Manager
 var testEnv *envtest.Environment
+var ctx context.Context
+var cancel context.CancelFunc
 
 func TestWorkspaceRoleBindingController(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "WorkspaceRoleBinding Controller Test Suite")
 }
 
-var _ = BeforeSuite(func(done Done) {
+var _ = BeforeSuite(func() {
 	logf.SetLogger(klog.NewKlogr())
 
 	By("bootstrapping test environment")
@@ -58,8 +57,10 @@ var _ = BeforeSuite(func(done Done) {
 			UseExistingCluster: &t,
 		}
 	} else {
+		crdDirPaths, err := controllertest.LoadCrdPath()
+		Expect(err).ToNot(HaveOccurred())
 		testEnv = &envtest.Environment{
-			CRDDirectoryPaths:        []string{filepath.Join("..", "..", "..", "config", "ks-core", "crds")},
+			CRDDirectoryPaths:        crdDirPaths,
 			AttachControlPlaneOutput: false,
 		}
 	}
@@ -68,32 +69,35 @@ var _ = BeforeSuite(func(done Done) {
 	Expect(err).ToNot(HaveOccurred())
 	Expect(cfg).ToNot(BeNil())
 
-	err = apis.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-
 	k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{
-		Scheme:             scheme.Scheme,
-		MetricsBindAddress: "0",
+		Scheme: scheme.Scheme,
+		Metrics: metricsserver.Options{
+			BindAddress: "0",
+		},
 	})
 	Expect(err).ToNot(HaveOccurred())
 
-	err = (&Reconciler{}).SetupWithManager(k8sManager)
+	clusterClient, err := clusterclient.NewClusterClientSet(k8sManager.GetCache())
 	Expect(err).ToNot(HaveOccurred())
 
+	err = (&Reconciler{}).SetupWithManager(&controller.Manager{ClusterClient: clusterClient, Manager: k8sManager,
+		Options: kscontroller.Options{MultiClusterOptions: &multicluster.Options{ClusterRole: string(clusterv1alpha1.ClusterRoleHost)}}})
+	Expect(err).ToNot(HaveOccurred())
+
+	ctx, cancel = context.WithCancel(context.Background())
 	go func() {
-		err = k8sManager.Start(ctrl.SetupSignalHandler())
+		err = k8sManager.Start(ctx)
 		Expect(err).ToNot(HaveOccurred())
 	}()
 
 	k8sClient = k8sManager.GetClient()
 	Expect(k8sClient).ToNot(BeNil())
-
-	close(done)
-}, 60)
+})
 
 var _ = AfterSuite(func() {
+	cancel()
 	By("tearing down the test environment")
-	gexec.KillAndWait(5 * time.Second)
-	err := testEnv.Stop()
-	Expect(err).ToNot(HaveOccurred())
+	Eventually(func() error {
+		return testEnv.Stop()
+	}, 30*time.Second, 5*time.Second).ShouldNot(HaveOccurred())
 })

pkg/controller/workspacerolebinding/workspacerolebinding_controller_test.go

@@ -1,18 +1,7 @@
 /*
-Copyright 2019 The KubeSphere Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
 
 package workspacerolebinding
 
@@ -20,15 +9,13 @@ import (
 	"context"
 	"time"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
-	iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
-	tenantv1alpha1 "kubesphere.io/api/tenant/v1alpha1"
-	tenantv1alpha2 "kubesphere.io/api/tenant/v1alpha2"
+	iamv1beta1 "kubesphere.io/api/iam/v1beta1"
+	tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
 )
 
 var _ = Describe("WorkspaceRoleBinding", func() {
@@ -36,7 +23,7 @@ var _ = Describe("WorkspaceRoleBinding", func() {
 	const timeout = time.Second * 30
 	const interval = time.Second * 1
 
-	workspace := &tenantv1alpha2.WorkspaceTemplate{
+	workspace := &tenantv1beta1.WorkspaceTemplate{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "workspace1",
 		},
@@ -52,15 +39,10 @@ var _ = Describe("WorkspaceRoleBinding", func() {
 	// test Kubernetes API server, which isn't the goal here.
 	Context("WorkspaceRoleBinding Controller", func() {
 		It("Should create successfully", func() {
-			workspaceAdminBinding := &iamv1alpha2.WorkspaceRoleBinding{
+			workspaceAdminBinding := &iamv1beta1.WorkspaceRoleBinding{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:   "admin-workspace1-admin",
-					Labels: map[string]string{tenantv1alpha1.WorkspaceLabel: workspace.Name},
-				},
-				RoleRef: rbacv1.RoleRef{
-					APIGroup: iamv1alpha2.SchemeGroupVersion.String(),
-					Kind:     iamv1alpha2.FedWorkspaceRoleKind,
-					Name:     "workspace1-admin",
+					Labels: map[string]string{tenantv1beta1.WorkspaceLabel: workspace.Name},
 				},
 			}