feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

pkg/controller/workspacetemplate/utils/helper.go

@@ -0,0 +1,34 @@
+/*
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
+
+package utils
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/klog/v2"
+	"kubesphere.io/api/cluster/v1alpha1"
+	tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
+)
+
+func WorkspaceTemplateMatchTargetCluster(workspaceTemplate *tenantv1beta1.WorkspaceTemplate, cluster *v1alpha1.Cluster) bool {
+	match := false
+	if len(workspaceTemplate.Spec.Placement.Clusters) > 0 {
+		for _, clusterRef := range workspaceTemplate.Spec.Placement.Clusters {
+			if clusterRef.Name == cluster.Name {
+				match = true
+				break
+			}
+		}
+	} else if workspaceTemplate.Spec.Placement.ClusterSelector != nil {
+		selector, err := metav1.LabelSelectorAsSelector(workspaceTemplate.Spec.Placement.ClusterSelector)
+		if err != nil {
+			klog.Errorf("failed to parse cluster selector: %s", err)
+			return false
+		}
+		match = selector.Matches(labels.Set(cluster.Labels))
+	}
+	return match
+}

pkg/controller/workspacetemplate/workspacetemplate_controller.go

@@ -1,18 +1,7 @@
 /*
-Copyright 2019 The KubeSphere Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
 
 package workspacetemplate
 
@@ -20,7 +9,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"reflect"
+	"strings"
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
@@ -31,106 +20,116 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/tools/record"
+	"k8s.io/klog/v2"
+	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
+	iamv1beta1 "kubesphere.io/api/iam/v1beta1"
+	tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
-	"kubesphere.io/api/application/v1alpha1"
-	iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
-	tenantv1alpha1 "kubesphere.io/api/tenant/v1alpha1"
-	tenantv1alpha2 "kubesphere.io/api/tenant/v1alpha2"
-	typesv1beta1 "kubesphere.io/api/types/v1beta1"
-
-	"kubesphere.io/kubesphere/pkg/constants"
-	controllerutils "kubesphere.io/kubesphere/pkg/controller/utils/controller"
-	"kubesphere.io/kubesphere/pkg/utils/sliceutil"
+	kscontroller "kubesphere.io/kubesphere/pkg/controller"
+	"kubesphere.io/kubesphere/pkg/controller/cluster/predicate"
+	clusterutils "kubesphere.io/kubesphere/pkg/controller/cluster/utils"
+	"kubesphere.io/kubesphere/pkg/controller/workspacetemplate/utils"
+	"kubesphere.io/kubesphere/pkg/utils/clusterclient"
 )
 
 const (
-	controllerName                  = "workspacetemplate-controller"
-	workspaceTemplateFinalizer      = "finalizers.workspacetemplate.kubesphere.io"
-	orphanFinalizer                 = "orphan.finalizers.kubesphere.io"
-	orphanDeleteOptionAnnotationKey = "kubefed.io/deleteoption"
-	orphanDeleteOptionAnnotation    = "{\"propagationPolicy\":\"Orphan\"}"
+	controllerName             = "workspacetemplate"
+	workspaceTemplateFinalizer = "finalizers.workspacetemplate.kubesphere.io"
+	orphanFinalizer            = "orphan.finalizers.kubesphere.io"
 )
 
 // Reconciler reconciles a WorkspaceRoleBinding object
 type Reconciler struct {
 	client.Client
-	Logger                  logr.Logger
-	Recorder                record.EventRecorder
-	MaxConcurrentReconciles int
-	MultiClusterEnabled     bool
+	logger        logr.Logger
+	recorder      record.EventRecorder
+	clusterClient clusterclient.Interface
 }
 
-func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
-	if r.Client == nil {
-		r.Client = mgr.GetClient()
-	}
-	if r.Logger.GetSink() == nil {
-		r.Logger = ctrl.Log.WithName("controllers").WithName(controllerName)
-	}
-	if r.Recorder == nil {
-		r.Recorder = mgr.GetEventRecorderFor(controllerName)
-	}
-	if r.MaxConcurrentReconciles <= 0 {
-		r.MaxConcurrentReconciles = 1
-	}
+func (r *Reconciler) Enabled(clusterRole string) bool {
+	return strings.EqualFold(clusterRole, string(clusterv1alpha1.ClusterRoleHost))
+}
+
+var _ kscontroller.Controller = &Reconciler{}
+var _ reconcile.Reconciler = &Reconciler{}
+
+func (r *Reconciler) Name() string {
+	return controllerName
+}
+
+func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error {
+	r.clusterClient = mgr.ClusterClient
+	r.Client = mgr.GetClient()
+	r.logger = ctrl.Log.WithName("controllers").WithName(controllerName)
+	r.recorder = mgr.GetEventRecorderFor(controllerName)
 	return ctrl.NewControllerManagedBy(mgr).
 		Named(controllerName).
-		WithOptions(controller.Options{
-			MaxConcurrentReconciles: r.MaxConcurrentReconciles,
-		}).
-		For(&tenantv1alpha2.WorkspaceTemplate{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: 2}).
+		For(&tenantv1beta1.WorkspaceTemplate{}).
+		Watches(
+			&clusterv1alpha1.Cluster{},
+			handler.EnqueueRequestsFromMapFunc(r.mapper),
+			builder.WithPredicates(predicate.ClusterStatusChangedPredicate{}),
+		).
 		Complete(r)
 }
 
+func (r *Reconciler) mapper(ctx context.Context, o client.Object) []reconcile.Request {
+	cluster := o.(*clusterv1alpha1.Cluster)
+	if !clusterutils.IsClusterReady(cluster) {
+		return []reconcile.Request{}
+	}
+	workspaceTemplates := &tenantv1beta1.WorkspaceTemplateList{}
+	if err := r.List(ctx, workspaceTemplates); err != nil {
+		r.logger.Error(err, "failed to list workspace templates")
+		return []reconcile.Request{}
+	}
+	var result []reconcile.Request
+	for _, workspaceTemplate := range workspaceTemplates.Items {
+		if utils.WorkspaceTemplateMatchTargetCluster(&workspaceTemplate, cluster) {
+			result = append(result, reconcile.Request{NamespacedName: types.NamespacedName{Name: workspaceTemplate.Name}})
+		}
+	}
+	return result
+}
+
 // +kubebuilder:rbac:groups=iam.kubesphere.io,resources=workspacerolebindings,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=types.kubefed.io,resources=federatedworkspacerolebindings,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=tenant.kubesphere.io,resources=workspaces,verbs=get;list;watch;
+
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	logger := r.Logger.WithValues("workspacetemplate", req.NamespacedName)
-	workspaceTemplate := &tenantv1alpha2.WorkspaceTemplate{}
+	logger := r.logger.WithValues("workspacetemplate", req.NamespacedName)
+	workspaceTemplate := &tenantv1beta1.WorkspaceTemplate{}
 	if err := r.Get(ctx, req.NamespacedName, workspaceTemplate); err != nil {
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	ctx = klog.NewContext(ctx, logger)
 	if workspaceTemplate.ObjectMeta.DeletionTimestamp.IsZero() {
 		// The object is not being deleted, so if it does not have our finalizer,
 		// then lets add the finalizer and update the object.
-		if !sliceutil.HasString(workspaceTemplate.ObjectMeta.Finalizers, workspaceTemplateFinalizer) {
-			workspaceTemplate.ObjectMeta.Finalizers = append(workspaceTemplate.ObjectMeta.Finalizers, workspaceTemplateFinalizer)
-			if err := r.Update(ctx, workspaceTemplate); err != nil {
-				return ctrl.Result{}, err
-			}
+		if !controllerutil.ContainsFinalizer(workspaceTemplate, workspaceTemplateFinalizer) {
+			updated := workspaceTemplate.DeepCopy()
+			controllerutil.AddFinalizer(updated, workspaceTemplateFinalizer)
+			return ctrl.Result{}, r.Patch(ctx, updated, client.MergeFrom(workspaceTemplate))
 		}
 	} else {
 		// The object is being deleted
-		if sliceutil.HasString(workspaceTemplate.ObjectMeta.Finalizers, workspaceTemplateFinalizer) ||
-			sliceutil.HasString(workspaceTemplate.ObjectMeta.Finalizers, orphanFinalizer) {
-			if err := r.deleteOpenPitrixResourcesInWorkspace(ctx, workspaceTemplate.Name); err != nil {
-				logger.Error(err, "failed to delete related openpitrix resource")
+		if controllerutil.ContainsFinalizer(workspaceTemplate, workspaceTemplateFinalizer) ||
+			controllerutil.ContainsFinalizer(workspaceTemplate, orphanFinalizer) {
+			if err := r.reconcileDelete(ctx, workspaceTemplate); err != nil {
 				return ctrl.Result{}, err
 			}
-
-			if err := r.deleteWorkspace(ctx, workspaceTemplate); err != nil {
-				if errors.IsNotFound(err) {
-					logger.V(4).Info("related workspace not found")
-				} else {
-					logger.Error(err, "failed to delete related workspace")
-					return ctrl.Result{}, nil
-				}
-			}
-
 			// remove our finalizer from the list and update it.
-			workspaceTemplate.ObjectMeta.Finalizers = sliceutil.RemoveString(workspaceTemplate.ObjectMeta.Finalizers, func(item string) bool {
-				return item == workspaceTemplateFinalizer || item == orphanFinalizer
-			})
-
-			logger.V(4).Info("update workspace template")
+			controllerutil.RemoveFinalizer(workspaceTemplate, workspaceTemplateFinalizer)
+			controllerutil.RemoveFinalizer(workspaceTemplate, orphanFinalizer)
 			if err := r.Update(ctx, workspaceTemplate); err != nil {
-				logger.Error(err, "update workspace template failed")
 				return ctrl.Result{}, err
 			}
 		}
@@ -138,380 +137,192 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, nil
 	}
 
-	if r.MultiClusterEnabled {
-		if err := r.multiClusterSync(ctx, logger, workspaceTemplate); err != nil {
-			return ctrl.Result{}, err
-		}
-	} else {
-		if err := r.singleClusterSync(ctx, logger, workspaceTemplate); err != nil {
-			return ctrl.Result{}, err
-		}
-	}
-	if err := r.initWorkspaceRoles(ctx, logger, workspaceTemplate); err != nil {
+	if err := r.initWorkspaceRoles(ctx, workspaceTemplate); err != nil {
 		return ctrl.Result{}, err
 	}
-	if err := r.initManagerRoleBinding(ctx, logger, workspaceTemplate); err != nil {
+	if err := r.initManagerRoleBinding(ctx, workspaceTemplate); err != nil {
 		return ctrl.Result{}, err
 	}
-	r.Recorder.Event(workspaceTemplate, corev1.EventTypeNormal, controllerutils.SuccessSynced, controllerutils.MessageResourceSynced)
+	if err := r.multiClusterSync(ctx, workspaceTemplate); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	r.recorder.Event(workspaceTemplate, corev1.EventTypeNormal, kscontroller.Synced, kscontroller.MessageResourceSynced)
 	return ctrl.Result{}, nil
 }
 
-func (r *Reconciler) singleClusterSync(ctx context.Context, logger logr.Logger, workspaceTemplate *tenantv1alpha2.WorkspaceTemplate) error {
-	workspace := &tenantv1alpha1.Workspace{}
-	if err := r.Get(ctx, types.NamespacedName{Name: workspaceTemplate.Name}, workspace); err != nil {
-		if errors.IsNotFound(err) {
-			if workspace, err := newWorkspace(workspaceTemplate); err != nil {
-				logger.Error(err, "generate workspace failed")
-				return err
-			} else {
-				if err := r.Create(ctx, workspace); err != nil {
-					logger.Error(err, "create workspace failed")
-					return err
-				}
-				return nil
-			}
-		}
-		logger.Error(err, "get workspace failed")
-		return err
+func (r *Reconciler) multiClusterSync(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
+	clusters, err := r.clusterClient.ListClusters(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to list clusters: %s", err)
 	}
-	if !reflect.DeepEqual(workspace.Spec, workspaceTemplate.Spec.Template.Spec) ||
-		!reflect.DeepEqual(workspace.Labels, workspaceTemplate.Spec.Template.Labels) {
-
-		workspace = workspace.DeepCopy()
-		workspace.Spec = workspaceTemplate.Spec.Template.Spec
-		workspace.Labels = workspaceTemplate.Spec.Template.Labels
-
-		if err := r.Update(ctx, workspace); err != nil {
-			logger.Error(err, "update workspace failed")
-			return err
+	var notReadyClusters []string
+	for _, cluster := range clusters {
+		// skip if cluster is not ready
+		if !clusterutils.IsClusterReady(&cluster) {
+			notReadyClusters = append(notReadyClusters, cluster.Name)
+			continue
 		}
+		if err := r.syncWorkspaceTemplate(ctx, cluster, workspaceTemplate); err != nil {
+			return fmt.Errorf("failed to sync workspace template %s to cluster %s: %s", workspaceTemplate.Name, cluster.Name, err)
+		}
+	}
+	if len(notReadyClusters) > 0 {
+		klog.FromContext(ctx).V(4).Info("cluster not ready", "clusters", strings.Join(notReadyClusters, ","))
+		r.recorder.Event(workspaceTemplate, corev1.EventTypeWarning, kscontroller.SyncFailed, fmt.Sprintf("cluster not ready: %s", strings.Join(notReadyClusters, ",")))
 	}
 	return nil
 }
 
-func (r *Reconciler) multiClusterSync(ctx context.Context, logger logr.Logger, workspaceTemplate *tenantv1alpha2.WorkspaceTemplate) error {
-	if err := r.ensureNotControlledByKubefed(ctx, logger, workspaceTemplate); err != nil {
-		return err
-	}
-	federatedWorkspace := &typesv1beta1.FederatedWorkspace{}
-	if err := r.Client.Get(ctx, types.NamespacedName{Name: workspaceTemplate.Name}, federatedWorkspace); err != nil {
-		if errors.IsNotFound(err) {
-			if federatedWorkspace, err := newFederatedWorkspace(workspaceTemplate); err != nil {
-				logger.Error(err, "generate federated workspace failed")
-				return err
-			} else {
-				if err := r.Create(ctx, federatedWorkspace); err != nil {
-					logger.Error(err, "create federated workspace failed")
-					return err
-				}
-				return nil
-			}
-		}
-		logger.Error(err, "get federated workspace failed")
-		return err
-	}
-
-	if !reflect.DeepEqual(federatedWorkspace.Spec, workspaceTemplate.Spec) ||
-		!reflect.DeepEqual(federatedWorkspace.Labels, workspaceTemplate.Labels) {
-
-		federatedWorkspace.Spec = workspaceTemplate.Spec
-		federatedWorkspace.Labels = workspaceTemplate.Labels
-
-		if err := r.Update(ctx, federatedWorkspace); err != nil {
-			logger.Error(err, "update federated workspace failed")
-			return err
-		}
-	}
-
-	return nil
-}
-
-func newFederatedWorkspace(template *tenantv1alpha2.WorkspaceTemplate) (*typesv1beta1.FederatedWorkspace, error) {
-	federatedWorkspace := &typesv1beta1.FederatedWorkspace{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:   template.Name,
-			Labels: template.Labels,
-		},
-		Spec: template.Spec,
-	}
-	return federatedWorkspace, nil
-}
-
-func newWorkspace(template *tenantv1alpha2.WorkspaceTemplate) (*tenantv1alpha1.Workspace, error) {
-	workspace := &tenantv1alpha1.Workspace{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:   template.Name,
-			Labels: template.Spec.Template.Labels,
-		},
-		Spec: template.Spec.Template.Spec,
-	}
-
-	return workspace, nil
-}
-
-func (r *Reconciler) deleteWorkspace(ctx context.Context, template *tenantv1alpha2.WorkspaceTemplate) error {
-	if r.MultiClusterEnabled {
-		federatedWorkspace := &typesv1beta1.FederatedWorkspace{}
-		if err := r.Client.Get(ctx, types.NamespacedName{Name: template.Name}, federatedWorkspace); err != nil {
-			return err
-		}
-
-		// Workspace will be deleted with Orphan Option when it has a orphan finalizer.
-		// Reousrces that owned by the Workspace will not be deleted.
-		if sliceutil.HasString(template.ObjectMeta.Finalizers, orphanFinalizer) {
-			if federatedWorkspace.Annotations == nil {
-				federatedWorkspace.Annotations = make(map[string]string, 1)
-			}
-			federatedWorkspace.Annotations[orphanDeleteOptionAnnotationKey] = orphanDeleteOptionAnnotation
-			if err := r.Update(ctx, federatedWorkspace); err != nil {
-				return err
-			}
-		} else {
-			// Usually namespace will bind the lifecycle of workspace with ownerReference,
-			// in multi-cluster environment workspace will not be created in host cluster
-			// if the cluster is not be granted or kubefed-controller-manager is unavailable,
-			// this will cause the federated namespace left an orphan object in host cluster.
-			// After workspaceTemplate deleted we need to deleted orphan namespace in host cluster directly.
-			if err := r.deleteNamespacesInWorkspace(ctx, template); err != nil {
-				return err
-			}
-		}
-
-		if err := r.Delete(ctx, federatedWorkspace); err != nil {
-			return err
-		}
-	}
-
-	opt := &client.DeleteOptions{}
-	// Dependents won't be deleted when it's has a orphanFinalizer
-	if sliceutil.HasString(template.ObjectMeta.Finalizers, orphanFinalizer) {
-		orphan := metav1.DeletePropagationOrphan
-		opt = &client.DeleteOptions{PropagationPolicy: &orphan}
-	}
-
-	ws := &tenantv1alpha1.Workspace{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: template.Name,
-		},
-	}
-	if err := r.Delete(ctx, ws, opt); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *Reconciler) ensureNotControlledByKubefed(ctx context.Context, logger logr.Logger, workspaceTemplate *tenantv1alpha2.WorkspaceTemplate) error {
-	if workspaceTemplate.Labels[constants.KubefedManagedLabel] != "false" {
-		if workspaceTemplate.Labels == nil {
-			workspaceTemplate.Labels = make(map[string]string)
-		}
-		workspaceTemplate.Labels[constants.KubefedManagedLabel] = "false"
-		logger.V(4).Info("update kubefed managed label")
-		if err := r.Update(ctx, workspaceTemplate); err != nil {
-			logger.Error(err, "update kubefed managed label failed")
-			return err
-		}
-	}
-	return nil
-}
-
-func (r *Reconciler) initWorkspaceRoles(ctx context.Context, logger logr.Logger, workspace *tenantv1alpha2.WorkspaceTemplate) error {
-	var templates iamv1alpha2.RoleBaseList
-	if err := r.List(ctx, &templates); err != nil {
-		logger.Error(err, "list role base failed")
-		return err
-	}
-	for _, template := range templates.Items {
-		var expected iamv1alpha2.WorkspaceRole
-		if err := yaml.NewYAMLOrJSONDecoder(bytes.NewBuffer(template.Role.Raw), 1024).Decode(&expected); err == nil && expected.Kind == iamv1alpha2.ResourceKindWorkspaceRole {
-			expected.Name = fmt.Sprintf("%s-%s", workspace.Name, expected.Name)
-			if expected.Labels == nil {
-				expected.Labels = make(map[string]string)
-			}
-			expected.Labels[tenantv1alpha1.WorkspaceLabel] = workspace.Name
-			workspaceRole := &iamv1alpha2.WorkspaceRole{}
-			if err := r.Get(ctx, types.NamespacedName{Name: expected.Name}, workspaceRole); err != nil {
-				if errors.IsNotFound(err) {
-					logger.V(4).Info("create workspace role", "workspacerole", expected.Name)
-					if err := r.Create(ctx, &expected); err != nil {
-						logger.Error(err, "create workspace role failed")
-						return err
-					}
-					continue
-				} else {
-					logger.Error(err, "get workspace role failed")
-					return err
-				}
-			}
-			if !reflect.DeepEqual(expected.Labels, workspaceRole.Labels) ||
-				!reflect.DeepEqual(expected.Annotations, workspaceRole.Annotations) ||
-				!reflect.DeepEqual(expected.Rules, workspaceRole.Rules) {
-				workspaceRole.Labels = expected.Labels
-				workspaceRole.Annotations = expected.Annotations
-				workspaceRole.Rules = expected.Rules
-				logger.V(4).Info("update workspace role", "workspacerole", workspaceRole.Name)
-				if err := r.Update(ctx, workspaceRole); err != nil {
-					logger.Error(err, "update workspace role failed")
-					return err
-				}
-			}
-		} else if err != nil {
-			logger.Error(fmt.Errorf("invalid role base found"), "init workspace roles failed", "name", template.Name)
-		}
-	}
-	return nil
-}
-
-func (r *Reconciler) initManagerRoleBinding(ctx context.Context, logger logr.Logger, workspace *tenantv1alpha2.WorkspaceTemplate) error {
-	manager := workspace.Spec.Template.Spec.Manager
-	if manager == "" {
-		return nil
-	}
-
-	var user iamv1alpha2.User
-	if err := r.Get(ctx, types.NamespacedName{Name: manager}, &user); err != nil {
-		return client.IgnoreNotFound(err)
-	}
-
-	// skip if user has been deleted
-	if !user.DeletionTimestamp.IsZero() {
-		return nil
-	}
-
-	workspaceAdminRoleName := fmt.Sprintf("%s-admin", workspace.Name)
-	managerRoleBinding := &iamv1alpha2.WorkspaceRoleBinding{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: workspaceAdminRoleName,
-		},
-	}
-
-	if _, err := ctrl.CreateOrUpdate(ctx, r.Client, managerRoleBinding, workspaceRoleBindingChanger(managerRoleBinding, workspace.Name, manager, workspaceAdminRoleName)); err != nil {
-		logger.Error(err, "create workspace manager role binding failed")
-		return err
-	}
-
-	return nil
-}
-func (r *Reconciler) deleteOpenPitrixResourcesInWorkspace(ctx context.Context, ws string) error {
-	if len(ws) == 0 {
-		return nil
-	}
-
-	var err error
-	// helm release, apps and appVersion only exist in host cluster. Delete these resource in workspace template controller
-	if err = r.deleteHelmReleases(ctx, ws); err != nil {
-		return err
-	}
-
-	if err = r.deleteHelmApps(ctx, ws); err != nil {
-		return err
-	}
-
-	if err = r.deleteHelmRepos(ctx, ws); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (r *Reconciler) deleteHelmApps(ctx context.Context, ws string) error {
-	if len(ws) == 0 {
-		return nil
-	}
-
-	apps := v1alpha1.HelmApplicationList{}
-	err := r.List(ctx, &apps, &client.ListOptions{LabelSelector: labels.SelectorFromSet(map[string]string{
-		constants.WorkspaceLabelKey: ws}),
-	})
+func (r *Reconciler) syncWorkspaceTemplate(ctx context.Context, cluster clusterv1alpha1.Cluster, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
+	clusterClient, err := r.clusterClient.GetRuntimeClient(cluster.Name)
 	if err != nil {
 		return err
 	}
-	for _, app := range apps.Items {
-		if app.Annotations == nil {
-			app.Annotations = map[string]string{}
-		}
-		if _, exists := app.Annotations[constants.DanglingAppCleanupKey]; !exists {
-			// Mark the app, the cleanup is in the application controller.
-			appCopy := app.DeepCopy()
-			appCopy.Annotations[constants.DanglingAppCleanupKey] = constants.CleanupDanglingAppOngoing
-			appPatch := client.MergeFrom(&app)
-			err = r.Patch(ctx, appCopy, appPatch)
-			if err != nil {
-				return err
+	if utils.WorkspaceTemplateMatchTargetCluster(workspaceTemplate, &cluster) {
+		target := &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}}
+		op, err := controllerutil.CreateOrUpdate(ctx, clusterClient, target, func() error {
+			for k, v := range workspaceTemplate.Spec.Template.Labels {
+				if target.Labels == nil {
+					target.Labels = make(map[string]string)
+				}
+				target.Labels[k] = v
 			}
-		}
-	}
-
-	return nil
-}
-
-// Delete all helm releases in the workspace ws
-func (r *Reconciler) deleteHelmReleases(ctx context.Context, ws string) error {
-	if len(ws) == 0 {
-		return nil
-	}
-	rls := &v1alpha1.HelmRelease{}
-	err := r.DeleteAllOf(ctx, rls, &client.DeleteAllOfOptions{
-		ListOptions: client.ListOptions{LabelSelector: labels.SelectorFromSet(map[string]string{
-			constants.WorkspaceLabelKey: ws,
-		}),
-		}})
-	return err
-}
-
-func (r *Reconciler) deleteHelmRepos(ctx context.Context, ws string) error {
-	if len(ws) == 0 {
-		return nil
-	}
-	rls := &v1alpha1.HelmRepo{}
-	err := r.DeleteAllOf(ctx, rls, &client.DeleteAllOfOptions{
-		ListOptions: client.ListOptions{LabelSelector: labels.SelectorFromSet(map[string]string{
-			constants.WorkspaceLabelKey: ws,
-		}),
-		}})
-
-	return err
-}
-
-// deleteNamespacesInWorkspace Deletes the namespace associated with the workspace, which match the workspace label selector
-func (r *Reconciler) deleteNamespacesInWorkspace(ctx context.Context, template *tenantv1alpha2.WorkspaceTemplate) error {
-	namespaceList := &corev1.NamespaceList{}
-	err := r.Client.List(ctx, namespaceList, client.MatchingLabels{tenantv1alpha1.WorkspaceLabel: template.Name})
-	if err != nil {
-		return err
-	}
-
-	for _, namespace := range namespaceList.Items {
-		err = r.Client.Delete(ctx, &namespace)
+			for k, v := range workspaceTemplate.Spec.Template.Annotations {
+				if target.Annotations == nil {
+					target.Annotations = make(map[string]string)
+				}
+				target.Annotations[k] = v
+			}
+			target.Spec = workspaceTemplate.Spec.Template.Spec
+			return nil
+		})
 		if err != nil {
 			return err
 		}
+		klog.FromContext(ctx).V(4).Info("workspace successfully synced", "operation", op)
+	} else {
+		orphan := metav1.DeletePropagationBackground
+		err = clusterClient.Delete(ctx, &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}},
+			&client.DeleteOptions{PropagationPolicy: &orphan})
+		return client.IgnoreNotFound(err)
 	}
 	return nil
 }
 
-func workspaceRoleBindingChanger(workspaceRoleBinding *iamv1alpha2.WorkspaceRoleBinding, workspace, username, workspaceRoleName string) controllerutil.MutateFn {
-	return func() error {
-		workspaceRoleBinding.Labels = map[string]string{
-			tenantv1alpha1.WorkspaceLabel:  workspace,
-			iamv1alpha2.UserReferenceLabel: username,
+func (r *Reconciler) initWorkspaceRoles(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
+	logger := klog.FromContext(ctx)
+	var templates iamv1beta1.BuiltinRoleList
+	// scope.iam.kubesphere.io/workspace: ""
+	if err := r.List(ctx, &templates, client.MatchingLabels{iamv1beta1.ScopeLabel: iamv1beta1.ScopeWorkspace}); err != nil {
+		return err
+	}
+	for _, template := range templates.Items {
+		selector, err := metav1.LabelSelectorAsSelector(&template.TargetSelector)
+		if err != nil {
+			logger.V(4).Error(err, "failed to pares target selector", "template", template.Name)
+			continue
+		}
+		if !selector.Matches(labels.Set(workspaceTemplate.Labels)) {
+			continue
+		}
+		var builtinWorkspaceRole iamv1beta1.WorkspaceRole
+		if err := yaml.NewYAMLOrJSONDecoder(bytes.NewBuffer(template.Role.Raw), 1024).Decode(&builtinWorkspaceRole); err == nil &&
+			builtinWorkspaceRole.Kind == iamv1beta1.ResourceKindWorkspaceRole {
+			target := &iamv1beta1.WorkspaceRole{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: fmt.Sprintf("%s-%s", workspaceTemplate.Name, builtinWorkspaceRole.Name),
+				},
+			}
+			op, err := controllerutil.CreateOrUpdate(ctx, r.Client, target, func() error {
+				target.Labels = builtinWorkspaceRole.Labels
+				if target.Labels == nil {
+					target.Labels = make(map[string]string)
+				}
+				target.Labels[tenantv1beta1.WorkspaceLabel] = workspaceTemplate.Name
+				target.Annotations = builtinWorkspaceRole.Annotations
+				target.AggregationRoleTemplates = builtinWorkspaceRole.AggregationRoleTemplates
+				target.Rules = builtinWorkspaceRole.Rules
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+			logger.V(4).Info("builtin workspace role successfully updated", "operation", op, "name", target.Name)
+		} else if err != nil {
+			logger.Error(err, "invalid builtin workspace role found", "name", template.Name)
+		}
+	}
+	return nil
+}
+
+func (r *Reconciler) initManagerRoleBinding(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
+	manager := workspaceTemplate.Spec.Template.Spec.Manager
+	if manager == "" {
+		return nil
+	}
+	workspaceAdminRoleName := fmt.Sprintf("%s-admin", workspaceTemplate.Name)
+	existWorkspaceRoleBinding := &iamv1beta1.WorkspaceRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: workspaceAdminRoleName}}
+	if _, err := ctrl.CreateOrUpdate(ctx, r.Client, existWorkspaceRoleBinding, func() error {
+		existWorkspaceRoleBinding.Labels = map[string]string{
+			tenantv1beta1.WorkspaceLabel:  workspaceTemplate.Name,
+			iamv1beta1.UserReferenceLabel: manager,
+			iamv1beta1.RoleReferenceLabel: workspaceAdminRoleName,
 		}
 
-		workspaceRoleBinding.RoleRef = rbacv1.RoleRef{
-			APIGroup: iamv1alpha2.SchemeGroupVersion.Group,
-			Kind:     iamv1alpha2.ResourceKindWorkspaceRole,
-			Name:     workspaceRoleName,
+		existWorkspaceRoleBinding.RoleRef = rbacv1.RoleRef{
+			APIGroup: iamv1beta1.SchemeGroupVersion.Group,
+			Kind:     iamv1beta1.ResourceKindWorkspaceRole,
+			Name:     workspaceAdminRoleName,
 		}
-
-		workspaceRoleBinding.Subjects = []rbacv1.Subject{
+		existWorkspaceRoleBinding.Subjects = []rbacv1.Subject{
 			{
-				Name:     username,
-				Kind:     rbacv1.UserKind,
-				APIGroup: rbacv1.GroupName,
+				Name:     manager,
+				Kind:     iamv1beta1.ResourceKindUser,
+				APIGroup: iamv1beta1.SchemeGroupVersion.Group,
 			},
 		}
 		return nil
+	}); err != nil {
+		return err
 	}
+	return nil
+}
+
+func (r *Reconciler) reconcileDelete(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
+	clusters, err := r.clusterClient.ListClusters(ctx)
+	if err != nil {
+		return err
+	}
+	var notReadyClusters []string
+	for _, cluster := range clusters {
+		// skip if cluster is not ready
+		if !clusterutils.IsClusterReady(&cluster) {
+			notReadyClusters = append(notReadyClusters, cluster.Name)
+			continue
+		}
+		clusterClient, err := r.clusterClient.GetRuntimeClient(cluster.Name)
+		if err != nil {
+			notReadyClusters = append(notReadyClusters, cluster.Name)
+			continue
+		}
+
+		if controllerutil.ContainsFinalizer(workspaceTemplate, orphanFinalizer) {
+			orphan := metav1.DeletePropagationOrphan
+			err = clusterClient.Delete(ctx, &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}}, &client.DeleteOptions{PropagationPolicy: &orphan})
+		} else {
+			err = clusterClient.Delete(ctx, &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}}, &client.DeleteOptions{})
+		}
+
+		if !errors.IsNotFound(err) {
+			notReadyClusters = append(notReadyClusters, cluster.Name)
+			continue
+		}
+	}
+	if len(notReadyClusters) > 0 {
+		klog.FromContext(ctx).V(4).Info("cluster not ready", "clusters", strings.Join(notReadyClusters, ","))
+		r.recorder.Event(workspaceTemplate, corev1.EventTypeWarning, kscontroller.SyncFailed, fmt.Sprintf("cluster not ready: %s", strings.Join(notReadyClusters, ",")))
+		return err
+	}
+	return nil
 }

pkg/controller/workspacetemplate/workspacetemplate_controller_suite_test.go

@@ -1,52 +0,0 @@
-/*
-Copyright 2019 The KubeSphere Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package workspacetemplate
-
-import (
-	"testing"
-
-	"k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/klog/v2"
-	logf "sigs.k8s.io/controller-runtime/pkg/log"
-
-	helmappscheme "kubesphere.io/api/application/v1alpha1"
-	typesv1beta1 "kubesphere.io/api/types/v1beta1"
-
-	"kubesphere.io/kubesphere/pkg/apis"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-func TestWorkspaceTemplateController(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "WorkspaceTemplate Controller Test Suite")
-}
-
-var _ = BeforeSuite(func(done Done) {
-	logf.SetLogger(klog.NewKlogr())
-
-	err := helmappscheme.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-	err = apis.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = typesv1beta1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-
-	close(done)
-}, 60)

pkg/controller/workspacetemplate/workspacetemplate_controller_test.go

@@ -1,18 +1,7 @@
 /*
-Copyright 2019 The KubeSphere Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
 
 package workspacetemplate
 
@@ -20,7 +9,7 @@ import (
 	"context"
 	"time"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -32,9 +21,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
-	iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
-	tenantv1alpha1 "kubesphere.io/api/tenant/v1alpha1"
-	tenantv1alpha2 "kubesphere.io/api/tenant/v1alpha2"
+	iamv1beta1 "kubesphere.io/api/iam/v1beta1"
+	tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
 )
 
 var reconciler *Reconciler
@@ -47,9 +35,9 @@ var _ = Describe("WorkspaceTemplate", func() {
 
 		reconciler = &Reconciler{
 			//nolint:staticcheck
-			Client:   fake.NewFakeClientWithScheme(scheme.Scheme),
-			Logger:   ctrl.Log.WithName("controllers").WithName("acrpullbinding-controller"),
-			Recorder: record.NewFakeRecorder(5),
+			Client:   fake.NewClientBuilder().WithScheme(scheme.Scheme).Build(),
+			logger:   ctrl.Log.WithName("controllers").WithName("workspacetemplate"),
+			recorder: record.NewFakeRecorder(5),
 		}
 
 		workspaceAdmin := newWorkspaceAdmin()
@@ -57,7 +45,7 @@ var _ = Describe("WorkspaceTemplate", func() {
 		err := reconciler.Create(context.Background(), &workspaceAdmin)
 		Expect(err).NotTo(HaveOccurred())
 
-		admin := iamv1alpha2.User{ObjectMeta: metav1.ObjectMeta{Name: "admin"}}
+		admin := iamv1beta1.User{ObjectMeta: metav1.ObjectMeta{Name: "admin"}}
 		err = reconciler.Create(context.Background(), &admin)
 		Expect(err).NotTo(HaveOccurred())
 	})
@@ -67,87 +55,83 @@ var _ = Describe("WorkspaceTemplate", func() {
 	// Avoid adding tests for vanilla CRUD operations because they would
 	// test Kubernetes API server, which isn't the goal here.
 	Context("WorkspaceTemplate Controller", func() {
-		for _, multiCluster := range []bool{true, false} {
-			enalbed := multiCluster
-			It("Should create successfully", func() {
-				reconciler.MultiClusterEnabled = enalbed
-				key := types.NamespacedName{
-					Name: "workspace-template",
-				}
+		It("Should create successfully", func() {
+			key := types.NamespacedName{
+				Name: "workspace-template",
+			}
 
-				created := &tenantv1alpha2.WorkspaceTemplate{
-					ObjectMeta: metav1.ObjectMeta{
-						Name: key.Name,
-					},
-				}
+			created := &tenantv1beta1.WorkspaceTemplate{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: key.Name,
+				},
+			}
 
-				// Create
-				Expect(reconciler.Create(context.Background(), created)).Should(Succeed())
+			// Create
+			Expect(reconciler.Create(context.Background(), created)).Should(Succeed())
 
-				req := ctrl.Request{
-					NamespacedName: key,
-				}
-				_, err := reconciler.Reconcile(context.Background(), req)
-				Expect(err).To(BeNil())
+			req := ctrl.Request{
+				NamespacedName: key,
+			}
+			_, err := reconciler.Reconcile(context.Background(), req)
+			Expect(err).To(BeNil())
 
-				By("Expecting to create workspace template successfully")
-				Expect(func() *tenantv1alpha2.WorkspaceTemplate {
-					f := &tenantv1alpha2.WorkspaceTemplate{}
-					reconciler.Get(context.Background(), key, f)
-					return f
-				}()).ShouldNot(BeNil())
+			By("Expecting to create workspace template successfully")
+			Expect(func() *tenantv1beta1.WorkspaceTemplate {
+				f := &tenantv1beta1.WorkspaceTemplate{}
+				reconciler.Get(context.Background(), key, f)
+				return f
+			}()).ShouldNot(BeNil())
 
-				By("Expecting to create workspace successfully")
-				Expect(func() *tenantv1alpha1.Workspace {
-					f := &tenantv1alpha1.Workspace{}
-					reconciler.Get(context.Background(), key, f)
-					return f
-				}()).ShouldNot(BeNil())
+			By("Expecting to create workspace successfully")
+			Expect(func() *tenantv1beta1.Workspace {
+				f := &tenantv1beta1.Workspace{}
+				reconciler.Get(context.Background(), key, f)
+				return f
+			}()).ShouldNot(BeNil())
 
-				// List workspace roles
-				By("Expecting to create workspace role successfully")
-				Eventually(func() bool {
-					f := &iamv1alpha2.WorkspaceRoleList{}
-					reconciler.List(context.Background(), f, &client.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{tenantv1alpha1.WorkspaceLabel: key.Name})})
-					return len(f.Items) == 1
-				}, timeout, interval).Should(BeTrue())
+			// List workspace roles
+			By("Expecting to create workspace role successfully")
+			Eventually(func() bool {
+				f := &iamv1beta1.WorkspaceRoleList{}
+				reconciler.List(context.Background(), f, &client.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{tenantv1beta1.WorkspaceLabel: key.Name})})
+				return len(f.Items) == 1
+			}, timeout, interval).Should(BeTrue())
 
-				// Update
-				updated := &tenantv1alpha2.WorkspaceTemplate{}
-				Expect(reconciler.Get(context.Background(), key, updated)).Should(Succeed())
-				updated.Spec.Template.Spec.Manager = "admin"
-				Expect(reconciler.Update(context.Background(), updated)).Should(Succeed())
+			// Update
+			updated := &tenantv1beta1.WorkspaceTemplate{}
+			Expect(reconciler.Get(context.Background(), key, updated)).Should(Succeed())
+			updated.Spec.Template.Spec.Manager = "admin"
+			Expect(reconciler.Update(context.Background(), updated)).Should(Succeed())
 
-				_, err = reconciler.Reconcile(context.Background(), req)
-				Expect(err).To(BeNil())
+			_, err = reconciler.Reconcile(context.Background(), req)
+			Expect(err).To(BeNil())
 
-				// List workspace role bindings
-				By("Expecting to create workspace manager role binding successfully")
-				Eventually(func() bool {
-					f := &iamv1alpha2.WorkspaceRoleBindingList{}
-					reconciler.List(context.Background(), f, &client.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{tenantv1alpha1.WorkspaceLabel: key.Name})})
-					return len(f.Items) == 1
-				}, timeout, interval).Should(BeTrue())
+			// List workspace role bindings
+			By("Expecting to create workspace manager role binding successfully")
+			Eventually(func() bool {
+				f := &iamv1beta1.WorkspaceRoleBindingList{}
+				reconciler.List(context.Background(), f, &client.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{tenantv1beta1.WorkspaceLabel: key.Name})})
+				return len(f.Items) == 1
+			}, timeout, interval).Should(BeTrue())
 
-				// Delete
-				By("Expecting to finalize workspace successfully")
-				Eventually(func() error {
-					f := &tenantv1alpha2.WorkspaceTemplate{}
-					reconciler.Get(context.Background(), key, f)
-					now := metav1.NewTime(time.Now())
-					f.DeletionTimestamp = &now
-					return reconciler.Update(context.Background(), f)
-				}, timeout, interval).Should(Succeed())
+			// Delete
+			By("Expecting to finalize workspace successfully")
+			Eventually(func() error {
+				f := &tenantv1beta1.WorkspaceTemplate{}
+				reconciler.Get(context.Background(), key, f)
+				now := metav1.NewTime(time.Now())
+				f.DeletionTimestamp = &now
+				return reconciler.Update(context.Background(), f)
+			}, timeout, interval).Should(Succeed())
 
-				_, err = reconciler.Reconcile(context.Background(), req)
-				Expect(err).To(BeNil())
-			})
-		}
+			_, err = reconciler.Reconcile(context.Background(), req)
+			Expect(err).To(BeNil())
+		})
 	})
 })
 
-func newWorkspaceAdmin() iamv1alpha2.RoleBase {
-	return iamv1alpha2.RoleBase{
+func newWorkspaceAdmin() iamv1beta1.BuiltinRole {
+	return iamv1beta1.BuiltinRole{
 		ObjectMeta: metav1.ObjectMeta{Name: "workspace-admin"},
 		Role: runtime.RawExtension{
 			Raw: []byte(`{