[release-4.1] refactor: workspace cascading deletion logic (#6251)

refactor: workspace cascading deletion logic

Signed-off-by: hongming <coder.scala@gmail.com>
Co-authored-by: hongming <coder.scala@gmail.com>

pkg/constants/constants.go

@@ -13,15 +13,17 @@ const (
 	KubeSphereConfigMapDataKey = "kubesphere.yaml"
 	KubectlPodNamePrefix       = "ks-managed-kubectl"
 
-	WorkspaceLabelKey        = "kubesphere.io/workspace"
-	DisplayNameAnnotationKey = "kubesphere.io/alias-name"
-	DescriptionAnnotationKey = "kubesphere.io/description"
-	CreatorAnnotationKey     = "kubesphere.io/creator"
-	UsernameLabelKey         = "kubesphere.io/username"
-	GenericConfigTypeLabel   = "config.kubesphere.io/type"
-	KubectlPodLabel          = "kubesphere.io/kubectl-pod"
-	ConfigHashAnnotation     = "kubesphere.io/config-hash"
-	KubeSphereManagedLabel   = "kubesphere.io/managed"
+	WorkspaceLabelKey             = "kubesphere.io/workspace"
+	DisplayNameAnnotationKey      = "kubesphere.io/alias-name"
+	DescriptionAnnotationKey      = "kubesphere.io/description"
+	CreatorAnnotationKey          = "kubesphere.io/creator"
+	UsernameLabelKey              = "kubesphere.io/username"
+	GenericConfigTypeLabel        = "config.kubesphere.io/type"
+	KubectlPodLabel               = "kubesphere.io/kubectl-pod"
+	ConfigHashAnnotation          = "kubesphere.io/config-hash"
+	KubeSphereManagedLabel        = "kubesphere.io/managed"
+	DeletionPropagationAnnotation = "kubesphere.io/deletion-propagation"
+	CascadingDeletionFinalizer    = "kubesphere.io/cascading-deletion"
 )
 
 var (

pkg/controller/namespace/namespace_controller.go

@@ -13,15 +13,12 @@ import (
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 	iamv1beta1 "kubesphere.io/api/iam/v1beta1"
-	"kubesphere.io/api/tenant/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -30,13 +27,10 @@ import (
 
 	"kubesphere.io/kubesphere/pkg/constants"
 	kscontroller "kubesphere.io/kubesphere/pkg/controller"
-	"kubesphere.io/kubesphere/pkg/scheme"
-	"kubesphere.io/kubesphere/pkg/utils/k8sutil"
 )
 
 const (
 	controllerName = "namespace"
-	finalizer      = "finalizers.kubesphere.io/namespaces"
 )
 
 var _ kscontroller.Controller = &Reconciler{}
@@ -65,10 +59,9 @@ func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error {
 }
 
 // +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=tenant.kubesphere.io,resources=workspaces,verbs=get;list;watch
-// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=rolebases,verbs=get;list;watch
-// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch
-// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=builtinroles,verbs=get;list;watch
+// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=roles,verbs=get;list;watch
+// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=rolebindings,verbs=get;list;watch
 
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	logger := r.logger.WithValues("namespace", req.NamespacedName)
@@ -78,21 +71,30 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	// Skip the namespace that is not created in workspace
+	if _, ok := namespace.Labels[constants.WorkspaceLabelKey]; !ok {
+		return ctrl.Result{}, nil
+	}
+
 	if namespace.ObjectMeta.DeletionTimestamp.IsZero() {
 		// The object is not being deleted, so if it does not have our finalizer,
 		// then lets add the finalizer and update the object.
-		if !controllerutil.ContainsFinalizer(namespace, finalizer) {
+		if !controllerutil.ContainsFinalizer(namespace, constants.CascadingDeletionFinalizer) {
 			if err := r.initCreatorRoleBinding(ctx, namespace); err != nil {
 				return ctrl.Result{}, err
 			}
 			updated := namespace.DeepCopy()
-			controllerutil.AddFinalizer(updated, finalizer)
+			// Remove legacy finalizer
+			controllerutil.RemoveFinalizer(updated, "finalizers.kubesphere.io/namespaces")
+			// Remove legacy ownerReferences
+			updated.OwnerReferences = make([]metav1.OwnerReference, 0)
+			controllerutil.AddFinalizer(updated, constants.CascadingDeletionFinalizer)
 			return ctrl.Result{}, r.Patch(ctx, updated, client.MergeFrom(namespace))
 		}
 	} else {
 		// The object is being deleted
-		if controllerutil.ContainsFinalizer(namespace, finalizer) {
-			controllerutil.RemoveFinalizer(namespace, finalizer)
+		if controllerutil.ContainsFinalizer(namespace, constants.CascadingDeletionFinalizer) {
+			controllerutil.RemoveFinalizer(namespace, constants.CascadingDeletionFinalizer)
 			if err := r.Update(ctx, namespace); err != nil {
 				return ctrl.Result{}, err
 			}
@@ -105,64 +107,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, err
 	}
 
-	if err := r.reconcileWorkspaceOwnerReference(ctx, namespace); err != nil {
-		return ctrl.Result{}, err
-	}
-
 	r.recorder.Event(namespace, corev1.EventTypeNormal, kscontroller.Synced, kscontroller.MessageResourceSynced)
 	return ctrl.Result{}, nil
 }
 
-func (r *Reconciler) reconcileWorkspaceOwnerReference(ctx context.Context, namespace *corev1.Namespace) error {
-	workspaceName, hasWorkspaceLabel := namespace.Labels[v1beta1.WorkspaceLabel]
-
-	if !hasWorkspaceLabel {
-		if k8sutil.IsControlledBy(namespace.OwnerReferences, v1beta1.ResourceKindWorkspace, workspaceName) {
-			namespace.OwnerReferences = k8sutil.RemoveWorkspaceOwnerReference(namespace.OwnerReferences)
-			return r.Update(ctx, namespace)
-		}
-		// noting to do
-		return nil
-	}
-
-	workspace := &v1beta1.Workspace{}
-	if err := r.Get(ctx, types.NamespacedName{Name: workspaceName}, workspace); err != nil {
-		owner := metav1.GetControllerOf(namespace)
-		if errors.IsNotFound(err) && owner != nil && owner.Kind == v1beta1.ResourceKindWorkspace {
-			namespace.OwnerReferences = k8sutil.RemoveWorkspaceOwnerReference(namespace.OwnerReferences)
-			return r.Update(ctx, namespace)
-		}
-		return client.IgnoreNotFound(err)
-	}
-
-	// workspace has been deleted
-	if !workspace.ObjectMeta.DeletionTimestamp.IsZero() {
-		return nil
-	}
-
-	if !metav1.IsControlledBy(namespace, workspace) && namespace.Labels[constants.KubeSphereManagedLabel] == "true" {
-		namespace = namespace.DeepCopy()
-		if err := controllerutil.SetControllerReference(workspace, namespace, scheme.Scheme); err != nil {
-			return err
-		}
-		if err := r.Update(ctx, namespace); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 func (r *Reconciler) initRoles(ctx context.Context, namespace *corev1.Namespace) error {
-	if _, ok := namespace.Labels[constants.WorkspaceLabelKey]; !ok {
-		return nil
-	}
-
 	logger := klog.FromContext(ctx)
 	var templates iamv1beta1.BuiltinRoleList
-	matchingLabels := client.MatchingLabels{iamv1beta1.ScopeLabel: iamv1beta1.ScopeNamespace}
-	if err := r.List(ctx, &templates, matchingLabels); err != nil {
-		return err
+	if err := r.List(ctx, &templates, client.MatchingLabels{iamv1beta1.ScopeLabel: iamv1beta1.ScopeNamespace}); err != nil {
+		return fmt.Errorf("failed to list builtin roles: %v", err)
 	}
 	for _, template := range templates.Items {
 		selector, err := metav1.LabelSelectorAsSelector(&template.TargetSelector)
@@ -185,9 +138,9 @@ func (r *Reconciler) initRoles(ctx context.Context, namespace *corev1.Namespace)
 				return nil
 			})
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to create or update builtin role: %v", err)
 			}
-			logger.V(4).Info("builtin role successfully initialized", "operation", op)
+			logger.V(4).Info("builtin role initialized", "operation", op)
 		} else if err != nil {
 			logger.Error(err, "invalid builtin role found", "name", template.Name)
 		}
@@ -228,6 +181,6 @@ func (r *Reconciler) initCreatorRoleBinding(ctx context.Context, namespace *core
 	if err != nil {
 		return err
 	}
-	klog.FromContext(ctx).V(4).Info("creator role binding successfully initialized", "operation", op)
+	klog.FromContext(ctx).V(4).Info("creator role binding initialized", "operation", op)
 	return nil
 }

pkg/controller/namespace/namespace_controller_test.go

@@ -54,41 +54,17 @@ var _ = Describe("Namespace", func() {
 			Expect(k8sClient.Create(context.Background(), namespace)).Should(Succeed())
 
 			By("Expecting to create namespace successfully")
-			Eventually(func() bool {
-				k8sClient.Get(context.Background(), types.NamespacedName{Name: namespace.Name}, namespace)
-				return !namespace.CreationTimestamp.IsZero()
-			}, timeout, interval).Should(BeTrue())
-
-			By("Expecting to set owner reference successfully")
-			Eventually(func() bool {
-				k8sClient.Get(context.Background(), types.NamespacedName{Name: namespace.Name}, namespace)
-				return len(namespace.OwnerReferences) > 0
-			}, timeout, interval).Should(BeTrue())
-
-			Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace)).Should(Succeed())
-
-			controlled := true
-			expectedOwnerReference := metav1.OwnerReference{
-				Kind:               workspace.Kind,
-				APIVersion:         workspace.APIVersion,
-				UID:                workspace.UID,
-				Name:               workspace.Name,
-				Controller:         &controlled,
-				BlockOwnerDeletion: &controlled,
-			}
-
-			By("Expecting to bind workspace successfully")
-			Expect(namespace.OwnerReferences).To(ContainElement(expectedOwnerReference))
-			Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace)).Should(Succeed())
+			Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: namespace.Name}, namespace)).Should(Succeed())
 
 			By("Expecting to update namespace successfully")
 			updated := namespace.DeepCopy()
 			updated.Labels[constants.WorkspaceLabelKey] = "workspace-not-exist"
+
 			Expect(k8sClient.Update(context.Background(), updated)).Should(Succeed())
 
 			By("Expecting to unbind workspace successfully")
 			Eventually(func() bool {
-				k8sClient.Get(context.Background(), types.NamespacedName{Name: namespace.Name}, namespace)
+				_ = k8sClient.Get(context.Background(), types.NamespacedName{Name: namespace.Name}, namespace)
 				return len(namespace.OwnerReferences) == 0
 			}, timeout, interval).Should(BeTrue())
 		})

pkg/controller/workspace/workspace_controller.go

@@ -7,9 +7,15 @@ package workspace
 
 import (
 	"context"
+	"fmt"
+
+	"kubesphere.io/kubesphere/pkg/constants"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 	tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
@@ -24,7 +30,6 @@ import (
 
 const (
 	controllerName = "workspace"
-	finalizer      = "finalizers.tenant.kubesphere.io"
 )
 
 var _ kscontroller.Controller = &Reconciler{}
@@ -53,11 +58,6 @@ func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error {
 }
 
 // +kubebuilder:rbac:groups=tenant.kubesphere.io,resources=workspaces,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=tenant.kubesphere.io,resources=workspaces/status,verbs=get;update;patch
-// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=users,verbs=get;list;watch
-// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=rolebases,verbs=get;list;watch
-// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=workspaceroles,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=workspacerolebindings,verbs=get;list;watch;create;update;patch;delete
 
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	logger := r.logger.WithValues("workspace", req.NamespacedName)
@@ -70,28 +70,71 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	if workspace.ObjectMeta.DeletionTimestamp.IsZero() {
 		// The object is not being deleted, so if it does not have our finalizer,
 		// then lets add the finalizer and update the object.
-		if !controllerutil.ContainsFinalizer(workspace, finalizer) {
+		if !controllerutil.ContainsFinalizer(workspace, constants.CascadingDeletionFinalizer) {
 			expected := workspace.DeepCopy()
-			controllerutil.AddFinalizer(expected, finalizer)
+			// Remove legacy finalizer
+			controllerutil.RemoveFinalizer(expected, "finalizers.tenant.kubesphere.io")
+			controllerutil.AddFinalizer(expected, constants.CascadingDeletionFinalizer)
 			if err := r.Patch(ctx, expected, client.MergeFrom(workspace)); err != nil {
-				return ctrl.Result{}, err
+				return ctrl.Result{}, fmt.Errorf("failed to add finalizer: %s", err)
 			}
 			workspaceOperation.WithLabelValues("create", workspace.Name).Inc()
 		}
 	} else {
-		// The object is being deleted
-		if controllerutil.ContainsFinalizer(workspace, finalizer) {
-			// remove our finalizer from the list and update it.
-			controllerutil.RemoveFinalizer(workspace, finalizer)
-			if err := r.Update(ctx, workspace); err != nil {
-				return ctrl.Result{}, err
+		if controllerutil.ContainsFinalizer(workspace, constants.CascadingDeletionFinalizer) {
+			ok, err := r.workspaceCascadingDeletion(ctx, workspace)
+			if err != nil {
+				return ctrl.Result{}, fmt.Errorf("failed to delete workspace: %s", err)
+			}
+			if ok {
+				controllerutil.RemoveFinalizer(workspace, constants.CascadingDeletionFinalizer)
+				if err := r.Update(ctx, workspace); err != nil {
+					return ctrl.Result{}, fmt.Errorf("failed to remove finalizer: %s", err)
+				}
+				workspaceOperation.WithLabelValues("delete", workspace.Name).Inc()
 			}
-			workspaceOperation.WithLabelValues("delete", workspace.Name).Inc()
 		}
 		// Our finalizer has finished, so the reconciler can do nothing.
 		return ctrl.Result{}, nil
 	}
 
-	r.recorder.Event(workspace, corev1.EventTypeNormal, kscontroller.Synced, kscontroller.MessageResourceSynced)
+	r.recorder.Event(workspace, corev1.EventTypeNormal, "Reconcile", "Reconcile workspace successfully")
 	return ctrl.Result{}, nil
 }
+
+// workspaceCascadingDeletion handles the cascading deletion of a workspace based on its deletion propagation policy.
+// It returns a boolean indicating whether the deletion was successful and an error if any occurred.
+func (r *Reconciler) workspaceCascadingDeletion(ctx context.Context, workspace *tenantv1beta1.Workspace) (bool, error) {
+	switch workspace.Annotations[constants.DeletionPropagationAnnotation] {
+	case string(metav1.DeletePropagationOrphan):
+		// If the deletion propagation policy is "Orphan", return true without deleting namespaces.
+		return true, nil
+	case string(metav1.DeletePropagationForeground), string(metav1.DeletePropagationBackground):
+		// If the deletion propagation policy is "Foreground" or "Background", delete the namespaces.
+		if err := r.deleteNamespaces(ctx, workspace); err != nil {
+			return false, fmt.Errorf("failed to delete namespaces in workspace %s: %s", workspace.Name, err)
+		}
+		return true, nil
+	default:
+		// If the deletion propagation policy is invalid, return an error.
+		return false, fmt.Errorf("invalid deletion propagation policy: %s", workspace.Annotations[constants.DeletionPropagationAnnotation])
+	}
+}
+
+// deleteNamespaces deletes all namespaces associated with the given workspace.
+// It uses the "Background" deletion propagation policy.
+func (r *Reconciler) deleteNamespaces(ctx context.Context, workspace *tenantv1beta1.Workspace) error {
+	namespaces := &corev1.NamespaceList{}
+	if err := r.List(ctx, namespaces, client.MatchingLabels{tenantv1beta1.WorkspaceLabel: workspace.Name}); err != nil {
+		return fmt.Errorf("failed to list namespaces in workspace %s: %s", workspace.Name, err)
+	}
+	for _, ns := range namespaces.Items {
+		if err := r.Delete(ctx, &ns); err != nil {
+			if apierrors.IsNotFound(err) {
+				continue
+			}
+			return fmt.Errorf("failed to delete namespace %s: %s", ns.Name, err)
+		}
+	}
+	return nil
+}

pkg/controller/workspace/workspace_controller_test.go

@@ -9,6 +9,8 @@ import (
 	"context"
 	"time"
 
+	"kubesphere.io/kubesphere/pkg/constants"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -39,9 +41,8 @@ var _ = Describe("Workspace", func() {
 
 			By("Expecting to create workspace successfully")
 			Eventually(func() bool {
-				f := &tenantv1beta1.Workspace{}
-				_ = k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, f)
-				return len(f.Finalizers) > 0
+				_ = k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace)
+				return len(workspace.Finalizers) > 0
 			}, timeout, interval).Should(BeTrue())
 
 			// Update
@@ -60,15 +61,28 @@ var _ = Describe("Workspace", func() {
 			// Delete
 			By("Expecting to delete workspace successfully")
 			Eventually(func() error {
-				f := &tenantv1beta1.Workspace{}
-				_ = k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, f)
-				return k8sClient.Delete(context.Background(), f)
+				if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace); err != nil {
+					return err
+				}
+				return k8sClient.Delete(context.Background(), workspace)
+			}, timeout, interval).Should(Succeed())
+
+			// Update DeletionPropagation
+			By("Expecting to delete workspace successfully")
+			Eventually(func() error {
+				if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace); err != nil {
+					return err
+				}
+				if workspace.Annotations == nil {
+					workspace.Annotations = make(map[string]string)
+				}
+				workspace.Annotations[constants.DeletionPropagationAnnotation] = string(metav1.DeletePropagationBackground)
+				return k8sClient.Update(context.Background(), workspace)
 			}, timeout, interval).Should(Succeed())
 
 			By("Expecting to delete workspace finish")
 			Eventually(func() error {
-				f := &tenantv1beta1.Workspace{}
-				return k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, f)
+				return k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace)
 			}, timeout, interval).ShouldNot(Succeed())
 		})
 	})

pkg/controller/workspacetemplate/workspacetemplate_controller.go

@@ -11,10 +11,11 @@ import (
 	"fmt"
 	"strings"
 
+	"kubesphere.io/kubesphere/pkg/constants"
+
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
@@ -40,17 +41,15 @@ import (
 )
 
 const (
-	controllerName             = "workspacetemplate"
-	workspaceTemplateFinalizer = "finalizers.workspacetemplate.kubesphere.io"
-	orphanFinalizer            = "orphan.finalizers.kubesphere.io"
+	controllerName = "workspacetemplate"
 )
 
 // Reconciler reconciles a WorkspaceRoleBinding object
 type Reconciler struct {
 	client.Client
-	logger        logr.Logger
-	recorder      record.EventRecorder
-	clusterClient clusterclient.Interface
+	logger           logr.Logger
+	recorder         record.EventRecorder
+	clusterClientSet clusterclient.Interface
 }
 
 func (r *Reconciler) Enabled(clusterRole string) bool {
@@ -65,7 +64,7 @@ func (r *Reconciler) Name() string {
 }
 
 func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error {
-	r.clusterClient = mgr.ClusterClient
+	r.clusterClientSet = mgr.ClusterClient
 	r.Client = mgr.GetClient()
 	r.logger = ctrl.Log.WithName("controllers").WithName(controllerName)
 	r.recorder = mgr.GetEventRecorderFor(controllerName)
@@ -114,23 +113,26 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	if workspaceTemplate.ObjectMeta.DeletionTimestamp.IsZero() {
 		// The object is not being deleted, so if it does not have our finalizer,
 		// then lets add the finalizer and update the object.
-		if !controllerutil.ContainsFinalizer(workspaceTemplate, workspaceTemplateFinalizer) {
+		if !controllerutil.ContainsFinalizer(workspaceTemplate, constants.CascadingDeletionFinalizer) {
 			updated := workspaceTemplate.DeepCopy()
-			controllerutil.AddFinalizer(updated, workspaceTemplateFinalizer)
+			// Remove legacy finalizer
+			controllerutil.RemoveFinalizer(updated, "finalizers.workspacetemplate.kubesphere.io")
+			controllerutil.AddFinalizer(updated, constants.CascadingDeletionFinalizer)
 			return ctrl.Result{}, r.Patch(ctx, updated, client.MergeFrom(workspaceTemplate))
 		}
 	} else {
 		// The object is being deleted
-		if controllerutil.ContainsFinalizer(workspaceTemplate, workspaceTemplateFinalizer) ||
-			controllerutil.ContainsFinalizer(workspaceTemplate, orphanFinalizer) {
-			if err := r.reconcileDelete(ctx, workspaceTemplate); err != nil {
-				return ctrl.Result{}, err
+		if controllerutil.ContainsFinalizer(workspaceTemplate, constants.CascadingDeletionFinalizer) {
+			ok, err := r.workspaceTemplateCascadingDeletion(ctx, workspaceTemplate)
+			if err != nil {
+				return ctrl.Result{}, fmt.Errorf("failed to cascade delete workspacetemplate %s: %s", workspaceTemplate.Name, err)
 			}
-			// remove our finalizer from the list and update it.
-			controllerutil.RemoveFinalizer(workspaceTemplate, workspaceTemplateFinalizer)
-			controllerutil.RemoveFinalizer(workspaceTemplate, orphanFinalizer)
-			if err := r.Update(ctx, workspaceTemplate); err != nil {
-				return ctrl.Result{}, err
+			if ok {
+				// remove our finalizer from the list and update it.
+				controllerutil.RemoveFinalizer(workspaceTemplate, constants.CascadingDeletionFinalizer)
+				if err := r.Update(ctx, workspaceTemplate); err != nil {
+					return ctrl.Result{}, fmt.Errorf("failed to remove finalizer: %s", err)
+				}
 			}
 		}
 		// Our finalizer has finished, so the reconciler can do nothing.
@@ -152,7 +154,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 }
 
 func (r *Reconciler) multiClusterSync(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
-	clusters, err := r.clusterClient.ListClusters(ctx)
+	clusters, err := r.clusterClientSet.ListClusters(ctx)
 	if err != nil {
 		return fmt.Errorf("failed to list clusters: %s", err)
 	}
@@ -175,9 +177,9 @@ func (r *Reconciler) multiClusterSync(ctx context.Context, workspaceTemplate *te
 }
 
 func (r *Reconciler) syncWorkspaceTemplate(ctx context.Context, cluster clusterv1alpha1.Cluster, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
-	clusterClient, err := r.clusterClient.GetRuntimeClient(cluster.Name)
+	clusterClient, err := r.clusterClientSet.GetRuntimeClient(cluster.Name)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to get cluster client for %s: %s", cluster.Name, err)
 	}
 	if utils.WorkspaceTemplateMatchTargetCluster(workspaceTemplate, &cluster) {
 		target := &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}}
@@ -200,11 +202,9 @@ func (r *Reconciler) syncWorkspaceTemplate(ctx context.Context, cluster clusterv
 		if err != nil {
 			return err
 		}
-		klog.FromContext(ctx).V(4).Info("workspace successfully synced", "operation", op)
+		klog.FromContext(ctx).V(4).Info("workspace successfully synced", "cluster", cluster.Name, "operation", op)
 	} else {
-		orphan := metav1.DeletePropagationBackground
-		err = clusterClient.Delete(ctx, &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}},
-			&client.DeleteOptions{PropagationPolicy: &orphan})
+		err = clusterClient.Delete(ctx, &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}})
 		return client.IgnoreNotFound(err)
 	}
 	return nil
@@ -289,10 +289,17 @@ func (r *Reconciler) initManagerRoleBinding(ctx context.Context, workspaceTempla
 	return nil
 }
 
-func (r *Reconciler) reconcileDelete(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
-	clusters, err := r.clusterClient.ListClusters(ctx)
+func (r *Reconciler) workspaceTemplateCascadingDeletion(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) (bool, error) {
+	switch workspaceTemplate.Annotations[constants.DeletionPropagationAnnotation] {
+	case string(metav1.DeletePropagationOrphan), string(metav1.DeletePropagationForeground), string(metav1.DeletePropagationBackground):
+	default:
+		klog.FromContext(ctx).V(4).Info(fmt.Sprintf("waiting for deletion propagation update, invalid deletion propagation policy found: %s", workspaceTemplate.Annotations[constants.DeletionPropagationAnnotation]))
+		return false, nil
+	}
+
+	clusters, err := r.clusterClientSet.ListClusters(ctx)
 	if err != nil {
-		return err
+		return false, fmt.Errorf("failed to list clusters: %s", err)
 	}
 	var notReadyClusters []string
 	for _, cluster := range clusters {
@@ -301,28 +308,42 @@ func (r *Reconciler) reconcileDelete(ctx context.Context, workspaceTemplate *ten
 			notReadyClusters = append(notReadyClusters, cluster.Name)
 			continue
 		}
-		clusterClient, err := r.clusterClient.GetRuntimeClient(cluster.Name)
+		clusterClient, err := r.clusterClientSet.GetRuntimeClient(cluster.Name)
 		if err != nil {
 			notReadyClusters = append(notReadyClusters, cluster.Name)
 			continue
 		}
-
-		if controllerutil.ContainsFinalizer(workspaceTemplate, orphanFinalizer) {
-			orphan := metav1.DeletePropagationOrphan
-			err = clusterClient.Delete(ctx, &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}}, &client.DeleteOptions{PropagationPolicy: &orphan})
-		} else {
-			err = clusterClient.Delete(ctx, &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}}, &client.DeleteOptions{})
-		}
-
-		if !errors.IsNotFound(err) {
-			notReadyClusters = append(notReadyClusters, cluster.Name)
-			continue
+		if err := r.workspaceCascadingDeletion(ctx, cluster.Name, clusterClient, workspaceTemplate); err != nil {
+			return false, fmt.Errorf("failed to delete workspace %s in cluster %s: %s", workspaceTemplate.Name, cluster.Name, err)
 		}
 	}
 	if len(notReadyClusters) > 0 {
 		klog.FromContext(ctx).V(4).Info("cluster not ready", "clusters", strings.Join(notReadyClusters, ","))
 		r.recorder.Event(workspaceTemplate, corev1.EventTypeWarning, kscontroller.SyncFailed, fmt.Sprintf("cluster not ready: %s", strings.Join(notReadyClusters, ",")))
-		return err
+		return false, fmt.Errorf("cluster not ready: %s", strings.Join(notReadyClusters, ","))
+	}
+	return true, nil
+}
+
+func (r *Reconciler) workspaceCascadingDeletion(ctx context.Context, clusterName string, clusterClient client.Client, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
+	workspace := &tenantv1beta1.Workspace{}
+	if err := clusterClient.Get(ctx, types.NamespacedName{Name: workspaceTemplate.Name}, workspace); err != nil {
+		return client.IgnoreNotFound(err)
+	}
+	if workspace.DeletionTimestamp.IsZero() {
+		if err := clusterClient.Delete(ctx, workspace); err != nil {
+			return fmt.Errorf("failed to delete workspace %s in cluster %s: %s", workspace.Name, clusterName, err)
+		}
+	}
+	if workspace.Annotations[constants.DeletionPropagationAnnotation] == workspaceTemplate.Annotations[constants.DeletionPropagationAnnotation] {
+		return nil
+	}
+	if workspace.Annotations == nil {
+		workspace.Annotations = make(map[string]string)
+	}
+	workspace.Annotations[constants.DeletionPropagationAnnotation] = workspaceTemplate.Annotations[constants.DeletionPropagationAnnotation]
+	if err := clusterClient.Update(ctx, workspace); err != nil {
+		return fmt.Errorf("failed to update workspace %s in cluster %s: %s", workspace.Name, clusterName, err)
 	}
 	return nil
 }

pkg/models/tenant/tenant.go

@@ -11,6 +11,8 @@ import (
 	"fmt"
 	"strings"
 
+	"kubesphere.io/kubesphere/pkg/constants"
+
 	"github.com/Masterminds/semver/v3"
 	"github.com/mitchellh/mapstructure"
 	corev1 "k8s.io/api/core/v1"
@@ -41,8 +43,6 @@ import (
 	jsonpatchutil "kubesphere.io/kubesphere/pkg/utils/josnpatchutil"
 )
 
-const orphanFinalizer = "orphan.finalizers.kubesphere.io"
-
 type Interface interface {
 	ListWorkspaces(user user.Info, queryParam *query.Query) (*api.ListResult, error)
 	GetWorkspace(workspace string) (*tenantv1beta1.Workspace, error)
@@ -561,16 +561,18 @@ func (t *tenantOperator) ListClusters(user user.Info, queryParam *query.Query) (
 func (t *tenantOperator) DeleteWorkspaceTemplate(workspaceName string, opts metav1.DeleteOptions) error {
 	workspace := &tenantv1beta1.WorkspaceTemplate{}
 	if err := t.client.Get(context.Background(), types.NamespacedName{Name: workspaceName}, workspace); err != nil {
-		return err
+		return fmt.Errorf("failed to get workspace template: %s", err)
 	}
-	if opts.PropagationPolicy != nil && *opts.PropagationPolicy == metav1.DeletePropagationOrphan {
-
-		workspace.Finalizers = append(workspace.Finalizers, orphanFinalizer)
+	if opts.PropagationPolicy != nil {
+		if workspace.Annotations == nil {
+			workspace.Annotations = make(map[string]string)
+		}
+		workspace.Annotations[constants.DeletionPropagationAnnotation] = string(*opts.PropagationPolicy)
 		if err := t.client.Update(context.Background(), workspace); err != nil {
-			return err
+			return fmt.Errorf("failed to update workspace template: %s", err)
 		}
 	}
-	return t.client.Delete(context.Background(), workspace, &runtimeclient.DeleteOptions{Raw: &opts})
+	return t.client.Delete(context.Background(), workspace)
 }
 
 func (t *tenantOperator) getClusterRoleBindingsByUser(clusterName, username string) (*iamv1beta1.ClusterRoleBindingList, error) {