diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index a533e7b61..ba30f9bad 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -13,15 +13,17 @@ const ( KubeSphereConfigMapDataKey = "kubesphere.yaml" KubectlPodNamePrefix = "ks-managed-kubectl" - WorkspaceLabelKey = "kubesphere.io/workspace" - DisplayNameAnnotationKey = "kubesphere.io/alias-name" - DescriptionAnnotationKey = "kubesphere.io/description" - CreatorAnnotationKey = "kubesphere.io/creator" - UsernameLabelKey = "kubesphere.io/username" - GenericConfigTypeLabel = "config.kubesphere.io/type" - KubectlPodLabel = "kubesphere.io/kubectl-pod" - ConfigHashAnnotation = "kubesphere.io/config-hash" - KubeSphereManagedLabel = "kubesphere.io/managed" + WorkspaceLabelKey = "kubesphere.io/workspace" + DisplayNameAnnotationKey = "kubesphere.io/alias-name" + DescriptionAnnotationKey = "kubesphere.io/description" + CreatorAnnotationKey = "kubesphere.io/creator" + UsernameLabelKey = "kubesphere.io/username" + GenericConfigTypeLabel = "config.kubesphere.io/type" + KubectlPodLabel = "kubesphere.io/kubectl-pod" + ConfigHashAnnotation = "kubesphere.io/config-hash" + KubeSphereManagedLabel = "kubesphere.io/managed" + DeletionPropagationAnnotation = "kubesphere.io/deletion-propagation" + CascadingDeletionFinalizer = "kubesphere.io/cascading-deletion" ) var ( diff --git a/pkg/controller/namespace/namespace_controller.go b/pkg/controller/namespace/namespace_controller.go index 2896b9508..7d9a7b37b 100644 --- a/pkg/controller/namespace/namespace_controller.go +++ b/pkg/controller/namespace/namespace_controller.go @@ -13,15 +13,12 @@ import ( "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" iamv1beta1 "kubesphere.io/api/iam/v1beta1" - "kubesphere.io/api/tenant/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -30,13 +27,10 @@ import ( "kubesphere.io/kubesphere/pkg/constants" kscontroller "kubesphere.io/kubesphere/pkg/controller" - "kubesphere.io/kubesphere/pkg/scheme" - "kubesphere.io/kubesphere/pkg/utils/k8sutil" ) const ( controllerName = "namespace" - finalizer = "finalizers.kubesphere.io/namespaces" ) var _ kscontroller.Controller = &Reconciler{} @@ -65,10 +59,9 @@ func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error { } // +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=tenant.kubesphere.io,resources=workspaces,verbs=get;list;watch -// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=rolebases,verbs=get;list;watch -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=builtinroles,verbs=get;list;watch +// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=roles,verbs=get;list;watch +// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=rolebindings,verbs=get;list;watch func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := r.logger.WithValues("namespace", req.NamespacedName) @@ -78,21 +71,30 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{}, client.IgnoreNotFound(err) } + // Skip the namespace that is not created in workspace + if _, ok := namespace.Labels[constants.WorkspaceLabelKey]; !ok { + return ctrl.Result{}, nil + } + if namespace.ObjectMeta.DeletionTimestamp.IsZero() { // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. - if !controllerutil.ContainsFinalizer(namespace, finalizer) { + if !controllerutil.ContainsFinalizer(namespace, constants.CascadingDeletionFinalizer) { if err := r.initCreatorRoleBinding(ctx, namespace); err != nil { return ctrl.Result{}, err } updated := namespace.DeepCopy() - controllerutil.AddFinalizer(updated, finalizer) + // Remove legacy finalizer + controllerutil.RemoveFinalizer(updated, "finalizers.kubesphere.io/namespaces") + // Remove legacy ownerReferences + updated.OwnerReferences = make([]metav1.OwnerReference, 0) + controllerutil.AddFinalizer(updated, constants.CascadingDeletionFinalizer) return ctrl.Result{}, r.Patch(ctx, updated, client.MergeFrom(namespace)) } } else { // The object is being deleted - if controllerutil.ContainsFinalizer(namespace, finalizer) { - controllerutil.RemoveFinalizer(namespace, finalizer) + if controllerutil.ContainsFinalizer(namespace, constants.CascadingDeletionFinalizer) { + controllerutil.RemoveFinalizer(namespace, constants.CascadingDeletionFinalizer) if err := r.Update(ctx, namespace); err != nil { return ctrl.Result{}, err } @@ -105,64 +107,15 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{}, err } - if err := r.reconcileWorkspaceOwnerReference(ctx, namespace); err != nil { - return ctrl.Result{}, err - } - r.recorder.Event(namespace, corev1.EventTypeNormal, kscontroller.Synced, kscontroller.MessageResourceSynced) return ctrl.Result{}, nil } -func (r *Reconciler) reconcileWorkspaceOwnerReference(ctx context.Context, namespace *corev1.Namespace) error { - workspaceName, hasWorkspaceLabel := namespace.Labels[v1beta1.WorkspaceLabel] - - if !hasWorkspaceLabel { - if k8sutil.IsControlledBy(namespace.OwnerReferences, v1beta1.ResourceKindWorkspace, workspaceName) { - namespace.OwnerReferences = k8sutil.RemoveWorkspaceOwnerReference(namespace.OwnerReferences) - return r.Update(ctx, namespace) - } - // noting to do - return nil - } - - workspace := &v1beta1.Workspace{} - if err := r.Get(ctx, types.NamespacedName{Name: workspaceName}, workspace); err != nil { - owner := metav1.GetControllerOf(namespace) - if errors.IsNotFound(err) && owner != nil && owner.Kind == v1beta1.ResourceKindWorkspace { - namespace.OwnerReferences = k8sutil.RemoveWorkspaceOwnerReference(namespace.OwnerReferences) - return r.Update(ctx, namespace) - } - return client.IgnoreNotFound(err) - } - - // workspace has been deleted - if !workspace.ObjectMeta.DeletionTimestamp.IsZero() { - return nil - } - - if !metav1.IsControlledBy(namespace, workspace) && namespace.Labels[constants.KubeSphereManagedLabel] == "true" { - namespace = namespace.DeepCopy() - if err := controllerutil.SetControllerReference(workspace, namespace, scheme.Scheme); err != nil { - return err - } - if err := r.Update(ctx, namespace); err != nil { - return err - } - } - - return nil -} - func (r *Reconciler) initRoles(ctx context.Context, namespace *corev1.Namespace) error { - if _, ok := namespace.Labels[constants.WorkspaceLabelKey]; !ok { - return nil - } - logger := klog.FromContext(ctx) var templates iamv1beta1.BuiltinRoleList - matchingLabels := client.MatchingLabels{iamv1beta1.ScopeLabel: iamv1beta1.ScopeNamespace} - if err := r.List(ctx, &templates, matchingLabels); err != nil { - return err + if err := r.List(ctx, &templates, client.MatchingLabels{iamv1beta1.ScopeLabel: iamv1beta1.ScopeNamespace}); err != nil { + return fmt.Errorf("failed to list builtin roles: %v", err) } for _, template := range templates.Items { selector, err := metav1.LabelSelectorAsSelector(&template.TargetSelector) @@ -185,9 +138,9 @@ func (r *Reconciler) initRoles(ctx context.Context, namespace *corev1.Namespace) return nil }) if err != nil { - return err + return fmt.Errorf("failed to create or update builtin role: %v", err) } - logger.V(4).Info("builtin role successfully initialized", "operation", op) + logger.V(4).Info("builtin role initialized", "operation", op) } else if err != nil { logger.Error(err, "invalid builtin role found", "name", template.Name) } @@ -228,6 +181,6 @@ func (r *Reconciler) initCreatorRoleBinding(ctx context.Context, namespace *core if err != nil { return err } - klog.FromContext(ctx).V(4).Info("creator role binding successfully initialized", "operation", op) + klog.FromContext(ctx).V(4).Info("creator role binding initialized", "operation", op) return nil } diff --git a/pkg/controller/namespace/namespace_controller_test.go b/pkg/controller/namespace/namespace_controller_test.go index 8f6c6ef79..4e61fe973 100644 --- a/pkg/controller/namespace/namespace_controller_test.go +++ b/pkg/controller/namespace/namespace_controller_test.go @@ -54,41 +54,17 @@ var _ = Describe("Namespace", func() { Expect(k8sClient.Create(context.Background(), namespace)).Should(Succeed()) By("Expecting to create namespace successfully") - Eventually(func() bool { - k8sClient.Get(context.Background(), types.NamespacedName{Name: namespace.Name}, namespace) - return !namespace.CreationTimestamp.IsZero() - }, timeout, interval).Should(BeTrue()) - - By("Expecting to set owner reference successfully") - Eventually(func() bool { - k8sClient.Get(context.Background(), types.NamespacedName{Name: namespace.Name}, namespace) - return len(namespace.OwnerReferences) > 0 - }, timeout, interval).Should(BeTrue()) - - Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace)).Should(Succeed()) - - controlled := true - expectedOwnerReference := metav1.OwnerReference{ - Kind: workspace.Kind, - APIVersion: workspace.APIVersion, - UID: workspace.UID, - Name: workspace.Name, - Controller: &controlled, - BlockOwnerDeletion: &controlled, - } - - By("Expecting to bind workspace successfully") - Expect(namespace.OwnerReferences).To(ContainElement(expectedOwnerReference)) - Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace)).Should(Succeed()) + Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: namespace.Name}, namespace)).Should(Succeed()) By("Expecting to update namespace successfully") updated := namespace.DeepCopy() updated.Labels[constants.WorkspaceLabelKey] = "workspace-not-exist" + Expect(k8sClient.Update(context.Background(), updated)).Should(Succeed()) By("Expecting to unbind workspace successfully") Eventually(func() bool { - k8sClient.Get(context.Background(), types.NamespacedName{Name: namespace.Name}, namespace) + _ = k8sClient.Get(context.Background(), types.NamespacedName{Name: namespace.Name}, namespace) return len(namespace.OwnerReferences) == 0 }, timeout, interval).Should(BeTrue()) }) diff --git a/pkg/controller/workspace/workspace_controller.go b/pkg/controller/workspace/workspace_controller.go index 3be2321c2..638ee783a 100644 --- a/pkg/controller/workspace/workspace_controller.go +++ b/pkg/controller/workspace/workspace_controller.go @@ -7,9 +7,15 @@ package workspace import ( "context" + "fmt" + + "kubesphere.io/kubesphere/pkg/constants" + + apierrors "k8s.io/apimachinery/pkg/api/errors" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" tenantv1beta1 "kubesphere.io/api/tenant/v1beta1" @@ -24,7 +30,6 @@ import ( const ( controllerName = "workspace" - finalizer = "finalizers.tenant.kubesphere.io" ) var _ kscontroller.Controller = &Reconciler{} @@ -53,11 +58,6 @@ func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error { } // +kubebuilder:rbac:groups=tenant.kubesphere.io,resources=workspaces,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=tenant.kubesphere.io,resources=workspaces/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=users,verbs=get;list;watch -// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=rolebases,verbs=get;list;watch -// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=workspaceroles,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=workspacerolebindings,verbs=get;list;watch;create;update;patch;delete func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := r.logger.WithValues("workspace", req.NamespacedName) @@ -70,28 +70,71 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if workspace.ObjectMeta.DeletionTimestamp.IsZero() { // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. - if !controllerutil.ContainsFinalizer(workspace, finalizer) { + if !controllerutil.ContainsFinalizer(workspace, constants.CascadingDeletionFinalizer) { expected := workspace.DeepCopy() - controllerutil.AddFinalizer(expected, finalizer) + // Remove legacy finalizer + controllerutil.RemoveFinalizer(expected, "finalizers.tenant.kubesphere.io") + controllerutil.AddFinalizer(expected, constants.CascadingDeletionFinalizer) if err := r.Patch(ctx, expected, client.MergeFrom(workspace)); err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to add finalizer: %s", err) } workspaceOperation.WithLabelValues("create", workspace.Name).Inc() } } else { - // The object is being deleted - if controllerutil.ContainsFinalizer(workspace, finalizer) { - // remove our finalizer from the list and update it. - controllerutil.RemoveFinalizer(workspace, finalizer) - if err := r.Update(ctx, workspace); err != nil { - return ctrl.Result{}, err + if controllerutil.ContainsFinalizer(workspace, constants.CascadingDeletionFinalizer) { + ok, err := r.workspaceCascadingDeletion(ctx, workspace) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to delete workspace: %s", err) + } + if ok { + controllerutil.RemoveFinalizer(workspace, constants.CascadingDeletionFinalizer) + if err := r.Update(ctx, workspace); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to remove finalizer: %s", err) + } + workspaceOperation.WithLabelValues("delete", workspace.Name).Inc() } - workspaceOperation.WithLabelValues("delete", workspace.Name).Inc() } // Our finalizer has finished, so the reconciler can do nothing. return ctrl.Result{}, nil } - r.recorder.Event(workspace, corev1.EventTypeNormal, kscontroller.Synced, kscontroller.MessageResourceSynced) + r.recorder.Event(workspace, corev1.EventTypeNormal, "Reconcile", "Reconcile workspace successfully") return ctrl.Result{}, nil } + +// workspaceCascadingDeletion handles the cascading deletion of a workspace based on its deletion propagation policy. +// It returns a boolean indicating whether the deletion was successful and an error if any occurred. +func (r *Reconciler) workspaceCascadingDeletion(ctx context.Context, workspace *tenantv1beta1.Workspace) (bool, error) { + switch workspace.Annotations[constants.DeletionPropagationAnnotation] { + case string(metav1.DeletePropagationOrphan): + // If the deletion propagation policy is "Orphan", return true without deleting namespaces. + return true, nil + case string(metav1.DeletePropagationForeground), string(metav1.DeletePropagationBackground): + // If the deletion propagation policy is "Foreground" or "Background", delete the namespaces. + if err := r.deleteNamespaces(ctx, workspace); err != nil { + return false, fmt.Errorf("failed to delete namespaces in workspace %s: %s", workspace.Name, err) + } + return true, nil + default: + // If the deletion propagation policy is invalid, return an error. + return false, fmt.Errorf("invalid deletion propagation policy: %s", workspace.Annotations[constants.DeletionPropagationAnnotation]) + } +} + +// deleteNamespaces deletes all namespaces associated with the given workspace. +// It uses the "Background" deletion propagation policy. +func (r *Reconciler) deleteNamespaces(ctx context.Context, workspace *tenantv1beta1.Workspace) error { + namespaces := &corev1.NamespaceList{} + if err := r.List(ctx, namespaces, client.MatchingLabels{tenantv1beta1.WorkspaceLabel: workspace.Name}); err != nil { + return fmt.Errorf("failed to list namespaces in workspace %s: %s", workspace.Name, err) + } + for _, ns := range namespaces.Items { + if err := r.Delete(ctx, &ns); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return fmt.Errorf("failed to delete namespace %s: %s", ns.Name, err) + } + } + return nil +} diff --git a/pkg/controller/workspace/workspace_controller_test.go b/pkg/controller/workspace/workspace_controller_test.go index 6ce26c76f..dd9ae9a47 100644 --- a/pkg/controller/workspace/workspace_controller_test.go +++ b/pkg/controller/workspace/workspace_controller_test.go @@ -9,6 +9,8 @@ import ( "context" "time" + "kubesphere.io/kubesphere/pkg/constants" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,9 +41,8 @@ var _ = Describe("Workspace", func() { By("Expecting to create workspace successfully") Eventually(func() bool { - f := &tenantv1beta1.Workspace{} - _ = k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, f) - return len(f.Finalizers) > 0 + _ = k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace) + return len(workspace.Finalizers) > 0 }, timeout, interval).Should(BeTrue()) // Update @@ -60,15 +61,28 @@ var _ = Describe("Workspace", func() { // Delete By("Expecting to delete workspace successfully") Eventually(func() error { - f := &tenantv1beta1.Workspace{} - _ = k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, f) - return k8sClient.Delete(context.Background(), f) + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace); err != nil { + return err + } + return k8sClient.Delete(context.Background(), workspace) + }, timeout, interval).Should(Succeed()) + + // Update DeletionPropagation + By("Expecting to delete workspace successfully") + Eventually(func() error { + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace); err != nil { + return err + } + if workspace.Annotations == nil { + workspace.Annotations = make(map[string]string) + } + workspace.Annotations[constants.DeletionPropagationAnnotation] = string(metav1.DeletePropagationBackground) + return k8sClient.Update(context.Background(), workspace) }, timeout, interval).Should(Succeed()) By("Expecting to delete workspace finish") Eventually(func() error { - f := &tenantv1beta1.Workspace{} - return k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, f) + return k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace) }, timeout, interval).ShouldNot(Succeed()) }) }) diff --git a/pkg/controller/workspacetemplate/workspacetemplate_controller.go b/pkg/controller/workspacetemplate/workspacetemplate_controller.go index 7d00b4ecd..7b0890958 100644 --- a/pkg/controller/workspacetemplate/workspacetemplate_controller.go +++ b/pkg/controller/workspacetemplate/workspacetemplate_controller.go @@ -11,10 +11,11 @@ import ( "fmt" "strings" + "kubesphere.io/kubesphere/pkg/constants" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" @@ -40,17 +41,15 @@ import ( ) const ( - controllerName = "workspacetemplate" - workspaceTemplateFinalizer = "finalizers.workspacetemplate.kubesphere.io" - orphanFinalizer = "orphan.finalizers.kubesphere.io" + controllerName = "workspacetemplate" ) // Reconciler reconciles a WorkspaceRoleBinding object type Reconciler struct { client.Client - logger logr.Logger - recorder record.EventRecorder - clusterClient clusterclient.Interface + logger logr.Logger + recorder record.EventRecorder + clusterClientSet clusterclient.Interface } func (r *Reconciler) Enabled(clusterRole string) bool { @@ -65,7 +64,7 @@ func (r *Reconciler) Name() string { } func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error { - r.clusterClient = mgr.ClusterClient + r.clusterClientSet = mgr.ClusterClient r.Client = mgr.GetClient() r.logger = ctrl.Log.WithName("controllers").WithName(controllerName) r.recorder = mgr.GetEventRecorderFor(controllerName) @@ -114,23 +113,26 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if workspaceTemplate.ObjectMeta.DeletionTimestamp.IsZero() { // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. - if !controllerutil.ContainsFinalizer(workspaceTemplate, workspaceTemplateFinalizer) { + if !controllerutil.ContainsFinalizer(workspaceTemplate, constants.CascadingDeletionFinalizer) { updated := workspaceTemplate.DeepCopy() - controllerutil.AddFinalizer(updated, workspaceTemplateFinalizer) + // Remove legacy finalizer + controllerutil.RemoveFinalizer(updated, "finalizers.workspacetemplate.kubesphere.io") + controllerutil.AddFinalizer(updated, constants.CascadingDeletionFinalizer) return ctrl.Result{}, r.Patch(ctx, updated, client.MergeFrom(workspaceTemplate)) } } else { // The object is being deleted - if controllerutil.ContainsFinalizer(workspaceTemplate, workspaceTemplateFinalizer) || - controllerutil.ContainsFinalizer(workspaceTemplate, orphanFinalizer) { - if err := r.reconcileDelete(ctx, workspaceTemplate); err != nil { - return ctrl.Result{}, err + if controllerutil.ContainsFinalizer(workspaceTemplate, constants.CascadingDeletionFinalizer) { + ok, err := r.workspaceTemplateCascadingDeletion(ctx, workspaceTemplate) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to cascade delete workspacetemplate %s: %s", workspaceTemplate.Name, err) } - // remove our finalizer from the list and update it. - controllerutil.RemoveFinalizer(workspaceTemplate, workspaceTemplateFinalizer) - controllerutil.RemoveFinalizer(workspaceTemplate, orphanFinalizer) - if err := r.Update(ctx, workspaceTemplate); err != nil { - return ctrl.Result{}, err + if ok { + // remove our finalizer from the list and update it. + controllerutil.RemoveFinalizer(workspaceTemplate, constants.CascadingDeletionFinalizer) + if err := r.Update(ctx, workspaceTemplate); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to remove finalizer: %s", err) + } } } // Our finalizer has finished, so the reconciler can do nothing. @@ -152,7 +154,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } func (r *Reconciler) multiClusterSync(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error { - clusters, err := r.clusterClient.ListClusters(ctx) + clusters, err := r.clusterClientSet.ListClusters(ctx) if err != nil { return fmt.Errorf("failed to list clusters: %s", err) } @@ -175,9 +177,9 @@ func (r *Reconciler) multiClusterSync(ctx context.Context, workspaceTemplate *te } func (r *Reconciler) syncWorkspaceTemplate(ctx context.Context, cluster clusterv1alpha1.Cluster, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error { - clusterClient, err := r.clusterClient.GetRuntimeClient(cluster.Name) + clusterClient, err := r.clusterClientSet.GetRuntimeClient(cluster.Name) if err != nil { - return err + return fmt.Errorf("failed to get cluster client for %s: %s", cluster.Name, err) } if utils.WorkspaceTemplateMatchTargetCluster(workspaceTemplate, &cluster) { target := &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}} @@ -200,11 +202,9 @@ func (r *Reconciler) syncWorkspaceTemplate(ctx context.Context, cluster clusterv if err != nil { return err } - klog.FromContext(ctx).V(4).Info("workspace successfully synced", "operation", op) + klog.FromContext(ctx).V(4).Info("workspace successfully synced", "cluster", cluster.Name, "operation", op) } else { - orphan := metav1.DeletePropagationBackground - err = clusterClient.Delete(ctx, &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}}, - &client.DeleteOptions{PropagationPolicy: &orphan}) + err = clusterClient.Delete(ctx, &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}}) return client.IgnoreNotFound(err) } return nil @@ -289,10 +289,17 @@ func (r *Reconciler) initManagerRoleBinding(ctx context.Context, workspaceTempla return nil } -func (r *Reconciler) reconcileDelete(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error { - clusters, err := r.clusterClient.ListClusters(ctx) +func (r *Reconciler) workspaceTemplateCascadingDeletion(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) (bool, error) { + switch workspaceTemplate.Annotations[constants.DeletionPropagationAnnotation] { + case string(metav1.DeletePropagationOrphan), string(metav1.DeletePropagationForeground), string(metav1.DeletePropagationBackground): + default: + klog.FromContext(ctx).V(4).Info(fmt.Sprintf("waiting for deletion propagation update, invalid deletion propagation policy found: %s", workspaceTemplate.Annotations[constants.DeletionPropagationAnnotation])) + return false, nil + } + + clusters, err := r.clusterClientSet.ListClusters(ctx) if err != nil { - return err + return false, fmt.Errorf("failed to list clusters: %s", err) } var notReadyClusters []string for _, cluster := range clusters { @@ -301,28 +308,42 @@ func (r *Reconciler) reconcileDelete(ctx context.Context, workspaceTemplate *ten notReadyClusters = append(notReadyClusters, cluster.Name) continue } - clusterClient, err := r.clusterClient.GetRuntimeClient(cluster.Name) + clusterClient, err := r.clusterClientSet.GetRuntimeClient(cluster.Name) if err != nil { notReadyClusters = append(notReadyClusters, cluster.Name) continue } - - if controllerutil.ContainsFinalizer(workspaceTemplate, orphanFinalizer) { - orphan := metav1.DeletePropagationOrphan - err = clusterClient.Delete(ctx, &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}}, &client.DeleteOptions{PropagationPolicy: &orphan}) - } else { - err = clusterClient.Delete(ctx, &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}}, &client.DeleteOptions{}) - } - - if !errors.IsNotFound(err) { - notReadyClusters = append(notReadyClusters, cluster.Name) - continue + if err := r.workspaceCascadingDeletion(ctx, cluster.Name, clusterClient, workspaceTemplate); err != nil { + return false, fmt.Errorf("failed to delete workspace %s in cluster %s: %s", workspaceTemplate.Name, cluster.Name, err) } } if len(notReadyClusters) > 0 { klog.FromContext(ctx).V(4).Info("cluster not ready", "clusters", strings.Join(notReadyClusters, ",")) r.recorder.Event(workspaceTemplate, corev1.EventTypeWarning, kscontroller.SyncFailed, fmt.Sprintf("cluster not ready: %s", strings.Join(notReadyClusters, ","))) - return err + return false, fmt.Errorf("cluster not ready: %s", strings.Join(notReadyClusters, ",")) + } + return true, nil +} + +func (r *Reconciler) workspaceCascadingDeletion(ctx context.Context, clusterName string, clusterClient client.Client, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error { + workspace := &tenantv1beta1.Workspace{} + if err := clusterClient.Get(ctx, types.NamespacedName{Name: workspaceTemplate.Name}, workspace); err != nil { + return client.IgnoreNotFound(err) + } + if workspace.DeletionTimestamp.IsZero() { + if err := clusterClient.Delete(ctx, workspace); err != nil { + return fmt.Errorf("failed to delete workspace %s in cluster %s: %s", workspace.Name, clusterName, err) + } + } + if workspace.Annotations[constants.DeletionPropagationAnnotation] == workspaceTemplate.Annotations[constants.DeletionPropagationAnnotation] { + return nil + } + if workspace.Annotations == nil { + workspace.Annotations = make(map[string]string) + } + workspace.Annotations[constants.DeletionPropagationAnnotation] = workspaceTemplate.Annotations[constants.DeletionPropagationAnnotation] + if err := clusterClient.Update(ctx, workspace); err != nil { + return fmt.Errorf("failed to update workspace %s in cluster %s: %s", workspace.Name, clusterName, err) } return nil } diff --git a/pkg/models/tenant/tenant.go b/pkg/models/tenant/tenant.go index 761065c42..526d6b5db 100644 --- a/pkg/models/tenant/tenant.go +++ b/pkg/models/tenant/tenant.go @@ -11,6 +11,8 @@ import ( "fmt" "strings" + "kubesphere.io/kubesphere/pkg/constants" + "github.com/Masterminds/semver/v3" "github.com/mitchellh/mapstructure" corev1 "k8s.io/api/core/v1" @@ -41,8 +43,6 @@ import ( jsonpatchutil "kubesphere.io/kubesphere/pkg/utils/josnpatchutil" ) -const orphanFinalizer = "orphan.finalizers.kubesphere.io" - type Interface interface { ListWorkspaces(user user.Info, queryParam *query.Query) (*api.ListResult, error) GetWorkspace(workspace string) (*tenantv1beta1.Workspace, error) @@ -561,16 +561,18 @@ func (t *tenantOperator) ListClusters(user user.Info, queryParam *query.Query) ( func (t *tenantOperator) DeleteWorkspaceTemplate(workspaceName string, opts metav1.DeleteOptions) error { workspace := &tenantv1beta1.WorkspaceTemplate{} if err := t.client.Get(context.Background(), types.NamespacedName{Name: workspaceName}, workspace); err != nil { - return err + return fmt.Errorf("failed to get workspace template: %s", err) } - if opts.PropagationPolicy != nil && *opts.PropagationPolicy == metav1.DeletePropagationOrphan { - - workspace.Finalizers = append(workspace.Finalizers, orphanFinalizer) + if opts.PropagationPolicy != nil { + if workspace.Annotations == nil { + workspace.Annotations = make(map[string]string) + } + workspace.Annotations[constants.DeletionPropagationAnnotation] = string(*opts.PropagationPolicy) if err := t.client.Update(context.Background(), workspace); err != nil { - return err + return fmt.Errorf("failed to update workspace template: %s", err) } } - return t.client.Delete(context.Background(), workspace, &runtimeclient.DeleteOptions{Raw: &opts}) + return t.client.Delete(context.Background(), workspace) } func (t *tenantOperator) getClusterRoleBindingsByUser(clusterName, username string) (*iamv1beta1.ClusterRoleBindingList, error) {