Merge pull request #4128 from xyz-li/cleanup_app

clean up app when workspace is deleted

pkg/constants/constants.go

@@ -41,6 +41,7 @@ const (
 	ChartApplicationIdLabelKey        = "application.kubesphere.io/app-id"
 	ChartApplicationVersionIdLabelKey = "application.kubesphere.io/app-version-id"
 	CategoryIdLabelKey                = "application.kubesphere.io/app-category-id"
+	DanglingAppCleanupKey             = "application.kubesphere.io/app-cleanup"
 	CreatorAnnotationKey              = "kubesphere.io/creator"
 	UsernameLabelKey                  = "kubesphere.io/username"
 	DevOpsProjectLabelKey             = "kubesphere.io/devopsproject"
@@ -71,6 +72,9 @@ const (
 	OpenpitrixRepositoryTag  = "Repository"
 	OpenpitrixManagementTag  = "App Management"
 
+	CleanupDanglingAppOngoing = "ongoing"
+	CleanupDanglingAppDone    = "done"
+
 	DevOpsCredentialTag  = "DevOps Credential"
 	DevOpsPipelineTag    = "DevOps Pipeline"
 	DevOpsWebhookTag     = "DevOps Webhook"

pkg/controller/openpitrix/helmapplication/helm_application_controller.go

@@ -22,6 +22,8 @@ import (
 	"strconv"
 	"strings"
 
+	"k8s.io/apimachinery/pkg/labels"
+
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog"
@@ -79,6 +81,11 @@ func (r *ReconcileHelmApplication) Reconcile(ctx context.Context, request reconc
 		}
 
 		if !inAppStore(app) {
+			// The workspace of this app is being deleting, clean up this app
+			if err := r.cleanupDanglingApp(context.TODO(), app); err != nil {
+				return reconcile.Result{}, err
+			}
+
 			if app.Status.State == v1alpha1.StateActive ||
 				app.Status.State == v1alpha1.StateSuspended {
 				if err := r.createAppCopyInAppStore(rootCtx, app); err != nil {
@@ -181,3 +188,88 @@ func (r *ReconcileHelmApplication) SetupWithManager(mgr ctrl.Manager) error {
 func inAppStore(app *v1alpha1.HelmApplication) bool {
 	return strings.HasSuffix(app.Name, v1alpha1.HelmApplicationAppStoreSuffix)
 }
+
+// cleanupDanglingApp deletes the app when it is not active and not suspended,
+// sets the workspace label to empty and remove parts of the appversion when app state is active or suspended.
+//
+// When one workspace is being deleting, we can delete all the app which are not active or suspended of this workspace,
+// but when an app has been promoted to app store, we have to deal with it specially.
+// If we just delete that app, then this app will be deleted from app store too.
+// If we leave it alone, and user creates a workspace with the same name sometime,
+// then this app will appear in this new workspace which confuses the user.
+// So we need to delete all the appversion which are not active or suspended first,
+// then remove the workspace label from the app. And on the console of ks, we will show something
+// like "(workspace deleted)" to user for this app.
+func (r *ReconcileHelmApplication) cleanupDanglingApp(ctx context.Context, app *v1alpha1.HelmApplication) error {
+	if app.Annotations != nil && app.Annotations[constants.DanglingAppCleanupKey] == constants.CleanupDanglingAppOngoing {
+		// Just delete the app when the state is not active or not suspended.
+		if app.Status.State != v1alpha1.StateActive && app.Status.State != v1alpha1.StateSuspended {
+			err := r.Delete(ctx, app)
+			if err != nil {
+				klog.Errorf("delete app: %s, state: %s, error: %s",
+					app.GetHelmApplicationId(), app.Status.State, err)
+				return err
+			}
+			return nil
+		}
+
+		var appVersions v1alpha1.HelmApplicationVersionList
+		err := r.List(ctx, &appVersions, &client.ListOptions{LabelSelector: labels.SelectorFromSet(map[string]string{
+			constants.ChartApplicationIdLabelKey: app.GetHelmApplicationId()})})
+		if err != nil {
+			klog.Errorf("list app version of %s failed, error: %s", app.GetHelmApplicationId(), err)
+			return err
+		}
+
+		// Delete app version where are not active and not suspended.
+		for _, version := range appVersions.Items {
+			if version.Status.State != v1alpha1.StateActive && version.Status.State != v1alpha1.StateSuspended {
+				err = r.Delete(ctx, &version)
+				if err != nil {
+					klog.Errorf("delete app version: %s, state: %s, error: %s",
+						version.GetHelmApplicationVersionId(), version.Status.State, err)
+					return err
+				}
+			}
+		}
+
+		// Mark the app that the workspace to which it belongs has been deleted.
+		var appInStore v1alpha1.HelmApplication
+		err = r.Get(ctx,
+			types.NamespacedName{Name: fmt.Sprintf("%s%s", app.GetHelmApplicationId(), v1alpha1.HelmApplicationAppStoreSuffix)}, &appInStore)
+		if err != nil {
+			if !apierrors.IsNotFound(err) {
+				return err
+			}
+		} else {
+			appCopy := appInStore.DeepCopy()
+			if appCopy.Annotations == nil {
+				appCopy.Annotations = map[string]string{}
+			}
+			appCopy.Annotations[constants.DanglingAppCleanupKey] = constants.CleanupDanglingAppDone
+
+			patchedApp := client.MergeFrom(&appInStore)
+			err = r.Patch(ctx, appCopy, patchedApp)
+			if err != nil {
+				klog.Errorf("patch app: %s failed, error: %s", app.GetHelmApplicationId(), err)
+				return err
+			}
+		}
+
+		appCopy := app.DeepCopy()
+		appCopy.Annotations[constants.DanglingAppCleanupKey] = constants.CleanupDanglingAppDone
+		// Remove the workspace label, or if user creates a workspace with the same name, this app will show in the new workspace.
+		if appCopy.Labels == nil {
+			appCopy.Labels = map[string]string{}
+		}
+		appCopy.Labels[constants.WorkspaceLabelKey] = ""
+		patchedApp := client.MergeFrom(app)
+		err = r.Patch(ctx, appCopy, patchedApp)
+		if err != nil {
+			klog.Errorf("patch app: %s failed, error: %s", app.GetHelmApplicationId(), err)
+			return err
+		}
+	}
+
+	return nil
+}

pkg/controller/openpitrix/helmapplication/helm_application_controller_suite_test.go

@@ -0,0 +1,105 @@
+/*
+Copyright 2019 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helmapplication
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/onsi/gomega/gexec"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/klog/klogr"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"kubesphere.io/kubesphere/pkg/apis"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
+)
+
+// These tests use Ginkgo (BDD-style Go testing framework). Refer to
+// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
+
+var k8sClient client.Client
+var k8sManager ctrl.Manager
+var testEnv *envtest.Environment
+
+func TestHelmApplicationController(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecsWithDefaultAndCustomReporters(t,
+		"HelmCategory Application Test Suite",
+		[]Reporter{printer.NewlineReporter{}})
+}
+
+var _ = BeforeSuite(func(done Done) {
+	logf.SetLogger(klogr.New())
+
+	By("bootstrapping test environment")
+	t := true
+	if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" {
+		testEnv = &envtest.Environment{
+			UseExistingCluster: &t,
+		}
+	} else {
+		testEnv = &envtest.Environment{
+			CRDDirectoryPaths:        []string{filepath.Join("..", "..", "..", "..", "config", "crds")},
+			AttachControlPlaneOutput: false,
+		}
+	}
+
+	cfg, err := testEnv.Start()
+	Expect(err).ToNot(HaveOccurred())
+	Expect(cfg).ToNot(BeNil())
+
+	err = apis.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{
+		MetricsBindAddress: "0",
+		Scheme:             scheme.Scheme,
+	})
+	Expect(err).ToNot(HaveOccurred())
+
+	err = (&ReconcileHelmApplication{}).SetupWithManager(k8sManager)
+	Expect(err).ToNot(HaveOccurred())
+
+	err = (&ReconcileHelmApplicationVersion{}).SetupWithManager(k8sManager)
+	Expect(err).ToNot(HaveOccurred())
+
+	go func() {
+		err = k8sManager.Start(ctrl.SetupSignalHandler())
+		Expect(err).ToNot(HaveOccurred())
+	}()
+
+	k8sClient = k8sManager.GetClient()
+	Expect(k8sClient).ToNot(BeNil())
+
+	close(done)
+}, 60)
+
+var _ = AfterSuite(func() {
+	By("tearing down the test environment")
+	gexec.KillAndWait(5 * time.Second)
+	err := testEnv.Stop()
+	Expect(err).ToNot(HaveOccurred())
+})

pkg/controller/openpitrix/helmapplication/helm_application_controller_test.go

@@ -0,0 +1,142 @@
+/*
+Copyright 2019 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helmapplication
+
+import (
+	"context"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	"kubesphere.io/api/application/v1alpha1"
+
+	"kubesphere.io/kubesphere/pkg/constants"
+	"kubesphere.io/kubesphere/pkg/utils/idutils"
+)
+
+var _ = Describe("helmApplication", func() {
+
+	const timeout = time.Second * 240
+	const interval = time.Second * 1
+
+	app := createApp()
+	appVer := createAppVersion(app.GetHelmApplicationId(), "0.0.1")
+	appVer2 := createAppVersion(app.GetHelmApplicationId(), "0.0.2")
+
+	BeforeEach(func() {
+		err := k8sClient.Create(context.Background(), app)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = k8sClient.Create(context.Background(), appVer)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = k8sClient.Create(context.Background(), appVer2)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	Context("Helm Application Controller", func() {
+		It("Should success", func() {
+
+			By("Update helm app version status")
+			Eventually(func() bool {
+				k8sClient.Get(context.Background(), types.NamespacedName{Name: appVer.Name}, appVer)
+				appVer.Status = v1alpha1.HelmApplicationVersionStatus{
+					State: v1alpha1.StateActive,
+				}
+				err := k8sClient.Status().Update(context.Background(), appVer)
+				return err == nil
+			}, timeout, interval).Should(BeTrue())
+
+			By("Wait for app status become active")
+			Eventually(func() bool {
+				var localApp v1alpha1.HelmApplication
+				appKey := types.NamespacedName{
+					Name: app.Name,
+				}
+				k8sClient.Get(context.Background(), appKey, &localApp)
+				return localApp.State() == v1alpha1.StateActive
+			}, timeout, interval).Should(BeTrue())
+
+			By("Mark workspace is deleted")
+			Eventually(func() bool {
+				var localApp v1alpha1.HelmApplication
+				err := k8sClient.Get(context.Background(), types.NamespacedName{Name: app.Name}, &localApp)
+				if err != nil {
+					return false
+				}
+				appCopy := localApp.DeepCopy()
+				appCopy.Annotations = map[string]string{}
+				appCopy.Annotations[constants.DanglingAppCleanupKey] = constants.CleanupDanglingAppOngoing
+				patchData := client.MergeFrom(&localApp)
+				err = k8sClient.Patch(context.Background(), appCopy, patchData)
+				return err == nil
+			}, timeout, interval).Should(BeTrue())
+
+			By("Draft app version are deleted")
+			Eventually(func() bool {
+				var ver v1alpha1.HelmApplicationVersion
+				err := k8sClient.Get(context.Background(), types.NamespacedName{Name: appVer2.Name}, &ver)
+				if apierrors.IsNotFound(err) {
+					return true
+				}
+				return false
+			}, timeout, interval).Should(BeTrue())
+
+			By("Active app version exists")
+			Eventually(func() bool {
+				var ver v1alpha1.HelmApplicationVersion
+				err := k8sClient.Get(context.Background(), types.NamespacedName{Name: appVer.Name}, &ver)
+				return err == nil
+			}, timeout, interval).Should(BeTrue())
+
+		})
+	})
+})
+
+func createApp() *v1alpha1.HelmApplication {
+	return &v1alpha1.HelmApplication{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: idutils.GetUuid36(v1alpha1.HelmApplicationIdPrefix),
+		},
+		Spec: v1alpha1.HelmApplicationSpec{
+			Name: "dummy-chart",
+		},
+	}
+}
+
+func createAppVersion(appId string, version string) *v1alpha1.HelmApplicationVersion {
+	return &v1alpha1.HelmApplicationVersion{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: idutils.GetUuid36(v1alpha1.HelmApplicationVersionIdPrefix),
+			Labels: map[string]string{
+				constants.ChartApplicationIdLabelKey: appId,
+			},
+		},
+		Spec: v1alpha1.HelmApplicationVersionSpec{
+			Metadata: &v1alpha1.Metadata{
+				Version: version,
+				Name:    "dummy-chart",
+			},
+		},
+	}
+}

pkg/controller/workspacetemplate/workspacetemplate_controller.go

@@ -424,11 +424,16 @@ func (r *Reconciler) deleteHelmApps(ctx context.Context, ws string) error {
 	if err != nil {
 		return err
 	}
-	for i := range apps.Items {
-		state := apps.Items[i].Status.State
-		// active and suspended applications belong to app store, they should not be removed here.
-		if !(state == v1alpha1.StateActive || state == v1alpha1.StateSuspended) {
-			err = r.Delete(ctx, &apps.Items[i])
+	for _, app := range apps.Items {
+		if app.Annotations == nil {
+			app.Annotations = map[string]string{}
+		}
+		if _, exists := app.Annotations[constants.DanglingAppCleanupKey]; !exists {
+			// Mark the app, the cleanup is in the application controller.
+			appCopy := app.DeepCopy()
+			appCopy.Annotations[constants.DanglingAppCleanupKey] = constants.CleanupDanglingAppOngoing
+			appPatch := client.MergeFrom(&app)
+			err = r.Patch(ctx, appCopy, appPatch)
 			if err != nil {
 				return err
 			}

pkg/models/openpitrix/utils.go

@@ -338,7 +338,10 @@ func convertApp(app *v1alpha1.HelmApplication, versions []*v1alpha1.HelmApplicat
 	}
 
 	out.AppVersionTypes = "helm"
-	out.Isv = app.GetWorkspace()
+	// If this keys exists, the workspace of this app has been deleted, set the isv to empty.
+	if _, exists := app.Annotations[constants.DanglingAppCleanupKey]; !exists {
+		out.Isv = app.GetWorkspace()
+	}
 
 	out.ClusterTotal = &rlsCount
 	out.Owner = app.GetCreator()